diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000000000..e1a5ac38d106d --- /dev/null +++ b/bench/README.md @@ -0,0 +1,36 @@ +# Benchmark charts + +Assets referenced by the benchmark comments of the short-lived double +write buffer pull request. Generated from the pgbench series described +there (104-thread NVMe stand, update-heavy pgbench, 1.5 TB cluster); +`vanilla` is the same tree with `io_torn_pages_protection = full_pages` +and data checksums enabled, converted from the same reference cluster. + +* `users-tps.png`, `users-lat.png` — TPS and average latency vs + connection count at `checkpoint_timeout = 300s`. +* `time-tps.png`, `time-lat.png` — TPS and average latency over the + 900 s run at 2700 connections (10 s pgbench samples, 30 s step). +* `repl-tps-wal.png`, `repl-lag.png` — the primary/synchronous-standby + series: throughput and shipped WAL per client count, and the standby's + replay backlog over the run. Both instances share the stand, split by + socket, with a netem-emulated 10 GbE hop between them. +* `repl-time-tps.png`, `repl-time-lat.png` — TPS and average latency over + the run at 750 connections against a synchronous standby, vanilla + against DWB with the replay warm pool (`replay_warm_workers = 12`). + The vanilla run is 900 s, the pool run 600 s. +* `repl2-tps-sockets.png`, `repl2-time-tps.png`, `repl2-time-lat.png` — + the two-host grid: the primary owns 1, 2 or 4 whole NUMA sockets of a + 240-thread host, the synchronous standby has a second host to itself, + and the link between them is a real 100 GbE hop (RTT 0.126 ms). The + bar chart is throughput per socket count; the two line charts follow + the two-socket point over its 600 s run. +* `ring-size.png`, `ring-time-tps.png`, `ring-time-lat.png` — the + `replay_warm_queue_size` series on the two-host pair: what each ring size + buys the standby's replay and what it costs the pool, and the primary's + throughput and latency over each 600 s run at 750 connections against a + synchronous standby. The four points ran back to back on one pair, so + only neighbours are comparable. +* `replay-lag-512.png`, `replay-time-budget.png` — the standby side of the + ring-512 point: the backlog it accumulates over the run, and where the + replay process's time goes — its wall clock split between running and + waiting, and the running part broken down by symbol group. diff --git a/bench/repl-lag.png b/bench/repl-lag.png new file mode 100644 index 0000000000000..cf0fc25164994 Binary files /dev/null and b/bench/repl-lag.png differ diff --git a/bench/repl-time-lat.png b/bench/repl-time-lat.png new file mode 100644 index 0000000000000..3011750d205f0 Binary files /dev/null and b/bench/repl-time-lat.png differ diff --git a/bench/repl-time-tps.png b/bench/repl-time-tps.png new file mode 100644 index 0000000000000..7f28d3cc39618 Binary files /dev/null and b/bench/repl-time-tps.png differ diff --git a/bench/repl-tps-wal.png b/bench/repl-tps-wal.png new file mode 100644 index 0000000000000..3ee8c9aec8078 Binary files /dev/null and b/bench/repl-tps-wal.png differ diff --git a/bench/repl2-time-lat.png b/bench/repl2-time-lat.png new file mode 100644 index 0000000000000..6e5a77c65c07d Binary files /dev/null and b/bench/repl2-time-lat.png differ diff --git a/bench/repl2-time-tps.png b/bench/repl2-time-tps.png new file mode 100644 index 0000000000000..83527b09ac76a Binary files /dev/null and b/bench/repl2-time-tps.png differ diff --git a/bench/repl2-tps-sockets.png b/bench/repl2-tps-sockets.png new file mode 100644 index 0000000000000..2a7c9c81b4db7 Binary files /dev/null and b/bench/repl2-tps-sockets.png differ diff --git a/bench/replay-lag-512.png b/bench/replay-lag-512.png new file mode 100644 index 0000000000000..a48098199defa Binary files /dev/null and b/bench/replay-lag-512.png differ diff --git a/bench/replay-time-budget.png b/bench/replay-time-budget.png new file mode 100644 index 0000000000000..14d317367b334 Binary files /dev/null and b/bench/replay-time-budget.png differ diff --git a/bench/ring-size.png b/bench/ring-size.png new file mode 100644 index 0000000000000..9613ff630d486 Binary files /dev/null and b/bench/ring-size.png differ diff --git a/bench/ring-time-lat.png b/bench/ring-time-lat.png new file mode 100644 index 0000000000000..36d3344c7c735 Binary files /dev/null and b/bench/ring-time-lat.png differ diff --git a/bench/ring-time-tps.png b/bench/ring-time-tps.png new file mode 100644 index 0000000000000..0d56f25bbba8e Binary files /dev/null and b/bench/ring-time-tps.png differ diff --git a/bench/time-lat.png b/bench/time-lat.png new file mode 100644 index 0000000000000..ad98779bc5b95 Binary files /dev/null and b/bench/time-lat.png differ diff --git a/bench/time-tps.png b/bench/time-tps.png new file mode 100644 index 0000000000000..4e659a0d1a7ef Binary files /dev/null and b/bench/time-tps.png differ diff --git a/bench/users-lat.png b/bench/users-lat.png new file mode 100644 index 0000000000000..c09fedd6bb8a1 Binary files /dev/null and b/bench/users-lat.png differ diff --git a/bench/users-tps.png b/bench/users-tps.png new file mode 100644 index 0000000000000..42a7526165cc6 Binary files /dev/null and b/bench/users-tps.png differ diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index 25b8904baf7cd..2c61260dd01ca 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -819,7 +819,11 @@ test ! -f /mnt/server/archivedir/00000001000000A900000065 && cp pg_wal/0 to make a base backup. However, if you normally run the server with full_page_writes disabled, you might notice a drop in performance while the backup runs since full_page_writes is - effectively forced on during backup mode. + effectively forced on during backup mode. The same applies when the + server runs with set to + double_writes: page images are forced back into WAL + for the duration of the backup, because a base backup can legitimately + copy a torn page that only WAL replay with page images can repair. @@ -1130,11 +1134,15 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true); - The contents of the directories pg_dynshmem/, + The contents of the directories pg_dwb/, + pg_dynshmem/, pg_notify/, pg_serial/, pg_snapshots/, pg_stat_tmp/, and pg_subtrans/ (but not the directories themselves) can be omitted from the backup as they will be initialized on postmaster startup. + The double write buffer ring in pg_dwb/ in particular + must never be restored to another cluster: its page copies only repair + torn writes of the instance that wrote them. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 607dafcb2ed16..51dff8b2280e0 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3402,6 +3402,13 @@ include_dir 'conf.d' (see ). + + This parameter has an effect only when is set to + full_pages; the other modes determine the + protection against partial page writes themselves and ignore it. + + This parameter can only be set in the postgresql.conf file or on the server command line. @@ -3410,6 +3417,395 @@ include_dir 'conf.d' + + io_torn_pages_protection (enum) + + io_torn_pages_protection configuration parameter + + + + + Selects the mechanism that protects data files against torn + (partially written) pages after an operating system crash. + Valid values are full_pages, + double_writes and off. + The default is full_pages. + This parameter can only be set at server start. + + + + full_pages is the traditional protection: a full + image of each page is written to WAL on its first modification after + a checkpoint, under the control of the + parameter. + + + + double_writes replaces the page images in WAL + with a double write buffer: every permanent + data page leaving shared buffers is first written and flushed to a + small reusable ring of files in the pg_dwb + directory, and only then written to its actual location. After a + crash, pages torn by interrupted writes are restored from their + ring copies before WAL replay begins. The automatic page images + that full_pages mode writes into WAL on the + first modification after a checkpoint are not written in this mode + (operations that explicitly request a page image still log one), + which can substantially reduce WAL volume and the associated commit + latency spikes after checkpoints, at the price of writing every + flushed data page twice. See + for discussion. This mode + requires data checksums (see ); + the server refuses to start without them. An active base backup + temporarily forces page images back into WAL, because a backup can + legitimately copy a torn page that only WAL replay can repair. + + + + off disables torn page protection entirely + (full_page_writes is ignored). Nothing can + repair pages torn by a crash in this mode; use it only on storage + that guarantees atomic 8kB writes, under the same considerations as + turning off full_page_writes. + + + + The mode in effect is recorded in pg_control + and in the WAL stream. A standby tracks the mode of its primary: + replaying WAL that was generated without page images is refused if + the standby itself expects full_pages protection, + because a crash of that standby could tear pages its own recovery + could not repair. A standby running double_writes + repairs its own torn pages from its own ring and can follow a + primary in any mode. + + + + + + dwb_num_batches (integer) + + dwb_num_batches configuration parameter + + + + + Number of batches in the double write buffer ring. Together with + this sets the ring capacity: + the on-disk size of pg_dwb is roughly + dwb_num_batches × + dwb_batch_pages × 8kB. A larger ring + absorbs longer bursts of page writes before writers have to wait + for batches to be retired. The default is 64. + This parameter can only be set at server start. + + + + + + dwb_batch_pages (integer) + + dwb_batch_pages configuration parameter + + + + + Number of pages in one double write buffer batch. A batch is + written to the ring and flushed as a single unit, so this is the + unit of grouping for the ring's writes and fsyncs. The default + is 64. This parameter can only be set at server start. + + + + + + dwb_max_segments (integer) + + dwb_max_segments configuration parameter + + + + + Capacity of the shared table that tracks which data file segments + still need an fsync before their ring batches + can be reused. If it overflows, batch publication falls back to + retiring the batch synchronously, which is safe but slower. The + default is 4096. This parameter can only be set at server start. + + + + + + dwb_retire_workers (integer) + + dwb_retire_workers configuration parameter + + + + + Number of background workers that retire double write buffer + batches by flushing the covered data files, freeing ring space for + new writes. The workers consume + slots. Setting it to 0 + disables the pool and makes writers retire batches synchronously; + this is meant for testing only. The default is 1. + This parameter can only be set at server start. + + + + + + dwb_cleaner_workers (integer) + + dwb_cleaner_workers configuration parameter + + + + + Number of background workers that execute the flush bins produced + by the background writer's LRU scan. Each bin is one double write + buffer batch write followed by its sync and the data-file writes; + a single background writer executing them serially cannot clean + more than a few thousand pages per second, and backends then evict + dirty buffers themselves at full double write buffer latency. + With a pool the scan keeps running in the background writer while + the bins are executed concurrently, and backends find clean + buffers instead. The background writer then writes no data pages + itself: when the pool falls momentarily behind, the scan pauses + instead of writing and resumes as soon as a worker frees queue + space. The checkpointer also defers to a loaded pool: while the + pool's queue is hot, checkpoint writes pause within a small, + bounded slice of the + slack, so + checkpoint traffic lands in the quieter moments of the write + window. The workers consume + slots. Setting it to 0 + (the default) disables the pool and the background writer flushes + its bins itself. + This parameter can only be set at server start. + + + The pool scales execution only; how much the LRU scan issues per + round is still governed by + , + and + . With their default + values the pool mostly idles, so raise them together with this + setting. Pages written by the pool are counted in + pg_stat_bgwriter.buffers_clean + as if the background writer had written them itself. + + + + + + dwb_retire_sync_method (enum) + + dwb_retire_sync_method configuration parameter + + + + + Selects how retirement makes the covered data files durable before + freeing a batch. With fsync each touched + data-file segment is synced individually. With + syncfs a retire round issues one + syncfs() call per file system holding data + files and then frees every batch whose data-file writes preceded + the round, which is much cheaper when a write-heavy workload + touches many segments between rounds. syncfs + is only available on Linux and is the default there; + fsync is the default elsewhere. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + The caveats of syncfs described under + apply here as + well. + + + + + + dwb_batch_timeout_ms (integer) + + dwb_batch_timeout_ms configuration parameter + + + + + Maximum time an open batch may collect pages before it is sealed + and written even if not full, bounding the latency a lone page + flush can spend waiting for company. A lone page flush seals its + batch immediately while its writer class is quiet; after a recent + full-batch seal — the sign of a dense concurrent stream — it waits + out this window instead, letting the batch fill. With + set to 0 and in + single-user mode a lone flush always seals immediately. + If this value is specified without units, it is taken as milliseconds. + The default is 10 milliseconds. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_retire_interval_ms (integer) + + dwb_retire_interval_ms configuration parameter + + + + + Cycle time of each retire worker: how long a worker sleeps when it + finds no batches waiting for retirement. + If this value is specified without units, it is taken as milliseconds. + The default is 50 milliseconds. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_writeback_after (integer) + + dwb_writeback_after configuration parameter + + + + + Whenever a process has written this many data pages through the + double write buffer, it asks the kernel to start writing them back to + disk. The synchronization that eventually retires their batch (see + ) is then more likely to + find the data already on its way, and to cost closer to a barrier + than to a full flush. The pending pages are handed over in block + order, and neighbouring ones as a single request. Has no effect on + platforms without such a request. + + + This is pacing, not ordering: a page waits in its process until the + count is reached, so a batch may well be retired before the kernel + has heard about the pages in it. Raising the value trades away more + of that overlap for fewer requests. The exception is a server + running without retire workers ( + set to 0), where a page write retires its own + batch before returning: there the pending pages are handed over at + every such write, so any positive value behaves as + 1 — and so, for pages the double write buffer + stages, do and + . + + + This parameter governs the pages the double write buffer stages on + paths that have no writeback pacing of their own — above all the + eviction a backend performs to make room for a page it needs. Pages + written by the checkpointer and the background writer keep following + and + . When a backend evicts a + page the double write buffer does not stage, such as a page of an + unlogged relation, that page keeps following + . + + + If this value is specified without units, it is taken as blocks, that + is BLCKSZ bytes, typically 8kB. The valid range is + between 0, which disables the double write buffer's + own writeback, and 2MB. The default is + 256kB on all platforms. (If BLCKSZ + is not 8kB, the default and maximum values scale proportionally to + it.) + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_slow_warn_ms (integer) + + dwb_slow_warn_ms configuration parameter + + + + + When a writer has waited this long for free ring space, a warning + is logged and non-critical background writing is paused so that the + remaining ring capacity serves user-facing work first. + If this value is specified without units, it is taken as milliseconds. + The default is 5 seconds. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_slot_stuck_timeout_ms (integer) + + dwb_slot_stuck_timeout_ms configuration parameter + + + + + Time the writer of a sealed batch waits for the batch's remaining + pages to be staged before treating the batch as wedged and raising + a PANIC. This is a defensive limit that should + never be reached. + If this value is specified without units, it is taken as milliseconds. + The default is 30 seconds. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_write_timeout_ms (integer) + + dwb_write_timeout_ms configuration parameter + + + + + When a writer has waited this long for free ring space, the action + selected by is taken. A wait + this long means batch retirement cannot keep up with the flow of + page writes (or the underlying storage has stalled). + If this value is specified without units, it is taken as milliseconds. + The default is 60 seconds. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + + + dwb_on_stall (enum) + + dwb_on_stall configuration parameter + + + + + Action taken when a double write buffer wait exceeds + : warn + logs and keeps waiting, error aborts the waiting + transaction, and panic restarts the server. + Processes whose page writes cannot be abandoned, such as the + checkpointer, always escalate to panic. The + default is panic: a stall this long indicates a + storage-level problem, and a restart with crash recovery is + preferable to an indefinite hang. + This parameter can only be set in the postgresql.conf + file or on the server command line. + + + + wal_log_hints (boolean) @@ -4000,6 +4396,67 @@ include_dir 'conf.d' + + replay_warm_workers (integer) + + replay_warm_workers configuration parameter + + + + + Number of background workers that read the blocks recovery is about to + modify into shared buffers, ahead of replay. The default is 0, which + disables the pool; recovery then relies on + advising the operating system + and reads the blocks in the startup process itself. + + + This matters most when the WAL stream carries no full-page images, + which is the case on a standby of a cluster running + set to + double_writes: every replayed record then needs its + page fetched, and doing that from the single replay process can make + it the limit on how fast the standby keeps up. With the pool enabled + that fetching, including its checksum verification and its buffer + allocation, happens in the workers instead. + + + The workers take slots from + , take no database + connection, and start with the postmaster, so they also serve crash + recovery. Their requests are advisory throughout: anything they do + not get to in time is simply read by the replay process, as it would + be with the pool disabled. This parameter can only be set at server + start. + + + + + + replay_warm_queue_size (integer) + + replay_warm_queue_size configuration parameter + + + + + How many block requests the pool enabled by + can hold at once, which also + bounds how far ahead of replay the server looks for blocks to warm. + The default is 512. This parameter can only be set at server start. + + + Every block a record refers to takes a request, whether or not the + block turns out to be in a buffer already: which of the two it is is + the pool's answer to give, and asking the question in the startup + process is the cost this arrangement avoids. Since most references + are to blocks already in buffers, the reads actually outstanding + ahead of replay are a fraction of this setting, and a pool of many + workers wants room for all of them. + + + + wal_decode_buffer_size (integer) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5ac0e54688cb4..eefac73f38f22 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -1854,6 +1854,15 @@ description | Waiting for a newly initialized WAL file to reach durable storage with the pg_stat_reset_shared function. + + When is set, whether a block was + already in the buffer pool is decided by the pool's workers rather than by + recovery itself, and prefetch and + hit are counted there. A request that no worker + ever serves — one withdrawn because recovery moved its read position, or + one whose worker exited while holding it — is counted in neither. + + <structname>pg_stat_recovery_prefetch</structname> View @@ -2606,8 +2615,9 @@ description | Waiting for a newly initialized WAL file to reach durable storage - Currently, I/O on relations (e.g. tables, indexes) and WAL activity are - tracked. However, relation I/O which bypasses shared buffers + Currently, I/O on relations (e.g. tables, indexes), WAL activity and the + double write buffer (see ) + are tracked. However, relation I/O which bypasses shared buffers (e.g. when moving a table from one tablespace to another) is currently not tracked. @@ -2662,6 +2672,13 @@ description | Waiting for a newly initialized WAL file to reach durable storage temp relation: Temporary relations. + + + dwb: The double write buffer ring in + pg_dwb (see + ). + + wal: Write Ahead Logs. @@ -3059,7 +3076,9 @@ description | Waiting for a newly initialized WAL file to reach durable storage buffers_clean bigint - Number of buffers written by the background writer + Number of buffers written by the background writer's LRU cleaning, + including buffers its scan handed to the cleaner worker pool + (see ) @@ -3069,7 +3088,8 @@ description | Waiting for a newly initialized WAL file to reach durable storage Number of times the background writer stopped a cleaning - scan because it had written too many buffers + scan because it had issued too many buffers, counting both its own + writes and bins handed to the cleaner worker pool diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index f0b29ed8cab9f..0b62fe02afb29 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -3388,7 +3388,8 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" - pg_dynshmem, pg_notify, + pg_dwb, pg_dynshmem, + pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, and pg_subtrans are copied as empty directories (even if diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 9659f76042c5b..aae8f4d4c8a77 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -115,7 +115,12 @@ PostgreSQL documentation All WAL records required for the backup must contain sufficient full-page writes, - which requires you to enable full_page_writes on the primary. + which requires you to enable full_page_writes on the + primary and to run it with + set to full_pages. In the other modes the primary + does not write the automatic page images the backup depends on, and a + standby has no way to request them, so a backup taken from the + standby is refused. diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index c696cec1a1c43..040ae5e2bd3ab 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -359,6 +359,19 @@ GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, b + + When the source is a running server, it must run with + set to + full_pages and full_page_writes + enabled: pg_rewind reads data blocks from the + source while they may be concurrently written, and such a torn read is + only repaired by the full page images the target replays afterwards. A + cleanly shut down source can be used whatever its mode. The double + write buffer ring in pg_dwb is never copied from + the source, and the target's own ring contents are discarded when the + rewound cluster first starts. + + How It Works diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 61250799ec076..198b36a3e3344 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -77,6 +77,13 @@ Item Subdirectory containing transaction commit timestamp data + + pg_dwb + Subdirectory containing the double write buffer ring + (when io_torn_pages_protection is set to + double_writes) + + pg_dynshmem Subdirectory containing files used by the dynamic shared memory diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index f3b86b26be905..92c763f08185a 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -179,6 +179,33 @@ (BBU) disk controllers do not prevent partial page writes unless they guarantee that data is written to the BBU as full (8kB) pages. + + An alternative protection against partial page writes is the + double write buffer, selected by setting + to + double_writes. Instead of recording page images in + WAL, every permanent data page leaving shared buffers is first written + and flushed to a small reusable ring of files in the + pg_dwb + directory, and only then written to its actual location; a batch of + pages is flushed to the ring with a single fsync, + and ring space is reused as soon as the covered data-file writes have + been made durable. After a crash, the server scans the ring before WAL + replay begins and rewrites every data page that is torn or older than + its ring copy, so replay always starts from intact pages. Since the + torn-page repair no longer depends on WAL contents, the automatic page + images written on the first modification after a checkpoint disappear + from WAL (operations that explicitly request a page image still log + one): WAL volume shrinks by their share, and the write + bursts that follow each checkpoint flatten out. In exchange every + flushed data page is written twice, which moves the cost from the WAL + device to the data-file write path. This mode relies on data checksums + to detect torn pages, so checksums must be enabled. The ring only + repairs pages torn by the local instance's own writes: it is excluded + from base backups, and a backup taken while this mode is active + temporarily carries forced page images in WAL instead (see + ). + PostgreSQL also protects against some kinds of data corruption on storage devices that may occur because of hardware errors or media failure over time, diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index 58040f28656fc..f5fb54d1bf4a0 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -124,7 +124,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "max_connections=%d max_worker_processes=%d " "max_wal_senders=%d max_prepared_xacts=%d " "max_locks_per_xact=%d wal_level=%s " - "wal_log_hints=%s track_commit_timestamp=%s", + "wal_log_hints=%s track_commit_timestamp=%s " + "io_torn_pages_protection=%s", xlrec.MaxConnections, xlrec.max_worker_processes, xlrec.max_wal_senders, @@ -132,7 +133,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record) xlrec.max_locks_per_xact, wal_level_str, xlrec.wal_log_hints ? "on" : "off", - xlrec.track_commit_timestamp ? "on" : "off"); + xlrec.track_commit_timestamp ? "on" : "off", + DWBProtectionModeName(xlrec.io_torn_pages_protection)); } else if (info == XLOG_FPW_CHANGE) { diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index 661c55a9db789..5daedb962e4b3 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -36,7 +36,8 @@ OBJS = \ xlogreader.o \ xlogrecovery.o \ xlogstats.o \ - xlogutils.o + xlogutils.o \ + xlogwarm.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/transam/meson.build b/src/backend/access/transam/meson.build index e8ae9b13c8e49..019992993ee61 100644 --- a/src/backend/access/transam/meson.build +++ b/src/backend/access/transam/meson.build @@ -24,6 +24,7 @@ backend_sources += files( 'xlogrecovery.c', 'xlogstats.c', 'xlogutils.c', + 'xlogwarm.c', ) # used by frontend programs to build a frontend xlogreader diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e07cb9103515f..afb355b18617f 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -83,6 +83,7 @@ #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/bufmgr.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/large_object.h" @@ -146,9 +147,12 @@ int wal_segment_size = DEFAULT_XLOG_SEG_SIZE; /* * Number of WAL insertion locks to use. A higher value allows more insertions * to happen concurrently, but adds some CPU overhead to flushing the WAL, - * which needs to iterate all the locks. + * which needs to iterate all the locks. Raised from 8 for double_writes + * workloads: with full-page images gone the record stream is made of many + * small records, so high-connection benchmarks bottleneck on insertion-slot + * contention well before the WAL device saturates. */ -#define NUM_XLOGINSERT_LOCKS 8 +#define NUM_XLOGINSERT_LOCKS 32 /* * Max distance from last checkpoint, before triggering a new xlog-based @@ -687,6 +691,7 @@ static void UpdateLastRemovedPtr(char *filename); static void ValidateXLOGDirectoryStructure(void); static void CleanupBackupHistory(void); static void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force); +static bool EffectiveFullPageWrites(void); static bool PerformRecoveryXLogAction(void); static void InitControlFile(uint64 sysidentifier, uint32 data_checksum_version); static void WriteControlFile(void); @@ -4228,6 +4233,7 @@ InitControlFile(uint64 sysidentifier, uint32 data_checksum_version) ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; ControlFile->track_commit_timestamp = track_commit_timestamp; + ControlFile->io_torn_pages_protection = io_torn_pages_protection; ControlFile->data_checksum_version = data_checksum_version; } @@ -5115,7 +5121,7 @@ BootStrapXLOG(uint32 data_checksum_version) checkPoint.redo = wal_segment_size + SizeOfXLogLongPHD; checkPoint.ThisTimeLineID = BootstrapTimeLineID; checkPoint.PrevTimeLineID = BootstrapTimeLineID; - checkPoint.fullPageWrites = fullPageWrites; + checkPoint.fullPageWrites = EffectiveFullPageWrites(); checkPoint.wal_level = wal_level; checkPoint.nextXid = FullTransactionIdFromEpochAndXid(0, FirstNormalTransactionId); @@ -5435,6 +5441,26 @@ CheckRequiredParameterValues(void) errhint("Use a backup taken after setting \"wal_level\" to higher than \"minimal\"."))); } + /* + * A server that believes full page images protect it must not replay WAL + * generated without them: its own crash would leave torn data pages that + * neither this WAL nor its configuration can repair. A local double + * write buffer repairs its own torn pages instead, and under "off" the + * user has explicitly waived the protection, so only the "full_pages" + * expectation is refused. + */ + if (ArchiveRecoveryRequested && + ControlFile->io_torn_pages_protection != DWB_PROTECT_FULL_PAGES && + io_torn_pages_protection == DWB_PROTECT_FULL_PAGES) + { + ereport(FATAL, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("WAL was generated with \"io_torn_pages_protection=%s\", cannot continue recovering with \"io_torn_pages_protection=full_pages\"", + DWBProtectionModeName(ControlFile->io_torn_pages_protection)), + errdetail("The WAL carries no full page images, so a crash of this server would leave torn data pages that nothing can repair."), + errhint("Set \"io_torn_pages_protection\" to \"double_writes\" on this server, or to \"full_pages\" on the server that generated the WAL."))); + } + /* * For Hot Standby, the WAL must be generated with 'replica' mode, and we * must have at least as many backend slots as the primary. @@ -5594,6 +5620,85 @@ StartupXLOG(void) else didCrash = false; + /* + * Create or validate the double write buffer ring, repair torn data pages + * from it if the previous run did not close it cleanly, and durably bump + * its generation before any of its slots can be written. This runs before + * InitWalRecovery: the repairs establish the base that WAL replay + * advances from, and the backup_label file (a "restoring from base + * backup" indicator, together with backupStartPoint) is still in place + * here. + */ + { + bool restoring_backup; + XLogRecPtr dwb_applied_upto; + + /* + * A failure to probe for backup_label must fail closed: reading its + * absence out of an EACCES/EIO would drop the one guard that keeps a + * ring shipped inside a base backup from being applied into the + * restored cluster. (read_backup_label treats a failing open the + * same way.) + */ + if (access(BACKUP_LABEL_FILE, F_OK) == 0) + restoring_backup = true; + else if (errno != ENOENT) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not access file \"%s\": %m", + BACKUP_LABEL_FILE))); + else + restoring_backup = + !XLogRecPtrIsInvalid(ControlFile->backupStartPoint); + + /* + * Crash recovery over WAL generated without any torn page protection + * cannot repair pages the crash tore, whatever the local mode says + * now. The mode-based FATAL in CheckRequiredParameterValues fires + * for archive recovery only, so this is the one transition that would + * otherwise be silent. On a standby the pg_control field describes + * the primary, not the run that crashed here — and a local + * double_writes standby of an "off" primary repairs its own torn + * pages from its ring — so the warning is limited to servers whose + * crashed run owned the field. + */ + if (didCrash && + ControlFile->state != DB_IN_ARCHIVE_RECOVERY && + ControlFile->io_torn_pages_protection == DWB_PROTECT_OFF && + io_torn_pages_protection != DWB_PROTECT_OFF) + ereport(WARNING, + (errmsg("database system was interrupted while torn page protection was disabled"), + errdetail("WAL generated with \"io_torn_pages_protection=off\" carries no full page images; pages torn by the crash cannot be repaired by this recovery."))); + + dwb_applied_upto = DWBStartup(restoring_backup); + + /* + * On a standby that did not durably retire the ring — whether it + * crashed or merely skipped its shutdown restartpoint — consistency + * must not be declared before the local WAL covers the repaired + * pages. The write path guarantees minRecoveryPoint already does — + * FlushBuffer's XLogFlush advances it durably before the page can + * enter the ring — so this raise is expected to be a no-op; it + * stays as a belt-and-braces enforcement of the invariant. The + * timeline is left alone: any LSN the ring can hold lies on a + * timeline minRecoveryPoint has already seen, by the same write-path + * argument. + */ + if ((ControlFile->state == DB_IN_ARCHIVE_RECOVERY || + ControlFile->state == DB_SHUTDOWNED_IN_RECOVERY) && + !XLogRecPtrIsInvalid(ControlFile->minRecoveryPoint) && + dwb_applied_upto > ControlFile->minRecoveryPoint) + { + ereport(LOG, + (errmsg("raising minimum recovery point to %X/%X to cover pages repaired from the double write buffer", + LSN_FORMAT_ARGS(dwb_applied_upto)))); + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + ControlFile->minRecoveryPoint = dwb_applied_upto; + UpdateControlFile(); + LWLockRelease(ControlFileLock); + } + } + /* * Prepare for WAL recovery if needed. * @@ -6680,6 +6785,15 @@ ShutdownXLOG(int code, Datum arg) CreateCheckPoint(CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_IMMEDIATE); } + + /* + * Retire what is left in the double write buffer ring — a shutdown + * restartpoint can be skipped entirely, leaving segment fsyncs pending — + * and mark the ring cleanly closed. The next start can then skip the + * apply-pass, and a start under a different io_torn_pages_protection mode + * is legal. + */ + DWBMarkCleanShutdown(); } /* @@ -8156,7 +8270,8 @@ XLogReportParameters(void) max_wal_senders != ControlFile->max_wal_senders || max_prepared_xacts != ControlFile->max_prepared_xacts || max_locks_per_xact != ControlFile->max_locks_per_xact || - track_commit_timestamp != ControlFile->track_commit_timestamp) + track_commit_timestamp != ControlFile->track_commit_timestamp || + io_torn_pages_protection != ControlFile->io_torn_pages_protection) { /* * The change in number of backend slots doesn't need to be WAL-logged @@ -8164,8 +8279,14 @@ XLogReportParameters(void) * with wal_level=minimal anyway. We don't really care about the * values in pg_control either if wal_level=minimal, but seems better * to keep them up-to-date to avoid confusion. + * + * An io_torn_pages_protection change is always WAL-logged: replay + * must learn the generating server's mode, because the FATAL in + * CheckRequiredParameterValues keys on it. */ - if (wal_level != ControlFile->wal_level || XLogIsNeeded()) + if (wal_level != ControlFile->wal_level || + io_torn_pages_protection != ControlFile->io_torn_pages_protection || + XLogIsNeeded()) { xl_parameter_change xlrec; XLogRecPtr recptr; @@ -8178,6 +8299,7 @@ XLogReportParameters(void) xlrec.wal_level = wal_level; xlrec.wal_log_hints = wal_log_hints; xlrec.track_commit_timestamp = track_commit_timestamp; + xlrec.io_torn_pages_protection = io_torn_pages_protection; XLogBeginInsert(); XLogRegisterData(&xlrec, sizeof(xlrec)); @@ -8196,12 +8318,31 @@ XLogReportParameters(void) ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; ControlFile->track_commit_timestamp = track_commit_timestamp; + ControlFile->io_torn_pages_protection = io_torn_pages_protection; UpdateControlFile(); LWLockRelease(ControlFileLock); } } +/* + * The effective value of the full_page_writes setting. + * + * io_torn_pages_protection selects the torn-page protection mechanism: under + * "double_writes" the durable copy in pg_dwb/ replaces FPIs and under "off" + * the user has declared torn writes impossible, so both force this off; the + * legacy full_page_writes GUC keeps its meaning under "full_pages" only. + * Online backups taken on a primary still force page images regardless of + * this value, through the runningBackups term of doPageWrites (see + * XLogInsertRecord); backups initiated on a standby cannot and are refused + * by do_pg_backup_start when the replayed WAL lacks page images. + */ +static bool +EffectiveFullPageWrites(void) +{ + return io_torn_pages_protection == DWB_PROTECT_FULL_PAGES && fullPageWrites; +} + /* * Update full_page_writes in shared memory, and write an * XLOG_FPW_CHANGE record if necessary. @@ -8213,6 +8354,7 @@ void UpdateFullPageWrites(void) { XLogCtlInsert *Insert = &XLogCtl->Insert; + bool newFullPageWrites = EffectiveFullPageWrites(); bool recoveryInProgress; /* @@ -8222,7 +8364,7 @@ UpdateFullPageWrites(void) * because we assume that there is no concurrently running process which * can update it. */ - if (fullPageWrites == Insert->fullPageWrites) + if (newFullPageWrites == Insert->fullPageWrites) return; /* @@ -8241,7 +8383,7 @@ UpdateFullPageWrites(void) * setting it to false, first write the WAL record and then set the global * flag. */ - if (fullPageWrites) + if (newFullPageWrites) { WALInsertLockAcquireExclusive(); Insert->fullPageWrites = true; @@ -8255,12 +8397,12 @@ UpdateFullPageWrites(void) if (XLogStandbyInfoActive() && !recoveryInProgress) { XLogBeginInsert(); - XLogRegisterData(&fullPageWrites, sizeof(bool)); + XLogRegisterData(&newFullPageWrites, sizeof(bool)); XLogInsert(RM_XLOG_ID, XLOG_FPW_CHANGE); } - if (!fullPageWrites) + if (!newFullPageWrites) { WALInsertLockAcquireExclusive(); Insert->fullPageWrites = false; @@ -8269,6 +8411,33 @@ UpdateFullPageWrites(void) END_CRIT_SECTION(); } +/* + * Track the last replayed WAL record declaring full-page writes disabled, + * for the standby backup guards in do_pg_backup_start/stop. Both + * XLOG_FPW_CHANGE records and checkpoint records can carry the declaration: + * a primary restarted into a mode without page images + * (io_torn_pages_protection = "double_writes"/"off", or full_page_writes = + * off) emits no XLOG_FPW_CHANGE — its checkpoints are the only replayed + * evidence of the change (see UpdateFullPageWrites). + * + * Only the mode-based loss of images is refused by + * CheckRequiredParameterValues (keyed on the generating server's + * io_torn_pages_protection in pg_control); the legacy case — a "full_pages" + * primary running with full_page_writes = off — still replays, and this + * tracking is what lets the backup guards reject it. + */ +static void +XLogTrackFullPageWritesDisabled(XLogReaderState *record, bool fpw) +{ + if (fpw) + return; + + SpinLockAcquire(&XLogCtl->info_lck); + if (XLogCtl->lastFpwDisableRecPtr < record->ReadRecPtr) + XLogCtl->lastFpwDisableRecPtr = record->ReadRecPtr; + SpinLockRelease(&XLogCtl->info_lck); +} + /* * XLOG resource manager's routines * @@ -8314,6 +8483,7 @@ xlog_redo(XLogReaderState *record) TimeLineID replayTLI; memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint)); + XLogTrackFullPageWritesDisabled(record, checkPoint.fullPageWrites); /* In a SHUTDOWN checkpoint, believe the counters exactly */ LWLockAcquire(XidGenLock, LW_EXCLUSIVE); TransamVariables->nextXid = checkPoint.nextXid; @@ -8420,6 +8590,7 @@ xlog_redo(XLogReaderState *record) TimeLineID replayTLI; memcpy(&checkPoint, XLogRecGetData(record), sizeof(CheckPoint)); + XLogTrackFullPageWritesDisabled(record, checkPoint.fullPageWrites); /* In an ONLINE checkpoint, treat the XID counter as a minimum */ LWLockAcquire(XidGenLock, LW_EXCLUSIVE); if (FullTransactionIdPrecedes(TransamVariables->nextXid, @@ -8587,6 +8758,7 @@ xlog_redo(XLogReaderState *record) ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact; ControlFile->wal_level = xlrec.wal_level; ControlFile->wal_log_hints = xlrec.wal_log_hints; + ControlFile->io_torn_pages_protection = xlrec.io_torn_pages_protection; /* * Update minRecoveryPoint to ensure that if recovery is aborted, we @@ -8627,17 +8799,11 @@ xlog_redo(XLogReaderState *record) memcpy(&fpw, XLogRecGetData(record), sizeof(bool)); /* - * Update the LSN of the last replayed XLOG_FPW_CHANGE record so that - * do_pg_backup_start() and do_pg_backup_stop() can check whether - * full_page_writes has been disabled during online backup. + * Track the disable point so that do_pg_backup_start() and + * do_pg_backup_stop() can check whether full-page writes were + * disabled during an online backup. */ - if (!fpw) - { - SpinLockAcquire(&XLogCtl->info_lck); - if (XLogCtl->lastFpwDisableRecPtr < record->ReadRecPtr) - XLogCtl->lastFpwDisableRecPtr = record->ReadRecPtr; - SpinLockRelease(&XLogCtl->info_lck); - } + XLogTrackFullPageWritesDisabled(record, fpw); /* Keep track of full_page_writes */ lastFullPageWrites = fpw; @@ -8932,6 +9098,7 @@ do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, do { bool checkpointfpw; + int primary_iotpp; /* * Force a CHECKPOINT. Aside from being necessary to prevent torn @@ -8965,6 +9132,7 @@ do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, state->startpoint = ControlFile->checkPointCopy.redo; state->starttli = ControlFile->checkPointCopy.ThisTimeLineID; checkpointfpw = ControlFile->checkPointCopy.fullPageWrites; + primary_iotpp = ControlFile->io_torn_pages_protection; LWLockRelease(ControlFileLock); if (backup_started_in_recovery) @@ -8983,12 +9151,19 @@ do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, if (!checkpointfpw || state->startpoint <= recptr) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("WAL generated with \"full_page_writes=off\" was replayed " + errmsg("WAL generated without full page images was replayed " "since last restartpoint"), - errhint("This means that the backup being taken on the standby " - "is corrupt and should not be used. " - "Enable \"full_page_writes\" and run CHECKPOINT on the primary, " - "and then try an online backup again."))); + primary_iotpp != DWB_PROTECT_FULL_PAGES + ? errdetail("The primary runs \"io_torn_pages_protection\" = \"%s\" " + "and does not write full page images.", + DWBProtectionModeName(primary_iotpp)) + : errdetail("The primary has \"full_page_writes\" disabled."), + errhint("A backup taken on a standby needs full page images in the " + "replayed WAL; the primary's double write buffer cannot " + "substitute for them. Set io_torn_pages_protection = " + "\"full_pages\" (with \"full_page_writes\" enabled) on the " + "primary and run CHECKPOINT there, or take the backup on " + "the primary."))); /* * During recovery, since we don't use the end-of-backup WAL @@ -9267,6 +9442,7 @@ do_pg_backup_stop(BackupState *state, bool waitforarchive) if (backup_stopped_in_recovery) { XLogRecPtr recptr; + int primary_iotpp; /* * Check to see if all WAL replayed during online backup contain @@ -9276,15 +9452,29 @@ do_pg_backup_stop(BackupState *state, bool waitforarchive) recptr = XLogCtl->lastFpwDisableRecPtr; SpinLockRelease(&XLogCtl->info_lck); + LWLockAcquire(ControlFileLock, LW_SHARED); + primary_iotpp = ControlFile->io_torn_pages_protection; + LWLockRelease(ControlFileLock); + if (state->startpoint <= recptr) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("WAL generated with \"full_page_writes=off\" was replayed " + errmsg("WAL generated without full page images was replayed " "during online backup"), - errhint("This means that the backup being taken on the standby " - "is corrupt and should not be used. " - "Enable \"full_page_writes\" and run CHECKPOINT on the primary, " - "and then try an online backup again."))); + primary_iotpp != DWB_PROTECT_FULL_PAGES + ? errdetail("The backup being taken on the standby is corrupt " + "and should not be used: the primary runs " + "\"io_torn_pages_protection\" = \"%s\".", + DWBProtectionModeName(primary_iotpp)) + : errdetail("The backup being taken on the standby is corrupt " + "and should not be used: the primary has " + "\"full_page_writes\" disabled."), + errhint("A backup taken on a standby needs full page images in the " + "replayed WAL; the primary's double write buffer cannot " + "substitute for them. Set io_torn_pages_protection = " + "\"full_pages\" (with \"full_page_writes\" enabled) on the " + "primary and run CHECKPOINT there, or take the backup on " + "the primary."))); LWLockAcquire(ControlFileLock, LW_SHARED); diff --git a/src/backend/access/transam/xlogprefetcher.c b/src/backend/access/transam/xlogprefetcher.c index 7735562db01d1..1d386191d7009 100644 --- a/src/backend/access/transam/xlogprefetcher.c +++ b/src/backend/access/transam/xlogprefetcher.c @@ -29,6 +29,7 @@ #include "access/xlogprefetcher.h" #include "access/xlogreader.h" +#include "access/xlogwarm.h" #include "catalog/pg_control.h" #include "catalog/storage_xlog.h" #include "commands/dbcommands_xlog.h" @@ -75,6 +76,16 @@ int recovery_prefetch = RECOVERY_PREFETCH_TRY; #define RecoveryPrefetchEnabled() false #endif +/* + * The lookahead machinery — decoding ahead, the relation filters, the + * distance logic — serves two consumers now, and runs if either wants it. + * The warm pool wants it whenever it is configured: it does not issue + * kernel advice, so neither USE_PREFETCH nor maintenance_io_concurrency has + * any say over it. + */ +#define RecoveryLookaheadEnabled() \ + (RecoveryPrefetchEnabled() || XLogWarmPoolActive()) + static int XLogPrefetchReconfigureCount = 0; /* @@ -286,7 +297,7 @@ lrq_complete_lsn(LsnReadQueue *lrq, XLogRecPtr lsn) if (lrq->tail == lrq->size) lrq->tail = 0; } - if (RecoveryPrefetchEnabled()) + if (RecoveryLookaheadEnabled()) lrq_prefetch(lrq); } @@ -354,6 +365,35 @@ XLogPrefetchIncrement(pg_atomic_uint64 *counter) pg_atomic_write_u64(counter, pg_atomic_read_u64(counter) + 1); } +/* + * Increment a counter that more than one process writes. + * + * The plain increment above belongs to the startup process alone and is a + * read followed by a write, which several writers would lose counts to. The + * replay warm pool has as many writers as it has workers, so it uses this. + */ +static inline void +XLogPrefetchIncrementShared(pg_atomic_uint64 *counter) +{ + pg_atomic_fetch_add_u64(counter, 1); +} + +/* + * Record what the warm pool found when it served a request: a block already + * in a buffer, or one it had to read. + */ +void +XLogPrefetchCountHit(void) +{ + XLogPrefetchIncrementShared(&SharedStats->hit); +} + +void +XLogPrefetchCountPrefetch(void) +{ + XLogPrefetchIncrementShared(&SharedStats->prefetch); +} + /* * Create a prefetcher that is ready to begin prefetching blocks referenced by * WAL records. @@ -389,6 +429,9 @@ XLogPrefetcherAllocate(XLogReaderState *reader) void XLogPrefetcherFree(XLogPrefetcher *prefetcher) { + /* the decoded records go away with the reader, so must their requests */ + XLogWarmCancelAll(); + lrq_free(prefetcher->streaming_read); hash_destroy(prefetcher->filter_table); pfree(prefetcher); @@ -503,11 +546,11 @@ XLogPrefetcherNextBlock(uintptr_t pgsr_private, XLogRecPtr *lsn) } /* - * If prefetching is disabled, we don't need to analyze the record - * or issue any prefetches. We just need to cause one record to - * be decoded. + * If neither the advice nor the warm pool wants blocks, we don't + * need to analyze the record or issue any prefetches. We just + * need to cause one record to be decoded. */ - if (!RecoveryPrefetchEnabled()) + if (!RecoveryLookaheadEnabled()) { *lsn = InvalidXLogRecPtr; return LRQ_NEXT_NO_IO; @@ -763,6 +806,41 @@ XLogPrefetcherNextBlock(uintptr_t pgsr_private, XLogRecPtr *lsn) return LRQ_NEXT_NO_IO; } + /* + * With the warm pool running, a miss is handed to a worker + * instead of being turned into kernel advice: the worker reads + * the page into a shared buffer, which is what replay actually + * needs, and none of that work lands on this process. + */ + if (XLogWarmPoolActive()) + { + uint64 request_id; + int slot_no; + + /* + * Whether the block is already in a buffer is a question with + * a hash lookup under a partition lock behind it, and three + * references in four answer yes. Asking it here spends that + * lookup in the one process replay cannot do without, so the + * question goes to the pool along with the block: a worker + * asks it, hands back whichever buffer the answer names, and + * counts it (XLogWarmDoOne()). Replay ends up with the same + * hint, produced beside it instead of by it. + */ + slot_no = XLogWarmPublish(block->rlocator, block->forknum, + block->blkno, &request_id); + if (slot_no == XLOGWARM_NO_SLOT) + { + /* pool behind: replay will read this block itself */ + return LRQ_NEXT_NO_IO; + } + + block->warm_slot = slot_no; + block->warm_request = request_id; + block->prefetch_buffer = InvalidBuffer; + return LRQ_NEXT_IO; + } + /* Try to initiate prefetching. */ result = PrefetchSharedBuffer(reln, block->forknum, block->blkno); if (BufferIsValid(result.recent_buffer)) @@ -961,6 +1039,13 @@ XLogPrefetcherIsFiltered(XLogPrefetcher *prefetcher, RelFileLocator rlocator, void XLogPrefetcherBeginRead(XLogPrefetcher *prefetcher, XLogRecPtr recPtr) { + /* + * This will forget about any in-flight IO, so the requests those decoded + * records referred to must be withdrawn: nobody will ever collect them, + * and slots nobody collects would eventually fill the ring. + */ + XLogWarmCancelAll(); + /* This will forget about any in-flight IO. */ prefetcher->reconfigure_count--; @@ -995,7 +1080,21 @@ XLogPrefetcherReadRecord(XLogPrefetcher *prefetcher, char **errmsg) if (prefetcher->streaming_read) lrq_free(prefetcher->streaming_read); - if (RecoveryPrefetchEnabled()) + if (XLogWarmPoolActive()) + { + /* + * The pool's ring is what bounds requests in flight, and it is + * sized at server start: maintenance_io_concurrency can be raised + * at runtime and must not be able to push the lookahead past the + * ring, which would only produce requests that get dropped for + * want of a slot. + */ + max_inflight = Max(replay_warm_queue_size / 2, 1); + max_distance = Min(max_inflight * XLOGPREFETCHER_DISTANCE_MULTIPLIER, + replay_warm_queue_size); + max_distance = Max(max_distance, max_inflight); + } + else if (RecoveryPrefetchEnabled()) { Assert(maintenance_io_concurrency > 0); max_inflight = maintenance_io_concurrency; @@ -1056,6 +1155,28 @@ XLogPrefetcherReadRecord(XLogPrefetcher *prefetcher, char **errmsg) */ Assert(record == prefetcher->reader->record); + /* + * Collect whatever the warm pool managed to read for this record. A + * buffer collected here is only a hint, exactly like the one the cache + * lookup above leaves behind, and XLogReadBufferExtended() validates it; + * an unanswered request just leaves replay to read the block itself. + */ + if (XLogWarmPoolActive()) + { + for (int block_id = 0; block_id <= record->max_block_id; block_id++) + { + DecodedBkpBlock *block = &record->blocks[block_id]; + + if (!block->in_use || block->warm_slot == XLOGWARM_NO_SLOT) + continue; + + block->prefetch_buffer = XLogWarmCollect(block->warm_slot, + block->warm_request); + block->warm_hint = BufferIsValid(block->prefetch_buffer); + block->warm_slot = XLOGWARM_NO_SLOT; + } + } + /* * If maintenance_io_concurrency is set very low, we might have started * prefetching some but not all of the blocks referenced in the record diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 29d4df7a996ef..d485dfdda2204 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1786,6 +1786,9 @@ DecodeXLogRecord(XLogReaderState *state, blk->has_data = ((fork_flags & BKPBLOCK_HAS_DATA) != 0); blk->prefetch_buffer = InvalidBuffer; + blk->warm_slot = XLOGWARM_NO_SLOT; + blk->warm_request = 0; + blk->warm_hint = false; COPY_HEADER_FIELD(&blk->data_len, sizeof(uint16)); /* cross-check that the HAS_DATA flag is set iff data_length > 0 */ diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 0d67f256afedf..50cf22ad1c341 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -23,6 +23,7 @@ #include "access/xlogrecovery.h" #include "access/xlog_internal.h" #include "access/xlogutils.h" +#include "access/xlogwarm.h" #include "miscadmin.h" #include "storage/fd.h" #include "storage/smgr.h" @@ -432,6 +433,18 @@ XLogReadBufferForRedoExtended(XLogReaderState *record, else { *buf = XLogReadBufferExtended(rlocator, forknum, blkno, mode, prefetch_buffer); + + /* + * A hint the pool produced and replay could not use means the page + * was evicted between the worker reading it and replay reaching it — + * the pool running too far ahead of replay, which is worth knowing + * about. Hints the buffer lookup left behind say nothing about the + * pool, so only the pool's own answers are counted. + */ + if (XLogRecGetBlock(record, block_id)->warm_hint && + mode == RBM_NORMAL && *buf != prefetch_buffer) + XLogWarmCountStale(); + if (BufferIsValid(*buf)) { if (mode != RBM_ZERO_AND_LOCK && mode != RBM_ZERO_AND_CLEANUP_LOCK) @@ -493,11 +506,13 @@ XLogReadBufferExtended(RelFileLocator rlocator, ForkNumber forknum, /* Do we have a clue where the buffer might be already? */ if (BufferIsValid(recent_buffer) && - mode == RBM_NORMAL && - ReadRecentBuffer(rlocator, forknum, blkno, recent_buffer)) + mode == RBM_NORMAL) { - buffer = recent_buffer; - goto recent_buffer_fast_path; + if (ReadRecentBuffer(rlocator, forknum, blkno, recent_buffer)) + { + buffer = recent_buffer; + goto recent_buffer_fast_path; + } } /* Open the relation at smgr level */ diff --git a/src/backend/access/transam/xlogwarm.c b/src/backend/access/transam/xlogwarm.c new file mode 100644 index 0000000000000..3eb1a651ac414 --- /dev/null +++ b/src/backend/access/transam/xlogwarm.c @@ -0,0 +1,1189 @@ +/*------------------------------------------------------------------------- + * + * xlogwarm.c + * Replay prefetch worker pool: reads the pages replay is about to need + * into shared buffers, so the startup process does not spend its single + * core fetching them. + * + * Without full-page images in the WAL stream — the standby of a cluster + * running io_torn_pages_protection = double_writes — replay has to fetch + * every page it modifies. Doing that from the startup process costs about + * half of its core: issuing kernel advice for each block, then reading the + * page with its copy out of the page cache, then verifying its checksum, + * then finding a victim buffer to put it in. All of that is work another + * process can do in parallel, ahead of replay. + * + * The prefetcher already decodes WAL ahead of replay and already filters + * the blocks that must not be touched. Where it would issue advice, it + * instead publishes the block here; a worker reads it into a shared buffer + * and records which buffer that was. Replay picks the answer up as a + * recent-buffer hint, which XLogReadBufferExtended() already knows how to + * validate, so the redo path itself is unchanged. + * + * Nothing here is an obligation. A slot that no worker got to, a read that + * failed, a buffer that was evicted before replay reached it — each simply + * means replay reads the page itself, exactly as it does with the pool + * disabled. Replay never sleeps on a slot. + * + * The queue is a ring of slots recycled in publication order. One + * publisher (the startup process) and several consumers coordinate through + * the slot state alone: + * + * FREE/DONE/FAILED --(publisher)--> PUBLISHED + * PUBLISHED --(worker)-----> CLAIMED + * CLAIMED --(worker)-----> DONE | FAILED + * + * Both worker transitions are compare-and-swap, so several consumers cannot + * claim one slot. A claimed slot belongs to its worker until that worker + * leaves it: nothing else ever writes it, which is what keeps the result and + * the request id it was produced for a consistent pair. The publisher + * ignores an answer whose id is not the one it published, so an answer that + * arrives after replay has moved on is simply not picked up. + * + * A worker that exits while holding a slot returns it on the way out, so the + * ring does not shrink when a worker is signalled or throws a FATAL error; a + * worker that dies in an uglier way takes the whole cluster through a restart + * cycle, which rebuilds this ring from scratch. The one case that does cost + * a slot for good is a worker wedged inside a read that never returns, and + * replay would be wedged on that page too. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/access/transam/xlogwarm.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/xlogprefetcher.h" +#include "access/xlogwarm.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/bufmgr.h" +#include "storage/condition_variable.h" +#include "storage/ipc.h" +#include "storage/proc.h" +#include "storage/shmem.h" +#include "storage/smgr.h" +#include "tcop/tcopprot.h" +#include "utils/guc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" +#include "utils/resowner.h" +#include "utils/wait_event.h" + +/* GUCs */ +int replay_warm_workers = 0; +int replay_warm_queue_size = 512; + + +typedef enum XLogWarmState +{ + XLOGWARM_FREE = 0, + XLOGWARM_PUBLISHED, + XLOGWARM_CLAIMED, + XLOGWARM_DONE, + XLOGWARM_FAILED, +} XLogWarmState; + +typedef struct XLogWarmSlot +{ + pg_atomic_uint32 state; /* XLogWarmState */ + + /* payload, written by the publisher while it owns the slot */ + uint64 request_id; + RelFileLocator rlocator; + ForkNumber forknum; + BlockNumber blkno; + + /* result, written by the claiming worker */ + uint64 result_id; + Buffer result_buffer; +} XLogWarmSlot; + +typedef struct XLogWarmCtl +{ + ConditionVariable cv_work; + + /* publisher side */ + pg_atomic_uint64 published; + pg_atomic_uint64 dropped_full; + pg_atomic_uint64 collected; + pg_atomic_uint64 missed; + pg_atomic_uint64 stale; + pg_atomic_uint64 cancelled; + pg_atomic_uint64 released; + + /* worker side */ + pg_atomic_uint64 claimed; + pg_atomic_uint64 reads; + pg_atomic_uint64 hits; + pg_atomic_uint64 failed; + pg_atomic_uint64 vanished; + pg_atomic_uint64 discarded; + + int capacity; + pg_atomic_uint32 hand; /* where consumers start scanning */ + + /* + * What keeps the publisher from spending a system call per block. + * + * Replay publishes on the order of a hundred thousand blocks a second, + * and a condition variable signal that finds a sleeper costs a kill(2) + * every time. So the publisher only signals when nobody is looking at + * the ring, and a worker that stops looking hands the ring over in its + * place. Between them the wakeup happens once per idle pool rather than + * once per block. + * + * scanners counts workers searching the ring, and deliberately not the + * ones inside a page read: a worker in a read cannot take new work, so + * counting it would let one busy worker silence the wakeups for a pool + * that is otherwise asleep. + * + * pending counts published requests nobody has claimed. It is what a + * searching worker reads instead of walking every slot, and what tells + * the process leaving the ring whether the ring still needs somebody. It + * is raised before its slot becomes visible, so it is never lower than + * the number of published slots and a claimer never takes it below zero; + * the two are equal only when no publication is in flight. + * + * sleepers decides nothing. It exists so that "the whole pool is asleep" + * is an observable fact from outside the pool. + */ + pg_atomic_uint32 scanners; + pg_atomic_uint32 pending; + pg_atomic_uint32 sleepers; + + /* + * Bumped under ReplayWarmReadLock whenever a relation or a database is + * about to lose its files. A worker has no database connection and so + * receives no cache invalidations: without this it could keep a + * relation's cached size — or its open segments — from before the + * file changed underneath it. Read under the lock held shared. + */ + uint64 drop_epoch; + + /* + * Worker pids, published by the workers themselves. Without a database + * connection there is no pg_stat_activity row to see them in, and a pool + * nobody can see is a pool nobody can diagnose. + */ + pg_atomic_uint32 worker_pids[XLOGWARM_MAX_WORKERS]; + + /* + * The slot each worker holds, one past its index, or zero for none. A + * request that stops moving belongs to somebody, and this is how the + * owner is found: a worker inside a read that never returns is the one + * case that costs the ring a slot for good, and without this the pid to + * look at is a guess. + */ + pg_atomic_uint32 worker_slots[XLOGWARM_MAX_WORKERS]; + + XLogWarmSlot slots[FLEXIBLE_ARRAY_MEMBER]; +} XLogWarmCtl; + +static XLogWarmCtl * XLogWarmQueue = NULL; + +/* one claimed request, in the hands of the worker that claimed it */ +typedef struct XLogWarmRequest +{ + XLogWarmSlot *slot; + uint64 request_id; + RelFileLocator rlocator; + ForkNumber forknum; + BlockNumber blkno; +} XLogWarmRequest; + +static void XLogWarmWorkerExit(int code, Datum arg); +static void XLogWarmHandOff(void); + +/* publisher-private state */ +static uint64 next_request_id = 1; +static int publish_hand = 0; + +/* + * Worker-private: the drop epoch this worker's smgr state is good for. + */ +static uint64 my_drop_epoch = 0; + +/* + * Worker-private: relation sizes this worker has measured. + * + * smgr keeps such a cache too, but hands it out only to the startup process + * (smgrnblocks_cached(), "due to lack of a shared invalidation mechanism for + * changes in file size"). This pool has that mechanism — the drop epoch — + * so it can keep its own answers, and it has to: measuring walks the segment + * chain, which on a terabyte relation is a thousand file opens, and paying + * that per request leaves a worker doing nothing else. + * + * A remembered size is only ever too small, never too large: within an epoch + * no relation lost blocks, so the entry is trusted for "the block is inside + * the relation" and re-measured for anything else. Direct-mapped and small + * on purpose — replay works through a handful of relations at a time. + */ +/* + * How long a worker keeps searching an empty ring before it sleeps, in + * pg_spin_delay() rounds. It buys the publisher its silence: a worker that + * stays in the search over the gap between two publications is one the + * publisher does not have to wake, and at replay's rate those gaps are + * microseconds. Large enough to cover them, small enough that a standby + * with nothing to replay settles into sleeping workers within a moment. + */ +#define XLOGWARM_SPINS 1000 + +/* + * Worker-private: the slot this worker holds, or -1. Read on the way out to + * hand the slot back, so a worker that is signalled away does not take a slot + * of the ring with it. + */ +static int my_claimed_slot = -1; + +/* + * Worker-private: which of the two shared counts this worker is part of. + * + * Both are read on the way out. A worker can be signalled away from either + * state, and a count left standing would say the pool has a searcher, or a + * sleeper, that no longer exists — the first silences the publisher's + * wakeups, the second makes "the whole pool is asleep" untrue where it is + * relied upon. + */ +static bool my_scanning = false; +static bool my_sleeping = false; + +/* Worker-private: this worker's index in the pool, or -1 outside one. */ +static int my_worker_id = -1; + +Size +XLogWarmShmemSize(void) +{ + if (replay_warm_workers == 0) + return 0; + + return add_size(offsetof(XLogWarmCtl, slots), + mul_size(replay_warm_queue_size, sizeof(XLogWarmSlot))); +} + +void +XLogWarmShmemInit(void) +{ + bool found; + + if (replay_warm_workers == 0) + return; + + XLogWarmQueue = (XLogWarmCtl *) + ShmemInitStruct("Replay Warm Queue", XLogWarmShmemSize(), &found); + + if (!found) + { + memset(XLogWarmQueue, 0, XLogWarmShmemSize()); + ConditionVariableInit(&XLogWarmQueue->cv_work); + pg_atomic_init_u64(&XLogWarmQueue->published, 0); + pg_atomic_init_u64(&XLogWarmQueue->dropped_full, 0); + pg_atomic_init_u64(&XLogWarmQueue->collected, 0); + pg_atomic_init_u64(&XLogWarmQueue->missed, 0); + pg_atomic_init_u64(&XLogWarmQueue->stale, 0); + pg_atomic_init_u64(&XLogWarmQueue->cancelled, 0); + pg_atomic_init_u64(&XLogWarmQueue->released, 0); + pg_atomic_init_u64(&XLogWarmQueue->claimed, 0); + pg_atomic_init_u64(&XLogWarmQueue->reads, 0); + pg_atomic_init_u64(&XLogWarmQueue->hits, 0); + pg_atomic_init_u64(&XLogWarmQueue->failed, 0); + pg_atomic_init_u64(&XLogWarmQueue->vanished, 0); + pg_atomic_init_u64(&XLogWarmQueue->discarded, 0); + pg_atomic_init_u32(&XLogWarmQueue->hand, 0); + pg_atomic_init_u32(&XLogWarmQueue->scanners, 0); + pg_atomic_init_u32(&XLogWarmQueue->pending, 0); + pg_atomic_init_u32(&XLogWarmQueue->sleepers, 0); + XLogWarmQueue->drop_epoch = 0; + XLogWarmQueue->capacity = replay_warm_queue_size; + + for (int i = 0; i < XLOGWARM_MAX_WORKERS; i++) + { + pg_atomic_init_u32(&XLogWarmQueue->worker_pids[i], 0); + pg_atomic_init_u32(&XLogWarmQueue->worker_slots[i], 0); + } + + for (int i = 0; i < replay_warm_queue_size; i++) + pg_atomic_init_u32(&XLogWarmQueue->slots[i].state, XLOGWARM_FREE); + } +} + +/* + * True when blocks may be handed to the pool. The pool exists for the + * duration of the postmaster; it is idle whenever nothing publishes. + */ +bool +XLogWarmPoolActive(void) +{ + return XLogWarmQueue != NULL; +} + +/* + * Publish one block for a worker to read. + * + * Returns the slot the request went into and sets *request_id, or + * XLOGWARM_NO_SLOT when the ring has no reusable slot, in which case the + * caller simply leaves the block unwarmed. + */ +int +XLogWarmPublish(RelFileLocator rlocator, ForkNumber forknum, + BlockNumber blkno, uint64 *request_id) +{ + XLogWarmSlot *slot = NULL; + int capacity = XLogWarmQueue->capacity; + int slot_no = XLOGWARM_NO_SLOT; + + Assert(XLogWarmQueue != NULL); + + /* + * A slot still PUBLISHED or CLAIMED belongs to an earlier request that + * has not been dealt with yet, and taking it back is never right — a + * claimed slot is being written by its worker, and a second writer would + * tear the result apart. So the publisher writes only slots nobody + * holds, and looks past the ones somebody does: a single slow worker must + * not stop the ring, which it would if publication insisted on one slot. + */ + for (int i = 0; i < capacity; i++) + { + int candidate = (publish_hand + i) % capacity; + uint32 state = pg_atomic_read_u32(&XLogWarmQueue->slots[candidate].state); + + if (state != XLOGWARM_PUBLISHED && state != XLOGWARM_CLAIMED) + { + slot_no = candidate; + slot = &XLogWarmQueue->slots[candidate]; + break; + } + } + + /* + * Every slot is spoken for: the pool is behind, and this block goes + * unwarmed. That is the pool's back-pressure — replay reads the block + * itself and never waits for a worker. + */ + if (slot == NULL) + { + pg_atomic_fetch_add_u64(&XLogWarmQueue->dropped_full, 1); + return XLOGWARM_NO_SLOT; + } + + slot->request_id = next_request_id; + slot->rlocator = rlocator; + slot->forknum = forknum; + slot->blkno = blkno; + + /* + * The count covers the slot before anybody else can see it. Raising it + * afterwards would let a worker claim the slot and lower a count that had + * not been raised yet, which on an unsigned counter is not a small error. + */ + pg_atomic_fetch_add_u32(&XLogWarmQueue->pending, 1); + + /* the payload must be visible before a worker can see the state */ + pg_write_barrier(); + pg_atomic_write_u32(&slot->state, XLOGWARM_PUBLISHED); + + *request_id = next_request_id++; + publish_hand = (slot_no + 1) % XLogWarmQueue->capacity; + pg_atomic_fetch_add_u64(&XLogWarmQueue->published, 1); + + /* + * Somebody already searching the ring will find this request without + * being told, so the signal — and the system call inside it — is only + * for a pool where nobody is. The barrier is what makes the two sides + * meet: a write barrier would order the publication, but not this load + * against it, and the pairing needs the load to come after. A worker on + * its way out of the search closes the other half of the window in + * XLogWarmHandOff(). + */ + pg_memory_barrier(); + if (pg_atomic_read_u32(&XLogWarmQueue->scanners) == 0) + ConditionVariableSignal(&XLogWarmQueue->cv_work); + + return slot_no; +} + +/* + * Collect the buffer a worker read for this request, if it has one. + * + * Returns InvalidBuffer when the request was never claimed, is still being + * read, failed, or the slot has moved on to another request: in every one + * of those cases replay reads the page itself. The returned buffer is only + * a hint and is not pinned — the caller validates it, as it does for any + * recent-buffer hint. + */ +Buffer +XLogWarmCollect(int slot_no, uint64 request_id) +{ + XLogWarmSlot *slot; + Buffer buffer; + + Assert(XLogWarmQueue != NULL); + Assert(slot_no >= 0 && slot_no < XLogWarmQueue->capacity); + + slot = &XLogWarmQueue->slots[slot_no]; + + if (pg_atomic_read_u32(&slot->state) != XLOGWARM_DONE) + { + pg_atomic_fetch_add_u64(&XLogWarmQueue->missed, 1); + return InvalidBuffer; + } + + /* the state must be observed before the result it advertises */ + pg_read_barrier(); + + if (slot->result_id != request_id) + { + pg_atomic_fetch_add_u64(&XLogWarmQueue->missed, 1); + return InvalidBuffer; + } + + buffer = slot->result_buffer; + if (!BufferIsValid(buffer)) + { + pg_atomic_fetch_add_u64(&XLogWarmQueue->missed, 1); + return InvalidBuffer; + } + + pg_atomic_fetch_add_u64(&XLogWarmQueue->collected, 1); + return buffer; +} + +/* + * Withdraw every outstanding request. + * + * The prefetcher throws its decoded records away when the read position + * moves (XLogPrefetcherBeginRead) and at the end of recovery, taking the + * slot references with them. Requests nobody claimed are freed here; + * requests a worker is inside of are left alone — that worker will finish + * into DONE, and the slot becomes reusable on the next pass of the ring. + */ +void +XLogWarmCancelAll(void) +{ + if (XLogWarmQueue == NULL) + return; + + for (int i = 0; i < XLogWarmQueue->capacity; i++) + { + XLogWarmSlot *slot = &XLogWarmQueue->slots[i]; + uint32 expected = XLOGWARM_PUBLISHED; + + if (pg_atomic_compare_exchange_u32(&slot->state, &expected, + XLOGWARM_FREE)) + { + /* the request is gone, and so is the need for somebody to take it */ + pg_atomic_fetch_sub_u32(&XLogWarmQueue->pending, 1); + pg_atomic_fetch_add_u64(&XLogWarmQueue->cancelled, 1); + } + } + + publish_hand = 0; +} + +/* + * Take the pool out of the way of a relation about to lose its buffers. + * + * DropRelationBuffers() requires that no other process be loading pages of + * the relation into buffers while it runs (bufmgr.c:5081-5083), and a worker + * reading ahead of replay is exactly such a process — one that usually wins + * the race, since its file descriptor keeps reaching an unlinked file and the + * read succeeds. Held across the buffer drop *and* the file operation that + * follows it, this leaves a worker two possibilities and no third: it either + * finished before the drop scanned the pool, in which case the scan removes + * its page, or it starts afterwards and finds the relation gone. + * + * The wait is one page read long, and only relation drops and truncations + * ever wait at all. + */ +void +XLogWarmDropBegin(void) +{ + if (XLogWarmQueue == NULL) + return; + + LWLockAcquire(ReplayWarmReadLock, LW_EXCLUSIVE); + XLogWarmQueue->drop_epoch++; +} + +void +XLogWarmDropEnd(void) +{ + if (XLogWarmQueue != NULL) + LWLockRelease(ReplayWarmReadLock); +} + +/* + * A warmed buffer no longer held the page by the time replay asked for it. + * Only the recent-buffer validation can tell, so it reports it here. + */ +void +XLogWarmCountStale(void) +{ + if (XLogWarmQueue != NULL) + pg_atomic_fetch_add_u64(&XLogWarmQueue->stale, 1); +} + +/* + * Report the pids of the running workers into caller-provided storage of + * XLOGWARM_MAX_WORKERS entries, returning how many were found. + */ +int +XLogWarmGetWorkerPids(int *pids, int *slots) +{ + int found = 0; + + if (XLogWarmQueue == NULL) + return 0; + + for (int i = 0; i < XLOGWARM_MAX_WORKERS; i++) + { + uint32 pid = pg_atomic_read_u32(&XLogWarmQueue->worker_pids[i]); + + if (pid != 0) + { + if (slots != NULL) + slots[found] = + (int) pg_atomic_read_u32(&XLogWarmQueue->worker_slots[i]) - 1; + pids[found++] = (int) pid; + } + } + + return found; +} + +/* + * Read the counters out, for monitoring and for the tests. Returns false + * when the pool is not configured, leaving *stats untouched. + */ +bool +XLogWarmGetStats(XLogWarmStats * stats) +{ + if (XLogWarmQueue == NULL) + return false; + + stats->published = pg_atomic_read_u64(&XLogWarmQueue->published); + stats->dropped_full = pg_atomic_read_u64(&XLogWarmQueue->dropped_full); + stats->collected = pg_atomic_read_u64(&XLogWarmQueue->collected); + stats->missed = pg_atomic_read_u64(&XLogWarmQueue->missed); + stats->stale = pg_atomic_read_u64(&XLogWarmQueue->stale); + stats->cancelled = pg_atomic_read_u64(&XLogWarmQueue->cancelled); + stats->released = pg_atomic_read_u64(&XLogWarmQueue->released); + stats->claimed = pg_atomic_read_u64(&XLogWarmQueue->claimed); + stats->reads = pg_atomic_read_u64(&XLogWarmQueue->reads); + stats->hits = pg_atomic_read_u64(&XLogWarmQueue->hits); + stats->failed = pg_atomic_read_u64(&XLogWarmQueue->failed); + stats->vanished = pg_atomic_read_u64(&XLogWarmQueue->vanished); + stats->discarded = pg_atomic_read_u64(&XLogWarmQueue->discarded); + + return true; +} + +/* + * How many slots are waiting for a worker, and how many a worker holds. + * + * A running total says what the pool has done; this says what it is doing, + * which is what a test needs to arrange anything around a request in flight. + */ +void +XLogWarmGetSlotCounts(int *published, int *claimed) +{ + *published = 0; + *claimed = 0; + + if (XLogWarmQueue == NULL) + return; + + /* + * This walk is not a snapshot: slots change state under it, so what it + * returns is what the ring looked like slot by slot rather than at any + * one instant. Good enough to see a request in flight, and not good + * enough to check a counter against while the pool is working. + */ + + for (int i = 0; i < XLogWarmQueue->capacity; i++) + { + switch (pg_atomic_read_u32(&XLogWarmQueue->slots[i].state)) + { + case XLOGWARM_PUBLISHED: + (*published)++; + break; + case XLOGWARM_CLAIMED: + (*claimed)++; + break; + default: + break; + } + } +} + +/* + * What the pool's processes are doing right now: how many are searching the + * ring, how many requests are waiting for one of them, and how many are + * asleep. Unlike the slot walk above these are single counters, so each is + * a real value rather than a scan; "the whole pool is asleep" is a fact a + * test can wait for here. + */ +void +XLogWarmGetPoolState(int *scanners, int *pending, int *sleepers) +{ + *scanners = 0; + *pending = 0; + *sleepers = 0; + + if (XLogWarmQueue == NULL) + return; + + *scanners = (int) pg_atomic_read_u32(&XLogWarmQueue->scanners); + *pending = (int) pg_atomic_read_u32(&XLogWarmQueue->pending); + *sleepers = (int) pg_atomic_read_u32(&XLogWarmQueue->sleepers); +} + +/* + * Read one published block into shared buffers. + * + * Runs inside the worker's own resource owner: replay may drop or truncate + * the relation between publication and this read, so any smgr error has to + * be survivable. + */ +static void +XLogWarmDoOne(XLogWarmSlot * slot, uint64 request_id, + RelFileLocator rlocator, ForkNumber forknum, BlockNumber blkno) +{ + SMgrRelation smgr; + BlockNumber nblocks; + Buffer buffer = InvalidBuffer; + uint32 expected; + bool failed = false; + + /* + * A test can park a worker here to hold a slot claimed while it arranges + * what happens next. It sits outside the interlock below on purpose: a + * worker parked while holding that lock would stop replay from dropping + * anything at all. + */ + INJECTION_POINT("replay-warm-before-read", NULL); + + PG_TRY(); + { + /* see XLogWarmDropBegin(): this is the whole reason it exists */ + LWLockAcquire(ReplayWarmReadLock, LW_SHARED); + + /* + * Something lost its files since this worker last looked. Nothing + * tells a process without a database connection that, so it throws + * its own smgr state away and starts from the files as they are now. + */ + if (XLogWarmQueue->drop_epoch != my_drop_epoch) + { + smgrreleaseall(); + my_drop_epoch = XLogWarmQueue->drop_epoch; + } + + smgr = smgropen(rlocator, INVALID_PROC_NUMBER); + + /* + * Is the block still there? Replay may have dropped or truncated the + * relation between publication and now — the ordinary outcome of + * running ahead of it, and the outcome the interlock guarantees for a + * request that gets here after a drop. + * + * Asking the file system outright costs more than it looks: + * smgrexists() closes the fork before answering (mdexists() skips + * that only in the startup process) and the smgrnblocks() behind it + * then reopens the segment chain from the beginning, so on a terabyte + * relation one question is a thousand file opens. + * + * The size this worker last saw is therefore taken from the relation + * itself: smgrnblocks() records it there, smgrrelease() clears it, + * and the smgrreleaseall() above is what clears it after a drop. + * Reading the field directly is how the rest of the tree uses it — + * see the comment on smgrnblocks_cached(), whose InRecovery test is + * about the startup process and so never lets a worker in. + */ + nblocks = smgr->smgr_cached_nblocks[forknum]; + + if (nblocks == InvalidBlockNumber) + { + /* + * Nothing known about this fork: either the worker has not + * touched it since the last drop, or it has never touched it at + * all. This is the one place that pays for the expensive + * question, and it is also the only place that can tell a + * relation whose files are gone from one that is merely shorter + * than the request expects. + */ + if (!smgrexists(smgr, forknum)) + { + failed = true; + pg_atomic_fetch_add_u64(&XLogWarmQueue->vanished, 1); + } + else + nblocks = smgrnblocks(smgr, forknum); + } + + if (!failed && blkno >= nblocks) + { + /* + * The remembered size may simply predate an extension: nothing + * tells a worker that replay has grown a relation, so a known + * size is a lower bound. mdnblocks() resumes from the last open + * segment, which makes this an lseek rather than another walk. + */ + nblocks = smgrnblocks(smgr, forknum); + + if (blkno >= nblocks) + { + failed = true; + pg_atomic_fetch_add_u64(&XLogWarmQueue->vanished, 1); + } + } + + if (failed) + { + /* the block is gone; there is nothing to warm */ + } + else if (BufferIsValid(buffer = LookupSharedBuffer(smgr, forknum, blkno))) + { + /* + * Already resident: not a read, but still the answer replay + * wants, so hand the buffer on as if we had read it. + * + * This is also where that question gets answered for + * pg_stat_recovery_prefetch. Replay used to ask it before + * publishing and count the answer itself; with the pool running + * it no longer asks, so the count belongs to whoever does. + */ + pg_atomic_fetch_add_u64(&XLogWarmQueue->hits, 1); + XLogPrefetchCountHit(); + } + else + { + buffer = ReadBufferWithoutRelcache(rlocator, forknum, blkno, + RBM_NORMAL, NULL, true); + pg_atomic_fetch_add_u64(&XLogWarmQueue->reads, 1); + XLogPrefetchCountPrefetch(); + + /* + * Hand the buffer number on and let go: holding pins ahead of + * replay would pin down a slice of the buffer pool, and replay + * validates the hint anyway. + */ + ReleaseBuffer(buffer); + } + + LWLockRelease(ReplayWarmReadLock); + } + PG_CATCH(); + { + /* an unreadable block is not this pool's problem to solve */ + buffer = InvalidBuffer; + failed = true; + pg_atomic_fetch_add_u64(&XLogWarmQueue->failed, 1); + + MemoryContextSwitchTo(TopMemoryContext); + FlushErrorState(); + + /* + * Whatever the failed read was holding goes back here: the + * lightweight locks it took, including the interlock above, and then + * the pins and the buffer I/O owned by the aux-process resource + * owner. Releasing the owner is what repairs a read interrupted + * mid-flight — it hands the buffer's I/O back, so whoever waits on + * that buffer can retry instead of waiting on a process that is no + * longer reading. + */ + LWLockReleaseAll(); + ReleaseAuxProcessResources(false); + } + PG_END_TRY(); + + slot->result_id = request_id; + slot->result_buffer = buffer; + + /* the result must be visible before the state that advertises it */ + pg_write_barrier(); + + /* + * The slot is ours until we leave it, so this compare-and-swap is a + * statement of that invariant rather than a race to win; a failure would + * mean somebody else wrote a claimed slot, and the counter says so. + */ + expected = XLOGWARM_CLAIMED; + if (!pg_atomic_compare_exchange_u32(&slot->state, &expected, + failed ? XLOGWARM_FAILED : XLOGWARM_DONE)) + pg_atomic_fetch_add_u64(&XLogWarmQueue->discarded, 1); +} + +/* + * Leave the pool tidily. + * + * Two things outlive this process if it does not: the pid it advertised, + * which would point at a process that no longer exists, and the slot it + * holds, which no one else may write and which would therefore shrink the + * ring for the rest of the cluster's life. This runs on the way out of a + * signalled or FATAL exit — the paths that leave shared memory in place. + */ +static void +XLogWarmWorkerExit(int code, Datum arg) +{ + int worker_id = DatumGetInt32(arg); + + if (XLogWarmQueue == NULL) + return; + + /* + * Leave the pool's counts, and leave them in the order a live worker + * would. This callback runs in the before_shmem_exit phase, and the + * teardown that takes a process off a condition variable's wait list + * happens later, in ProcKill(): until then a publisher's signal can still + * land on this process, which is about to stop reading its latch. So the + * wait list goes first, then the counts, then the hand-off — a searcher + * that leaves without one takes the ring's only promised searcher with + * it. + */ + if (my_scanning || my_sleeping) + { + ConditionVariableCancelSleep(); + + if (my_sleeping) + { + pg_atomic_fetch_sub_u32(&XLogWarmQueue->sleepers, 1); + my_sleeping = false; + } + if (my_scanning) + { + pg_atomic_fetch_sub_u32(&XLogWarmQueue->scanners, 1); + my_scanning = false; + } + + XLogWarmHandOff(); + } + + if (my_claimed_slot >= 0) + { + XLogWarmSlot *slot = &XLogWarmQueue->slots[my_claimed_slot]; + uint32 expected = XLOGWARM_CLAIMED; + + my_claimed_slot = -1; + + if (pg_atomic_compare_exchange_u32(&slot->state, &expected, + XLOGWARM_FAILED)) + pg_atomic_fetch_add_u64(&XLogWarmQueue->released, 1); + } + + pg_atomic_write_u32(&XLogWarmQueue->worker_slots[worker_id], 0); + pg_atomic_write_u32(&XLogWarmQueue->worker_pids[worker_id], 0); +} + +/* + * Claim one published slot. Returns false when the ring holds nothing to do. + * + * The caller must be counted in scanners while this runs: that is what tells + * a publisher it need not spend a wakeup, and the promise behind it is that + * this process looks at the ring after the publication became visible. + */ +static bool +XLogWarmClaimOne(XLogWarmRequest * req) +{ + int capacity = XLogWarmQueue->capacity; + uint32 start; + + /* + * No request outstanding, and the counter says so without touching a + * slot. A worker that searched the whole ring every time it looked would + * spend the pool's cores dragging several hundred shared cache lines + * between them, which is what makes waiting here cheap enough to prefer + * to sleeping. + */ + if (pg_atomic_read_u32(&XLogWarmQueue->pending) == 0) + return false; + + start = pg_atomic_fetch_add_u32(&XLogWarmQueue->hand, 1); + + for (int i = 0; i < capacity; i++) + { + XLogWarmSlot *slot = &XLogWarmQueue->slots[(start + i) % capacity]; + uint32 expected = XLOGWARM_PUBLISHED; + + if (!pg_atomic_compare_exchange_u32(&slot->state, &expected, + XLOGWARM_CLAIMED)) + continue; + + pg_atomic_fetch_sub_u32(&XLogWarmQueue->pending, 1); + + /* the state was observed before the payload it advertises */ + pg_read_barrier(); + + req->slot = slot; + req->request_id = slot->request_id; + req->rlocator = slot->rlocator; + req->forknum = slot->forknum; + req->blkno = slot->blkno; + + /* + * From here until the slot is finished this worker owns it, and says + * so where its exit callback can see it. + */ + my_claimed_slot = (start + i) % capacity; + if (my_worker_id >= 0) + pg_atomic_write_u32(&XLogWarmQueue->worker_slots[my_worker_id], + (uint32) my_claimed_slot + 1); + + pg_atomic_fetch_add_u64(&XLogWarmQueue->claimed, 1); + return true; + } + + return false; +} + +/* + * Hand the ring over on the way out of the search. + * + * A publisher that saw this process searching stayed quiet, so a process + * that stops searching — to read a page, or for good — has to make sure + * somebody else is looking if anything is still outstanding. Otherwise the + * work it was trusted to find would sit in front of a sleeping pool until + * the next publication happened to wake somebody. + * + * Two things must already be true at the call: this process has left the + * scanners count, and it is not itself on the wait list — a signal issued + * while still registered could pick the signaller and leave the others + * asleep. + */ +static void +XLogWarmHandOff(void) +{ + /* + * Pairs with the publisher: it publishes and then reads scanners, this + * side leaves scanners and then reads pending, and a full barrier on both + * sides is what guarantees at least one of the two sees the other. + */ + pg_memory_barrier(); + + if (pg_atomic_read_u32(&XLogWarmQueue->pending) > 0 && + pg_atomic_read_u32(&XLogWarmQueue->scanners) == 0) + ConditionVariableSignal(&XLogWarmQueue->cv_work); +} + +/* + * Stop and start searching, as the states above are entered and left. + */ +static void +XLogWarmStopScanning(void) +{ + Assert(my_scanning); + pg_atomic_fetch_sub_u32(&XLogWarmQueue->scanners, 1); + my_scanning = false; + XLogWarmHandOff(); +} + +static void +XLogWarmStartScanning(void) +{ + Assert(!my_scanning); + pg_atomic_fetch_add_u32(&XLogWarmQueue->scanners, 1); + my_scanning = true; +} + +/* + * Serve a claimed request and go back to searching. + */ +static void +XLogWarmServe(XLogWarmRequest * req) +{ + XLogWarmDoOne(req->slot, req->request_id, req->rlocator, req->forknum, + req->blkno); + my_claimed_slot = -1; + if (my_worker_id >= 0) + pg_atomic_write_u32(&XLogWarmQueue->worker_slots[my_worker_id], 0); +} + +/* + * Leave the search to serve what this worker just claimed, then rejoin it. + * + * The injection point catches a worker in the state the hand-off exists for: + * holding a request, and still counted as a searcher, so a publication + * landing now is one the publisher will leave to this process. It is safe + * to park here — the worker is not on the pool's wait list at this point, so + * the waiting the injection point does of its own cannot disturb it. + */ +static void +XLogWarmServeAsScanner(XLogWarmRequest * req) +{ + INJECTION_POINT("replay-warm-claimed", NULL); + XLogWarmStopScanning(); + XLogWarmServe(req); + XLogWarmStartScanning(); +} + +/* + * Register the pool. Like the DWB cleaner pool, a worker slot shortage is + * fatal rather than silent: a smaller pool than the operator configured is + * a performance surprise nobody asked for. + */ +void +XLogWarmWorkersRegister(void) +{ + BackgroundWorker bgw; + int free_slots; + + if (replay_warm_workers == 0) + return; + + free_slots = max_worker_processes - GetNumRegisteredBackgroundWorkers(); + if (replay_warm_workers > free_slots) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"replay_warm_workers\" (%d) needs more \"max_worker_processes\" slots than remain free (%d)", + replay_warm_workers, free_slots), + errhint("Increase \"max_worker_processes\" or decrease \"replay_warm_workers\"."))); + + for (int i = 0; i < replay_warm_workers; i++) + { + memset(&bgw, 0, sizeof(bgw)); + + /* + * No database connection: the workers deal in relation locators and + * shared buffers only. That also lets them start at postmaster + * start, so they serve crash recovery on a primary from the first + * record, not only a standby past consistency. + */ + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS; + bgw.bgw_start_time = BgWorkerStart_PostmasterStart; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "XLogWarmWorkerMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, "replay warm worker %d", i); + snprintf(bgw.bgw_type, BGW_MAXLEN, "replay warm worker"); + bgw.bgw_restart_time = 1; + bgw.bgw_notify_pid = 0; + bgw.bgw_main_arg = Int32GetDatum(i); + + RegisterBackgroundWorker(&bgw); + } +} + +/* + * Main loop: serve published slots, sleep when there is nothing published. + */ +void +XLogWarmWorkerMain(Datum main_arg) +{ + int worker_id; + + pqsignal(SIGHUP, SignalHandlerForConfigReload); + + /* + * die, not a shutdown flag: a worker exiting with code 0 is unregistered + * for good, so one stray SIGTERM would permanently shrink the pool. + */ + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* + * ReadBufferWithoutRelcache pins buffers and registers its buffer I/O + * with CurrentResourceOwner; that registration is what releases an + * interrupted read if an ERROR throws the worker out of a slot. The + * aux-process owner provides both the owner and its shmem-exit release. + */ + CreateAuxProcessResourceOwner(); + + Assert(XLogWarmQueue != NULL); + + worker_id = DatumGetInt32(main_arg); + Assert(worker_id >= 0 && worker_id < XLOGWARM_MAX_WORKERS); + my_worker_id = worker_id; + pg_atomic_write_u32(&XLogWarmQueue->worker_pids[worker_id], MyProcPid); + before_shmem_exit(XLogWarmWorkerExit, Int32GetDatum(worker_id)); + + /* this process is searching the ring from here on */ + XLogWarmStartScanning(); + + for (;;) + { + XLogWarmRequest req; + int spins; + + /* the CFI is what turns a pending die() into the FATAL exit */ + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + + if (XLogWarmClaimOne(&req)) + { + XLogWarmServeAsScanner(&req); + continue; + } + + /* + * Nothing to do this instant, which at replay's publication rate + * usually means "not yet" rather than "not at all". Stay in the + * search for a while: a worker that is still counted is a worker the + * publisher does not have to wake, and the whole point of the counts + * is to keep that system call out of replay's way. The budget is + * small enough that an idle standby settles into sleeping workers + * rather than spinning ones. + */ + for (spins = XLOGWARM_SPINS; spins > 0; spins--) + { + if (pg_atomic_read_u32(&XLogWarmQueue->pending) > 0) + break; + pg_spin_delay(); + } + + if (spins > 0 && XLogWarmClaimOne(&req)) + { + XLogWarmServeAsScanner(&req); + continue; + } + + /* + * Give up and sleep, without losing a wakeup: join the wait list + * first, then leave the searchers, then recheck. In that order a + * publisher that reads no searchers is reading about a process that + * is already waiting, and a publisher that reads one is reading about + * a process that has yet to look again. + */ + ConditionVariablePrepareToSleep(&XLogWarmQueue->cv_work); + pg_atomic_fetch_sub_u32(&XLogWarmQueue->scanners, 1); + my_scanning = false; + + if (XLogWarmClaimOne(&req)) + { + /* leave the wait list before the hand-off can pick this process */ + ConditionVariableCancelSleep(); + XLogWarmHandOff(); + XLogWarmServe(&req); + XLogWarmStartScanning(); + continue; + } + + pg_atomic_fetch_add_u32(&XLogWarmQueue->sleepers, 1); + my_sleeping = true; + + ConditionVariableSleep(&XLogWarmQueue->cv_work, + WAIT_EVENT_REPLAY_WARM_MAIN); + + /* + * ConditionVariableSleep() puts this process back on the wait list + * before it returns, so become a searcher while still registered: + * between the two counts there must be no moment where this process + * is neither searching nor waiting, or a publisher could look at that + * moment and decide the ring needs nobody. + */ + pg_atomic_fetch_sub_u32(&XLogWarmQueue->sleepers, 1); + my_sleeping = false; + pg_atomic_fetch_add_u32(&XLogWarmQueue->scanners, 1); + my_scanning = true; + ConditionVariableCancelSleep(); + } +} diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index f0f88838dc21a..16fddfd855c2f 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -41,6 +41,7 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" +#include "storage/dwb.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" @@ -166,6 +167,14 @@ static const char *const excludeDirContents[] = /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ PG_DYNSHMEM_DIR, + /* + * The double write buffer ring holds page copies belonging to the + * instance being backed up; they are meaningless anywhere else and must + * never be applied to a restored cluster. A restored cluster cold-starts + * a fresh ring instead, see DWBStartup(). + */ + DWB_DIR, + /* Contents removed on startup, see AsyncShmemInit(). */ "pg_notify", diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5eb6caffe6d1a..c2610d565fef6 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -32,6 +32,7 @@ #include "access/xloginsert.h" #include "access/xlogrecovery.h" #include "access/xlogutils.h" +#include "access/xlogwarm.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -1851,6 +1852,14 @@ dropdb(const char *dbname, bool missing_ok, bool force) */ ReplicationSlotsDropDBSlots(db_id); + /* + * As in dbase_redo(): hold the replay warm pool off until the files are + * gone. A worker that was still finishing a read when recovery ended + * could otherwise put a page of this database back into the buffer pool + * after the drop below (see XLogWarmDropBegin()). + */ + XLogWarmDropBegin(); + /* * Drop pages for this database that are in the shared buffer cache. This * is important to ensure that no remaining backend tries to write out a @@ -1880,6 +1889,8 @@ dropdb(const char *dbname, bool missing_ok, bool force) */ remove_dbtablespaces(db_id); + XLogWarmDropEnd(); + /* * Close pg_database, but keep lock till commit. */ @@ -3431,6 +3442,13 @@ dbase_redo(XLogReaderState *record) /* Drop any database-specific replication slots */ ReplicationSlotsDropDBSlots(xlrec->db_id); + /* + * Keep the replay warm pool away until the directories are gone, so a + * worker cannot load a page of this database into buffers behind the + * drop below (see XLogWarmDropBegin()). + */ + XLogWarmDropBegin(); + /* Drop pages for this database that are in the shared buffer cache */ DropDatabaseBuffers(xlrec->db_id); @@ -3455,6 +3473,8 @@ dbase_redo(XLogReaderState *record) pfree(dst_path); } + XLogWarmDropEnd(); + if (InHotStandby) { /* diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index 1ad65c237c34e..fe49d10783f60 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -21,6 +21,8 @@ #include "postmaster/postmaster.h" #include "replication/logicallauncher.h" #include "replication/logicalworker.h" +#include "access/xlogwarm.h" +#include "storage/dwb.h" #include "storage/ipc.h" #include "storage/latch.h" #include "storage/lwlock.h" @@ -124,6 +126,15 @@ static const struct { "ApplyLauncherMain", ApplyLauncherMain }, + { + "DWBRetireWorkerMain", DWBRetireWorkerMain + }, + { + "DWBCleanerWorkerMain", DWBCleanerWorkerMain + }, + { + "XLogWarmWorkerMain", XLogWarmWorkerMain + }, { "ApplyWorkerMain", ApplyWorkerMain }, @@ -936,11 +947,27 @@ BackgroundWorkerUnblockSignals(void) * function of a module library that's loaded by shared_preload_libraries; * otherwise it will have no effect. */ +/* static background workers registered so far (against max_worker_processes) */ +static int numworkers = 0; + +/* + * Report how many static background workers have been registered so far. + * + * RegisterBackgroundWorker only LOGs when the limit is exceeded, so an + * in-core pool registered late in startup (after the logical replication + * launcher) uses this to verify that its workers actually fit and to fail + * loudly otherwise. + */ +int +GetNumRegisteredBackgroundWorkers(void) +{ + return numworkers; +} + void RegisterBackgroundWorker(BackgroundWorker *worker) { RegisteredBgWorker *rw; - static int numworkers = 0; /* * Static background workers can only be registered in the postmaster @@ -997,7 +1024,7 @@ RegisterBackgroundWorker(BackgroundWorker *worker) * towards the MAX_BACKENDS limit elsewhere. For now, it doesn't seem * important to relax this restriction. */ - if (++numworkers > max_worker_processes) + if (numworkers >= max_worker_processes) { ereport(LOG, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), @@ -1030,6 +1057,12 @@ RegisterBackgroundWorker(BackgroundWorker *worker) rw->rw_terminate = false; dlist_push_head(&BackgroundWorkerList, &rw->rw_lnode); + + /* + * Count only successful registrations, so that + * GetNumRegisteredBackgroundWorkers() reflects the actual list. + */ + numworkers++; } /* diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index 72f5acceec78d..8efa39ea4054a 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -42,6 +42,7 @@ #include "storage/buf_internals.h" #include "storage/bufmgr.h" #include "storage/condition_variable.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/lwlock.h" #include "storage/proc.h" @@ -231,9 +232,15 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len) ProcessMainLoopInterrupts(); /* - * Do one cycle of dirty-buffer writing. + * Do one cycle of dirty-buffer writing. While a double write buffer + * stall has us paused (Stage A backpressure), sit the round out + * instead of queueing more flushes behind an exhausted ring; + * user-facing paths keep their reserve, we retry after the delay. */ - can_hibernate = BgBufferSync(&wb_context); + if (DWBIsEnabled() && DWBWritesPaused()) + can_hibernate = false; + else + can_hibernate = BgBufferSync(&wb_context); /* Report pending statistics to the cumulative stats system */ pgstat_report_bgwriter(); diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 3914ac8da2622..d093623bc2d7b 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -52,6 +52,7 @@ #include "storage/aio_subsys.h" #include "storage/bufmgr.h" #include "storage/condition_variable.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -163,7 +164,7 @@ static pg_time_t last_xlog_switch_time; static void ProcessCheckpointerInterrupts(void); static void CheckArchiveTimeout(void); -static bool IsCheckpointOnSchedule(double progress); +static bool IsCheckpointOnSchedule(double progress, double slack); static bool ImmediateCheckpointRequested(void); static bool CompactCheckpointerRequestQueue(void); static void UpdateSharedMemoryConfig(void); @@ -772,6 +773,7 @@ void CheckpointWriteDelay(int flags, double progress) { static int absorb_counter = WRITES_PER_ABSORB; + bool nap; /* Do nothing if checkpoint is being executed by non-checkpointer process */ if (!AmCheckpointerProcess()) @@ -780,12 +782,38 @@ CheckpointWriteDelay(int flags, double progress) /* * Perform the usual duties and take a nap, unless we're behind schedule, * in which case we just try to catch up as quickly as possible. + * + * Under double_writes an active cleaner pool competes with us for the + * ring and the array; while its bin queue is hot we keep napping a little + * past the schedule, spending a bounded slice of the completion-target + * slack so our writes land in the quieter phases of the window. The + * margin is recomputed from the live target on every check (it is + * SIGHUP-reloadable mid checkpoint) and caps the extra schedule lag; once + * it is used up, pacing is the stock behavior no matter the pressure. */ + nap = false; if (!(flags & CHECKPOINT_IMMEDIATE) && !ShutdownXLOGPending && !ShutdownRequestPending && - !ImmediateCheckpointRequested() && - IsCheckpointOnSchedule(progress)) + !ImmediateCheckpointRequested()) + { + if (IsCheckpointOnSchedule(progress, 0.0)) + nap = true; + else + { + double margin = Min(0.05, + (1.0 - CheckPointCompletionTarget) / 2.0); + + if (margin > 0.0 && DWBCleanerQueueHot() && + IsCheckpointOnSchedule(progress, margin)) + { + DWBCleanerCountPressureNap(); + nap = true; + } + } + } + + if (nap) { if (ConfigReloadPending) { @@ -839,7 +867,7 @@ CheckpointWriteDelay(int flags, double progress) * than the elapsed time/segments. */ static bool -IsCheckpointOnSchedule(double progress) +IsCheckpointOnSchedule(double progress, double slack) { XLogRecPtr recptr; struct timeval now; @@ -848,8 +876,12 @@ IsCheckpointOnSchedule(double progress) Assert(ckpt_active); - /* Scale progress according to checkpoint_completion_target. */ - progress *= CheckPointCompletionTarget; + /* + * Scale progress according to checkpoint_completion_target. The slack + * term is added after the scaling: it grants the caller that much extra + * elapsed fraction before the answer flips to "behind". + */ + progress = progress * CheckPointCompletionTarget + slack; /* * Check against the cached value first. Only do the more expensive diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 5124d39e5c2fb..0ae9718f3fb92 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -110,6 +110,8 @@ #include "replication/slotsync.h" #include "replication/walsender.h" #include "storage/aio_subsys.h" +#include "access/xlogwarm.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/io_worker.h" #include "storage/ipc.h" @@ -927,6 +929,22 @@ PostmasterMain(int argc, char *argv[]) */ ApplyLauncherRegister(); + /* + * Register the double write buffer retire workers, for the same reason: + * the ring cannot circulate without them. + */ + DWBRetireWorkersRegister(); + + /* And the double write buffer cleaner pool feeding off the bgwriter. */ + DWBCleanerWorkersRegister(); + + /* + * The replay warm pool, which fetches pages ahead of redo. It takes no + * database connection, so it can start now and serve crash recovery from + * the first record. + */ + XLogWarmWorkersRegister(); + /* * process any libraries that should be preloaded at postmaster start */ diff --git a/src/backend/storage/Makefile b/src/backend/storage/Makefile index eec03f6f2b4c5..46c960f2248a6 100644 --- a/src/backend/storage/Makefile +++ b/src/backend/storage/Makefile @@ -8,6 +8,6 @@ subdir = src/backend/storage top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = aio buffer file freespace ipc large_object lmgr page smgr sync +SUBDIRS = aio buffer dwb file freespace ipc large_object lmgr page smgr sync include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index ed1dc488a42b4..39b0f7f404ced 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -17,11 +17,13 @@ #include "storage/aio.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" +#include "storage/dwb.h" BufferDescPadded *BufferDescriptors; char *BufferBlocks; ConditionVariableMinimallyPadded *BufferIOCVArray; WritebackContext BackendWritebackContext; +WritebackContext DwbWritebackContext; CkptSortItem *CkptBufferIds; @@ -150,6 +152,13 @@ BufferManagerShmemInit(void) /* Initialize per-backend file flush context */ WritebackContextInit(&BackendWritebackContext, &backend_flush_after); + + /* + * The double write buffer paces the writeback of the pages it staged with + * a parameter of its own, so that the hint it wants started before a + * batch retires does not turn into one syscall per page in every backend. + */ + WritebackContextInit(&DwbWritebackContext, &dwb_writeback_after); } /* diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 27fd7e9720a48..d2bb85f6dafb5 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -54,6 +54,8 @@ #include "storage/aio.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" +#include "storage/checksum.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lmgr.h" @@ -61,7 +63,9 @@ #include "storage/read_stream.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "utils/injection_point.h" #include "utils/memdebug.h" +#include "utils/memutils.h" #include "utils/ps_status.h" #include "utils/rel.h" #include "utils/resowner.h" @@ -76,9 +80,11 @@ #define LocalBufHdrGetBlock(bufHdr) \ LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] -/* Bits in SyncOneBuffer's return value */ +/* Bits in SyncOneBuffer's (and BgSyncPeekBuffer's) return value */ #define BUF_WRITTEN 0x01 #define BUF_REUSABLE 0x02 +#define BUF_BINNABLE 0x04 /* would-write candidate for the + * vectored DWB flush bin */ #define RELS_BSEARCH_THRESHOLD 20 @@ -518,6 +524,7 @@ static void UnpinBuffer(BufferDesc *buf); static void UnpinBufferNoOwner(BufferDesc *buf); static void BufferSync(int flags); static uint32 WaitBufHdrUnlocked(BufferDesc *buf); +static int BgSyncPeekBuffer(int buf_id); static int SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context); static void WaitIO(BufferDesc *buf); @@ -533,8 +540,9 @@ static inline BufferDesc *BufferAlloc(SMgrRelation smgr, static bool AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress); static void CheckReadBuffersOperation(ReadBuffersOperation *operation, bool is_complete); static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); -static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, - IOObject io_object, IOContext io_context); +static bool FlushBuffer(BufferDesc *buf, SMgrRelation reln, + IOObject io_object, IOContext io_context, + WritebackContext *wb_context); static void FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, @@ -555,14 +563,22 @@ static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); /* - * Implementation of PrefetchBuffer() for shared buffers. + * Look up a shared buffer without touching storage. + * + * Returns the buffer the block was found in, or InvalidBuffer. As with + * PrefetchSharedBuffer(), the buffer is not pinned and the answer is only a + * hint: the caller must recheck, typically through ReadRecentBuffer(). + * + * Callers that want the residency answer alone use this: the recovery + * prefetcher, to decide whether a block is worth handing to the warm pool, + * and a warm worker, to tell a real read from a hit (ReadBufferWithoutRelcache + * does not report that). */ -PrefetchBufferResult -PrefetchSharedBuffer(SMgrRelation smgr_reln, - ForkNumber forkNum, - BlockNumber blockNum) +Buffer +LookupSharedBuffer(SMgrRelation smgr_reln, + ForkNumber forkNum, + BlockNumber blockNum) { - PrefetchBufferResult result = {InvalidBuffer, false}; BufferTag newTag; /* identity of requested block */ uint32 newHash; /* hash value for newTag */ LWLock *newPartitionLock; /* buffer partition lock for it */ @@ -583,8 +599,24 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln, buf_id = BufTableLookup(&newTag, newHash); LWLockRelease(newPartitionLock); + return buf_id < 0 ? InvalidBuffer : buf_id + 1; +} + +/* + * Implementation of PrefetchBuffer() for shared buffers. + */ +PrefetchBufferResult +PrefetchSharedBuffer(SMgrRelation smgr_reln, + ForkNumber forkNum, + BlockNumber blockNum) +{ + PrefetchBufferResult result = {InvalidBuffer, false}; + Buffer recent_buffer; + + recent_buffer = LookupSharedBuffer(smgr_reln, forkNum, blockNum); + /* If not in buffers, initiate prefetch */ - if (buf_id < 0) + if (!BufferIsValid(recent_buffer)) { #ifdef USE_PREFETCH /* @@ -605,7 +637,7 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln, * to avoid a buffer table lookup, but it's not pinned and it must be * rechecked! */ - result.recent_buffer = buf_id + 1; + result.recent_buffer = recent_buffer; } /* @@ -2357,6 +2389,7 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) Buffer buf; uint32 buf_state; bool from_ring; + bool staged; /* * Ensure, while the spinlock's not yet held, that there's a free refcount @@ -2450,11 +2483,17 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) } /* OK, do the I/O */ - FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context); + staged = FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context, + &DwbWritebackContext); LWLockRelease(content_lock); - ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, - &buf_hdr->tag); + /* + * A staged page has already been scheduled, paced by the double write + * buffer's own parameter; only the rest is ours to pace. + */ + if (!staged) + ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, + &buf_hdr->tag); } @@ -3364,6 +3403,9 @@ BufferSync(int flags) int i; int mask = BM_DIRTY; WritebackContext wb_context; + int *dwb_bin = NULL; + int dwb_bin_n = 0; + int dwb_bin_size = 0; /* * Unless this is a shutdown checkpoint or we have been explicitly told, @@ -3427,6 +3469,17 @@ BufferSync(int flags) WritebackContextInit(&wb_context, &checkpoint_flush_after); + /* + * With the double write buffer active, permanent buffers are flushed in + * bins of up to a batch: one batch write and one fdatasync cover the + * whole bin instead of one per page (see FlushBufferBin). + */ + if (DWBIsEnabled() && !IsBootstrapProcessingMode()) + { + dwb_bin_size = Min(dwb_batch_pages, DWB_FLUSH_BIN_MAX); + dwb_bin = palloc(dwb_bin_size * sizeof(int)); + } + TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan); /* @@ -3560,7 +3613,27 @@ BufferSync(int flags) */ if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + if (dwb_bin != NULL && + (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) + { + /* + * Vectored flush: collect permanent buffers into a bin and + * write them through the double write buffer with a single + * fdatasync. Nothing is locked while the bin fills; the + * members are re-checked when it flushes. + */ + dwb_bin[dwb_bin_n++] = buf_id; + if (dwb_bin_n == dwb_bin_size) + { + int nw = FlushBufferBin(dwb_bin, dwb_bin_n, + false, &wb_context); + + PendingCheckpointerStats.buffers_written += nw; + num_written += nw; + dwb_bin_n = 0; + } + } + else if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); PendingCheckpointerStats.buffers_written++; @@ -3595,6 +3668,20 @@ BufferSync(int flags) CheckpointWriteDelay(flags, (double) num_processed / num_to_scan); } + /* flush the residual bin of the vectored path */ + if (dwb_bin != NULL) + { + if (dwb_bin_n > 0) + { + int nw = FlushBufferBin(dwb_bin, dwb_bin_n, + false, &wb_context); + + PendingCheckpointerStats.buffers_written += nw; + num_written += nw; + } + pfree(dwb_bin); + } + /* * Issue all pending flushes. Only checkpointer calls BufferSync(), so * IOContext will always be IOCONTEXT_NORMAL. @@ -3663,8 +3750,21 @@ BgBufferSync(WritebackContext *wb_context) /* Variables for the scanning loop proper */ int num_to_scan; int num_written; + int num_issued; int reusable_buffers; + /* + * Vectored DWB flush bin (bin_size stays 0 without the DWB). The bin is + * static so that a bin refused by a full cleaner queue survives the round + * and is re-offered at the top of the next one. + */ + static int bin[DWB_FLUSH_BIN_MAX]; + static int bin_n = 0; + int bin_size = 0; + bool use_cleaners; + bool bin_deferred = false; + bool skip_scan = false; + /* Variables for final smoothed_density update */ long new_strategy_delta; uint32 new_recent_alloc; @@ -3678,6 +3778,16 @@ BgBufferSync(WritebackContext *wb_context) /* Report buffer alloc counts to pgstat */ PendingBgWriterStats.buf_alloc += recent_alloc; + /* + * Fold the cleaner pool's completed writes into buf_written_clean: + * pg_stat_bgwriter keeps counting pages written by LRU cleaning no matter + * which process executed the write. + */ + use_cleaners = DWBCleanersActive(); + if (use_cleaners) + PendingBgWriterStats.buf_written_clean += + DWBCleanerFetchPoolWritten(); + /* * If we're not running the LRU scan, just stop after doing the stats * stuff. We mark the saved state invalid so that we can recover sanely @@ -3686,6 +3796,13 @@ BgBufferSync(WritebackContext *wb_context) if (bgwriter_lru_maxpages <= 0) { saved_info_valid = false; + + /* + * A disabled scan feeds the pool nothing; a bin carried over from + * before the disable was only ever a hint, so drop it — the pages + * stay dirty for later scans, backends or the next checkpoint. + */ + bin_n = 0; return true; } @@ -3843,13 +3960,80 @@ BgBufferSync(WritebackContext *wb_context) num_to_scan = bufs_to_lap; num_written = 0; + num_issued = 0; reusable_buffers = reusable_buffers_est; + /* + * With the double write buffer active, would-write buffers are collected + * into bins and flushed as one batch each: one batch write and one + * fdatasync cover the whole bin instead of one per page (the LRU scan's + * scattered singleton writes otherwise degenerate to lone-writer batches; + * see FlushBufferBin). With a cleaner pool the bgwriter writes nothing + * itself: bins are handed to the pool's queue, and a refused bin (queue + * full: the pool is saturated) is carried over to the next round while + * the scan ends early — scanning further ahead would only produce bins + * nobody can drain, and flushing here would stall the scan behind serial + * batch fsyncs, starving the pool of fresh bins until the strategy clock + * hand catches the scan point and evictions land on the backends. The + * bgwriter_lru_maxpages budget caps the pages ISSUED per round — bins + * accepted by the queue, plus everything written in pool-less mode — + * while buf_written_clean counts actual writes only (the pool's + * completions are folded in at the top of the next round). + */ + if (DWBIsEnabled()) + bin_size = Min(dwb_batch_pages, DWB_FLUSH_BIN_MAX); + + /* + * Offer a bin carried over from a deferred round before scanning anew. + * Refused again: no scan this round, the queue is simply polled once per + * bgwriter_delay while the pool is saturated. Accepted: it spends this + * round's issue budget, and a budget shrunk below the bin size meanwhile + * (SIGHUP) ends the round before any scanning. + */ + if (use_cleaners && bin_n > 0) + { + if (DWBCleanerEnqueueBin(bin, bin_n)) + { + num_issued += bin_n; + bin_n = 0; + if (num_issued >= bgwriter_lru_maxpages) + { + PendingBgWriterStats.maxwritten_clean++; + skip_scan = true; + } + } + else + { + DWBCleanerCountDeferral(); + bin_deferred = true; + skip_scan = true; + } + } + /* Execute the LRU scan */ - while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) + while (!skip_scan && num_to_scan > 0 && + reusable_buffers < upcoming_alloc_est) { - int sync_state = SyncOneBuffer(next_to_clean, true, - wb_context); + int sync_state; + + if (bin_size > 0) + sync_state = BgSyncPeekBuffer(next_to_clean); + else + sync_state = SyncOneBuffer(next_to_clean, true, wb_context); + + if (sync_state & BUF_BINNABLE) + { + bin[bin_n++] = next_to_clean; + reusable_buffers++; + } + else if (sync_state & BUF_WRITTEN) + { + reusable_buffers++; + num_written++; + num_issued++; + } + else if (sync_state & BUF_REUSABLE) + reusable_buffers++; if (++next_to_clean >= NBuffers) { @@ -3858,17 +4042,67 @@ BgBufferSync(WritebackContext *wb_context) } num_to_scan--; - if (sync_state & BUF_WRITTEN) + /* + * Flush a full bin, and any partial one that already covers the + * remaining issue budget: the cap check below must see the true + * issued count, not a deferred bin. + */ + if (bin_n > 0 && + (bin_n == bin_size || + num_issued + bin_n >= bgwriter_lru_maxpages)) { - reusable_buffers++; - if (++num_written >= bgwriter_lru_maxpages) + if (use_cleaners) { - PendingBgWriterStats.maxwritten_clean++; - break; + if (DWBCleanerEnqueueBin(bin, bin_n)) + { + num_issued += bin_n; + bin_n = 0; + } + else + { + /* queue full: carry the bin over, end the round */ + DWBCleanerCountDeferral(); + bin_deferred = true; + break; + } + } + else + { + int nw = FlushBufferBin(bin, bin_n, false, wb_context); + + num_written += nw; + num_issued += nw; + bin_n = 0; } } - else if (sync_state & BUF_REUSABLE) - reusable_buffers++; + + if (num_issued >= bgwriter_lru_maxpages) + { + PendingBgWriterStats.maxwritten_clean++; + break; + } + } + + if (bin_n > 0 && !bin_deferred) + { + if (use_cleaners) + { + if (DWBCleanerEnqueueBin(bin, bin_n)) + { + num_issued += bin_n; + bin_n = 0; + } + else + DWBCleanerCountDeferral(); /* carry the bin over */ + } + else + { + int nw = FlushBufferBin(bin, bin_n, false, wb_context); + + num_written += nw; + num_issued += nw; + bin_n = 0; + } } PendingBgWriterStats.buf_written_clean += num_written; @@ -3905,8 +4139,49 @@ BgBufferSync(WritebackContext *wb_context) #endif } - /* Return true if OK to hibernate */ - return (bufs_to_lap == 0 && recent_alloc == 0); + /* Return true if OK to hibernate; a carried bin is pending work */ + return (bufs_to_lap == 0 && recent_alloc == 0 && bin_n == 0); +} + +/* + * BgSyncPeekBuffer -- the check half of SyncOneBuffer (with its + * skip_recently_used semantics) without the write. Classifies a buffer for + * the bgwriter's vectored flush: returns BUF_REUSABLE exactly as + * SyncOneBuffer would, plus BUF_BINNABLE when the buffer would have been + * written — the caller collects those into a bin and flushes them through + * the double write buffer as one batch (FlushBufferBin). The bin flush + * re-checks validity, dirtiness and permanence under the header lock, so a + * buffer that changes between the peek and the flush is handled there: + * clean again is skipped, recycled to unlogged goes to the per-page + * fallback. Pin and usage counts are not re-checked anywhere past this + * peek — a fresh pin or usage bump before the flush is the same benign + * race SyncOneBuffer itself has between its check and its write; + * skip_recently_used is an optimization, not a correctness contract. + */ +static int +BgSyncPeekBuffer(int buf_id) +{ + BufferDesc *bufHdr = GetBufferDescriptor(buf_id); + int result = 0; + uint32 buf_state; + + buf_state = LockBufHdr(bufHdr); + + if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && + BUF_STATE_GET_USAGECOUNT(buf_state) == 0) + result |= BUF_REUSABLE; + else + { + /* recently used: not a replacement candidate, nothing to write */ + UnlockBufHdr(bufHdr, buf_state); + return result; + } + + if ((buf_state & BM_VALID) && (buf_state & BM_DIRTY)) + result |= BUF_BINNABLE; + + UnlockBufHdr(bufHdr, buf_state); + return result; } /* @@ -3930,6 +4205,7 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) int result = 0; uint32 buf_state; BufferTag tag; + bool staged; /* Make sure we can handle the pin */ ReservePrivateRefCountEntry(); @@ -3972,7 +4248,8 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); - FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + staged = FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL, + wb_context); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); @@ -3982,13 +4259,266 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) /* * SyncOneBuffer() is only called by checkpointer and bgwriter, so - * IOContext will always be IOCONTEXT_NORMAL. + * IOContext will always be IOCONTEXT_NORMAL. A staged page went into the + * same context already, before its batch could retire. */ - ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag); + if (!staged) + ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag); return result | BUF_WRITTEN; } +/* + * FlushBufferBin -- flush a bin of buffers through the double write buffer + * as one batch. Serves both background flushers: the checkpointer's + * BufferSync and the bgwriter's LRU scan. + * + * The per-page write protocol cannot amortize the batch fdatasync for a + * sequential stream: each page waits for its own batch copy to become + * durable before its data-file write, and the lone-writer seal then closes + * the batch over that single page — a background flusher would pay one + * fdatasync per page. Here the whole bin is staged first, sealed and + * fdatasynced once, and only then written to the data files (the vectored + * background flush of the design, 3.4). + * + * All lock acquisitions in the gather phase are non-blocking: waiting for a + * content lock or for somebody's buffer I/O while already holding shared + * content locks of earlier bin members could deadlock against backends that + * take multiple buffer locks in their own order. Buffers that cannot be + * claimed without waiting fall back to the ordinary per-page SyncOneBuffer + * path after the bin is done, when nothing is held. + * + * A caller may pre-filter for BM_PERMANENT (BufferSync does, the bgwriter's + * peek deliberately does not), but that is only ever an optimization: the + * authoritative check is made here under the buffer header lock, because a + * captured buffer can be recycled for an unlogged page before the bin + * flushes (the same benign window BufferSync already tolerates for the + * checkpoint-needed bit). Non-permanent buffers go to the per-page + * fallback, whose FlushBuffer skips both the WAL flush and the DWB for + * them. Pin counts and usage counts are NOT re-checked here — writing a + * buffer that became recently-used after the caller picked it is the same + * benign race the per-page paths have between their check and their write. + * Returns the number of buffers written. + * + * All of the above describes the mandatory mode (opportunistic = false): + * the caller's bins are fresh and every member must be flushed or handed + * to the blocking per-page fallback — checkpointer semantics. The + * cleaner worker pool executes bins that sat in a queue for arbitrarily + * long, so it passes opportunistic = true: each member is reclassified + * under the buffer header lock with the LRU-candidate predicate of the + * scan that produced it (unpinned, unused, valid, dirty — see + * BgSyncPeekBuffer), and a member that fails the predicate, lost + * BM_PERMANENT, or cannot be claimed without waiting is dropped instead + * of written: the page stays dirty for the next scan pass or checkpoint, + * and the cleaner never blocks on somebody's content lock or I/O. In + * this mode every bin member ends up either written (counted in the + * return value) or dropped, so callers derive the skip count as + * nbuf minus the result. + */ +int +FlushBufferBin(const int *buf_ids, int nbuf, bool opportunistic, + WritebackContext *wb_context) +{ + static char *bin_buf = NULL; + + BufferDesc *bufs[DWB_FLUSH_BIN_MAX]; + XLogRecPtr lsns[DWB_FLUSH_BIN_MAX]; + DWBSlotRef refs[DWB_FLUSH_BIN_MAX]; + int fb_ids[DWB_FLUSH_BIN_MAX]; + int gathered = 0; + int nfallback = 0; + int written = 0; + XLogRecPtr max_lsn = InvalidXLogRecPtr; + ErrorContextCallback errcallback; + + Assert(nbuf > 0 && nbuf <= DWB_FLUSH_BIN_MAX); + + if (bin_buf == NULL) + bin_buf = MemoryContextAllocAligned(TopMemoryContext, + (Size) DWB_FLUSH_BIN_MAX * BLCKSZ, + PG_IO_ALIGN_SIZE, 0); + + /* Phase 1: claim and copy what can be claimed without waiting */ + for (int i = 0; i < nbuf; i++) + { + BufferDesc *bufHdr = GetBufferDescriptor(buf_ids[i]); + uint32 buf_state; + char *dst; + + /* Make sure we can handle the pin */ + ReservePrivateRefCountEntry(); + ResourceOwnerEnlarge(CurrentResourceOwner); + + buf_state = LockBufHdr(bufHdr); + if (opportunistic && + (BUF_STATE_GET_REFCOUNT(buf_state) != 0 || + BUF_STATE_GET_USAGECOUNT(buf_state) != 0)) + { + /* a stale claim: the buffer became hot since it was queued */ + UnlockBufHdr(bufHdr, buf_state); + continue; + } + if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY)) + { + /* clean already: nothing to do */ + UnlockBufHdr(bufHdr, buf_state); + continue; + } + if (!(buf_state & BM_PERMANENT)) + { + /* + * Recycled for an unlogged page after the bin captured it, or a + * shutdown checkpoint's unlogged buffer raced past the unlocked + * pre-check. Staging it would feed the DWB — and XLogFlush — + * a fake unlogged LSN, so route it to the per-page path instead. + */ + UnlockBufHdr(bufHdr, buf_state); + if (!opportunistic) + fb_ids[nfallback++] = buf_ids[i]; + continue; + } + PinBuffer_Locked(bufHdr); + + if (!LWLockConditionalAcquire(BufferDescriptorGetContentLock(bufHdr), + LW_SHARED)) + { + UnpinBuffer(bufHdr); + if (!opportunistic) + fb_ids[nfallback++] = buf_ids[i]; + continue; + } + if (!StartBufferIO(bufHdr, false, true)) + { + /* + * Either somebody else's I/O is in flight (fall back per-page: + * SyncOneBuffer may wait and rechecks dirtiness) or the buffer + * went clean; the fallback handles both. The opportunistic + * caller waits for neither and leaves the page to a later pass. + */ + LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); + UnpinBuffer(bufHdr); + if (!opportunistic) + fb_ids[nfallback++] = buf_ids[i]; + continue; + } + + /* as in FlushBuffer: read the LSN under the header lock */ + buf_state = LockBufHdr(bufHdr); + lsns[gathered] = BufferGetLSN(bufHdr); + buf_state &= ~BM_JUST_DIRTIED; + UnlockBufHdr(bufHdr, buf_state); + + TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&bufHdr->tag), + bufHdr->tag.blockNum, + BufTagGetRelFileLocator(&bufHdr->tag).spcOid, + BufTagGetRelFileLocator(&bufHdr->tag).dbOid, + BufTagGetRelFileLocator(&bufHdr->tag).relNumber); + + /* + * The private copy decouples the image from concurrent hint-bit + * updates, like PageSetChecksumCopy in the per-page path; an all-zero + * page must stay all-zero, so it gets no checksum. + */ + dst = bin_buf + (Size) gathered * BLCKSZ; + memcpy(dst, BufHdrGetBlock(bufHdr), BLCKSZ); + if (DataChecksumsEnabled() && !PageIsNew((Page) dst)) + ((PageHeader) dst)->pd_checksum = + pg_checksum_page(dst, bufHdr->tag.blockNum); + + if (lsns[gathered] > max_lsn) + max_lsn = lsns[gathered]; + bufs[gathered] = bufHdr; + gathered++; + } + + if (gathered > 0) + { + /* Phase 2: one WAL flush covers the whole bin (WAL before data) */ + if (!XLogRecPtrIsInvalid(max_lsn)) + XLogFlush(max_lsn); + + /* Setup error traceback support for ereport() */ + errcallback.callback = shared_buffer_write_error_callback; + errcallback.arg = NULL; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + /* Phase 3: stage everything, then one seal + one fdatasync */ + for (int i = 0; i < gathered; i++) + { + errcallback.arg = bufs[i]; + DWBStagePageWriteNoWait(&bufs[i]->tag, + bin_buf + (Size) i * BLCKSZ, + lsns[i], &refs[i]); + } + errcallback.arg = NULL; + DWBWaitStagedWrites(refs, gathered); + + /* Phase 4: the data-file writes */ + for (int i = 0; i < gathered; i++) + { + BufferDesc *bufHdr = bufs[i]; + SMgrRelation reln; + instr_time io_start; + BufferTag tag; + + errcallback.arg = bufHdr; + reln = smgropen(BufTagGetRelFileLocator(&bufHdr->tag), + INVALID_PROC_NUMBER); + + io_start = pgstat_prepare_io_time(track_io_timing); + smgrwrite(reln, + BufTagGetForkNum(&bufHdr->tag), + bufHdr->tag.blockNum, + bin_buf + (Size) i * BLCKSZ, + false); + pgstat_count_io_op_time(IOOBJECT_RELATION, IOCONTEXT_NORMAL, + IOOP_WRITE, io_start, 1, BLCKSZ); + + /* queue, and where the retirement is inline also issue */ + tag = bufHdr->tag; + ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag); + if (DWBRetiresInline()) + { + IssuePendingWritebacks(wb_context, IOCONTEXT_NORMAL); + INJECTION_POINT("dwb-inline-retire", wb_context); + } + DWBFinishPageWrite(&refs[i]); + + pgBufferUsage.shared_blks_written++; + + TerminateBufferIO(bufHdr, true, 0, true, false); + + TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&bufHdr->tag), + bufHdr->tag.blockNum, + BufTagGetRelFileLocator(&bufHdr->tag).spcOid, + BufTagGetRelFileLocator(&bufHdr->tag).dbOid, + BufTagGetRelFileLocator(&bufHdr->tag).relNumber); + + LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); + TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(bufHdr->buf_id); + UnpinBuffer(bufHdr); + + written++; + } + + error_context_stack = errcallback.previous; + } + + /* Phase 5: per-page fallback for the contended buffers, nothing held */ + Assert(!opportunistic || nfallback == 0); + for (int i = 0; i < nfallback; i++) + { + if (SyncOneBuffer(fb_ids[i], false, wb_context) & BUF_WRITTEN) + { + TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(fb_ids[i]); + written++; + } + } + + return written; +} + /* * AtEOXact_Buffers - clean up at end of transaction. * @@ -4288,10 +4818,17 @@ BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, * * If the caller has an smgr reference for the buffer's relation, pass it * as the second parameter. If not, pass NULL. + * + * A page written through the double write buffer wants a kernel writeback + * started before its batch retires, so that the sync retiring the batch is a + * cheap barrier rather than a full flush. Such a page is queued into + * wb_context, and true is returned so the caller knows not to queue it a + * second time; a page written without the double write buffer is left to the + * caller entirely and returns false. */ -static void +static bool FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, - IOContext io_context) + IOContext io_context, WritebackContext *wb_context) { XLogRecPtr recptr; ErrorContextCallback errcallback; @@ -4299,6 +4836,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, Block bufBlock; char *bufToWrite; uint32 buf_state; + DWBSlotRef dwbref; + bool staged; /* * Try to start an I/O operation. If StartBufferIO returns false, then @@ -4306,7 +4845,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * anything. */ if (!StartBufferIO(buf, false, false)) - return; + return false; /* Setup error traceback support for ereport() */ errcallback.callback = shared_buffer_write_error_callback; @@ -4371,6 +4910,23 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, */ bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum); + /* + * Double write buffer path: before the data-file write, make the copy + * durable in pg_dwb/ so that a torn smgrwrite can always be repaired from + * there (full_page_writes replacement, see storage/dwb.h). Only + * BM_PERMANENT buffers need this: unlogged relations are reset from their + * init fork after a crash, so their torn writes don't matter. Data + * checksums are required by the DWB, so for any page with content + * bufToWrite is a private copy, stable regardless of concurrent hint-bit + * updates; PageSetChecksumCopy returns the shared page only when it is + * all-zero new, where there are no tuples for hint bits to touch. + */ + if (DWBIsEnabled() && (buf_state & BM_PERMANENT) && + !IsBootstrapProcessingMode()) + DWBStagePageWrite(&buf->tag, bufToWrite, recptr, &dwbref); + else + dwbref.batch_idx = -1; + io_start = pgstat_prepare_io_time(track_io_timing); /* @@ -4403,6 +4959,31 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITE, io_start, 1, BLCKSZ); + staged = dwbref.batch_idx >= 0; + if (staged) + { + /* + * Step 6b: queue the page for kernel writeback, so that the sync + * retiring its batch becomes a cheap barrier instead of a full flush. + * Not durability — that comes from the sync. + * + * Queueing is not handing over: the context holds the tag until + * dwb_writeback_after of them have accumulated, so under a retire + * pool a batch may retire before the kernel has heard about its + * pages. That is the price of not making one syscall per page, and + * the sync is correct either way. When the retirement runs inline + * there is nothing to gamble on — the fsync is a few statements + * below — so the queue is emptied here instead. + */ + ScheduleBufferTagForWriteback(wb_context, io_context, &buf->tag); + if (DWBRetiresInline()) + { + IssuePendingWritebacks(wb_context, io_context); + INJECTION_POINT("dwb-inline-retire", wb_context); + } + DWBFinishPageWrite(&dwbref); + } + pgBufferUsage.shared_blks_written++; /* @@ -4419,6 +5000,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, /* Pop the error context stack */ error_context_stack = errcallback.previous; + + return staged; } /* @@ -4903,6 +5486,7 @@ DropDatabaseBuffers(Oid dbid) * database isn't our own. */ + for (i = 0; i < NBuffers; i++) { BufferDesc *bufHdr = GetBufferDescriptor(i); @@ -5013,7 +5597,8 @@ FlushRelationBuffers(Relation rel) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); - FlushBuffer(bufHdr, srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + FlushBuffer(bufHdr, srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL, + &DwbWritebackContext); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); } @@ -5110,7 +5695,8 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); - FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, + IOCONTEXT_NORMAL, &DwbWritebackContext); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); } @@ -5338,7 +5924,8 @@ FlushDatabaseBuffers(Oid dbid) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); - FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL, + &DwbWritebackContext); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); } @@ -5365,7 +5952,8 @@ FlushOneBuffer(Buffer buffer) Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr))); - FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL, + &DwbWritebackContext); } /* @@ -6615,7 +7203,8 @@ EvictUnpinnedBufferInternal(BufferDesc *desc, bool *buffer_flushed) if (buf_state & BM_DIRTY) { LWLockAcquire(BufferDescriptorGetContentLock(desc), LW_SHARED); - FlushBuffer(desc, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + FlushBuffer(desc, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL, + &DwbWritebackContext); *buffer_flushed = true; LWLockRelease(BufferDescriptorGetContentLock(desc)); } diff --git a/src/backend/storage/dwb/Makefile b/src/backend/storage/dwb/Makefile new file mode 100644 index 0000000000000..0b1cf707cd923 --- /dev/null +++ b/src/backend/storage/dwb/Makefile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for storage/dwb +# +# IDENTIFICATION +# src/backend/storage/dwb/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/storage/dwb +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = \ + dwb.o \ + dwb_cleaner.o \ + dwb_ctl.o \ + dwb_file.o \ + dwb_recovery.o \ + dwb_retire.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/storage/dwb/dwb.c b/src/backend/storage/dwb/dwb.c new file mode 100644 index 0000000000000..ff6ca409ff841 --- /dev/null +++ b/src/backend/storage/dwb/dwb.c @@ -0,0 +1,1416 @@ +/*------------------------------------------------------------------------- + * + * dwb.c + * Batch state machine of the short-lived double write buffer. + * + * Batch lifecycle: FREE -> ALLOCATED -> SEALED -> WRITTEN -> FSYNCED -> + * DATA_WRITTEN -> RETIRING -> FREE. Writers reserve slots with an atomic + * CAS on next_slot_idx (30-bit index + writer-class bit + SEAL_BIT + * sentinel; see dwb.h), publish their page image with a plain memcpy into + * the batch's staging buffer and set their bit in slots_written_bitmap. + * The SEAL initiator becomes the leader: it waits for bitmap coverage of + * capped_slots, then writes the whole batch — in this write order: the + * contiguous image stream, then the meta region, then fdatasync (the + * on-disk layout puts the meta region first; see dwb.h) — and broadcasts + * DWB_FSYNCED. + * + * FlushBuffer drives this through DWBStagePageWrite/DWBFinishPageWrite; + * retirement (segment fsyncs, the worker pool) lives in dwb_retire.c. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "common/relpath.h" +#include "miscadmin.h" +#include "port/pg_bitutils.h" +#include "storage/dwb.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/proc.h" +#include "utils/injection_point.h" +#include "utils/memutils.h" +#include "utils/resowner.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +/* + * Slot refs held by this backend. A ref lives from DWBAcquireSlot to + * DWBReleaseSlot; the write path also attaches it to the current + * ResourceOwner, so that a transaction abort (e.g. an ERROR out of + * smgrwrite) releases it long before process exit. Entries have stable + * addresses — the ResourceOwner remembers a pointer — so freeing is a flag, + * not compaction. Sized to two full batches because a backend can hold + * refs on a sealed batch and on its successor at the same time. + */ +typedef struct DWBPendingRef +{ + DWBSlotRef ref; + ResourceOwner owner; /* owner the ref is registered with, or NULL */ + bool in_use; +} DWBPendingRef; + +static DWBPendingRef pendingRefs[2 * DWB_BATCH_MAX_PAGES]; + +/* leader-side meta assembly area, allocated before the seal is attempted */ +static DWSlotMeta *leader_metas = NULL; + +/* Stage A pause of the bgwriter (see DWBWritesPaused) */ +static bool bgwriter_paused = false; +static uint64 bgwriter_pause_snap = 0; + +/* + * Escalation clock of one wait for ring space. The clock re-arms whenever + * freed_events moves: escalation fires only when retirement as a whole has + * made no progress for the full window, i.e. "broken", not "slow". + */ +typedef struct DWBStallState +{ + TimestampTz start; + uint64 freed_snap; + bool warned; +} DWBStallState; + +static void DWBProcExit(int code, Datum arg); +static void DWBLeaderWriteBatch(int batch_idx); +static bool DWBSealBatch(int batch_idx, DWBSealReason reason); +static void DWBFinishBatchData(DWBatchCtl *batch); +static void DWBAbandonRef(DWBPendingRef *pref); +static void ResOwnerReleaseDWBRef(Datum res); + +/* + * Released BEFORE the buffer-IO cleanup (smaller priority runs first): + * DWBAbandonRef may repair the data page from the batch file, which is only + * race-free while BM_IO_IN_PROGRESS of the aborted flush is still ours. + */ +static const ResourceOwnerDesc dwb_ref_resowner_desc = +{ + .name = "double write buffer slot ref", + .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .release_priority = RELEASE_PRIO_BUFFER_IOS - 10, + .ReleaseResource = ResOwnerReleaseDWBRef, + .DebugPrint = NULL, +}; + +static inline char * +DWBStagingSlotPtr(int staging_idx, int slot_idx) +{ + return DWBStagingBase + + (Size) staging_idx * dwb_batch_pages * BLCKSZ + + (Size) slot_idx * BLCKSZ; +} + +/* ---------------------------------------------------------------- + * backpressure (3.6) + * ---------------------------------------------------------------- + */ + +static void +DWBStallInit(DWBStallState *st) +{ + st->start = GetCurrentTimestamp(); + st->freed_snap = pg_atomic_read_u64(&DWBCtl->freed_events); + st->warned = false; +} + +/* + * Escalate one iteration of a ring-space wait. Stage A after + * dwb_slow_warn_ms: WARNING, and the bgwriter additionally pauses its own + * future flush rounds (DWBWritesPaused). Stage B after + * dwb_write_timeout_ms without a single retired batch: dwb_on_stall, except + * that the checkpointer and the startup process always PANIC — an ERROR + * there would fail the checkpoint or recovery anyway, without the fresh + * start that crash recovery gives (the explicit safety policy of 3.6, + * consistent with data_sync_elevel for a checkpoint-phase fsync failure). + * + * The dwb-force-stall injection point makes the current wait escalate to + * Stage B immediately: the role policy and dwb_on_stall handling stay + * exactly the production code paths, only the clock is bypassed. + */ +static void +DWBStallCheck(DWBStallState *st) +{ + uint64 freed = pg_atomic_read_u64(&DWBCtl->freed_events); + TimestampTz now = GetCurrentTimestamp(); + long waited; + bool forced; + + if (freed != st->freed_snap) + { + /* retirement made progress: re-arm */ + st->freed_snap = freed; + st->start = now; + st->warned = false; + return; + } + + waited = TimestampDifferenceMilliseconds(st->start, now); + forced = IS_INJECTION_POINT_ATTACHED("dwb-force-stall"); + + if (waited >= dwb_slow_warn_ms && !st->warned) + { + st->warned = true; + ereport(WARNING, + (errmsg("double write buffer has no free batch after %ld ms", + waited))); + if (MyBackendType == B_BG_WRITER) + { + bgwriter_paused = true; + bgwriter_pause_snap = freed; + } + } + + if (waited >= dwb_write_timeout_ms || forced) + { + if (AmCheckpointerProcess() || AmStartupProcess() || + CritSectionCount > 0 || dwb_on_stall == DWB_ON_STALL_PANIC) + ereport(PANIC, + (errmsg("double write buffer retirement made no progress within \"dwb_write_timeout_ms\""), + errdetail("No batch was retired while a %s process waited for ring space.", + GetBackendTypeDesc(MyBackendType)))); + if (dwb_on_stall == DWB_ON_STALL_ERROR) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("double write buffer retirement made no progress within \"dwb_write_timeout_ms\""))); + /* DWB_ON_STALL_WARN: complain and keep waiting */ + ereport(WARNING, + (errmsg("double write buffer retirement made no progress within \"dwb_write_timeout_ms\", still waiting"))); + st->start = now; + } +} + +/* + * Has a Stage A stall told the bgwriter to sit out its flush rounds? + * Clears itself as soon as any batch retires. + */ +bool +DWBWritesPaused(void) +{ + if (!bgwriter_paused) + return false; + if (pg_atomic_read_u64(&DWBCtl->freed_events) != bgwriter_pause_snap) + { + bgwriter_paused = false; + return false; + } + return true; +} + +/* ---------------------------------------------------------------- + * staging pool + * ---------------------------------------------------------------- + */ +/* + * Non-blocking staging reservation: returns a buffer index, or -1 when the + * pool is empty. Waiting for a buffer happens in DWBOpenNewBatch's outer + * loop, together with the wait for ring space: a would-be opener needs both + * resources and re-checks both conditions on every wake-up. + */ +static int +DWBStagingTryAlloc(void) +{ + int idx = -1; + + SpinLockAcquire(&DWBCtl->staging_lock); + if (DWBCtl->staging_free != 0) + { + idx = pg_rightmost_one_pos32(DWBCtl->staging_free); + DWBCtl->staging_free &= ~(1U << idx); + } + SpinLockRelease(&DWBCtl->staging_lock); + return idx; +} + +/* + * Return a staging buffer to the pool. Deliberately silent: whether the + * return is a wake-worthy event depends on the caller. A leader finishing + * its image pwrite adds capacity and wakes (DWBWakeRingWaiters); a would-be + * opener returning a probe-acquired buffer it could not use must NOT wake — + * under a full ring the wake token would then circulate forever through the + * waiters (each woken prober re-signals on its own release, and a + * timeout-woken process is even still queued on the condition variable, so + * it can pop itself), turning the paced 1s waits into a busy rotation of + * DWBRingOpenLock acquisitions. + */ +static void +DWBStagingRelease(int idx) +{ + SpinLockAcquire(&DWBCtl->staging_lock); + DWBCtl->staging_free |= 1U << idx; + SpinLockRelease(&DWBCtl->staging_lock); +} + +/* + * Wake one would-be batch opener of each writer class. Called on real + * capacity transitions only — a leader finished its image pwrite (the + * staging buffer serves the next batch) or a batch returned to FREE — never + * on a probe-acquired staging buffer bouncing back unused (see + * DWBStagingRelease). One targeted signal per class replaces a broadcast + * to every waiter, which collapses under thousands of ring-space waiters: + * each free event would wake them all just to re-queue on the condition + * variable's spinlock (3.6). Signalling per class rather than once overall + * is what makes a wake-up impossible to lose across the class boundary, + * where the sliced reserves (dwb.h) may forbid the woken class to open. A + * signal to an empty queue is a cheap no-op, every sleeper re-checks on a + * 1s timeout anyway, so over- and under-waking are both harmless. + * Allocation-free: legal inside critical sections. + */ +void +DWBWakeRingWaiters(void) +{ + for (int c = 0; c < DWB_NUM_WCLASSES; c++) + ConditionVariableSignal(&DWBCtl->cv_want_batch[c]); +} + +/* + * After opening a fresh batch, wake enough same-class waiters to fill it. + * The opener consumes one slot itself, so batch_pages - 1 joiners are the + * most that can make progress; the rest keep sleeping until the next open. + * The pipeline is self-clocking: the writer that overflows this batch seals + * it and opens the next one while already awake, waking the next portion. + */ +static void +DWBWakeJoiners(int wclass) +{ + for (int i = 0; i < dwb_batch_pages - 1; i++) + ConditionVariableSignal(&DWBCtl->cv_want_batch[wclass]); +} + +/* ---------------------------------------------------------------- + * batch opening + * ---------------------------------------------------------------- + */ + +/* + * Does open_batch_idx[wclass] still name the stale batch old_idx? Comparing + * the index alone is not enough: the ring reuses indexes, so by the time a + * slow opener asks, old_idx may name a NEW live incarnation of the same slot + * (sealed, retired, freed and reopened behind its back), and replacing it + * would orphan that live batch together with its staging buffer. SEAL_BIT + * plus the class bit disambiguate the incarnations: SEAL_BIT is set from + * SEAL through FREE and cleared only by the re-initialization in + * DWBOpenNewBatch (under DWBRingOpenLock), which also stamps the opening + * class — so the open batch needs replacing if and only if it is sealed or + * belongs to the other class (a reused index that the other class reopened + * while our pointer kept naming it). + * + * Callers outside DWBRingOpenLock use this as an opportunistic fast path; + * the opener re-checks under the lock before acting on the answer. + */ +static bool +DWBOpenBatchIsStale(int wclass, uint32 old_idx) +{ + uint32 cur = pg_atomic_read_u32(&DWBCtl->open_batch_idx[wclass]); + uint32 nsi; + + if (cur != old_idx) + return false; /* someone already replaced it */ + if (cur == DWB_INVALID_BATCH) + return true; /* nothing open yet */ + + nsi = pg_atomic_read_u32(&DWBCtl->batches[cur].next_slot_idx); + return (nsi & DWB_SEAL_BIT) || + (nsi & DWB_WCLASS_BIT) != DWBWClassBit(wclass); +} + +/* + * Make open_batch_idx[wclass] point at an ALLOCATED batch, if it currently + * points at old_idx (a sealed, foreign-class or invalid batch). Serialized + * by DWBRingOpenLock; sleeps on cv_want_batch[wclass] when the staging pool + * or the whole ring is busy. + * + * Ordering note for stale writers: a batch keeps SEAL_BIT in next_slot_idx + * from its SEAL until we finish re-initializing it here, and the + * re-initialization stamps the opening class into DWB_WCLASS_BIT, so a + * stale reservation attempt against a reused batch either sees SEAL_BIT or + * a foreign class bit (and retries) or lands on a valid slot of a new + * same-class incarnation — never on a slot that a concurrent reset can + * wipe, and never in a batch the other class is filling. + * + * Non-static only for test_dwb's stale-open regression test. + */ +void +DWBOpenNewBatch(int wclass, uint32 old_idx) +{ + DWBStallState stall; + + DWBStallInit(&stall); + for (;;) + { + int free_idx = -1; + int nfree = 0; + int staging_idx; + + /* + * Fast path: if another opener already replaced the open batch, there + * is nothing left to do here. Checking this before the staging + * reservation matters under pressure: every SEAL pushes all + * concurrent same-class writers into this function at once, and all + * but one of them only need to learn the new index — sending them + * through the DWB_STAGING_BUFFERS-deep staging pool first would + * serialize the whole herd on it. The racy read is fine: whoever + * proceeds re-checks under DWBRingOpenLock below. + */ + if (!DWBOpenBatchIsStale(wclass, old_idx)) + { + ConditionVariableCancelSleep(); + return; + } + + /* + * Try to become the opener. The staging buffer is reserved before + * taking the lock: no sleeping (or interruptible) point may exist + * below, where we hold DWBRingOpenLock with a batch already taken out + * of DWB_FREE. An empty pool is waited out in this outer loop: a + * leader wakes cv_want_batch when its image pwrite returns a buffer + * to the pool. + */ + staging_idx = DWBStagingTryAlloc(); + if (staging_idx < 0) + { + pg_atomic_fetch_add_u64(&DWBCtl->ring_wait_retries, 1); + (void) ConditionVariableTimedSleep(&DWBCtl->cv_want_batch[wclass], + 1000, + WAIT_EVENT_DWB_FREE_BATCH); + DWBStallCheck(&stall); + continue; + } + + LWLockAcquire(DWBRingOpenLock, LW_EXCLUSIVE); + + /* authoritative staleness re-check under the lock */ + if (!DWBOpenBatchIsStale(wclass, old_idx)) + { + LWLockRelease(DWBRingOpenLock); + DWBStagingRelease(staging_idx); + ConditionVariableCancelSleep(); + return; + } + + /* + * Count FREE batches first and apply the sliced reserves (see dwb.h): + * the eviction class may not consume the bottom DWB_BG_RESERVE + * batches, and the background class skips the middle + * DWB_EVICT_RESERVE slice — it opens either above both slices or + * inside its own bottom one. Leaving a slice to the other class + * alone would not be enough: under saturation the free count hovers + * at the throttle line of the greedier class, and a rule that only + * says "leave some behind" never lets the background stream reach a + * batch at all. FREE->ALLOCATED happens only under DWBRingOpenLock, + * and concurrent retirements only grow the count, so the check cannot + * overestimate. + */ + for (int i = 0; i < dwb_num_batches; i++) + if (pg_atomic_read_u32(&DWBCtl->batches[i].state) == DWB_FREE) + nfree++; + + if (wclass == DWB_WCLASS_BACKGROUND ? + (nfree >= 1 && (nfree > DWB_BG_RESERVE + DWB_EVICT_RESERVE || + nfree <= DWB_BG_RESERVE)) : + nfree > DWB_BG_RESERVE) + { + for (int i = 0; i < dwb_num_batches; i++) + { + uint32 expected = DWB_FREE; + + if (pg_atomic_compare_exchange_u32(&DWBCtl->batches[i].state, + &expected, DWB_ALLOCATED)) + { + free_idx = i; + break; + } + } + Assert(free_idx >= 0); + } + + if (free_idx >= 0) + { + DWBatchCtl *batch = &DWBCtl->batches[free_idx]; + + for (int w = 0; w < DWB_BITMAP_WORDS; w++) + pg_atomic_write_u64(&batch->slots_written_bitmap[w], 0); + pg_atomic_write_u32(&batch->capped_slots, 0); + pg_atomic_write_u32(&batch->ref_count, 0); + pg_atomic_write_u32(&batch->seg_pending_count, 0); + batch->n_segs = 0; + batch->max_page_lsn = InvalidXLogRecPtr; + batch->batch_id = pg_atomic_fetch_add_u64(&DWBCtl->next_batch_id, 1); + batch->open_time = GetCurrentTimestamp(); + batch->staging_idx = staging_idx; + + /* + * Open for reservations only after everything above is visible: + * clearing SEAL_BIT is the point where writers may enter. The + * write also stamps the opening class into DWB_WCLASS_BIT, which + * reservations validate atomically with their increment. + */ + pg_write_barrier(); + pg_atomic_write_u32(&batch->next_slot_idx, DWBWClassBit(wclass)); + + pg_atomic_write_u32(&DWBCtl->open_batch_idx[wclass], free_idx); + LWLockRelease(DWBRingOpenLock); + ConditionVariableCancelSleep(); + + /* let retire workers re-time the force-seal deadline */ + ConditionVariableBroadcast(&DWBCtl->cv_retire_wake); + + /* wake enough same-class waiters to fill the new batch */ + DWBWakeJoiners(wclass); + return; + } + + LWLockRelease(DWBRingOpenLock); + DWBStagingRelease(staging_idx); + + /* + * No usable FREE batch. Help ourselves before waiting: sweep the + * RETIRING batches synchronously. One sweeper at a time: with + * thousands of writers parked on a full ring, a sweep by every waiter + * is pure lock traffic — they hammer the per-batch publish locks + * and the segment hash while losing every fsync claim to whoever got + * there first (measured at ~half the CPU of a 104-core machine). A + * trylock loser skips straight to the sleep below and is woken + * through cv_want_batch by the winner's frees; the winner still + * shares the fsync work with the worker pool through the per-segment + * claims. The gate is an LWLock, not an atomic flag, so an ERROR + * inside the sweep releases it in the unwind. The self-help is also + * what keeps the ring alive with dwb_retire_workers = 0 and in + * single-user mode. + */ + if (LWLockConditionalAcquire(DWBSelfSweepLock, LW_EXCLUSIVE)) + { + int swept; + + /* test hook: proves at most one waiter ever gets here at a time */ + INJECTION_POINT("dwb-self-sweep", NULL); + + swept = DWBRetireAllSync(); + LWLockRelease(DWBSelfSweepLock); + if (swept > 0) + continue; + } + + pg_atomic_fetch_add_u64(&DWBCtl->ring_wait_retries, 1); + (void) ConditionVariableTimedSleep(&DWBCtl->cv_want_batch[wclass], + 1000, + WAIT_EVENT_DWB_FREE_BATCH); + DWBStallCheck(&stall); + } +} + +/* ---------------------------------------------------------------- + * sealing and the leader write + * ---------------------------------------------------------------- + */ + +/* + * Seal a batch. Returns true if we won the seal race and performed the + * leader duties (the batch is DWB_FSYNCED — or fully cascaded to FREE for + * the defensive capped_slots == 0 case — on return). + */ +static bool +DWBSealBatch(int batch_idx, DWBSealReason reason) +{ + DWBatchCtl *batch = &DWBCtl->batches[batch_idx]; + uint32 prev; + uint32 capped; + uint32 expected; + int wclass; + + /* + * Get everything the critical section below could fail at out of the way + * while failure is still harmless (the seal has not been attempted yet, + * so on ERROR the batch stays ALLOCATED and any other writer can seal it + * later): the one-time leader allocations, the batch file VFD, and this + * backend's condition-variable wait event set (the first + * ConditionVariablePrepareToSleep of a backend allocates it). + */ + if (leader_metas == NULL) + leader_metas = MemoryContextAllocZero(TopMemoryContext, + DWB_BATCH_MAX_PAGES * sizeof(DWSlotMeta)); + DWBPrepareBatchWrite(batch_idx); + ConditionVariablePrepareToSleep(&batch->cv_state); + ConditionVariableCancelSleep(); + + prev = pg_atomic_fetch_or_u32(&batch->next_slot_idx, DWB_SEAL_BIT); + if (prev & DWB_SEAL_BIT) + return false; /* somebody else is the leader */ + + /* + * We are the leader: nobody else can advance this batch anymore. A + * failure between here and DWB_FSYNCED would leave the batch wedged + * forever, its waiters stuck and its staging buffer lost, so run the + * whole span as a critical section: any error escalates to PANIC and + * crash recovery resets the ring. (This also suspends interrupt + * processing, making the coverage wait in DWBLeaderWriteBatch + * non-interruptible; its dwb_slot_stuck_timeout_ms PANIC is the + * backstop.) + */ + START_CRIT_SECTION(); + + capped = Min(prev & DWB_IDX_MASK, (uint32) dwb_batch_pages); + pg_atomic_write_u32(&batch->capped_slots, capped); + pg_write_barrier(); + + /* diagnostic accounting: who seals, and how full the batches are */ + wclass = (prev & DWB_WCLASS_BIT) ? DWB_WCLASS_BACKGROUND : DWB_WCLASS_EVICTION; + pg_atomic_fetch_add_u64(&DWBCtl->seal_count[wclass][reason], 1); + pg_atomic_fetch_add_u64(&DWBCtl->seal_pages[wclass][reason], capped); + + /* + * An overflow seal marks the class as HOT: writers are streaming in + * faster than a batch fills, so a lone writer in a fresh batch must not + * fast-seal it (see DWBWaitBatchFsynced). Stamped again after the leader + * write below: the hot window has to survive a slow batch fdatasync, or + * the first writer after it would find a stale stamp. + */ + if (reason == DWB_SEAL_OVERFLOW) + pg_atomic_write_u64(&DWBCtl->last_overflow_seal[wclass], + (uint64) GetCurrentTimestamp()); + + expected = DWB_ALLOCATED; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, DWB_SEALED)) + elog(PANIC, "DWB batch %d sealed in unexpected state %u", + batch_idx, expected); + ConditionVariableBroadcast(&batch->cv_state); + + if (capped == 0) + { + /* + * Defensive: a seal without a single reservation. No leader write; + * cascade to FREE without I/O or publication. + */ + DWBStagingRelease(batch->staging_idx); + batch->staging_idx = -1; + pg_atomic_write_u32(&batch->state, DWB_FREE); + pg_atomic_fetch_add_u64(&DWBCtl->freed_events, 1); + DWBWakeRingWaiters(); + END_CRIT_SECTION(); + return true; + } + + /* + * Pin the batch with a leader ref for the duration of the write. All + * writers may exit while we write (dropping their refs), and the FSYNCED + * -> RETIRING hand-off runs when the last ref drops: the pin guarantees + * ref_count stays above zero until DWB_FSYNCED is reached, so the + * hand-off always has exactly one well-defined owner. + */ + pg_atomic_fetch_add_u32(&batch->ref_count, 1); + + DWBLeaderWriteBatch(batch_idx); + + END_CRIT_SECTION(); + + /* the hot window starts over once the overflow's fdatasync is done */ + if (reason == DWB_SEAL_OVERFLOW) + pg_atomic_write_u64(&DWBCtl->last_overflow_seal[wclass], + (uint64) GetCurrentTimestamp()); + + if (pg_atomic_fetch_sub_u32(&batch->ref_count, 1) == 1) + DWBFinishBatchData(batch); + + return true; +} + +/* + * Leader: wait for bitmap coverage of capped_slots, write the batch, + * fdatasync, broadcast DWB_FSYNCED. + * + * Runs inside the leader's critical section (see DWBSealBatch); everything + * it needs was allocated and opened before the seal, so no palloc happens + * here. + */ +static void +DWBLeaderWriteBatch(int batch_idx) +{ + DWBatchCtl *batch = &DWBCtl->batches[batch_idx]; + uint32 capped = pg_atomic_read_u32(&batch->capped_slots); + DWBBatchHeader hdr; + TimestampTz wait_start = GetCurrentTimestamp(); + uint32 expected; + + Assert(CritSectionCount > 0); + Assert(leader_metas != NULL); + + /* + * Coverage wait is memcpy-bound: writers do no I/O between reserving a + * slot and setting their bit. The timeout is a defensive backstop (e.g. + * a writer stopped in a debugger); dead writers are covered by ref + * cleanup marking their slots DWB_SLOT_ABORTED. + */ + ConditionVariablePrepareToSleep(&batch->cv_state); + for (;;) + { + bool covered = true; + uint32 full_words = capped / 64; + uint32 tail_bits = capped % 64; + + for (uint32 i = 0; covered && i < full_words; i++) + if (pg_atomic_read_u64(&batch->slots_written_bitmap[i]) != + PG_UINT64_MAX) + covered = false; + if (covered && tail_bits > 0) + { + uint64 mask = (UINT64CONST(1) << tail_bits) - 1; + + if ((pg_atomic_read_u64(&batch->slots_written_bitmap[full_words]) & + mask) != mask) + covered = false; + } + if (covered) + break; + + if (ConditionVariableTimedSleep(&batch->cv_state, + dwb_slot_stuck_timeout_ms, + WAIT_EVENT_DWB_BATCH_COVERAGE) && + TimestampDifferenceExceeds(wait_start, GetCurrentTimestamp(), + dwb_slot_stuck_timeout_ms)) + elog(PANIC, "DWB batch %d coverage wait exceeded %d ms", + batch_idx, dwb_slot_stuck_timeout_ms); + } + ConditionVariableCancelSleep(); + + expected = DWB_SEALED; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, DWB_WRITTEN)) + elog(PANIC, "DWB batch %d written in unexpected state %u", + batch_idx, expected); + + /* assemble slot metas entirely from shmem arrays */ + memset(leader_metas, 0, capped * sizeof(DWSlotMeta)); + batch->max_page_lsn = InvalidXLogRecPtr; + for (uint32 i = 0; i < capped; i++) + { + DWSlotMeta *meta = &leader_metas[i]; + + meta->tag = batch->pages[i]; + meta->page_lsn = batch->page_lsns[i]; + meta->generation = DWBCtl->ring_generation; + meta->flags = batch->slot_flags[i]; + meta->image_crc = batch->image_crcs[i]; + meta->meta_crc = DWBSlotMetaCrc(meta); + if (batch->page_lsns[i] > batch->max_page_lsn) + batch->max_page_lsn = batch->page_lsns[i]; + } + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = DWB_BATCH_MAGIC; + hdr.version = DWB_VERSION; + hdr.batch_id = batch->batch_id; + hdr.n_slots = capped; + hdr.crc = DWBBatchHeaderCrc(&hdr); + + DWBWriteBatch(batch_idx, &hdr, leader_metas, + DWBStagingSlotPtr(batch->staging_idx, 0)); + + /* image pwrite done — staging can serve the next batch */ + DWBStagingRelease(batch->staging_idx); + batch->staging_idx = -1; + DWBWakeRingWaiters(); + + expected = DWB_WRITTEN; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, DWB_FSYNCED)) + elog(PANIC, "DWB batch %d fsynced in unexpected state %u", + batch_idx, expected); + ConditionVariableBroadcast(&batch->cv_state); +} + +/* ---------------------------------------------------------------- + * writer API + * ---------------------------------------------------------------- + */ + +/* + * Try to seal a batch if it is still an open, non-empty ALLOCATED one; the + * guards make it safe to call speculatively against any state. Entry point + * of the decentralized seal triggers: the lone-writer fast seal and the + * timeout seal in DWBWaitBatchFsynced, the no-worker-pool seal in + * DWBStagePageWrite, DWBForceSealOpenBatch, and the retire workers' + * force-seal on dwb_batch_timeout_ms. The overflow writer in + * DWBAcquireSlot calls DWBSealBatch directly instead: it has just consumed + * the first slot index past the cap, so it already knows the batch is full + * and non-empty. + */ +bool +DWBTrySealBatch(int batch_idx, DWBSealReason reason) +{ + DWBatchCtl *batch = &DWBCtl->batches[batch_idx]; + uint32 nsi = pg_atomic_read_u32(&batch->next_slot_idx); + + if (nsi & DWB_SEAL_BIT) + return false; /* sealed already (or FREE: the bit is held + * through FREE until reopen) */ + if ((nsi & DWB_IDX_MASK) == 0) + return false; /* empty: sealing buys nothing */ + if (pg_atomic_read_u32(&batch->state) != DWB_ALLOCATED) + return false; + return DWBSealBatch(batch_idx, reason); +} + +/* + * Reserve a slot in the open batch of the given writer class, record the + * page tag and the segment ref, and take a batch ref. With use_resowner + * the ref is also attached to CurrentResourceOwner, so a transaction abort + * releases it (the write path always does this; tests exercising proc-exit + * cleanup do not). + */ +void +DWBAcquireSlot(const BufferTag *tag, int wclass, bool use_resowner, + DWBSlotRef *ref) +{ + DWBPendingRef *pref = NULL; + + Assert(DWBIsEnabled()); + Assert(wclass >= 0 && wclass < DWB_NUM_WCLASSES); + + for (int i = 0; i < (int) lengthof(pendingRefs); i++) + { + if (!pendingRefs[i].in_use) + { + pref = &pendingRefs[i]; + break; + } + } + /* hard bound: overflowing the static array would corrupt memory */ + if (pref == NULL) + elog(ERROR, "too many pending double write buffer slot refs held by one backend"); + + /* no failure window between the reservation below and remembering it */ + if (use_resowner) + ResourceOwnerEnlarge(CurrentResourceOwner); + + for (;;) + { + uint32 idx = pg_atomic_read_u32(&DWBCtl->open_batch_idx[wclass]); + DWBatchCtl *batch; + uint32 prev; + uint32 slot; + + if (idx == DWB_INVALID_BATCH) + { + DWBOpenNewBatch(wclass, idx); + continue; + } + + batch = &DWBCtl->batches[idx]; + + /* + * Reserve with a CAS rather than a plain fetch_add: the seal and + * class bits are validated atomically with the increment. The class + * check is what makes a stale open pointer safe: the ring reuses + * indexes, so idx may name a batch that was freed and reopened under + * the OTHER class while our per-class pointer kept naming it, and a + * blind increment there would consume a slot nobody ever publishes — + * the leader would wait for its coverage forever. + */ + prev = pg_atomic_read_u32(&batch->next_slot_idx); + for (;;) + { + if (prev & DWB_SEAL_BIT) + break; /* sealed: reopen and retry */ + if ((prev & DWB_WCLASS_BIT) != DWBWClassBit(wclass)) + break; /* foreign incarnation: our pointer is stale */ + if (pg_atomic_compare_exchange_u32(&batch->next_slot_idx, + &prev, prev + 1)) + break; /* reserved */ + } + + if ((prev & DWB_SEAL_BIT) || + (prev & DWB_WCLASS_BIT) != DWBWClassBit(wclass)) + { + DWBOpenNewBatch(wclass, idx); + continue; + } + + slot = prev & DWB_IDX_MASK; + if (slot >= (uint32) dwb_batch_pages) + { + /* overflow: this writer seals and (if it wins) leads */ + DWBSealBatch(idx, DWB_SEAL_OVERFLOW); + DWBOpenNewBatch(wclass, idx); + continue; + } + + /* valid reservation */ + batch->pages[slot] = *tag; + batch->slot_flags[slot] = 0; + + { + DWSegRef seg; + bool found = false; + + /* becomes a HASH_BLOBS key at publication: no padding garbage */ + memset(&seg, 0, sizeof(seg)); + seg.rlocator = BufTagGetRelFileLocator(tag); + seg.forknum = BufTagGetForkNum(tag); + seg.segno = tag->blockNum / RELSEG_SIZE; + + /* + * The dedup scan is O(n_segs), far too long for a spinlock; + * publish_lock is this batch's LWLock over n_segs/seg_set. + */ + LWLockAcquire(&batch->publish_lock, LW_EXCLUSIVE); + for (uint32 i = 0; i < batch->n_segs; i++) + { + if (RelFileLocatorEquals(batch->seg_set[i].rlocator, seg.rlocator) && + batch->seg_set[i].forknum == seg.forknum && + batch->seg_set[i].segno == seg.segno) + { + found = true; + break; + } + } + if (!found) + batch->seg_set[batch->n_segs++] = seg; + LWLockRelease(&batch->publish_lock); + } + + pg_atomic_fetch_add_u32(&batch->ref_count, 1); + + ref->batch_idx = (int) idx; + ref->slot_idx = (int) slot; + ref->batch_id = batch->batch_id; + + pref->ref = *ref; + pref->owner = use_resowner ? CurrentResourceOwner : NULL; + pref->in_use = true; + if (pref->owner) + ResourceOwnerRemember(pref->owner, PointerGetDatum(pref), + &dwb_ref_resowner_desc); + return; + } +} + +/* + * Publish the page image: memcpy into the batch's staging slot, record + * LSN and image CRC in shmem, set our bitmap bit. + */ +void +DWBPublishImage(const DWBSlotRef *ref, const char *image, XLogRecPtr page_lsn) +{ + DWBatchCtl *batch = &DWBCtl->batches[ref->batch_idx]; + + /* a held ref pins the batch, so its incarnation cannot have changed */ + Assert(ref->batch_id == batch->batch_id); + + memcpy(DWBStagingSlotPtr(batch->staging_idx, ref->slot_idx), + image, BLCKSZ); + batch->page_lsns[ref->slot_idx] = page_lsn; + batch->image_crcs[ref->slot_idx] = DWBImageCrc(image); + + pg_write_barrier(); + pg_atomic_fetch_or_u64(&batch->slots_written_bitmap[ref->slot_idx / 64], + UINT64CONST(1) << (ref->slot_idx % 64)); + ConditionVariableBroadcast(&batch->cv_state); +} + +/* + * Is the writer class's demand hot — was its last overflow seal younger + * than the rendezvous window? Hot only when the clock reads at or past + * the stamp: a stamp from the future (a backward system-clock step) must + * read as QUIET, or the immediate lone seal would stay disabled until the + * clock catches up, taxing every write of a sequential stream with the + * timeout. Exported for the test module, which plants a future stamp to + * pin exactly that branch. + */ +bool +DWBClassIsHot(int wclass) +{ + TimestampTz stamp; + TimestampTz now; + + stamp = (TimestampTz) pg_atomic_read_u64(&DWBCtl->last_overflow_seal[wclass]); + now = GetCurrentTimestamp(); + return now >= stamp && + !TimestampDifferenceExceeds(stamp, now, dwb_batch_timeout_ms); +} + +/* + * Wait until the batch's DWB copy is durable. The caller holds a batch + * ref, so the batch cannot be retired or reused under us. + * + * A batch that nobody seals would leave its writers waiting forever, so + * after dwb_batch_timeout_ms of waiting on a still-open batch the waiter + * seals it itself. The retire workers force-seal on the same timeout; + * this decentralized backstop keeps the write path independent of the + * worker pool (dwb_retire_workers = 0, single-user mode, a stuck worker). + */ +void +DWBWaitBatchFsynced(const DWBSlotRef *ref) +{ + DWBatchCtl *batch = &DWBCtl->batches[ref->batch_idx]; + + Assert(ref->batch_id == batch->batch_id); + + /* + * A lone writer has nobody to batch with: sequential flush streams + * (recovery, a backend evicting page after page, BufferSync) reach this + * wait one page at a time, and paying dwb_batch_timeout_ms per page would + * dominate the stream. If our ref is the only one on a still-open batch, + * seal right away; under concurrency ref_count > 1 keeps the rendezvous + * window open for the timeout. A racing second writer merely bounces to + * the next batch — sealing is valid at any moment. + * + * The fast seal only applies while the class is QUIET. Under a dense + * concurrent stream the FIRST writer of every freshly opened batch also + * finds ref_count == 1 — it published within microseconds and nobody + * joined yet — and fast-sealing there halves the ring into one-page + * batches (54% of eviction batches at 1.33 slots, measured). A class + * whose last overflow seal is younger than the rendezvous window is + * clearly hot: skip the fast seal and let the batch fill. If the stream + * dies right here, the timeout seal below and the retire workers' + * force-seal still fire after dwb_batch_timeout_ms. The class bit of a + * held-ref batch is stable (reopen is fenced by the ref); the stamp is + * advisory, so a stale read just mis-decides one seal. + */ + if (pg_atomic_read_u32(&batch->ref_count) == 1) + { + int wclass; + + wclass = (pg_atomic_read_u32(&batch->next_slot_idx) & DWB_WCLASS_BIT) ? + DWB_WCLASS_BACKGROUND : DWB_WCLASS_EVICTION; + if (!DWBClassIsHot(wclass)) + (void) DWBTrySealBatch(ref->batch_idx, DWB_SEAL_LONE); + } + + ConditionVariablePrepareToSleep(&batch->cv_state); + while (pg_atomic_read_u32(&batch->state) < DWB_FSYNCED) + { + if (ConditionVariableTimedSleep(&batch->cv_state, + dwb_batch_timeout_ms, + WAIT_EVENT_DWB_BATCH_FSYNC)) + (void) DWBTrySealBatch(ref->batch_idx, DWB_SEAL_WAIT_TIMEOUT); + } + ConditionVariableCancelSleep(); +} + +/* + * Step 7 of the write path: the last ref hands the batch over to + * retirement — FSYNCED -> DATA_WRITTEN, then the seg_set publication into + * DWSegmentHash and the RETIRING transition (dwb_retire.c). + */ +static void +DWBFinishBatchData(DWBatchCtl *batch) +{ + uint32 expected = DWB_FSYNCED; + + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, + DWB_DATA_WRITTEN)) + elog(PANIC, "DWB batch data-written in unexpected state %u", expected); + + DWBPublishBatchSegSet((int) (batch - DWBCtl->batches)); +} + +/* + * Drop our batch ref after the data-file write. The last ref finishes the + * batch (see DWBFinishBatchData). + */ +void +DWBReleaseSlot(const DWBSlotRef *ref) +{ + DWBatchCtl *batch = &DWBCtl->batches[ref->batch_idx]; + + Assert(ref->batch_id == batch->batch_id); + + for (int i = 0; i < (int) lengthof(pendingRefs); i++) + { + DWBPendingRef *pref = &pendingRefs[i]; + + if (pref->in_use && + pref->ref.batch_idx == ref->batch_idx && + pref->ref.slot_idx == ref->slot_idx) + { + if (pref->owner != NULL) + ResourceOwnerForget(pref->owner, PointerGetDatum(pref), + &dwb_ref_resowner_desc); + pref->owner = NULL; + pref->in_use = false; + break; + } + } + + if (pg_atomic_fetch_sub_u32(&batch->ref_count, 1) == 1) + DWBFinishBatchData(batch); +} + +/* + * Force-seal the currently open batch of a writer class. Returns true if + * a batch was sealed by us. + */ +bool +DWBForceSealOpenBatch(int wclass) +{ + uint32 idx = pg_atomic_read_u32(&DWBCtl->open_batch_idx[wclass]); + + if (idx == DWB_INVALID_BATCH) + return false; + return DWBTrySealBatch((int) idx, DWB_SEAL_FORCED); +} + +DWBatchState +DWBGetBatchState(int batch_idx) +{ + return (DWBatchState) pg_atomic_read_u32(&DWBCtl->batches[batch_idx].state); +} + +/* ---------------------------------------------------------------- + * FlushBuffer entry points + * ---------------------------------------------------------------- + */ + +/* + * The checkpointer's BufferSync, the bgwriter's flush rounds, the + * cleaner worker pool executing the bgwriter's bins and autovacuum + * workers flushing their private ring strategy form the background + * stream; everything else — ordinary backend evictions above all — is + * the latency-critical class with first claim on FREE batches. + * Autovacuum belongs there because it is a scheduled sequential writer: + * in a busy class its pages ride the pool's bin batches instead of + * sealing one-page batches of their own, and in a cold class the + * lone-writer fast seal keeps its per-page latency unchanged. Manual + * VACUUM stays with its client backend's class. Cleaners are ordinary + * background workers, invisible to MyBackendType, hence the + * process-local flag. + */ +static int +DWBWriterClass(void) +{ + if (MyBackendType == B_CHECKPOINTER || MyBackendType == B_BG_WRITER || + MyBackendType == B_AUTOVAC_WORKER || DWBAmCleanerWorker) + return DWB_WCLASS_BACKGROUND; + return DWB_WCLASS_EVICTION; +} + +/* + * Steps 3-5 of the write path (3.4): reserve a slot in this writer class's + * open batch, publish the private page copy, and wait until the batch copy + * is durable in pg_dwb/. On return the caller may write the same copy to + * the data file. The caller must already have flushed WAL up to page_lsn. + */ +void +DWBStagePageWrite(const BufferTag *tag, const char *image, + XLogRecPtr page_lsn, DWBSlotRef *ref) +{ + Assert(DWBCtl->ring_generation > 0); + + DWBAcquireSlot(tag, DWBWriterClass(), true, ref); + DWBPublishImage(ref, image, page_lsn); + + /* + * With no worker pool (dwb_retire_workers = 0, single-user mode) a lonely + * batch would only seal via the wait timeout below; seal it right away + * instead of paying dwb_batch_timeout_ms per page. + */ + if (dwb_retire_workers == 0 || !IsUnderPostmaster) + (void) DWBTrySealBatch(ref->batch_idx, DWB_SEAL_LONE); + + DWBWaitBatchFsynced(ref); + + INJECTION_POINT("dwb-after-batch-fsynced", NULL); +} + +/* + * Steps 3-4 for a vectored caller: reserve and publish without waiting for + * durability. A sequential stream gets no rendezvous from the per-page + * protocol — each page would seal and fdatasync a batch of its own — so the + * background flushers stage a whole bin of pages first and then make them + * durable in one place with DWBWaitStagedWrites (one batch write and one + * fdatasync per bin; see "vectored background flush" in 3.4 of the design). + */ +void +DWBStagePageWriteNoWait(const BufferTag *tag, const char *image, + XLogRecPtr page_lsn, DWBSlotRef *ref) +{ + Assert(DWBCtl->ring_generation > 0); + + DWBAcquireSlot(tag, DWBWriterClass(), true, ref); + DWBPublishImage(ref, image, page_lsn); +} + +/* + * Step 5 for a vectored caller: seal every batch the bin's slots landed in + * and wait until they are all durable. On return the caller may write the + * staged copies to the data files. + * + * Slots were acquired in order and a batch held by our refs cannot recycle, + * so slots of the same batch are consecutive and comparing with the previous + * ref finds every batch boundary (normally none: one bin, one batch). + */ +void +DWBWaitStagedWrites(const DWBSlotRef *refs, int nrefs) +{ + for (int i = 0; i < nrefs; i++) + if (i == 0 || refs[i].batch_idx != refs[i - 1].batch_idx) + (void) DWBTrySealBatch(refs[i].batch_idx, DWB_SEAL_BIN); + + for (int i = 0; i < nrefs; i++) + if (i == 0 || refs[i].batch_idx != refs[i - 1].batch_idx) + DWBWaitBatchFsynced(&refs[i]); + + if (nrefs > 0) + INJECTION_POINT("dwb-after-batch-fsynced", NULL); +} + +/* + * Step 7: release the ref after smgrwrite returned. Without a worker + * pool, also retire synchronously so the ring keeps circulating (and, in + * the TAP tests, returns to all-FREE after every flush). + */ +void +DWBFinishPageWrite(const DWBSlotRef *ref) +{ + DWBReleaseSlot(ref); + + if (DWBRetiresInline()) + (void) DWBRetireAllSync(); +} + +/* ---------------------------------------------------------------- + * abort / process exit cleanup + * ---------------------------------------------------------------- + */ + +/* + * Repair the data page of an abandoned ref from the batch file. + * + * If the writer died out of a failed smgrwrite (step 6), the data page may + * be torn on disk while the batch — and with it the only whole copy — is + * about to retire and recycle. Overwriting the page with the durable batch + * copy makes the disk page whole again; the shared buffer is still dirty + * (the abort path never clears BM_DIRTY), so newer content still reaches + * the disk through a later flush. + * + * This runs for refs that were attached to a ResourceOwner, either from the + * owner's release (BEFORE the buffer-IO cleanup) or from the proc-exit + * backstop when abort cleanup was cut short (see DWBProcExit): in both + * cases BM_IO_IN_PROGRESS of the failed flush is still ours, so no + * concurrent flush of the same page can be in flight and writing the + * (possibly stale) batch copy cannot overwrite a newer image. For the same + * reason the relation cannot be dropped or truncated under us — both + * invalidate the buffer first and that waits for our IO flag — so the + * ENOENT/short-file exits are pure defense (and serve test refs pointing at + * fake relations). + * + * Durability: our segment is in the batch's seg_set, and the seg_set is + * published only after every ref (ours included) is gone, so retirement + * fsyncs this segment strictly after this write. + * + * Runs from release callbacks that must not fail, so the whole path is + * allocation-free: BasicOpenFile + raw pg_pwrite here (and the same inside + * DWBReadSlotImage) instead of OpenTransientFile/VFD, whose descriptor + * reservation and name bookkeeping can throw ERROR. Every failure other + * than the dropped/truncated-relation exits is PANIC. + */ +static void +DWBRewriteAbandonedSlot(const DWBSlotRef *ref) +{ + DWBatchCtl *batch = &DWBCtl->batches[ref->batch_idx]; + BufferTag tag = batch->pages[ref->slot_idx]; + uint32 segno = tag.blockNum / ((BlockNumber) RELSEG_SIZE); + PGAlignedBlock image; + RelPathStr relpath; + char path[MAXPGPATH]; + int fd; + struct stat st; + off_t off; + ssize_t written; + + DWBReadSlotImage(ref->batch_idx, ref->slot_idx, image.data); + + relpath = relpathperm(BufTagGetRelFileLocator(&tag), + BufTagGetForkNum(&tag)); + if (segno == 0) + snprintf(path, MAXPGPATH, "%s", relpath.str); + else + snprintf(path, MAXPGPATH, "%s.%u", relpath.str, segno); + + fd = BasicOpenFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + { + if (errno == ENOENT) + return; /* relation dropped: the write is moot */ + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not open file \"%s\" to repair an abandoned double write buffer slot: %m", + path))); + } + + off = (off_t) (tag.blockNum % ((BlockNumber) RELSEG_SIZE)) * BLCKSZ; + if (fstat(fd, &st) < 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", path))); + if (off + BLCKSZ > st.st_size) + { + /* segment truncated: the write is moot */ + close(fd); + return; + } + + errno = 0; + written = pg_pwrite(fd, image.data, BLCKSZ, off); + if (written != BLCKSZ) + { + if (errno == 0) + errno = ENOSPC; + /* the page may now be torn with its DWB copy about to recycle */ + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not repair block %u of file \"%s\" from the double write buffer: %m", + tag.blockNum, path))); + } + + if (close(fd) != 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); +} + +/* + * Release a ref whose owner is going away without the normal + * DWBReleaseSlot: transaction abort (ResourceOwner) or process exit. No + * ResourceOwnerForget here — the ResourceOwner path removes the resource + * implicitly, and at process exit the owner dies with the process. Works + * strictly on shmem DWB state plus the batch file — the private page copy + * of the writer is gone. + */ +static void +DWBAbandonRef(DWBPendingRef *pref) +{ + DWBSlotRef ref = pref->ref; + DWBatchCtl *batch = &DWBCtl->batches[ref.batch_idx]; + uint64 bit = UINT64CONST(1) << (ref.slot_idx % 64); + pg_atomic_uint64 *word = + &batch->slots_written_bitmap[ref.slot_idx / 64]; + bool had_owner = (pref->owner != NULL); + + /* + * Idempotent: the exit backstop (before_shmem_exit) and a later + * ResourceOwner release may both reach the same entry — the relative + * order of exit callbacks is not fixed across process types. + */ + if (!pref->in_use) + return; + + pref->owner = NULL; + pref->in_use = false; + + /* a held ref pins the batch, so its incarnation cannot have changed */ + Assert(ref.batch_id == batch->batch_id); + + if (!(pg_atomic_read_u64(word) & bit)) + { + /* + * Copy never published: poison the slot so the seal-waiter wakes up + * and recovery ignores it. + */ + batch->slot_flags[ref.slot_idx] |= DWB_SLOT_ABORTED; + pg_write_barrier(); + pg_atomic_fetch_or_u64(word, bit); + ConditionVariableBroadcast(&batch->cv_state); + } + else if (had_owner && + pg_atomic_read_u32(&batch->state) >= DWB_FSYNCED) + { + /* + * Copy published and the batch is durable, which means the writer was + * at or past step 6: its smgrwrite may have failed halfway. Make the + * data page whole again from the batch copy. + * + * Only for refs that were attached to a ResourceOwner: those are real + * write-path refs, and their BM_IO_IN_PROGRESS is still held here (on + * the proc-exit path too, see DWBProcExit). An ownerless (test) ref + * never had the buffer-IO interlock, so the repair write would race a + * concurrent flush of the same page. + */ + DWBRewriteAbandonedSlot(&ref); + } + + /* + * The last ref finishes the batch only once it is FSYNCED. A sealed + * batch cannot lose its last ref earlier — the leader holds its own pin + * from SEAL to FSYNCED (see DWBSealBatch) — so reaching zero refs in an + * earlier state means the batch is not sealed yet: it stays open and a + * later seal completes it normally. + */ + if (pg_atomic_fetch_sub_u32(&batch->ref_count, 1) == 1 && + pg_atomic_read_u32(&batch->state) == DWB_FSYNCED) + DWBFinishBatchData(batch); +} + +/* + * ResourceOwner release of one ref: the abort path of the write path. + */ +static void +ResOwnerReleaseDWBRef(Datum res) +{ + DWBAbandonRef((DWBPendingRef *) DatumGetPointer(res)); +} + +/* + * Exit backstop for refs that no ResourceOwner released. Runs as a + * before_shmem_exit callback: the PGPROC is still alive, so the LWLocks + * taken by a last-ref publication (publish_lock, DWBSegHashLock) are legal + * here — unlike in on_proc_exit callbacks, which run after ProcKill. It + * may run BEFORE the ResourceOwner release of the same refs (callback + * registration order); DWBAbandonRef is idempotent, so whichever side runs + * second is a no-op. + * + * An owned ref can only get here when abort cleanup was cut short before + * the ResourceOwner release phase (e.g. a FATAL thrown out of the abort + * path itself). In that case the buffer-IO resource of the failed flush + * was not released either: it lives in the SAME owner and releases AFTER + * the DWB ref (ascending priority within the phase, RELEASE_PRIO_BUFFER_IOS + * - 10 before RELEASE_PRIO_BUFFER_IOS; on the success path + * DWBFinishPageWrite likewise precedes TerminateBufferIO). So whenever an + * owned ref is still alive, BM_IO_IN_PROGRESS is still ours and the + * abandoned-slot repair is exactly as race-free as on the ResourceOwner + * path. It is also the last chance to repair: a FATAL exit does not + * trigger crash recovery, so no apply-pass would ever fix a torn page. + * + * Ownerless refs are test refs (DWBAcquireSlot with use_resowner = false); + * they never had the interlock and DWBAbandonRef skips the repair write for + * them. + */ +static void +DWBProcExit(int code, Datum arg) +{ + for (int i = 0; i < (int) lengthof(pendingRefs); i++) + { + if (pendingRefs[i].in_use) + DWBAbandonRef(&pendingRefs[i]); + } +} + +/* + * Per-process initialization: arrange for the refs this process is holding + * to be given back if it exits still holding them. + * + * before_shmem_exit, NOT on_proc_exit: dropping the last ref of a durable + * batch publishes its seg_set under LWLocks, which is only legal while our + * PGPROC is alive — on_proc_exit callbacks run after ProcKill has released + * it. + * + * The callback belongs here, among the other process-wide registrations, + * rather than at the first slot a process takes. A command that registers + * a cleanup callback of its own and then cancels it — PG_ENSURE_ERROR_CLEANUP, + * as CREATE DATABASE uses — requires its callback to still be the last one + * registered, and a first write staged through the buffer between the two + * would leave ours on top of it. + */ +void +DWBInitBackend(void) +{ + if (DWBIsEnabled()) + before_shmem_exit(DWBProcExit, 0); +} diff --git a/src/backend/storage/dwb/dwb_cleaner.c b/src/backend/storage/dwb/dwb_cleaner.c new file mode 100644 index 0000000000000..29a0c62ae8fcd --- /dev/null +++ b/src/backend/storage/dwb/dwb_cleaner.c @@ -0,0 +1,375 @@ +/*------------------------------------------------------------------------- + * + * dwb_cleaner.c + * Cleaner worker pool of the short-lived double write buffer: takes + * the flush bins the bgwriter's LRU scan produces and executes them, + * so the scan's issue rate is no longer capped by one process + * serially waiting out a batch fdatasync per bin. + * + * The bgwriter stays the only LRU scanner and pacing estimator (the + * allocation counter of StrategySyncStart is consumed on read, so the + * estimator cannot be split across processes). What scales here is + * execution only: bins travel through a small shared-memory queue and + * any pool worker flushes them through FlushBufferBin's opportunistic + * mode. + * + * Queue entries are hints, not obligations. Every claim is + * reclassified under the buffer header lock right before the write; a + * buffer that was recycled, became hot, went clean or is busy with + * somebody's I/O is skipped, never waited on. Hence no draining on + * shutdown and no meaning to queue contents after a crash: whatever was + * queued is still dirty and the next checkpoint covers it. + * + * Backpressure is the enqueue refusing a full queue: the bgwriter keeps + * the bin, stops scanning and re-offers it next round. With an active + * pool the bgwriter never writes data pages itself — a scan stalled + * behind serial batch fsyncs starves the pool of fresh bins and lets + * the strategy clock hand catch the scan point, pushing evictions onto + * the backends. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb_cleaner.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/buf_internals.h" +#include "storage/bufmgr.h" +#include "storage/condition_variable.h" +#include "storage/dwb.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "tcop/tcopprot.h" +#include "utils/guc.h" +#include "utils/injection_point.h" +#include "utils/resowner.h" +#include "utils/wait_event.h" + +DWBCleanerCtl *DWBCleanerQueue = NULL; + +/* set for the lifetime of a cleaner worker; DWBWriterClass consults it */ +bool DWBAmCleanerWorker = false; + +/* + * Queue capacity in bins. A fixed burst absorber, deliberately not + * scaled by the worker count: at observed pool drain rates even a full + * queue empties in tens of milliseconds, so entries stay fresh, while a + * queue sized to the pool would overflow on demand bursts exactly when + * the scan must not stall. 64 bins is ~17 kB of shared memory. + */ +#define DWB_CLEANER_QUEUE_CAPACITY 64 + +Size +DWBCleanerShmemSize(void) +{ + if (!DWBIsEnabled() || dwb_cleaner_workers == 0) + return 0; + + return add_size(offsetof(DWBCleanerCtl, bins), + mul_size(DWB_CLEANER_QUEUE_CAPACITY, + sizeof(DWBCleanerBin))); +} + +void +DWBCleanerShmemInit(void) +{ + bool found; + + if (!DWBIsEnabled() || dwb_cleaner_workers == 0) + return; + + DWBCleanerQueue = (DWBCleanerCtl *) + ShmemInitStruct("DWB Cleaner Queue", DWBCleanerShmemSize(), &found); + + if (!found) + { + memset(DWBCleanerQueue, 0, DWBCleanerShmemSize()); + pg_atomic_init_u64(&DWBCleanerQueue->enqueued_pages, 0); + pg_atomic_init_u64(&DWBCleanerQueue->pool_written, 0); + pg_atomic_init_u64(&DWBCleanerQueue->pool_written_total, 0); + pg_atomic_init_u64(&DWBCleanerQueue->skipped_pages, 0); + pg_atomic_init_u64(&DWBCleanerQueue->deferred_bins, 0); + pg_atomic_init_u64(&DWBCleanerQueue->pressure_naps, 0); + pg_atomic_init_u32(&DWBCleanerQueue->depth, 0); + ConditionVariableInit(&DWBCleanerQueue->cv_work); + DWBCleanerQueue->capacity = DWB_CLEANER_QUEUE_CAPACITY; + } +} + +/* + * True when bins may be handed to the pool instead of flushed in place. + */ +bool +DWBCleanersActive(void) +{ + return DWBCleanerQueue != NULL; +} + +/* + * Hand one bin to the pool. The queue lock is taken unconditionally — + * the critical section is one bin copy — so a false return means + * exactly one thing: the queue is full. The caller keeps the bin and + * re-offers it later (the bgwriter counts the refusal as a deferral). + */ +bool +DWBCleanerEnqueueBin(const int *buf_ids, int nbuf) +{ + DWBCleanerCtl *ctl = DWBCleanerQueue; + DWBCleanerBin *bin; + + Assert(ctl != NULL); + Assert(nbuf > 0 && nbuf <= DWB_FLUSH_BIN_MAX); + + LWLockAcquire(DWBCleanerQueueLock, LW_EXCLUSIVE); + if (ctl->nqueued == ctl->capacity) + { + LWLockRelease(DWBCleanerQueueLock); + return false; + } + + bin = &ctl->bins[(ctl->head + ctl->nqueued) % ctl->capacity]; + bin->nbuf = nbuf; + memcpy(bin->buf_ids, buf_ids, nbuf * sizeof(int)); + ctl->nqueued++; + pg_atomic_write_u32(&ctl->depth, ctl->nqueued); + LWLockRelease(DWBCleanerQueueLock); + + pg_atomic_fetch_add_u64(&ctl->enqueued_pages, nbuf); + ConditionVariableSignal(&ctl->cv_work); + return true; +} + +/* + * Take the oldest bin, if any. A local copy is returned so the queue + * lock is never held across the flush. + */ +static bool +DWBCleanerDequeueBin(DWBCleanerBin *bin) +{ + DWBCleanerCtl *ctl = DWBCleanerQueue; + bool got = false; + + LWLockAcquire(DWBCleanerQueueLock, LW_EXCLUSIVE); + if (ctl->nqueued > 0) + { + *bin = ctl->bins[ctl->head]; + ctl->head = (ctl->head + 1) % ctl->capacity; + ctl->nqueued--; + pg_atomic_write_u32(&ctl->depth, ctl->nqueued); + got = true; + } + LWLockRelease(DWBCleanerQueueLock); + return got; +} + +/* + * Advisory pressure signal for the checkpointer: is the bin queue at + * least half full? Reads only the lock-free depth mirror — a stale + * answer merely shifts one 100ms pacing decision, so no lock is taken; + * nqueued itself stays under DWBCleanerQueueLock. + */ +bool +DWBCleanerQueueHot(void) +{ + DWBCleanerCtl *ctl = DWBCleanerQueue; + + if (ctl == NULL || !DWBCleanersActive()) + return false; + return pg_atomic_read_u32(&ctl->depth) >= ctl->capacity / 2; +} + +/* + * Count a checkpointer nap taken only because the queue was hot (the + * base schedule check alone would have kept writing). + */ +void +DWBCleanerCountPressureNap(void) +{ + if (DWBCleanerQueue != NULL) + pg_atomic_fetch_add_u64(&DWBCleanerQueue->pressure_naps, 1); +} + +/* + * The bgwriter folds the pool's completed writes into + * PendingBgWriterStats.buf_written_clean once per round, keeping + * pg_stat_bgwriter's counter "pages written by LRU cleaning" no matter + * which process executed the write. + */ +uint64 +DWBCleanerFetchPoolWritten(void) +{ + if (!DWBCleanersActive()) + return 0; + return pg_atomic_exchange_u64(&DWBCleanerQueue->pool_written, 0); +} + +/* + * The bgwriter counts a bin the pool's queue refused; the bin itself is + * carried over to the next round, so this is a pure saturation gauge — + * nothing gets written on this path. Counted by the bgwriter, not + * inside the failed enqueue: a refused test claim defers nothing. + */ +void +DWBCleanerCountDeferral(void) +{ + pg_atomic_fetch_add_u64(&DWBCleanerQueue->deferred_bins, 1); +} + +/* + * Register dwb_cleaner_workers static background workers. Called from + * PostmasterMain right after the retire pool registration, before + * extensions get a chance at the worker slots. + */ +void +DWBCleanerWorkersRegister(void) +{ + BackgroundWorker bgw; + int free_slots; + + if (!DWBIsEnabled() || dwb_cleaner_workers == 0) + return; + + /* + * RegisterBackgroundWorker only LOGs on overflow, so check the slots + * actually left and fail loudly: a silently missing cleaner would ship a + * smaller pool than the operator configured. + */ + free_slots = max_worker_processes - GetNumRegisteredBackgroundWorkers(); + if (dwb_cleaner_workers > free_slots) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"dwb_cleaner_workers\" (%d) needs more \"max_worker_processes\" slots than remain free (%d)", + dwb_cleaner_workers, free_slots), + errhint("Increase \"max_worker_processes\" or decrease \"dwb_cleaner_workers\"."))); + + for (int i = 0; i < dwb_cleaner_workers; i++) + { + memset(&bgw, 0, sizeof(bgw)); + + /* + * The database-less connection gives the worker a pg_stat_activity + * entry; it forces BgWorkerStart_ConsistentState, which is fine: the + * bgwriter that feeds the queue starts even later, and until then the + * queue simply stays empty. + */ + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "DWBCleanerWorkerMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, "dwb cleaner %d", i); + snprintf(bgw.bgw_type, BGW_MAXLEN, "dwb cleaner"); + bgw.bgw_restart_time = 1; + bgw.bgw_notify_pid = 0; + bgw.bgw_main_arg = Int32GetDatum(i); + + RegisterBackgroundWorker(&bgw); + } +} + +/* + * Main loop: flush queued bins, sleep when the queue is empty. + */ +void +DWBCleanerWorkerMain(Datum main_arg) +{ + WritebackContext wb_context; + + pqsignal(SIGHUP, SignalHandlerForConfigReload); + + /* + * die, not a shutdown flag: a worker that exits with code 0 is + * unregistered for good, so one stray SIGTERM would permanently shrink + * the pool. The FATAL exit restarts after bgw_restart_time outside a + * postmaster shutdown and is simply the end during one. + */ + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* no database, just shared state and pg_stat_activity visibility */ + BackgroundWorkerInitializeConnection(NULL, NULL, 0); + + /* + * FlushBufferBin pins buffers and registers its buffer I/O with + * CurrentResourceOwner — that registration is what repairs an + * interrupted data-file write from the batch copy if an ERROR throws the + * worker out mid-bin. The aux-process owner provides both the owner and + * its shmem-exit release. + */ + CreateAuxProcessResourceOwner(); + + DWBAmCleanerWorker = true; + WritebackContextInit(&wb_context, &bgwriter_flush_after); + + for (;;) + { + DWBCleanerBin bin; + int written; + + /* the CFI is what turns a pending die() into the FATAL exit */ + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + + INJECTION_POINT("dwb-cleaner-loop", NULL); + + if (!DWBCleanerDequeueBin(&bin)) + { + /* + * Sleep without losing a wakeup: get onto the wait list first, + * then recheck under the queue lock, then sleep. A signal sent + * after the recheck is kept by the prepared state; a bin enqueued + * before it is seen by the recheck. The sleep itself checks for + * interrupts, so a pending die() cuts it short. + */ + ConditionVariablePrepareToSleep(&DWBCleanerQueue->cv_work); + if (!DWBCleanerDequeueBin(&bin)) + { + IssuePendingWritebacks(&wb_context, IOCONTEXT_NORMAL); + + /* + * Flush I/O statistics while idle: nothing else in this loop + * reports them, and the worker's pg_stat_io rows are how an + * operator sees the pool actually writing. Forced, because a + * deferred report would sit on local counters through the + * whole open-ended sleep that follows. The injection point + * lets a test park the worker right after the report; a + * parked worker switches its prepared condition-variable + * sleep to the injection one, which the sleep below repairs + * by re-preparing and returning for another loop. + */ + pgstat_report_stat(true); + INJECTION_POINT("dwb-cleaner-reported", NULL); + ConditionVariableSleep(&DWBCleanerQueue->cv_work, + WAIT_EVENT_DWB_CLEANER_MAIN); + continue; + } + } + + /* off the wait list while flushing (no-op if never prepared) */ + ConditionVariableCancelSleep(); + + /* + * In opportunistic mode every bin member ends up either written or + * skipped, so the skip count needs no extra plumbing. + */ + written = FlushBufferBin(bin.buf_ids, bin.nbuf, true, &wb_context); + pg_atomic_fetch_add_u64(&DWBCleanerQueue->pool_written, written); + pg_atomic_fetch_add_u64(&DWBCleanerQueue->pool_written_total, written); + pg_atomic_fetch_add_u64(&DWBCleanerQueue->skipped_pages, + bin.nbuf - written); + } +} diff --git a/src/backend/storage/dwb/dwb_ctl.c b/src/backend/storage/dwb/dwb_ctl.c new file mode 100644 index 0000000000000..83e17e0727e5d --- /dev/null +++ b/src/backend/storage/dwb/dwb_ctl.c @@ -0,0 +1,151 @@ +/*------------------------------------------------------------------------- + * + * dwb_ctl.c + * Shared-memory state and GUC variables of the short-lived double + * write buffer. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb_ctl.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "storage/dwb.h" +#include "storage/shmem.h" +#include "utils/guc.h" + +/* GUC variables (see 3.10 of the design plan) */ +int io_torn_pages_protection = DWB_PROTECT_FULL_PAGES; +int dwb_num_batches = 64; +int dwb_batch_pages = 64; +int dwb_max_segments = 4096; +int dwb_retire_workers = 1; +int dwb_cleaner_workers = 0; +int dwb_retire_sync_method = DWB_RETIRE_SYNC_METHOD_DEFAULT; +int dwb_batch_timeout_ms = 10; +int dwb_retire_interval_ms = 50; +int dwb_writeback_after = DEFAULT_DWB_WRITEBACK_AFTER; +int dwb_slow_warn_ms = 5000; +int dwb_slot_stuck_timeout_ms = 30000; +int dwb_write_timeout_ms = 60000; +int dwb_on_stall = DWB_ON_STALL_PANIC; + +DWCtl *DWBCtl = NULL; +char *DWBStagingBase = NULL; +HTAB *DWSegmentHash = NULL; + +static Size +DWBCtlSize(void) +{ + return offsetof(DWCtl, batches) + + mul_size(dwb_num_batches, sizeof(DWBatchCtl)); +} + +static Size +DWBSegEntrySize(void) +{ + return offsetof(DWSegEntry, batch_bitmap) + + mul_size(DWBSegBitmapWords(), sizeof(pg_atomic_uint64)); +} + +static Size +DWBStagingSize(void) +{ + /* IO-aligned staging buffers of one batch worth of pages each */ + return add_size(mul_size(DWB_STAGING_BUFFERS, + mul_size(dwb_batch_pages, BLCKSZ)), + PG_IO_ALIGN_SIZE); +} + +Size +DWBShmemSize(void) +{ + Size size; + + if (!DWBIsEnabled()) + return 0; + + size = add_size(DWBCtlSize(), DWBStagingSize()); + size = add_size(size, hash_estimate_size(dwb_max_segments, + DWBSegEntrySize())); + size = add_size(size, DWBCleanerShmemSize()); + return size; +} + +void +DWBShmemInit(void) +{ + bool found; + + if (!DWBIsEnabled()) + return; + + DWBCtl = (DWCtl *) ShmemInitStruct("DWB Ctl", DWBCtlSize(), &found); + + if (!found) + { + memset(DWBCtl, 0, DWBCtlSize()); + + for (int i = 0; i < DWB_NUM_WCLASSES; i++) + pg_atomic_init_u32(&DWBCtl->open_batch_idx[i], DWB_INVALID_BATCH); + pg_atomic_init_u64(&DWBCtl->next_batch_id, 1); + pg_atomic_init_u64(&DWBCtl->freed_events, 0); + pg_atomic_init_u64(&DWBCtl->ring_wait_retries, 0); + for (int c = 0; c < DWB_NUM_WCLASSES; c++) + for (int r = 0; r < DWB_SEAL_NREASONS; r++) + { + pg_atomic_init_u64(&DWBCtl->seal_count[c][r], 0); + pg_atomic_init_u64(&DWBCtl->seal_pages[c][r], 0); + } + for (int c = 0; c < DWB_NUM_WCLASSES; c++) + pg_atomic_init_u64(&DWBCtl->last_overflow_seal[c], 0); + for (int c = 0; c < DWB_NUM_WCLASSES; c++) + ConditionVariableInit(&DWBCtl->cv_want_batch[c]); + ConditionVariableInit(&DWBCtl->cv_retire_wake); + SpinLockInit(&DWBCtl->staging_lock); + DWBCtl->staging_free = (1U << DWB_STAGING_BUFFERS) - 1; + + for (int i = 0; i < dwb_num_batches; i++) + { + DWBatchCtl *batch = &DWBCtl->batches[i]; + + pg_atomic_init_u32(&batch->state, DWB_FREE); + pg_atomic_init_u32(&batch->next_slot_idx, 0); + pg_atomic_init_u32(&batch->capped_slots, 0); + for (int w = 0; w < DWB_BITMAP_WORDS; w++) + pg_atomic_init_u64(&batch->slots_written_bitmap[w], 0); + pg_atomic_init_u32(&batch->ref_count, 0); + pg_atomic_init_u32(&batch->seg_pending_count, 0); + LWLockInitialize(&batch->publish_lock, LWTRANCHE_DWB_PUBLISH); + ConditionVariableInit(&batch->cv_state); + batch->staging_idx = -1; + } + } + + { + char *base; + + base = (char *) ShmemInitStruct("DWB Staging", DWBStagingSize(), + &found); + DWBStagingBase = (char *) TYPEALIGN(PG_IO_ALIGN_SIZE, base); + } + + { + HASHCTL info; + + info.keysize = sizeof(DWSegRef); + info.entrysize = DWBSegEntrySize(); + + DWSegmentHash = ShmemInitHash("DWB Segment Hash", + dwb_max_segments, dwb_max_segments, + &info, + HASH_ELEM | HASH_BLOBS | HASH_FIXED_SIZE); + } + + DWBCleanerShmemInit(); +} diff --git a/src/backend/storage/dwb/dwb_file.c b/src/backend/storage/dwb/dwb_file.c new file mode 100644 index 0000000000000..b10d36433f07a --- /dev/null +++ b/src/backend/storage/dwb/dwb_file.c @@ -0,0 +1,440 @@ +/*------------------------------------------------------------------------- + * + * dwb_file.c + * On-disk format of the short-lived double write buffer: ring creation + * and preallocation, control file, batch writes. + * + * Ring files are fully preallocated with zeros at creation (same contract + * as WAL segments): every block is allocated and the file size never + * changes, so batch durability only needs fdatasync. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb_file.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "common/file_utils.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/bufmgr.h" +#include "storage/dwb.h" +#include "storage/fd.h" +#include "utils/memutils.h" +#include "utils/wait_event.h" + +/* backend-local cache of open batch-file VFDs; only leaders and retire + * workers ever open batch files */ +static File *batch_files = NULL; + +/* IO-aligned meta-region assembly buffer, allocated by DWBPrepareBatchWrite */ +static char *meta_buf = NULL; + +pg_crc32c +DWBImageCrc(const char *image) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, image, BLCKSZ); + FIN_CRC32C(crc); + return crc; +} + +pg_crc32c +DWBSlotMetaCrc(const DWSlotMeta *meta) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, meta, offsetof(DWSlotMeta, meta_crc)); + FIN_CRC32C(crc); + return crc; +} + +pg_crc32c +DWBControlCrc(const DWBControlFileData *control) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, control, offsetof(DWBControlFileData, crc)); + FIN_CRC32C(crc); + return crc; +} + +pg_crc32c +DWBBatchHeaderCrc(const DWBBatchHeader *hdr) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, hdr, offsetof(DWBBatchHeader, crc)); + FIN_CRC32C(crc); + return crc; +} + +void +DWBBatchFilePath(char *path, int batch_idx) +{ + snprintf(path, MAXPGPATH, DWB_DIR "/batch_%04d", batch_idx); +} + +/* + * Read pg_dwb/control. Returns false if the file does not exist and + * missing_ok; any other failure (including a CRC mismatch) is FATAL — + * a damaged control file must not silently degrade the apply-pass. + * + * A caller that can refuse startup with a more helpful message than the + * low-level FATALs may pass corruptp: an unreadable or corrupt file then + * sets *corruptp and returns false instead, with the specific cause + * reported at LOG so it is not lost behind the caller's summary. A + * too-new format version is FATAL either way: that ring is intact, and + * "corrupt" advice would invite discarding it. + */ +bool +DWBReadControlFile(DWBControlFileData *control, bool missing_ok, + bool *corruptp) +{ + /* with elevel < ERROR the ereports return and the *corruptp tails run */ + int elevel = corruptp ? LOG : FATAL; + int fd; + int r; + + if (corruptp) + *corruptp = false; + + fd = OpenTransientFile(DWB_CONTROL_FILE, O_RDONLY | PG_BINARY); + if (fd < 0) + { + if (errno == ENOENT && missing_ok) + return false; + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", DWB_CONTROL_FILE))); + *corruptp = true; + return false; + } + + pgstat_report_wait_start(WAIT_EVENT_DWB_CONTROL_READ); + errno = 0; + r = read(fd, control, sizeof(DWBControlFileData)); + pgstat_report_wait_end(); + if (r != sizeof(DWBControlFileData)) + { + /* distinguish a real read error from a truncated file */ + if (r < 0) + ereport(elevel, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", + DWB_CONTROL_FILE))); + else + ereport(elevel, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read file \"%s\": read %d of %zu", + DWB_CONTROL_FILE, r, + sizeof(DWBControlFileData)))); + if (CloseTransientFile(fd) != 0) + ereport(LOG, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", + DWB_CONTROL_FILE))); + *corruptp = true; + return false; + } + if (CloseTransientFile(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", DWB_CONTROL_FILE))); + + if (control->magic != DWB_CONTROL_MAGIC || + !EQ_CRC32C(control->crc, DWBControlCrc(control))) + { + ereport(elevel, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid checksum or magic number in file \"%s\"", + DWB_CONTROL_FILE))); + *corruptp = true; + return false; + } + if (control->min_version > DWB_VERSION) + ereport(FATAL, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("file \"%s\" requires format version at least %u, but this server supports %u", + DWB_CONTROL_FILE, control->min_version, DWB_VERSION))); + + return true; +} + +/* + * Write pg_dwb/control atomically: tmp file + fsync + durable_rename, + * all inside pg_dwb/. + */ +void +DWBWriteControlFile(const DWBControlFileData *control) +{ + const char *tmppath = DWB_DIR "/control.tmp"; + int fd; + + fd = OpenTransientFile(tmppath, + O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", tmppath))); + + pgstat_report_wait_start(WAIT_EVENT_DWB_CONTROL_WRITE); + errno = 0; + if (write(fd, control, sizeof(DWBControlFileData)) != + sizeof(DWBControlFileData)) + { + if (errno == 0) + errno = ENOSPC; + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", tmppath))); + } + pgstat_report_wait_end(); + + pgstat_report_wait_start(WAIT_EVENT_DWB_CONTROL_SYNC); + if (pg_fsync(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", tmppath))); + pgstat_report_wait_end(); + + if (CloseTransientFile(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", tmppath))); + + durable_rename(tmppath, DWB_CONTROL_FILE, FATAL); +} + +/* + * Create pg_dwb/ from scratch: directory, zero-preallocated batch files, + * control with generation 0. + */ +void +DWBCreateRing(void) +{ + DWBControlFileData control; + + if (MakePGDirectory(DWB_DIR) < 0 && errno != EEXIST) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", DWB_DIR))); + + for (int i = 0; i < dwb_num_batches; i++) + { + char path[MAXPGPATH]; + int fd; + int rc; + + DWBBatchFilePath(path, i); + fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_RDWR | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + + pgstat_report_wait_start(WAIT_EVENT_DWB_RING_INIT); + rc = pg_pwrite_zeros(fd, DWBBatchFileSize(dwb_batch_pages), 0); + if (rc < 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", path))); + if (pg_fsync(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + pgstat_report_wait_end(); + + if (CloseTransientFile(fd) != 0) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + } + + memset(&control, 0, sizeof(control)); + control.magic = DWB_CONTROL_MAGIC; + control.version = DWB_VERSION; + control.min_version = DWB_MIN_VERSION; + control.num_batches = dwb_num_batches; + control.batch_pages = dwb_batch_pages; + control.generation = 0; + control.crc = DWBControlCrc(&control); + DWBWriteControlFile(&control); + + fsync_fname(DWB_DIR, true); +} + +/* + * Return an open VFD for a batch file, from the backend-local cache. + */ +int +DWBOpenBatchFile(int batch_idx) +{ + char path[MAXPGPATH]; + + if (batch_files == NULL) + { + batch_files = (File *) + MemoryContextAllocZero(TopMemoryContext, + dwb_num_batches * sizeof(File)); + for (int i = 0; i < dwb_num_batches; i++) + batch_files[i] = -1; + } + + if (batch_files[batch_idx] >= 0) + return batch_files[batch_idx]; + + DWBBatchFilePath(path, batch_idx); + batch_files[batch_idx] = PathNameOpenFile(path, O_RDWR | PG_BINARY); + if (batch_files[batch_idx] < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + return batch_files[batch_idx]; +} + +/* + * Pre-open the batch file and pre-allocate the meta-region buffer, so that + * DWBWriteBatch can run inside the leader's critical section without + * allocating anything. Called before the seal is attempted, where an + * ERROR is still harmless. + */ +void +DWBPrepareBatchWrite(int batch_idx) +{ + if (meta_buf == NULL) + meta_buf = MemoryContextAllocAligned(TopMemoryContext, + DWBMetaRegionSize(dwb_batch_pages), + PG_IO_ALIGN_SIZE, 0); + (void) DWBOpenBatchFile(batch_idx); +} + +/* + * Read one slot's page image back from a batch file. Used by the abort + * cleanup of a published ref whose batch is already durable (>= FSYNCED): + * the staged copy in shmem is gone by then, the batch file is the + * authoritative source. Failure — the open included — is PANIC: the + * caller is about to repair a possibly-torn data page, has no fallback, + * and may be running from a ResourceOwner release callback. + * + * Deliberately avoids the VFD layer (BasicOpenFile + raw pg_pread): a + * release callback must not fail, and PathNameOpenFile can throw ERROR + * from its internal allocations. BasicOpenFile allocates nothing and + * still recovers from EMFILE/ENFILE by closing LRU VFDs. + */ +void +DWBReadSlotImage(int batch_idx, int slot_idx, char *dst) +{ + char path[MAXPGPATH]; + int fd; + off_t off = DWBMetaRegionSize(dwb_batch_pages) + + (off_t) slot_idx * BLCKSZ; + ssize_t r; + + DWBBatchFilePath(path, batch_idx); + fd = BasicOpenFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + + pgstat_report_wait_start(WAIT_EVENT_DWB_BATCH_READ); + r = pg_pread(fd, dst, BLCKSZ, off); + pgstat_report_wait_end(); + if (r != BLCKSZ) + { + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not read slot %d of batch %d in \"%s\": %m", + slot_idx, batch_idx, DWB_DIR))); + ereport(PANIC, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read slot %d of batch %d in \"%s\": read %zd of %d", + slot_idx, batch_idx, DWB_DIR, r, BLCKSZ))); + } + if (close(fd) != 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); +} + +/* + * Leader write of one batch: (a) one contiguous pwrite of the image stream + * from staging, (b) one pwrite of the meta region, (c) fdatasync. Exactly + * this order: a crash while reusing a slot must never leave valid-looking + * meta over a torn or foreign image (any partially-persistent mix is + * rejected locally by meta_crc/generation/image_crc, see 3.2/3.4). + * + * Runs inside the leader's critical section: every ereport here escalates + * to PANIC, which is deliberate — an incomplete leader write cannot be + * unwound (see DWBSealBatch). + */ +void +DWBWriteBatch(int batch_idx, const DWBBatchHeader *hdr, + const DWSlotMeta *metas, const char *images) +{ + Size meta_region = DWBMetaRegionSize(dwb_batch_pages); + File file = DWBOpenBatchFile(batch_idx); + Size image_bytes = (Size) hdr->n_slots * BLCKSZ; + ssize_t nwritten; + int fd; + instr_time io_start; + + /* DWBPrepareBatchWrite has run */ + Assert(meta_buf != NULL); + + io_start = pgstat_prepare_io_time(track_io_timing); + + nwritten = FileWrite(file, images, image_bytes, meta_region, + WAIT_EVENT_DWB_BATCH_WRITE); + if (nwritten != (ssize_t) image_bytes) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write batch %d of \"%s\": %m", + batch_idx, DWB_DIR))); + + memset(meta_buf, 0, meta_region); + memcpy(meta_buf, hdr, sizeof(DWBBatchHeader)); + memcpy(meta_buf + sizeof(DWBBatchHeader), metas, + hdr->n_slots * sizeof(DWSlotMeta)); + + nwritten = FileWrite(file, meta_buf, meta_region, 0, + WAIT_EVENT_DWB_BATCH_WRITE); + if (nwritten != (ssize_t) meta_region) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write batch %d of \"%s\": %m", + batch_idx, DWB_DIR))); + + pgstat_count_io_op_time(IOOBJECT_DWB, IOCONTEXT_NORMAL, IOOP_WRITE, + io_start, 1, image_bytes + meta_region); + + /* + * fdatasync suffices: the file was fully preallocated at ring creation, + * its size and block layout never change (WAL-segment contract). + */ + io_start = pgstat_prepare_io_time(track_io_timing); + pgstat_report_wait_start(WAIT_EVENT_DWB_BATCH_SYNC); + fd = FileGetRawDesc(file); + if (fd < 0 || pg_fdatasync(fd) != 0) + ereport(data_sync_elevel(ERROR), + (errcode_for_file_access(), + errmsg("could not fsync batch %d of \"%s\": %m", + batch_idx, DWB_DIR))); + pgstat_report_wait_end(); + pgstat_count_io_op_time(IOOBJECT_DWB, IOCONTEXT_NORMAL, IOOP_FSYNC, + io_start, 1, 0); +} diff --git a/src/backend/storage/dwb/dwb_recovery.c b/src/backend/storage/dwb/dwb_recovery.c new file mode 100644 index 0000000000000..bdc7df7c2909e --- /dev/null +++ b/src/backend/storage/dwb/dwb_recovery.c @@ -0,0 +1,649 @@ +/*------------------------------------------------------------------------- + * + * dwb_recovery.c + * Startup-time handling of the short-lived double write buffer ring: + * the apply-pass that repairs torn data pages, and the durable + * generation protocol around it. + * + * On every double_writes start (clean, unclean or cold) the durable + * generation in pg_dwb/control is bumped BEFORE the ring opens for new + * writes, so slots left behind by the previous run can never masquerade as + * current after a future crash. Order: read G -> (RING_CLEAN not set) + * apply-pass over generation G + fsync -> durable control.generation := G+1 + * -> open ring. Starts in the other modes leave the ring untouched, + * except that a restored base backup's ring contents are discarded. + * + * The apply-pass runs before WAL replay and repairs the data files + * directly: a candidate slot must carry a valid meta_crc, the current + * generation and a valid image_crc; candidates are deduplicated per page + * keeping the highest LSN, and a page is rewritten from its slot copy when + * the on-disk version fails verification or carries an older LSN. This is + * the only repair path — the runtime read path never consults the ring — + * and it covers pages replay never reads, such as hint-bit-only pages + * logged as XLOG_FPI_FOR_HINT without an image. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb_recovery.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include + +#include "access/xlog.h" +#include "pgstat.h" +#include "storage/bufpage.h" +#include "storage/dwb.h" +#include "storage/fd.h" +#include "storage/smgr.h" +#include "utils/hsearch.h" +#include "utils/wait_event.h" + +/* dedup table entry: the best candidate slot seen for one page */ +typedef struct DWBApplyCandidate +{ + BufferTag tag; /* hash key */ + XLogRecPtr lsn; + uint64 batch_id; /* LSN tie-breaker; see the dedup comment in + * the scan loop below */ + uint32 batch_idx; + uint32 slot_idx; + pg_crc32c image_crc; /* revalidates the image on re-read */ +} DWBApplyCandidate; + +/* one fork the apply-pass has written to and must fsync (HASH_BLOBS key) */ +typedef struct DWBAppliedFork +{ + RelFileLocator rlocator; + ForkNumber forknum; +} DWBAppliedFork; + +StaticAssertDecl(sizeof(DWBAppliedFork) == + sizeof(RelFileLocator) + sizeof(ForkNumber), + "DWBAppliedFork has padding; unsafe as a HASH_BLOBS key"); + +static XLogRecPtr DWBApplyPass(const DWBControlFileData *control); +static bool DWBWipeRing(void); +static bool DWBRingIsQuiescent(void); + +/* + * Read one batch file's page image into an aligned buffer. The apply-pass + * variant of DWBReadSlotImage: geometry comes from the on-disk control + * file, not the GUCs, and failures are plain ERRORs (startup context, no + * critical section). + */ +static void +DWBApplyReadImage(int fd, const char *path, uint32 batch_pages, + uint32 slot_idx, char *dst) +{ + off_t off = DWBMetaRegionSize(batch_pages) + + (off_t) slot_idx * BLCKSZ; + ssize_t r; + + pgstat_report_wait_start(WAIT_EVENT_DWB_BATCH_READ); + r = pg_pread(fd, dst, BLCKSZ, off); + pgstat_report_wait_end(); + if (r != BLCKSZ) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", path))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read file \"%s\": read %zd of %d", + path, r, BLCKSZ))); + } +} + +/* + * The eager repair pass (3.8 of the design plan). Scans the whole ring, + * selects candidate slots of the current generation, dedups them per page + * and rewrites the data pages that are torn or older than their copy. + * Returns the highest LSN actually applied, or InvalidXLogRecPtr. + * + * Runs in the startup process before WAL replay, strictly read-only with + * respect to the ring: a crash in the middle leaves control.generation + * untouched and the next start simply repeats the pass (already-repaired + * pages then carry disk_lsn >= slot_lsn and are skipped). + */ +static XLogRecPtr +DWBApplyPass(const DWBControlFileData *control) +{ + Size meta_region = DWBMetaRegionSize(control->batch_pages); + char *meta_buf = palloc(meta_region); + char *image_buf = palloc_aligned(BLCKSZ, PG_IO_ALIGN_SIZE, 0); + char *disk_buf = palloc_aligned(BLCKSZ, PG_IO_ALIGN_SIZE, 0); + HASHCTL info; + HTAB *candidates; + HTAB *applied_forks; + HASH_SEQ_STATUS seq; + DWBApplyCandidate *cand; + DWBAppliedFork *fork; + int n_candidates = 0; + int n_applied = 0; + XLogRecPtr applied_upto = InvalidXLogRecPtr; + + info.keysize = sizeof(BufferTag); + info.entrysize = sizeof(DWBApplyCandidate); + candidates = hash_create("DWB apply-pass candidates", + (long) control->num_batches * control->batch_pages, + &info, + HASH_ELEM | HASH_BLOBS); + + /* forks written to, for the final fsync sweep */ + info.keysize = sizeof(DWBAppliedFork); + info.entrysize = sizeof(DWBAppliedFork); + applied_forks = hash_create("DWB apply-pass forks", 16, &info, + HASH_ELEM | HASH_BLOBS); + + /* + * Scan every batch file with the geometry recorded in control. A batch + * or slot that fails any local validity check is skipped, not an error: + * by the write protocol a torn or half-written slot means the + * corresponding data-file write never started, so the disk holds an older + * durable version that replay can advance from. + */ + for (uint32 batch_idx = 0; batch_idx < control->num_batches; batch_idx++) + { + char path[MAXPGPATH]; + int fd; + ssize_t r; + DWBBatchHeader hdr; + DWSlotMeta *metas; + + DWBBatchFilePath(path, batch_idx); + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + + pgstat_report_wait_start(WAIT_EVENT_DWB_BATCH_READ); + r = pg_pread(fd, meta_buf, meta_region, 0); + pgstat_report_wait_end(); + if (r != (ssize_t) meta_region) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", path))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read file \"%s\": read %zd of %zu", + path, r, meta_region))); + } + + memcpy(&hdr, meta_buf, sizeof(DWBBatchHeader)); + if (hdr.magic != DWB_BATCH_MAGIC || + hdr.version != DWB_VERSION || + hdr.n_slots > control->batch_pages || + !EQ_CRC32C(hdr.crc, DWBBatchHeaderCrc(&hdr))) + { + /* never sealed, or torn mid-write: nothing durable depends on it */ + CloseTransientFile(fd); + continue; + } + + metas = (DWSlotMeta *) (meta_buf + sizeof(DWBBatchHeader)); + for (uint32 slot_idx = 0; slot_idx < hdr.n_slots; slot_idx++) + { + DWSlotMeta *meta = &metas[slot_idx]; + DWBApplyCandidate *entry; + bool found; + + if (!EQ_CRC32C(meta->meta_crc, DWBSlotMetaCrc(meta))) + continue; + if (meta->generation != control->generation) + continue; + if (meta->flags & DWB_SLOT_ABORTED) + continue; + + DWBApplyReadImage(fd, path, control->batch_pages, slot_idx, + image_buf); + if (!EQ_CRC32C(meta->image_crc, DWBImageCrc(image_buf))) + continue; + + /* + * Keep the highest LSN; on equal LSNs the higher batch_id wins + * (equal LSNs with different contents are real: a re-flush after + * hint-bit-only changes does not move the LSN — but such copies + * differ only in hint bits, so any of them is a valid redo base + * and the id merely makes the pick deterministic; with two writer + * classes a later flush can even land in a batch opened earlier). + * Within one batch the later slot wins by plain overwrite, + * matching the order the slots were filled in. + */ + n_candidates++; + entry = hash_search(candidates, &meta->tag, HASH_ENTER, &found); + if (found && + (entry->lsn > meta->page_lsn || + (entry->lsn == meta->page_lsn && + entry->batch_id > hdr.batch_id))) + continue; + entry->lsn = meta->page_lsn; + entry->batch_id = hdr.batch_id; + entry->batch_idx = batch_idx; + entry->slot_idx = slot_idx; + entry->image_crc = meta->image_crc; + } + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + } + + /* + * Repair. For each winning candidate, rewrite the data page when the + * on-disk version fails verification or is older than the copy. A + * dropped relation (no file) or a truncated one (block beyond EOF) is + * skipped: there is nothing to repair and replay or a replayed truncate + * drives the final state. + */ + hash_seq_init(&seq, candidates); + while ((cand = hash_seq_search(&seq)) != NULL) + { + RelFileLocator rlocator = BufTagGetRelFileLocator(&cand->tag); + ForkNumber forknum = BufTagGetForkNum(&cand->tag); + BlockNumber blkno = cand->tag.blockNum; + SMgrRelation reln; + char path[MAXPGPATH]; + int fd; + DWBAppliedFork fkey; + + reln = smgropen(rlocator, INVALID_PROC_NUMBER); + if (!smgrexists(reln, forknum)) + continue; + if (blkno >= smgrnblocks(reln, forknum)) + continue; + + smgrread(reln, forknum, blkno, disk_buf); + + /* + * A "new" page (empty header) is never repaired. For a candidate + * holding an initialized image, an empty on-disk header means the + * covered write's first sector never reached disk and the block had + * never held an initialized page before — its init record therefore + * lies after the last checkpoint, and replay recreates the page + * without reading the current contents. (A staged image can itself + * be all-zero — FlushBuffer may flush a still-new page — but such + * a copy carries LSN 0: skipping it here changes nothing, and on the + * repair branch below it would merely complete an intended zeroing.) + * The skip is also required for correctness in the other direction: + * after a truncate + re-extend within one generation the ring can + * hold a pre-truncate copy of this block, and the re-extended zeroed + * page (LSN 0) would lose the LSN comparison below to that stale + * image, which nothing would then replay over. + */ + if (PageIsNew((Page) disk_buf)) + continue; + + if (PageIsVerified((Page) disk_buf, blkno, PIV_LOG_LOG, NULL) && + PageGetLSN((Page) disk_buf) >= cand->lsn) + continue; + + /* re-read the winning copy; the scan buffer is long overwritten */ + DWBBatchFilePath(path, (int) cand->batch_idx); + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + DWBApplyReadImage(fd, path, control->batch_pages, cand->slot_idx, + image_buf); + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + /* + * The image was CRC-checked during the scan, but this is a second + * physical read; a divergence means the storage returned different + * bytes twice, and writing them over a data page would defeat the + * pass's whole purpose. + */ + if (!EQ_CRC32C(cand->image_crc, DWBImageCrc(image_buf))) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("page image in file \"%s\" slot %u failed verification on re-read", + path, cand->slot_idx))); + + elog(DEBUG1, "double write buffer recovery: restoring page %u of relation %u/%u/%u fork %d from batch %u slot %u (LSN %X/%X)", + blkno, rlocator.spcOid, rlocator.dbOid, rlocator.relNumber, + forknum, cand->batch_idx, cand->slot_idx, + LSN_FORMAT_ARGS(cand->lsn)); + + smgrwrite(reln, forknum, blkno, image_buf, true); + + n_applied++; + if (cand->lsn > applied_upto) + applied_upto = cand->lsn; + + fkey.rlocator = rlocator; + fkey.forknum = forknum; + (void) hash_search(applied_forks, &fkey, HASH_ENTER, NULL); + } + + /* make the repairs durable before the generation moves on */ + hash_seq_init(&seq, applied_forks); + while ((fork = hash_seq_search(&seq)) != NULL) + smgrimmedsync(smgropen(fork->rlocator, INVALID_PROC_NUMBER), + fork->forknum); + + ereport(LOG, + (errmsg("double write buffer recovery: %d of %d candidate pages restored, generation " UINT64_FORMAT, + n_applied, n_candidates, control->generation))); + + hash_destroy(candidates); + hash_destroy(applied_forks); + pfree(meta_buf); + pfree(image_buf); + pfree(disk_buf); + + return applied_upto; +} + +/* + * Durably remove the contents of pg_dwb/, keeping the directory (or the + * symlink to it) in place, and report whether there was anything to + * remove — a restored backup normally ships pg_dwb/ empty, and the + * callers must not claim to have discarded ring contents that never + * existed. Callers: the restored-backup branches (a shipped ring must + * not survive), the geometry change, and the cold-create sweep that + * clears leftovers of an interrupted wipe. + * + * The control file goes first, durably: a crash in the middle of the batch + * sweep must not leave a readable control beside missing batch files, or a + * retried apply-pass would hard-fail on the ENOENT forever. With the + * control gone first, a retry takes the cold-create path instead, which is + * correct for every caller, since the apply-pass, if one was needed, ran + * to completion before any wipe starts. + */ +static bool +DWBWipeRing(void) +{ + struct stat st; + DIR *dir; + struct dirent *de; + bool had_contents = false; + + if (lstat(DWB_DIR, &st) < 0) + { + if (errno == ENOENT) + return false; + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not stat directory \"%s\": %m", DWB_DIR))); + } + + dir = AllocateDir(DWB_DIR); + while ((de = ReadDir(dir, DWB_DIR)) != NULL) + { + if (strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) + { + had_contents = true; + break; + } + } + FreeDir(dir); + + if (!had_contents) + return false; + + if (unlink(DWB_CONTROL_FILE) < 0 && errno != ENOENT) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not remove file \"%s\": %m", + DWB_CONTROL_FILE))); + fsync_fname(DWB_DIR, true); + + if (!rmtree(DWB_DIR, false)) + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not remove contents of directory \"%s\"", + DWB_DIR))); + fsync_fname(DWB_DIR, true); + + return true; +} + +/* + * Called from StartupXLOG before WAL recovery is initialized. Creates or + * validates the ring, enforces data checksums, repairs torn data pages + * from the ring unless it was cleanly closed, performs the durable + * generation bump and publishes ring_generation for the leaders' slot + * metas. Returns the highest LSN the apply-pass wrote to a data file, or + * InvalidXLogRecPtr. + * + * Whether the apply-pass must run is decided by the ring's own RING_CLEAN + * marker alone, never by the pg_control state. The marker is the exact + * certificate: it is set only after full retirement (so while it is set, + * no slot covers a data write that has not reached disk) and cleared + * before the ring reopens (so slots of the clearing run are covered until + * the next clean shutdown re-sets it). pg_control can be both cleaner and + * dirtier than the ring: a standby's shutdown restartpoint can be skipped + * entirely, leaving retirement fsyncs pending behind a clean pg_control — + * and a crash under an interim full_pages/off run (which touches neither + * the marker nor the generation) leaves an unclean pg_control over a fully + * retired ring whose stale slots still match the current generation, where + * an apply would resurrect ancient pages over blocks torn long after the + * ring was closed. + * + * restoring_backup means the data directory is a restored base backup + * (backup_label present, or pg_control still carrying backupStartPoint + * after a crash mid-backup-recovery). A ring found in that case — shipped + * by a third-party backup tool, or this server's own from a crashed + * backup-recovery run — must not be applied: its slots carry the restored + * control's own generation, and the restored data files are legitimately + * older than the slot copies, so both staleness defences pass — an + * unguarded apply would push pages from the future of the backup into a + * PITR target. The WAL of the backup window carries forced full page + * images instead, so the ring is not needed for this recovery; it is + * wiped and recreated cold. + */ +XLogRecPtr +DWBStartup(bool restoring_backup) +{ + DWBControlFileData control; + XLogRecPtr applied_upto = InvalidXLogRecPtr; + bool created = false; + bool corrupt; + bool need_apply; + + /* + * A ring shipped inside a restored backup must not survive in any mode: + * dormant, it would greet a much later switch to double_writes with a + * plausible control file, and under double_writes it must not be applied + * (see the header comment). Discard it before anything reads the ring + * state. + */ + if (restoring_backup && DWBWipeRing()) + ereport(LOG, + (errmsg("discarding double write buffer ring contents restored from a base backup"))); + + if (!DWBIsEnabled()) + { + /* + * The most dangerous mode must not be the quietest one: with "off" + * neither page images nor the ring protect data files, and the legacy + * full_page_writes GUC may still read "on". + */ + if (io_torn_pages_protection == DWB_PROTECT_OFF) + ereport(LOG, + (errmsg("torn page protection is disabled (io_torn_pages_protection = \"off\")"), + errdetail("WAL carries no full page images; \"full_page_writes\" is ignored in this mode."))); + + if (restoring_backup) + return InvalidXLogRecPtr; + + /* + * Mode-downgrade guard: a ring that was not cleanly closed may hold + * repairs of torn data pages that only a double_writes start can + * apply. This intentionally does not consider pg_control: a clean + * server shutdown that failed to retire the ring (e.g. a soft fsync + * failure under data_sync_retry) leaves RING_CLEAN unset, and the + * pending data writes it covers are exactly as unprotected. + */ + if (DWBReadControlFile(&control, true, &corrupt)) + { + if ((control.flags & DWB_CONTROL_RING_CLEAN) == 0) + ereport(FATAL, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the double write buffer ring was not cleanly shut down, cannot start with \"io_torn_pages_protection=%s\"", + DWBProtectionModeName(io_torn_pages_protection)), + errdetail("The ring in \"%s\" may hold repairs of torn data pages that have not been applied.", + DWB_DIR), + errhint("Start the server once with \"io_torn_pages_protection=double_writes\" and shut it down cleanly, or remove \"%s\" if you accept the risk of torn data pages.", + DWB_DIR))); + } + else if (corrupt) + { + /* + * An unreadable ring state must not block modes that never touch + * the ring with a bare low-level error: name the way out. (A + * double_writes start would refuse too, so the only cure is + * removal.) + */ + ereport(FATAL, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the double write buffer ring state could not be validated, cannot start with \"io_torn_pages_protection=%s\"", + DWBProtectionModeName(io_torn_pages_protection)), + errdetail("The control file in \"%s\" is unreadable or corrupt, and the ring may hold repairs of torn data pages that have not been applied.", + DWB_DIR), + errhint("Remove \"%s\" if you accept the risk of torn data pages.", + DWB_DIR))); + } + return InvalidXLogRecPtr; + } + + /* 3.1.7: a torn page with an intact header must never pass unnoticed */ + if (!DataChecksumsEnabled()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("io_torn_pages_protection = \"double_writes\" requires data checksums"), + errhint("Enable checksums with initdb -k or pg_checksums."))); + + if (!DWBReadControlFile(&control, true, NULL)) + { + /* + * Cold start: no ring yet. Sweep the directory first — an + * interrupted wipe can leave batch files behind after the control + * file is gone. + */ + (void) DWBWipeRing(); + DWBCreateRing(); + created = true; + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + } + + /* + * The RING_CLEAN marker alone decides (see the DWBStartup comment above + * for why pg_control must not weigh in). A freshly created ring has + * nothing to apply even though its marker is unset. + */ + need_apply = !created && + (control.flags & DWB_CONTROL_RING_CLEAN) == 0; + + if (!created && + (control.num_batches != (uint32) dwb_num_batches || + control.batch_pages != (uint32) dwb_batch_pages)) + { + /* + * The geometry GUCs changed. The old ring must still be applied + * first — its batch files follow the recorded geometry — and only + * then can the ring be recreated under the new one. The fresh + * control restarts the generation from zero, which is safe exactly + * because the wipe left no slot behind. + */ + if (need_apply) + applied_upto = DWBApplyPass(&control); + ereport(LOG, + (errmsg("recreating double write buffer ring: geometry changed from %u batches of %u pages to %d batches of %d pages", + control.num_batches, control.batch_pages, + dwb_num_batches, dwb_batch_pages))); + (void) DWBWipeRing(); + DWBCreateRing(); + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + } + else if (need_apply) + applied_upto = DWBApplyPass(&control); + + control.generation++; + control.flags &= ~DWB_CONTROL_RING_CLEAN; + control.crc = DWBControlCrc(&control); + DWBWriteControlFile(&control); + + DWBCtl->ring_generation = control.generation; + + ereport(LOG, + (errmsg("double write buffer ring opened: %d batches of %d pages, generation " UINT64_FORMAT, + dwb_num_batches, dwb_batch_pages, control.generation))); + + return applied_upto; +} + +/* + * True when no on-disk slot covers a data write that could still be + * pending: every batch is FREE, or ALLOCATED (an open batch whose staged + * copies live in shared memory only — nothing of it has been written to + * the ring files, and its file still holds the fully retired slots of the + * previous incarnation). + */ +static bool +DWBRingIsQuiescent(void) +{ + for (int i = 0; i < dwb_num_batches; i++) + { + DWBatchState state = DWBGetBatchState(i); + + if (state != DWB_FREE && state != DWB_ALLOCATED) + return false; + } + return true; +} + +/* + * Called at the tail of a clean shutdown, after the shutdown checkpoint or + * restartpoint. Retires whatever the checkpoint left behind (a shutdown + * restartpoint can be skipped entirely, leaving segment fsyncs pending) + * and then sets RING_CLEAN in the ring's control file, entitling the next + * start to skip the apply-pass and legalizing a start under a different + * io_torn_pages_protection mode. If the ring cannot be fully retired the + * marker simply stays unset — the next start applies the ring, which is + * always safe. + */ +void +DWBMarkCleanShutdown(void) +{ + DWBControlFileData control; + + if (!DWBIsEnabled()) + return; + + if (!DWBRingIsQuiescent()) + { + DWBRetireAllSync(); + if (!DWBRingIsQuiescent()) + { + ereport(LOG, + (errmsg("double write buffer ring could not be fully retired; not marking it cleanly shut down"))); + return; + } + } + + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + control.flags |= DWB_CONTROL_RING_CLEAN; + control.crc = DWBControlCrc(&control); + DWBWriteControlFile(&control); +} diff --git a/src/backend/storage/dwb/dwb_retire.c b/src/backend/storage/dwb/dwb_retire.c new file mode 100644 index 0000000000000..c027aa065de97 --- /dev/null +++ b/src/backend/storage/dwb/dwb_retire.c @@ -0,0 +1,949 @@ +/*------------------------------------------------------------------------- + * + * dwb_retire.c + * Retirement of double write buffer batches: the segment->batch + * back-reference hash, the durability accounting that frees batches, + * and the retire worker pool. + * + * A batch reaches FREE only after every segment in its seg_set has been + * fsynced after the batch's data-file writes. The accounting protocol + * (3.5 of the design plan): + * + * - publication: at DATA_WRITTEN -> RETIRING, under the batch's + * publish_lock, seg_pending_count := n_segs and one bit per segment is + * set in the segment's DWSegmentHash entry; + * - decrement: a successful segment fsync clears the bits that were + * already set BEFORE the fsync started (a bit published mid-fsync may + * cover a write the fsync missed) and decrements seg_pending_count of + * the owning batches; the decrement to zero frees the batch; + * - ABA guard: bits address batches by ring index, which is reused, so + * the fsyncer snapshots (batch_idx, batch_id) before the fsync and, + * under that batch's publish_lock, decrements only if batch_id still + * matches. + * + * Fsyncs come from three independent sources: the retire worker pool + * (proactive, partitioned by segment hash), ProcessSyncRequests in the + * checkpointer (opportunistic, wrapped by DWBSegmentFsyncBegin/End), and + * writers stuck on a full ring helping themselves (DWBRetireAllSync). + * All of them share this accounting; duplicate fsyncs are wasted work at + * worst, never a correctness problem. + * + * With dwb_retire_sync_method = syncfs the worker pool and the self-help + * skip the per-segment protocol entirely: one syncfs() round makes every + * file system holding data files durable and frees all batches that were + * RETIRING when the round began (DWBRetireRoundSyncfs). The checkpointer + * piggyback keeps using the per-segment accounting in both modes. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/dwb/dwb_retire.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "common/hashfn.h" +#include "common/int.h" +#include "common/relpath.h" +#include "miscadmin.h" +#include "port/pg_bitutils.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/dwb.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/md.h" +#include "storage/sync.h" +#include "utils/guc.h" +#include "utils/injection_point.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +static void DWBMaybeRemoveSegEntry(DWSegEntry *entry); +static int DWBRetireSweep(int worker_id); +static int DWBRetireRoundSyncfs(void); + +/* + * Snapshot of one segment's back-references, taken before an fsync of that + * segment and consumed after it. One at a time per process: a process + * fsyncs one segment at a time on every path that uses this. + */ +typedef struct DWBSegSyncSnap +{ + bool active; + DWSegRef seg; + int npairs; + struct + { + int batch_idx; + uint64 batch_id; + } pairs[DWB_NUM_BATCHES_MAX]; +} DWBSegSyncSnap; + +static DWBSegSyncSnap seg_sync_snap; + +static DWSegRef +DWBSegRefFromFileTag(const FileTag *ftag) +{ + DWSegRef seg; + + memset(&seg, 0, sizeof(seg)); /* keyed by memcmp: no padding garbage */ + seg.rlocator = ftag->rlocator; + seg.forknum = (ForkNumber) ftag->forknum; + seg.segno = (uint32) ftag->segno; + return seg; +} + +static FileTag +DWBFileTagFromSegRef(const DWSegRef *seg) +{ + FileTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.handler = SYNC_HANDLER_MD; + tag.forknum = (int16) seg->forknum; + tag.rlocator = seg->rlocator; + tag.segno = seg->segno; + return tag; +} + +/* ---------------------------------------------------------------- + * publication + * ---------------------------------------------------------------- + */ + +/* + * Free a batch and wake one would-be opener per writer class. The caller + * has already moved the state to DWB_FREE. + */ +static void +DWBNoteBatchFreed(void) +{ + pg_atomic_fetch_add_u64(&DWBCtl->freed_events, 1); + DWBWakeRingWaiters(); +} + +/* + * Fsync one segment for retirement purposes, tolerating a concurrently + * dropped relation: the data-file writes of a dropped segment are moot, so + * ENOENT counts as covered. Returns true if the segment is covered (fsynced + * or dropped). + * + * A real fsync failure follows the vanilla data_sync_retry policy: PANIC by + * default, but with data_sync_retry = on the kernel is trusted to keep the + * dirty pages, so this must NOT throw -- the callers hold accounting state + * (the advisory fsync claim, the OOM batch state) that a longjmp would leak + * forever. Instead it WARNs and returns false; the segment's back-reference + * bits stay set and a later fsyncer retries. force_panic is for the one + * caller that has no later fsyncer to fall back on (DWBRetireBatchSyncOOM). + */ +static bool +DWBRetireSyncSegment(const DWSegRef *seg, bool force_panic) +{ + FileTag tag = DWBFileTagFromSegRef(seg); + char path[MAXPGPATH]; + + if (mdsyncfiletag(&tag, path) < 0) + { + if (errno == ENOENT) + { + elog(DEBUG1, "DWB: segment \"%s\" dropped during retire, skipping fsync", + path); + return true; + } + ereport(force_panic ? PANIC : data_sync_elevel(WARNING), + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", path))); + return false; + } + return true; +} + +/* + * Synchronous retire of a batch whose seg_set could not be published into a + * full DWSegmentHash: the publisher itself fsyncs every segment and frees + * the batch, so an undersized hash degrades throughput instead of wedging + * the ring. Runs with no locks held; the DWB_OOM_RETIRING state keeps + * everyone else away from the batch. + * + * An fsync failure here is a PANIC even under data_sync_retry = on: nothing + * ever revisits a DWB_OOM_RETIRING batch (retire sweeps only collect + * DWB_RETIRING, and the hash bits were rolled back), so a soft failure would + * leak the batch until restart -- and this can run inside a ResourceOwner + * release callback, which must not fail (resowner.h). + */ +static void +DWBRetireBatchSyncOOM(DWBatchCtl *batch) +{ + uint32 expected; + + for (uint32 i = 0; i < batch->n_segs; i++) + (void) DWBRetireSyncSegment(&batch->seg_set[i], true); + + expected = DWB_OOM_RETIRING; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, DWB_FREE)) + elog(PANIC, "DWB batch freed in unexpected state %u", expected); + DWBNoteBatchFreed(); + + ereport(WARNING, + (errmsg("double write buffer segment hash is full"), + errhint("Consider increasing \"dwb_max_segments\"."))); +} + +/* + * Publish a batch's seg_set into DWSegmentHash and hand the batch over to + * retirement (DATA_WRITTEN -> RETIRING). Called by whoever drops the last + * ref (see DWBFinishBatchData). + */ +void +DWBPublishBatchSegSet(int batch_idx) +{ + DWBatchCtl *batch = &DWBCtl->batches[batch_idx]; + uint32 published = 0; + bool oom = false; + uint32 expected; + + /* + * A sealed non-empty batch always has at least one segment (every slot + * reservation dedup-inserts its segment). Publishing an empty seg_set + * would move the batch to DWB_RETIRING with nothing to ever decrement it + * to FREE. + */ + Assert(batch->n_segs > 0); + + LWLockAcquire(&batch->publish_lock, LW_EXCLUSIVE); + + pg_atomic_write_u32(&batch->seg_pending_count, batch->n_segs); + pg_write_barrier(); + + LWLockAcquire(DWBSegHashLock, LW_EXCLUSIVE); + for (published = 0; published < batch->n_segs; published++) + { + bool found; + DWSegEntry *entry; + + entry = (DWSegEntry *) hash_search(DWSegmentHash, + &batch->seg_set[published], + HASH_ENTER_NULL, &found); + if (entry == NULL) + { + oom = true; + break; + } + if (!found) + { + pg_atomic_init_u32(&entry->fsync_in_progress, 0); + for (uint32 w = 0; w < DWBSegBitmapWords(); w++) + pg_atomic_init_u64(&entry->batch_bitmap[w], 0); + } + pg_atomic_fetch_or_u64(&entry->batch_bitmap[batch_idx / 64], + UINT64CONST(1) << (batch_idx % 64)); + } + + if (oom) + { + /* + * Take the partial publication back. Nobody saw those bits: they + * were set and are removed under one continuous exclusive hold of + * DWBSegHashLock. + */ + for (uint32 i = 0; i < published; i++) + { + bool found; + DWSegEntry *entry; + + entry = (DWSegEntry *) hash_search(DWSegmentHash, + &batch->seg_set[i], + HASH_FIND, &found); + if (entry == NULL) + continue; + pg_atomic_fetch_and_u64(&entry->batch_bitmap[batch_idx / 64], + ~(UINT64CONST(1) << (batch_idx % 64))); + DWBMaybeRemoveSegEntry(entry); + } + pg_atomic_write_u32(&batch->seg_pending_count, 0); + + expected = DWB_DATA_WRITTEN; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, + DWB_OOM_RETIRING)) + elog(PANIC, "DWB batch OOM-retiring in unexpected state %u", + expected); + } + LWLockRelease(DWBSegHashLock); + + if (!oom) + { + pg_write_barrier(); + expected = DWB_DATA_WRITTEN; + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, + DWB_RETIRING)) + elog(PANIC, "DWB batch retiring in unexpected state %u", expected); + } + LWLockRelease(&batch->publish_lock); + + if (oom) + DWBRetireBatchSyncOOM(batch); + else + ConditionVariableBroadcast(&DWBCtl->cv_retire_wake); +} + +/* ---------------------------------------------------------------- + * decrement + * ---------------------------------------------------------------- + */ + +/* + * Remove a segment entry once its bitmap is empty. Caller holds + * DWBSegHashLock exclusive. Safe regardless of fsync_in_progress: nobody + * keeps entry pointers across the lock, claim holders re-look-up by key. + */ +static void +DWBMaybeRemoveSegEntry(DWSegEntry *entry) +{ + for (uint32 w = 0; w < DWBSegBitmapWords(); w++) + if (pg_atomic_read_u64(&entry->batch_bitmap[w]) != 0) + return; + if (hash_search(DWSegmentHash, &entry->key, HASH_REMOVE, NULL) == NULL) + elog(PANIC, "DWB segment hash entry vanished under exclusive lock"); +} + +/* + * Snapshot the back-references of one segment before fsyncing it. + */ +static void +DWBSegSnapBegin(const DWSegRef *seg) +{ + DWSegEntry *entry; + + /* + * overwriting a leftover snapshot (an fsync that errored out between + * Begin and End) is a correct drop: its bits were never cleared and a + * later fsyncer covers them; see also DWBSegmentFsyncBegin + */ + seg_sync_snap.active = true; + seg_sync_snap.seg = *seg; + seg_sync_snap.npairs = 0; + + LWLockAcquire(DWBSegHashLock, LW_SHARED); + entry = (DWSegEntry *) hash_search(DWSegmentHash, seg, HASH_FIND, NULL); + if (entry != NULL) + { + for (uint32 w = 0; w < DWBSegBitmapWords(); w++) + { + uint64 word = pg_atomic_read_u64(&entry->batch_bitmap[w]); + + while (word != 0) + { + int bit = pg_rightmost_one_pos64(word); + int idx = (int) (w * 64) + bit; + + word &= word - 1; + Assert(seg_sync_snap.npairs < (int) lengthof(seg_sync_snap.pairs)); + seg_sync_snap.pairs[seg_sync_snap.npairs].batch_idx = idx; + + /* + * Racy read of a 64-bit batch_id outside the publish_lock: a + * torn or stale value only makes the guarded re-check below + * skip the decrement, never decrement a wrong batch. + */ + seg_sync_snap.pairs[seg_sync_snap.npairs].batch_id = + DWBCtl->batches[idx].batch_id; + seg_sync_snap.npairs++; + } + } + } + LWLockRelease(DWBSegHashLock); +} + +/* + * Consume the snapshot after the fsync. If synced is false (the fsync did + * not happen and the segment still exists), the snapshot is discarded and + * the bits stay for a later fsyncer. Returns the number of batches this + * call moved RETIRING -> FREE. + */ +static int +DWBSegSnapEnd(bool synced) +{ + int freed = 0; + + Assert(seg_sync_snap.active); + seg_sync_snap.active = false; + + if (!synced) + return 0; + + for (int i = 0; i < seg_sync_snap.npairs; i++) + { + int idx = seg_sync_snap.pairs[i].batch_idx; + DWBatchCtl *batch = &DWBCtl->batches[idx]; + bool cleared = false; + + LWLockAcquire(&batch->publish_lock, LW_EXCLUSIVE); + if (batch->batch_id == seg_sync_snap.pairs[i].batch_id) + { + DWSegEntry *entry; + uint64 bit = UINT64CONST(1) << (idx % 64); + + LWLockAcquire(DWBSegHashLock, LW_EXCLUSIVE); + entry = (DWSegEntry *) hash_search(DWSegmentHash, + &seg_sync_snap.seg, + HASH_FIND, NULL); + if (entry != NULL) + { + uint64 prev; + + prev = pg_atomic_fetch_and_u64(&entry->batch_bitmap[idx / 64], + ~bit); + if (prev & bit) + { + cleared = true; + DWBMaybeRemoveSegEntry(entry); + } + } + LWLockRelease(DWBSegHashLock); + + if (cleared && + pg_atomic_fetch_sub_u32(&batch->seg_pending_count, 1) == 1) + { + uint32 expected = DWB_RETIRING; + + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, + DWB_FREE)) + elog(PANIC, "DWB batch freed in unexpected state %u", + expected); + freed++; + } + } + LWLockRelease(&batch->publish_lock); + } + + if (freed > 0) + DWBNoteBatchFreed(); + + return freed; +} + +/* + * Wrap an external fsync of a relation segment (ProcessSyncRequests in the + * checkpointer). Begin before the fsync attempt; End after it, with + * synced = true if the segment was fsynced OR turned out to be dropped + * (a dropped segment's writes are moot). Non-md tags and disabled DWB are + * handled here so the caller stays a two-liner. + */ +void +DWBSegmentFsyncBegin(const FileTag *ftag) +{ + DWSegRef seg; + + /* + * Drop any leftover snapshot BEFORE deciding whether to take a new one. + * If a previous fsync ERROR'ed out between Begin and End (possible in the + * checkpointer with data_sync_retry = on, which survives the ERROR and + * keeps this process-local state), the early return below would otherwise + * leave the stale snapshot armed, and the End of the next successful + * fsync of an unrelated non-MD tag would decrement the stale segment's + * back-references -- freeing batches whose data-file fsync never + * succeeded. + */ + seg_sync_snap.active = false; + + if (!DWBIsEnabled() || ftag->handler != SYNC_HANDLER_MD) + return; + + seg = DWBSegRefFromFileTag(ftag); + DWBSegSnapBegin(&seg); +} + +int +DWBSegmentFsyncEnd(bool synced) +{ + if (!seg_sync_snap.active) + return 0; + return DWBSegSnapEnd(synced); +} + +/* ---------------------------------------------------------------- + * proactive retire + * ---------------------------------------------------------------- + */ + +/* + * Copy the seg_set of a batch if it is still the expected RETIRING + * incarnation. Returns the number of segments, 0 if the batch moved on. + */ +static uint32 +DWBCollectBatchSegs(int batch_idx, uint64 batch_id, DWSegRef *segs) +{ + DWBatchCtl *batch = &DWBCtl->batches[batch_idx]; + uint32 n = 0; + + LWLockAcquire(&batch->publish_lock, LW_SHARED); + if (batch->batch_id == batch_id && + pg_atomic_read_u32(&batch->state) == DWB_RETIRING) + { + n = batch->n_segs; + memcpy(segs, batch->seg_set, n * sizeof(DWSegRef)); + } + LWLockRelease(&batch->publish_lock); + return n; +} + +/* + * Fsync one segment of a RETIRING batch and decrement its back-references. + * Skips the segment when another fsyncer holds the claim (they will cover + * it) or when its bits are already gone. Returns batches freed. + */ +static int +DWBRetireSegment(const DWSegRef *seg) +{ + DWSegEntry *entry; + bool claimed = false; + bool covered; + uint32 zero = 0; + int freed; + + /* claim the segment; a busy or vanished entry means nothing to do */ + LWLockAcquire(DWBSegHashLock, LW_SHARED); + entry = (DWSegEntry *) hash_search(DWSegmentHash, seg, HASH_FIND, NULL); + if (entry != NULL) + claimed = pg_atomic_compare_exchange_u32(&entry->fsync_in_progress, + &zero, 1); + LWLockRelease(DWBSegHashLock); + if (!claimed) + return 0; + + /* + * DWBRetireSyncSegment does not throw on a soft (data_sync_retry = on) + * fsync failure, so the claim reset below always runs; on covered = false + * the snapshot is discarded and the bits stay for a retry. + */ + DWBSegSnapBegin(seg); + covered = DWBRetireSyncSegment(seg, false); + freed = DWBSegSnapEnd(covered); + + /* + * Release the claim. The entry may have been removed (and even + * re-created for a new batch) meanwhile; re-look-up by key and reset + * whatever is there -- the flag is advisory, an over-reset only costs a + * duplicate fsync. + */ + LWLockAcquire(DWBSegHashLock, LW_SHARED); + entry = (DWSegEntry *) hash_search(DWSegmentHash, seg, HASH_FIND, NULL); + if (entry != NULL) + pg_atomic_write_u32(&entry->fsync_in_progress, 0); + LWLockRelease(DWBSegHashLock); + + return freed; +} + +/* + * Retire everything that can be retired right now. Called by the self-help + * of a writer stuck on a full ring (DWBOpenNewBatch) and by the synchronous + * retire in DWBFinishPageWrite when there is no worker pool + * (dwb_retire_workers = 0, single-user mode). Returns batches freed. + */ +int +DWBRetireAllSync(void) +{ + if (dwb_retire_sync_method == DATA_DIR_SYNC_METHOD_SYNCFS) + return DWBRetireRoundSyncfs(); + return DWBRetireSweep(-1); +} + +typedef struct DWBRetiringBatch +{ + int idx; + uint64 id; +} DWBRetiringBatch; + +static int +dwb_retiring_batch_cmp(const void *a, const void *b) +{ + return pg_cmp_u64(((const DWBRetiringBatch *) a)->id, + ((const DWBRetiringBatch *) b)->id); +} + +/* + * One per-segment retire sweep over all RETIRING batches, oldest first. + * worker_id >= 0 restricts the sweep to that worker's segment partition; + * -1 sweeps everything. Returns batches freed. + */ +static int +DWBRetireSweep(int worker_id) +{ + DWBRetiringBatch *retiring; + int nretiring = 0; + int freed = 0; + DWSegRef *segs; + + retiring = palloc(dwb_num_batches * sizeof(DWBRetiringBatch)); + segs = palloc(dwb_batch_pages * sizeof(DWSegRef)); + + for (int i = 0; i < dwb_num_batches; i++) + { + if (pg_atomic_read_u32(&DWBCtl->batches[i].state) == DWB_RETIRING) + { + retiring[nretiring].idx = i; + retiring[nretiring].id = DWBCtl->batches[i].batch_id; + nretiring++; + } + } + + /* oldest first: smaller batch_id was opened earlier */ + qsort(retiring, nretiring, sizeof(DWBRetiringBatch), + dwb_retiring_batch_cmp); + + for (int i = 0; i < nretiring; i++) + { + uint32 nsegs = DWBCollectBatchSegs(retiring[i].idx, + retiring[i].id, segs); + + for (uint32 s = 0; s < nsegs; s++) + { + if (worker_id >= 0 && + (int) (hash_bytes((const unsigned char *) &segs[s], + sizeof(DWSegRef)) % + (uint32) dwb_retire_workers) != worker_id) + continue; + freed += DWBRetireSegment(&segs[s]); + } + } + + pfree(retiring); + pfree(segs); + return freed; +} + +#ifdef HAVE_SYNCFS +/* + * syncfs() one directory's file system. Follows the vanilla data_sync_retry + * policy of DWBRetireSyncSegment: a failure PANICs by default, or WARNs and + * returns false under data_sync_retry = on so the caller retries the round + * later. (The usual retry caveat applies doubly here: a second syncfs may + * report success after the kernel already dropped the dirty pages the first + * failure was about.) missing_ok tolerates a dangling pg_tblspc entry: a + * vanished tablespace took its data files with it, so their writes are as + * moot as a dropped segment's on the per-segment path. + */ +static bool +DWBSyncfsPath(const char *path, bool missing_ok) +{ + int fd; + + fd = OpenTransientFile(path, O_RDONLY); + if (fd < 0) + { + if (missing_ok && errno == ENOENT) + return true; + ereport(data_sync_elevel(WARNING), + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + return false; + } + pgstat_report_wait_start(WAIT_EVENT_DWB_SYNCFS); + if (syncfs(fd) < 0) + { + pgstat_report_wait_end(); + CloseTransientFile(fd); + ereport(data_sync_elevel(WARNING), + (errcode_for_file_access(), + errmsg("could not synchronize file system for file \"%s\": %m", + path))); + return false; + } + pgstat_report_wait_end(); + CloseTransientFile(fd); + return true; +} +#endif /* HAVE_SYNCFS */ + +/* + * Make every file system that can hold data files durable: the one under + * the data directory (the process is chdir'd into it) and each tablespace + * mount. Returns true only if every syncfs succeeded — anything less and + * no batch may be freed on its account. An unreadable pg_tblspc raises an + * ERROR (never a wrong free): the worker restarts, a self-helping writer + * aborts its statement. + */ +static bool +DWBSyncfsAllFilesystems(void) +{ +#ifdef HAVE_SYNCFS + DIR *dir; + struct dirent *de; + bool ok = true; + + if (!enableFsync) + return true; + + if (!DWBSyncfsPath(".", false)) + ok = false; + + dir = AllocateDir(PG_TBLSPC_DIR); + while ((de = ReadDir(dir, PG_TBLSPC_DIR)) != NULL) + { + char path[MAXPGPATH]; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + snprintf(path, MAXPGPATH, "%s/%s", PG_TBLSPC_DIR, de->d_name); + if (!DWBSyncfsPath(path, true)) + ok = false; + } + FreeDir(dir); + return ok; +#else + /* the GUC cannot be set to syncfs without HAVE_SYNCFS */ + elog(PANIC, "syncfs is not supported on this platform"); + return false; /* keep the compiler happy */ +#endif +} + +/* + * One wholesale retire round: syncfs the file systems and free every batch + * that was already RETIRING when the round began. Returns batches freed. + * + * Correctness of the wholesale free: a batch observed RETIRING completed + * ALL its data-file writes before the last ref drop published it (write + * path steps 6-7), so those writes were submitted before syncfs() started + * and are durable when it returns. A batch that reaches RETIRING while + * the syncfs runs is not on the list and waits for the next round. + * + * The per-segment accounting stays consistent with the concurrent + * checkpointer piggyback: each freed batch's bits are cleared under the + * same publish_lock + DWBSegHashLock the decrement path takes, and the + * (batch_idx, batch_id) snapshot re-check under publish_lock is the same + * ABA guard the snapshot protocol uses (batch_id is read racily here, like + * in DWBRetireSweep's collect; a torn read only makes the re-check skip). + */ +static int +DWBRetireRoundSyncfs(void) +{ + DWBRetiringBatch *retiring; + int nretiring = 0; + int freed = 0; + + /* + * One round at a time: every retire worker wakes on the same broadcast, + * and the ring-full self-help can race the pool, but concurrent rounds + * would only duplicate a whole-file-system syncfs. A loser returns at + * once — the winner's round covers everything that was RETIRING when it + * collected, and batches published after that wake the pool again. The + * collect below runs under the lock, so no process can free a batch some + * other round's syncfs did not cover. An ERROR inside the round (e.g. an + * unreadable pg_tblspc) releases the gate in the unwind. + */ + if (!LWLockConditionalAcquire(DWBSyncfsRoundLock, LW_EXCLUSIVE)) + return 0; + + /* + * Test hook: proves the gate admits one process at a time. Sits before + * the collect, so a batch published while a test holds a round parked + * here is still picked up once the round resumes. NB: parking here + * freezes ALL wholesale retirement, so a test must not generate ring + * traffic while the point is armed. + */ + INJECTION_POINT("dwb-syncfs-round", NULL); + + retiring = palloc(dwb_num_batches * sizeof(DWBRetiringBatch)); + + for (int i = 0; i < dwb_num_batches; i++) + { + if (pg_atomic_read_u32(&DWBCtl->batches[i].state) == DWB_RETIRING) + { + retiring[nretiring].idx = i; + retiring[nretiring].id = DWBCtl->batches[i].batch_id; + nretiring++; + } + } + + if (nretiring == 0 || !DWBSyncfsAllFilesystems()) + { + LWLockRelease(DWBSyncfsRoundLock); + pfree(retiring); + return 0; + } + + for (int i = 0; i < nretiring; i++) + { + int idx = retiring[i].idx; + DWBatchCtl *batch = &DWBCtl->batches[idx]; + + LWLockAcquire(&batch->publish_lock, LW_EXCLUSIVE); + if (batch->batch_id == retiring[i].id && + pg_atomic_read_u32(&batch->state) == DWB_RETIRING) + { + uint32 expected = DWB_RETIRING; + uint64 bit = UINT64CONST(1) << (idx % 64); + + LWLockAcquire(DWBSegHashLock, LW_EXCLUSIVE); + for (uint32 s = 0; s < batch->n_segs; s++) + { + DWSegEntry *entry; + + entry = (DWSegEntry *) hash_search(DWSegmentHash, + &batch->seg_set[s], + HASH_FIND, NULL); + if (entry != NULL) + { + uint64 prev; + + prev = pg_atomic_fetch_and_u64(&entry->batch_bitmap[idx / 64], + ~bit); + if (prev & bit) + DWBMaybeRemoveSegEntry(entry); + } + } + LWLockRelease(DWBSegHashLock); + + pg_atomic_write_u32(&batch->seg_pending_count, 0); + if (!pg_atomic_compare_exchange_u32(&batch->state, &expected, + DWB_FREE)) + elog(PANIC, "DWB batch freed in unexpected state %u", expected); + DWBNoteBatchFreed(); + freed++; + } + LWLockRelease(&batch->publish_lock); + } + + LWLockRelease(DWBSyncfsRoundLock); + pfree(retiring); + return freed; +} + +/* ---------------------------------------------------------------- + * retire worker pool + * ---------------------------------------------------------------- + */ + +/* + * Register dwb_retire_workers static background workers. Called from + * PostmasterMain before extensions get a chance at the worker slots. + */ +void +DWBRetireWorkersRegister(void) +{ + BackgroundWorker bgw; + int free_slots; + + if (!DWBIsEnabled() || dwb_retire_workers == 0) + return; + + /* + * RegisterBackgroundWorker only LOGs on overflow, so check the slots + * actually left after the earlier internal registrations (the logical + * replication launcher above all) and fail loudly: a silently missing + * retire worker would ship a smaller pool than the operator configured. + */ + free_slots = max_worker_processes - GetNumRegisteredBackgroundWorkers(); + if (dwb_retire_workers > free_slots) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"dwb_retire_workers\" (%d) needs more \"max_worker_processes\" slots than remain free (%d)", + dwb_retire_workers, free_slots), + errhint("Increase \"max_worker_processes\" or decrease \"dwb_retire_workers\"."))); + + for (int i = 0; i < dwb_retire_workers; i++) + { + memset(&bgw, 0, sizeof(bgw)); + + /* + * The database-less connection gives the worker a pg_stat_activity + * entry; it forces BgWorkerStart_ConsistentState, so during the + * pre-consistency part of recovery the write path relies on its + * built-in self service (waiters seal on timeout, ring-full writers + * retire inline) — the pool is throughput, not correctness. + */ + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_ConsistentState; + snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "DWBRetireWorkerMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, "dwb retire worker %d", i); + snprintf(bgw.bgw_type, BGW_MAXLEN, "dwb retire worker"); + bgw.bgw_restart_time = 1; + bgw.bgw_notify_pid = 0; + bgw.bgw_main_arg = Int32GetDatum(i); + + RegisterBackgroundWorker(&bgw); + } +} + +/* + * Main loop: force-SEAL non-empty batches that outlived + * dwb_batch_timeout_ms, then proactively fsync this worker's segment + * partition of every RETIRING batch. Woken by cv_retire_wake (publication + * of a seg_set, opening of a batch) or by timeout. + */ +void +DWBRetireWorkerMain(Datum main_arg) +{ + int my_id = DatumGetInt32(main_arg); + + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGTERM, SignalHandlerForShutdownRequest); + BackgroundWorkerUnblockSignals(); + + /* no database, just shared state and pg_stat_activity visibility */ + BackgroundWorkerInitializeConnection(NULL, NULL, 0); + + ConditionVariablePrepareToSleep(&DWBCtl->cv_retire_wake); + + for (;;) + { + long timeout; + TimestampTz now; + + if (ShutdownRequestPending) + break; + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* + * Force-SEAL pass: writers waiting on a half-filled batch seal it + * themselves after the same timeout, so this only matters for batches + * whose writers all went away before sealing. open_time is read + * unlocked; a torn read can only mis-time the seal, which is always a + * valid action on a non-empty ALLOCATED batch. + */ + now = GetCurrentTimestamp(); + timeout = dwb_retire_interval_ms; + for (int i = 0; i < dwb_num_batches; i++) + { + DWBatchCtl *batch = &DWBCtl->batches[i]; + long age_ms; + + if (pg_atomic_read_u32(&batch->state) != DWB_ALLOCATED) + continue; + if ((pg_atomic_read_u32(&batch->next_slot_idx) & DWB_IDX_MASK) == 0) + continue; /* empty: sealing it buys nothing */ + + age_ms = TimestampDifferenceMilliseconds(batch->open_time, now); + if (age_ms >= dwb_batch_timeout_ms) + (void) DWBTrySealBatch(i, DWB_SEAL_WORKER_TIMEOUT); + else if (dwb_batch_timeout_ms - age_ms < timeout) + timeout = dwb_batch_timeout_ms - age_ms; + } + + if (dwb_retire_sync_method == DATA_DIR_SYNC_METHOD_SYNCFS) + (void) DWBRetireRoundSyncfs(); + else + (void) DWBRetireSweep(my_id); + + (void) ConditionVariableTimedSleep(&DWBCtl->cv_retire_wake, + Max(timeout, 1), + WAIT_EVENT_DWB_RETIRE_MAIN); + } + + ConditionVariableCancelSleep(); + proc_exit(0); +} diff --git a/src/backend/storage/dwb/meson.build b/src/backend/storage/dwb/meson.build new file mode 100644 index 0000000000000..d59954d5ce75a --- /dev/null +++ b/src/backend/storage/dwb/meson.build @@ -0,0 +1,10 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +backend_sources += files( + 'dwb.c', + 'dwb_cleaner.c', + 'dwb_ctl.c', + 'dwb_file.c', + 'dwb_recovery.c', + 'dwb_retire.c', +) diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 8d2b0f1193d9f..7a8fef25d5031 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -39,6 +39,8 @@ #include "replication/walsender.h" #include "storage/aio_subsys.h" #include "storage/bufmgr.h" +#include "access/xlogwarm.h" +#include "storage/dwb.h" #include "storage/dsm.h" #include "storage/dsm_registry.h" #include "storage/ipc.h" @@ -114,6 +116,8 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, dsm_estimate_size()); size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferManagerShmemSize()); + size = add_size(size, DWBShmemSize()); + size = add_size(size, XLogWarmShmemSize()); size = add_size(size, LockManagerShmemSize()); size = add_size(size, PredicateLockShmemSize()); size = add_size(size, ProcGlobalShmemSize()); @@ -293,6 +297,8 @@ CreateOrAttachShmemStructs(void) SUBTRANSShmemInit(); MultiXactShmemInit(); BufferManagerShmemInit(); + DWBShmemInit(); + XLogWarmShmemInit(); /* * Set up lock manager diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index c3d4b7275ecc1..1c53553aa2508 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -178,6 +178,7 @@ static const char *const BuiltinTrancheNames[] = { [LWTRANCHE_XACT_SLRU] = "XactSLRU", [LWTRANCHE_PARALLEL_VACUUM_DSA] = "ParallelVacuumDSA", [LWTRANCHE_AIO_URING_COMPLETION] = "AioUringCompletion", + [LWTRANCHE_DWB_PUBLISH] = "DWBPublish", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/meson.build b/src/backend/storage/meson.build index 0cd48844f1d9b..52ce2bbdc4548 100644 --- a/src/backend/storage/meson.build +++ b/src/backend/storage/meson.build @@ -2,6 +2,7 @@ subdir('aio') subdir('buffer') +subdir('dwb') subdir('file') subdir('freespace') subdir('ipc') diff --git a/src/backend/storage/smgr/smgr.c b/src/backend/storage/smgr/smgr.c index 37b99fa319852..017eb2cf8f858 100644 --- a/src/backend/storage/smgr/smgr.c +++ b/src/backend/storage/smgr/smgr.c @@ -64,6 +64,7 @@ #include "postgres.h" #include "access/xlogutils.h" +#include "access/xlogwarm.h" #include "lib/ilist.h" #include "miscadmin.h" #include "storage/aio.h" @@ -551,6 +552,14 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo) */ HOLD_INTERRUPTS(); + /* + * Keep the replay warm pool out of these relations until the files are + * gone: a worker reading one of their pages right now would otherwise + * leave that page in the buffer pool, which is exactly what + * DropRelationsAllBuffers() must not have happen behind it. + */ + XLogWarmDropBegin(); + /* * Get rid of any remaining buffers for the relations. bufmgr will just * drop them without bothering to write the contents. @@ -603,6 +612,8 @@ smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo) pfree(rlocators); + XLogWarmDropEnd(); + RESUME_INTERRUPTS(); } @@ -877,6 +888,13 @@ smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, { int i; + /* + * As in smgrdounlinkall(): a warm-pool worker must not be able to load a + * page of this relation into buffers between the drop below and the + * truncation that follows it. + */ + XLogWarmDropBegin(); + /* * Get rid of any buffers for the about-to-be-deleted blocks. bufmgr will * just drop them without bothering to write the contents. @@ -922,6 +940,8 @@ smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, reln->smgr_cached_nblocks[forknum[i]] = nblocks[i] > old_nblocks[i] ? old_nblocks[i] : nblocks[i]; } + + XLogWarmDropEnd(); } /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c index fc16db90133bb..3c374fdb15dde 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -26,6 +26,7 @@ #include "pgstat.h" #include "portability/instr_time.h" #include "postmaster/bgwriter.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/latch.h" #include "storage/md.h" @@ -407,6 +408,16 @@ ProcessSyncRequests(void) * DROP DATABASE likewise has to tell us to forget fsync requests * before it starts deletions. */ + + /* + * This fsync also retires double write buffer batches: snapshot + * the segment's DWB back-references now — bits published while + * the fsync runs may cover writes it missed — and decrement + * them once the segment is durable (or turns out dropped, which + * makes its data-file writes moot). + */ + DWBSegmentFsyncBegin(&entry->tag); + for (failures = 0; !entry->canceled; failures++) { char path[MAXPGPATH]; @@ -458,6 +469,9 @@ ProcessSyncRequests(void) AbsorbSyncRequests(); absorb_counter = FSYNCS_PER_ABSORB; /* might as well... */ } /* end retry loop */ + + /* durable or dropped either way: retire DWB references */ + (void) DWBSegmentFsyncEnd(true); } /* We are done with this entry, remove it */ diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 13ae57ed6498d..98524ee17f284 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -129,14 +129,20 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_SET_CURRENT(io_time); INSTR_TIME_SUBTRACT(io_time, start_time); - if (io_object != IOOBJECT_WAL) + /* + * pg_stat_database's blk_read_time/blk_write_time count data-block IO + * only: relation and temp-relation objects. WAL and double write + * buffer IO have their own accounting, and counting the DWB copy of a + * page here would double the apparent block write time. + */ + if (io_object == IOOBJECT_RELATION || io_object == IOOBJECT_TEMP_RELATION) { if (io_op == IOOP_WRITE || io_op == IOOP_EXTEND) { pgstat_count_buffer_write_time(INSTR_TIME_GET_MICROSEC(io_time)); if (io_object == IOOBJECT_RELATION) INSTR_TIME_ADD(pgBufferUsage.shared_blk_write_time, io_time); - else if (io_object == IOOBJECT_TEMP_RELATION) + else INSTR_TIME_ADD(pgBufferUsage.local_blk_write_time, io_time); } else if (io_op == IOOP_READ) @@ -144,7 +150,7 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, pgstat_count_buffer_read_time(INSTR_TIME_GET_MICROSEC(io_time)); if (io_object == IOOBJECT_RELATION) INSTR_TIME_ADD(pgBufferUsage.shared_blk_read_time, io_time); - else if (io_object == IOOBJECT_TEMP_RELATION) + else INSTR_TIME_ADD(pgBufferUsage.local_blk_read_time, io_time); } } @@ -268,6 +274,8 @@ pgstat_get_io_object_name(IOObject io_object) return "temp relation"; case IOOBJECT_WAL: return "wal"; + case IOOBJECT_DWB: + return "dwb"; } elog(ERROR, "unrecognized IOObject value: %d", io_object); @@ -418,6 +426,12 @@ pgstat_tracks_io_object(BackendType bktype, IOObject io_object, io_object == IOOBJECT_TEMP_RELATION) return false; + /* + * IO on the double write buffer ring only occurs in IOCONTEXT_NORMAL. + */ + if (io_object == IOOBJECT_DWB && io_context != IOCONTEXT_NORMAL) + return false; + /* * In core Postgres, only regular backends and WAL Sender processes * executing queries will use local buffers and operate on temporary @@ -516,6 +530,13 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, (io_op == IOOP_FSYNC || io_op == IOOP_WRITEBACK)) return false; + /* + * The double write buffer ring only sees batch writes and fdatasyncs. + */ + if (io_object == IOOBJECT_DWB && + !(io_op == IOOP_WRITE || io_op == IOOP_FSYNC)) + return false; + /* * Some IOOps are not valid in certain IOContexts and some IOOps are only * valid in certain contexts. diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index b9c1e6900ec1b..2d0b75d0964e2 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -57,11 +57,14 @@ BGWRITER_HIBERNATE "Waiting in background writer process, hibernating." BGWRITER_MAIN "Waiting in main loop of background writer process." CHECKPOINTER_MAIN "Waiting in main loop of checkpointer process." CHECKPOINTER_SHUTDOWN "Waiting for checkpointer process to be terminated." +DWB_CLEANER_MAIN "Waiting in main loop of a double write buffer cleaner worker." +DWB_RETIRE_MAIN "Waiting in main loop of a double write buffer retire worker." IO_WORKER_MAIN "Waiting in main loop of IO Worker process." LOGICAL_APPLY_MAIN "Waiting in main loop of logical replication apply process." LOGICAL_LAUNCHER_MAIN "Waiting in main loop of logical replication launcher process." LOGICAL_PARALLEL_APPLY_MAIN "Waiting in main loop of logical replication parallel apply process." RECOVERY_WAL_STREAM "Waiting in main loop of startup process for WAL to arrive, during streaming recovery." +REPLAY_WARM_MAIN "Waiting in main loop of a replay warm worker." REPLICATION_SLOTSYNC_MAIN "Waiting in main loop of slot sync worker." REPLICATION_SLOTSYNC_SHUTDOWN "Waiting for slot sync worker to shut down." SYSLOGGER_MAIN "Waiting in main loop of syslogger process." @@ -116,6 +119,9 @@ CHECKPOINT_DELAY_COMPLETE "Waiting for a backend that blocks a checkpoint from c CHECKPOINT_DELAY_START "Waiting for a backend that blocks a checkpoint from starting." CHECKPOINT_DONE "Waiting for a checkpoint to complete." CHECKPOINT_START "Waiting for a checkpoint to start." +DWB_BATCH_COVERAGE "Waiting for writers of a double write buffer batch to publish their page images." +DWB_BATCH_FSYNC "Waiting for a double write buffer batch to reach durable storage." +DWB_FREE_BATCH "Waiting for a free double write buffer batch." EXECUTE_GATHER "Waiting for activity from a child process while executing a Gather plan node." HASH_BATCH_ALLOCATE "Waiting for an elected Parallel Hash participant to allocate a hash table." HASH_BATCH_ELECT "Waiting to elect a Parallel Hash participant to allocate a hash table." @@ -219,6 +225,14 @@ DATA_FILE_TRUNCATE "Waiting for a relation data file to be truncated." DATA_FILE_WRITE "Waiting for a write to a relation data file." DSM_ALLOCATE "Waiting for a dynamic shared memory segment to be allocated." DSM_FILL_ZERO_WRITE "Waiting to fill a dynamic shared memory backing file with zeroes." +DWB_BATCH_READ "Waiting for a read from a double write buffer batch file." +DWB_BATCH_SYNC "Waiting for a double write buffer batch file to reach durable storage." +DWB_BATCH_WRITE "Waiting for a write to a double write buffer batch file." +DWB_CONTROL_READ "Waiting for a read of the double write buffer control file." +DWB_CONTROL_SYNC "Waiting for the double write buffer control file to reach durable storage." +DWB_CONTROL_WRITE "Waiting for a write to the double write buffer control file." +DWB_RING_INIT "Waiting for preallocation of the double write buffer ring files." +DWB_SYNCFS "Waiting for a syncfs() call that retires double write buffer batches." LOCK_FILE_ADDTODATADIR_READ "Waiting for a read while adding a line to the data directory lock file." LOCK_FILE_ADDTODATADIR_SYNC "Waiting for data to reach durable storage while adding a line to the data directory lock file." LOCK_FILE_ADDTODATADIR_WRITE "Waiting for a write while adding a line to the data directory lock file." @@ -352,6 +366,12 @@ DSMRegistry "Waiting to read or update the dynamic shared memory registry." InjectionPoint "Waiting to read or update information related to injection points." SerialControl "Waiting to read or update shared pg_serial state." AioWorkerSubmissionQueue "Waiting to access AIO worker submission queue." +DWBRingOpen "Waiting to open a new double write buffer batch." +DWBSegHash "Waiting to read or update the double write buffer segment hash table." +DWBSelfSweep "Waiting to run the double write buffer self-help retirement sweep." +DWBSyncfsRound "Waiting to run a wholesale double write buffer retirement round." +DWBCleanerQueue "Waiting to access the double write buffer cleaner work queue." +ReplayWarmRead "Waiting for replay prefetch workers to finish reading pages of a relation being dropped or truncated." # # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE) @@ -402,6 +422,7 @@ SubtransSLRU "Waiting to access the sub-transaction SLRU cache." XactSLRU "Waiting to access the transaction status SLRU cache." ParallelVacuumDSA "Waiting for parallel vacuum dynamic shared memory allocation." AioUringCompletion "Waiting for another process to complete IO via io_uring." +DWBPublish "Waiting to publish a double write buffer batch's segment set." # No "ABI_compatibility" region here as WaitEventLWLock has its own C code. diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index c86ceefda940b..5375f30ef360c 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -45,6 +45,7 @@ #include "replication/walsender.h" #include "storage/aio_subsys.h" #include "storage/bufmgr.h" +#include "storage/dwb.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/lmgr.h" @@ -663,6 +664,12 @@ BaseInit(void) * drop ephemeral slots, which in turn triggers stats reporting. */ ReplicationSlotInitialize(); + + /* + * Initialize the double write buffer's process-wide state, so that a + * process holding staged writes when it exits gives them back. + */ + DWBInitBackend(); } diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6b82a23435efe..ffea9f1f6ca7c 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -37,6 +37,7 @@ #include "access/xlog_internal.h" #include "access/xlogprefetcher.h" #include "access/xlogrecovery.h" +#include "access/xlogwarm.h" #include "access/xlogutils.h" #include "archive/archive_module.h" #include "catalog/namespace.h" @@ -78,6 +79,7 @@ #include "replication/syncrep.h" #include "storage/aio.h" #include "storage/bufmgr.h" +#include "storage/dwb.h" #include "storage/bufpage.h" #include "storage/copydir.h" #include "storage/fd.h" @@ -355,6 +357,21 @@ static const struct config_enum_entry synchronous_commit_options[] = { {NULL, 0, false} }; +/* keep the spellings in sync with DWBProtectionModeName() in pg_control.h */ +static const struct config_enum_entry io_torn_pages_protection_options[] = { + {"off", DWB_PROTECT_OFF, false}, + {"full_pages", DWB_PROTECT_FULL_PAGES, false}, + {"double_writes", DWB_PROTECT_DOUBLE_WRITES, false}, + {NULL, 0, false} +}; + +static const struct config_enum_entry dwb_on_stall_options[] = { + {"warn", DWB_ON_STALL_WARN, false}, + {"error", DWB_ON_STALL_ERROR, false}, + {"panic", DWB_ON_STALL_PANIC, false}, + {NULL, 0, false} +}; + /* * Although only "on", "off", "try" are documented, we accept all the likely * variants of "on" and "off". @@ -436,6 +453,7 @@ static const struct config_enum_entry debug_logical_replication_streaming_option StaticAssertDecl(lengthof(ssl_protocol_versions_info) == (PG_TLS1_3_VERSION + 2), "array length mismatch"); +/* shared by recovery_init_sync_method and dwb_retire_sync_method */ static const struct config_enum_entry recovery_init_sync_method_options[] = { {"fsync", DATA_DIR_SYNC_METHOD_FSYNC, false}, #ifdef HAVE_SYNCFS @@ -1204,7 +1222,6 @@ struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, - { {"wal_log_hints", PGC_POSTMASTER, WAL_SETTINGS, gettext_noop("Writes full pages to WAL when first modified after a checkpoint, even for a non-critical modification."), @@ -2172,6 +2189,131 @@ struct config_int ConfigureNamesInt[] = 0, 0, INT_MAX / 2, NULL, NULL, NULL }, + { + {"dwb_num_batches", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Number of batches in the double write buffer ring."), + NULL + }, + &dwb_num_batches, + 64, 16, DWB_NUM_BATCHES_MAX, + NULL, NULL, NULL + }, + { + {"dwb_batch_pages", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Number of pages per double write buffer batch."), + NULL + }, + &dwb_batch_pages, + 64, 16, DWB_BATCH_MAX_PAGES, + NULL, NULL, NULL + }, + { + {"dwb_max_segments", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Capacity of the double write buffer segment hash table."), + NULL + }, + &dwb_max_segments, + 4096, 1024, 1048576, + NULL, NULL, NULL + }, + { + {"dwb_retire_workers", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Number of double write buffer retire worker processes."), + gettext_noop("The workers consume \"max_worker_processes\" slots. " + "0 disables the pool and makes writers retire batches " + "synchronously; meant for testing only.") + }, + &dwb_retire_workers, + 1, 0, 32, + NULL, NULL, NULL + }, + { + {"dwb_cleaner_workers", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Number of double write buffer cleaner worker processes."), + gettext_noop("The pool executes the flush bins the background " + "writer's LRU scan produces, so the scan's issue rate " + "is not capped by one process. The workers consume " + "\"max_worker_processes\" slots. 0 disables the pool " + "and the background writer flushes its bins itself.") + }, + &dwb_cleaner_workers, + 0, 0, 64, + NULL, NULL, NULL + }, + { + {"replay_warm_workers", PGC_POSTMASTER, WAL_RECOVERY, + gettext_noop("Number of replay warm worker processes."), + gettext_noop("The pool reads the pages replay is about to modify " + "into shared buffers ahead of it, which matters when " + "the WAL stream carries no full-page images. The " + "workers consume \"max_worker_processes\" slots. 0 " + "disables the pool and recovery prefetching falls " + "back to advising the operating system.") + }, + &replay_warm_workers, + 0, 0, 64, + NULL, NULL, NULL + }, + { + {"replay_warm_queue_size", PGC_POSTMASTER, WAL_RECOVERY, + gettext_noop("Number of block requests the replay warm pool can hold."), + gettext_noop("This also bounds how far ahead of replay the " + "prefetcher looks when the pool is enabled.") + }, + &replay_warm_queue_size, + 512, 16, 8192, + NULL, NULL, NULL + }, + { + {"dwb_batch_timeout_ms", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Maximum time an open double write buffer batch may wait before being sealed."), + NULL, + GUC_UNIT_MS + }, + &dwb_batch_timeout_ms, + 10, 1, 1000, + NULL, NULL, NULL + }, + { + {"dwb_retire_interval_ms", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Cycle time of each double write buffer retire worker."), + NULL, + GUC_UNIT_MS + }, + &dwb_retire_interval_ms, + 50, 5, 5000, + NULL, NULL, NULL + }, + { + {"dwb_slow_warn_ms", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Double write buffer wait time after which throttling of non-critical writers begins."), + NULL, + GUC_UNIT_MS + }, + &dwb_slow_warn_ms, + 5000, 100, 60000, + NULL, NULL, NULL + }, + { + {"dwb_slot_stuck_timeout_ms", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Time a double write buffer batch leader waits for slot coverage before PANIC."), + NULL, + GUC_UNIT_MS + }, + &dwb_slot_stuck_timeout_ms, + 30000, 1000, 600000, + NULL, NULL, NULL + }, + { + {"dwb_write_timeout_ms", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Double write buffer wait time after which dwb_on_stall applies."), + NULL, + GUC_UNIT_MS + }, + &dwb_write_timeout_ms, + 60000, 1000, 600000, + NULL, NULL, NULL + }, { {"post_auth_delay", PGC_BACKEND, DEVELOPER_OPTIONS, gettext_noop("Sets the amount of time to wait after " @@ -3015,6 +3157,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"dwb_writeback_after", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Number of pages the double write buffer accumulates before starting kernel writeback of them."), + gettext_noop("Lets the sync that retires a batch cost closer to a barrier than to a full flush. " + "0 disables the double write buffer's own writeback; the checkpointer " + "and the background writer keep using their own parameters."), + GUC_UNIT_BLOCKS + }, + &dwb_writeback_after, + DEFAULT_DWB_WRITEBACK_AFTER, 0, WRITEBACK_MAX_PENDING_FLUSHES, + NULL, NULL, NULL + }, + { {"wal_buffers", PGC_POSTMASTER, WAL_SETTINGS, gettext_noop("Sets the number of disk-page buffers in shared memory for WAL."), @@ -5003,6 +5158,38 @@ struct config_string ConfigureNamesString[] = struct config_enum ConfigureNamesEnum[] = { + { + {"io_torn_pages_protection", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Selects the protection against torn (partially written) data pages."), + gettext_noop("\"full_pages\" writes full page images to WAL after a checkpoint, " + "\"double_writes\" uses the double write buffer in pg_dwb, " + "\"off\" disables protection.") + }, + &io_torn_pages_protection, + DWB_PROTECT_FULL_PAGES, io_torn_pages_protection_options, + NULL, NULL, NULL + }, + + { + {"dwb_on_stall", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Action to take when a double write buffer wait exceeds dwb_write_timeout_ms."), + NULL + }, + &dwb_on_stall, + DWB_ON_STALL_PANIC, dwb_on_stall_options, + NULL, NULL, NULL + }, + + { + {"dwb_retire_sync_method", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Selects how double write buffer retirement makes data files durable."), + gettext_noop("\"fsync\" syncs the touched data-file segments one by one; \"syncfs\" syncs their whole file systems per retire round.") + }, + &dwb_retire_sync_method, + DWB_RETIRE_SYNC_METHOD_DEFAULT, recovery_init_sync_method_options, + NULL, NULL, NULL + }, + { {"backslash_quote", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, gettext_noop("Sets whether \"\\'\" is allowed in string literals."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d91133dbd7357..709012b839d75 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -247,6 +247,28 @@ # fsync_writethrough # open_sync #full_page_writes = on # recover from partial page writes +#io_torn_pages_protection = full_pages # off, full_pages, or double_writes + # (change requires restart) +#dwb_num_batches = 64 # batches in the double write ring + # (change requires restart) +#dwb_batch_pages = 64 # pages per batch + # (change requires restart) +#dwb_max_segments = 4096 # segment hash capacity + # (change requires restart) +#dwb_retire_workers = 1 # retire worker processes + # (change requires restart) +#dwb_cleaner_workers = 0 # cleaner worker processes executing the + # background writer's flush bins + # (change requires restart) +#dwb_retire_sync_method = syncfs # syncfs where supported (Linux; the + # default there), fsync elsewhere +#dwb_batch_timeout_ms = 10ms # force-seal an open batch after this time +#dwb_retire_interval_ms = 50ms # retire worker cycle +#dwb_writeback_after = 32 # measured in pages, 0 disables +#dwb_slow_warn_ms = 5s # throttle non-critical writers after this wait +#dwb_slot_stuck_timeout_ms = 30s # PANIC on stuck batch coverage +#dwb_write_timeout_ms = 60s # apply dwb_on_stall after this wait +#dwb_on_stall = panic # panic, error, or warn #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_compression = off # enables compression of full-page writes; @@ -276,6 +298,10 @@ #recovery_prefetch = try # prefetch pages referenced in the WAL? #wal_decode_buffer_size = 512kB # lookahead window used for prefetching # (change requires restart) +#replay_warm_workers = 0 # workers reading pages ahead of replay + # (change requires restart) +#replay_warm_queue_size = 512 # block requests the warm pool can hold + # (change requires restart) # - Archiving - diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 7bb801bb88612..63d368d874ec8 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -296,6 +296,8 @@ main(int argc, char *argv[]) ControlFile->backupEndRequired ? _("yes") : _("no")); printf(_("wal_level setting: %s\n"), wal_level_str(ControlFile->wal_level)); + printf(_("io_torn_pages_protection setting: %s\n"), + DWBProtectionModeName(ControlFile->io_torn_pages_protection)); printf(_("wal_log_hints setting: %s\n"), ControlFile->wal_log_hints ? _("on") : _("off")); printf(_("max_connections setting: %d\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index efb515e8a1ac2..31ad3e0686555 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -708,6 +708,7 @@ GuessControlValues(void) ControlFile.wal_level = WAL_LEVEL_MINIMAL; ControlFile.wal_log_hints = false; ControlFile.track_commit_timestamp = false; + ControlFile.io_torn_pages_protection = DWB_PROTECT_FULL_PAGES; ControlFile.MaxConnections = 100; ControlFile.max_wal_senders = 10; ControlFile.max_worker_processes = 8; @@ -917,6 +918,7 @@ RewriteControlFile(void) ControlFile.wal_level = WAL_LEVEL_MINIMAL; ControlFile.wal_log_hints = false; ControlFile.track_commit_timestamp = false; + ControlFile.io_torn_pages_protection = DWB_PROTECT_FULL_PAGES; ControlFile.MaxConnections = 100; ControlFile.max_wal_senders = 10; ControlFile.max_worker_processes = 8; diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c index 074ac41b026fc..b1360b75367cc 100644 --- a/src/bin/pg_rewind/file_ops.c +++ b/src/bin/pg_rewind/file_ops.c @@ -37,7 +37,8 @@ static void create_target_symlink(const char *path, const char *link); static void remove_target_symlink(const char *path); static void recurse_dir(const char *datadir, const char *parentpath, - process_file_callback_t callback); + process_file_callback_t callback, + bool follow_dwb_symlink); /* * Open a target file for writing. If 'trunc' is true and the file already @@ -380,11 +381,17 @@ slurpFile(const char *datadir, const char *path, size_t *filesize) /* * Traverse through all files in a data directory, calling 'callback' * for each file. + * + * 'follow_dwb_symlink' says whether to follow a symlinked pg_dwb: the + * target's ring must be enumerated so that the rewind wipes it, but the + * source's ring is never used, so a broken link there must not fail the + * traversal. */ void -traverse_datadir(const char *datadir, process_file_callback_t callback) +traverse_datadir(const char *datadir, process_file_callback_t callback, + bool follow_dwb_symlink) { - recurse_dir(datadir, NULL, callback); + recurse_dir(datadir, NULL, callback, follow_dwb_symlink); } /* @@ -395,7 +402,7 @@ traverse_datadir(const char *datadir, process_file_callback_t callback) */ static void recurse_dir(const char *datadir, const char *parentpath, - process_file_callback_t callback) + process_file_callback_t callback, bool follow_dwb_symlink) { DIR *xldir; struct dirent *xlde; @@ -452,7 +459,7 @@ recurse_dir(const char *datadir, const char *parentpath, { callback(path, FILE_TYPE_DIRECTORY, 0, NULL); /* recurse to handle subdirectories */ - recurse_dir(datadir, path, callback); + recurse_dir(datadir, path, callback, follow_dwb_symlink); } else if (S_ISLNK(fst.st_mode)) { @@ -473,11 +480,15 @@ recurse_dir(const char *datadir, const char *parentpath, /* * If it's a symlink within pg_tblspc, we need to recurse into it, * to process all the tablespaces. We also follow a symlink if - * it's for pg_wal. Symlinks elsewhere are ignored. + * it's for pg_wal, or — when requested — for pg_dwb so that + * the target's double write buffer ring is enumerated (and thus + * wiped) even when the ring lives behind a symlink. Symlinks + * elsewhere are ignored. */ if ((parentpath && strcmp(parentpath, PG_TBLSPC_DIR) == 0) || - strcmp(path, "pg_wal") == 0) - recurse_dir(datadir, path, callback); + strcmp(path, "pg_wal") == 0 || + (follow_dwb_symlink && strcmp(path, "pg_dwb") == 0)) + recurse_dir(datadir, path, callback, follow_dwb_symlink); } } diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h index ee0d01df1ae51..e6ad847d87c6c 100644 --- a/src/bin/pg_rewind/file_ops.h +++ b/src/bin/pg_rewind/file_ops.h @@ -24,6 +24,7 @@ extern void sync_target_dir(void); extern char *slurpFile(const char *datadir, const char *path, size_t *filesize); typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target); -extern void traverse_datadir(const char *datadir, process_file_callback_t callback); +extern void traverse_datadir(const char *datadir, process_file_callback_t callback, + bool follow_dwb_symlink); #endif /* FILE_OPS_H */ diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index c933871ca9fda..4ebda55e2df32 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -132,6 +132,16 @@ static const char *const excludeDirContents[] = /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ "pg_dynshmem", /* defined as PG_DYNSHMEM_DIR */ + /* + * The double write buffer ring is local to an instance: its slots are + * page copies of that cluster's own in-flight writes. Excluding it keeps + * the source's ring off the target and, because decide_file_action() + * removes excluded paths that exist in the target, also wipes the + * target's own ring — the rewound cluster cold-starts a fresh one, see + * DWBStartup(). + */ + "pg_dwb", /* defined as DWB_DIR */ + /* Contents removed on startup, see AsyncShmemInit(). */ "pg_notify", @@ -708,6 +718,18 @@ decide_file_action(file_entry_t *entry) if (strcmp(path, XLOG_CONTROL_FILE) == 0) return FILE_ACTION_NONE; + /* + * Never touch the pg_dwb entry itself: either side may have it as a plain + * directory, as a symlink, or (before its first double_writes startup) + * not at all, and the server (re)creates it lazily, see DWBCreateRing(). + * Its contents match the exclusion filters and are removed from the + * target below. The target's entry and ring contents are validated up + * front by checkTargetDwb() before the traversal; the source's entry + * needs no validation, its ring is never used. + */ + if (strcmp(path, "pg_dwb") == 0) + return FILE_ACTION_NONE; + /* Skip macOS system files */ if (strstr(path, ".DS_Store") != NULL) return FILE_ACTION_NONE; diff --git a/src/bin/pg_rewind/libpq_source.c b/src/bin/pg_rewind/libpq_source.c index 56c2ad55d4a67..95a31f4387d69 100644 --- a/src/bin/pg_rewind/libpq_source.c +++ b/src/bin/pg_rewind/libpq_source.c @@ -132,9 +132,12 @@ init_libpq_conn(PGconn *conn) PQclear(res); /* - * Also check that full_page_writes is enabled. We can get torn pages if - * a page is modified while we read it with pg_read_binary_file(), and we - * rely on full page images to fix them. + * Also check that the source server actually writes full page images. We + * can get torn pages if a page is modified while we read it with + * pg_read_binary_file(), and we rely on full page images to fix them. + * This GUC only has its usual meaning under io_torn_pages_protection = + * "full_pages"; the other modes are refused outright based on the + * source's pg_control (see the up-front check in pg_rewind.c). */ str = run_simple_query(conn, "SHOW full_page_writes"); if (strcmp(str, "on") != 0) diff --git a/src/bin/pg_rewind/local_source.c b/src/bin/pg_rewind/local_source.c index 5a6e805c15833..89420a5e7d13c 100644 --- a/src/bin/pg_rewind/local_source.c +++ b/src/bin/pg_rewind/local_source.c @@ -57,7 +57,8 @@ init_local_source(const char *datadir) static void local_traverse_files(rewind_source *source, process_file_callback_t callback) { - traverse_datadir(((local_source *) source)->datadir, callback); + /* the source's double write buffer ring is never used, don't enter it */ + traverse_datadir(((local_source *) source)->datadir, callback, false); } static char * diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 9d16c1e6b4757..6aed326a9a5ef 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include +#include #include #include #include @@ -46,6 +47,7 @@ static void digestControlFile(ControlFileData *ControlFile, const char *content, size_t size); static void getRestoreCommand(const char *argv0); static void sanityChecks(void); +static void checkTargetDwb(void); static TimeLineHistoryEntry *getTimelineHistory(TimeLineID tli, bool is_source, int *nentries); static void findCommonAncestorTimeline(TimeLineHistoryEntry *a_history, @@ -318,6 +320,33 @@ main(int argc, char **argv) else source = init_local_source(datadir_source); + /* + * A live source must itself be protected by full page images: reading + * files from a running server can catch pages mid-write, and only WAL + * page images repair such torn reads on the rewound target. Under + * io_torn_pages_protection = "double_writes" or "off" the source's WAL + * has no images (its double write buffer repairs its own torn writes, not + * our torn reads), so refuse up front, before the target is touched in + * any way. A stopped source has no such requirement. The mode is read + * from the source's pg_control — the authoritative record, unlike the + * legacy full_page_writes GUC, which only matters under "full_pages" and + * is checked by init_libpq_conn when the connection is made (so a + * double_writes source with full_page_writes=off draws that message, not + * this one). + */ + if (connstr_source) + { + buffer = source->fetch_file(source, XLOG_CONTROL_FILE, &size); + digestControlFile(&ControlFile_source, buffer, size); + pg_free(buffer); + + if (ControlFile_source.io_torn_pages_protection != DWB_PROTECT_FULL_PAGES) + pg_fatal("\"io_torn_pages_protection\" must be \"full_pages\" in the source server, not \"%s\"", + DWBProtectionModeName(ControlFile_source.io_torn_pages_protection)); + } + + checkTargetDwb(); + /* * Check the status of the target instance. * @@ -476,7 +505,7 @@ main(int argc, char **argv) if (showprogress) pg_log_info("reading target file list"); - traverse_datadir(datadir_target, &process_target_file); + traverse_datadir(datadir_target, &process_target_file, true); /* * Read the target WAL from last checkpoint before the point of fork, to @@ -780,6 +809,80 @@ sanityChecks(void) pg_fatal("source data directory must be shut down cleanly"); } +/* + * Validate the target's pg_dwb entry before the target is touched in any + * way — in particular before the single-user recovery run and the + * no-rewind-required exit. + * + * The file-list traversal classifies only directories, symlinks and regular + * files, so garbage in place of pg_dwb or inside it (a FIFO, a socket...) + * would go unnoticed and survive the rewind, only to fail the next + * double_writes startup. The entry must be a directory or a symlink to an + * accessible directory — or absent, since the server creates the ring + * lazily — and, the ring being flat, nothing but regular files belongs + * inside; anything else is rejected here, which lets the traversal's + * exclusion-driven removal wipe the ring completely. + */ +static void +checkTargetDwb(void) +{ + char dwb_path[MAXPGPATH]; + struct stat st; + DIR *dir; + struct dirent *de; + + snprintf(dwb_path, sizeof(dwb_path), "%s/pg_dwb", datadir_target); + + if (lstat(dwb_path, &st) < 0) + { + if (errno == ENOENT) + return; + pg_fatal("could not stat file \"%s\": %m", dwb_path); + } + + if (!S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + pg_fatal("\"%s\" in target is not a directory or symbolic link", + "pg_dwb"); + + if (S_ISLNK(st.st_mode)) + { + if (stat(dwb_path, &st) < 0) + { + if (errno == ENOENT || errno == ENOTDIR) + pg_fatal("\"%s\" in target is a symbolic link that does not point to a directory", + "pg_dwb"); + pg_fatal("could not stat file \"%s\": %m", dwb_path); + } + if (!S_ISDIR(st.st_mode)) + pg_fatal("\"%s\" in target is a symbolic link that does not point to a directory", + "pg_dwb"); + } + + dir = opendir(dwb_path); + if (dir == NULL) + pg_fatal("could not open directory \"%s\": %m", dwb_path); + + while (errno = 0, (de = readdir(dir)) != NULL) + { + char entry_path[MAXPGPATH * 2]; + + if (strcmp(de->d_name, ".") == 0 || + strcmp(de->d_name, "..") == 0) + continue; + + snprintf(entry_path, sizeof(entry_path), "%s/%s", dwb_path, de->d_name); + if (lstat(entry_path, &st) < 0) + pg_fatal("could not stat file \"%s\": %m", entry_path); + if (!S_ISREG(st.st_mode)) + pg_fatal("\"%s/%s\" in target is not a regular file", + "pg_dwb", de->d_name); + } + if (errno) + pg_fatal("could not read directory \"%s\": %m", dwb_path); + + (void) closedir(dir); +} + /* * Print a progress report based on the fetch_size and fetch_done variables. * diff --git a/src/common/file_utils.c b/src/common/file_utils.c index 7b62687a2aa75..af31d4b962609 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -86,9 +86,9 @@ do_syncfs(const char *path) * Synchronize PGDATA and all its contents. * * We sync regular files and directories wherever they are, but we follow - * symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc. - * Other symlinks are presumed to point at files we're not responsible for - * syncing, and might not have privileges to write at all. + * symlinks only for pg_wal (or pg_xlog), pg_dwb and immediately under + * pg_tblspc. Other symlinks are presumed to point at files we're not + * responsible for syncing, and might not have privileges to write at all. * * serverVersion indicates the version of the server to be sync'd. * @@ -102,12 +102,15 @@ sync_pgdata(const char *pg_data, bool sync_data_files) { bool xlog_is_symlink; + bool dwb_is_symlink; char pg_wal[MAXPGPATH]; + char pg_dwb[MAXPGPATH]; char pg_tblspc[MAXPGPATH]; /* handle renaming of pg_xlog to pg_wal in post-10 clusters */ snprintf(pg_wal, MAXPGPATH, "%s/%s", pg_data, serverVersion < MINIMUM_VERSION_FOR_PG_WAL ? "pg_xlog" : "pg_wal"); + snprintf(pg_dwb, MAXPGPATH, "%s/%s", pg_data, "pg_dwb"); snprintf(pg_tblspc, MAXPGPATH, "%s/%s", pg_data, PG_TBLSPC_DIR); /* @@ -125,6 +128,25 @@ sync_pgdata(const char *pg_data, xlog_is_symlink = true; } + /* + * Likewise for the double write buffer ring. Unlike pg_wal, pg_dwb is + * created lazily at the first double_writes startup, so its absence is + * normal and not worth a complaint; any other lstat() failure is. + */ + dwb_is_symlink = false; + + { + struct stat st; + + if (lstat(pg_dwb, &st) < 0) + { + if (errno != ENOENT) + pg_log_error("could not stat file \"%s\": %m", pg_dwb); + } + else if (S_ISLNK(st.st_mode)) + dwb_is_symlink = true; + } + switch (sync_method) { case DATA_DIR_SYNC_METHOD_SYNCFS: @@ -141,8 +163,9 @@ sync_pgdata(const char *pg_data, * On Linux, we don't have to open every single file one by * one. We can use syncfs() to sync whole filesystems. We * only expect filesystem boundaries to exist where we - * tolerate symlinks, namely pg_wal and the tablespaces, so we - * call syncfs() for each of those directories. + * tolerate symlinks, namely pg_wal, pg_dwb and the + * tablespaces, so we call syncfs() for each of those + * directories. */ /* Sync the top level pgdata directory. */ @@ -181,6 +204,10 @@ sync_pgdata(const char *pg_data, /* If pg_wal is a symlink, process that too. */ if (xlog_is_symlink) do_syncfs(pg_wal); + + /* Likewise for a symlinked double write buffer ring. */ + if (dwb_is_symlink) + do_syncfs(pg_dwb); #endif /* HAVE_SYNCFS */ } break; @@ -200,6 +227,8 @@ sync_pgdata(const char *pg_data, walkdir(pg_data, pre_sync_fname, false, exclude_dir); if (xlog_is_symlink) walkdir(pg_wal, pre_sync_fname, false, NULL); + if (dwb_is_symlink) + walkdir(pg_dwb, pre_sync_fname, false, NULL); if (sync_data_files) walkdir(pg_tblspc, pre_sync_fname, true, NULL); #endif @@ -208,15 +237,17 @@ sync_pgdata(const char *pg_data, * Now we do the fsync()s in the same order. * * The main call ignores symlinks, so in addition to specially - * processing pg_wal if it's a symlink, pg_tblspc has to be - * visited separately with process_symlinks = true. Note that - * if there are any plain directories in pg_tblspc, they'll - * get fsync'd twice. That's not an expected case so we don't - * worry about optimizing it. + * processing pg_wal and pg_dwb if they are symlinks, + * pg_tblspc has to be visited separately with + * process_symlinks = true. Note that if there are any plain + * directories in pg_tblspc, they'll get fsync'd twice. That's + * not an expected case so we don't worry about optimizing it. */ walkdir(pg_data, fsync_fname, false, exclude_dir); if (xlog_is_symlink) walkdir(pg_wal, fsync_fname, false, NULL); + if (dwb_is_symlink) + walkdir(pg_dwb, fsync_fname, false, NULL); if (sync_data_files) walkdir(pg_tblspc, fsync_fname, true, NULL); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 2cf8d55d706d1..f5fc6adf405dd 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -31,7 +31,7 @@ /* * Each page of XLOG file has a header like this: */ -#define XLOG_PAGE_MAGIC 0xD118 /* can be used as WAL version indicator */ +#define XLOG_PAGE_MAGIC 0xD11A /* can be used as WAL version indicator */ typedef struct XLogPageHeaderData { @@ -278,6 +278,7 @@ typedef struct xl_parameter_change int max_prepared_xacts; int max_locks_per_xact; int wal_level; + int io_torn_pages_protection; /* DWBTornPageProtection */ bool wal_log_hints; bool track_commit_timestamp; } xl_parameter_change; diff --git a/src/include/access/xlogprefetcher.h b/src/include/access/xlogprefetcher.h index 50b39c1fb0d77..a7e55dcbe61f7 100644 --- a/src/include/access/xlogprefetcher.h +++ b/src/include/access/xlogprefetcher.h @@ -39,6 +39,17 @@ extern void XLogPrefetchShmemInit(void); extern void XLogPrefetchResetStats(void); +/* + * Counting from outside the startup process. + * + * The replay warm pool decides in its workers what replay used to decide for + * itself — whether a block was already in a buffer — so the two counters that + * record that decision are incremented from there. They keep their meaning; + * only the process holding the answer has changed. + */ +extern void XLogPrefetchCountHit(void); +extern void XLogPrefetchCountPrefetch(void); + extern XLogPrefetcher *XLogPrefetcherAllocate(XLogReaderState *reader); extern void XLogPrefetcherFree(XLogPrefetcher *prefetcher); diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h index 9738462d3c9f1..3dadff389d432 100644 --- a/src/include/access/xlogreader.h +++ b/src/include/access/xlogreader.h @@ -116,6 +116,9 @@ typedef struct XLogReaderRoutine #define XL_ROUTINE(...) &(XLogReaderRoutine){__VA_ARGS__} +/* a block that was not published to the replay warm pool */ +#define XLOGWARM_NO_SLOT (-1) + typedef struct { /* Is this block ref in use? */ @@ -129,6 +132,23 @@ typedef struct /* Prefetching workspace. */ Buffer prefetch_buffer; + /* + * Warm pool workspace: the slot this block was published to and the + * request it was published as, so the answer can be told apart from a + * later request that recycled the slot. XLOGWARM_NO_SLOT when the block + * was never published (which is always the case in frontend code). + */ + int warm_slot; + uint64 warm_request; + + /* + * True when prefetch_buffer above is an answer collected from the pool + * rather than a buffer the cache lookup happened to find. Only the + * pool's own answers say anything about the pool when they turn out to be + * stale. + */ + bool warm_hint; + /* copy of the fork_flags field from the XLogRecordBlockHeader */ uint8 flags; diff --git a/src/include/access/xlogwarm.h b/src/include/access/xlogwarm.h new file mode 100644 index 0000000000000..e7b45b76f3e57 --- /dev/null +++ b/src/include/access/xlogwarm.h @@ -0,0 +1,63 @@ +/*------------------------------------------------------------------------- + * + * xlogwarm.h + * Replay prefetch worker pool. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/xlogwarm.h + *------------------------------------------------------------------------- + */ +#ifndef XLOGWARM_H +#define XLOGWARM_H + +#include "access/xlogreader.h" +#include "storage/block.h" +#include "storage/bufmgr.h" +#include "storage/relfilelocator.h" + +/* GUCs */ +extern PGDLLIMPORT int replay_warm_workers; +extern PGDLLIMPORT int replay_warm_queue_size; + +/* a snapshot of the pool's counters */ +typedef struct XLogWarmStats +{ + uint64 published; /* blocks handed to the pool */ + uint64 dropped_full; /* blocks left unwarmed, no free slot */ + uint64 collected; /* answers replay picked up */ + uint64 missed; /* requests with no answer to pick up */ + uint64 stale; /* answers whose buffer had been evicted */ + uint64 cancelled; /* requests withdrawn on a prefetcher reset */ + uint64 released; /* slots handed back by a departing worker */ + uint64 claimed; /* requests a worker took */ + uint64 reads; /* pages a worker actually read */ + uint64 hits; /* pages already resident when claimed */ + uint64 failed; /* reads that errored out */ + uint64 vanished; /* relations gone or too short by read time */ + uint64 discarded; /* results dropped, slot no longer theirs */ +} XLogWarmStats; + +extern Size XLogWarmShmemSize(void); +extern void XLogWarmShmemInit(void); +extern void XLogWarmWorkersRegister(void); +pg_noreturn extern void XLogWarmWorkerMain(Datum main_arg); + +extern bool XLogWarmPoolActive(void); +extern int XLogWarmPublish(RelFileLocator rlocator, ForkNumber forknum, + BlockNumber blkno, uint64 *request_id); +extern Buffer XLogWarmCollect(int slot_no, uint64 request_id); +extern void XLogWarmCancelAll(void); +extern void XLogWarmCountStale(void); +extern void XLogWarmDropBegin(void); +extern void XLogWarmDropEnd(void); +extern bool XLogWarmGetStats(XLogWarmStats * stats); +extern void XLogWarmGetSlotCounts(int *published, int *claimed); +extern void XLogWarmGetPoolState(int *scanners, int *pending, int *sleepers); +extern int XLogWarmGetWorkerPids(int *pids, int *slots); + +/* the pool size ceiling, matching the setting's maximum */ +#define XLOGWARM_MAX_WORKERS 64 + +#endif /* XLOGWARM_H */ diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index 63e834a6ce477..041266a7402f4 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -22,11 +22,49 @@ /* Version identifier for this pg_control format */ -#define PG_CONTROL_VERSION 1800 +#define PG_CONTROL_VERSION 1802 /* Nonce key length, see below */ #define MOCK_AUTH_NONCE_LEN 32 +/* + * The torn-page protection mechanism (GUC io_torn_pages_protection). Like + * wal_level, the value in force on the WAL-generating server is a protocol + * fact: it decides whether the WAL can carry full page images at all (under + * "full_pages" the legacy full_page_writes GUC still chooses whether it + * actually does), so it is recorded in pg_control and in + * XLOG_PARAMETER_CHANGE records for replay to track. Defined here rather + * than in storage/dwb.h so that frontend code reading pg_control can use it. + * + * The numeric values are stored on disk and in WAL; never renumber the + * members. Keep the names in sync with the GUC option list in + * guc_tables.c. + */ +typedef enum +{ + DWB_PROTECT_OFF = 0, + DWB_PROTECT_FULL_PAGES = 1, + DWB_PROTECT_DOUBLE_WRITES = 2, +} DWBTornPageProtection; + +/* GUC-spelling name of a DWBTornPageProtection value, for messages */ +static inline const char * +DWBProtectionModeName(int mode) +{ + /* the cast keeps -Wswitch honest about newly added members */ + switch ((DWBTornPageProtection) mode) + { + case DWB_PROTECT_OFF: + return "off"; + case DWB_PROTECT_FULL_PAGES: + return "full_pages"; + case DWB_PROTECT_DOUBLE_WRITES: + return "double_writes"; + } + /* garbage read from disk or WAL must not turn into UB */ + return "unrecognized"; +} + /* * Body of CheckPoint XLOG records. This is declared here because we keep * a copy of the latest one in pg_control for possible disaster recovery. @@ -176,6 +214,7 @@ typedef struct ControlFileData * or hot standby. */ int wal_level; + int io_torn_pages_protection; /* DWBTornPageProtection */ bool wal_log_hints; int MaxConnections; int max_worker_processes; diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 3a302c2cab022..e95a82bde8501 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -275,9 +275,10 @@ typedef enum IOObject IOOBJECT_RELATION, IOOBJECT_TEMP_RELATION, IOOBJECT_WAL, + IOOBJECT_DWB, } IOObject; -#define IOOBJECT_NUM_TYPES (IOOBJECT_WAL + 1) +#define IOOBJECT_NUM_TYPES (IOOBJECT_DWB + 1) typedef enum IOContext { diff --git a/src/include/postmaster/bgworker.h b/src/include/postmaster/bgworker.h index 058667a47a0a1..f897b7f093822 100644 --- a/src/include/postmaster/bgworker.h +++ b/src/include/postmaster/bgworker.h @@ -114,6 +114,9 @@ typedef struct BackgroundWorkerHandle BackgroundWorkerHandle; /* Register a new bgworker during shared_preload_libraries */ extern void RegisterBackgroundWorker(BackgroundWorker *worker); +/* How many static bgworkers have been registered so far (postmaster only) */ +extern int GetNumRegisteredBackgroundWorkers(void); + /* Register a new bgworker from a regular backend */ extern bool RegisterDynamicBackgroundWorker(BackgroundWorker *worker, BackgroundWorkerHandle **handle); diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index 0dec7d93b3b27..c52c231e88523 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -325,6 +325,7 @@ typedef struct WritebackContext extern PGDLLIMPORT BufferDescPadded *BufferDescriptors; extern PGDLLIMPORT ConditionVariableMinimallyPadded *BufferIOCVArray; extern PGDLLIMPORT WritebackContext BackendWritebackContext; +extern PGDLLIMPORT WritebackContext DwbWritebackContext; /* in localbuf.c */ extern PGDLLIMPORT BufferDesc *LocalBufferDescriptors; @@ -434,6 +435,14 @@ extern void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_co extern void ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context, BufferTag *tag); +/* + * The vectored DWB flush of one bin of buffers; exported for the cleaner + * worker pool (dwb_cleaner.c), which executes the bgwriter's queued bins + * in opportunistic mode. + */ +extern int FlushBufferBin(const int *buf_ids, int nbuf, bool opportunistic, + WritebackContext *wb_context); + /* solely to make it easier to write tests */ extern bool StartBufferIO(BufferDesc *buf, bool forInput, bool nowait); extern void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits, diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 41fdc1e76938e..988d81a8f543c 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -201,6 +201,9 @@ extern PGDLLIMPORT int32 *LocalRefCount; /* * prototypes for functions in bufmgr.c */ +extern Buffer LookupSharedBuffer(struct SMgrRelationData *smgr_reln, + ForkNumber forkNum, + BlockNumber blockNum); extern PrefetchBufferResult PrefetchSharedBuffer(struct SMgrRelationData *smgr_reln, ForkNumber forkNum, BlockNumber blockNum); diff --git a/src/include/storage/dwb.h b/src/include/storage/dwb.h new file mode 100644 index 0000000000000..e8fc83b5892c1 --- /dev/null +++ b/src/include/storage/dwb.h @@ -0,0 +1,579 @@ +/*------------------------------------------------------------------------- + * + * dwb.h + * Short-lived double write buffer (DWB). + * + * A DWB slot lives for the duration of one flush batch and is reused only + * after the data-file fsync covering its page has become durable. Strict + * durability order per batch: + * + * XLogFlush(page LSN) -> batch write + fdatasync (leader) -> + * smgrwrite (kernel cache) -> eventual segment fsync -> slot reuse + * + * See .plan/short-lived-dwb_REL_18.md for the full design. + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dwb.h + * + *------------------------------------------------------------------------- + */ +#ifndef DWB_H +#define DWB_H + +#include "access/xlogdefs.h" +#include "catalog/pg_control.h" +#include "common/file_utils.h" +#include "port/pg_crc32c.h" +#include "storage/buf_internals.h" +#include "storage/condition_variable.h" +#include "storage/lwlock.h" +#include "storage/relfilelocator.h" +#include "storage/s_lock.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" + +/* + * The io_torn_pages_protection GUC values (DWBTornPageProtection) live in + * catalog/pg_control.h: the mode is recorded in pg_control. + */ + +/* GUC: dwb_on_stall (Stage B backpressure behaviour) */ +typedef enum +{ + DWB_ON_STALL_WARN, + DWB_ON_STALL_ERROR, + DWB_ON_STALL_PANIC, +} DWBOnStall; + +/* + * GUC: dwb_retire_sync_method. Shares the DataDirSyncMethod values of + * recovery_init_sync_method (common/file_utils.h): "fsync" retires by + * fsyncing each touched data-file segment, "syncfs" makes whole file + * systems durable per retire round. syncfs is the default where the + * syscall exists: a shared random workload touches nearly every segment + * of a large table between rounds, and one syncfs replaces hundreds of + * per-segment fdatasync calls on the same file system. + */ +#ifdef HAVE_SYNCFS +#define DWB_RETIRE_SYNC_METHOD_DEFAULT DATA_DIR_SYNC_METHOD_SYNCFS +#else +#define DWB_RETIRE_SYNC_METHOD_DEFAULT DATA_DIR_SYNC_METHOD_FSYNC +#endif + +/* GUC variables (defined in dwb_ctl.c) */ +extern PGDLLIMPORT int io_torn_pages_protection; +extern PGDLLIMPORT int dwb_num_batches; +extern PGDLLIMPORT int dwb_batch_pages; +extern PGDLLIMPORT int dwb_max_segments; +extern PGDLLIMPORT int dwb_retire_workers; +extern PGDLLIMPORT int dwb_cleaner_workers; +extern PGDLLIMPORT int dwb_retire_sync_method; +extern PGDLLIMPORT int dwb_batch_timeout_ms; +extern PGDLLIMPORT int dwb_retire_interval_ms; +extern PGDLLIMPORT int dwb_writeback_after; +extern PGDLLIMPORT int dwb_slow_warn_ms; +extern PGDLLIMPORT int dwb_slot_stuck_timeout_ms; +extern PGDLLIMPORT int dwb_write_timeout_ms; +extern PGDLLIMPORT int dwb_on_stall; + +#define DWBIsEnabled() (io_torn_pages_protection == DWB_PROTECT_DOUBLE_WRITES) + +/* + * Does a page write retire its own batch inline? Without a retire pool + * DWBFinishPageWrite fsyncs before it returns (see dwb.c), so a writeback + * hint that is still sitting in a pending array when it is called has already + * missed its purpose. Callers hand the page to the kernel there and then. + * Requires miscadmin.h for IsUnderPostmaster. + */ +#define DWBRetiresInline() (dwb_retire_workers == 0 || !IsUnderPostmaster) + +/* + * Blocks a process accumulates before it hands the double write buffer's + * writeback hints to the kernel. The same count as the checkpointer's + * default, for the same reason: the hint is worth starting early, and worth + * starting in block order, but not worth a syscall per page. Like that one + * it is a block count, so what it is worth in bytes follows BLCKSZ. + */ +#define DEFAULT_DWB_WRITEBACK_AFTER 32 + +/* + * Compile-time capacity limits (GUC maxima). Statically sized arrays + * (per-batch shmem arrays, the retire-side segment snapshot) rely on these, + * so the GUC bounds in guc_tables.c must use them, never bare literals. + */ +#define DWB_BATCH_MAX_PAGES 256 +#define DWB_BATCH_MAX_SEGS DWB_BATCH_MAX_PAGES +#define DWB_BITMAP_WORDS (DWB_BATCH_MAX_PAGES / 64) +#define DWB_NUM_BATCHES_MAX 1024 +/* staging pool: 2 writer classes + 2 in-flight leader writes */ +#define DWB_STAGING_BUFFERS 4 +/* writer classes (3.6) */ +#define DWB_NUM_WCLASSES 2 +#define DWB_WCLASS_EVICTION 0 +#define DWB_WCLASS_BACKGROUND 1 + +/* + * Bin size cap for the vectored background flush (FlushBufferBin, used by + * the checkpointer's BufferSync, the bgwriter's LRU scan and the cleaner + * worker pool): the flush holds a pin, a shared content lock and + * BM_IO_IN_PROGRESS per bin member at once, so the cap must leave + * MAX_SIMUL_LWLOCKS (200) plenty of headroom. 64 matches the default + * dwb_batch_pages; larger batch_pages settings seal their batches at + * bin-sized fills. Shared here because the cleaner work queue stores + * bins of this size. + */ +#define DWB_FLUSH_BIN_MAX 64 + +/* + * Why a batch was sealed. Purely diagnostic: per-class seal and page + * counters in DWCtl attribute batch turnover to its trigger, which is how + * a half-filled average (batch fsyncs paid for underfilled batches) is + * told apart from healthy overflow sealing. + */ +typedef enum DWBSealReason +{ + DWB_SEAL_OVERFLOW, /* a reservation ran past the last slot */ + DWB_SEAL_LONE, /* solo-stream fast seal: a lone waiter, or + * every page when there is no worker pool */ + DWB_SEAL_WAIT_TIMEOUT, /* a waiting writer hit dwb_batch_timeout_ms */ + DWB_SEAL_WORKER_TIMEOUT, /* a retire worker force-sealed on age */ + DWB_SEAL_BIN, /* a background bin flush sealed its batches */ + DWB_SEAL_FORCED, /* explicit DWBForceSealOpenBatch */ +} DWBSealReason; + +#define DWB_SEAL_NREASONS (DWB_SEAL_FORCED + 1) + +#define DWB_DIR "pg_dwb" +#define DWB_CONTROL_FILE DWB_DIR "/control" + +/* + * On-disk format. + * + * pg_dwb/control - geometry + durable generation, written atomically + * pg_dwb/batch_NNNN - meta region (header + slot metas, padded to + * PG_IO_ALIGN_SIZE) followed by a contiguous + * PG_IO_ALIGN_SIZE-aligned stream of BLCKSZ page + * images + * + * Slot validity is locally verifiable: meta_crc rejects a torn meta write + * (including any old/new field mix on slot reuse), image_crc rejects a torn + * image; an apply-pass candidate must pass meta_crc + generation + image_crc. + */ +#define DWB_CONTROL_MAGIC 0x44574243 /* "DWBC" */ +#define DWB_BATCH_MAGIC 0x44574242 /* "DWBB" */ +#define DWB_VERSION 1 +#define DWB_MIN_VERSION 1 + +typedef struct DWBControlFileData +{ + uint32 magic; + uint32 version; + uint32 min_version; + uint32 num_batches; + uint32 batch_pages; + uint32 flags; /* DWB_CONTROL_* */ + uint64 generation; /* apply-pass horizon: bumped durably on every + * double_writes start before the ring opens. + * Monotonic within one ring incarnation; a + * geometry-change recreate restarts it at + * zero, which is safe because the wipe leaves + * no CRC-valid slot behind */ + pg_crc32c crc; /* CRC of all preceding fields */ +} DWBControlFileData; + +/* + * DWBControlFileData.flags. RING_CLEAN certifies that every data-file write + * covered by an on-disk slot had been fsynced when the server shut down: it + * is written at the end of a clean shutdown after the ring is fully retired, + * and cleared by the next double_writes startup before the ring reopens. + * While it is set, the ring holds no unapplied repairs, so the apply-pass + * must be skipped (non-double_writes runs in between leave the generation + * untouched, so old slots would otherwise still match it) and a start under + * a different io_torn_pages_protection mode is legal. Any new code path + * that opens the ring for writes must clear the marker in the same control + * write that bumps the generation — the marker-only apply decision is + * sound only while set-marker implies untouched-since-retirement. + * + * The flags field occupies what was interior alignment padding in version-1 + * control files; those read back with flags == 0 (the padding was always + * memset and CRC-covered), which is the safe "not clean" state, so filling + * the hole needed no DWB_VERSION bump. + */ +#define DWB_CONTROL_RING_CLEAN 0x0001 + +typedef struct DWBBatchHeader +{ + uint32 magic; + uint32 version; + uint64 batch_id; /* incarnation id, assigned at batch open, + * monotonic in open order within one server + * run (next_batch_id restarts at 1 with each + * start); the apply-pass dedup uses it as an + * LSN tie-breaker — see DWBApplyPass */ + uint32 n_slots; /* capped_slots at seal time */ + pg_crc32c crc; /* CRC of all preceding fields */ +} DWBBatchHeader; + +typedef struct DWSlotMeta +{ + BufferTag tag; + XLogRecPtr page_lsn; + uint64 generation; /* ring generation at write time */ + uint16 flags; + pg_crc32c image_crc; /* CRC of the BLCKSZ page image */ + pg_crc32c meta_crc; /* CRC of all preceding fields */ +} DWSlotMeta; + +/* DWSlotMeta.flags */ +#define DWB_SLOT_ABORTED 0x0001 /* writer died before publishing */ + +/* + * The on-disk layout is pinned: any change to these sizes or offsets is an + * on-disk format change and requires a DWB_VERSION bump. + */ +StaticAssertDecl(sizeof(DWBControlFileData) == 40, + "DWBControlFileData on-disk size changed"); +StaticAssertDecl(offsetof(DWBControlFileData, crc) == 32, + "DWBControlFileData crc offset changed"); +StaticAssertDecl(sizeof(DWBBatchHeader) == 24, + "DWBBatchHeader on-disk size changed"); +StaticAssertDecl(offsetof(DWBBatchHeader, crc) == 20, + "DWBBatchHeader crc offset changed"); +StaticAssertDecl(sizeof(DWSlotMeta) == 56, + "DWSlotMeta on-disk size changed"); +StaticAssertDecl(offsetof(DWSlotMeta, meta_crc) == 48, + "DWSlotMeta meta_crc offset changed"); + +#define DWBMetaRegionSize(batch_pages) \ + TYPEALIGN(PG_IO_ALIGN_SIZE, \ + sizeof(DWBBatchHeader) + (batch_pages) * sizeof(DWSlotMeta)) +#define DWBBatchFileSize(batch_pages) \ + (DWBMetaRegionSize(batch_pages) + (Size) (batch_pages) * BLCKSZ) + +/* + * Batch lifecycle. A slot is reused only via DWB_FREE. + * + * The numeric order of the happy-path states is semantic: the code uses + * comparisons like "state < DWB_FSYNCED" as progress tests. Insert new + * states only in lifecycle order; DWB_OOM_RETIRING is a side fork of + * DWB_RETIRING and must stay numerically last. + */ +typedef enum DWBatchState +{ + DWB_FREE = 0, + DWB_ALLOCATED, /* writers fill slots */ + DWB_SEALED, /* no new writers; waiting for bitmap + * coverage, then the leader writes */ + DWB_WRITTEN, /* images + meta written, fdatasync pending */ + DWB_FSYNCED, /* batch durable; writers do smgrwrite */ + DWB_DATA_WRITTEN, /* all smgrwrite + sync requests done */ + DWB_RETIRING, /* waiting for fsync of seg_set segments */ + DWB_OOM_RETIRING, /* publisher retires synchronously after a + * DWSegmentHash OOM (3.5) */ +} DWBatchState; + +StaticAssertDecl(DWB_FREE < DWB_ALLOCATED && + DWB_ALLOCATED < DWB_SEALED && + DWB_SEALED < DWB_WRITTEN && + DWB_WRITTEN < DWB_FSYNCED && + DWB_FSYNCED < DWB_DATA_WRITTEN && + DWB_DATA_WRITTEN < DWB_RETIRING && + DWB_RETIRING < DWB_OOM_RETIRING, + "DWBatchState numeric order is semantic (progress tests)"); + +typedef struct DWSegRef +{ + RelFileLocator rlocator; + ForkNumber forknum; + uint32 segno; +} DWSegRef; + +/* + * DWSegRef is a HASH_BLOBS key: hashed and compared as raw bytes, so it must + * not contain padding (which field-wise construction would leave undefined). + */ +StaticAssertDecl(sizeof(DWSegRef) == + sizeof(RelFileLocator) + sizeof(ForkNumber) + sizeof(uint32), + "DWSegRef has padding; unsafe as a HASH_BLOBS key"); + +/* + * Segment -> batch back-reference (3.5): one shmem hash entry per segment + * that at least one RETIRING batch still needs fsynced. The bitmap is + * indexed by ring batch index; a bit is set exactly once per batch life, at + * the DATA_WRITTEN -> RETIRING transition, and cleared by the fsyncer that + * covered it. Batch index reuse is disambiguated by snapshotting batch_id + * before the fsync and re-checking it under the batch's publish_lock before + * decrementing (the ABA guard of 3.5). + * + * The entry size depends on dwb_num_batches, so the bitmap is a flexible + * array of DWBSegBitmapWords() words; hash lookups/inserts/removals are + * serialized by DWBSegHashLock. fsync_in_progress is a best-effort claim + * that lets concurrent fsyncers skip a segment somebody is already syncing; + * races on it are benign because the bit-clear + decrement is idempotent. + */ +typedef struct DWSegEntry +{ + DWSegRef key; + pg_atomic_uint32 fsync_in_progress; + pg_atomic_uint64 batch_bitmap[FLEXIBLE_ARRAY_MEMBER]; +} DWSegEntry; + +#define DWBSegBitmapWords() (((uint32) dwb_num_batches + 63) / 64) + +/* + * Sliced reserves of FREE batches, by the free count F at open time: the + * bottom slice [1 .. DWB_BG_RESERVE] may be opened only by the background + * class, the middle slice (.. DWB_BG_RESERVE + DWB_EVICT_RESERVE] only by + * the eviction class, anything above by both. The middle slice keeps a + * checkpoint's BufferSync storm from eating the ring from under user + * evictions; the bottom slice keeps a crowd of evicting backends from + * starving the checkpointer outright (each class needs just one open batch, + * so a non-empty bottom slice is a progress guarantee for the background + * stream). + */ +#define DWB_EVICT_RESERVE Max(2, dwb_num_batches / 8) +#define DWB_BG_RESERVE Max(1, dwb_num_batches / 32) + +/* + * next_slot_idx encoding: 30-bit index + writer-class bit + seal sentinel. + * + * The class bit records which writer class opened this incarnation of the + * batch. A reservation validates it atomically with the increment (CAS in + * DWBAcquireSlot), so a stale per-class open pointer can never join a batch + * that was freed and reopened under the other class: the ring reuses batch + * indexes, and open_batch_idx[] of an idle class keeps naming its last batch + * long after that batch was retired. + */ +#define DWB_SEAL_BIT (1U << 31) +#define DWB_WCLASS_BIT (1U << 30) +#define DWB_IDX_MASK (DWB_WCLASS_BIT - 1) + +/* one flag bit encodes the opening class: works for exactly two classes */ +StaticAssertDecl(DWB_NUM_WCLASSES == 2, + "next_slot_idx has a single writer-class bit"); + +#define DWBWClassBit(wclass) \ + ((wclass) == DWB_WCLASS_BACKGROUND ? DWB_WCLASS_BIT : 0) + +typedef struct DWBatchCtl +{ + pg_atomic_uint32 state; /* DWBatchState */ + pg_atomic_uint32 next_slot_idx; /* CAS-incremented while open, see + * DWBAcquireSlot() */ + pg_atomic_uint32 capped_slots; /* fixed by SEAL; leader waits for exactly + * this many bitmap bits */ + pg_atomic_uint64 slots_written_bitmap[DWB_BITMAP_WORDS]; + pg_atomic_uint32 ref_count; /* writers holding the batch from slot + * reservation to smgrwrite done */ + pg_atomic_uint32 seg_pending_count; /* seg_set entries not yet fsynced; + * decremented once per covered + * segment, the decrement to zero + * frees the batch */ + LWLock publish_lock; /* protects n_segs/seg_set (dedup insert), + * serializes seg_set publication into + * DWSegmentHash and the batch_id-guarded + * seg_pending_count decrement (3.5) */ + ConditionVariable cv_state; /* broadcast on state change */ + uint32 n_segs; + DWSegRef seg_set[DWB_BATCH_MAX_SEGS]; + BufferTag pages[DWB_BATCH_MAX_PAGES]; + XLogRecPtr page_lsns[DWB_BATCH_MAX_PAGES]; + pg_crc32c image_crcs[DWB_BATCH_MAX_PAGES]; /* computed by writers at + * publication */ + uint16 slot_flags[DWB_BATCH_MAX_PAGES]; /* same width as + * DWSlotMeta.flags */ + int staging_idx; /* staging buffer; held from ALLOCATED until + * the leader finishes the image pwrite */ + XLogRecPtr max_page_lsn; + uint64 batch_id; /* monotonic incarnation id. Written only at + * reopen, under DWBRingOpenLock; read under a + * held ref (which pins the incarnation) or + * racily by the retire side. The ABA + * re-check in DWBSegSnapEnd reads it under + * publish_lock, but that lock does not + * serialize against the reopen write: safety + * comes from the idempotent bitmap re-check + * plus id monotonicity (3.5). */ + TimestampTz open_time; /* FREE -> ALLOCATED instant; drives + * force-SEAL via dwb_batch_timeout_ms */ +} DWBatchCtl; + +typedef struct DWCtl +{ + pg_atomic_uint32 open_batch_idx[DWB_NUM_WCLASSES]; /* current ALLOCATED + * batch per writer + * class, or + * DWB_INVALID_BATCH */ + pg_atomic_uint64 next_batch_id; + uint64 ring_generation; /* = control.generation after the startup + * bump; constant until restart, stamped + * into DWSlotMeta by the leader */ + pg_atomic_uint64 freed_events; /* monotonic count of batches that reached + * FREE; backpressure waiters treat a + * change as retire progress */ + pg_atomic_uint64 ring_wait_retries; /* monotonic count of DWBOpenNewBatch + * iterations that went to sleep; a + * parked waiter on an unchanged ring + * must accrue these at the sleep + * timeout pace, not spin (see the + * silent-probe-release rule in + * DWBStagingRelease) */ + /* diagnostic seal accounting: [writer class][DWBSealReason] */ + pg_atomic_uint64 seal_count[DWB_NUM_WCLASSES][DWB_SEAL_NREASONS]; + pg_atomic_uint64 seal_pages[DWB_NUM_WCLASSES][DWB_SEAL_NREASONS]; + + /* + * TimestampTz of the class's last overflow seal: the "demand is hot" + * marker that suppresses the lone-writer fast seal (see + * DWBWaitBatchFsynced). Advisory — read and written without barriers; + * a stale value mis-decides at most one seal in either direction. + */ + pg_atomic_uint64 last_overflow_seal[DWB_NUM_WCLASSES]; + ConditionVariable cv_want_batch[DWB_NUM_WCLASSES]; /* per-class "want a + * batch" queue: both + * staging and + * ring-space waiters + * sleep here; woken by + * targeted signals, not + * broadcast (3.6) */ + ConditionVariable cv_retire_wake; /* wakes retire workers */ + slock_t staging_lock; /* protects staging_free bitmap */ + uint32 staging_free; /* bitmap of free staging buffers */ + DWBatchCtl batches[FLEXIBLE_ARRAY_MEMBER]; /* dwb_num_batches entries */ +} DWCtl; + +#define DWB_INVALID_BATCH PG_UINT32_MAX + +/* Writer-side handle for one reserved slot */ +typedef struct DWBSlotRef +{ + int batch_idx; + int slot_idx; + uint64 batch_id; +} DWBSlotRef; + +/* + * Work queue between the bgwriter's LRU scan and the cleaner worker pool + * (dwb_cleaner.c). An entry is one flush bin: buffer ids the scan + * classified as cold dirty candidates. Entries are hints, not + * obligations — every claim is reclassified under the buffer header lock + * right before the write (FlushBufferBin's opportunistic mode), so a + * stale entry is skipped, never wrongly written, and the queue needs no + * draining on shutdown: whatever it held stays dirty and is covered by + * the next checkpoint. + * + * In error-free operation every accepted page ends up counted as either + * written or skipped, so enqueued_pages = pool_written_total + + * skipped_pages once the queue is empty. A worker error mid-bin + * abandons the bin's remainder — those pages stay dirty and are simply + * rescanned later, but they leave the counters short of the identity. + */ +typedef struct DWBCleanerBin +{ + int nbuf; + int buf_ids[DWB_FLUSH_BIN_MAX]; +} DWBCleanerBin; + +typedef struct DWBCleanerCtl +{ + /* counters are monotonic except pool_written, which bgwriter drains */ + pg_atomic_uint64 enqueued_pages; /* pages ever accepted into the queue */ + pg_atomic_uint64 pool_written; /* pages written by cleaners since the + * bgwriter last folded them into + * buf_written_clean */ + pg_atomic_uint64 pool_written_total; /* same, never reset (tests, + * diagnostics) */ + pg_atomic_uint64 skipped_pages; /* stale claims dropped by + * reclassification */ + pg_atomic_uint64 deferred_bins; /* bins refused by a full queue and + * carried over by the bgwriter */ + pg_atomic_uint64 pressure_naps; /* checkpointer naps taken only because + * the queue was hot */ + pg_atomic_uint32 depth; /* lock-free mirror of nqueued for the + * checkpointer's advisory pressure check */ + ConditionVariable cv_work; /* one targeted signal per enqueued bin */ + int capacity; + /* head/nqueued and the bins are protected by DWBCleanerQueueLock */ + int head; + int nqueued; + DWBCleanerBin bins[FLEXIBLE_ARRAY_MEMBER]; /* capacity entries */ +} DWBCleanerCtl; + +extern PGDLLIMPORT DWCtl *DWBCtl; +extern PGDLLIMPORT char *DWBStagingBase; +extern PGDLLIMPORT HTAB *DWSegmentHash; + +/* dwb_ctl.c */ +extern Size DWBShmemSize(void); +extern void DWBShmemInit(void); + +/* dwb.c — write path */ +extern void DWBInitBackend(void); +extern void DWBStagePageWrite(const BufferTag *tag, const char *image, + XLogRecPtr page_lsn, DWBSlotRef *ref); +extern void DWBStagePageWriteNoWait(const BufferTag *tag, const char *image, + XLogRecPtr page_lsn, DWBSlotRef *ref); +extern void DWBWaitStagedWrites(const DWBSlotRef *refs, int nrefs); +extern void DWBFinishPageWrite(const DWBSlotRef *ref); +extern bool DWBWritesPaused(void); +extern void DWBAcquireSlot(const BufferTag *tag, int wclass, + bool use_resowner, DWBSlotRef *ref); +extern void DWBPublishImage(const DWBSlotRef *ref, const char *image, + XLogRecPtr page_lsn); +extern void DWBWaitBatchFsynced(const DWBSlotRef *ref); +extern void DWBReleaseSlot(const DWBSlotRef *ref); +extern bool DWBForceSealOpenBatch(int wclass); +extern bool DWBTrySealBatch(int batch_idx, DWBSealReason reason); +extern bool DWBClassIsHot(int wclass); +extern DWBatchState DWBGetBatchState(int batch_idx); + +/* internal; exported for test_dwb's stale-open regression test */ +extern void DWBOpenNewBatch(int wclass, uint32 old_idx); +extern void DWBWakeRingWaiters(void); + +/* dwb_retire.c — segment hash, retirement, worker pool */ +struct FileTag; /* avoid dragging storage/sync.h in here */ +extern void DWBPublishBatchSegSet(int batch_idx); +extern void DWBSegmentFsyncBegin(const struct FileTag *ftag); +extern int DWBSegmentFsyncEnd(bool synced); +extern int DWBRetireAllSync(void); +extern void DWBRetireWorkersRegister(void); +pg_noreturn extern void DWBRetireWorkerMain(Datum main_arg); + +/* dwb_cleaner.c — bgwriter bin queue and the cleaner worker pool */ +extern PGDLLIMPORT DWBCleanerCtl *DWBCleanerQueue; +extern PGDLLIMPORT bool DWBAmCleanerWorker; +extern Size DWBCleanerShmemSize(void); +extern void DWBCleanerShmemInit(void); +extern bool DWBCleanersActive(void); +extern bool DWBCleanerEnqueueBin(const int *buf_ids, int nbuf); +extern uint64 DWBCleanerFetchPoolWritten(void); +extern void DWBCleanerCountDeferral(void); +extern bool DWBCleanerQueueHot(void); +extern void DWBCleanerCountPressureNap(void); +extern void DWBCleanerWorkersRegister(void); +pg_noreturn extern void DWBCleanerWorkerMain(Datum main_arg); + +/* dwb_file.c */ +extern void DWBCreateRing(void); +extern void DWBBatchFilePath(char *path, int batch_idx); +extern bool DWBReadControlFile(DWBControlFileData *control, bool missing_ok, + bool *corruptp); +extern void DWBWriteControlFile(const DWBControlFileData *control); +extern int DWBOpenBatchFile(int batch_idx); +extern void DWBPrepareBatchWrite(int batch_idx); +extern void DWBReadSlotImage(int batch_idx, int slot_idx, char *dst); +extern void DWBWriteBatch(int batch_idx, const DWBBatchHeader *hdr, + const DWSlotMeta *metas, const char *images); +extern pg_crc32c DWBImageCrc(const char *image); +extern pg_crc32c DWBSlotMetaCrc(const DWSlotMeta *meta); +extern pg_crc32c DWBControlCrc(const DWBControlFileData *control); +extern pg_crc32c DWBBatchHeaderCrc(const DWBBatchHeader *hdr); + +/* dwb_recovery.c */ +extern XLogRecPtr DWBStartup(bool restoring_backup); +extern void DWBMarkCleanShutdown(void); + +#endif /* DWB_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 08a72569ae5fd..c65ea79edadab 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -221,6 +221,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_XACT_SLRU, LWTRANCHE_PARALLEL_VACUUM_DSA, LWTRANCHE_AIO_URING_COMPLETION, + LWTRANCHE_DWB_PUBLISH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/include/storage/lwlocklist.h b/src/include/storage/lwlocklist.h index 932024b1b0ba5..bee80d842c7c2 100644 --- a/src/include/storage/lwlocklist.h +++ b/src/include/storage/lwlocklist.h @@ -84,3 +84,9 @@ PG_LWLOCK(50, DSMRegistry) PG_LWLOCK(51, InjectionPoint) PG_LWLOCK(52, SerialControl) PG_LWLOCK(53, AioWorkerSubmissionQueue) +PG_LWLOCK(54, DWBRingOpen) +PG_LWLOCK(55, DWBSegHash) +PG_LWLOCK(56, DWBSelfSweep) +PG_LWLOCK(57, DWBSyncfsRound) +PG_LWLOCK(58, DWBCleanerQueue) +PG_LWLOCK(59, ReplayWarmRead) diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 4e82d6f151732..93251cc6e90ac 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -23,6 +23,7 @@ SUBDIRS = \ test_ddl_deparse \ test_dsa \ test_dsm_registry \ + test_dwb \ test_escape \ test_extensions \ test_ginpostinglist \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 9a957351ab693..846fd4a9e0015 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -22,6 +22,7 @@ subdir('test_custom_types') subdir('test_ddl_deparse') subdir('test_dsa') subdir('test_dsm_registry') +subdir('test_dwb') subdir('test_escape') subdir('test_extensions') subdir('test_ginpostinglist') diff --git a/src/test/modules/test_dwb/.gitignore b/src/test/modules/test_dwb/.gitignore new file mode 100644 index 0000000000000..4c5757d966fad --- /dev/null +++ b/src/test/modules/test_dwb/.gitignore @@ -0,0 +1,8 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ + +# Generated by the pg_upgrade run in t/014_pg_upgrade.pl +/delete_old_cluster.sh +/delete_old_cluster.bat diff --git a/src/test/modules/test_dwb/Makefile b/src/test/modules/test_dwb/Makefile new file mode 100644 index 0000000000000..a7d2f04813490 --- /dev/null +++ b/src/test/modules/test_dwb/Makefile @@ -0,0 +1,34 @@ +# src/test/modules/test_dwb/Makefile + +MODULE_big = test_dwb +OBJS = \ + $(WIN32RES) \ + test_dwb.o +PGFILEDESC = "test_dwb - test module for the short-lived double write buffer" + +TAP_TESTS = 1 + +EXTENSION = test_dwb +DATA = test_dwb--1.0.sql + +# 003_backpressure.pl uses the injection_points extension, +# 022_writeback_pacing.pl evicts named buffers with pg_buffercache +EXTRA_INSTALL = src/test/modules/injection_points contrib/pg_buffercache +export enable_injection_points + +REGRESS_OPTS = --temp-config $(top_srcdir)/src/test/modules/test_dwb/test_dwb.conf +REGRESS = test_dwb +# Disabled because these tests require io_torn_pages_protection=double_writes, +# which typical installcheck users do not have. +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dwb +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dwb/expected/test_dwb.out b/src/test/modules/test_dwb/expected/test_dwb.out new file mode 100644 index 0000000000000..3e1fa377cd942 --- /dev/null +++ b/src/test/modules/test_dwb/expected/test_dwb.out @@ -0,0 +1,144 @@ +CREATE EXTENSION test_dwb; +-- ring is idle after startup +SELECT test_dwb_states(); + test_dwb_states +---------------------------------------------------------------------------- + free=16 allocated=0 sealed=0 written=0 fsynced=0 data_written=0 retiring=0 +(1 row) + +-- a partial batch, sealed by force (ring: 16 batches x 16 pages) +SELECT test_dwb_cycle(5); + test_dwb_cycle +---------------- + 1 +(1 row) + +-- overflow-sealed batches: 40 pages = 16 + 16 + 8 (tail force-sealed) +SELECT test_dwb_cycle(40); + test_dwb_cycle +---------------- + 3 +(1 row) + +-- every surviving slot validates against meta_crc, generation and +-- image_crc: eager retirement lets the cycles reuse batch file 0, so its +-- final content is the 8-slot tail write, plus 16 slots in batch file 1 +SELECT test_dwb_ring_slots(true); + test_dwb_ring_slots +--------------------- + 24 +(1 row) + +-- and the ring is fully retired again +SELECT test_dwb_states(); + test_dwb_states +---------------------------------------------------------------------------- + free=16 allocated=0 sealed=0 written=0 fsynced=0 data_written=0 retiring=0 +(1 row) + +-- the cycles sealed deterministically: two overflow seals from the 40-page +-- run (2 x 16 slots) and two forced tail seals (5 + 8 slots) +SELECT wclass, reason, seals, pages FROM test_dwb_seal_stats() + WHERE seals > 0 ORDER BY wclass, reason; + wclass | reason | seals | pages +----------+----------+-------+------- + eviction | forced | 2 | 13 + eviction | overflow | 2 | 32 +(2 rows) + +-- lone-writer fast seal in a QUIET class: get clear of the cycles' overflow +-- stamps first, then a single staged page must seal immediately as "lone" +SELECT pg_sleep(0.3); + pg_sleep +---------- + +(1 row) + +CREATE TEMP TABLE seal_before_quiet AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_stage_lone_wait(); + test_dwb_stage_lone_wait +-------------------------- + +(1 row) + +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_quiet b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; + wclass | reason | dseals | dpages +----------+--------+--------+-------- + eviction | lone | 1 | 1 +(1 row) + +SELECT test_dwb_retire() >= 0 AS drained; + drained +--------- + t +(1 row) + +-- HOT class: an overflow seal microseconds before the lone attempt must +-- suppress the fast seal into the waiter's timeout seal +CREATE TEMP TABLE seal_before_hot AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_overflow_lone_wait(); + test_dwb_overflow_lone_wait +----------------------------- + +(1 row) + +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_hot b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; + wclass | reason | dseals | dpages +----------+--------------+--------+-------- + eviction | overflow | 1 | 16 + eviction | wait_timeout | 1 | 1 +(2 rows) + +SELECT test_dwb_retire() >= 0 AS drained; + drained +--------- + t +(1 row) + +-- the hot test reads the stamp through the same helper: a fresh stamp is +-- hot, a stamp from the FUTURE (a backward clock step) must read as quiet +-- and keep the fast seal immediate +SELECT test_dwb_set_overflow_stamp(0) AS hot_now; + hot_now +--------- + t +(1 row) + +SELECT test_dwb_set_overflow_stamp(60000) AS hot_future; + hot_future +------------ + f +(1 row) + +CREATE TEMP TABLE seal_before_future AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_stage_lone_wait(); + test_dwb_stage_lone_wait +-------------------------- + +(1 row) + +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_future b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; + wclass | reason | dseals | dpages +----------+--------+--------+-------- + eviction | lone | 1 | 1 +(1 row) + +SELECT test_dwb_retire() >= 0 AS drained; + drained +--------- + t +(1 row) + +-- and the ring is idle again +SELECT test_dwb_states(); + test_dwb_states +---------------------------------------------------------------------------- + free=16 allocated=0 sealed=0 written=0 fsynced=0 data_written=0 retiring=0 +(1 row) + diff --git a/src/test/modules/test_dwb/meson.build b/src/test/modules/test_dwb/meson.build new file mode 100644 index 0000000000000..1ad7ad517d375 --- /dev/null +++ b/src/test/modules/test_dwb/meson.build @@ -0,0 +1,64 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +test_dwb_sources = files( + 'test_dwb.c', +) + +if host_system == 'windows' + test_dwb_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dwb', + '--FILEDESC', 'test_dwb - test module for the short-lived double write buffer',]) +endif + +test_dwb = shared_module('test_dwb', + test_dwb_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dwb + +test_install_data += files( + 'test_dwb.control', + 'test_dwb--1.0.sql', +) + +tests += { + 'name': 'test_dwb', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dwb', + ], + 'regress_args': ['--temp-config', files('test_dwb.conf')], + 'runningcheck': false, + }, + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_dwb.pl', + 't/002_flushbuffer.pl', + 't/003_backpressure.pl', + 't/004_retire_paths.pl', + 't/005_standby.pl', + 't/006_backup.pl', + 't/007_rewind.pl', + 't/008_modes.pl', + 't/009_fpw_transition.pl', + 't/010_recovery.pl', + 't/011_geometry_recovery.pl', + 't/012_apply_crafted.pl', + 't/013_backup_start_point.pl', + 't/014_pg_upgrade.pl', + 't/015_vectored_flush.pl', + 't/016_bgwriter_bin.pl', + 't/017_syncfs_retire.pl', + 't/018_cleaners.pl', + 't/019_autovacuum_class.pl', + 't/020_ckpt_yield.pl', + 't/021_replay_warm.pl', + 't/022_writeback_pacing.pl', + ], + }, +} diff --git a/src/test/modules/test_dwb/sql/test_dwb.sql b/src/test/modules/test_dwb/sql/test_dwb.sql new file mode 100644 index 0000000000000..a583ac543b5bb --- /dev/null +++ b/src/test/modules/test_dwb/sql/test_dwb.sql @@ -0,0 +1,57 @@ +CREATE EXTENSION test_dwb; + +-- ring is idle after startup +SELECT test_dwb_states(); + +-- a partial batch, sealed by force (ring: 16 batches x 16 pages) +SELECT test_dwb_cycle(5); + +-- overflow-sealed batches: 40 pages = 16 + 16 + 8 (tail force-sealed) +SELECT test_dwb_cycle(40); + +-- every surviving slot validates against meta_crc, generation and +-- image_crc: eager retirement lets the cycles reuse batch file 0, so its +-- final content is the 8-slot tail write, plus 16 slots in batch file 1 +SELECT test_dwb_ring_slots(true); + +-- and the ring is fully retired again +SELECT test_dwb_states(); + +-- the cycles sealed deterministically: two overflow seals from the 40-page +-- run (2 x 16 slots) and two forced tail seals (5 + 8 slots) +SELECT wclass, reason, seals, pages FROM test_dwb_seal_stats() + WHERE seals > 0 ORDER BY wclass, reason; + +-- lone-writer fast seal in a QUIET class: get clear of the cycles' overflow +-- stamps first, then a single staged page must seal immediately as "lone" +SELECT pg_sleep(0.3); +CREATE TEMP TABLE seal_before_quiet AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_stage_lone_wait(); +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_quiet b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; +SELECT test_dwb_retire() >= 0 AS drained; + +-- HOT class: an overflow seal microseconds before the lone attempt must +-- suppress the fast seal into the waiter's timeout seal +CREATE TEMP TABLE seal_before_hot AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_overflow_lone_wait(); +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_hot b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; +SELECT test_dwb_retire() >= 0 AS drained; + +-- the hot test reads the stamp through the same helper: a fresh stamp is +-- hot, a stamp from the FUTURE (a backward clock step) must read as quiet +-- and keep the fast seal immediate +SELECT test_dwb_set_overflow_stamp(0) AS hot_now; +SELECT test_dwb_set_overflow_stamp(60000) AS hot_future; +CREATE TEMP TABLE seal_before_future AS SELECT * FROM test_dwb_seal_stats(); +SELECT test_dwb_stage_lone_wait(); +SELECT s.wclass, s.reason, s.seals - b.seals AS dseals, s.pages - b.pages AS dpages + FROM test_dwb_seal_stats() s JOIN seal_before_future b USING (wclass, reason) + WHERE s.seals <> b.seals ORDER BY s.wclass, s.reason; +SELECT test_dwb_retire() >= 0 AS drained; + +-- and the ring is idle again +SELECT test_dwb_states(); diff --git a/src/test/modules/test_dwb/t/001_dwb.pl b/src/test/modules/test_dwb/t/001_dwb.pl new file mode 100644 index 0000000000000..1d4b7b21524f5 --- /dev/null +++ b/src/test/modules/test_dwb/t/001_dwb.pl @@ -0,0 +1,328 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Concurrency, restart (generation) and enforcement tests for the +# short-lived double write buffer. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb'); +$node->init; +# dwb_retire_workers = 0 keeps batch sealing and retirement fully under the +# test's control (writers retire synchronously); the quiescing settings keep +# background flushes from opening batches between the state assertions. +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 1h +autovacuum = off +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# --- single-session cycles --------------------------------------------- + +is($node->safe_psql('postgres', 'SELECT test_dwb_cycle(40)'), + '3', 'overflow-sealed cycle retires three batches'); +# eager retirement reuses batch file 0 within the cycle: its final content +# is the 8-slot tail write, plus 16 slots in batch file 1 +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '24', 'all surviving slots validate (meta_crc, generation, image_crc)'); + +# --- torn copies are rejected by the on-disk CRCs ---------------------- + +# Batch file layout with dwb_batch_pages = 16: a 24-byte header, 16 slot +# metas of 56 bytes each (sizes pinned by StaticAssertDecl in dwb.h), the +# meta region padded to 4096; page images follow at 4096 + slot * 8192. +my $bfile = $node->data_dir . '/pg_dwb/batch_0001'; + +sub flip_byte +{ + my ($file, $offset) = @_; + open my $bf, '+<:raw', $file or die "open $file: $!"; + sysseek($bf, $offset, 0) or die "seek: $!"; + die "read: $!" unless sysread($bf, my $byte, 1) == 1; + sysseek($bf, $offset, 0) or die "seek: $!"; + die "write: $!" unless syswrite($bf, chr(ord($byte) ^ 0xFF), 1) == 1; + close $bf; + return; +} + +# a torn image: one flipped byte inside slot 0's page image +flip_byte($bfile, 4096 + 100); +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '23', 'image_crc rejects a torn page image'); + +# a torn meta: one flipped byte inside slot 1's meta (offset 24 + 56) +flip_byte($bfile, 24 + 56); +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '22', 'meta_crc rejects a torn slot meta'); + +# --- concurrent writers over a small ring ------------------------------ + +# 3 clients x 30 transactions x 40 pages through a 16x16 ring +my $script = $node->basedir . '/stress.sql'; +open my $fh, '>', $script or die $!; +print $fh "SELECT test_dwb_stress(1, 40);\n"; +close $fh; +$node->command_ok( + [ + 'pgbench', '-n', '-c', '3', '-j', '3', + '-t', '30', '-f', $script, 'postgres' + ], + 'concurrent stress over a small ring'); +like( + $node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=16 allocated=0 sealed=0 written=0 fsynced=0 data_written=0 retiring=0/, + 'ring fully retired after concurrent stress'); +my $valid = $node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'); +cmp_ok($valid, '>', 0, + 'ring holds valid current-generation slots after stress'); +cmp_ok($valid, '<=', 16 * 16, 'slot count bounded by the ring capacity'); + +# --- restart bumps the durable generation ------------------------------ + +# The shutdown checkpoint itself streams pages through the ring, so exact +# slot counts cannot survive a restart; the invariants that must hold are +# that CRC-valid slots exist and that none of them belongs to the new +# generation. +my $stale = $node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(false)'); +cmp_ok($stale, '>', 0, 'ring holds slots before the restart check'); +$node->restart; +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '0', 'no slot belongs to the new generation after restart'); +cmp_ok($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(false)'), + '>', 0, 'stale slots still CRC-valid, only the generation gates them'); + +# --- process exit cleanup ---------------------------------------------- + +# A backend dies holding unpublished slots: DWBProcExit poisons them, the +# next seal's coverage wait is satisfied by the poison bits, the batch +# completes and the aborted slots never validate. +my $bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT test_dwb_leak(3, false)'); +$bg->quit; +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/allocated=1/, 'abandoned batch stays open'); +is($node->safe_psql('postgres', 'SELECT test_dwb_force_seal()'), + 't', 'abandoned batch seals'); +$node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE '%retiring=1%'") + or die 'timed out waiting for the abandoned batch to reach RETIRING'; +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'abandoned batch retires'); +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '0', 'aborted slots are not apply candidates'); + +# A backend dies after publishing: the copies are still written out and +# validate; the leader's own pin hands the batch over to retirement even +# though no writer is left alive. +$bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT test_dwb_leak(3, true)'); +$bg->quit; +is($node->safe_psql('postgres', 'SELECT test_dwb_force_seal()'), + 't', 'orphaned batch seals'); +# the dead backend's ProcExit may still be releasing its refs: wait for +# the FSYNCED -> RETIRING hand-off instead of assuming it already happened +$node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE '%retiring=1%'") + or die 'timed out waiting for the orphaned batch to reach RETIRING'; +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'orphaned batch retires'); +is($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '3', 'published slots of a dead backend survive and validate'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=16/, 'ring fully idle after the orphan hand-off'); + +# A backend dies holding the LAST ref of an already-durable batch: the exit +# backstop itself performs the FSYNCED -> RETIRING hand-off (seg_set +# publication under LWLocks), which is only legal because it runs as a +# before_shmem_exit callback while the PGPROC is still alive. +$bg = $node->background_psql('postgres'); +$bg->query_safe('SELECT test_dwb_leak_fsynced()'); +$bg->quit; +$node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE '%retiring=1%'") + or die 'timed out waiting for the exit-time publication'; +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'batch published from the exit backstop retires'); + +# A command that registers an exit callback of its own and cancels it again +# on the way out -- PG_ENSURE_ERROR_CLEANUP, as CREATE DATABASE uses -- can +# only cancel it while it is still the last one registered. The backstop +# above is registered once, when the process starts, so that a write staged +# in between does not land on top of it. This session has staged none yet, +# which is exactly the case that would. +$node->safe_psql('postgres', 'CREATE DATABASE dwb_createdb'); +is( $node->safe_psql('dwb_createdb', 'SELECT count(*) FROM pg_class'), + $node->safe_psql('template1', 'SELECT count(*) FROM pg_class'), + 'a database copied through the buffer cache is complete and usable'); +$node->safe_psql('postgres', 'DROP DATABASE dwb_createdb'); + +# --- transaction abort releases refs (ResourceOwner path) --------------- + +# An ERROR with unpublished refs: the abort poisons the slots, and the +# batch seals and completes later exactly like the dead-backend case — +# without a process exit. +my ($rc, $out, $err) = + $node->psql('postgres', 'SELECT test_dwb_abort_release(3, false)'); +isnt($rc, 0, 'deliberate abort with pending refs reported'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/allocated=1/, 'batch of the aborted transaction stays open'); +is($node->safe_psql('postgres', 'SELECT test_dwb_force_seal()'), + 't', 'batch of the aborted transaction seals'); +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'batch of the aborted transaction retires'); + +# An ERROR after the batch is durable: the abort cleanup goes through the +# abandoned-slot ref hand-off (the fake relation exits via the +# dropped-relation branch) and must still hand the batch over to retirement. +($rc, $out, $err) = + $node->psql('postgres', 'SELECT test_dwb_abort_after_fsync()'); +isnt($rc, 0, 'deliberate abort after batch fsync reported'); +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'batch of the post-fsync abort retires'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=16/, 'ring idle after the abort scenarios'); + +# --- torn data page repaired from the batch copy on abort ---------------- + +# A REAL relation this time: test_dwb_torn_repair stages the pristine +# on-disk image of block 0 into the DWB, tears the block on disk, and +# aborts. The ResourceOwner release must rewrite the block from the +# durable batch copy (DWBRewriteAbandonedSlot). The restart proves the +# repair reached the data file: the buffer cache is dropped, and with data +# checksums a block left torn would make the read below fail. +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_repair AS + SELECT g AS id, repeat('r', 64) AS pad FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $filenode = + $node->safe_psql('postgres', "SELECT pg_relation_filenode('dwb_repair')"); +($rc, $out, $err) = + $node->psql('postgres', "SELECT test_dwb_torn_repair($filenode, 0)"); +isnt($rc, 0, 'deliberate abort after tearing the data page reported'); +like($err, qr/deliberate abort after tearing/, 'the tear scenario ran'); +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'batch of the torn-page scenario retires'); +$node->restart; +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_repair'), + '100', 'torn block repaired from the batch copy (checksum-clean read)'); + +# --- the sliced reserves shape who may open what -------------------------- + +# On this geometry DWB_BG_RESERVE = Max(1, 16/32) = 1 and DWB_EVICT_RESERVE +# = Max(2, 16/8) = 2: a background-class writer filling a fresh ring stops +# above the middle eviction slice, an eviction-class writer consumes +# everything but the bottom background slice, and the background class can +# still open that last batch — the starvation-proof lane of the +# checkpointer. +$bg = $node->background_psql('postgres'); +my $bg_taken = $bg->query_safe('SELECT test_dwb_fill_ring(true)'); +cmp_ok($bg_taken, '>', 0, 'background class filled the ring'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=3 /, 'background fill stops above the eviction slice'); +my $ev_taken = $bg->query_safe('SELECT test_dwb_fill_ring(false)'); +cmp_ok($ev_taken, '>', 0, 'eviction class still opens batches'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=1 /, 'eviction leaves the bottom background slice'); +my $lane_taken = $bg->query_safe('SELECT test_dwb_fill_ring(true)'); +cmp_ok($lane_taken, '>', 0, 'background class opens its reserved lane'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=0 /, '... consuming the ring fully'); +$bg->quit; +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal(false) IS NOT NULL THEN " + . "CASE WHEN test_dwb_force_seal(true) IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=16 %' END END END") + or die 'timed out waiting for the ring to drain after the reserve scenario'; + +# --- stale open must not hijack a reopened index ------------------------ + +($rc, $out, $err) = $node->psql('postgres', 'SELECT test_dwb_open_stale()'); +is($rc, 0, 'stale open leaves the live reopened batch alone') + or diag($err); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/free=16/, 'ring idle after the stale-open scenario'); + +# --- a geometry change recreates the ring ------------------------------ + +# The on-disk layout follows the geometry GUCs, so a change rebuilds the +# ring from scratch (after applying the old one if needed — exercised in +# t/011_geometry_recovery.pl); the generation restarts with the fresh +# control file. +$node->stop; +my $log_offset = -s $node->logfile; +$node->append_conf('postgresql.conf', 'dwb_num_batches = 32'); +$node->start; +ok( $node->log_contains( + 'recreating double write buffer ring: geometry changed from 16 batches of 16 pages to 32 batches of 16 pages', + $log_offset), + 'geometry change recreates the ring'); +ok( $node->log_contains( + qr/ring opened: 32 batches of 16 pages, generation 1\b/, $log_offset), + 'recreated ring opens with a fresh generation'); +$node->stop; +$log_offset = -s $node->logfile; +$node->append_conf( + 'postgresql.conf', + 'dwb_num_batches = 16 +dwb_batch_pages = 32'); +$node->start; +ok( $node->log_contains( + 'geometry changed from 32 batches of 16 pages to 16 batches of 32 pages', + $log_offset), + 'batch_pages change recreates the ring too'); +$node->stop; +$node->append_conf('postgresql.conf', 'dwb_batch_pages = 16'); +$node->start; +$node->stop; + +# --- the retire worker pool must actually fit into the worker slots ------ + +# The logical replication launcher takes a slot before the pool registers; +# RegisterBackgroundWorker itself only LOGs on overflow, so the pool checks +# the remaining capacity and refuses to start a silently smaller pool. +$log_offset = -s $node->logfile; +$node->append_conf( + 'postgresql.conf', qq( +max_worker_processes = 1 +dwb_retire_workers = 1 +)); +my $ret = $node->start(fail_ok => 1); +is($ret, 0, 'start refused when the pool does not fit into worker slots'); +ok( $node->log_contains( + 'needs more "max_worker_processes" slots than remain free', + $log_offset), + 'worker slot shortage reported'); +$node->append_conf( + 'postgresql.conf', qq( +max_worker_processes = 8 +dwb_retire_workers = 0 +)); +$node->start; +$node->stop; + +# --- double_writes requires data checksums ----------------------------- + +my $node2 = PostgreSQL::Test::Cluster->new('dwb_nochecksums'); +$node2->init(extra => ['--no-data-checksums']); +$node2->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$ret = $node2->start(fail_ok => 1); +is($ret, 0, 'start refused without data checksums'); +ok($node2->log_contains('requires data checksums'), + 'checksum requirement reported'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/002_flushbuffer.pl b/src/test/modules/test_dwb/t/002_flushbuffer.pl new file mode 100644 index 0000000000000..ecb6656582398 --- /dev/null +++ b/src/test/modules/test_dwb/t/002_flushbuffer.pl @@ -0,0 +1,154 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# End-to-end tests of the FlushBuffer integration: real pages flow through +# the double write buffer, the retire worker pool frees the ring, pg_stat_io +# accounts the batch IO, and the cluster survives a crash. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_flush'); +$node->init; +# A tiny buffer pool forces evictions through the DWB on a modest workload; +# the short batch timeout keeps the worker pool sealing and retiring briskly. +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 2 +dwb_batch_timeout_ms = 20 +shared_buffers = 2MB +autovacuum = off +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# --- the worker pool is running ----------------------------------------- + +# the workers start asynchronously once the server is up +$node->poll_query_until('postgres', + "SELECT count(*) = 2 FROM pg_stat_activity WHERE backend_type = 'dwb retire worker'" +) or die 'timed out waiting for the retire workers to start'; +pass('both retire workers are running'); + +# --- a real workload flows through the ring ------------------------------ + +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_t AS + SELECT g AS id, repeat('x', 300) AS filler + FROM generate_series(1, 50000) g; + UPDATE dwb_t SET filler = repeat('y', 300) WHERE id % 10 = 0; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); + +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '50000', 'workload survived the DWB write path'); + +# The workload far exceeds shared_buffers, so evictions must have staged +# real pages into the ring under the current generation. +cmp_ok($node->safe_psql('postgres', 'SELECT test_dwb_ring_slots(true)'), + '>', 0, 'real pages were staged into the ring'); + +# --- the worker pool retires everything ---------------------------------- + +$node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE 'free=16 %'") + or die 'timed out waiting for the retire workers to free the ring'; +pass('retire workers returned the ring to all-free'); + +# --- pg_stat_io accounts the batch writes and fdatasyncs ----------------- + +is( $node->safe_psql( + 'postgres', + "SELECT sum(writes) > 0 AND sum(fsyncs) > 0 FROM pg_stat_io WHERE object = 'dwb'" + ), + 't', + 'pg_stat_io shows double write buffer writes and fsyncs'); + +# --- crash recovery: data intact, generation bumped ---------------------- + +# Until the Stage 4 apply-pass lands this is a WAL-replay smoke test: it +# asserts that the DWB write path corrupts nothing and replay still works, +# not that torn pages get repaired from the ring. +$node->safe_psql('postgres', + "UPDATE dwb_t SET filler = repeat('z', 300) WHERE id % 7 = 0"); +$node->stop('immediate'); +$node->start; + +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '50000', 'data intact after crash recovery'); + +# --- unlogged relations bypass the double write buffer ------------------- + +# A normal CHECKPOINT skips unlogged buffers entirely (BufferSync writes +# only BM_PERMANENT ones), so the flush that exercises the FlushBuffer gate +# is the SHUTDOWN checkpoint of a clean restart, which writes all dirty +# buffers. The assertion reads the ring itself: a broken gate would leave +# dwb_ul's tags in batch files, and no stray permanent-page flush can fake +# that. +$node->safe_psql( + 'postgres', q( + CREATE UNLOGGED TABLE dwb_ul AS + SELECT g AS id, repeat('u', 300) AS filler + FROM generate_series(1, 1000) g; + UPDATE dwb_ul SET filler = repeat('w', 300); +)); +my $ul_filenode = + $node->safe_psql('postgres', "SELECT pg_relation_filenode('dwb_ul')"); +my $t_filenode = + $node->safe_psql('postgres', "SELECT pg_relation_filenode('dwb_t')"); +my $rel_pre = $node->safe_psql('postgres', + "SELECT COALESCE(sum(writes), 0) FROM pg_stat_io " + . "WHERE object = 'relation' AND backend_type = 'checkpointer'"); +$node->restart; +cmp_ok( + $node->safe_psql( + 'postgres', + "SELECT COALESCE(sum(writes), 0) FROM pg_stat_io " + . "WHERE object = 'relation' AND backend_type = 'checkpointer'"), + '>', $rel_pre, + 'shutdown checkpoint flushed the unlogged pages'); +is( $node->safe_psql( + 'postgres', "SELECT test_dwb_ring_rel_slots($ul_filenode)"), + '0', + 'no unlogged page ever entered the ring'); +cmp_ok( + $node->safe_psql( + 'postgres', "SELECT test_dwb_ring_rel_slots($t_filenode)"), + '>', 0, + 'permanent pages did enter the ring (control)'); + +# --- the write path is self-sufficient without the worker pool ----------- + +# dwb_retire_workers = 0: writers seal immediately and retire synchronously +# in DWBFinishPageWrite; a real eviction workload must keep circulating and +# the ring must drain to all-free without any worker. +$node->append_conf('postgresql.conf', 'dwb_retire_workers = 0'); +$node->restart; +$node->safe_psql('postgres', + "UPDATE dwb_t SET filler = repeat('n', 300) WHERE id % 5 = 0"); +$node->safe_psql('postgres', 'CHECKPOINT'); +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '50000', 'workload survived the no-pool write path'); +$node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE 'free=16 %'") + or die 'timed out waiting for the ring to drain without a worker pool'; +pass('ring drained to all-free without a worker pool'); + +# The no-pool path seals every staged page right away through the +# solo-stream fast seal; the seal accounting must attribute them to it. +cmp_ok( + $node->safe_psql( + 'postgres', + "SELECT seals FROM test_dwb_seal_stats() " + . "WHERE wclass = 'eviction' AND reason = 'lone'"), + '>', 0, + 'no-pool seals are accounted as lone-writer fast seals'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/003_backpressure.pl b/src/test/modules/test_dwb/t/003_backpressure.pl new file mode 100644 index 0000000000000..918d243980a56 --- /dev/null +++ b/src/test/modules/test_dwb/t/003_backpressure.pl @@ -0,0 +1,266 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Stage B backpressure policy tests: with the ring exhausted, a stalled +# non-critical writer gets an ERROR (dwb_on_stall = error) and the cluster +# stays up, while a stalled checkpointer always PANICs by role policy and +# the cluster crash-recovers. The dwb-force-stall injection point makes +# the current wait escalate immediately instead of after +# dwb_write_timeout_ms; everything else is the production code path. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if (!defined $ENV{enable_injection_points} + || $ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('dwb_stall'); +$node->init; +# No retire workers and no background flushers: the ring stays exactly as +# exhausted as test_dwb_fill_ring() leaves it. shared_buffers is sized so +# that ONLY the deliberately oversized victim workload evicts dirty pages — +# incidental sessions (attach/detach, liveness probes) must never touch the +# exhausted ring, or they would stall in its place. +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_on_stall = error +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 1h +autovacuum = off +shared_buffers = 16MB +restart_after_crash = on +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + +# Dirty pages for the checkpointer scenario, created while the ring is +# still healthy and small enough to stay in shared_buffers. +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_dirty AS + SELECT g AS id, repeat('d', 300) AS filler + FROM generate_series(1, 1000) g; +)); + +# --- the self-help sweep admits exactly one sweeper ----------------------- + +# The dwb-self-sweep point sits INSIDE the trylock-guarded section, so a +# process can only park there after winning DWBSelfSweepLock. With the +# ring exhausted and two writers stalled, exactly one may hold the gate: +# the loser must be asleep in the ring wait, not sweeping. +$node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-self-sweep', 'wait')"); + +my $filler = $node->background_psql('postgres'); +my $taken = $filler->query_safe('SELECT test_dwb_fill_ring()'); +cmp_ok($taken, '>', 0, 'ring exhausted for the sweep-gate scenario'); + +# Two victims, each forced to evict dirty pages through the full ring. +my $victim1 = $node->background_psql('postgres'); +$victim1->query_until( + qr/starting_victim1/, q( +\echo starting_victim1 +CREATE TABLE dwb_sweep_v1 AS + SELECT g AS id, repeat('1', 300) AS filler + FROM generate_series(1, 80000) g; +)); +$node->wait_for_event('client backend', 'dwb-self-sweep'); + +my $victim2 = $node->background_psql('postgres'); +$victim2->query_until( + qr/starting_victim2/, q( +\echo starting_victim2 +CREATE TABLE dwb_sweep_v2 AS + SELECT g AS id, repeat('2', 300) AS filler + FROM generate_series(1, 80000) g; +)); + +# The loser's sleep in the ring wait proves it took the trylock-failed +# path: the winner's path parks at the injection point before any sleep. +$node->wait_for_event('client backend', 'DwbFreeBatch'); +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_stat_activity WHERE wait_event = 'dwb-self-sweep'" + ), + '1', + 'exactly one process is inside the self-help sweep gate'); + +# Detach BEFORE waking: the winner re-enters the gate on its retry loop, +# and with the point still attached it would park again with no wakeup +# left to release it. +$node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-self-sweep')"); +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('dwb-self-sweep')"); + +# Release the filler's leaked refs: the victims then finish on their own +# (no worker pool, so they seal and retire synchronously as they write). +# The abandoned batches need a seal/retire nudge from the poll, but the +# ring must NOT be required to go idle here — the victims keep batches in +# flight until they commit, so wait for their tables first. pg_class only +# shows them once the statements committed. +$filler->quit; +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "count(*) = 2 END END FROM pg_class " + . "WHERE relname IN ('dwb_sweep_v1', 'dwb_sweep_v2')") + or die 'timed out waiting for the sweep-gate victims to finish'; +is( $node->safe_psql( + 'postgres', + 'SELECT (SELECT count(*) FROM dwb_sweep_v1) + (SELECT count(*) FROM dwb_sweep_v2)' + ), + '160000', + 'both stalled victims completed after the ring drained'); +$victim1->quit; +$victim2->quit; +$node->safe_psql('postgres', 'DROP TABLE dwb_sweep_v1, dwb_sweep_v2'); + +# With the victims gone the ring drains to idle for the next scenario. +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=16 %' END END") + or die + 'timed out waiting for the ring to drain after the sweep-gate scenario'; + +# --- ERROR in a non-critical writer keeps the cluster alive -------------- + +# Attach while the ring is still healthy; the point only fires for a +# process already stuck waiting for ring space. +$node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-force-stall', 'notice')"); + +$filler = $node->background_psql('postgres'); +$taken = $filler->query_safe('SELECT test_dwb_fill_ring()'); +cmp_ok($taken, '>', 0, 'ring exhausted by leaked refs'); + +# The victim outgrows shared_buffers, so it must evict its own dirty pages +# through the exhausted ring. Its rollback drops its buffers unwritten, +# leaving the pool clean for the sessions that follow. +my ($rc, $out, $err) = $node->psql( + 'postgres', q( + CREATE TABLE dwb_victim AS + SELECT g AS id, repeat('v', 300) AS filler + FROM generate_series(1, 80000) g; +)); +isnt($rc, 0, 'stalled eviction fails instead of hanging'); +like( + $err, + qr/double write buffer retirement made no progress/, + 'stall ERROR reported to the writer'); + +is($node->safe_psql('postgres', 'SELECT 1'), + '1', 'cluster alive after the writer ERROR'); + +$node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-force-stall')"); + +# Releasing the leaked refs lets the abandoned batches finish. With no +# worker pool, drive sealing and retirement from the poll itself (nested +# CASEs order the side effects before the state probe). +$filler->quit; +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=16 %' END END") + or die 'timed out waiting for the ring to drain after the ERROR scenario'; + +# --- a stalled checkpointer PANICs by role policy ------------------------- + +# Dirty the pages for BufferSync while the ring is still healthy, THEN +# exhaust it: the UPDATE itself must not stall. +$node->safe_psql('postgres', + "UPDATE dwb_dirty SET filler = repeat('e', 300) WHERE id % 2 = 0"); + +$filler = $node->background_psql('postgres'); +$taken = $filler->query_safe('SELECT test_dwb_fill_ring()'); +cmp_ok($taken, '>', 0, 'ring exhausted again for the checkpointer scenario'); + +# An eviction fill stops at the bottom background slice, which is exactly +# the checkpointer's guaranteed lane — consume it too, or the checkpoint +# below would simply proceed through it instead of stalling. +$taken = $filler->query_safe('SELECT test_dwb_fill_ring(true)'); +cmp_ok($taken, '>', 0, 'the background lane is consumed as well'); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-force-stall', 'notice')"); + +my $log_offset = -s $node->logfile; +($rc, $out, $err) = $node->psql('postgres', 'CHECKPOINT'); +isnt($rc, 0, 'CHECKPOINT fails when the checkpointer PANICs'); + +# probing with psql during the restart window trips over dying sockets; +# wait for the crash-recovery cycle in the log instead +$node->wait_for_log(qr/database system is ready to accept connections/, + $log_offset); +pass('cluster restarted after the checkpointer PANIC'); +ok( $node->log_contains( + 'double write buffer retirement made no progress', $log_offset), + 'checkpointer stall escalated to the role-policy PANIC'); + +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_dirty'), + '1000', 'data intact after crash recovery'); + +# --- Stage A warning fires on the real clock ------------------------------ + +# No injection point this time: shrink the real thresholds and let a victim +# writer walk through Stage A (WARNING after dwb_slow_warn_ms) into Stage B +# (ERROR after dwb_write_timeout_ms, dwb_on_stall = error). The bgwriter +# pause of Stage A has no SQL-visible probe and stays untested here. +$node->append_conf( + 'postgresql.conf', qq( +dwb_slow_warn_ms = 100 +dwb_write_timeout_ms = 1000 +)); +$node->reload; + +$filler = $node->background_psql('postgres'); +$taken = $filler->query_safe('SELECT test_dwb_fill_ring()'); +cmp_ok($taken, '>', 0, 'ring exhausted for the slow-warn scenario'); + +my $retries0 = + $node->safe_psql('postgres', 'SELECT test_dwb_ring_wait_retries()'); + +($rc, $out, $err) = $node->psql('postgres', 'SELECT test_dwb_cycle(1)'); +isnt($rc, 0, 'victim writer errors out on the real stall clock'); + +# Anti-spin regression: nothing woke the victim during its ~1s of waiting +# (no retire, no leader write), so its wait iterations must be paced by the +# 1s sleep timeout — a handful, not the thousands a self-waking rotation of +# the probe-released staging buffer would produce. +my $retries1 = + $node->safe_psql('postgres', 'SELECT test_dwb_ring_wait_retries()'); +cmp_ok($retries1 - $retries0, '>=', 1, + 'the stalled victim slept in the wait'); +cmp_ok($retries1 - $retries0, + '<=', 10, 'ring wait paced by the sleep timeout, not a busy rotation'); +like( + $err, + qr/double write buffer has no free batch after/, + 'Stage A warning reached the writer'); +like( + $err, + qr/double write buffer retirement made no progress/, + 'Stage B error reached the writer'); + +$filler->quit; +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=16 %' END END") + or die + 'timed out waiting for the ring to drain after the slow-warn scenario'; + +done_testing(); diff --git a/src/test/modules/test_dwb/t/004_retire_paths.pl b/src/test/modules/test_dwb/t/004_retire_paths.pl new file mode 100644 index 0000000000000..882156172aeb3 --- /dev/null +++ b/src/test/modules/test_dwb/t/004_retire_paths.pl @@ -0,0 +1,172 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Retirement fallback paths: the checkpointer's ProcessSyncRequests as the +# only retire point, and the synchronous OOM retire when DWSegmentHash +# overflows. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_retire'); +$node->init; +# The default 64x64 geometry lets RETIRING batches pin more distinct +# segments (64 x 64 = 4096) than the smallest segment hash can hold; no +# workers and no background flushers keep retirement fully under the test's +# control. fsync must be ON: with fsync = off ProcessSyncRequests skips +# its whole per-file block, including the DWBSegmentFsyncBegin/End wrap +# this test exists to exercise. +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 64 +dwb_batch_pages = 64 +dwb_max_segments = 1024 +dwb_retire_workers = 0 +dwb_retire_sync_method = fsync +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 1h +autovacuum = off +fsync = on +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# --- a CHECKPOINT alone retires a batch (ProcessSyncRequests path) ------- + +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_ckpt AS + SELECT g AS id, repeat('c', 64) AS pad FROM generate_series(1, 100) g; +)); +my $filenode = + $node->safe_psql('postgres', "SELECT pg_relation_filenode('dwb_ckpt')"); + +# Warmup: run the same statements once and CHECKPOINT, so every catalog +# hint bit they dirty is flushed now. The real run below then leaves no +# dirty buffer behind, the final CHECKPOINT has nothing to feed through the +# DWB write path (whose no-pool finish would retire our batch as a side +# effect), and only the DWBSegmentFsyncBegin/End wrap of +# ProcessSyncRequests can free the parked batch. +$node->safe_psql('postgres', "SELECT test_dwb_checkpoint_pending($filenode)"); +$node->safe_psql('postgres', 'SELECT test_dwb_states()'); +$node->safe_psql('postgres', 'CHECKPOINT'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/^free=64 /, 'warmup batch retired'); + +$node->safe_psql('postgres', "SELECT test_dwb_checkpoint_pending($filenode)"); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/retiring=1$/, + 'one batch parked in RETIRING with a pending sync request'); +$node->safe_psql('postgres', 'CHECKPOINT'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/^free=64 /, 'CHECKPOINT alone retired the parked batch'); + +# --- a checkpoint tolerates a live ALLOCATED batch ------------------------ + +# Checkpoints take no DWB barrier. With no retire workers nothing seals +# behind our back, so the holder's open batch must stay ALLOCATED across a +# CHECKPOINT; the test seals and retires it explicitly once the holder is +# gone. +# +# This doubles as the regression test for cross-class open-pointer +# aliasing: the checkpoints above left open_batch_idx[BACKGROUND] naming a +# long-freed batch index, the holder's EVICTION-class open reuses exactly +# that index (lowest FREE), and the CHECKPOINT below makes the checkpointer +# flush the holder's login hint bits through the DWB. Without the writer +# class stamp in next_slot_idx the checkpointer would join the holder's +# batch and, on the no-pool path, seal it. +my $holder = $node->background_psql('postgres'); +$holder->query_safe('SELECT test_dwb_leak(1, true)'); +my $one_open = + 'free=63 allocated=1 sealed=0 written=0 fsynced=0 data_written=0 retiring=0'; +is($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + $one_open, 'an open ALLOCATED batch is live before the checkpoint'); +$node->safe_psql('postgres', 'CHECKPOINT'); +is($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + $one_open, 'CHECKPOINT completed and left the open batch alone'); +$holder->quit; +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=64 %' END END") + or die + 'timed out waiting for the open batch to drain after the holder quit'; +pass('abandoned open batch drained'); + +# --- segment hash overflow degrades to synchronous retire ---------------- + +my $log_offset = -s $node->logfile; +my ($rc, $out, $err) = + $node->psql('postgres', 'SELECT test_dwb_fill_segments(30)'); +is($rc, 0, 'segment fill survived the hash overflow'); +is($out, 30 * 64, 'thirty batches of unique segments published'); +like( + $err, + qr/double write buffer segment hash is full/, + 'hash overflow warning reached the publisher'); +ok( $node->log_contains( + 'double write buffer segment hash is full', $log_offset), + 'hash overflow logged'); + +# The batches parked in RETIRING drain through the normal sweep (dropped +# fake segments count as covered); the OOM-retired ones are already free. +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=64 %' END") + or die 'timed out waiting for the ring to drain after the hash overflow'; +pass('ring drained after the hash overflow'); + +# --- a leftover fsync snapshot must not be consumed by a foreign End ----- + +# test_dwb_stale_snapshot replays the checkpointer hazard: Begin for the +# parked segment without the matching End (the state an fsync ERROR under +# data_sync_retry = on leaves behind), then a successful Begin/End of an +# unrelated non-MD entry. The parked batch must still be RETIRING — a +# consumed stale snapshot would have freed it without durability. +$node->safe_psql('postgres', 'SELECT test_dwb_park(98000)'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/retiring=1$/, 'batch parked for the stale-snapshot scenario'); +$node->safe_psql('postgres', 'SELECT test_dwb_stale_snapshot(98000)'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/retiring=1$/, 'stale snapshot dropped, parked batch still RETIRING'); +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_retire() >= 0 THEN " + . "test_dwb_states() LIKE 'free=64 %' END") + or die 'timed out waiting for the stale-snapshot batch to drain'; + +# --- a soft retire-fsync failure keeps the batch and releases the claim -- + +# With data_sync_retry = on a failed segment fsync must not throw: the +# batch stays RETIRING for a later retry and the advisory claim is +# released. A directory planted at the fake segment's path makes the +# fsync fail deterministically (EISDIR); removing it lets the next sweep +# cover the segment — which only works if the failed attempt released the +# claim. +$node->append_conf('postgresql.conf', 'data_sync_retry = on'); +$node->restart; + +my $segdir = $node->data_dir . '/base/1/99000'; +mkdir $segdir or die "mkdir $segdir: $!"; + +$node->safe_psql('postgres', 'SELECT test_dwb_park(99000)'); +($rc, $out, $err) = $node->psql('postgres', 'SELECT test_dwb_retire()'); +is($rc, 0, 'retire sweep survives the failing segment fsync'); +is($out, '0', 'no batch freed while the segment fsync fails'); +like( + $err, + qr/could not fsync file/, + 'soft fsync failure reported as a WARNING'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/retiring=1$/, 'batch stays RETIRING after the soft fsync failure'); + +rmdir $segdir or die "rmdir $segdir: $!"; +is($node->safe_psql('postgres', 'SELECT test_dwb_retire()'), + '1', 'released claim lets the next sweep cover the segment'); +like($node->safe_psql('postgres', 'SELECT test_dwb_states()'), + qr/^free=64 /, 'ring drained after the soft-failure scenario'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/005_standby.pl b/src/test/modules/test_dwb/t/005_standby.pl new file mode 100644 index 0000000000000..ad2fa036b542d --- /dev/null +++ b/src/test/modules/test_dwb/t/005_standby.pl @@ -0,0 +1,272 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Hot standby under io_torn_pages_protection = double_writes: the standby +# runs its own ring while replaying, both sides survive crashes, promotion +# drains a replay backlog through the ring, the minRecoveryPoint contract +# holds for replay-driven flushes, and a base backup initiated on the +# standby is refused loudly. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use lib $FindBin::RealBin; +use DWBTest; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('dwb_primary'); +$primary->init(allows_streaming => 1); +# The workload table (~3.4 MB) exceeds shared_buffers, so replay on the +# standby must evict through its ring; fsync stays ON so the restartpoint +# ProcessSyncRequests path runs its DWB wrap for real. +$primary->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_batch_timeout_ms = 20 +shared_buffers = 2MB +autovacuum = off +fsync = on +)); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# --- a standby provisioned from a base backup cold-starts its ring ------- + +$primary->backup('bkp'); +my $standby = PostgreSQL::Test::Cluster->new('dwb_standby'); +$standby->init_from_backup($primary, 'bkp', has_streaming => 1); +my $standby_log_offset = (-s $standby->logfile) // 0; +$standby->start; + +# pg_dwb/ is excluded from the backup, so the standby must open a fresh +# ring rather than inherit the primary's. +ok( $standby->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 1\b/, + $standby_log_offset), + 'standby cold-started a fresh ring from the base backup'); + +# --- the retire worker pool runs during recovery ------------------------- + +$standby->poll_query_until('postgres', + "SELECT count(*) = 1 FROM pg_stat_activity WHERE backend_type = 'dwb retire worker'" +) or die 'timed out waiting for the standby retire worker to start'; +pass('retire worker is running on the standby during recovery'); + +# --- replay traffic flows through the standby ring ----------------------- + +$primary->safe_psql( + 'postgres', q( + CREATE TABLE dwb_t AS + SELECT g AS id, repeat('x', 300) AS filler + FROM generate_series(1, 10000) g; + UPDATE dwb_t SET filler = repeat('y', 300) WHERE id % 10 = 0; +)); +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($standby); + +# The workload exceeds the standby's shared_buffers, so the startup +# process itself must have evicted dirty pages through the standby's own +# DWB write path. Startup flushes its stats when it replays the +# XLOG_RUNNING_XACTS record the primary's CHECKPOINT above emitted, but +# that is asynchronous to wait_for_catchup — hence the poll. +$standby->poll_query_until('postgres', + "SELECT COALESCE(sum(writes), 0) > 0 FROM pg_stat_io " + . "WHERE object = 'dwb' AND backend_type = 'startup'") + or die 'timed out waiting for startup-process DWB writes on the standby'; +pass('replay evictions flowed through the standby ring'); + +is( $standby->safe_psql( + 'postgres', + "SELECT count(*) FROM dwb_t WHERE filler = repeat('y', 300)"), + '1000', + 'replayed page contents are correct'); + +# --- FlushBuffer on the standby advances minRecoveryPoint ---------------- + +# Take the baseline right after a restartpoint, then push replay-eviction +# traffic with NO further checkpoint or restartpoint anywhere: any advance +# past the baseline can then come only from buffer flushes — XLogFlush in +# recovery does not fsync WAL, it calls UpdateMinRecoveryPoint instead +# (see 3.9 of the design plan). +$standby->safe_psql('postgres', 'CHECKPOINT'); +my $mrp_before = $standby->safe_psql('postgres', + 'SELECT min_recovery_end_lsn FROM pg_control_recovery()'); +$primary->safe_psql('postgres', + "UPDATE dwb_t SET filler = repeat('m', 300) WHERE id % 9 = 0"); +$primary->wait_for_catchup($standby); +$standby->poll_query_until('postgres', + "SELECT min_recovery_end_lsn > '$mrp_before'::pg_lsn FROM pg_control_recovery()" + ) + or die 'minRecoveryPoint did not advance from replay-driven flushes alone'; +pass( + 'replay-driven flushes advanced minRecoveryPoint without a restartpoint'); + +# and the ring keeps circulating: the worker drains it back to all-free +$standby->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE 'free=16 %'") + or die 'timed out waiting for the standby ring to drain'; +pass('standby ring drained back to all-free'); + +# --- a base backup initiated on the standby is refused loudly ------------ + +# The replayed WAL carries no page images and the primary's ring cannot +# substitute for them, so the vanilla do_pg_backup_start guard must refuse +# with a hint that names the real knob. +my $refused_path = $primary->backup_dir . '/standby_backup'; +my ($out, $err) = run_command( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $refused_path, + '--host' => $standby->host, + '--port' => $standby->port, + '--checkpoint' => 'fast' + ]); +ok(!-f "$refused_path/PG_VERSION", 'base backup from the standby is refused'); +like($err, qr/WAL generated without full page images was replayed/, + '... loudly'); +like($err, qr/io_torn_pages_protection/, + '... with a hint naming the real knob'); + +# --- the standby survives its own crash ---------------------------------- + +$standby->stop('immediate'); +$standby_log_offset = -s $standby->logfile; +$standby->start; +ok( $standby->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 2\b/, + $standby_log_offset), + 'crashed standby reopened its ring under a bumped generation'); + +$primary->safe_psql('postgres', + "INSERT INTO dwb_t VALUES (100001, 'after standby crash')"); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '10001', 'replication resumed after the standby crash'); + +# --- the primary survives its own crash ---------------------------------- + +$primary->stop('immediate'); +$primary->start; +$primary->safe_psql('postgres', + "INSERT INTO dwb_t VALUES (100002, 'after primary crash')"); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '10002', 'replication resumed after the primary crash'); + +# --- a torn page on the standby is repaired by its own apply-pass --------- + +# The apply-pass on a crashed standby runs against pg_control state +# DB_IN_ARCHIVE_RECOVERY — the branch that may raise minRecoveryPoint — +# and must repair from the standby's OWN ring: the replayed WAL carries no +# page images that could do it instead. +$primary->safe_psql( + 'postgres', q( + CREATE TABLE ts_repair AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($standby); +my $ts_path = + $primary->safe_psql('postgres', "SELECT pg_relation_filepath('ts_repair')"); +my $ts_relnum = $primary->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'ts_repair'"); + +# a restartpoint flushes the replayed pages through the standby's ring; +# crash right after, while the table's slot is still on disk +$standby->append_conf('postgresql.conf', 'log_min_messages = debug1'); +$standby->safe_psql('postgres', 'CHECKPOINT'); +$standby->stop('immediate'); + +my $ts_file = $standby->data_dir . '/' . $ts_path; +write_block($ts_file, 0, + substr(read_block($ts_file, 0), 0, 4096) . ("\0" x 4096)); + +$standby_log_offset = -s $standby->logfile; +$standby->start; +ok( $standby->log_contains( + qr!restoring page 0 of relation \d+/\d+/$ts_relnum fork 0!, + $standby_log_offset), + 'the crashed standby repaired its torn page from its own ring'); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM ts_repair'), + '100', 'the repaired standby page reads whole'); + +# --- promotion with a replay backlog ------------------------------------- + +# Pause replay, pile up a burst, make sure it is flushed to the standby's +# local WAL, and promote with the pause still in effect: promotion breaks +# the pause (recoveryPausesHere exits on the standby trigger), so the +# whole backlog demonstrably replays through the standby's DWB write path +# before the timeline switch. +$standby->safe_psql('postgres', 'SELECT pg_wal_replay_pause()'); +$standby->poll_query_until('postgres', + "SELECT pg_get_wal_replay_pause_state() = 'paused'") + or die 'timed out waiting for replay to pause'; +$primary->safe_psql( + 'postgres', q( + UPDATE dwb_t SET filler = repeat('p', 300) WHERE id % 3 = 0; + INSERT INTO dwb_t VALUES (100003, 'burst tail'); +)); +$primary->wait_for_catchup($standby, 'flush', $primary->lsn('write')); +is( $standby->safe_psql( + 'postgres', + 'SELECT pg_last_wal_replay_lsn() < pg_last_wal_receive_lsn()'), + 't', + 'a real replay backlog exists at promotion time'); +$standby->promote; + +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '10003', 'promoted standby replayed the whole backlog'); +is( $standby->safe_psql( + 'postgres', + "SELECT count(*) FROM dwb_t WHERE filler = repeat('p', 300)"), + '3334', + 'backlog page contents are correct'); +is($standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 'f', 'standby left recovery'); + +# --- the promoted node is a full DWB primary ----------------------------- + +my $tl2_start = + $standby->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +$standby->safe_psql( + 'postgres', q( + UPDATE dwb_t SET filler = repeat('q', 300) WHERE id % 5 = 0; + INSERT INTO dwb_t VALUES (100004, 'after promotion'); +)); +my $tl2_end = $standby->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +$standby->safe_psql('postgres', 'CHECKPOINT'); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_t'), + '10004', 'promoted node accepts writes'); +$standby->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE 'free=16 %'") + or die 'timed out waiting for the promoted ring to drain'; +pass('promoted ring drained back to all-free'); + +# the new timeline still carries no page images +my ($waldump, $walerr) = run_command( + [ + 'pg_waldump', + '--path' => $standby->data_dir . '/pg_wal', + '--timeline' => 2, + '--start' => $tl2_start, + '--end' => $tl2_end + ]); +# pg_current_wal_lsn() above may land exactly on a WAL page boundary +# while concurrent records (imageless FPI_FOR_HINT from the count(*) +# checks) are still being inserted: the write position advances in +# whole pages. +# pg_waldump then skips the page header to the first whole record and +# reports that with a benign informational line on stderr. Tolerate +# exactly that line; anything else on stderr is still a real failure. +$walerr =~ + s/^pg_waldump: first record is after \S+, at \S+, skipping over \d+ bytes?\n?//; +is($walerr, '', 'pg_waldump read the post-promotion window cleanly'); +like($waldump, qr/Heap/, 'the window covers the post-promotion update'); +unlike($waldump, qr/\bFPW\b/, 'no full-page images after promotion'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/006_backup.pl b/src/test/modules/test_dwb/t/006_backup.pl new file mode 100644 index 0000000000000..b2fbff633252b --- /dev/null +++ b/src/test/modules/test_dwb/t/006_backup.pl @@ -0,0 +1,250 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Online backup under io_torn_pages_protection = double_writes: WAL carries +# no full-page images in normal running, but an active backup forces them +# back on (the ring only repairs local torn writes; a backup copied +# mid-write can hold a torn page that only WAL replay with FPIs can +# repair). The backup itself excludes pg_dwb/ contents, tolerates pg_dwb +# being a symlink, and a cluster restored from it cold-starts a fresh ring. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_backup'); +$node->init(allows_streaming => 1); +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_batch_timeout_ms = 20 +autovacuum = off +)); +$node->start; + +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_fpi AS + SELECT g AS id, repeat('f', 64) AS pad FROM generate_series(1, 100) g; +)); +my $fpi_filenode = + $node->safe_psql('postgres', "SELECT pg_relation_filenode('dwb_fpi')"); + +# --- normal running writes no full-page images --------------------------- + +# The first touch of a page after a checkpoint is exactly where an FPI +# would go; under double_writes none may appear. Hint-bit records +# (FPI_FOR_HINT) can still show up in the window, but without doPageWrites +# they carry no page image and thus no FPW block flag — the probe below +# stays meaningful. +$node->safe_psql('postgres', 'CHECKPOINT'); +my $lsn0 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +$node->safe_psql('postgres', + "UPDATE dwb_fpi SET pad = repeat('a', 64) WHERE id = 1"); +my $lsn1 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); + +my ($waldump, $walerr) = run_command( + [ + 'pg_waldump', + '--path' => $node->data_dir . '/pg_wal', + '--start' => $lsn0, + '--end' => $lsn1 + ]); +is($walerr, '', 'pg_waldump read the no-backup window cleanly'); +like($waldump, qr/Heap/, 'the WAL window covers the update'); +unlike($waldump, qr/\bFPW\b/, 'no full-page image outside a backup'); + +# --- an active backup forces full-page images back on --------------------- + +# pg_backup_start checkpoints and raises runningBackups; the first touch +# of any page after that must carry an FPI (doPageWrites = +# Insert->fullPageWrites || runningBackups > 0). +my $bk = $node->background_psql('postgres'); +$bk->query_safe('SET client_min_messages = warning'); +$bk->query_safe("SELECT pg_backup_start('dwb_fpi_probe', true)"); + +my $lsn2 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +$node->safe_psql('postgres', + "UPDATE dwb_fpi SET pad = repeat('b', 64) WHERE id = 2"); +my $lsn3 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); + +($waldump, $walerr) = run_command( + [ + 'pg_waldump', + '--path' => $node->data_dir . '/pg_wal', + '--start' => $lsn2, + '--end' => $lsn3 + ]); +is($walerr, '', 'pg_waldump read the backup window cleanly'); +like( + $waldump, + qr!rel \d+/\d+/$fpi_filenode .* FPW!, + 'an active backup forces a full-page image of the touched page'); + +$bk->query_safe('SELECT pg_backup_stop()'); +$bk->quit; + +# --- the backup keeps pg_dwb as an empty directory ------------------------ + +# guard against a vacuous emptiness assert: the source ring is non-empty +ok( -f $node->data_dir . '/pg_dwb/control', + 'the source cluster has ring files to exclude'); + +my $backup_path = $node->backup_dir . '/content_check'; +my ($out, $err) = run_command( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $backup_path, + '--host' => $node->host, + '--port' => $node->port, + '--checkpoint' => 'fast' + ]); +ok(-f "$backup_path/PG_VERSION", 'backup completed'); +unlike( + $err, + qr/WARNING|skipping special file/, + 'pg_basebackup issued no warnings'); +ok(-d "$backup_path/pg_dwb", 'backup contains a pg_dwb directory'); +{ + opendir(my $dh, "$backup_path/pg_dwb") or die "opendir: $!"; + my @entries = grep { !/^\.\.?$/ } readdir($dh); + closedir($dh); + is(scalar(@entries), 0, 'the pg_dwb directory in the backup is empty'); +} + +# --- a cluster restored from the backup cold-starts its ring -------------- + +my $restored = PostgreSQL::Test::Cluster->new('dwb_restored'); +$restored->init_from_backup($node, 'content_check'); +my $restored_log_offset = -s $restored->logfile; +$restored->start; +ok( $restored->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 1\b/, + $restored_log_offset), + 'restored cluster opened a fresh ring'); +ok( !$restored->log_contains( + qr/discarding double write buffer ring contents/, + $restored_log_offset), + '... without claiming to discard the empty restored pg_dwb'); +is($restored->safe_psql('postgres', 'SELECT count(*) FROM dwb_fpi'), + '100', 'restored data is intact'); +$restored->stop; + +# --- a ring shipped into a restore by a foreign tool is discarded --------- + +# pg_basebackup excludes the ring, but a third-party backup tool may ship +# pg_dwb/ contents into the restore. Neither staleness defence works +# there — the slots carry the restored control's own generation, and the +# restored data files are legitimately older than the slot copies — so a +# start from a base backup (backup_label present) must not apply the ring: +# it is wiped and recreated cold. The planted control file is short +# garbage; reading it would be a fatal "read 16 of 40" error, so this also +# pins that the wipe comes before any ring-state read. +my $planted = PostgreSQL::Test::Cluster->new('dwb_planted'); +$planted->init_from_backup($node, 'content_check'); +ok( -f $planted->data_dir . '/backup_label', + 'the restore still carries backup_label'); +append_to_file($planted->data_dir . '/pg_dwb/control', 'torn by the tool'); +append_to_file($planted->data_dir . '/pg_dwb/batch_9999', 'foreign slots'); + +my $planted_log_offset = -s $planted->logfile; +$planted->start; +ok( $planted->log_contains( + qr/discarding double write buffer ring contents restored from a base backup/, + $planted_log_offset), + 'the restored ring is discarded'); +ok( !$planted->log_contains( + qr/double write buffer recovery:/, + $planted_log_offset), + '... without an apply-pass over it'); +ok( $planted->log_contains( + qr/ring opened: 16 batches of 16 pages, generation 1\b/, + $planted_log_offset), + '... and a fresh ring is created cold'); +ok(!-f $planted->data_dir . '/pg_dwb/batch_9999', + 'the foreign ring files are gone'); +is($planted->safe_psql('postgres', 'SELECT count(*) FROM dwb_fpi'), + '100', 'restored data is intact'); + +# once the backup recovery is over the guard is gone: an ordinary crash +# of this cluster is served by the apply-pass again +$planted->stop('immediate'); +$planted_log_offset = -s $planted->logfile; +$planted->start; +ok( $planted->log_contains( + qr/double write buffer recovery: \d+ of \d+ candidate pages restored/, + $planted_log_offset), + 'a later crash of the restored cluster applies the ring normally'); +$planted->stop; + +# --- the wipe also covers restores in the non-ring modes ------------------ + +# A planted ring is dangerous even to a cluster restored under +# "full_pages": left dormant, it would greet a much later switch to +# double_writes with a plausible control file. The restore start must +# discard it — before any ring-state read, and without tripping the +# downgrade guard on it. +my $planted_fp = PostgreSQL::Test::Cluster->new('dwb_planted_fp'); +$planted_fp->init_from_backup($node, 'content_check'); +$planted_fp->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +append_to_file($planted_fp->data_dir . '/pg_dwb/control', 'torn by the tool'); +append_to_file($planted_fp->data_dir . '/pg_dwb/batch_9999', 'foreign slots'); + +my $planted_fp_log_offset = -s $planted_fp->logfile; +$planted_fp->start; +ok( $planted_fp->log_contains( + qr/discarding double write buffer ring contents restored from a base backup/, + $planted_fp_log_offset), + 'a full_pages restore discards the planted ring too'); +ok(!-f $planted_fp->data_dir . '/pg_dwb/batch_9999', + '... removing the foreign files'); +ok(!$planted_fp->log_contains(qr/ring opened/, $planted_fp_log_offset), + '... without creating a ring it will not use'); +is($planted_fp->safe_psql('postgres', 'SELECT count(*) FROM dwb_fpi'), + '100', 'restored data is intact under full_pages'); +$planted_fp->stop; + +# --- pg_dwb as a symlink backs up as an empty real directory -------------- + +SKIP: +{ + skip 'symlinks are not portable to Windows', 4 if $windows_os; + + $node->stop; + my $dwb_store = $node->basedir . '/dwb_store'; + rename($node->data_dir . '/pg_dwb', $dwb_store) + or die "rename pg_dwb: $!"; + symlink($dwb_store, $node->data_dir . '/pg_dwb') + or die "symlink pg_dwb: $!"; + $node->start; + + my $link_backup = $node->backup_dir . '/symlink_check'; + ($out, $err) = run_command( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $link_backup, + '--host' => $node->host, + '--port' => $node->port, + '--checkpoint' => 'fast' + ]); + ok(-f "$link_backup/PG_VERSION", + 'backup of the symlinked ring completed'); + unlike( + $err, + qr/WARNING|skipping special file/, + 'no warnings for the symlinked pg_dwb'); + ok( -d "$link_backup/pg_dwb" && !-l "$link_backup/pg_dwb", + 'symlinked pg_dwb became a real directory in the backup'); + opendir(my $dh, "$link_backup/pg_dwb") or die "opendir: $!"; + my @entries = grep { !/^\.\.?$/ } readdir($dh); + closedir($dh); + is(scalar(@entries), 0, 'the symlinked pg_dwb backed up empty'); +} + +done_testing(); diff --git a/src/test/modules/test_dwb/t/007_rewind.pl b/src/test/modules/test_dwb/t/007_rewind.pl new file mode 100644 index 0000000000000..9b6e30031128a --- /dev/null +++ b/src/test/modules/test_dwb/t/007_rewind.pl @@ -0,0 +1,231 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# pg_rewind on a double_writes pair: a live source is refused (its WAL has +# no full-page images to repair pages read mid-write), a stopped source +# works, the target's own ring is wiped by the rewind — also when pg_dwb +# is a symlink — and the rewound node cold-starts a fresh ring and follows +# the promoted primary. A regular file in place of pg_dwb is refused, and +# a broken ring symlink on the source is ignored. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node_a = PostgreSQL::Test::Cluster->new('dwb_rewind_a'); +$node_a->init(allows_streaming => 1); +$node_a->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_batch_timeout_ms = 20 +autovacuum = off +wal_keep_size = 64MB +)); +$node_a->start; +$node_a->safe_psql( + 'postgres', q( + CREATE TABLE dwb_r AS SELECT g AS id FROM generate_series(1, 100) g; +)); + +$node_a->backup('bkp'); +my $node_b = PostgreSQL::Test::Cluster->new('dwb_rewind_b'); +$node_b->init_from_backup($node_a, 'bkp', has_streaming => 1); +$node_b->start; +$node_a->wait_for_catchup($node_b); + +# --- diverge the timelines ----------------------------------------------- + +$node_b->promote; +$node_b->safe_psql('postgres', "INSERT INTO dwb_r VALUES (100001)"); +# A keeps running as the old primary and diverges past the fork point. +# Crash it: the refusal scenarios below must fire before pg_rewind's +# single-user recovery of the target gets a chance to run. +$node_a->safe_psql('postgres', "INSERT INTO dwb_r VALUES (200001)"); +$node_a->stop('immediate'); + +# --- relocate the target ring behind a symlink --------------------------- + +# Like pg_wal, pg_dwb may be a symlink to a directory on other storage; +# the rewind must wipe the ring through the link and leave the link +# itself in place. Creating symlinks requires a privilege on Windows, +# so the plain-directory layout is exercised there instead. +my $dwb_is_symlinked = 0; +unless ($windows_os) +{ + my $ring_home = PostgreSQL::Test::Utils::tempdir('dwb_ring'); + my $dwb_path = $node_a->data_dir . '/pg_dwb'; + rename($dwb_path, "$ring_home/pg_dwb") + or BAIL_OUT("could not move $dwb_path: $!"); + symlink("$ring_home/pg_dwb", $dwb_path) + or BAIL_OUT("could not symlink $dwb_path: $!"); + $dwb_is_symlinked = 1; +} + +# --- a live double_writes source is refused ------------------------------ + +command_fails_like( + [ + 'pg_rewind', + '--target-pgdata' => $node_a->data_dir, + '--source-server' => $node_b->connstr('postgres') + ], + qr/"io_torn_pages_protection" must be "full_pages" in the source server/, + 'pg_rewind refuses a live source that writes no full-page images'); + +# --- a stopped source works and wipes the target ring -------------------- + +# leave proof on the target that the rewind, not a later cold start, +# removed the ring files +ok( -f $node_a->data_dir . '/pg_dwb/control', + 'the target has ring files before the rewind'); + +$node_b->stop('fast'); + +my @rewind_from_b = ( + 'pg_rewind', + '--target-pgdata' => $node_a->data_dir, + '--source-pgdata' => $node_b->data_dir); + +# --- a garbage entry in place of pg_dwb is refused ----------------------- + +# Anything but a directory, a symlink or nothing at all would survive the +# rewind only to fail the next double_writes startup, so the rewind must +# reject it before touching the target. +{ + my $dwb_path = $node_a->data_dir . '/pg_dwb'; + my $stash = $node_a->basedir . '/pg_dwb_stash'; + rename($dwb_path, $stash) or BAIL_OUT("could not move $dwb_path: $!"); + append_to_file($dwb_path, "not a ring\n"); + command_fails_like( + [@rewind_from_b], + qr/"pg_dwb" in target is not a directory or symbolic link/, + 'pg_rewind refuses a regular file in place of pg_dwb'); + unlink($dwb_path) or BAIL_OUT("could not remove $dwb_path: $!"); + + unless ($windows_os) + { + require POSIX; + + # a FIFO never even reaches the file map, so it takes the up-front + # check + POSIX::mkfifo($dwb_path, 0700) + or BAIL_OUT("could not create FIFO $dwb_path: $!"); + command_fails_like( + [@rewind_from_b], + qr/"pg_dwb" in target is not a directory or symbolic link/, + 'pg_rewind refuses a FIFO in place of pg_dwb'); + unlink($dwb_path) or BAIL_OUT("could not remove $dwb_path: $!"); + + # a symlink is only as good as what it points to + symlink('/nonexistent/dwb_ring_target', $dwb_path) + or BAIL_OUT("could not symlink $dwb_path: $!"); + command_fails_like( + [@rewind_from_b], + qr/"pg_dwb" in target is a symbolic link that does not point to a directory/, + 'pg_rewind refuses a broken pg_dwb symlink'); + unlink($dwb_path) or BAIL_OUT("could not remove $dwb_path: $!"); + + append_to_file("$stash.file", "not a ring\n"); + symlink("$stash.file", $dwb_path) + or BAIL_OUT("could not symlink $dwb_path: $!"); + command_fails_like( + [@rewind_from_b], + qr/"pg_dwb" in target is a symbolic link that does not point to a directory/, + 'pg_rewind refuses a pg_dwb symlink to a regular file'); + unlink($dwb_path) or BAIL_OUT("could not remove $dwb_path: $!"); + unlink("$stash.file") or BAIL_OUT("could not remove $stash.file: $!"); + + # garbage inside the ring is refused too: it would silently survive + # the wipe + mkdir($dwb_path) or BAIL_OUT("could not create $dwb_path: $!"); + POSIX::mkfifo("$dwb_path/control", 0700) + or BAIL_OUT("could not create FIFO $dwb_path/control: $!"); + command_fails_like( + [@rewind_from_b], + qr!"pg_dwb/control" in target is not a regular file!, + 'pg_rewind refuses a FIFO inside pg_dwb'); + unlink("$dwb_path/control") + or BAIL_OUT("could not remove $dwb_path/control: $!"); + rmdir($dwb_path) or BAIL_OUT("could not remove $dwb_path: $!"); + } + + # the crashed target was left alone throughout: every refusal came + # before the single-user recovery run + command_like( + [ 'pg_controldata', $node_a->data_dir ], + qr/Database cluster state:\s+in production/, + 'the refusals precede the single-user recovery of the target'); + + rename($stash, $dwb_path) or BAIL_OUT("could not restore $dwb_path: $!"); +} + +# The target stays crashed on purpose: the rewind itself drives it to a +# clean shutdown through ensureCleanShutdown's single-user run, which +# exercises the apply-pass and the ring drain with no worker pool at all — +# and with the ring already behind the symlink. + +# --- a broken ring symlink on the source is harmless --------------------- + +# The source's ring is never used, so only the target's pg_dwb link may be +# entered. Point the source's at nowhere and let the rewind below prove +# it: if the traversal followed the link, it would fail outright. +my $b_dwb_stash; +unless ($windows_os) +{ + my $b_dwb = $node_b->data_dir . '/pg_dwb'; + $b_dwb_stash = $node_b->basedir . '/pg_dwb_stash'; + rename($b_dwb, $b_dwb_stash) or BAIL_OUT("could not move $b_dwb: $!"); + symlink('/nonexistent/dwb_ring', $b_dwb) + or BAIL_OUT("could not symlink $b_dwb: $!"); +} + +command_ok([@rewind_from_b], 'pg_rewind from a stopped source succeeds'); + +ok(-d $node_a->data_dir . '/pg_dwb', + 'the target still has a pg_dwb directory'); +{ + opendir(my $dh, $node_a->data_dir . '/pg_dwb') or die "opendir: $!"; + my @entries = grep { !/^\.\.?$/ } readdir($dh); + closedir($dh); + is(scalar(@entries), 0, 'the rewind wiped the target ring'); +} +ok(-l $node_a->data_dir . '/pg_dwb', 'the pg_dwb symlink survived the rewind') + if $dwb_is_symlinked; + +# --- the rewound node cold-starts a ring and follows the new primary ----- + +if (defined $b_dwb_stash) +{ + my $b_dwb = $node_b->data_dir . '/pg_dwb'; + unlink($b_dwb) or BAIL_OUT("could not remove $b_dwb: $!"); + rename($b_dwb_stash, $b_dwb) or BAIL_OUT("could not restore $b_dwb: $!"); +} +$node_b->start; +# the rewind copied the source's configuration; restore this node's port +$node_a->append_conf('postgresql.conf', 'port = ' . $node_a->port); +$node_a->enable_streaming($node_b); +my $a_log_offset = -s $node_a->logfile; +$node_a->start; +ok( $node_a->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 1\b/, + $a_log_offset), + 'rewound node cold-started a fresh ring'); + +$node_b->wait_for_catchup($node_a); +is($node_a->safe_psql('postgres', 'SELECT count(*) FROM dwb_r'), + '101', 'rewound node converged on the new primary timeline'); +is( $node_a->safe_psql( + 'postgres', 'SELECT count(*) FROM dwb_r WHERE id = 200001'), + '0', + 'the divergent row is gone'); +is( $node_a->safe_psql( + 'postgres', 'SELECT count(*) FROM dwb_r WHERE id = 100001'), + '1', + "the new primary's row is present"); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/008_modes.pl b/src/test/modules/test_dwb/t/008_modes.pl new file mode 100644 index 0000000000000..b8974fb1611a2 --- /dev/null +++ b/src/test/modules/test_dwb/t/008_modes.pl @@ -0,0 +1,285 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The io_torn_pages_protection modes that do not use the ring: "off" +# forces full-page images off no matter what the legacy GUC says (and is +# loud about it), "full_pages" defers to the legacy full_page_writes GUC, +# and under "double_writes" a SIGHUP of the legacy GUC is a no-op. + +use strict; +use warnings FATAL => 'all'; +use File::Path qw(rmtree); +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_modes'); +$node->init; + +# A tiny helper: pg_waldump over the WAL the given statement generated. +sub wal_window +{ + my ($stmt) = @_; + my $lsn0 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); + $node->safe_psql('postgres', $stmt); + my $lsn1 = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); + my ($out, $err) = run_command( + [ + 'pg_waldump', + '--path' => $node->data_dir . '/pg_wal', + '--start' => $lsn0, + '--end' => $lsn1 + ]); + is($err, '', "pg_waldump read the window of: $stmt"); + return $out; +} + +# --- mode "off": no page images regardless of the legacy GUC ------------- + +$node->append_conf('postgresql.conf', 'io_torn_pages_protection = off'); +my $log_offset = (-s $node->logfile) // 0; +$node->start; +ok($node->log_contains(qr/torn page protection is disabled/, $log_offset), + 'mode "off" announces itself'); +ok(!-d $node->data_dir . '/pg_dwb', 'mode "off" creates no ring'); +is($node->safe_psql('postgres', 'SHOW full_page_writes'), + 'on', 'the legacy GUC still reads on...'); + +$node->safe_psql('postgres', + 'CREATE TABLE dwb_m AS SELECT g AS id FROM generate_series(1, 100) g'); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $dump = wal_window('UPDATE dwb_m SET id = id WHERE id = 1'); +like($dump, qr/Heap/, '...the window covers the update...'); +unlike($dump, qr/\bFPW\b/, '...but no page image is written'); + +# --- mode "full_pages": the legacy GUC keeps its vanilla meaning --------- + +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = full_pages +full_page_writes = off +)); +$node->restart; +$node->safe_psql('postgres', 'CHECKPOINT'); +$dump = wal_window('UPDATE dwb_m SET id = id WHERE id = 2'); +like($dump, qr/Heap/, 'full_pages + legacy off: window covers the update'); +unlike($dump, qr/\bFPW\b/, 'full_pages + legacy off: no page image'); + +$node->append_conf('postgresql.conf', 'full_page_writes = on'); +$node->restart; +$node->safe_psql('postgres', 'CHECKPOINT'); +$dump = wal_window('UPDATE dwb_m SET id = id WHERE id = 3'); +like($dump, qr/\bFPW\b/, 'full_pages + legacy on: page image written'); + +# --- mode "double_writes": a SIGHUP of the legacy GUC is a no-op --------- + +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +full_page_writes = on +)); +$node->restart; +$node->safe_psql('postgres', 'CHECKPOINT'); +$dump = wal_window('UPDATE dwb_m SET id = id WHERE id = 4'); +unlike($dump, qr/\bFPW\b/, 'double_writes ignores the legacy on'); + +my $reload_lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +$node->append_conf('postgresql.conf', 'full_page_writes = off'); +$node->reload; +# a CHECKPOINT forces a checkpointer cycle, which processes the pending +# SIGHUP (and would emit XLOG_FPW_CHANGE if the reload were not a no-op) +$node->safe_psql('postgres', 'CHECKPOINT'); +$dump = wal_window('UPDATE dwb_m SET id = id WHERE id = 5'); +unlike($dump, qr/\bFPW\b/, 'reloading the legacy GUC changes nothing'); + +my $lsn_end = $node->safe_psql('postgres', 'SELECT pg_current_wal_lsn()'); +my ($out, $err) = run_command( + [ + 'pg_waldump', + '--path' => $node->data_dir . '/pg_wal', + '--start' => $reload_lsn, + '--end' => $lsn_end + ]); +is($err, '', 'pg_waldump read the reload window cleanly'); +unlike($out, qr/FPW_CHANGE/, + 'the no-op reload emitted no XLOG_FPW_CHANGE record'); + +# --- leaving double_writes takes one clean shutdown ---------------------- + +# After a crash the ring may hold repairs only a double_writes start can +# apply, so a start in any other mode is refused until the ring has been +# closed cleanly once. +$node->stop('immediate'); + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +my $ret = $node->start(fail_ok => 1); +is($ret, 0, 'crashed ring refuses a full_pages start'); +ok( $node->log_contains( + qr/FATAL: .* the double write buffer ring was not cleanly shut down, cannot start with "io_torn_pages_protection=full_pages"/ + ), + '... naming the mode change as the problem'); + +$node->append_conf('postgresql.conf', 'io_torn_pages_protection = off'); +$ret = $node->start(fail_ok => 1); +is($ret, 0, 'crashed ring refuses an off start too'); + +# one double_writes start applies the ring, and a clean stop releases it +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$log_offset = -s $node->logfile; +$node->start; +ok($node->log_contains(qr/double write buffer recovery:/, $log_offset), + 'the double_writes start runs the apply-pass'); +$node->stop; + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +$log_offset = -s $node->logfile; +$node->start; +is($node->safe_psql('postgres', 'SHOW io_torn_pages_protection'), + 'full_pages', 'after a clean stop the mode change is legal'); +ok( !$node->log_contains(qr/ring opened/, $log_offset), + '... and the leftover ring stays closed'); + +# --- reopening the ring re-arms the guard -------------------------------- + +# A double_writes start clears RING_CLEAN before the ring reopens, so a +# crash of that run leaves the marker unset and the guard must fire again: +# the marker certifies one clean closure, not a permanent state. +$node->stop; +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$node->start; +$node->stop('immediate'); + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +$ret = $node->start(fail_ok => 1); +is($ret, 0, 'a crash after reopening the ring re-arms the guard'); + +# close it cleanly once more +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$node->start; +$node->stop; + +# --- a standing marker suppresses the pass even over a crash ------------- + +# The reverse direction of the marker keying: a crash under an interim +# full_pages run (which touches neither the marker nor the generation) +# must NOT re-arm the ring — an apply here would resurrect ancient +# same-generation slots over pages torn long after the ring was closed. +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +$node->start; +$node->stop('immediate'); + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$log_offset = -s $node->logfile; +$node->start; +ok( !$node->log_contains(qr/double write buffer recovery:/, $log_offset), + 'a crash under an interim mode does not re-arm the apply-pass'); +$node->stop; + +# --- a corrupt ring control is refused, with a way out ------------------- + +# Modes that never touch the ring must still refuse an unreadable control +# (the ring may hold unapplied repairs), but with a message naming the +# removal recipe instead of a bare low-level read error. +my $control = $node->data_dir . '/pg_dwb/control'; +open my $fh, '>', $control or die "open $control: $!"; +binmode $fh; +print $fh "\x00" x 16; +close $fh; + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +$log_offset = -s $node->logfile; +$ret = $node->start(fail_ok => 1); +is($ret, 0, 'a corrupt ring control refuses a full_pages start'); +ok( $node->log_contains( + qr/FATAL: .* the double write buffer ring state could not be validated, cannot start with "io_torn_pages_protection=full_pages"/, + $log_offset), + '... naming the ring state as the problem'); + +# the hint's recipe: removing pg_dwb unblocks the start +rmtree($node->data_dir . '/pg_dwb'); +$node->start; +is($node->safe_psql('postgres', 'SHOW io_torn_pages_protection'), + 'full_pages', 'removing pg_dwb unblocks the non-ring mode'); + +# --- a crash under "off" is announced on the next protected start -------- + +# Nothing can repair pages torn by a crash that happened while WAL carried +# no images and no ring was active; the restart into a protected mode must +# say so instead of recovering in silence. +$node->append_conf('postgresql.conf', 'io_torn_pages_protection = off'); +$node->restart; +$node->stop('immediate'); + +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = full_pages'); +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/WARNING: .* database system was interrupted while torn page protection was disabled/, + $log_offset), + 'crash under "off" draws a warning on the protected restart'); + +# --- a ring of a newer format version is refused intact ------------------ + +# A binary downgrade can meet a ring whose min_version exceeds what this +# server reads. That ring is intact and may hold unapplied repairs only +# the newer server understands, so the refusal must name the version gap — +# never the "corrupt, remove pg_dwb" advice, which would invite discarding +# it. +$node->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$node->restart; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); +$node->safe_psql('postgres', 'SELECT test_dwb_set_control_min_version(2)'); +# an immediate stop: a clean shutdown would try to rewrite the control +$node->stop('immediate'); + +$log_offset = -s $node->logfile; +$ret = $node->start(fail_ok => 1); +is($ret, 0, 'a too-new ring format refuses the start'); +ok( $node->log_contains( + qr!FATAL: .* file "pg_dwb/control" requires format version at least 2, but this server supports 1!, + $log_offset), + '... naming the version gap'); +ok(!$node->log_contains(qr/could not be validated/, $log_offset), + '... and not the corrupt-ring advice'); + +# the intact-but-unreadable ring can only be resolved by removal +rmtree($node->data_dir . '/pg_dwb'); +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains(qr/ring opened: .* generation 1\b/, $log_offset), + 'removing the newer ring unblocks a fresh double_writes start'); + +# --- leftovers of an interrupted wipe are swept, not fatal --------------- + +# DWBWipeRing removes the control first, durably: a crash between that and +# the batch sweep leaves batch files behind a missing control. The next +# start must take the cold-create path and clear them — a start that +# trusted the batch files would fail on the missing control forever. +$node->stop; +unlink($node->data_dir . '/pg_dwb/control') + or die "unlink pg_dwb/control: $!"; +my $leftover = $node->data_dir . '/pg_dwb/batch_9999'; +open my $lf, '>', $leftover or die "open $leftover: $!"; +print $lf 'leftover of an interrupted wipe'; +close $lf; + +$log_offset = -s $node->logfile; +$node->start; +ok(!$node->log_contains(qr/double write buffer recovery:/, $log_offset), + 'no apply-pass over the swept leftovers'); +ok($node->log_contains(qr/ring opened: .* generation 1\b/, $log_offset), + 'the interrupted-wipe state cold-starts a fresh ring'); +ok(!-e $leftover, 'the leftover batch file is gone'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/009_fpw_transition.pl b/src/test/modules/test_dwb/t/009_fpw_transition.pl new file mode 100644 index 0000000000000..8eca8558fe406 --- /dev/null +++ b/src/test/modules/test_dwb/t/009_fpw_transition.pl @@ -0,0 +1,149 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Two ways a primary can stop writing page images, and how a full_pages +# standby reacts to each. Disabling the legacy full_page_writes GUC (the +# primary staying in full_pages mode) is the vanilla situation: the standby +# keeps replaying, but an online backup opened on it cannot be closed. The +# transition is carried by checkpoint records only — a restart emits no +# XLOG_FPW_CHANGE (UpdateFullPageWrites at startup runs before recovery is +# marked done, and the checkpointer's later call sees no remaining change). +# Switching the primary to io_torn_pages_protection = double_writes is a +# protocol change recorded in pg_control and XLOG_PARAMETER_CHANGE, and a +# standby still expecting full-page protection is refused outright: its own +# crash would leave torn pages nothing can repair. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +my $primary = PostgreSQL::Test::Cluster->new('dwb_fpw_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = full_pages +autovacuum = off +)); +$primary->start; + +$primary->backup('bkp'); +my $standby = PostgreSQL::Test::Cluster->new('dwb_fpw_standby'); +$standby->init_from_backup($primary, 'bkp', has_streaming => 1); +$standby->start; + +$primary->safe_psql('postgres', + 'CREATE TABLE dwb_fpw AS SELECT g AS id FROM generate_series(1, 1000) g'); +$primary->wait_for_catchup($standby); + +# --- an online backup opened on the standby while page images flow ------- + +# on_error_stop off: the session must survive the expected pg_backup_stop() +# error below +my $backer = $standby->background_psql('postgres', on_error_stop => 0); +$backer->query_safe("SELECT pg_backup_start('dwb_fpw_transition')"); + +# --- the legacy GUC is disabled under full_pages ------------------------- + +$primary->append_conf('postgresql.conf', 'full_page_writes = off'); +$primary->restart; + +$primary->safe_psql('postgres', + 'INSERT INTO dwb_fpw SELECT g FROM generate_series(1001, 2000) g'); +# the post-restart checkpoint records are the only replayed evidence +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($standby); + +# --- the open backup cannot be closed cleanly ---------------------------- + +my $stop_offset = -s $standby->logfile; +my ($out, $errored) = $backer->query('SELECT * FROM pg_backup_stop()'); +ok($errored, 'pg_backup_stop() on the standby fails across the transition'); +ok( $standby->log_contains( + qr/WAL generated without full page images was replayed during online backup/, + $stop_offset), + '... naming the replayed image-less WAL'); +ok( $standby->log_contains( + qr/the primary has "full_page_writes" disabled/, $stop_offset), + '... and blaming the legacy GUC, not the mode'); +$backer->quit; + +# --- the primary switches to double_writes ------------------------------- + +# The standby still runs full_pages: replaying the XLOG_PARAMETER_CHANGE +# that announces the mode must be fatal, and the whole standby exits. +my $fatal_offset = -s $standby->logfile; +$primary->append_conf('postgresql.conf', + 'io_torn_pages_protection = double_writes'); +$primary->restart; +$primary->safe_psql('postgres', + 'INSERT INTO dwb_fpw SELECT g FROM generate_series(2001, 3000) g'); + +foreach my $i (1 .. 300) +{ + last unless -f $standby->data_dir . '/postmaster.pid'; + usleep(100_000); +} +ok(!-f $standby->data_dir . '/postmaster.pid', + 'full_pages standby dies replaying the double_writes transition'); +# the node died on its own; let the harness notice before restarting it +$standby->stop('fast', fail_ok => 1); +ok( $standby->log_contains( + qr/FATAL: .* WAL was generated with "io_torn_pages_protection=double_writes", cannot continue recovering with "io_torn_pages_protection=full_pages"/, + $fatal_offset), + '... with the incompatibility spelled out'); + +# The refusal is durable: the mode is in the standby's pg_control now, so a +# restart is refused up front, before any replay. +$fatal_offset = -s $standby->logfile; +my $ret = $standby->start(fail_ok => 1); +is($ret, 0, 'restarting the full_pages standby is refused up front'); +ok( $standby->log_contains( + qr/WAL was generated with "io_torn_pages_protection=double_writes"/, + $fatal_offset), + '... for the same reason'); + +# --- a double_writes standby follows the same primary -------------------- + +$standby->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +)); +my $ring_offset = -s $standby->logfile; +$standby->start; +ok( $standby->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 1\b/, + $ring_offset), + 'reconfigured standby cold-starts a ring of its own'); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_fpw'), + '3000', 'and replays the image-less WAL'); + +# --- a crash of a double_writes standby of an "off" primary is quiet ------ + +# The crash-under-off warning keys on pg_control's io_torn_pages_protection, +# which on a standby describes the PRIMARY's run, not the one that crashed +# here. A double_writes standby of an "off" primary repairs its own torn +# pages from its own ring, so its crash restart must stay silent. +$primary->append_conf('postgresql.conf', 'io_torn_pages_protection = off'); +$primary->restart; +$primary->safe_psql('postgres', + 'INSERT INTO dwb_fpw SELECT g FROM generate_series(3001, 4000) g'); +$primary->wait_for_catchup($standby); + +$standby->stop('immediate'); +my $warn_offset = -s $standby->logfile; +$standby->start; +ok( !$standby->log_contains( + qr/interrupted while torn page protection was disabled/, $warn_offset + ), + 'crashed double_writes standby of an "off" primary draws no warning'); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM dwb_fpw'), + '4000', 'and keeps replaying'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/010_recovery.pl b/src/test/modules/test_dwb/t/010_recovery.pl new file mode 100644 index 0000000000000..e4ff42ee65ed7 --- /dev/null +++ b/src/test/modules/test_dwb/t/010_recovery.pl @@ -0,0 +1,282 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The startup apply-pass: torn data pages are repaired from the ring before +# WAL replay. The scenarios pick pages replay itself can never fix — a +# hint-bit-only page is logged as XLOG_FPI_FOR_HINT without an image and +# redo does not even read it, and the other damaged pages predate the last +# checkpoint — so any repair observed here came from the ring. Also pins +# the boundaries of the pass: a second pass over the same ring is a no-op +# (idempotence after a crash mid-pass), a clean start skips the pass, and a +# slot of a past generation is never applied, even to a page that fails +# verification. +# +# With dwb_retire_workers = 0 every publish retires its batch on the spot, +# so a sequential writer keeps reusing the lowest ring index and only the +# LAST page written before a crash still has its slot on disk. Each damage +# scenario therefore ends its run with the target page's flush: +# BufferSync sorts a checkpoint's writes by relfilenode, and the user +# tables here sort after every catalog page a session may have hint-dirtied. + +use strict; +use warnings FATAL => 'all'; +use File::Path qw(rmtree); +use FindBin; +use lib $FindBin::RealBin; +use DWBTest; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_recovery'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_batch_timeout_ms = 20 +autovacuum = off +bgwriter_lru_maxpages = 0 +log_min_messages = debug1 +)); +$node->start; + +$node->safe_psql( + 'postgres', q( + CREATE TABLE thint AS SELECT g AS id FROM generate_series(1, 100) g; + CREATE TABLE told AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); + +my $thint_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('thint')"); +my $told_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('told')"); +my $thint_relnum = $node->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'thint'"); +my $told_relnum = $node->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'told'"); + +# --- a torn hint-bit-only page is repaired before replay ----------------- + +# With checksums on and page images off, setting hint bits logs +# XLOG_FPI_FOR_HINT without an image and still advances the page LSN, and +# redo of such a record never reads the page — the original silent-loss +# hole. Dirty thint's page with hint bits only, flush it through the ring, +# crash, and tear the on-disk page as if that flush had been cut short. +$node->safe_psql('postgres', 'SELECT count(*) FROM thint'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->stop('immediate'); + +write_block($thint_file, 0, + substr(read_block($thint_file, 0), 0, 4096) . ("\0" x 4096)); + +my $log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 1 of \d+ candidate pages restored/, + $log_offset), + 'apply-pass restored the torn page'); +ok( $node->log_contains( + qr!restoring page 0 of relation \d+/\d+/$thint_relnum fork 0!, + $log_offset), + '... and it was the hint-bit page'); +ok( $node->log_contains( + qr/ring opened: 16 batches of 16 pages, generation 2\b/, $log_offset), + 'generation bumped after the pass'); +is($node->safe_psql('postgres', 'SELECT count(*) FROM thint'), + '100', 'torn hint page is whole again'); + +# --- a checksum-valid but stale page is repaired by its LSN -------------- + +# Put the pre-update page image back after the crash: bytewise it verifies +# fine, so only the LSN comparison can see that the ring copy is newer. +my $told_v1 = read_block($told_file, 0); +$node->safe_psql('postgres', 'UPDATE told SET id = id + 1000 WHERE id <= 50'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->stop('immediate'); + +write_block($told_file, 0, $told_v1); + +# keep a copy of the whole ring for the re-apply scenario below +my $ring_stash = $node->basedir . '/ring_stash'; +PostgreSQL::Test::RecursiveCopy::copypath($node->data_dir . '/pg_dwb', + $ring_stash); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 1 of \d+ candidate pages restored/, + $log_offset), + 'apply-pass restored the stale page'); +ok( $node->log_contains( + qr!restoring page 0 of relation \d+/\d+/$told_relnum fork 0!, + $log_offset), + '... by its LSN — the page verified fine'); +is( $node->safe_psql('postgres', 'SELECT count(*) FROM told WHERE id > 1000'), + '50', + 'stale page carries the update again'); + +# --- a repeated pass over the same ring is a no-op ----------------------- + +# Equivalent to a crash in the middle of the pass: the ring is untouched +# and the generation not yet bumped, so the next start sees the very same +# candidate — now against a repaired, newer-or-equal page. +$node->stop('immediate'); +rmtree($node->data_dir . '/pg_dwb'); +PostgreSQL::Test::RecursiveCopy::copypath($ring_stash, + $node->data_dir . '/pg_dwb'); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 0 of \d+ candidate pages restored/, + $log_offset), + 're-applied pass sees the same candidate and rewrites nothing'); +is( $node->safe_psql('postgres', 'SELECT count(*) FROM told WHERE id > 1000'), + '50', + 'data intact after the repeated pass'); + +# --- a clean start skips the pass, but still bumps the generation -------- + +$node->safe_psql( + 'postgres', q( + CREATE TABLE tstale AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $tstale_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('tstale')"); +$node->stop; + +my $tstale_good = read_block($tstale_file, 0); + +$log_offset = -s $node->logfile; +$node->start; +ok(!$node->log_contains(qr/double write buffer recovery:/, $log_offset), + 'clean start runs no apply-pass'); +ok( $node->log_contains(qr/ring opened: .* generation 4\b/, $log_offset), + '... yet the generation still moves, expiring the old slots'); + +# --- a slot of a past generation is never applied ------------------------ + +# tstale's only slot is one generation behind now. Corrupt its page so +# that it fails verification — the branch where an LSN comparison cannot +# veto a repair — and crash: the pass must leave the page alone anyway. +$node->stop('immediate'); +write_block($tstale_file, 0, chr(0xAB) x 8192); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 0 of 0 candidate pages restored/, + $log_offset), + 'no current-generation candidates after the idle crash'); +is( read_block($tstale_file, 0), + chr(0xAB) x 8192, + 'the stale slot was not applied to the corrupted page'); + +# put the good page back so the cluster winds down healthy +write_block($tstale_file, 0, $tstale_good); +is($node->safe_psql('postgres', 'SELECT count(*) FROM tstale'), + '100', 'page manually restored, cluster consistent'); + +# --- the marker alone triggers the pass, not the pg_control state -------- + +# Leave a crashed ring behind a CLEAN pg_control: stash the ring right +# after a crash, run a clean stop cycle, then put the crashed ring back. +# Only the unset RING_CLEAN marker knows this ring was never retired — a +# standby whose shutdown restartpoint was skipped leaves exactly this +# combination, and the pass must key on the marker, not on pg_control. +$node->safe_psql( + 'postgres', q( + CREATE TABLE tmark AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $tmark_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('tmark')"); +$node->stop('immediate'); + +my $mark_stash = $node->basedir . '/mark_stash'; +PostgreSQL::Test::RecursiveCopy::copypath($node->data_dir . '/pg_dwb', + $mark_stash); + +$node->start; +$node->stop; + +rmtree($node->data_dir . '/pg_dwb'); +PostgreSQL::Test::RecursiveCopy::copypath($mark_stash, + $node->data_dir . '/pg_dwb'); +write_block($tmark_file, 0, + substr(read_block($tmark_file, 0), 0, 4096) . ("\0" x 4096)); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 1 of \d+ candidate pages restored/, + $log_offset), + 'unretired ring is applied despite a clean pg_control'); +is($node->safe_psql('postgres', 'SELECT count(*) FROM tmark'), + '100', 'torn page behind a clean shutdown is whole again'); + +# --- a slot for a dropped relation is skipped ---------------------------- + +# The relation's file may survive as an empty tombstone until the next +# checkpoint, or be gone entirely; either way there is nothing to repair +# and the pass must not trip over it. +$node->safe_psql( + 'postgres', q( + CREATE TABLE tdrop AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', 'DROP TABLE tdrop'); +$node->stop('immediate'); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 0 of \d+ candidate pages restored/, + $log_offset), + 'a candidate for a dropped relation is counted but skipped'); + +# --- an all-zero on-disk page is never repaired -------------------------- + +# Zero the whole block: an empty header means replay recreates the page +# from its init record without reading it, and a stale slot must not +# resurrect on it — the zeroed page's LSN 0 would lose the LSN comparison +# that this skip protects. +$node->safe_psql( + 'postgres', q( + CREATE TABLE tzero AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $tzero_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('tzero')"); +$node->stop('immediate'); + +my $tzero_good = read_block($tzero_file, 0); +write_block($tzero_file, 0, "\0" x 8192); + +$log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 0 of \d+ candidate pages restored/, + $log_offset), + 'a zeroed page is not repaired from its slot'); +is( read_block($tzero_file, 0), + "\0" x 8192, + '... and stays zero for replay to drive'); + +# put the good page back so the cluster winds down healthy +write_block($tzero_file, 0, $tzero_good); +is($node->safe_psql('postgres', 'SELECT count(*) FROM tzero'), + '100', 'page manually restored, cluster consistent'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/011_geometry_recovery.pl b/src/test/modules/test_dwb/t/011_geometry_recovery.pl new file mode 100644 index 0000000000000..e1cef37ff9a74 --- /dev/null +++ b/src/test/modules/test_dwb/t/011_geometry_recovery.pl @@ -0,0 +1,76 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# A geometry change over a ring that was not cleanly closed: the apply-pass +# must run with the OLD geometry recorded in pg_dwb/control — not the new +# GUCs — before the ring is recreated. A regression that read the GUCs +# instead would compute a wrong meta-region size, fail every slot CRC, +# report nothing to restore and leave the damage in place, silently: only +# the combination "page repaired AND ring recreated in one start" pins the +# ordering. +# +# The damage technique follows t/010_recovery.pl: dwb_retire_workers = 0 +# retires every batch on the spot, so the last page flushed before the +# crash — the user table's, sorted last in BufferSync by relfilenode — is +# the one slot still on disk. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use lib $FindBin::RealBin; +use DWBTest; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_geometry_recovery'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_batch_timeout_ms = 20 +autovacuum = off +bgwriter_lru_maxpages = 0 +)); +$node->start; + +$node->safe_psql( + 'postgres', q( + CREATE TABLE tgeo AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +my $tgeo_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('tgeo')"); + +# stale-page damage: put the pre-update image back after the crash, so the +# repair can only come from the ring copy's higher LSN +my $tgeo_v1 = read_block($tgeo_file, 0); +$node->safe_psql('postgres', 'UPDATE tgeo SET id = id + 1000 WHERE id <= 50'); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->stop('immediate'); + +write_block($tgeo_file, 0, $tgeo_v1); + +$node->append_conf('postgresql.conf', 'dwb_batch_pages = 32'); +my $log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 1 of \d+ candidate pages restored/, + $log_offset), + 'the crashed ring is applied with its recorded geometry'); +ok( $node->log_contains( + qr/recreating double write buffer ring: geometry changed from 16 batches of 16 pages to 16 batches of 32 pages/, + $log_offset), + '... and only then recreated under the new GUCs'); +ok( $node->log_contains( + qr/ring opened: 16 batches of 32 pages, generation 1\b/, $log_offset), + '... with a fresh generation'); +is( $node->safe_psql('postgres', 'SELECT count(*) FROM tgeo WHERE id > 1000'), + '50', + 'the stale page carries the update again'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/012_apply_crafted.pl b/src/test/modules/test_dwb/t/012_apply_crafted.pl new file mode 100644 index 0000000000000..2b167dd2271e1 --- /dev/null +++ b/src/test/modules/test_dwb/t/012_apply_crafted.pl @@ -0,0 +1,163 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The apply-pass dedup comparator, exercised through crafted batch files. +# The runtime write path cannot produce two surviving copies of one page — +# with dwb_retire_workers = 0 a sequential writer keeps reusing the lowest +# ring index — so test_dwb_craft_batch() writes the competing candidates +# directly: same page, two batches, chosen LSNs and batch_ids. The DEBUG1 +# "restoring ... from batch N" line names the winning slot exactly. +# +# The same helper drives the standby scenario: a crafted slot whose LSN lies +# beyond the standby's minRecoveryPoint must make the startup apply-pass +# raise it before consistency can be declared. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use lib $FindBin::RealBin; +use DWBTest; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('dwb_craft'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_batch_timeout_ms = 20 +autovacuum = off +bgwriter_lru_maxpages = 0 +log_min_messages = debug1 +)); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +$primary->safe_psql( + 'postgres', q( + CREATE TABLE tlsn AS SELECT g AS id FROM generate_series(1, 100) g; + CREATE TABLE ttie AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$primary->safe_psql('postgres', 'CHECKPOINT'); + +my $tlsn_file = + $primary->data_dir . '/' + . $primary->safe_psql('postgres', "SELECT pg_relation_filepath('tlsn')"); +my $ttie_file = + $primary->data_dir . '/' + . $primary->safe_psql('postgres', "SELECT pg_relation_filepath('ttie')"); +my $tlsn_relnum = $primary->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'tlsn'"); +my $ttie_relnum = $primary->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'ttie'"); + +# --- higher LSN wins the dedup; on a tie the higher batch_id does --------- + +# All crafted LSNs must stay at or below the real WAL insert position (they +# become page LSNs of live pages) and above the blocks' current disk LSNs: +# pad the WAL a little and take LSNs from just below the insert position. +$primary->safe_psql('postgres', + "SELECT pg_logical_emit_message(false, 'dwb', repeat('x', 256))") + for (1 .. 2); +my $insert_lsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +my ($lsn_a, $lsn_b) = split /\|/, + $primary->safe_psql('postgres', + "SELECT '$insert_lsn'::pg_lsn - 16, '$insert_lsn'::pg_lsn - 8"); + +# The LSN pair goes into batches 14/15. The tie pair goes into 12/13 with +# the HIGHER batch_id in the LOWER batch index, so a comparator that merely +# kept the later-scanned candidate would pick the wrong slot. +$primary->safe_psql( + 'postgres', qq( + SELECT test_dwb_craft_batch(14, 501, $tlsn_relnum, 0, '$lsn_a', 'DWBLSNLOSER'); + SELECT test_dwb_craft_batch(15, 502, $tlsn_relnum, 0, '$lsn_b', 'DWBLSNWINNER'); + SELECT test_dwb_craft_batch(12, 601, $ttie_relnum, 0, '$insert_lsn', 'DWBTIEWINNER'); + SELECT test_dwb_craft_batch(13, 600, $ttie_relnum, 0, '$insert_lsn', 'DWBTIELOSER'); +)); +$primary->stop('immediate'); + +my $log_offset = -s $primary->logfile; +$primary->start; +ok( $primary->log_contains( + qr!restoring page 0 of relation \d+/\d+/$tlsn_relnum fork 0 from batch 15!, + $log_offset), + 'the higher-LSN copy won the dedup'); +ok( $primary->log_contains( + qr!restoring page 0 of relation \d+/\d+/$ttie_relnum fork 0 from batch 12!, + $log_offset), + 'on equal LSNs the higher batch_id won, against scan order'); + +like(read_block($tlsn_file, 0), qr/DWBLSNWINNER/, 'winning image is on disk'); +unlike(read_block($tlsn_file, 0), + qr/DWBLSNLOSER/, '... and the losing image is not'); +like(read_block($ttie_file, 0), + qr/DWBTIEWINNER/, 'winning tie image is on disk'); +unlike(read_block($ttie_file, 0), + qr/DWBTIELOSER/, '... and the losing tie image is not'); + +is( $primary->safe_psql( + 'postgres', + 'SELECT count(*) FROM tlsn UNION ALL SELECT count(*) FROM ttie'), + "100\n100", + 'both repaired pages read back fine'); + +# --- a crafted slot beyond minRecoveryPoint raises it on the standby ------ + +$primary->backup('bkp'); +my $standby = PostgreSQL::Test::Cluster->new('dwb_craft_standby'); +$standby->init_from_backup($primary, 'bkp', has_streaming => 1); +$standby->start; + +$primary->safe_psql( + 'postgres', q( + CREATE TABLE tmrp AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($standby); +# the restartpoint flushes tmrp's block to the standby's disk +$standby->safe_psql('postgres', 'CHECKPOINT'); +my $tmrp_relnum = $standby->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'tmrp'"); +my $tmrp_file = + $standby->data_dir . '/' + . $standby->safe_psql('postgres', "SELECT pg_relation_filepath('tmrp')"); + +# Hold replay while the primary moves ahead: the standby then holds +# received-but-unreplayed WAL, and any LSN inside it lies beyond the +# standby's minRecoveryPoint yet within the WAL it can replay to. +$standby->safe_psql('postgres', 'SELECT pg_wal_replay_pause()'); +$primary->safe_psql('postgres', + "SELECT pg_logical_emit_message(false, 'dwb', repeat('x', 1024))"); +my $mrp_lsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$primary->wait_for_catchup($standby, 'flush', $mrp_lsn); + +$standby->safe_psql('postgres', + "SELECT test_dwb_craft_batch(15, 700, $tmrp_relnum, 0, '$mrp_lsn', 'DWBMRPMARK')" +); +$standby->stop('immediate'); + +$log_offset = -s $standby->logfile; +$standby->start; +ok( $standby->log_contains( + qr!restoring page 0 of relation \d+/\d+/$tmrp_relnum fork 0 from batch 15!, + $log_offset), + 'crafted slot applied on the standby'); +my $mrp_re = quotemeta($mrp_lsn); +ok( $standby->log_contains( + qr/raising minimum recovery point to $mrp_re to cover pages repaired from the double write buffer/, + $log_offset), + 'minimum recovery point raised to the applied LSN'); +like(read_block($tmrp_file, 0), + qr/DWBMRPMARK/, 'crafted image is on the standby disk'); + +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM tmrp'), + '100', 'standby reads the repaired page fine'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/013_backup_start_point.pl b/src/test/modules/test_dwb/t/013_backup_start_point.pl new file mode 100644 index 0000000000000..0ab645ae79911 --- /dev/null +++ b/src/test/modules/test_dwb/t/013_backup_start_point.pl @@ -0,0 +1,104 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The second arm of the restored-backup detection: backup_label is renamed +# away early in the first recovery attempt, so a crash mid-backup-recovery +# leaves only pg_control's backupStartPoint to say "this is still a restored +# backup". A low-level backup copied without the backup-end WAL fails its +# recovery deterministically ("WAL ends before end of online backup"), which +# yields both states in turn: the first start sees the label, the second +# start sees no label but a set backupStartPoint — and both must discard +# the ring instead of applying it. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use lib $FindBin::RealBin; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_bsp'); +$node->init(allows_streaming => 1); +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_batch_timeout_ms = 20 +autovacuum = off +)); +$node->start; + +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_bsp AS SELECT g AS id FROM generate_series(1, 100) g; +)); + +# A low-level backup: open the window, OS-copy the whole data directory, +# close the window. The copy predates pg_backup_stop, so it cannot contain +# the backup-end WAL record — recovery from it must run out of WAL inside +# the backup window. +my $bk = $node->background_psql('postgres'); +$bk->query_safe('SET client_min_messages = warning'); +$bk->query_safe("SELECT pg_backup_start('dwb_bsp_wide', true)"); +$node->safe_psql('postgres', + 'INSERT INTO dwb_bsp SELECT g FROM generate_series(101, 200) g'); + +my $backup_path = $node->backup_dir . '/wide'; +PostgreSQL::Test::RecursiveCopy::copypath($node->data_dir, $backup_path); +unlink("$backup_path/postmaster.pid", "$backup_path/postmaster.opts"); + +my $label = $bk->query_safe('SELECT labelfile FROM pg_backup_stop()'); +$bk->quit; +like($label, qr/^START WAL LOCATION/, 'pg_backup_stop returned the label'); +open my $lf, '>', "$backup_path/backup_label" or die "backup_label: $!"; +print $lf $label; +print $lf "\n" unless $label =~ /\n$/; +close $lf; + +my $restored = PostgreSQL::Test::Cluster->new('dwb_bsp_restored'); +$restored->init_from_backup($node, 'wide'); +my $pgdata = $restored->data_dir; + +# --- first start: the label arm ------------------------------------------ + +my $log_offset = -s $restored->logfile; +my $ret = $restored->start(fail_ok => 1); +is($ret, 0, 'recovery without the backup-end WAL fails'); +ok( $restored->log_contains( + qr/FATAL: .* WAL ends before end of online backup/, $log_offset), + '... for the expected reason'); +ok( $restored->log_contains( + qr/discarding double write buffer ring contents restored from a base backup/, + $log_offset), + 'the label start discarded the copied ring'); +ok( !$restored->log_contains(qr/double write buffer recovery:/, $log_offset), + '... and ran no apply-pass'); + +ok(!-f "$pgdata/backup_label", 'the failed recovery consumed backup_label'); +ok(-f "$pgdata/backup_label.old", '... renaming it out of the way'); +my ($cd, $cderr) = run_command([ 'pg_controldata', $pgdata ]); +like( + $cd, + qr/Backup start location:\s+(?!0\/0)\S/, + 'pg_control still carries backupStartPoint'); + +# --- second start: the backupStartPoint arm ------------------------------ + +# No label anymore; only the control flag says this recovery still belongs +# to a restored backup. The ring recreated by the first start must be +# discarded again, not applied. +$log_offset = -s $restored->logfile; +$ret = $restored->start(fail_ok => 1); +is($ret, 0, 'the second recovery attempt fails the same way'); +ok( $restored->log_contains( + qr/discarding double write buffer ring contents restored from a base backup/, + $log_offset), + 'backupStartPoint alone still discards the ring'); +ok( !$restored->log_contains(qr/double write buffer recovery:/, $log_offset), + '... and no apply-pass ran on the second start either'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/014_pg_upgrade.pl b/src/test/modules/test_dwb/t/014_pg_upgrade.pl new file mode 100644 index 0000000000000..5f076674d1558 --- /dev/null +++ b/src/test/modules/test_dwb/t/014_pg_upgrade.pl @@ -0,0 +1,71 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# pg_upgrade and the ring: the new cluster is a fresh initdb, so nothing of +# pg_dwb/ ever transfers — the old ring stays with the old cluster, and the +# upgraded cluster cold-starts a ring of its own on its first double_writes +# start. A same-version upgrade exercises the whole path, including +# pg_upgrade's own starts of the cleanly-stopped double_writes old cluster. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $dwb_conf = qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 0 +dwb_batch_timeout_ms = 20 +autovacuum = off +); + +my $old = PostgreSQL::Test::Cluster->new('dwb_upgrade_old'); +$old->init; +$old->append_conf('postgresql.conf', $dwb_conf); +$old->start; +$old->safe_psql( + 'postgres', q( + CREATE TABLE dwb_up AS SELECT g AS id FROM generate_series(1, 100) g; +)); +$old->safe_psql('postgres', 'CHECKPOINT'); +$old->stop; +ok( -f $old->data_dir . '/pg_dwb/control', + 'the old cluster leaves a ring behind'); + +my $new = PostgreSQL::Test::Cluster->new('dwb_upgrade_new'); +$new->init; + +my $bindir = $new->config_data('--bindir'); +command_ok( + [ + 'pg_upgrade', '--no-sync', + '--old-datadir' => $old->data_dir, + '--new-datadir' => $new->data_dir, + '--old-bindir' => $bindir, + '--new-bindir' => $bindir, + '--socketdir' => $new->host, + '--old-port' => $old->port, + '--new-port' => $new->port, + ], + 'pg_upgrade from a double_writes cluster succeeds'); + +ok( -f $old->data_dir . '/pg_dwb/control', + 'the old ring stays with the old cluster'); +ok(!-d $new->data_dir . '/pg_dwb', + 'nothing of the ring was shipped into the new cluster'); + +# the upgraded cluster starts its double_writes life cold +$new->append_conf('postgresql.conf', $dwb_conf); +my $log_offset = -s $new->logfile; +$new->start; +ok( $new->log_contains( + qr/double write buffer ring opened: 16 batches of 16 pages, generation 1\b/, + $log_offset), + 'the upgraded cluster cold-starts a fresh ring'); +is($new->safe_psql('postgres', 'SELECT count(*) FROM dwb_up'), + '100', 'the upgraded data is intact'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/015_vectored_flush.pl b/src/test/modules/test_dwb/t/015_vectored_flush.pl new file mode 100644 index 0000000000000..de05b0f9aeda6 --- /dev/null +++ b/src/test/modules/test_dwb/t/015_vectored_flush.pl @@ -0,0 +1,102 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The vectored background flush: a checkpoint writes its buffers through +# the double write buffer in bins — one batch write and one fdatasync per +# bin instead of one per page. pg_stat_io proves the batching (the dwb +# "writes" counter is per batch, so write_bytes/writes is the batch size), +# and a torn page written by the vectored path is restored by the +# apply-pass like any other. + +use strict; +use warnings FATAL => 'all'; +use FindBin; +use lib $FindBin::RealBin; +use DWBTest; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_vectored'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +autovacuum = off +bgwriter_lru_maxpages = 0 +log_min_messages = debug1 +)); +$node->start; + +# ~200 heap pages, all dirty: enough for a dozen full bins +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_vec (id int, pad text) WITH (fillfactor = 10); + INSERT INTO dwb_vec SELECT g, repeat('v', 256) FROM generate_series(1, 1000) g; +)); +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', "UPDATE dwb_vec SET pad = repeat('w', 256)"); +my $npages = + $node->safe_psql('postgres', "SELECT pg_relation_size('dwb_vec') / 8192"); +cmp_ok($npages, '>', 150, 'the table spans enough pages for full bins'); + +$node->safe_psql('postgres', 'CHECKPOINT'); + +# --- the checkpointer writes multi-slot batches --------------------------- + +# One dwb "write" is one batch write by its leader, so write_bytes/writes +# is the average batch size: the meta region (4096 bytes on this geometry) +# plus one 8 KB image per slot. The per-page protocol pins this ratio at +# exactly one slot for the checkpointer's sequential stream; the vectored +# flush must push it to bin-sized fills. Checkpointer stats reach the +# collector with a delay, so poll. +$node->poll_query_until( + 'postgres', q( + SELECT writes > 0 FROM pg_stat_io + WHERE backend_type = 'checkpointer' AND object = 'dwb' + AND context = 'normal' +)) or die 'timed out waiting for checkpointer dwb stats'; + +my $avg_slots = $node->safe_psql( + 'postgres', q( + SELECT round((write_bytes::numeric / writes - 4096) / 8192, 1) + FROM pg_stat_io + WHERE backend_type = 'checkpointer' AND object = 'dwb' + AND context = 'normal' +)); +cmp_ok($avg_slots, '>=', 4, + "checkpointer batches average $avg_slots slots, not one per page"); + +# --- a torn page of the vectored path is repaired ------------------------- + +# The last heap block sorts last in BufferSync, so it lands in the final +# bin and its slot survives any batch-index reuse by earlier bins. +my $vec_file = + $node->data_dir . '/' + . $node->safe_psql('postgres', "SELECT pg_relation_filepath('dwb_vec')"); +my $vec_relnum = $node->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'dwb_vec'"); +my $sum_before = $node->safe_psql('postgres', 'SELECT sum(id) FROM dwb_vec'); +my $last_block = $npages - 1; + +$node->stop('immediate'); +write_block($vec_file, $last_block, + substr(read_block($vec_file, $last_block), 0, 4096) . ("\0" x 4096)); + +my $log_offset = -s $node->logfile; +$node->start; +ok( $node->log_contains( + qr/double write buffer recovery: 1 of \d+ candidate pages restored/, + $log_offset), + 'apply-pass restored the page torn under the vectored flush'); +ok( $node->log_contains( + qr!restoring page $last_block of relation \d+/\d+/$vec_relnum fork 0!, + $log_offset), + '... and it was the torn heap page'); +is($node->safe_psql('postgres', 'SELECT sum(id) FROM dwb_vec'), + $sum_before, 'data intact after the repair'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/016_bgwriter_bin.pl b/src/test/modules/test_dwb/t/016_bgwriter_bin.pl new file mode 100644 index 0000000000000..7118b69cbfebe --- /dev/null +++ b/src/test/modules/test_dwb/t/016_bgwriter_bin.pl @@ -0,0 +1,73 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The bgwriter's LRU scan flushes through the double write buffer in bins +# (FlushBufferBin), like the checkpointer in 015: without the bins every +# scattered singleton write pays a full batch fdatasync through the +# lone-writer seal. pg_stat_io proves the batching: one dwb "write" is one +# batch, so write_bytes/writes is the average batch size. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +my $node = PostgreSQL::Test::Cluster->new('dwb_bgwriter'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +autovacuum = off +shared_buffers = 2MB +bgwriter_delay = 10ms +bgwriter_lru_maxpages = 1000 +bgwriter_lru_multiplier = 10 +checkpoint_timeout = 1h +)); +$node->start; + +# A table several times larger than shared_buffers: every UPDATE pass +# streams allocations through the small pool, which is what makes the +# bgwriter clean dirty buffers ahead of the clock sweep. +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_bgw (id int, pad text) WITH (fillfactor = 50); + INSERT INTO dwb_bgw SELECT g, repeat('b', 500) FROM generate_series(1, 20000) g; +)); + +# Only the bgwriter's own writes may enter the average: drop what the +# initial data load accumulated. +$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); + +# The bgwriter's write volume per round depends on its allocation estimator, +# and its statistics reach the collector asynchronously: drive passes of +# fresh allocations and poll between them until the dwb row carries the +# proof. writes >= 10 skips the noise of the first few partial bins. +my $binned = 0; +OUTER: for my $pass (1 .. 8) +{ + $node->safe_psql('postgres', + "UPDATE dwb_bgw SET pad = repeat(chr(96 + $pass), 500)"); + for my $probe (1 .. 25) + { + $binned = $node->safe_psql( + 'postgres', q( + SELECT COALESCE(bool_or( + writes >= 10 + AND (write_bytes::numeric / writes - 4096) / 8192 >= 4), false) + FROM pg_stat_io + WHERE backend_type = 'background writer' AND object = 'dwb' + AND context = 'normal' + )); + last OUTER if $binned eq 't'; + usleep(200_000); + } +} +is($binned, 't', 'bgwriter dwb batches average >= 4 slots, not one per page'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/017_syncfs_retire.pl b/src/test/modules/test_dwb/t/017_syncfs_retire.pl new file mode 100644 index 0000000000000..b8e76c96a32e9 --- /dev/null +++ b/src/test/modules/test_dwb/t/017_syncfs_retire.pl @@ -0,0 +1,131 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# dwb_retire_sync_method = syncfs: one syncfs() per retire round replaces +# the per-segment fdatasync protocol, and every batch that was RETIRING +# when the round began is freed wholesale. Two workers prove the round +# gate (DWBSyncfsRoundLock) admits one process at a time; fsync stays ON +# so the rounds issue real syncfs() calls; the crash at the end proves +# the durability chain end to end. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_syncfs'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 2 +fsync = on +shared_buffers = 2MB +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 1h +autovacuum = off +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# syncfs is a build-time option (Linux); skip where the GUC cannot take it +if ($node->safe_psql('postgres', + "SELECT 'syncfs' = ANY(enumvals) FROM pg_settings " + . "WHERE name = 'dwb_retire_sync_method'") ne 't') +{ + plan skip_all => 'syncfs not supported by this build'; +} + +$node->safe_psql('postgres', + 'ALTER SYSTEM SET dwb_retire_sync_method = syncfs'); +$node->reload; +$node->poll_query_until('postgres', + "SELECT current_setting('dwb_retire_sync_method') = 'syncfs'") + or die 'timed out waiting for the syncfs retire method to apply'; + +$node->poll_query_until('postgres', + "SELECT count(*) = 2 FROM pg_stat_activity WHERE backend_type = 'dwb retire worker'" +) or die 'timed out waiting for the retire workers to start'; + +# --- the round gate admits exactly one process --------------------------- + +# The dwb-syncfs-round point sits INSIDE the trylock-guarded round, so a +# process can only park there after winning DWBSyncfsRoundLock. While a +# winner is parked there, ALL wholesale retirement is frozen, so this +# scenario must stay ring-quiet: one parked batch via test_dwb_park() and +# pg_stat_activity polls, no eviction workload. +SKIP: +{ + skip 'injection points not supported by this build', 3 + unless defined $ENV{enable_injection_points} + && $ENV{enable_injection_points} eq 'yes'; + + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-syncfs-round', 'wait')"); + + # One RETIRING batch gives the winner's resumed round something to + # free; the client never touches the gate (park does not retire). + $node->safe_psql('postgres', 'SELECT test_dwb_park(99001)'); + + # The workers race the gate on their own interval clock: one parks at + # the point, the loser's tries all fail and put it back to sleep. + $node->wait_for_event('dwb retire worker', 'dwb-syncfs-round'); + pass('a worker won the gate and parked at the point'); + + $node->poll_query_until( + 'postgres', q( + SELECT count(*) FILTER (WHERE wait_event = 'dwb-syncfs-round') = 1 + AND count(*) FILTER (WHERE wait_event = 'DwbRetireMain') = 1 + FROM pg_stat_activity WHERE backend_type = 'dwb retire worker' + )) + or die 'timed out waiting for the gate loser to sleep in its main loop'; + pass('exactly one process is inside the syncfs round gate'); + + # Detach BEFORE waking: the winner re-enters the round on its next + # cycle, and with the point still attached it would park again with no + # wakeup left to release it. + $node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-syncfs-round')"); + $node->safe_psql('postgres', + "SELECT injection_points_wakeup('dwb-syncfs-round')"); + + # The resumed round collects and frees the parked batch. + $node->poll_query_until('postgres', + "SELECT test_dwb_states() LIKE 'free=16 %'") + or die 'timed out waiting for the resumed round to free the batch'; + pass('the resumed round freed the parked batch'); +} + +# --- syncfs rounds drain a real workload --------------------------------- + +# A workload well past shared_buffers streams evictions through the ring; +# the workers' syncfs rounds must keep freeing batches for it to finish. +$node->safe_psql( + 'postgres', q( + CREATE TABLE dwb_syncfs_t AS + SELECT g AS id, repeat('s', 300) AS filler + FROM generate_series(1, 20000) g; + UPDATE dwb_syncfs_t SET filler = repeat('f', 300) WHERE id % 5 = 0; +)); +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_syncfs_t'), + '20000', 'workload survived the syncfs retire path'); + +# The workers alone must drain the ring: the poll only nudges the tail +# batch closed, all the freeing is the pool's wholesale rounds. +$node->poll_query_until('postgres', + "SELECT CASE WHEN test_dwb_force_seal() IS NOT NULL THEN " + . "test_dwb_states() LIKE 'free=16 %' END") + or die 'timed out waiting for syncfs rounds to drain the ring'; +pass('the workers drained the ring through syncfs rounds'); + +# Crash recovery on top of syncfs-retired data: intact. +$node->stop('immediate'); +$node->start; +is($node->safe_psql('postgres', 'SELECT count(*) FROM dwb_syncfs_t'), + '20000', 'data intact after crash recovery'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/018_cleaners.pl b/src/test/modules/test_dwb/t/018_cleaners.pl new file mode 100644 index 0000000000000..b9442c6d49863 --- /dev/null +++ b/src/test/modules/test_dwb/t/018_cleaners.pl @@ -0,0 +1,462 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The cleaner worker pool: the bgwriter's LRU scan hands its flush bins +# to a shared-memory queue and dwb_cleaner_workers background workers +# execute them. Queue entries are hints — every claim is reclassified +# right before the write — so the scenarios here drive the queue with +# deterministic one-page claims: a cold dirty page is written, a pinned +# page and an already-clean page are skipped, a full queue makes the +# bgwriter defer its bin and pause the scan instead of writing. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_cleaners'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_cleaner_workers = 2 +shared_buffers = 2MB +bgwriter_delay = 10ms +bgwriter_lru_maxpages = 1000 +bgwriter_lru_multiplier = 10 +checkpoint_timeout = 1h +autovacuum = off +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +$node->poll_query_until('postgres', + "SELECT count(*) = 2 FROM pg_stat_activity WHERE backend_type = 'dwb cleaner'" +) or die 'timed out waiting for the cleaner workers to start'; +pass('two cleaner workers are visible in pg_stat_activity'); + +sub counter +{ + my ($name) = @_; + return $node->safe_psql('postgres', + "SELECT $name FROM test_dwb_cleaner_counters()"); +} + +# A quiet baseline: nothing dirty, the bgwriter has nothing to scan. +$node->safe_psql('postgres', 'CHECKPOINT'); + +# --- a stale-claim queue entry naming a cold dirty page is written ------- + +# enqueue_block cools the page as it queues the claim (production pages +# go cold when the clock hand sweeps past); the page is dirty and nobody +# holds it, so this claim must be written. The system is quiet after +# the checkpoint, so background-class DWB pages can only come from the +# cleaner executing this claim — pinned by the class-page growth with +# the deferral counter standing still (the queue was never full, so the +# bgwriter had nothing to defer and writes nothing itself anyway). +my $written_before = counter('written'); +my $deferred_before = counter('deferred'); +my $bg_pages_before = $node->safe_psql('postgres', + "SELECT sum(pages) FROM test_dwb_seal_stats() WHERE wclass = 'background'" +); +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_cold (id int); + INSERT INTO t_cold VALUES (1); +)); +is( $node->safe_psql( + 'postgres', "SELECT test_dwb_enqueue_block('t_cold', 0)"), + 't', + 'the queue accepted a claim on a cold dirty page'); +$node->poll_query_until('postgres', + "SELECT written > $written_before FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for a cleaner to write the cold dirty page'; +pass('a cleaner wrote the queued cold dirty page'); +$node->poll_query_until( + 'postgres', qq( + SELECT sum(pages) > $bg_pages_before FROM test_dwb_seal_stats() + WHERE wclass = 'background' +)) or die 'timed out waiting for the background-class DWB batch'; +is(counter('deferred'), $deferred_before, + 'the background-class write was the cleaner, with no deferrals'); + +# --- a claim on a page that became hot is skipped ------------------------ + +# The pin is held across statements by an open transaction in a second +# session; the enqueue then races nothing — the claim is stale from the +# start and the reclassification must drop it without waiting. +my $skipped_before = counter('skipped'); +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_hot (id int); + INSERT INTO t_hot VALUES (1); +)); +my $pinner = $node->background_psql('postgres'); +$pinner->query_safe('BEGIN'); +$pinner->query_safe("SELECT test_dwb_pin_block('t_hot', 0)"); + +is( $node->safe_psql('postgres', "SELECT test_dwb_enqueue_block('t_hot', 0)"), + 't', + 'the queue accepted a claim on a pinned page'); +$node->poll_query_until('postgres', + "SELECT skipped > $skipped_before FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for the pinned page claim to be skipped'; +pass('the claim on the pinned page was skipped, not written'); + +$pinner->query_safe('SELECT test_dwb_unpin_block()'); +$pinner->query_safe('COMMIT'); + +# A rollback releases the pin through the resource owner; the helpers +# must notice and be usable again in the next transaction. +$pinner->query_safe('BEGIN'); +$pinner->query_safe("SELECT test_dwb_pin_block('t_hot', 0)"); +$pinner->query_safe('ROLLBACK'); +$pinner->query_safe('BEGIN'); +$pinner->query_safe("SELECT test_dwb_pin_block('t_hot', 0)"); +$pinner->query_safe('SELECT test_dwb_unpin_block()'); +$pinner->query_safe('COMMIT'); +$pinner->quit; +pass('the pin helpers survive a rollback and pin again'); + +# --- a duplicate claim finds the page clean and is skipped --------------- + +# t_cold's page was written by the first scenario; a second claim on the +# same block must resolve as a skip (clean already), never a rewrite. +$skipped_before = counter('skipped'); +is( $node->safe_psql( + 'postgres', "SELECT test_dwb_enqueue_block('t_cold', 0)"), + 't', + 'the queue accepted a duplicate claim'); +$node->poll_query_until('postgres', + "SELECT skipped > $skipped_before FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for the duplicate claim to be skipped'; +pass('the duplicate claim on the clean page was skipped'); + +# --- the production path: scan feeds the queue, the pool executes -------- + +# A workload well past shared_buffers leaves plenty of cold dirty pages +# behind; the bgwriter's scan bins them into the queue and the pool +# must execute them (the class and statistics attribution have their +# own quiet-window scenarios). +my $organic_before = counter('written'); +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_organic AS + SELECT g AS id, repeat('o', 300) AS filler + FROM generate_series(1, 20000) g; + UPDATE t_organic SET filler = repeat('p', 300) WHERE id % 5 = 0; +)); +$node->poll_query_until('postgres', + "SELECT written >= $organic_before + 16 FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for the pool to execute scan-produced bins'; +pass('the pool executed at least one full scan-produced bin'); +# pg_stat_io aggregates by process type, so the pool shows up under +# 'background worker' (bgw_type granularity exists only in +# pg_stat_activity); nothing else of that type writes DWB batches here. +$node->poll_query_until( + 'postgres', q( + SELECT sum(writes) > 0 FROM pg_stat_io + WHERE object = 'dwb' AND backend_type = 'background worker' +)) or die 'timed out waiting for the cleaner pg_stat_io dwb rows'; +pass('the cleaners show their DWB writes in pg_stat_io'); + +# --- a full queue makes the bgwriter defer bins, never write them -------- + +SKIP: +{ + skip 'injection points not supported by this build', 8 + unless defined $ENV{enable_injection_points} + && $ENV{enable_injection_points} eq 'yes'; + + my $bgw_io_before = $node->safe_psql('postgres', + "SELECT coalesce(sum(writes), 0) FROM pg_stat_io WHERE object = 'dwb' AND backend_type = 'background writer'" + ); + + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-cleaner-loop', 'wait')"); + + # Idle workers sleep on the queue's condition variable and only pass + # the loop top — where the point sits — when woken; one targeted + # signal per enqueued claim wakes them one by one. The two wake-up + # claims park both workers BEFORE any dequeue, so they stay queued. + $node->safe_psql('postgres', + "SELECT test_dwb_enqueue_block('t_cold', 0)"); + $node->safe_psql('postgres', + "SELECT test_dwb_enqueue_block('t_cold', 0)"); + $node->poll_query_until( + 'postgres', q( + SELECT count(*) FILTER (WHERE wait_event = 'dwb-cleaner-loop') = 2 + FROM pg_stat_activity WHERE backend_type = 'dwb cleaner' + )) or die 'timed out waiting for both cleaners to park at the point'; + pass('both cleaners parked at the injection point'); + + # Capacity is a fixed 64 bins, two of which the wake-up claims hold; + # 67 claims into a parked queue must overflow it (the bgwriter may + # race bins in as well, so the accepted count is bounded, not exact). + # A refused test claim defers nothing — the deferral counter is the + # bgwriter's own bookkeeping, checked right below. + $node->safe_psql( + 'postgres', q( + CREATE TABLE t_fill AS + SELECT g AS id, repeat('x', 800) AS filler + FROM generate_series(1, 800) g; + )); + my $accepted = 0; + for my $blk (0 .. 66) + { + $accepted++ + if $node->safe_psql('postgres', + "SELECT test_dwb_enqueue_block('t_fill', $blk)") eq 't'; + } + cmp_ok($accepted, '<=', 62, + 'the queue turned the overflow claims away at its capacity'); + + # With the pool parked and the queue full, the bgwriter defers: each + # refused bin bumps the counter, the bin is carried over, nothing is + # written by the bgwriter itself (checked once the dust settles). + my $deferred_solo = counter('deferred'); + $node->safe_psql( + 'postgres', q( + CREATE TABLE t_solo AS + SELECT g AS id, repeat('y', 800) AS filler + FROM generate_series(1, 2000) g; + UPDATE t_solo SET filler = repeat('z', 800) WHERE id % 3 = 0; + )); + $node->poll_query_until('postgres', + "SELECT deferred > $deferred_solo FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for the bgwriter to defer bins'; + pass('the bgwriter deferred bins while the pool was parked'); + + # Disabling the LRU scan must stop the deferral stream: the carried + # bin is dropped, the queue is no longer polled. The workers are + # still parked, so nothing else can move the counter. + $node->append_conf('postgresql.conf', 'bgwriter_lru_maxpages = 0'); + $node->reload; + my ($def_prev, $def_now) = (-1, -2); + my $deadline = time() + 30; + while (time() < $deadline) + { + $def_now = counter('deferred'); + last if $def_now == $def_prev; + $def_prev = $def_now; + sleep 1; + } + is(counter('deferred'), $def_now, + 'the deferral stream stopped once the scan was disabled'); + + # Detach BEFORE waking: a woken worker loops back to the point, and + # with it still attached it would park again with no wakeup left. + # Each wakeup releases ONE waiter (the first waiter slot matching the + # point name), so keep nudging until both workers are off the point — + # a worker left parked here would sleep on the injection DSM forever + # and its eventual FATAL exit would touch the detached segment. + $node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-cleaner-loop')"); + $deadline = time() + 30; + while (time() < $deadline) + { + last + if $node->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_activity WHERE wait_event = 'dwb-cleaner-loop'" + ) == 0; + $node->psql('postgres', + "SELECT injection_points_wakeup('dwb-cleaner-loop')"); + } + is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_stat_activity WHERE wait_event = 'dwb-cleaner-loop'" + ), + '0', + 'both cleaners left the injection point'); + + # The released pool drains the queue, but the disabled scan feeds it + # nothing: enqueued freezes even under a dirty workload (the dropped + # carry-over never lands either — it would show up right here). + $node->poll_query_until('postgres', + 'SELECT queued = 0 FROM test_dwb_cleaner_counters()') + or die 'timed out waiting for the released pool to drain the queue'; + my $enq_frozen = counter('enqueued'); + $node->safe_psql('postgres', + "UPDATE t_solo SET filler = repeat('w', 800) WHERE id % 4 = 0"); + sleep 2; + is(counter('enqueued'), $enq_frozen, + 'a disabled scan feeds the pool nothing'); + + # By now seconds have passed since the deferral workload, well past + # the statistics flush interval: had the bgwriter written any bin + # itself, its pg_stat_io row would show it. + is( $node->safe_psql( + 'postgres', + "SELECT coalesce(sum(writes), 0) FROM pg_stat_io WHERE object = 'dwb' AND backend_type = 'background writer'" + ), + $bgw_io_before, + 'the bgwriter wrote no bins itself throughout'); + + # Re-enabling the scan resumes the feed. + $node->append_conf('postgresql.conf', 'bgwriter_lru_maxpages = 1000'); + $node->reload; + $node->safe_psql('postgres', + "UPDATE t_solo SET filler = repeat('v', 800) WHERE id % 5 = 0"); + $node->poll_query_until('postgres', + "SELECT enqueued > $enq_frozen FROM test_dwb_cleaner_counters()") + or die 'timed out waiting for the re-enabled scan to feed the pool'; + pass('the re-enabled scan resumed feeding the pool'); +} + +# --- the queue drains and the counters reconcile ------------------------- + +# Every accepted claim ends as written or skipped, nothing else: once the +# queue is empty, enqueued = written + skipped exactly. +$node->poll_query_until( + 'postgres', q( + SELECT queued = 0 AND enqueued = written + skipped + FROM test_dwb_cleaner_counters() +)) or die 'timed out waiting for the queue to drain and reconcile'; +pass('the drained queue reconciles: enqueued = written + skipped'); + +# --- one quiet claim moves buffers_clean and republishes pg_stat_io ------ + +# With the queue drained and the system quiet, let buffers_clean settle +# (pending folds of the workload above trickle in with the bgwriter's +# reporting), then drive exactly one pool write and pin the attribution: +# buffers_clean grows while the deferral counter stands still, and the +# worker's pg_stat_io row reflects the new write too (that the report is +# forced, not merely allowed by the stats interval, has its own +# injection-point scenario below). +my ($bclean_base, $bclean_prev) = (0, -1); +my $deadline = time() + 60; +while (time() < $deadline) +{ + $bclean_base = $node->safe_psql('postgres', + 'SELECT buffers_clean FROM pg_stat_bgwriter'); + last if $bclean_base == $bclean_prev; + $bclean_prev = $bclean_base; + sleep 1; +} +my $deferred_base = counter('deferred'); +my $io_base = $node->safe_psql('postgres', + "SELECT sum(writes) FROM pg_stat_io WHERE object = 'dwb' AND backend_type = 'background worker'" +); +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_fold (id int); + INSERT INTO t_fold VALUES (1); +)); +is( $node->safe_psql( + 'postgres', "SELECT test_dwb_enqueue_block('t_fold', 0)"), + 't', + 'the queue accepted the attribution claim'); +$node->poll_query_until( + 'postgres', qq( + SELECT buffers_clean > $bclean_base FROM pg_stat_bgwriter +)) or die 'timed out waiting for the pool write to reach buffers_clean'; +is(counter('deferred'), $deferred_base, + 'the buffers_clean growth came through the pool fold alone'); +$node->poll_query_until( + 'postgres', qq( + SELECT sum(writes) > $io_base FROM pg_stat_io + WHERE object = 'dwb' AND backend_type = 'background worker' +)) or die 'timed out waiting for the pg_stat_io row to grow again'; +pass('the attribution write reached pg_stat_io as well'); + +# --- a terminated worker is restarted, the pool keeps its size ----------- + +# The workers exit FATAL on SIGTERM (exit code 1): a zero exit would +# unregister the worker for good and one stray terminate would shrink +# the pool permanently. +my $victim = $node->safe_psql('postgres', + "SELECT pid FROM pg_stat_activity WHERE backend_type = 'dwb cleaner' LIMIT 1" +); +$node->safe_psql('postgres', "SELECT pg_terminate_backend($victim)"); +$node->poll_query_until( + 'postgres', qq( + SELECT count(*) = 2 AND count(*) FILTER (WHERE pid = $victim) = 0 + FROM pg_stat_activity WHERE backend_type = 'dwb cleaner' +)) or die 'timed out waiting for the terminated cleaner to be replaced'; +pass('a terminated cleaner was restarted and the pool is back to size'); + +# --- crash recovery on top of pool-cleaned data -------------------------- + +$node->stop('immediate'); +$node->start; +is($node->safe_psql('postgres', 'SELECT count(*) FROM t_cold'), + '1', 'data intact after crash recovery'); + +# --- the idle-time report is forced, not deferred ------------------------ + +# The dwb-cleaner-reported point sits right AFTER pgstat_report_stat: +# a parked worker has published everything it wrote so far. The next +# claim is executed milliseconds after the wakeup, so the worker's next +# report attempt lands well inside PGSTAT_MIN_INTERVAL of its previous +# one — a non-forced report would be suppressed there, and since the +# worker then sleeps indefinitely, the write would never surface. The +# delta becoming visible is therefore the forced report and nothing +# else. One worker, so no second cleaner can publish the delta on its +# own schedule. +SKIP: +{ + skip 'injection points not supported by this build', 2 + unless defined $ENV{enable_injection_points} + && $ENV{enable_injection_points} eq 'yes'; + + $node->stop; + $node->append_conf('postgresql.conf', 'dwb_cleaner_workers = 1'); + $node->start; + + $node->safe_psql( + 'postgres', q( + CREATE TABLE t_pub (id int); + INSERT INTO t_pub VALUES (1); + )); + $node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-cleaner-reported', 'wait')"); + + # The waker claim: the sleeping worker only reaches the point after + # processing something and going idle again. + $node->safe_psql('postgres', "SELECT test_dwb_enqueue_block('t_pub', 0)"); + $node->wait_for_event('dwb cleaner', 'dwb-cleaner-reported'); + pass('the worker parked right after publishing its statistics'); + + my $io_pub = $node->safe_psql('postgres', + "SELECT sum(writes) FROM pg_stat_io WHERE object = 'dwb' AND backend_type = 'background worker'" + ); + $node->safe_psql( + 'postgres', q( + INSERT INTO t_pub VALUES (2); + )); + $node->safe_psql('postgres', "SELECT test_dwb_enqueue_block('t_pub', 0)"); + + $node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-cleaner-reported')"); + $node->safe_psql('postgres', + "SELECT injection_points_wakeup('dwb-cleaner-reported')"); + + $node->poll_query_until( + 'postgres', qq( + SELECT sum(writes) > $io_pub FROM pg_stat_io + WHERE object = 'dwb' AND backend_type = 'background worker' + )) or die 'timed out waiting for the forced back-to-back publication'; + pass('the report published a delta inside the minimum stats interval'); +} + +# --- a pool larger than the free worker slots refuses to start ----------- + +$node->stop; +$node->append_conf( + 'postgresql.conf', qq( +max_worker_processes = 2 +dwb_cleaner_workers = 8 +)); +my $ret = $node->start(fail_ok => 1); +is($ret, 0, 'start with an oversized cleaner pool fails'); +ok( $node->log_contains( + qr/"dwb_cleaner_workers" \(8\) needs more "max_worker_processes" slots/ + ), + 'the refusal names the pool size and the slot shortage'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/019_autovacuum_class.pl b/src/test/modules/test_dwb/t/019_autovacuum_class.pl new file mode 100644 index 0000000000000..cb3bf9cbb21b4 --- /dev/null +++ b/src/test/modules/test_dwb/t/019_autovacuum_class.pl @@ -0,0 +1,90 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Autovacuum workers write the ring through the BACKGROUND class: their +# private ring strategy makes them scheduled sequential writers, not +# latency-critical evictors. The node is configured so that background +# class traffic can come from nothing else — no cleaner pool, no LRU +# scan, checkpoints an hour away — and the vacuum buffer ring is shrunk +# so autovacuum must flush the pages it dirties. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_av_class'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_cleaner_workers = 0 +shared_buffers = 64MB +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 1h +autovacuum_naptime = 1s +vacuum_buffer_usage_limit = 128kB +log_autovacuum_min_duration = 0 +# vacuum's ring only reuses a dirty buffer whose WAL is already flushed +# (StrategyRejectBuffer); keep the flushed LSN hard on the ring's heels +# so the reject path stays cold and the flushes really happen +wal_writer_delay = 1ms +wal_writer_flush_after = 0 +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# Several hundred pages, well past the 16-buffer vacuum ring. The ring +# strategy only kicks in on reads, and pruning must be what dirties the +# pages, so the workload is staged cold: delete with autovacuum held +# off, then a clean restart empties shared buffers (and flushes the +# delete's dirt through the shutdown checkpoint). The autovacuum pass +# then reads every page through its ring, prunes it dirty, and the ring +# wrap forces the per-page DWB flushes under test. +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_av (id int, filler text) + WITH (autovacuum_enabled = off, + autovacuum_vacuum_threshold = 1, + autovacuum_vacuum_scale_factor = 0); + INSERT INTO t_av SELECT g, repeat('a', 100) FROM generate_series(1, 20000) g; + DELETE FROM t_av WHERE id % 2 = 0; +)); +$node->restart; + +my $bg_before = $node->safe_psql('postgres', + "SELECT coalesce(sum(pages), 0) FROM test_dwb_seal_stats() WHERE wclass = 'background'" +); +my $ev_before = $node->safe_psql('postgres', + "SELECT coalesce(sum(pages), 0) FROM test_dwb_seal_stats() WHERE wclass = 'eviction'" +); + +$node->safe_psql('postgres', + 'ALTER TABLE t_av SET (autovacuum_enabled = on)'); + +$node->poll_query_until( + 'postgres', qq( + SELECT coalesce(sum(pages), 0) > $bg_before FROM test_dwb_seal_stats() + WHERE wclass = 'background' +)) or die 'timed out waiting for autovacuum to write the background class'; +pass('autovacuum flushed its ring through the background class'); + +is( $node->safe_psql( + 'postgres', + "SELECT coalesce(sum(pages), 0) FROM test_dwb_seal_stats() WHERE wclass = 'eviction'" + ), + $ev_before, + 'the eviction class saw none of the autovacuum writes'); + +$node->poll_query_until( + 'postgres', q( + SELECT coalesce(sum(writes), 0) > 0 FROM pg_stat_io + WHERE object = 'dwb' AND backend_type = 'autovacuum worker' +)) or die 'timed out waiting for the autovacuum pg_stat_io dwb row'; +pass('the autovacuum worker reports its DWB batches in pg_stat_io'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/020_ckpt_yield.pl b/src/test/modules/test_dwb/t/020_ckpt_yield.pl new file mode 100644 index 0000000000000..d8ce0dedebe72 --- /dev/null +++ b/src/test/modules/test_dwb/t/020_ckpt_yield.pl @@ -0,0 +1,160 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Pressure-aware checkpoint pacing: while the cleaner pool's bin queue +# is hot, the checkpointer keeps napping slightly past its schedule, +# inside a bounded slice of the completion-target slack. The scenario +# parks the pool at an injection point, fills the queue past the hot +# threshold, and lets a timed (non-immediate — SQL CHECKPOINT would set +# CHECKPOINT_IMMEDIATE and bypass the branch) checkpoint pace itself +# over a large dirty set: stock pacing oscillates around the schedule +# boundary, writing roughly one page per nap, so with the queue hot a +# stream of pressure naps must register — and the checkpoint must still +# complete while the pressure persists, proving the yield budget is +# bounded rather than a stall. A second timed checkpoint with the pool +# released and the queue drained must add no pressure naps. +# +# The file needs real timed-checkpoint cycles (checkpoint_timeout has a +# 30 s floor), so it runs for a bit over a minute by construction. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if (!defined $ENV{enable_injection_points} + || $ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('dwb_ckpt_yield'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_cleaner_workers = 2 +shared_buffers = 32MB +bgwriter_lru_maxpages = 0 +checkpoint_timeout = 30s +checkpoint_completion_target = 0.5 +log_checkpoints = on +autovacuum = off +)); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + +$node->poll_query_until('postgres', + "SELECT count(*) = 2 FROM pg_stat_activity WHERE backend_type = 'dwb cleaner'" +) or die 'timed out waiting for the cleaner workers to start'; + +sub counter +{ + my ($name) = @_; + return $node->safe_psql('postgres', + "SELECT $name FROM test_dwb_cleaner_counters()"); +} + +sub ckpts_done +{ + return $node->safe_psql('postgres', + 'SELECT num_done FROM pg_stat_checkpointer'); +} + +# Park both workers before any dequeue: idle workers sleep on the queue +# condition variable and pass the loop top — where the point sits — +# only when woken, one targeted signal per enqueued claim. +$node->safe_psql('postgres', + q(CREATE TABLE t_park AS SELECT 1 AS id, repeat('p', 100) AS filler)); +$node->safe_psql('postgres', + "SELECT injection_points_attach('dwb-cleaner-loop', 'wait')"); +$node->safe_psql('postgres', "SELECT test_dwb_enqueue_block('t_park', 0)"); +$node->safe_psql('postgres', "SELECT test_dwb_enqueue_block('t_park', 0)"); +$node->poll_query_until( + 'postgres', q( + SELECT count(*) FILTER (WHERE wait_event = 'dwb-cleaner-loop') = 2 + FROM pg_stat_activity WHERE backend_type = 'dwb cleaner' +)) or die 'timed out waiting for both cleaners to park at the point'; +pass('both cleaners parked at the injection point'); + +# Fill the queue to capacity and keep it there: the scan is disabled +# and the pool is parked, so nothing drains it below the hot threshold +# (half of the 64-bin capacity) for the rest of the pressure phase. +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_fill AS + SELECT g AS id, repeat('x', 800) AS filler + FROM generate_series(1, 800) g; +)); +for my $blk (0 .. 66) +{ + $node->psql('postgres', "SELECT test_dwb_enqueue_block('t_fill', $blk)"); +} +cmp_ok(counter('queued'), '>=', 32, 'the bin queue is hot'); + +# The dirty set the checkpoint paces over; it stays in shared buffers +# (no LRU scan, pool parked). It must be much larger than the pacing +# can absorb at one page per nap (completion window / nap quantum ≈ 150 +# slots), or the checkpointer never falls behind schedule at an +# evaluation point and the pressure branch is never reached — ~3000 +# pages against 150 slots keeps it behind for most of the window. +$node->safe_psql( + 'postgres', q( + CREATE TABLE t_dirt AS + SELECT g AS id, repeat('d', 800) AS filler + FROM generate_series(1, 24000) g; +)); + +my $naps_before = counter('pressure_naps'); +my $done_before = ckpts_done(); + +# The next timed checkpoint runs with the queue hot the whole way (the +# workers stay parked through the assertion). Completing under +# sustained pressure is itself the boundedness proof. +$node->poll_query_until('postgres', + "SELECT num_done > $done_before FROM pg_stat_checkpointer") + or die 'timed out waiting for the timed checkpoint under pressure'; +pass('the timed checkpoint completed under sustained queue pressure'); + +cmp_ok(counter('pressure_naps'), + '>', $naps_before, + 'the checkpointer took pressure naps while the queue was hot'); + +# Release the pool: detach BEFORE waking (a woken worker loops back to +# the point and would re-park with no wakeup left), then nudge until +# both are off the point — each wakeup releases one waiter. +$node->safe_psql('postgres', + "SELECT injection_points_detach('dwb-cleaner-loop')"); +my $deadline = time() + 30; +while (time() < $deadline) +{ + last + if $node->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_activity WHERE wait_event = 'dwb-cleaner-loop'" + ) == 0; + $node->psql('postgres', + "SELECT injection_points_wakeup('dwb-cleaner-loop')"); +} +$node->poll_query_until('postgres', + 'SELECT queued = 0 FROM test_dwb_cleaner_counters()') + or die 'timed out waiting for the released pool to drain the queue'; +pass('the released pool drained the queue'); + +# Control: a comparable dirty set, the queue empty — the next timed +# (again non-immediate) checkpoint must add no pressure naps. +$node->safe_psql('postgres', + "UPDATE t_dirt SET filler = repeat('e', 800) WHERE id % 2 = 0"); +my $naps_quiet = counter('pressure_naps'); +my $done_quiet = ckpts_done(); +$node->poll_query_until('postgres', + "SELECT num_done > $done_quiet FROM pg_stat_checkpointer") + or die 'timed out waiting for the control timed checkpoint'; +is(counter('pressure_naps'), + $naps_quiet, 'no pressure naps without queue pressure'); + +done_testing(); diff --git a/src/test/modules/test_dwb/t/021_replay_warm.pl b/src/test/modules/test_dwb/t/021_replay_warm.pl new file mode 100644 index 0000000000000..d2c67cc4e8311 --- /dev/null +++ b/src/test/modules/test_dwb/t/021_replay_warm.pl @@ -0,0 +1,883 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The replay warm pool: with no full-page images in the stream, a standby +# has to fetch every page it replays, and this pool does that fetching in +# background workers instead of in the startup process. The scenarios +# here check that the pool is actually used, that over one and the same +# stretch of WAL it takes reads off the startup process, that a request a +# worker is holding survives the relation being dropped, that a worker +# killed while holding a request gives the slot back, that the pool works +# on its own with the kernel-advice prefetcher turned off, and that +# promotion with requests still outstanding is clean. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +my $injection_points = defined $ENV{enable_injection_points} + && $ENV{enable_injection_points} eq 'yes'; + +# The workload must not fit in the standby's buffer cache, or replay would +# find every page resident and the pool would have nothing to do. +my $primary = PostgreSQL::Test::Cluster->new('warm_primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +shared_buffers = 2MB +autovacuum = off +fsync = off +wal_keep_size = 256MB +)); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + +# The standbys park their workers at an injection point, so the extension +# has to exist in their catalogs; a standby cannot create it itself. +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points') + if $injection_points; + +# --- a standby with the pool enabled ------------------------------------ + +$primary->backup('bkp'); + +sub make_standby +{ + my ($name, $warm_workers, %extra) = @_; + my $node = PostgreSQL::Test::Cluster->new($name); + $node->init_from_backup($primary, 'bkp', has_streaming => 1); + $node->append_conf( + 'postgresql.conf', qq( +shared_buffers = 2MB +max_worker_processes = 16 +replay_warm_workers = $warm_workers +replay_warm_queue_size = 64 +)); + # The workers hold no database connection, so a lazily loaded injection + # point would palloc outside a transaction; preloading gives them the + # library from the postmaster instead. + $node->append_conf('postgresql.conf', + "shared_preload_libraries = 'injection_points'") + if $injection_points; + $node->append_conf('postgresql.conf', $extra{conf}) if $extra{conf}; + return $node; +} + +my $standby = make_standby('warm_standby', 2); +$standby->start; + +# The workers hold no database connection, so they have no pg_stat_activity +# row; they advertise themselves in shared memory instead. +$standby->poll_query_until('postgres', + 'SELECT count(*) = 2 FROM test_dwb_warm_worker_pids()') + or die 'timed out waiting for the warm workers to start'; +pass('both warm workers are running'); + +# a table several times the buffer cache, so replay must fetch pages +$primary->safe_psql( + 'postgres', q( + CREATE TABLE t AS + SELECT g AS id, repeat('x', 200) AS filler + FROM generate_series(1, 40000) g; + CHECKPOINT; +)); +$primary->wait_for_catchup($standby, 'replay'); +$primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('y', 200) WHERE id % 3 = 0"); +$primary->wait_for_catchup($standby, 'replay'); + +sub warm_counters +{ + my ($node) = @_; + my %c; + @c{ + qw(published dropped_full collected missed stale cancelled + released claimed reads hits failed discarded vanished) + } + = split /\|/, + $node->safe_psql('postgres', 'SELECT * FROM test_dwb_warm_counters()'); + return \%c; +} + +my $warm = warm_counters($standby); + +cmp_ok($warm->{published}, '>', 0, 'blocks were published to the warm pool'); +cmp_ok($warm->{claimed}, '>', 0, 'workers claimed published blocks'); +cmp_ok($warm->{reads}, '>', 0, 'workers read pages for replay'); +cmp_ok($warm->{collected}, '>', 0, + 'replay collected pages the workers had read'); +is($warm->{discarded}, 0, 'no worker lost its slot under a healthy run'); + +# --- the same stretch of WAL, with the pool and without ----------------- + +# One wakeup releases one waiter, and a waiter clears its registration only +# once it runs, so waking a parked pool takes as many wakeups as it takes: +# keep at it until no worker holds a request any more. Detach the point +# first, or a woken worker parks again on its next request. +sub wake_parked_workers +{ + my ($node) = @_; + + foreach my $attempt (1 .. 300) + { + return + if $node->safe_psql('postgres', + 'SELECT claimed FROM test_dwb_warm_slot_states()') == 0; + + # not safe_psql: with the last waiter already gone this errors out + $node->psql('postgres', + "SELECT injection_points_wakeup('replay-warm-before-read')"); + usleep(100_000); + } + die 'the parked workers would not wake up'; +} + +sub startup_reads +{ + my ($node) = @_; + return $node->safe_psql( + 'postgres', + q(SELECT coalesce(sum(reads), 0) FROM pg_stat_io + WHERE backend_type = 'startup' AND object = 'relation')); +} + +# Both standbys are caught up to the same point before the measured +# statement, and both are measured by their own increment over it, so the +# two numbers cover one and the same WAL. +# +# A startup process reports its statistics when it replays a running-xacts +# record, which a checkpoint emits; without one on each side of the +# measured statement the numbers would be whatever happened to have been +# reported by then. +my $plain = make_standby('plain_standby', 0); +$plain->start; +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($plain, 'replay'); +$primary->wait_for_catchup($standby, 'replay'); + +my $warm_reads_before = startup_reads($standby); +my $plain_reads_before = startup_reads($plain); + +$primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('z', 200) WHERE id % 3 = 1"); +$primary->safe_psql('postgres', 'CHECKPOINT'); +$primary->wait_for_catchup($plain, 'replay'); +$primary->wait_for_catchup($standby, 'replay'); + +$plain->poll_query_until( + 'postgres', + "SELECT coalesce(sum(reads), 0) > $plain_reads_before FROM pg_stat_io + WHERE backend_type = 'startup' AND object = 'relation'" +) or die 'timed out waiting for the unaided standby to report its reads'; + +my $warm_delta = startup_reads($standby) - $warm_reads_before; +my $plain_delta = startup_reads($plain) - $plain_reads_before; + +cmp_ok($plain_delta, '>', 0, + 'an unaided startup process reads pages over this WAL'); +cmp_ok($warm_delta, '<', $plain_delta, + 'the warm pool keeps reads off the startup process'); +$plain->stop; + +# --- a relation dropped while a worker holds a request for it ----------- + +SKIP: +{ + skip 'injection points not supported by this build', 8 + unless $injection_points; + + # The table exists on both sides before the pool is parked, so the only + # WAL left in flight — and so the only thing the parked workers can be + # holding — is the update of the table about to be dropped. + $primary->safe_psql( + 'postgres', q( + CREATE TABLE doomed AS + SELECT g AS id, repeat('d', 200) AS filler + FROM generate_series(1, 20000) g; + CHECKPOINT; + )); + $primary->wait_for_catchup($standby, 'replay'); + + # First make the workers read this table for real, so each of them holds + # open segments and a cached size for it. Creating the table would not + # have done that: its pages arrive as initialised pages, which replay + # never needs read. A worker still carrying that state into the drop is + # what the scenario is about. + my $reads_before_doomed = warm_counters($standby)->{reads}; + $primary->safe_psql('postgres', + "UPDATE doomed SET filler = repeat('e', 200) WHERE id % 2 = 0"); + $primary->wait_for_catchup($standby, 'replay'); + cmp_ok(warm_counters($standby)->{reads}, + '>', $reads_before_doomed, + 'the workers read the table before it is doomed'); + + $standby->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-before-read', 'wait')"); + + $primary->safe_psql('postgres', + "UPDATE doomed SET filler = repeat('f', 200) WHERE id % 2 = 1"); + + $standby->poll_query_until('postgres', + 'SELECT claimed = 2 FROM test_dwb_warm_slot_states()') + or die 'timed out waiting for the workers to hold requests'; + pass('both workers are parked holding a request for the doomed table'); + + my $doomed_file = $primary->safe_psql('postgres', + "SELECT relfilenode FROM pg_class WHERE relname = 'doomed'"); + my $vanished_before = warm_counters($standby)->{vanished}; + + # Replay never waits for the pool, so it drops the relation out from + # under the parked request and moves on. + $primary->safe_psql('postgres', 'DROP TABLE doomed'); + $primary->wait_for_catchup($standby, 'replay'); + pass('replay went past the drop while a request was held'); + + $standby->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-before-read')"); + wake_parked_workers($standby); + + # Each woken worker starts its read after the drop has finished, which is + # the interesting order: the interlock makes it find the relation gone + # rather than read a page through a descriptor that still reaches the + # unlinked file. + $standby->poll_query_until('postgres', + "SELECT vanished > $vanished_before FROM test_dwb_warm_counters()") + or die 'timed out waiting for the held request to fail on the drop'; + pass('a read starting after the drop finds the relation gone'); + + $standby->poll_query_until('postgres', + 'SELECT claimed = 0 FROM test_dwb_warm_slot_states()') + or die 'timed out waiting for the workers to let their slots go'; + pass('the workers gave their slots back'); + + is( $standby->safe_psql( + 'postgres', "SELECT test_dwb_count_rel_buffers($doomed_file)"), + 0, + 'no buffer was left behind for the dropped relation'); + + # a pool that only ever fails would be silently useless + my $reads_before = warm_counters($standby)->{reads}; + $primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('k', 200) WHERE id % 3 = 2"); + $primary->wait_for_catchup($standby, 'replay'); + cmp_ok(warm_counters($standby)->{reads}, + '>', $reads_before, 'the pool kept reading after the drop'); + + is(warm_counters($standby)->{discarded}, + 0, 'no result was written into a slot its worker had lost'); +} + +# --- a relation truncated under a request for a block past its new end -- + +SKIP: +{ + skip 'injection points not supported by this build', 4 + unless $injection_points; + + $primary->safe_psql( + 'postgres', q( + CREATE TABLE shrunk AS + SELECT g AS id, repeat('s', 200) AS filler + FROM generate_series(1, 20000) g; + CHECKPOINT; + )); + $primary->wait_for_catchup($standby, 'replay'); + + # Warm the workers on the full-length relation, so each of them is + # carrying its size from before the truncation. + my $reads_before = warm_counters($standby)->{reads}; + $primary->safe_psql('postgres', + "UPDATE shrunk SET filler = repeat('u', 200) WHERE id % 2 = 0"); + $primary->wait_for_catchup($standby, 'replay'); + cmp_ok(warm_counters($standby)->{reads}, + '>', $reads_before, + 'the workers read the relation at its full length'); + + # Park them on blocks near the end — the part about to be cut off. + $standby->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-before-read', 'wait')"); + $primary->safe_psql('postgres', + "UPDATE shrunk SET filler = repeat('v', 200) WHERE id > 18000"); + $standby->poll_query_until('postgres', + 'SELECT claimed = 2 FROM test_dwb_warm_slot_states()') + or die 'timed out waiting for the workers to hold requests'; + + my $vanished_before = warm_counters($standby)->{vanished}; + my $failed_before = warm_counters($standby)->{failed}; + # Vacuum gives back the empty tail. Every row goes, so the file ends up + # empty and every parked request points past its end — with rows left + # behind, the tail could still hold live versions of them and the + # truncation would stop short of the blocks the workers are holding. + $primary->safe_psql('postgres', 'DELETE FROM shrunk'); + $primary->safe_psql('postgres', 'VACUUM shrunk'); + $primary->wait_for_catchup($standby, 'replay'); + + is( $standby->safe_psql('postgres', "SELECT pg_relation_size('shrunk')"), + 0, + 'replay truncated the relation away while requests were held'); + + $standby->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-before-read')"); + wake_parked_workers($standby); + + # The size each worker remembers is from before the truncation, and + # nothing tells a process without a database connection to forget it — + # except the pool itself, which is what this asserts: the block is + # recognised as past the end instead of being read against a stale size. + $standby->poll_query_until('postgres', + "SELECT vanished > $vanished_before FROM test_dwb_warm_counters()") + or die + 'timed out waiting for the held requests to notice the truncation' + . '; counters: ' + . $standby->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_counters()') + . " (vanished was $vanished_before)"; + pass('a request for a block past the new end is dropped, not read'); + + # wake_parked_workers() returned only once no worker held a request, so + # every parked request has been dealt with by now. A worker that carried + # its old size past the check would have gone on to read past the end of + # the file and errored out, which lands in a different counter; that this + # one did not move is what says the size was refreshed rather than the + # read merely failing somewhere else. + is(warm_counters($standby)->{failed}, + $failed_before, 'no worker read against the size it remembered'); +} + +# --- a worker killed while it holds a request --------------------------- + +SKIP: +{ + skip 'injection points not supported by this build', 3 + unless $injection_points; + + $standby->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-before-read', 'wait')"); + + $primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('m', 200) WHERE id % 5 = 0"); + $standby->poll_query_until('postgres', + 'SELECT claimed = 2 FROM test_dwb_warm_slot_states()') + or die 'timed out waiting for the workers to hold requests'; + + my $released_before = warm_counters($standby)->{released}; + my $victim = $standby->safe_psql('postgres', + 'SELECT pid FROM test_dwb_warm_worker_pids() ORDER BY worker LIMIT 1' + ); + kill 'TERM', $victim; + + $standby->poll_query_until('postgres', + "SELECT released > $released_before FROM test_dwb_warm_counters()") + or die 'the killed worker did not give its slot back'; + pass('a worker killed mid-request gives its slot back'); + + # The killed worker never got to clear its registration among the point's + # waiters, and a wakeup goes to the first registration under that name — + # so it would keep going to a process that no longer exists. Nothing + # below needs the surviving worker to move: replay does not wait for the + # pool, and the pool's return is the restarted worker's doing. + $standby->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-before-read')"); + + $primary->wait_for_catchup($standby, 'replay'); + pass('replay continued across a warm worker that died'); + + $standby->poll_query_until('postgres', + 'SELECT count(*) = 2 FROM test_dwb_warm_worker_pids()') + or die 'the pool did not come back after losing a worker'; + pass('the pool restored its worker'); +} + +# --- the wakeup protocol ------------------------------------------------ + +# Replay publishes on the order of a hundred thousand blocks a second, and +# waking a worker for each of them costs a system call each time. So the +# publisher stays quiet while somebody is searching the ring, and a worker +# that stops searching hands the ring on in its place. Driving that from +# replay would mean driving it thousands of requests at a time; these +# scenarios hand the pool one request at a time instead. + +my $proto = make_standby('warm_proto', 2); +$proto->start; +$proto->poll_query_until('postgres', + 'SELECT count(*) = 2 FROM test_dwb_warm_worker_pids()') + or die 'timed out waiting for the protocol standby to start its workers'; +$primary->wait_for_catchup($proto, 'replay'); + +sub pool_state +{ + my ($node) = @_; + my %s; + @s{qw(published claimed scanners pending sleepers)} = split /\|/, + $node->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + return \%s; +} + +# With nothing being replayed the pool has nothing to search for, and the +# spin that keeps a busy worker out of the wait list runs out. +$proto->poll_query_until('postgres', + 'SELECT sleepers = 2 AND scanners = 0 FROM test_dwb_warm_slot_states()') + or die 'the idle pool never settled into sleeping workers'; +pass('an idle pool settles into sleeping workers'); + +# Quiesced, the count and the ring agree. Under load they need not: a +# publication raises the count before its slot becomes visible, so the count +# leads by whatever is in flight, and the ring walk is not a snapshot anyway. +my $quiet = pool_state($proto); +is($quiet->{pending}, $quiet->{published}, + 'the pending count matches the published slots with the pool quiesced'); + +# The edge a lost wakeup would show at: a request arriving at a pool where +# nobody is searching has to be one the publisher wakes somebody for. +my $idle_before = warm_counters($proto); +my $pf_before = $proto->safe_psql('postgres', + q{SELECT prefetch || ' ' || hit FROM pg_stat_recovery_prefetch}); +my ($pf_prefetch, $pf_hit) = split / /, $pf_before; + +my $slot = + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 0)"); +cmp_ok($slot, '>=', 0, 'the published request got a slot'); +$proto->poll_query_until('postgres', + "SELECT claimed > $idle_before->{claimed} FROM test_dwb_warm_counters()") + or die 'a request published to a sleeping pool was never claimed'; +pass('a request published to a sleeping pool is served'); + +# Whether a block was already in a buffer is decided in the pool now, and +# pg_stat_recovery_prefetch is where that decision has always been counted. +# The block above was not resident, so it was read; asking for the same block +# again is the other answer. +$proto->poll_query_until('postgres', + "SELECT prefetch > $pf_prefetch FROM pg_stat_recovery_prefetch") + or die 'the view did not count the page the pool read'; +pass('the view counts a page the pool read'); + +$proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 0)"); +$proto->poll_query_until('postgres', + "SELECT hit > $pf_hit FROM pg_stat_recovery_prefetch") + or die 'the view did not count the page the pool found resident'; +pass('the view counts a page the pool found already in a buffer'); + +# A worker can be signalled away while it sleeps, and the count it is part of +# has to go with it: the decrement that follows the sleep never runs in that +# case, and a count left standing says the pool has a sleeper it does not +# have — which is the very fact the scenarios above wait on. +$proto->poll_query_until('postgres', + 'SELECT sleepers = 2 FROM test_dwb_warm_slot_states()') + or die 'the pool did not settle before the worker was killed; state: ' + . $proto->safe_psql('postgres', 'SELECT * FROM test_dwb_warm_slot_states()') + . ' workers: ' + . $proto->safe_psql( + 'postgres', + q{SELECT string_agg(worker || ':' || pid || ':' || holding, ',') + FROM test_dwb_warm_worker_pids()}); + +my $doomed = $proto->safe_psql('postgres', + 'SELECT pid FROM test_dwb_warm_worker_pids() ORDER BY worker LIMIT 1'); +kill 'TERM', $doomed; + +$proto->poll_query_until('postgres', + 'SELECT count(*) = 2 FROM test_dwb_warm_worker_pids()') + or die 'the protocol standby did not restore its worker'; +$proto->poll_query_until('postgres', + 'SELECT sleepers = 2 FROM test_dwb_warm_slot_states()') + or die 'a worker killed in its sleep left its count behind; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); +pass('a worker killed in its sleep leaves no count behind'); + +# The pool still works afterwards, which is what the counts are for. +my $after_kill = warm_counters($proto); +$proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 4)"); +$proto->poll_query_until('postgres', + "SELECT claimed > $after_kill->{claimed} FROM test_dwb_warm_counters()") + or die 'the pool stopped serving after losing a sleeping worker'; +pass('the pool serves again after losing a sleeping worker'); + +SKIP: +{ + skip 'injection points not supported by this build', 2 + unless $injection_points; + + # This scenario holds a process still with SIGSTOP, which Windows has no + # equivalent of. + skip 'stopping and continuing a process is not portable to Windows', 2 + if $PostgreSQL::Test::Utils::windows_os; + + # The worse version of the same thing: the worker that dies is the one + # the publisher has just woken. A signal is delivered to the head of the + # wait list and takes it off that list, so if the head is on its way out, + # the request it was woken for has been told to nobody. Handing the ring + # on from the exit path is what covers that, and it is the only thing + # that can. + # + # Freezing the process is what makes the race observable. A stopped + # worker stays on the wait list and stays asleep, so the wakeup is spent + # on it and the rest of the pool hears nothing. An injection point could + # not stand in for this: preparing to sleep on the point's own variable + # cancels the registration on the pool's — see + # ConditionVariablePrepareToSleep — and that registration is the whole + # subject. + $proto->poll_query_until('postgres', + 'SELECT sleepers = 2 AND pending = 0 FROM test_dwb_warm_slot_states()' + ) or die 'the pool did not settle before the head-of-queue scenario'; + + my @pids = split /\n/, + $proto->safe_psql('postgres', + 'SELECT pid FROM test_dwb_warm_worker_pids() ORDER BY worker'); + + # The list is joined at the tail and served from the head, so replacing + # one of the two workers leaves the other at the head for certain: + # whatever the order was, the replacement can only have joined behind it. + kill 'TERM', $pids[1]; + $proto->poll_query_until( + 'postgres', + "SELECT count(*) = 2 AND count(*) FILTER (WHERE pid = $pids[1]) = 0 + FROM test_dwb_warm_worker_pids()" + ) or die 'the protocol standby did not replace the worker'; + $proto->poll_query_until('postgres', + 'SELECT sleepers = 2 FROM test_dwb_warm_slot_states()') + or die 'the pool did not settle after the worker was replaced'; + + my $head = $pids[0]; + my $rest = $proto->safe_psql('postgres', + "SELECT pid FROM test_dwb_warm_worker_pids() WHERE pid <> $head"); + + # The dying worker is replaced within a second, and a replacement is free + # to walk the ring and take whatever it finds — which would drain the + # request whether or not anybody was ever told about it. So the point + # stays attached: the worker that claims parks while still holding the + # slot, and the pool then says which pid that is. A served request is + # not the evidence here; who served it is. + $proto->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-claimed', 'wait')"); + + kill 'STOP', $head; + + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 6)"); + is( $proto->safe_psql( + 'postgres', 'SELECT pending FROM test_dwb_warm_slot_states()'), + 1, + 'the one wakeup goes to the worker at the head of the wait list'); + + # That worker never gets to serve it: it is signalled away before it runs + # again, so the request now depends entirely on what its exit path does. + kill 'TERM', $head; + kill 'CONT', $head; + + $proto->poll_query_until('postgres', + 'SELECT count(*) = 1 FROM test_dwb_warm_worker_pids() WHERE holding >= 0' + ) + or die 'nobody took the request the dying worker was woken for; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + is( $proto->safe_psql( + 'postgres', + 'SELECT pid FROM test_dwb_warm_worker_pids() WHERE holding >= 0'), + $rest, + 'a worker that dies holding the wakeup hands the ring to the sleeper' + ); + + # Let the parked worker out and put the pool back to sleep, the same way + # the scenarios below do: a detached point can still be reached by a + # worker that looked it up a moment earlier. + $proto->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-claimed')"); + foreach my $attempt (1 .. 600) + { + last + if $proto->safe_psql( + 'postgres', + 'SELECT claimed = 0 AND sleepers = 2 + FROM test_dwb_warm_slot_states()') eq 't'; + $proto->psql('postgres', + "SELECT injection_points_wakeup('replay-warm-claimed')"); + usleep(100_000); + } + $proto->poll_query_until('postgres', + 'SELECT claimed = 0 AND sleepers = 2 FROM test_dwb_warm_slot_states()' + ) + or die + 'the pool stayed parked after the head-of-queue scenario; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); +} + +SKIP: +{ + skip 'injection points not supported by this build', 3 + unless $injection_points; + + # Park a worker where it holds a request and still counts as a searcher. + # Everything published while it sits there is something the publisher + # leaves to it, so the only way the rest of the ring gets served is if + # that worker hands the ring on when it stops searching. + $proto->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-claimed', 'wait')"); + $proto->poll_query_until('postgres', + 'SELECT sleepers = 2 FROM test_dwb_warm_slot_states()') + or die 'the pool did not go back to sleep before the hand-off scenario'; + + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 1)"); + $proto->poll_query_until( + 'postgres', + 'SELECT scanners = 1 AND sleepers = 1 AND claimed = 1 + FROM test_dwb_warm_slot_states()') + or die 'no worker parked holding a request; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + pass('a worker parks holding a request, still counted as a searcher'); + + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 2)"); + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 3)"); + + # Two requests, one searcher the publisher trusted, and a worker asleep + # that was told nothing. + is( $proto->safe_psql( + 'postgres', 'SELECT pending FROM test_dwb_warm_slot_states()'), + 2, + 'requests wait while the publisher leaves them to the searcher'); + + # Let the parked worker go, but leave the point attached: it takes one + # of the two waiting requests and parks again. The other one can only + # be served by the worker that is asleep, and nothing has woken it but + # the hand-off. + $proto->safe_psql('postgres', + "SELECT injection_points_wakeup('replay-warm-claimed')"); + $proto->poll_query_until('postgres', + 'SELECT pending = 0 FROM test_dwb_warm_slot_states()') + or die 'the worker leaving the search did not hand the ring on; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + pass('a worker leaving the search hands the ring to a sleeping one'); + + # Let the parked workers out and put the pool back to sleep before the + # next scenario builds its own state. + $proto->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-claimed')"); + # "Nobody is holding a request" is not the same as "nobody is parked": + # a detached point can still be reached by a worker that looked it up a + # moment earlier, so a loop that stops at the first idle instant can + # leave the next claim parked with nothing left to wake it. Waiting for + # the pool to be asleep is the state that cannot be a gap between two + # claims. + foreach my $attempt (1 .. 600) + { + last + if $proto->safe_psql( + 'postgres', + 'SELECT claimed = 0 AND sleepers = 2 + FROM test_dwb_warm_slot_states()') eq 't'; + $proto->psql('postgres', + "SELECT injection_points_wakeup('replay-warm-claimed')"); + usleep(100_000); + } + $proto->poll_query_until('postgres', + 'SELECT claimed = 0 AND sleepers = 2 FROM test_dwb_warm_slot_states()' + ) + or die 'workers stayed parked after the point was detached; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + $proto->poll_query_until('postgres', + 'SELECT sleepers = 2 AND pending = 0 FROM test_dwb_warm_slot_states()' + ) or die 'the pool did not settle before the dying-searcher scenario'; + + # The same hand-off from the other side: the searcher the publisher + # trusted does not stop searching, it dies. Whatever it was trusted to + # find has to be picked up by the worker asleep beside it, and the only + # thing that can tell that worker is the exit path. + # + # The point stays attached on purpose. The dying worker is replaced + # within a second, and a replacement free to drain the ring would hide a + # missing hand-off: it parks on its first claim instead, so of the two + # requests waiting it can take only one, and the other is left where + # nothing but the hand-off reaches it. + $proto->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-claimed', 'wait')"); + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 5)"); + $proto->poll_query_until('postgres', + 'SELECT scanners = 1 AND claimed = 1 FROM test_dwb_warm_slot_states()' + ) or die 'no worker parked for the dying-searcher scenario'; + + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 6)"); + $proto->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', 7)"); + + # The workers advertise which slot each of them holds, so the one to end + # is the one parked rather than the one asleep. + my $victim = $proto->safe_psql('postgres', + 'SELECT pid FROM test_dwb_warm_worker_pids() WHERE holding >= 0'); + like($victim, qr/^\d+$/, 'exactly one worker is holding a request'); + kill 'TERM', $victim; + + $proto->poll_query_until('postgres', + 'SELECT pending = 0 FROM test_dwb_warm_slot_states()') + or die 'a searcher that died did not hand the ring on; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + pass('a searcher that dies hands the ring to a sleeping worker'); + + # Nobody is woken from here on, and nothing below needs a worker to + # move. The worker killed above was waiting at the point, and a waiter + # that dies leaves its registration behind: a wakeup goes to the first + # registration under that name, so it would keep going to a process that + # no longer exists. This node is finished after the check below. + $proto->safe_psql('postgres', + "SELECT injection_points_detach('replay-warm-claimed')"); + + # A worker died holding a request, which is one of the two ways a + # published request leaves the ring without a claim behind it. + $proto->poll_query_until('postgres', + 'SELECT count(*) = 2 FROM test_dwb_warm_worker_pids()') + or die 'the pool did not come back after the dying-searcher scenario'; + $proto->poll_query_until('postgres', + 'SELECT pending = published FROM test_dwb_warm_slot_states()') + or die 'the pending count and the ring disagree after a worker died ' + . 'holding a request; state: ' + . $proto->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); + pass('the pending count survives a worker dying with a request'); +} + +$proto->stop; + +# --- the pool on its own, with kernel advice turned off ----------------- + +my $noadvice = + make_standby('noadvice_standby', 2, conf => "recovery_prefetch = off\n"); +$noadvice->start; +$primary->wait_for_catchup($noadvice, 'replay'); + +$primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('n', 200) WHERE id % 7 = 0"); +$primary->wait_for_catchup($noadvice, 'replay'); + +my $na = warm_counters($noadvice); +cmp_ok($na->{published}, '>', 0, + 'blocks reach the pool with the advice prefetcher off'); +cmp_ok($na->{collected}, '>', 0, + 'replay collects the pool answers with the advice prefetcher off'); +$noadvice->stop; + +# --- a relation that grew after a worker learned its size --------------- + +# The size a worker goes by is the one the storage manager recorded when it +# last asked, and nothing tells a worker that replay has extended a +# relation. A request past the end it knows must therefore make it ask +# again rather than refuse on what it remembers. +# +# The remembered size belongs to the process that asked, so this runs with +# a single worker: with more of them the second half could land in a +# process that never saw the first and would pass without asking anything. +my $onework = make_standby('warm_onework', 1); +$onework->start; +$onework->poll_query_until('postgres', + 'SELECT count(*) = 1 FROM test_dwb_warm_worker_pids()') + or die 'timed out waiting for the single warm worker to start'; +$primary->wait_for_catchup($onework, 'replay'); + +my $past = $onework->safe_psql('postgres', + q{SELECT (pg_relation_size('t') / current_setting('block_size')::int)::int} +); + +my $before = warm_counters($onework); +$onework->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', $past)"); +$onework->poll_query_until('postgres', + "SELECT vanished > $before->{vanished} FROM test_dwb_warm_counters()") + or die 'a request past the end of a relation was not refused'; +pass('a block past the end of a relation is refused'); + +$primary->safe_psql('postgres', + q{INSERT INTO t SELECT g, repeat('z', 200) FROM generate_series(40001, 60000) g} +); +$primary->wait_for_catchup($onework, 'replay'); + +cmp_ok( + $onework->safe_psql( + 'postgres', + q{SELECT (pg_relation_size('t') / current_setting('block_size')::int)::int} + ), + '>', $past, + 'the relation grew past the refused block'); + +$before = warm_counters($onework); +$onework->safe_psql('postgres', "SELECT test_dwb_warm_publish('t', $past)"); +$onework->poll_query_until( + 'postgres', + "SELECT hits + reads > @{[ $before->{hits} + $before->{reads} ]} + FROM test_dwb_warm_counters()") + or die 'the worker went by the size it learned before the relation grew; ' + . 'counters: ' + . $onework->safe_psql('postgres', 'SELECT * FROM test_dwb_warm_counters()'); +pass('a worker asks again for a block past the size it knows'); +$onework->stop; + +# --- promotion with requests still outstanding -------------------------- + +SKIP: +{ + skip 'injection points not supported by this build', 1 + unless $injection_points; + + # Counted against what the queue already holds: a worker left parked by + # the scenario above still holds its slot, and that one is not what this + # is about. + my $held_before = $standby->safe_psql('postgres', + 'SELECT published + claimed FROM test_dwb_warm_slot_states()'); + + $standby->safe_psql('postgres', + "SELECT injection_points_attach('replay-warm-before-read', 'wait')"); + $primary->safe_psql('postgres', + "UPDATE t SET filler = repeat('p', 200) WHERE id % 11 = 0"); + $standby->poll_query_until('postgres', + "SELECT published + claimed > $held_before FROM test_dwb_warm_slot_states()" + ) + or die 'timed out waiting for the queue to hold requests; slots: ' + . $standby->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()') + . ' counters: ' + . $standby->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_counters()') + . ' pids: ' + . $standby->safe_psql('postgres', + 'SELECT count(*) FROM test_dwb_warm_worker_pids()') + . ' received/replayed: ' + . $standby->safe_psql( + 'postgres', + q{SELECT pg_last_wal_receive_lsn() || ' ' || pg_last_wal_replay_lsn()} + ); + pass('the queue holds requests going into promotion'); +} + +$standby->promote; +$standby->safe_psql('postgres', 'SELECT 1'); +pass('the standby promoted with warm requests outstanding'); + +cmp_ok( + $standby->safe_psql( + 'postgres', "SELECT count(*) FROM t WHERE filler LIKE 'y%'"), + '>', 0, + 'the promoted node has the replayed data'); + +# The end of recovery withdraws every request nobody claimed, which is the +# other way the count of outstanding requests goes down without a claim +# behind it. A withdrawal that forgot the count would leave the pool +# claiming to owe work it has thrown away. +$standby->poll_query_until('postgres', + 'SELECT pending = published FROM test_dwb_warm_slot_states()') + or die 'the pending count and the ring disagree after promotion withdrew ' + . 'the outstanding requests; state: ' + . $standby->safe_psql('postgres', + 'SELECT * FROM test_dwb_warm_slot_states()'); +pass('the pending count comes down with the requests promotion withdraws'); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_dwb/t/022_writeback_pacing.pl b/src/test/modules/test_dwb/t/022_writeback_pacing.pl new file mode 100644 index 0000000000000..f9bb1ffbccfe8 --- /dev/null +++ b/src/test/modules/test_dwb/t/022_writeback_pacing.pl @@ -0,0 +1,253 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +# The double write buffer asks the kernel to start writing a staged page back +# before the batch retires, so that the retiring fsync is a cheap barrier +# rather than a full flush. dwb_writeback_after says how many pages a process +# accumulates before it hands them over. pg_stat_io's writebacks column +# counts the pages handed over, which is what this test reads back. +# +# The threshold is visible in the counter's arithmetic: a process hands its +# pages over in whole batches of dwb_writeback_after, so what a run adds up to +# is a multiple of whatever the parameter says, which is what tells the +# thresholds apart. At a threshold of one, and whenever the batch retires +# inline (dwb_retire_workers = 0, where waiting for the array to fill would +# mean handing the page over after its own sync), every staged write is handed +# over as it happens and the two counters meet. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('dwb_writeback'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq( +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +dwb_retire_workers = 1 +dwb_writeback_after = 32 +autovacuum = off +shared_buffers = 1MB +bgwriter_lru_maxpages = 0 +backend_flush_after = 0 +checkpoint_timeout = 1h +# ScheduleBufferTagForWriteback is a no-op without it, so the counters this +# test reads would all be zero on the usual TAP setting. +fsync = on +)); +$node->start; + +# Client-backend writebacks of relation pages: what the parameter paces. +# Every page of a logged table is staged, so the writes of the same rows are +# what the hand-over is measured against. Both counters have to come out of +# one query: opening a session evicts pages of its own, so reading them one +# after another would compare two different moments. +my $counters = q( + SELECT COALESCE(sum(writebacks), 0) || ' ' || COALESCE(sum(writes), 0) + FROM pg_stat_io + WHERE backend_type = 'client backend' AND object = 'relation'); + +# A table well past shared_buffers, so an UPDATE pass has to evict. Returns +# the pages handed over and the pages written, in that order. +sub churn +{ + my ($table, $mark) = @_; + $node->safe_psql('postgres', + "UPDATE $table SET pad = repeat('$mark', 400)"); + # statistics reach the collector when the session ends + $node->safe_psql('postgres', 'SELECT 1'); + return split(/ /, $node->safe_psql('postgres', $counters)); +} + +$node->safe_psql( + 'postgres', q( + CREATE TABLE wb_logged (id int, pad text); + INSERT INTO wb_logged SELECT g, repeat('a', 400) FROM generate_series(1, 20000) g; +)); + +# --- bounds ----------------------------------------------------------------- + +is( $node->safe_psql( + 'postgres', q( + SELECT unit = (current_setting('block_size')::int / 1024) || 'kB' + FROM pg_settings WHERE name = 'dwb_writeback_after')), + 't', + 'dwb_writeback_after is measured in blocks'); + +my ($rc, $stdout, $stderr) = + $node->psql('postgres', 'SET dwb_writeback_after = 1'); +like( + $stderr, + qr/cannot be changed now/, + 'dwb_writeback_after cannot be set from a session'); + +# pg_settings.setting is the raw block count; current_setting() would render +# it with a unit, which is not what the bounds are expressed in. +sub setting +{ + return $node->safe_psql('postgres', + "SELECT setting FROM pg_settings WHERE name = 'dwb_writeback_after'"); +} + +sub set_to +{ + my ($value) = @_; + + $node->adjust_conf('postgresql.conf', 'dwb_writeback_after', $value); + $node->reload; + $node->poll_query_until('postgres', + "SELECT setting = '$value' FROM pg_settings" + . " WHERE name = 'dwb_writeback_after'") + or die "dwb_writeback_after did not reach $value"; +} + +for my $ok (0, 1, 256) +{ + set_to($ok); + is(setting(), "$ok", "dwb_writeback_after accepts $ok"); +} + +my $offset = -s $node->logfile; +$node->adjust_conf('postgresql.conf', 'dwb_writeback_after', 257); +$node->reload; +$node->wait_for_log( + qr/is outside the valid range for parameter "dwb_writeback_after"/, + $offset); +is(setting(), '256', + 'a value past the maximum is refused and the old one kept'); + +# --- the parameter's effect across a reload --------------------------------- + +set_to(0); +$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); +my ($handed, $written) = churn('wb_logged', 'b'); +is($handed, '0', 'no page is handed over while the parameter is zero'); +cmp_ok($written, '>', 1000, 'though the pass did evict'); + +# Each threshold leaves its own signature in the total, so an implementation +# that ignored the parameter and used a fixed size would fail all but one. +my $mark = 'c'; +for my $n (32, 256, 1) +{ + set_to($n); + $node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); + ($handed, $written) = churn('wb_logged', $mark++); + cmp_ok($handed, '>', 0, "pages are handed over at a threshold of $n"); + is($handed % $n, 0, "and in whole batches of $n"); + if ($n == 1) + { + is($handed, $written, + 'at a threshold of one, every staged write as it happens'); + } + else + { + cmp_ok($handed, '<=', $written, + "and never more than what was written"); + } +} + +# --- scope: only what the double write buffer stages ------------------------- + +# Built here and dropped again, so that no page of it is left in the pool to +# be evicted by a later pass and counted as a write nobody staged. +$node->safe_psql( + 'postgres', q( + CREATE UNLOGGED TABLE wb_unlogged (id int, pad text); + INSERT INTO wb_unlogged SELECT g, repeat('a', 400) FROM generate_series(1, 20000) g; +)); +set_to(32); +$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); +my ($unlogged, $writes) = churn('wb_unlogged', 'd'); +cmp_ok($writes, '>', 1000, 'the unlogged pass did evict'); +cmp_ok($unlogged, '<', $writes / 10, + 'pages the double write buffer does not stage are left to backend_flush_after' +); +$node->safe_psql('postgres', 'DROP TABLE wb_unlogged'); + +# --- synchronous retirement -------------------------------------------------- + +$node->adjust_conf('postgresql.conf', 'dwb_retire_workers', '0'); +$node->restart; +$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); +($handed, $written) = churn('wb_logged', 'e'); +cmp_ok($handed, '>', 0, + 'pages are handed over when the batch retires inline'); +is($handed, $written, + 'every one of them, at a threshold that would otherwise hold them back'); + +# The counters above cannot tell "issued before the sync" from "issued after +# it": both leave the same totals. This can. The watch runs inside whichever +# process is writing, is handed the very context that process queued into, and +# reports through the server log — so it covers the bin-gather path in the +# checkpointer as well as the eviction path in a backend, neither of which can +# be read out of the test session's memory. +SKIP: +{ + skip 'injection points not supported by this build', 3 + unless $ENV{enable_injection_points} + && $ENV{enable_injection_points} eq 'yes'; + + $node->safe_psql('postgres', 'CREATE EXTENSION test_dwb'); + my $offset = -s $node->logfile; + $node->safe_psql('postgres', 'SELECT test_dwb_watch_inline_retire()'); + + # a backend evicting, then the checkpointer flushing in bins + churn('wb_logged', 'g'); + $node->safe_psql('postgres', 'CHECKPOINT'); + + $node->wait_for_log(qr/dwb-inline-retire watch armed in client backend/, + $offset); + ok(1, 'the watch ran on the eviction path'); + $node->wait_for_log(qr/dwb-inline-retire watch armed in checkpointer/, + $offset); + ok(1, 'and on the bin-gather path'); + ok( !$node->log_contains(qr/writebacks still queued/, $offset), + 'nothing was still queued when a batch was made durable'); + + $node->safe_psql('postgres', 'SELECT test_dwb_unwatch_inline_retire()'); +} + +# --- the threshold, one page at a time --------------------------------------- + +# Everything above reads totals, which a fixed internal size could match by +# luck. This evicts named buffers one by one in a session of its own: one +# short of the threshold nothing has been handed over, and the page that +# reaches it hands over the whole array. +$node->adjust_conf('postgresql.conf', 'dwb_retire_workers', '1'); +$node->adjust_conf('postgresql.conf', 'shared_buffers', '16MB'); +$node->restart; +set_to(8); + +$node->safe_psql( + 'postgres', q( + CREATE EXTENSION pg_buffercache; + CREATE TABLE wb_exact (id int, pad text); + INSERT INTO wb_exact SELECT g, repeat('a', 400) FROM generate_series(1, 400) g; + CHECKPOINT;)); + +my $evict = q{ + SELECT count(*) FROM ( + SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache + WHERE relfilenode = pg_relation_filenode('wb_exact') AND isdirty + LIMIT }; + +my $bg = $node->background_psql('postgres'); +$bg->query("UPDATE wb_exact SET pad = repeat('h', 400)"); +$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('io')"); + +$bg->query($evict . '7) x'); +$node->poll_query_until('postgres', "SELECT ($counters) = '0 7'") + or die 'the first seven evictions did not settle'; +ok(1, 'one page short of the threshold, nothing has been handed over'); + +$bg->query($evict . '1) x'); +$node->poll_query_until('postgres', "SELECT ($counters) = '8 8'") + or die 'the eighth eviction did not settle'; +ok(1, 'the page that reaches it hands over the whole array'); +$bg->quit; + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_dwb/t/DWBTest.pm b/src/test/modules/test_dwb/t/DWBTest.pm new file mode 100644 index 0000000000000..614f89091ecde --- /dev/null +++ b/src/test/modules/test_dwb/t/DWBTest.pm @@ -0,0 +1,38 @@ + +# Copyright (c) 2025, PostgreSQL Global Development Group + +# Shared helpers for the test_dwb TAP suite: raw 8 kB block I/O on relation +# files, for damaging and inspecting pages behind the server's back. + +package DWBTest; + +use strict; +use warnings FATAL => 'all'; +use Exporter 'import'; + +our @EXPORT = qw(read_block write_block); + +sub read_block +{ + my ($file, $blkno) = @_; + my $buf; + + open my $fh, '<:raw', $file or die "could not open $file: $!"; + sysseek($fh, $blkno * 8192, 0) or die "could not seek $file: $!"; + sysread($fh, $buf, 8192) == 8192 or die "short read from $file: $!"; + close $fh; + return $buf; +} + +sub write_block +{ + my ($file, $blkno, $buf) = @_; + + open my $fh, '+<:raw', $file or die "could not open $file: $!"; + sysseek($fh, $blkno * 8192, 0) or die "could not seek $file: $!"; + syswrite($fh, $buf) == length($buf) or die "short write to $file: $!"; + close $fh; + return; +} + +1; diff --git a/src/test/modules/test_dwb/test_dwb--1.0.sql b/src/test/modules/test_dwb/test_dwb--1.0.sql new file mode 100644 index 0000000000000..5edc062d25d02 --- /dev/null +++ b/src/test/modules/test_dwb/test_dwb--1.0.sql @@ -0,0 +1,160 @@ +/* src/test/modules/test_dwb/test_dwb--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dwb" to load this file. \quit + +CREATE FUNCTION test_dwb_cycle(npages int) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_stress(loops int, npages int) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_ring_slots(current_only bool) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_ring_rel_slots(relnumber oid) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_states() + RETURNS text STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_ring_wait_retries() + RETURNS bigint STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_seal_stats( + OUT wclass text, OUT reason text, OUT seals bigint, OUT pages bigint) + RETURNS SETOF record STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_stage_lone_wait() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_overflow_lone_wait() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_set_overflow_stamp(delta_ms int) + RETURNS bool STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_leak(npages int, do_publish bool) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_force_seal(background bool DEFAULT false) + RETURNS bool STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_retire() + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_open_stale() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_fill_ring(background bool DEFAULT false) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_abort_release(npages int, do_publish bool) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_abort_after_fsync() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_torn_repair(relnumber oid, blkno int) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_checkpoint_pending(relnumber oid) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_fill_segments(nbatches int) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_leak_fsynced() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_park(relnumber oid) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_stale_snapshot(relnumber oid) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_craft_batch(batch_idx int, batch_id int8, + relnumber oid, blkno int, lsn pg_lsn, marker text) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_set_control_min_version(min_version int) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_cleaner_counters( + OUT enqueued bigint, OUT written bigint, OUT skipped bigint, + OUT deferred bigint, OUT queued int, OUT pressure_naps bigint) + RETURNS record STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_warm_counters( + OUT published bigint, OUT dropped_full bigint, OUT collected bigint, + OUT missed bigint, OUT stale bigint, OUT cancelled bigint, + OUT released bigint, OUT claimed bigint, OUT reads bigint, + OUT hits bigint, OUT failed bigint, OUT discarded bigint, + OUT vanished bigint) + RETURNS record STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_count_rel_buffers(relnumber oid) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_warm_slot_states( + OUT published int, OUT claimed int, + OUT scanners int, OUT pending int, OUT sleepers int) + RETURNS record STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_warm_publish(rel regclass, blkno int) + RETURNS int STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_warm_worker_pids( + OUT worker int, OUT pid int, OUT holding int) + RETURNS SETOF record + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_pin_block(rel regclass, blkno int) + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_unpin_block() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_enqueue_block(rel regclass, blkno int) + RETURNS bool STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_watch_inline_retire() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION test_dwb_unwatch_inline_retire() + RETURNS void STRICT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dwb/test_dwb.c b/src/test/modules/test_dwb/test_dwb.c new file mode 100644 index 0000000000000..d4569681ebcd8 --- /dev/null +++ b/src/test/modules/test_dwb/test_dwb.c @@ -0,0 +1,1523 @@ +/*-------------------------------------------------------------------------- + * + * test_dwb.c + * Test module for the short-lived double write buffer. + * + * Drives the DWB batch state machine directly with synthetic page tags and + * images — independently of the FlushBuffer integration — and validates + * the on-disk ring format independently of the server-side write path. + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dwb/test_dwb.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/xact.h" +#include "access/xlogwarm.h" +#include "catalog/pg_tablespace_d.h" +#include "common/relpath.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "storage/buf_internals.h" +#include "storage/bufmgr.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "storage/dwb.h" +#include "storage/fd.h" +#include "storage/lwlock.h" +#include "storage/smgr.h" +#include "storage/sync.h" +#include "utils/builtins.h" +#include "utils/injection_point.h" +#include "utils/pg_lsn.h" +#include "utils/rel.h" +#include "utils/resowner.h" +#include "utils/timestamp.h" +#include "varatt.h" + +PG_MODULE_MAGIC; + +/* + * Ordering watch for the inline-retirement path. + * + * Without a retire pool a page write makes its own batch durable before it + * returns, so the kernel writeback the double write buffer wants started + * ahead of that sync has to leave the pending array first. The callback + * fires in whichever process is doing the writing — a client backend for + * FlushBuffer, the checkpointer or the background writer for the bin path — + * and is handed the very context that process queued into, so it reports + * through the server log rather than through memory the test session could + * read: one line the first time it runs in a process, and a warning every + * time something was still queued, which is the ordering mistake a later + * edit could reintroduce. + */ +#ifdef USE_INJECTION_POINTS +PGDLLEXPORT void test_dwb_inline_retire_cb(const char *name, + const void *private_data, + void *arg); + +void +test_dwb_inline_retire_cb(const char *name, const void *private_data, void *arg) +{ + static bool announced = false; + WritebackContext *wb_context = (WritebackContext *) arg; + + if (!announced) + { + announced = true; + elog(LOG, "dwb-inline-retire watch armed in %s", + GetBackendTypeDesc(MyBackendType)); + } + + if (wb_context != NULL && wb_context->nr_pending > 0) + elog(WARNING, "dwb-inline-retire: %d writebacks still queued", + wb_context->nr_pending); +} +#endif + +PG_FUNCTION_INFO_V1(test_dwb_watch_inline_retire); +Datum +test_dwb_watch_inline_retire(PG_FUNCTION_ARGS) +{ +#ifdef USE_INJECTION_POINTS + InjectionPointAttach("dwb-inline-retire", "test_dwb", + "test_dwb_inline_retire_cb", NULL, 0); + PG_RETURN_VOID(); +#else + elog(ERROR, "injection points are not supported by this build"); +#endif +} + +PG_FUNCTION_INFO_V1(test_dwb_unwatch_inline_retire); +Datum +test_dwb_unwatch_inline_retire(PG_FUNCTION_ARGS) +{ +#ifdef USE_INJECTION_POINTS + (void) InjectionPointDetach("dwb-inline-retire"); + PG_RETURN_VOID(); +#else + elog(ERROR, "injection points are not supported by this build"); +#endif +} + +static void +check_dwb_enabled(void) +{ + if (!DWBIsEnabled()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("test_dwb requires io_torn_pages_protection = \"double_writes\""))); +} + +static void +wait_and_release(DWBSlotRef *refs, int nrefs) +{ + for (int i = 0; i < nrefs; i++) + { + DWBWaitBatchFsynced(&refs[i]); + DWBReleaseSlot(&refs[i]); + } +} + +/* build a MAIN_FORKNUM page tag for (relnumber, blkno) in database dboid */ +static BufferTag +make_tag(Oid dboid, Oid relnumber, BlockNumber blkno) +{ + BufferTag tag; + RelFileLocator rlocator; + + rlocator.spcOid = DEFAULTTABLESPACE_OID; + rlocator.dbOid = dboid; + rlocator.relNumber = relnumber; + InitBufferTag(&tag, &rlocator, MAIN_FORKNUM, blkno); + return tag; +} + +/* + * Acquire one slot for the tag, publish the image, seal the batch and wait + * until it is durable: the shared prologue of the single-page scenarios. + */ +static void +stage_one_page(const BufferTag *tag, const char *image, XLogRecPtr page_lsn, + bool use_resowner, DWBSlotRef *ref) +{ + DWBAcquireSlot(tag, DWB_WCLASS_EVICTION, use_resowner, ref); + DWBPublishImage(ref, image, page_lsn); + if (!DWBTrySealBatch(ref->batch_idx, DWB_SEAL_FORCED)) + ereport(ERROR, (errmsg("could not seal the batch under test"))); + DWBWaitBatchFsynced(ref); +} + +/* + * Acquire (and optionally publish) npages slots and return with the refs + * still pending: the shared body of the leak / abort-release scenarios. + */ +static void +leak_refs(int npages, bool do_publish, bool use_resowner, Oid relnumber) +{ + static char page[BLCKSZ]; + + /* stay below the batch size so this backend never seals as leader */ + if (npages < 1 || npages >= dwb_batch_pages) + ereport(ERROR, (errmsg("npages out of range"))); + + for (int i = 0; i < npages; i++) + { + BufferTag tag = make_tag(1, relnumber, (BlockNumber) i); + DWBSlotRef ref; + + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, use_resowner, &ref); + if (do_publish) + { + memset(page, 'L', BLCKSZ); + DWBPublishImage(&ref, page, (XLogRecPtr) 0x2000000 + i); + } + } +} + +/* + * One full write cycle over npages synthetic pages: acquire, publish, + * seal (by overflow or forced), wait durable, release, retire. + * Returns the number of batches retired. + */ +static int +dwb_cycle_internal(int npages) +{ + DWBSlotRef refs[DWB_BATCH_MAX_PAGES]; + int nrefs = 0; + int last_batch = -1; + int retired = 0; + static char page[BLCKSZ]; + + for (int i = 0; i < npages; i++) + { + BufferTag tag = make_tag(1, 90000 + (i % 3), (BlockNumber) i); + DWBSlotRef ref; + + CHECK_FOR_INTERRUPTS(); + + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, false, &ref); + + /* + * A batch switch means the previous batch overflowed and was sealed + * by its overflow writer: its refs can be waited for and dropped. + */ + if (last_batch >= 0 && ref.batch_idx != last_batch) + { + wait_and_release(refs, nrefs); + nrefs = 0; + retired += DWBRetireAllSync(); + } + last_batch = ref.batch_idx; + + memset(page, 'A' + (i % 26), BLCKSZ); + DWBPublishImage(&ref, page, (XLogRecPtr) 0x1000000 + i); + refs[nrefs++] = ref; + } + + /* seal the tail batch and drain */ + DWBForceSealOpenBatch(DWB_WCLASS_EVICTION); + wait_and_release(refs, nrefs); + retired += DWBRetireAllSync(); + return retired; +} + +PG_FUNCTION_INFO_V1(test_dwb_cycle); +Datum +test_dwb_cycle(PG_FUNCTION_ARGS) +{ + int npages = PG_GETARG_INT32(0); + + check_dwb_enabled(); + if (npages < 1 || npages > 100000) + ereport(ERROR, (errmsg("npages out of range"))); + + PG_RETURN_INT32(dwb_cycle_internal(npages)); +} + +PG_FUNCTION_INFO_V1(test_dwb_stress); +Datum +test_dwb_stress(PG_FUNCTION_ARGS) +{ + int loops = PG_GETARG_INT32(0); + int npages = PG_GETARG_INT32(1); + + check_dwb_enabled(); + for (int i = 0; i < loops; i++) + { + CHECK_FOR_INTERRUPTS(); + (void) dwb_cycle_internal(npages); + } + PG_RETURN_VOID(); +} + +/* + * Validate the on-disk ring the way the apply-pass will: read every batch + * file, check header, then count slots passing meta_crc (+ generation if + * current_only) + flags + image_crc. With have_filter, count only slots + * whose tag belongs to the given relation (any generation) — used to prove + * that pages of a relation never entered the ring. + */ +static int +count_ring_slots(bool current_only, bool have_filter, Oid relnumber) +{ + DWBControlFileData control; + Size meta_region; + DWSlotMeta *metas; + char *image; + int valid = 0; + + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + + meta_region = DWBMetaRegionSize(control.batch_pages); + metas = palloc(control.batch_pages * sizeof(DWSlotMeta)); + image = palloc(BLCKSZ); + + for (uint32 b = 0; b < control.num_batches; b++) + { + char path[MAXPGPATH]; + int fd; + ssize_t r; + DWBBatchHeader hdr; + + snprintf(path, MAXPGPATH, DWB_DIR "/batch_%04u", b); + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + + errno = 0; + r = pg_pread(fd, &hdr, sizeof(hdr), 0); + if (r != sizeof(hdr)) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", path))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("short read of header in \"%s\"", path))); + } + + /* an all-zero (never written) batch fails the header check */ + if (hdr.magic == DWB_BATCH_MAGIC && + EQ_CRC32C(hdr.crc, DWBBatchHeaderCrc(&hdr))) + { + ssize_t nbytes = hdr.n_slots * sizeof(DWSlotMeta); + + /* + * A CRC-valid header with out-of-range n_slots cannot happen + * under the startup geometry check; report the anomaly instead of + * silently contributing zero slots. + */ + if (hdr.n_slots > control.batch_pages) + { + ereport(WARNING, + (errmsg("batch file \"%s\" has out-of-range n_slots %u", + path, hdr.n_slots))); + nbytes = -1; + } + + if (nbytes >= 0) + { + errno = 0; + r = pg_pread(fd, metas, nbytes, sizeof(DWBBatchHeader)); + if (r != nbytes) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read metas in \"%s\": %m", + path))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("short read of metas in \"%s\"", path))); + } + } + + for (uint32 i = 0; nbytes >= 0 && i < hdr.n_slots; i++) + { + DWSlotMeta *meta = &metas[i]; + + if (!EQ_CRC32C(meta->meta_crc, DWBSlotMetaCrc(meta))) + continue; + if (meta->flags & DWB_SLOT_ABORTED) + continue; + if (current_only && meta->generation != control.generation) + continue; + if (have_filter && + BufTagGetRelNumber(&meta->tag) != (RelFileNumber) relnumber) + continue; + + errno = 0; + r = pg_pread(fd, image, BLCKSZ, + meta_region + (off_t) i * BLCKSZ); + if (r != BLCKSZ) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read image in \"%s\": %m", + path))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("short read of image in \"%s\"", path))); + } + if (!EQ_CRC32C(meta->image_crc, DWBImageCrc(image))) + continue; + valid++; + } + } + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + } + + pfree(metas); + pfree(image); + return valid; +} + +PG_FUNCTION_INFO_V1(test_dwb_ring_slots); +Datum +test_dwb_ring_slots(PG_FUNCTION_ARGS) +{ + bool current_only = PG_GETARG_BOOL(0); + + check_dwb_enabled(); + PG_RETURN_INT32(count_ring_slots(current_only, false, InvalidOid)); +} + +/* slots of one relation, any generation: 0 = never entered the ring */ +PG_FUNCTION_INFO_V1(test_dwb_ring_rel_slots); +Datum +test_dwb_ring_rel_slots(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + + check_dwb_enabled(); + PG_RETURN_INT32(count_ring_slots(false, true, relnumber)); +} + +/* + * Cumulative count of DWBOpenNewBatch iterations that went to sleep. The + * anti-spin regression in 003 asserts that a waiter parked on an unchanged + * full ring accrues these at the 1s sleep-timeout pace instead of busily + * rotating a wake token. + */ +PG_FUNCTION_INFO_V1(test_dwb_ring_wait_retries); +Datum +test_dwb_ring_wait_retries(PG_FUNCTION_ARGS) +{ + check_dwb_enabled(); + PG_RETURN_INT64((int64) pg_atomic_read_u64(&DWBCtl->ring_wait_retries)); +} + +/* + * Cumulative seal accounting: one row per (writer class, seal reason) with + * the number of seal wins and the sum of slots the sealed batches carried. + * pages/seals is the average fill a reason is responsible for. + */ +PG_FUNCTION_INFO_V1(test_dwb_seal_stats); +Datum +test_dwb_seal_stats(PG_FUNCTION_ARGS) +{ + static const char *const wclass_names[DWB_NUM_WCLASSES] = { + "eviction", "background", + }; + static const char *const reason_names[DWB_SEAL_NREASONS] = { + "overflow", "lone", "wait_timeout", "worker_timeout", "bin", "forced", + }; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + check_dwb_enabled(); + InitMaterializedSRF(fcinfo, 0); + + for (int c = 0; c < DWB_NUM_WCLASSES; c++) + for (int r = 0; r < DWB_SEAL_NREASONS; r++) + { + Datum values[4]; + bool nulls[4] = {0}; + + values[0] = CStringGetTextDatum(wclass_names[c]); + values[1] = CStringGetTextDatum(reason_names[r]); + values[2] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCtl->seal_count[c][r])); + values[3] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCtl->seal_pages[c][r])); + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + + return (Datum) 0; +} + +PG_FUNCTION_INFO_V1(test_dwb_states); +Datum +test_dwb_states(PG_FUNCTION_ARGS) +{ + int counts[DWB_OOM_RETIRING + 1] = {0}; + StringInfoData buf; + + check_dwb_enabled(); + for (int i = 0; i < dwb_num_batches; i++) + { + DWBatchState state = DWBGetBatchState(i); + + if (state <= DWB_OOM_RETIRING) + counts[state]++; + } + + initStringInfo(&buf); + appendStringInfo(&buf, + "free=%d allocated=%d sealed=%d written=%d fsynced=%d data_written=%d retiring=%d", + counts[DWB_FREE], counts[DWB_ALLOCATED], + counts[DWB_SEALED], counts[DWB_WRITTEN], + counts[DWB_FSYNCED], counts[DWB_DATA_WRITTEN], + counts[DWB_RETIRING] + counts[DWB_OOM_RETIRING]); + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + +/* + * Acquire (and optionally publish) npages slots and return WITHOUT + * releasing them: the refs stay pending, so closing the session exercises + * DWBProcExit. Unpublished slots get poisoned; published ownerless refs + * just drop their batch ref (the repair write is reserved for + * ResourceOwner-attached refs), leaving the batch completable by a later + * seal. + */ +PG_FUNCTION_INFO_V1(test_dwb_leak); +Datum +test_dwb_leak(PG_FUNCTION_ARGS) +{ + int npages = PG_GETARG_INT32(0); + bool do_publish = PG_GETARG_BOOL(1); + + check_dwb_enabled(); + leak_refs(npages, do_publish, false, 91000); + PG_RETURN_VOID(); +} + +/* + * Occupy the ring without blocking: acquire and publish slots until no + * openable FREE batch remains and the open batch is full, keeping every ref + * (the refs die with the session). Sets up ring exhaustion for the + * backpressure tests. The sliced reserves (dwb.h) shape where each class + * stops: an eviction fill may not consume the bottom DWB_BG_RESERVE FREE + * batches, a background fill from a fresh ring stops above the middle + * DWB_EVICT_RESERVE slice. Meant for dwb_retire_workers = 0, where nothing + * seals or retires behind our back. Returns the number of slots taken. + */ +PG_FUNCTION_INFO_V1(test_dwb_fill_ring); +Datum +test_dwb_fill_ring(PG_FUNCTION_ARGS) +{ + bool background = PG_GETARG_BOOL(0); + int wclass = background ? DWB_WCLASS_BACKGROUND : DWB_WCLASS_EVICTION; + int taken = 0; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + for (;;) + { + int nfree = 0; + bool can_open; + uint32 open_idx; + BufferTag tag; + DWBSlotRef ref; + + CHECK_FOR_INTERRUPTS(); + + /* hard bound of the backend-local ref array */ + if (taken >= 2 * DWB_BATCH_MAX_PAGES - 1) + break; + + for (int i = 0; i < dwb_num_batches; i++) + if (DWBGetBatchState(i) == DWB_FREE) + nfree++; + can_open = background + ? (nfree > DWB_BG_RESERVE + DWB_EVICT_RESERVE || + (nfree >= 1 && nfree <= DWB_BG_RESERVE)) + : nfree > DWB_BG_RESERVE; + open_idx = pg_atomic_read_u32(&DWBCtl->open_batch_idx[wclass]); + if (!can_open && + (open_idx == DWB_INVALID_BATCH || + (pg_atomic_read_u32(&DWBCtl->batches[open_idx].next_slot_idx) & + (DWB_SEAL_BIT | DWB_IDX_MASK)) >= (uint32) dwb_batch_pages)) + break; /* one more acquire would block */ + + tag = make_tag(1, 95000 + (background ? 1000 : 0), (BlockNumber) taken); + + DWBAcquireSlot(&tag, wclass, false, &ref); + memset(page, 'X', BLCKSZ); + DWBPublishImage(&ref, page, (XLogRecPtr) 0x6000000 + taken); + taken++; + } + + PG_RETURN_INT32(taken); +} + +/* + * Acquire (and optionally publish) npages slots WITH a ResourceOwner + * attachment, then raise an ERROR: the transaction abort must release the + * refs (poisoning unpublished slots), leaving the batch completable by a + * later seal. Exercises the abort path of the write path without a + * process exit. + */ +PG_FUNCTION_INFO_V1(test_dwb_abort_release); +Datum +test_dwb_abort_release(PG_FUNCTION_ARGS) +{ + int npages = PG_GETARG_INT32(0); + bool do_publish = PG_GETARG_BOOL(1); + + check_dwb_enabled(); + leak_refs(npages, do_publish, true, 93000); + + ereport(ERROR, (errmsg("test_dwb: deliberate abort with pending refs"))); + PG_RETURN_VOID(); /* unreachable */ +} + +/* + * Abort AFTER the batch is durable: acquire one slot with a ResourceOwner + * attachment, publish, seal, wait for DWB_FSYNCED, then ERROR. This + * exercises the REF HAND-OFF of the abort path: the fake relation makes the + * repair exit through the dropped-relation branch, and the last-ref drop + * must still finish the batch (publication, RETIRING). The repair write + * itself is exercised by test_dwb_torn_repair on a real relation. + */ +PG_FUNCTION_INFO_V1(test_dwb_abort_after_fsync); +Datum +test_dwb_abort_after_fsync(PG_FUNCTION_ARGS) +{ + BufferTag tag = make_tag(1, 94000, 0); + DWBSlotRef ref; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + memset(page, 'F', BLCKSZ); + stage_one_page(&tag, page, (XLogRecPtr) 0x5000000, true, &ref); + + ereport(ERROR, (errmsg("test_dwb: deliberate abort after batch fsync"))); + PG_RETURN_VOID(); /* unreachable */ +} + +/* + * Torn-page repair end to end on a REAL relation: read the current on-disk + * image of one block, stage it into the DWB with a ResourceOwner-attached + * ref, make the batch durable, then deliberately tear the block on disk and + * abort. The ResourceOwner release must rewrite the block from the batch + * copy (DWBRewriteAbandonedSlot); the TAP test verifies the on-disk content + * after a restart, where a failed repair surfaces as a checksum error. + */ +PG_FUNCTION_INFO_V1(test_dwb_torn_repair); +Datum +test_dwb_torn_repair(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + BlockNumber blkno = (BlockNumber) PG_GETARG_INT32(1); + BufferTag tag = make_tag(MyDatabaseId, relnumber, blkno); + DWBSlotRef ref; + RelPathStr relpath; + static PGAlignedBlock image; + static char junk[BLCKSZ / 2]; + int fd; + + check_dwb_enabled(); + + relpath = relpathperm(BufTagGetRelFileLocator(&tag), MAIN_FORKNUM); + fd = OpenTransientFile(relpath.str, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", relpath.str))); + errno = 0; + if (pg_pread(fd, image.data, BLCKSZ, (off_t) blkno * BLCKSZ) != BLCKSZ) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read block %u of file \"%s\": %m", + blkno, relpath.str))); + + /* stage the pristine image; the abort below must put it back */ + stage_one_page(&tag, image.data, PageGetLSN((Page) image.data), true, &ref); + + /* simulate a torn smgrwrite: clobber the second half of the block */ + memset(junk, 0x7F, sizeof(junk)); + errno = 0; + if (pg_pwrite(fd, junk, sizeof(junk), + (off_t) blkno * BLCKSZ + BLCKSZ / 2) != (ssize_t) sizeof(junk)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not tear block %u of file \"%s\": %m", + blkno, relpath.str))); + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", relpath.str))); + + ereport(ERROR, + (errmsg("test_dwb: deliberate abort after tearing the data page"))); + PG_RETURN_VOID(); /* unreachable */ +} + +/* + * Leave one batch RETIRING with a REAL segment in DWSegmentHash and a + * pending checkpointer sync request for that segment: stage one real block, + * make the batch durable, write the block through smgrwrite (which + * registers the sync request), and release the ref. With + * dwb_retire_workers = 0 and no explicit test_dwb_retire() call, only the + * checkpointer's ProcessSyncRequests -- wrapped by + * DWBSegmentFsyncBegin/End -- can retire the batch: the TAP test asserts + * that a CHECKPOINT alone frees the ring. + */ +PG_FUNCTION_INFO_V1(test_dwb_checkpoint_pending); +Datum +test_dwb_checkpoint_pending(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + BufferTag tag = make_tag(MyDatabaseId, relnumber, 0); + DWBSlotRef ref; + SMgrRelation reln; + + /* + * This one goes through smgr, which may reach the file with direct I/O + * and requires the alignment that needs. The other page buffers here are + * read and written through this test's own descriptors, where the block + * alignment is all that is asked for. + */ + static PGIOAlignedBlock image; + + check_dwb_enabled(); + + reln = smgropen(BufTagGetRelFileLocator(&tag), INVALID_PROC_NUMBER); + smgrread(reln, MAIN_FORKNUM, 0, image.data); + + stage_one_page(&tag, image.data, PageGetLSN((Page) image.data), false, + &ref); + + /* the data-file write; registers the checkpointer sync request */ + smgrwrite(reln, MAIN_FORKNUM, 0, image.data, false); + + DWBReleaseSlot(&ref); /* last ref: publication, RETIRING */ + PG_RETURN_VOID(); +} + +/* + * Acquire one ownerless ref, publish, seal and wait until the batch is + * durable, then return WITHOUT releasing: closing the session leaves the + * exit backstop holding the LAST ref of a DWB_FSYNCED batch, so the + * FSYNCED -> RETIRING hand-off (seg_set publication under publish_lock and + * DWBSegHashLock) runs inside the exit callback itself. This is only + * legal from before_shmem_exit, while the PGPROC is still alive. + */ +PG_FUNCTION_INFO_V1(test_dwb_leak_fsynced); +Datum +test_dwb_leak_fsynced(PG_FUNCTION_ARGS) +{ + BufferTag tag = make_tag(1, 97000, 0); + DWBSlotRef ref; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + memset(page, 'E', BLCKSZ); + stage_one_page(&tag, page, (XLogRecPtr) 0x8000000, false, &ref); + PG_RETURN_VOID(); +} + +/* + * Park one batch in RETIRING on a single fake segment (relnumber, block 0, + * database oid 1): acquire, publish, seal, wait durable, release. The + * batch stays RETIRING until something fsyncs the segment (the fake + * relation makes that an ENOENT = covered, unless the test planted a real + * obstacle at the segment path). + */ +PG_FUNCTION_INFO_V1(test_dwb_park); +Datum +test_dwb_park(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + BufferTag tag = make_tag(1, relnumber, 0); + DWBSlotRef ref; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + memset(page, 'P', BLCKSZ); + stage_one_page(&tag, page, (XLogRecPtr) 0x9000000, false, &ref); + DWBReleaseSlot(&ref); /* last ref: publication, RETIRING */ + PG_RETURN_VOID(); +} + +/* + * Replay the checkpointer's stale-snapshot hazard against a parked batch: + * DWBSegmentFsyncBegin for the parked segment WITHOUT the matching End + * (exactly the state an fsync ERROR under data_sync_retry = on leaves + * behind), then a successful Begin/End of an unrelated non-MD sync entry. + * The leftover snapshot must be dropped, not consumed: the parked batch + * has to stay RETIRING. + */ +PG_FUNCTION_INFO_V1(test_dwb_stale_snapshot); +Datum +test_dwb_stale_snapshot(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + FileTag md_tag; + FileTag clog_tag; + + check_dwb_enabled(); + + memset(&md_tag, 0, sizeof(md_tag)); + md_tag.handler = SYNC_HANDLER_MD; + md_tag.rlocator.spcOid = DEFAULTTABLESPACE_OID; + md_tag.rlocator.dbOid = 1; + md_tag.rlocator.relNumber = relnumber; + md_tag.forknum = MAIN_FORKNUM; + md_tag.segno = 0; + + /* arm the snapshot; no End, as if the fsync threw an ERROR */ + DWBSegmentFsyncBegin(&md_tag); + + /* an unrelated non-MD entry syncs successfully */ + memset(&clog_tag, 0, sizeof(clog_tag)); + clog_tag.handler = SYNC_HANDLER_CLOG; + DWBSegmentFsyncBegin(&clog_tag); + (void) DWBSegmentFsyncEnd(true); + + PG_RETURN_VOID(); +} + +/* + * Publish nbatches full batches whose slots all point at DISTINCT fake + * segments, so that RETIRING batches accumulate DWSegmentHash entries until + * the hash overflows and publication degrades to the synchronous OOM retire + * (WARNING "segment hash is full", DWB_OOM_RETIRING, batch freed by the + * publisher). Meant for dwb_retire_workers = 0 so the RETIRING batches + * keep their entries pinned. Returns the number of slots published. + */ +PG_FUNCTION_INFO_V1(test_dwb_fill_segments); +Datum +test_dwb_fill_segments(PG_FUNCTION_ARGS) +{ + int nbatches = PG_GETARG_INT32(0); + int nsegs = 0; + static char page[BLCKSZ]; + static uint32 next_relnumber = 200000; + + check_dwb_enabled(); + if (nbatches < 1 || nbatches > dwb_num_batches) + ereport(ERROR, (errmsg("nbatches out of range"))); + + for (int b = 0; b < nbatches; b++) + { + DWBSlotRef refs[DWB_BATCH_MAX_PAGES]; + + CHECK_FOR_INTERRUPTS(); + + for (int i = 0; i < dwb_batch_pages; i++) + { + BufferTag tag = make_tag(1, next_relnumber++, 0); + + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, false, &refs[i]); + /* the whole batch must be ours for the seal below to cover it */ + if (refs[i].batch_idx != refs[0].batch_idx) + ereport(ERROR, + (errmsg("segment-fill batch split unexpectedly"))); + memset(page, 'S', BLCKSZ); + DWBPublishImage(&refs[i], page, (XLogRecPtr) 0x7000000 + nsegs); + nsegs++; + } + if (!DWBTrySealBatch(refs[0].batch_idx, DWB_SEAL_FORCED)) + ereport(ERROR, (errmsg("could not seal a segment-fill batch"))); + DWBWaitBatchFsynced(&refs[0]); + for (int i = 0; i < dwb_batch_pages; i++) + DWBReleaseSlot(&refs[i]); + } + PG_RETURN_INT32(nsegs); +} + +/* + * Acquire and publish ONE synthetic page and enter the fsync wait WITHOUT + * sealing first: the only SQL driver of the lone-writer fast-seal path + * (stage_one_page force-seals and never reaches it). In a quiet class the + * wait returns through the immediate lone seal; in a hot one it sleeps + * until the timeout seal. + */ +PG_FUNCTION_INFO_V1(test_dwb_stage_lone_wait); +Datum +test_dwb_stage_lone_wait(PG_FUNCTION_ARGS) +{ + BufferTag tag = make_tag(1, 93000, 0); + DWBSlotRef ref; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + memset(page, 'Q', BLCKSZ); + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, false, &ref); + DWBPublishImage(&ref, page, (XLogRecPtr) 0xA000000); + DWBWaitBatchFsynced(&ref); + DWBReleaseSlot(&ref); + PG_RETURN_VOID(); +} + +/* + * Plant the eviction class's last-overflow-seal stamp delta_ms from now. + * A positive delta puts the stamp in the FUTURE — the backward-clock-step + * shape that DWBClassIsHot must read as quiet. + */ +PG_FUNCTION_INFO_V1(test_dwb_set_overflow_stamp); +Datum +test_dwb_set_overflow_stamp(PG_FUNCTION_ARGS) +{ + int32 delta_ms = PG_GETARG_INT32(0); + TimestampTz stamp; + + check_dwb_enabled(); + stamp = GetCurrentTimestamp() + (TimestampTz) delta_ms * 1000; + pg_atomic_write_u64(&DWBCtl->last_overflow_seal[DWB_WCLASS_EVICTION], + (uint64) stamp); + PG_RETURN_BOOL(DWBClassIsHot(DWB_WCLASS_EVICTION)); +} + +/* + * The hot-window driver: fill and overflow one batch in this backend — the + * overflow seal runs the leader write and the batch fdatasync synchronously + * right here and leaves the extra slot in the next batch — then IMMEDIATELY + * enter the fsync wait on that next-batch ref while its ref_count is 1. + * The stamp-to-check gap is a few in-process reads after the overflow's + * fdatasync (the post-write re-stamp), so the hot suppression must turn + * the would-be lone seal into the waiter's timeout seal. + */ +PG_FUNCTION_INFO_V1(test_dwb_overflow_lone_wait); +Datum +test_dwb_overflow_lone_wait(PG_FUNCTION_ARGS) +{ + DWBSlotRef refs[DWB_BATCH_MAX_PAGES + 1]; + static char page[BLCKSZ]; + int npages; + + check_dwb_enabled(); + npages = dwb_batch_pages + 1; + + for (int i = 0; i < npages; i++) + { + BufferTag tag = make_tag(1, (Oid) (93100 + i), 0); + + memset(page, 'H', BLCKSZ); + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, false, &refs[i]); + DWBPublishImage(&refs[i], page, (XLogRecPtr) 0xB000000 + i); + } + if (refs[npages - 1].batch_idx == refs[0].batch_idx) + ereport(ERROR, + (errmsg("overflow did not move the extra slot to a fresh batch"))); + + DWBWaitBatchFsynced(&refs[npages - 1]); + wait_and_release(refs, npages); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(test_dwb_force_seal); +Datum +test_dwb_force_seal(PG_FUNCTION_ARGS) +{ + bool background = PG_GETARG_BOOL(0); + + check_dwb_enabled(); + PG_RETURN_BOOL(DWBForceSealOpenBatch(background ? DWB_WCLASS_BACKGROUND + : DWB_WCLASS_EVICTION)); +} + +PG_FUNCTION_INFO_V1(test_dwb_retire); +Datum +test_dwb_retire(PG_FUNCTION_ARGS) +{ + check_dwb_enabled(); + PG_RETURN_INT32(DWBRetireAllSync()); +} + +/* + * Deterministic regression test for the stale-open ABA race: a writer that + * bounced off a sealed batch calls DWBOpenNewBatch only after the ring has + * reused the same index for a NEW live incarnation. The replacement guard + * must recognize the reuse and leave the live batch alone; the buggy + * index-only comparison would repoint open_batch_idx and orphan it. + * + * Single-backend and timing-free: we replay the loser's exact interleaving + * instead of racing two sessions. + */ +PG_FUNCTION_INFO_V1(test_dwb_open_stale); +Datum +test_dwb_open_stale(PG_FUNCTION_ARGS) +{ + uint32 stale_idx; + uint32 reopened_idx; + DWBSlotRef ref; + BufferTag tag; + static char page[BLCKSZ]; + + check_dwb_enabled(); + + /* + * Cycle once so the open batch goes through seal and retire: + * open_batch_idx afterwards still names it, sealed (SEAL_BIT is held + * through FREE) — exactly a bounced writer's stale view. + */ + (void) dwb_cycle_internal(1); + stale_idx = pg_atomic_read_u32(&DWBCtl->open_batch_idx[DWB_WCLASS_EVICTION]); + + /* + * Acquire one slot: the reservation CAS bounces on SEAL_BIT and reopens + * the lowest FREE index — the same index again, as a new live + * incarnation. + */ + tag = make_tag(1, 92000, 0); + DWBAcquireSlot(&tag, DWB_WCLASS_EVICTION, false, &ref); + memset(page, 'S', BLCKSZ); + DWBPublishImage(&ref, page, (XLogRecPtr) 0x3000000); + + reopened_idx = (uint32) ref.batch_idx; + if (reopened_idx != stale_idx) + ereport(ERROR, + (errmsg("stale-open scenario not reproduced: reopened %u, stale %u", + reopened_idx, stale_idx))); + + /* + * The ABA moment: a stale opener calls with old_idx naming the live + * reopened incarnation. The guard must not replace it. + */ + DWBOpenNewBatch(DWB_WCLASS_EVICTION, stale_idx); + + if (pg_atomic_read_u32(&DWBCtl->open_batch_idx[DWB_WCLASS_EVICTION]) != + reopened_idx) + ereport(ERROR, + (errmsg("stale open hijacked the live open batch"))); + + /* drain: seal, wait durable, release, retire */ + (void) DWBForceSealOpenBatch(DWB_WCLASS_EVICTION); + DWBWaitBatchFsynced(&ref); + DWBReleaseSlot(&ref); + (void) DWBRetireAllSync(); + + PG_RETURN_VOID(); +} + +/* + * Write a synthetic single-slot batch file directly into pg_dwb/, bypassing + * the ring state machine. The apply-pass dedup scenarios need on-disk + * layouts — the same page in two batches with chosen LSNs and batch_ids — + * that the runtime write path cannot be steered into: a sequential writer + * keeps reusing the lowest free ring index, so only the last copy of a page + * survives on disk. + * + * The image is the target block's current on-disk content with the given + * LSN, the marker planted in the page hole and the checksum recomputed, so + * an applied image is a valid page the server can read back afterwards. + * The caller keeps the batch index away from runtime traffic (quiet server, + * high index) and must not hand out an LSN beyond the current WAL insert + * position: it ends up as a real page LSN, and a later flush of that page + * would ask XLogFlush for WAL that does not exist. + */ +PG_FUNCTION_INFO_V1(test_dwb_craft_batch); +Datum +test_dwb_craft_batch(PG_FUNCTION_ARGS) +{ + int batch_idx = PG_GETARG_INT32(0); + uint64 batch_id = (uint64) PG_GETARG_INT64(1); + Oid relnumber = PG_GETARG_OID(2); + BlockNumber blkno = (BlockNumber) PG_GETARG_INT32(3); + XLogRecPtr lsn = PG_GETARG_LSN(4); + text *marker = PG_GETARG_TEXT_PP(5); + BufferTag tag = make_tag(MyDatabaseId, relnumber, blkno); + DWBControlFileData control; + DWBBatchHeader hdr; + DWSlotMeta meta; + static PGAlignedBlock image; + PageHeader ph = (PageHeader) image.data; + RelPathStr relpath; + char path[MAXPGPATH]; + char *region; + Size region_size; + int fd; + + check_dwb_enabled(); + + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + if (batch_idx < 0 || (uint32) batch_idx >= control.num_batches) + ereport(ERROR, (errmsg("batch index out of range"))); + + /* base image: the block's current on-disk content */ + relpath = relpathperm(BufTagGetRelFileLocator(&tag), MAIN_FORKNUM); + fd = OpenTransientFile(relpath.str, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", relpath.str))); + errno = 0; + if (pg_pread(fd, image.data, BLCKSZ, (off_t) blkno * BLCKSZ) != BLCKSZ) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read block %u of file \"%s\": %m", + blkno, relpath.str))); + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", relpath.str))); + + if (PageIsNew((Page) image.data)) + ereport(ERROR, + (errmsg("block %u of \"%s\" is empty on disk; CHECKPOINT first", + blkno, relpath.str))); + if ((Size) (ph->pd_upper - ph->pd_lower) < VARSIZE_ANY_EXHDR(marker)) + ereport(ERROR, (errmsg("marker does not fit into the page hole"))); + + memcpy(image.data + ph->pd_lower, VARDATA_ANY(marker), + VARSIZE_ANY_EXHDR(marker)); + PageSetLSN((Page) image.data, lsn); + ph->pd_checksum = pg_checksum_page(image.data, blkno); + + memset(&meta, 0, sizeof(meta)); + meta.tag = tag; + meta.page_lsn = lsn; + meta.generation = control.generation; + meta.image_crc = DWBImageCrc(image.data); + meta.meta_crc = DWBSlotMetaCrc(&meta); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = DWB_BATCH_MAGIC; + hdr.version = DWB_VERSION; + hdr.batch_id = batch_id; + hdr.n_slots = 1; + hdr.crc = DWBBatchHeaderCrc(&hdr); + + region_size = DWBMetaRegionSize(control.batch_pages); + region = palloc0(region_size); + memcpy(region, &hdr, sizeof(hdr)); + memcpy(region + sizeof(hdr), &meta, sizeof(meta)); + + DWBBatchFilePath(path, batch_idx); + fd = OpenTransientFile(path, O_RDWR | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + errno = 0; + if (pg_pwrite(fd, region, region_size, 0) != (ssize_t) region_size || + pg_pwrite(fd, image.data, BLCKSZ, (off_t) region_size) != BLCKSZ) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", path))); + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + pfree(region); + + PG_RETURN_VOID(); +} + +/* + * Rewrite pg_dwb/control with the given min_version (and a matching CRC): + * the state a ring left behind by a newer server would present after a + * binary downgrade. The next start must refuse it with the format-version + * FATAL, never with the "corrupt, remove pg_dwb" advice — the ring is + * intact and may hold unapplied repairs only the newer server can read. + */ +PG_FUNCTION_INFO_V1(test_dwb_set_control_min_version); +Datum +test_dwb_set_control_min_version(PG_FUNCTION_ARGS) +{ + DWBControlFileData control; + + check_dwb_enabled(); + + if (!DWBReadControlFile(&control, false, NULL)) + pg_unreachable(); + control.min_version = (uint32) PG_GETARG_INT32(0); + control.crc = DWBControlCrc(&control); + DWBWriteControlFile(&control); + + PG_RETURN_VOID(); +} + +/* ---------------------------------------------------------------- + * cleaner worker pool helpers + * ---------------------------------------------------------------- + */ + +static void +check_cleaners_enabled(void) +{ + check_dwb_enabled(); + if (!DWBCleanersActive()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the cleaner pool is not configured"), + errhint("Set \"dwb_cleaner_workers\" above 0."))); +} + +/* + * Counters of the cleaner work queue: enqueued/written/skipped pages, + * bins the bgwriter deferred against a full queue, bins currently + * queued. The written count is the never-reset total (the drainable + * one feeds pg_stat_bgwriter). + */ +PG_FUNCTION_INFO_V1(test_dwb_cleaner_counters); +Datum +test_dwb_cleaner_counters(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[6]; + bool nulls[6] = {0}; + int queued; + + check_cleaners_enabled(); + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + LWLockAcquire(DWBCleanerQueueLock, LW_SHARED); + queued = DWBCleanerQueue->nqueued; + LWLockRelease(DWBCleanerQueueLock); + + values[0] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCleanerQueue->enqueued_pages)); + values[1] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCleanerQueue->pool_written_total)); + values[2] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCleanerQueue->skipped_pages)); + values[3] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCleanerQueue->deferred_bins)); + values[4] = Int32GetDatum(queued); + values[5] = Int64GetDatum( + (int64) pg_atomic_read_u64(&DWBCleanerQueue->pressure_naps)); + + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +/* + * Counters of the replay warm pool: what the publisher handed over and + * what came back, and what the workers did with it. Errors out when the + * pool is not configured, so a test cannot mistake "off" for "idle". + */ +PG_FUNCTION_INFO_V1(test_dwb_warm_counters); +Datum +test_dwb_warm_counters(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[13]; + bool nulls[13] = {0}; + XLogWarmStats stats; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (!XLogWarmGetStats(&stats)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the replay warm pool is not configured"), + errhint("Set \"replay_warm_workers\" above 0."))); + + values[0] = Int64GetDatum((int64) stats.published); + values[1] = Int64GetDatum((int64) stats.dropped_full); + values[2] = Int64GetDatum((int64) stats.collected); + values[3] = Int64GetDatum((int64) stats.missed); + values[4] = Int64GetDatum((int64) stats.stale); + values[5] = Int64GetDatum((int64) stats.cancelled); + values[6] = Int64GetDatum((int64) stats.released); + values[7] = Int64GetDatum((int64) stats.claimed); + values[8] = Int64GetDatum((int64) stats.reads); + values[9] = Int64GetDatum((int64) stats.hits); + values[10] = Int64GetDatum((int64) stats.failed); + values[11] = Int64GetDatum((int64) stats.discarded); + values[12] = Int64GetDatum((int64) stats.vanished); + + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +/* + * Pids of the running warm workers, and the slot each one holds. They keep + * no database connection, so pg_stat_activity cannot show them; this is how + * a test finds one to kill, how it finds the one holding a given request, + * and how an operator sees the pool is alive. A worker holding nothing + * reports -1. + */ +PG_FUNCTION_INFO_V1(test_dwb_warm_worker_pids); +Datum +test_dwb_warm_worker_pids(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + int pids[XLOGWARM_MAX_WORKERS]; + int slots[XLOGWARM_MAX_WORKERS]; + int nworkers; + + InitMaterializedSRF(fcinfo, 0); + + nworkers = XLogWarmGetWorkerPids(pids, slots); + for (int i = 0; i < nworkers; i++) + { + Datum values[3]; + bool nulls[3] = {0}; + + values[0] = Int32GetDatum(i); + values[1] = Int32GetDatum(pids[i]); + values[2] = Int32GetDatum(slots[i]); + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); + } + + PG_RETURN_VOID(); +} + +/* + * What the pool is doing right now: requests waiting for a worker, requests + * a worker holds, and what the workers themselves are up to. The running + * totals say what has happened; this is what a test needs to catch a request + * in flight, or to wait until the whole pool is asleep. + * + * The first two come from a walk of the ring, which slots change state + * under, so they are an impression rather than an instant. The last three + * are counters and are exact. + */ +PG_FUNCTION_INFO_V1(test_dwb_warm_slot_states); +Datum +test_dwb_warm_slot_states(PG_FUNCTION_ARGS) +{ + TupleDesc tupdesc; + Datum values[5]; + bool nulls[5] = {0}; + int published; + int claimed; + int scanners; + int pending; + int sleepers; + + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + XLogWarmGetSlotCounts(&published, &claimed); + XLogWarmGetPoolState(&scanners, &pending, &sleepers); + + values[0] = Int32GetDatum(published); + values[1] = Int32GetDatum(claimed); + values[2] = Int32GetDatum(scanners); + values[3] = Int32GetDatum(pending); + values[4] = Int32GetDatum(sleepers); + + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + +/* + * How many buffers still hold pages of this relation file. After replay has + * dropped a relation the answer must be zero, whatever the warm pool was + * doing at the time — a page left behind for a relation that no longer exists + * is the failure this counts. + */ +PG_FUNCTION_INFO_V1(test_dwb_count_rel_buffers); +Datum +test_dwb_count_rel_buffers(PG_FUNCTION_ARGS) +{ + Oid relnumber = PG_GETARG_OID(0); + int count = 0; + + for (int i = 0; i < NBuffers; i++) + { + BufferDesc *desc = GetBufferDescriptor(i); + uint32 state = LockBufHdr(desc); + + if ((state & BM_TAG_VALID) && + desc->tag.relNumber == relnumber) + count++; + + UnlockBufHdr(desc, state); + } + + PG_RETURN_INT32(count); +} + +/* + * Resolve (relation, block) to the buffer currently holding it. The + * transient pin is dropped before returning; the id is a hint exactly + * like a queued bin entry. + */ +static int +lookup_block_buf_id(Oid relid, BlockNumber blkno) +{ + Relation rel; + Buffer buf; + int buf_id; + + rel = relation_open(relid, AccessShareLock); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, NULL); + buf_id = buf - 1; + ReleaseBuffer(buf); + /* keep the relation lock till end of transaction */ + relation_close(rel, NoLock); + return buf_id; +} + +/* + * Pin one block for the rest of the current transaction (the pin is + * registered with the top transaction's resource owner, so it survives + * statement end). The deterministic way to make a queued page "hot": + * an open cursor does not promise which buffer it pins. Pair with + * test_dwb_unpin_block in the SAME transaction; at transaction end the + * owner releases the pin itself and the callback below drops the stale + * reference, so a commit or rollback with the pin still "held" leaves + * the helpers reusable (the commit prints the owner's leak warning). + */ +static Buffer test_pinned_buf = InvalidBuffer; +static bool test_pin_callback_registered = false; + +static void +test_dwb_pin_xact_callback(XactEvent event, void *arg) +{ + switch (event) + { + case XACT_EVENT_COMMIT: + case XACT_EVENT_PARALLEL_COMMIT: + case XACT_EVENT_ABORT: + case XACT_EVENT_PARALLEL_ABORT: + case XACT_EVENT_PREPARE: + test_pinned_buf = InvalidBuffer; + break; + default: + break; + } +} + +/* + * Hand the pool one request, from here instead of from replay. + * + * The pool's wakeup protocol is about what happens between a publication and + * the workers, and driving it from replay means driving it from a stream of + * thousands a second — nothing a test can aim. This publishes exactly one + * request, so a test can set the pool up in a known state and then watch what + * a single block does to it. Meant for a pool that is otherwise idle: the + * publisher's slot cursor and request ids are per-process, so a second + * publisher alongside a busy replay would be publishing into the same ring + * with a cursor of its own. + */ +PG_FUNCTION_INFO_V1(test_dwb_warm_publish); +Datum +test_dwb_warm_publish(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + BlockNumber blkno = (BlockNumber) PG_GETARG_INT32(1); + Relation rel; + RelFileLocator rlocator; + uint64 request_id; + int slot_no; + + if (!XLogWarmPoolActive()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("the replay warm pool is not configured"), + errhint("Set \"replay_warm_workers\" above 0."))); + + rel = relation_open(relid, AccessShareLock); + rlocator = rel->rd_locator; + relation_close(rel, NoLock); + + slot_no = XLogWarmPublish(rlocator, MAIN_FORKNUM, blkno, &request_id); + + PG_RETURN_INT32(slot_no); +} + +PG_FUNCTION_INFO_V1(test_dwb_pin_block); +Datum +test_dwb_pin_block(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + BlockNumber blkno = (BlockNumber) PG_GETARG_INT32(1); + Relation rel; + ResourceOwner oldowner; + + if (BufferIsValid(test_pinned_buf)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("a block is already pinned"))); + + if (!test_pin_callback_registered) + { + RegisterXactCallback(test_dwb_pin_xact_callback, NULL); + test_pin_callback_registered = true; + } + + rel = relation_open(relid, AccessShareLock); + oldowner = CurrentResourceOwner; + CurrentResourceOwner = TopTransactionResourceOwner; + test_pinned_buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, + RBM_NORMAL, NULL); + CurrentResourceOwner = oldowner; + relation_close(rel, NoLock); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(test_dwb_unpin_block); +Datum +test_dwb_unpin_block(PG_FUNCTION_ARGS) +{ + ResourceOwner oldowner; + + if (!BufferIsValid(test_pinned_buf)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("no block is pinned"))); + + oldowner = CurrentResourceOwner; + CurrentResourceOwner = TopTransactionResourceOwner; + ReleaseBuffer(test_pinned_buf); + CurrentResourceOwner = oldowner; + test_pinned_buf = InvalidBuffer; + + PG_RETURN_VOID(); +} + +/* + * Hand a one-entry bin naming (relation, block) straight to the cleaner + * queue, bypassing the bgwriter's scan: the deterministic driver for the + * stale-claim scenarios. Returns whether the queue accepted it. + * + * The lookup's own transient pin bumps the usage count, which would make + * every claim read as hot, so the count is zeroed after the pin drops — + * the same cooling the clock hand performs when it sweeps past. A page + * some other session holds pinned stays hot through its refcount. + */ +PG_FUNCTION_INFO_V1(test_dwb_enqueue_block); +Datum +test_dwb_enqueue_block(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + BlockNumber blkno = (BlockNumber) PG_GETARG_INT32(1); + int buf_id; + BufferDesc *bufHdr; + uint32 buf_state; + + check_cleaners_enabled(); + + buf_id = lookup_block_buf_id(relid, blkno); + bufHdr = GetBufferDescriptor(buf_id); + buf_state = LockBufHdr(bufHdr); + buf_state &= ~BUF_USAGECOUNT_MASK; + UnlockBufHdr(bufHdr, buf_state); + + PG_RETURN_BOOL(DWBCleanerEnqueueBin(&buf_id, 1)); +} diff --git a/src/test/modules/test_dwb/test_dwb.conf b/src/test/modules/test_dwb/test_dwb.conf new file mode 100644 index 0000000000000..6dd4c21593b5c --- /dev/null +++ b/src/test/modules/test_dwb/test_dwb.conf @@ -0,0 +1,10 @@ +io_torn_pages_protection = double_writes +dwb_num_batches = 16 +dwb_batch_pages = 16 +# keep sealing and retirement under the test's control +dwb_retire_workers = 0 +# wide margin for the hot/quiet lone-seal scenarios: the hot check must run +# within this window of its overflow, the quiet one clearly outside it +dwb_batch_timeout_ms = 200ms +bgwriter_lru_maxpages = 0 +autovacuum = off diff --git a/src/test/modules/test_dwb/test_dwb.control b/src/test/modules/test_dwb/test_dwb.control new file mode 100644 index 0000000000000..3063bd3cc54c8 --- /dev/null +++ b/src/test/modules/test_dwb/test_dwb.control @@ -0,0 +1,4 @@ +comment = 'Test code for the short-lived double write buffer' +default_version = '1.0' +module_pathname = '$libdir/test_dwb' +relocatable = true diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 37df27351d8af..300c8a4997e95 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -16,17 +16,20 @@ SHOW track_counts; -- must be on SELECT backend_type, object, context FROM pg_stat_io ORDER BY backend_type COLLATE "C", object COLLATE "C", context COLLATE "C"; backend_type|object|context +autovacuum launcher|dwb|normal autovacuum launcher|relation|bulkread autovacuum launcher|relation|init autovacuum launcher|relation|normal autovacuum launcher|wal|init autovacuum launcher|wal|normal +autovacuum worker|dwb|normal autovacuum worker|relation|bulkread autovacuum worker|relation|init autovacuum worker|relation|normal autovacuum worker|relation|vacuum autovacuum worker|wal|init autovacuum worker|wal|normal +background worker|dwb|normal background worker|relation|bulkread background worker|relation|bulkwrite background worker|relation|init @@ -35,14 +38,17 @@ background worker|relation|vacuum background worker|temp relation|normal background worker|wal|init background worker|wal|normal +background writer|dwb|normal background writer|relation|init background writer|relation|normal background writer|wal|init background writer|wal|normal +checkpointer|dwb|normal checkpointer|relation|init checkpointer|relation|normal checkpointer|wal|init checkpointer|wal|normal +client backend|dwb|normal client backend|relation|bulkread client backend|relation|bulkwrite client backend|relation|init @@ -51,6 +57,7 @@ client backend|relation|vacuum client backend|temp relation|normal client backend|wal|init client backend|wal|normal +io worker|dwb|normal io worker|relation|bulkread io worker|relation|bulkwrite io worker|relation|init @@ -59,6 +66,7 @@ io worker|relation|vacuum io worker|temp relation|normal io worker|wal|init io worker|wal|normal +slotsync worker|dwb|normal slotsync worker|relation|bulkread slotsync worker|relation|bulkwrite slotsync worker|relation|init @@ -67,6 +75,7 @@ slotsync worker|relation|vacuum slotsync worker|temp relation|normal slotsync worker|wal|init slotsync worker|wal|normal +standalone backend|dwb|normal standalone backend|relation|bulkread standalone backend|relation|bulkwrite standalone backend|relation|init @@ -74,6 +83,7 @@ standalone backend|relation|normal standalone backend|relation|vacuum standalone backend|wal|init standalone backend|wal|normal +startup|dwb|normal startup|relation|bulkread startup|relation|bulkwrite startup|relation|init @@ -83,6 +93,7 @@ startup|wal|init startup|wal|normal walreceiver|wal|init walreceiver|wal|normal +walsender|dwb|normal walsender|relation|bulkread walsender|relation|bulkwrite walsender|relation|init @@ -95,7 +106,7 @@ walsummarizer|wal|init walsummarizer|wal|normal walwriter|wal|init walwriter|wal|normal -(79 rows) +(90 rows) \a -- ensure that both seqscan and indexscan plans are allowed SET enable_seqscan TO on; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 8cd74c4e5b6eb..837561d23638e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -600,7 +600,27 @@ DR_sqlfunction DR_transientrel DSMRegistryCtxStruct DSMRegistryEntry +DWBAppliedFork +DWBApplyCandidate +DWBBatchHeader +DWBCleanerBin +DWBCleanerCtl +DWBControlFileData +DWBOnStall +DWBPendingRef +DWBRetiringBatch +DWBSealReason +DWBSegSyncSnap +DWBSlotRef +DWBStallState +DWBTornPageProtection +DWBatchCtl +DWBatchState +DWCtl DWORD +DWSegEntry +DWSegRef +DWSlotMeta DataDirSyncMethod DataDumperPtr DataPageDeleteStack