From c9eccda22df620a7a6a6100a5e1ae56673460b8c Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 01:08:17 +0200 Subject: [PATCH 01/55] monitor: add Archiving & Disaster Recovery schema (M1: schema + monitor API) Adds the SQL-only foundation for the Archiver process identity / ARCHIVING node membership / base-backup policy / PITR schema described in the archiving-disaster-recovery design doc, milestone 1 (schema + monitor API only -- no service_archiver process involved yet, everything is exercised via direct SQL calls against a plain cluster). - pgautofailover.replication_state gains a new 'archiving' terminal state. - pgautofailover.node gains haspgdata bool, distinguishing ordinary Postgres instances from lightweight ARCHIVING membership rows (a pg_receivewal client, no PGDATA). The old unconditional UNIQUE (nodehost, nodeport) constraint is replaced with a partial unique index scoped to haspgdata rows, since one archiver's (hostname, 0) pair is deliberately shared across every group it serves. - New types, tables and ~26 plpgsql/SQL functions covering: archiver registration and storage targets (local + rclone), formation/group archiver policy (quorum, base-backup policy, replication-quorum eligibility), WAL capture confirmation (wal_archived()/ report_wal_received()), base-backup lifecycle and pruning, warm-standby archiver_node rows with a maxresidentreplay cap, and PITR node lifecycle + command queue. - pgautofailover--2.2--2.3.sql mirrors the same DDL incrementally, since 2.3 hasn't shipped yet; verified end-to-end against a real 1.0 -> ... -> 2.2 -> 2.3 upgrade (including the pre-existing node_nodehost_nodeport_key1 constraint name quirk from two earlier migrations each recreating the table). - New archiving_schema regress test exercising the full schema end-to-end via direct SQL, added at the end of regress_schedule (after cluster_init_failover_rule_attribution, before the dummy_update/ drop_extension/upgrade trio that must stay last) since its expected output pins literal id values tied to its exact position in the shared contrib_regression database, same as every other test in this schedule. Full local regress (20/20) + isolation (6/6) schedules pass, plus a verified real extension upgrade from 2.2 to 2.3. --- src/monitor/expected/archiving_schema.out | 289 +++++ src/monitor/pgautofailover--2.2--2.3.sql | 1282 +++++++++++++++++++++ src/monitor/pgautofailover.sql | 1257 +++++++++++++++++++- src/monitor/regress_schedule | 1 + src/monitor/sql/archiving_schema.sql | 189 +++ 5 files changed, 3015 insertions(+), 3 deletions(-) create mode 100644 src/monitor/expected/archiving_schema.out create mode 100644 src/monitor/sql/archiving_schema.sql diff --git a/src/monitor/expected/archiving_schema.out b/src/monitor/expected/archiving_schema.out new file mode 100644 index 000000000..7743cae6a --- /dev/null +++ b/src/monitor/expected/archiving_schema.out @@ -0,0 +1,289 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. +\x on +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+------------------------------------ +create_formation | (archiving_test,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); +-- ── register_archiver ──────────────────────────────────────────────────── +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset +SELECT archiverid, archivername, hostname, basebackuppolicyid, autoregister, + maxresidentreplay + FROM pgautofailover.archiver; +-[ RECORD 1 ]------+---------------- +archiverid | 1 +archivername | archiver1 +hostname | archiver1.local +basebackuppolicyid | 1 +autoregister | t +maxresidentreplay | 1 + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; +-[ RECORD 1 ]-----+------ +archiverstorageid | 1 +archiverid | 1 +storagemethod | local +storagepath | +rcloneconfigid | + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 40 + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; +-[ RECORD 1 ]-+---------------- +nodeid | 40 +formationid | archiving_test +groupid | 0 +nodename | archiver-1-0 +nodehost | archiver1.local +nodeport | 0 +goalstate | wait_standby +reportedstate | wait_standby +haspgdata | f + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; +-[ RECORD 1 ]--+------------- +archivernodeid | 1 +archiverid | 1 +kind | wal-receiver +nodeid | 40 + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 41 + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); +-[ RECORD 1 ]-------+- +report_wal_received | + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | t + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | f + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-[ RECORD 1 ]-------------+-- +archiverquorum | 1 +basebackuppolicyid | +replicationquorumeligible | f + +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); +-[ RECORD 1 ]-------------+-- +archiverquorum | 2 +basebackuppolicyid | +replicationquorumeligible | f + +-- ── base backup lifecycle ─────────────────────────────────────────────────── +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); +-[ RECORD 1 ]---------------+- +report_basebackup_completed | + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+---------- +basebackupid | 1 +status | complete +startlsn | 0/500000 +endlsn | 0/1000000 +sizebytes | 123456789 + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); +-[ RECORD 1 ]+--------------- +basebackupid | 1 +formationid | archiving_test +groupid | 0 +status | complete + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); +-[ RECORD 1 ]------+-- +prune_archiver_wal | 0 + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +-[ RECORD 1 ]-------------+- +report_basebackup_deleted | + +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+-------- +basebackupid | 1 +status | deleted +was_deleted | t + +-- ── rclone_config + archiver_storage ───────────────────────────────────── +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; +-[ RECORD 1 ]-----+------- +archiverstorageid | 1 +storagemethod | local +rcloneconfigid | +-[ RECORD 2 ]-----+------- +archiverstorageid | 3 +storagemethod | rclone +rcloneconfigid | 1 + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset +SELECT pgautofailover.archiver_remove_storage(:local_storageid); +ERROR: archiver_storage 1 does not exist, or is the mandatory local target +CONTEXT: PL/pgSQL function pgautofailover.archiver_remove_storage(bigint) line 8 at RAISE +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +-[ RECORD 1 ]-----------+- +archiver_remove_storage | + +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; +-[ RECORD 1 ]-------------+-- +remaining_storage_targets | 1 + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); +ERROR: archiver 1 is already at its maxresidentreplay cap (1) +CONTEXT: PL/pgSQL function pgautofailover.create_archiver_node(bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,integer,pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) line 18 at RAISE +-- ── PITR lifecycle ─────────────────────────────────────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); +-[ RECORD 1 ]-----------------+- +set_archiver_node_pitr_status | + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; +-[ RECORD 1 ]------+--------- +archivernodeid | 4 +archiverid | 1 +pitrstatus | paused +lastoperation | status +observedlsn | 0/900000 +observedpausestate | paused + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +-[ RECORD 1 ]------+- +pitr_queue_command | + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+-------- +pitr_next_command | promote + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+----- +pitr_next_command | none + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; +-[ RECORD 1 ]--+-- +should_be_zero | 0 + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; +-[ RECORD 1 ]-------+-- +should_also_be_zero | 0 + diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index 871e6b83a..f39cfeded 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -689,3 +689,1285 @@ with last_events as ) select * from last_events order by eventtime, eventid; $$; + + +-- +-- Archiving & Disaster Recovery, milestone 1: schema + monitor API only +-- (#TODO -- update with the actual PR number once opened). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design, and +-- pgautofailover.sql's own comments on each object below (this mirrors +-- that file's DDL, applied incrementally to an existing 2.2 install +-- instead of as part of a fresh CREATE EXTENSION). +-- + +-- New terminal state for a pgautofailover.node row representing an +-- ARCHIVING membership (see haspgdata below) rather than an ordinary +-- Postgres instance. Safe to add live: nothing in this script uses the +-- new value in the same transaction it's added in. +ALTER TYPE pgautofailover.replication_state ADD VALUE 'archiving'; + +-- true for every ordinary Postgres node (its own PGDATA, promotable); +-- false only for an ARCHIVING membership row (a pg_receivewal client, +-- no PGDATA, no postmaster to manage). See archiving-disaster-recovery +-- design: this single boolean is what candidate_priority enforcement, +-- keeper_ensure_current_state's liveness check, and the FAST_FORWARD +-- source-selection branch all key off, instead of a third node-kind +-- value -- a cascading follower is still haspgdata = true, and a +-- future proxy never becomes a pgautofailover.node row at all. +ALTER TABLE pgautofailover.node + ADD COLUMN IF NOT EXISTS haspgdata bool NOT NULL DEFAULT true; + +-- The old "any nodehost:port can only be a unique node in the system" +-- constraint (unconditional UNIQUE (nodehost, nodeport), added in +-- 1.5--1.6) can't hold for ARCHIVING rows: one archiver's (hostname, 0) +-- pair is deliberately shared across every group it serves. Replace it +-- with the same partial unique index pgautofailover.sql's fresh-install +-- table definition uses, scoped to haspgdata rows only. +-- +-- The live constraint name is node_nodehost_nodeport_key1, not the +-- "expected" node_nodehost_nodeport_key: both 1.3--1.4 and 1.5--1.6 +-- separately recreate pgautofailover.node with their own unnamed +-- UNIQUE (nodehost, nodeport), so by the time a real install reaches +-- 2.2 (via the only upgrade path that exists -- there is no standalone +-- "--2.2.sql", so even a "fresh" VERSION '2.2' install runs this same +-- incremental chain from 1.0), Postgres has already disambiguated the +-- second one with a "1" suffix. Verified empirically against a real +-- 1.0 -> ... -> 2.2 -> 2.3 upgrade, not guessed from naming convention. +ALTER TABLE pgautofailover.node + DROP CONSTRAINT IF EXISTS node_nodehost_nodeport_key1; + +CREATE UNIQUE INDEX IF NOT EXISTS node_nodehost_nodeport_haspgdata_idx + ON pgautofailover.node (nodehost, nodeport) + WHERE haspgdata; + +-- +-- +-- Archiving & Disaster Recovery: schema for the Archiver process identity, +-- ARCHIVING node memberships, base-backup policy/history, and PITR. +-- See ~/dev/temp/archiving-disaster-recovery.md for the full design. +-- +-- Milestone 1 (schema + monitor API only): every function here is plain +-- plpgsql/SQL, callable directly with no service_archiver process running +-- -- the pgaftest coverage for this milestone exercises these functions +-- via direct SQL calls against a plain cluster. +-- + +CREATE TYPE pgautofailover.storage_method + AS ENUM ('local', 'rclone'); + +CREATE TYPE pgautofailover.basebackup_source + AS ENUM ('live', 'replay'); + +CREATE TYPE pgautofailover.basebackup_replay_mode + AS ENUM ('volatile', 'persistent'); + +CREATE TYPE pgautofailover.basebackup_cache + AS ENUM ('local', 'none'); + +CREATE TYPE pgautofailover.basebackup_status + AS ENUM ('in_progress', 'complete', 'failed', 'deleted'); + -- 'deleted' is what makes basebackup a full history rather + -- than just a live catalog + +-- shared or per-archiver base-backup production/retention policy +CREATE TABLE pgautofailover.basebackup_policy + ( + basebackuppolicyid bigserial PRIMARY KEY, + policyname text UNIQUE, + + source pgautofailover.basebackup_source + NOT NULL DEFAULT 'replay', + replaymode pgautofailover.basebackup_replay_mode + DEFAULT 'volatile', + cache pgautofailover.basebackup_cache + NOT NULL DEFAULT 'local', + + -- strong, ready-to-use-as-is defaults -- nightly, 3 days retention + frequency interval NOT NULL DEFAULT '24 hours', + maxcount int NOT NULL DEFAULT 3, + maxage interval NOT NULL DEFAULT '3 days', + onpromotion bool NOT NULL DEFAULT true, + + -- backpressure: cap on simultaneous base-backup production jobs, + -- per archiver, per referencing policy + concurrency int NOT NULL DEFAULT 1, + + CHECK (source <> 'replay' OR replaymode IS NOT NULL), + CHECK (concurrency >= 1) + ); + +INSERT INTO pgautofailover.basebackup_policy (policyname) VALUES ('default'); + +-- the physical Archiver entity: one row per archiver host/process +CREATE TABLE pgautofailover.archiver + ( + archiverid bigserial PRIMARY KEY, + archivername text NOT NULL, + hostname text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now(), + + -- NULL by default, same convention as pgautofailover.node.region; + -- unused until the cascading-replication design ships + region text, + + basebackuppolicyid bigint NOT NULL + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + + autoregister bool NOT NULL DEFAULT true, + + -- cap on resident 'warm-standby' archiver_node rows (either cadence) + -- this host is allowed to keep running at once + maxresidentreplay int NOT NULL DEFAULT 1, + + lastreporttime timestamptz, + + UNIQUE (archivername), + CHECK (maxresidentreplay >= 0) + ); + +-- a named, shareable rclone remote configuration -- the literal contents +-- of an rclone config file (real INI format, exactly as rclone itself +-- reads it: https://rclone.org/docs/#config-file). `config` should hold +-- only the non-secret, architectural half of an rclone remote (type, +-- provider, endpoint, region, acl, and a `type = alias` remote baking in +-- the bucket/prefix) -- credentials belong in the archiver process's own +-- environment (RCLONE_CONFIG__), never in this column, which +-- is backed up and readable by anyone with SQL access to the monitor. +CREATE TABLE pgautofailover.rclone_config + ( + rcloneconfigid bigserial PRIMARY KEY, + name text UNIQUE NOT NULL, + config text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now() + ); + +-- 1-N: an archiver's storage targets. Exactly one 'local' row always +-- exists (the mandatory default); adding cloud storage means adding one +-- or more 'rclone' rows, each an independent push target, each +-- referencing a (possibly shared) rclone_config row +CREATE TABLE pgautofailover.archiver_storage + ( + archiverstorageid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + storagemethod pgautofailover.storage_method NOT NULL, + + storagepath text, -- 'local' only: override the default topdir path + rcloneconfigid bigint REFERENCES pgautofailover.rclone_config (rcloneconfigid), + -- 'rclone' only: which named config this target uses + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (storagemethod <> 'local' OR rcloneconfigid IS NULL), + CHECK (storagemethod <> 'rclone' OR rcloneconfigid IS NOT NULL) + ); + +CREATE UNIQUE INDEX archiver_storage_one_local + ON pgautofailover.archiver_storage (archiverid) + WHERE storagemethod = 'local'; + +-- formation-granularity attachment. Only holds explicit rows for the +-- restricted case -- when autoregister is true this table isn't consulted +CREATE TABLE pgautofailover.archiver_formation + ( + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + attachedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (archiverid, formationid) + ); + +-- policy override, resolved formation-default then group-specific; +-- groupid IS NULL means "the formation-wide default for this archiver" +CREATE TABLE pgautofailover.archiver_policy + ( + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + groupid int, + archiverquorum int NOT NULL DEFAULT 1, + basebackuppolicyid bigint + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + replicationquorumeligible bool NOT NULL DEFAULT false + ); + +-- A plain UNIQUE (formationid, groupid) constraint would not actually +-- enforce "at most one formation-wide default row": Postgres treats every +-- NULL groupid as distinct from every other NULL for uniqueness purposes, +-- so two formation-wide rows for the same formation would never conflict. +-- coalesce(groupid, -1) normalizes NULL to a real, comparable value +-- instead -- -1 is safe as a stand-in since groupid is otherwise always +-- >= 0. set_archiver_policy's own ON CONFLICT targets this index. +CREATE UNIQUE INDEX archiver_policy_formation_group_idx + ON pgautofailover.archiver_policy (formationid, coalesce(groupid, -1)); + +-- one row per base backup taken by any archiver -- full history, not just +-- a live catalog: rows are never deleted by retention, only marked +-- status = 'deleted'; get_latest_basebackup filters on status = 'complete' +CREATE TABLE pgautofailover.basebackup + ( + basebackupid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL, + groupid int NOT NULL, + label text NOT NULL, + timeline int NOT NULL, + startlsn pg_lsn NOT NULL, + endlsn pg_lsn, + + period tstzrange NOT NULL DEFAULT tstzrange(now(), NULL), + + -- snapshot of how this specific backup was produced, independent of + -- whatever basebackup_policy says *now* + source pgautofailover.basebackup_source NOT NULL, + replaymode pgautofailover.basebackup_replay_mode, + + sizebytes bigint, + storagelocation text NOT NULL, -- local path, or object-storage URI + status pgautofailover.basebackup_status + NOT NULL DEFAULT 'in_progress', + deletedat timestamptz + ); + +CREATE INDEX basebackup_group_idx + ON pgautofailover.basebackup (formationid, groupid, lower(period) DESC); + +-- remote-side sync/prune tracking, one row per (basebackup, remote +-- storage target) -- a single backup can sync to several remotes +CREATE TABLE pgautofailover.basebackup_storage + ( + basebackupid bigint NOT NULL REFERENCES pgautofailover.basebackup (basebackupid) + ON DELETE CASCADE, + archiverstorageid bigint NOT NULL REFERENCES pgautofailover.archiver_storage (archiverstorageid) + ON DELETE CASCADE, + + syncedat timestamptz, + remotelocation text, + deletedat timestamptz, + + PRIMARY KEY (basebackupid, archiverstorageid) + ); + +-- one row per (archiver, WAL segment) durably captured -- the real +-- backing store wal_archived() queries. +-- +-- PRIMARY KEY is (formationid, groupid, walfilename, archiverid) -- the +-- hot path is wal_archived()'s lookup across every archiver holding %f +-- for this group, so this ordering makes it a direct index range scan. +-- +-- FILLFACTOR 20: traffic is INSERT + DELETE, never UPDATE, but is +-- continuous and high-throughput -- a low fillfactor spreads rows across +-- more pages, reducing buffer-lock contention between concurrently +-- inserting archivers and easing autovacuum on a table that's never +-- write-quiet. +CREATE TABLE pgautofailover.archiver_wal + ( + formationid text NOT NULL, + groupid int NOT NULL, + walfilename text NOT NULL, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + + lsn pg_lsn NOT NULL, + receivedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (formationid, groupid, walfilename, archiverid) + ) WITH (fillfactor = 20); + +CREATE TYPE pgautofailover.archiver_node_kind + AS ENUM ('wal-receiver', 'warm-standby', 'pitr'); + -- 'staging' anticipated for a later, not-yet-designed feature + -- (periodic dev/test environments refreshed from the archiver) + +CREATE TYPE pgautofailover.archiver_node_cadence + AS ENUM ('continuous', 'scheduled'); + -- 'manual' considered (operator-driven "advance only when I say so"), + -- not added yet -- same one-value-enum-addition cost as 'staging' + +CREATE TYPE pgautofailover.pitr_status + AS ENUM ('restoring', 'paused', 'registered', 'discarded'); + +-- every concrete Postgres instance an archiver hosts, derives, or is +-- otherwise associated with, beyond the archiver process itself +CREATE TABLE pgautofailover.archiver_node + ( + archivernodeid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + kind pgautofailover.archiver_node_kind NOT NULL, + + -- placement, uniform across every kind: NULL = colocated (local file + -- reads, zero network); non-NULL = a separate node (remote fetch) + hostname text, + pgdata text NOT NULL, + + -- 'wal-receiver' only: which ARCHIVING row this instance backs. + -- ON DELETE CASCADE: the ARCHIVING node row can be removed through + -- more than one path (this schema's own archiver_remove_formation, + -- or the ordinary pgautofailover.remove_node() every other node type + -- already goes through) -- cascading here means every path safely + -- cleans up this row too, instead of only the one this schema + -- controls directly. + nodeid bigint REFERENCES pgautofailover.node (nodeid) + ON DELETE CASCADE, + + -- 'warm-standby' only: which group's WAL cache this instance replays + formationid text REFERENCES pgautofailover.formation (formationid), + groupid int, + + -- 'warm-standby' only: continuous (chases the primary continuously, + -- eligible for nodecluster read exposure) or scheduled (advances only + -- at basebackup_policy.frequency's cadence, paused via + -- recovery_target_action = pause in between) + cadence pgautofailover.archiver_node_cadence, + + -- 'warm-standby' + cadence = 'continuous' only: opt-in read-only + -- exposure. Enforced by CHECK, not just CLI convention -- a + -- 'scheduled' instance is stale by up to a full frequency between + -- cycles and must never be reachable as an ordinary read-replica + -- connection string without that caveat + nodecluster text, + + -- 'pitr' only: lifecycle (restoring -> paused -> registered/discarded) + pitrstatus pgautofailover.pitr_status, + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (kind <> 'wal-receiver' OR nodeid IS NOT NULL), + CHECK (kind = 'wal-receiver' OR nodeid IS NULL), + CHECK (kind <> 'warm-standby' + OR (formationid IS NOT NULL AND groupid IS NOT NULL AND cadence IS NOT NULL)), + CHECK (kind = 'warm-standby' + OR (formationid IS NULL AND groupid IS NULL AND cadence IS NULL)), + CHECK (nodecluster IS NULL OR (kind = 'warm-standby' AND cadence = 'continuous')), + CHECK (kind = 'pitr' OR pitrstatus IS NULL) + ); + +CREATE TYPE pgautofailover.pitr_operation + AS ENUM ('create', 'status', 'retarget', 'resume', 'promote', + 'register', 'discard'); + +-- every PITR operation, recorded -- not just current status +CREATE TABLE pgautofailover.pitr_history + ( + pitrhistoryid bigserial PRIMARY KEY, + archivernodeid bigint NOT NULL + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + operation pgautofailover.pitr_operation NOT NULL, + occurredat timestamptz NOT NULL DEFAULT now(), + + requestedspec jsonb, -- what was asked for + observedlsn pg_lsn, -- what Postgres actually reported afterward + observedtimestamp timestamptz, + observedpausestate text, -- verbatim: 'not paused'/'pause requested'/'paused' + + note text + ); + +CREATE INDEX pitr_history_node_idx + ON pgautofailover.pitr_history (archivernodeid, occurredat); + +CREATE VIEW pgautofailover.pitr_node_status AS + SELECT n.archivernodeid, n.archiverid, n.hostname, n.pgdata, + n.pitrstatus, h.operation AS lastoperation, + h.observedlsn, h.observedtimestamp, h.observedpausestate, + h.occurredat AS lastupdatedat + FROM pgautofailover.archiver_node n + LEFT JOIN LATERAL ( + SELECT * FROM pgautofailover.pitr_history + WHERE archivernodeid = n.archivernodeid + ORDER BY occurredat DESC LIMIT 1 + ) h ON true + WHERE n.kind = 'pitr'; + +-- opt-in monitor-mediated PITR command queue, for the headless, +-- no-interactive-access deployment shape only (pg_autoctl node run +-- against a node.ini declaring kind = pitr) +CREATE TYPE pgautofailover.pitr_command + AS ENUM ('none', 'retarget', 'pause', 'resume', 'promote', + 'register', 'discard'); + +CREATE TABLE pgautofailover.pitr_pending_command + ( + archivernodeid bigint PRIMARY KEY + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + command pgautofailover.pitr_command NOT NULL DEFAULT 'none', + commandspec jsonb, + queuedat timestamptz NOT NULL DEFAULT now() + ); + + +-- +-- Functions +-- + +CREATE FUNCTION pgautofailover.create_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup_policy + (policyname, source, replaymode, cache, + frequency, maxcount, maxage, onpromotion, concurrency) + SELECT policyname, + coalesce((policyspec->>'source')::pgautofailover.basebackup_source, + 'replay'), + coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, + 'volatile'), + coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, + 'local'), + coalesce((policyspec->>'frequency')::interval, '24 hours'), + coalesce((policyspec->>'maxcount')::int, 3), + coalesce((policyspec->>'maxage')::interval, '3 days'), + coalesce((policyspec->>'onpromotion')::bool, true), + coalesce((policyspec->>'concurrency')::int, 1) + RETURNING basebackuppolicyid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_basebackup_policy(text,jsonb) + is 'create a named, shareable base-backup production/retention policy'; + +grant execute on function + pgautofailover.create_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_policy + SET source = coalesce((policyspec->>'source')::pgautofailover.basebackup_source, source), + replaymode = coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, replaymode), + cache = coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, cache), + frequency = coalesce((policyspec->>'frequency')::interval, frequency), + maxcount = coalesce((policyspec->>'maxcount')::int, maxcount), + maxage = coalesce((policyspec->>'maxage')::interval, maxage), + onpromotion = coalesce((policyspec->>'onpromotion')::bool, onpromotion), + concurrency = coalesce((policyspec->>'concurrency')::int, concurrency) + WHERE basebackup_policy.policyname = set_basebackup_policy.policyname; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup_policy "%" does not exist', policyname; + END IF; +END; +$$; + +comment on function pgautofailover.set_basebackup_policy(text,jsonb) + is 'update an existing named base-backup production/retention policy'; + +grant execute on function + pgautofailover.set_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.basebackup_policy + WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; +$$; + +comment on function pgautofailover.get_basebackup_policy(text) + is 'fetch a named base-backup production/retention policy'; + +grant execute on function pgautofailover.get_basebackup_policy(text) + to autoctl_node; + +-- creates the physical Archiver entity plus its mandatory 'local' +-- archiver_storage row. basebackuppolicyid NULL resolves to 'default'. +-- rcloneconfigname, when given, also attaches an additional 'rclone' row +-- referencing that existing, already-created rclone_config -- the +-- one-command way to "start a new archiver with the same shared rclone +-- setup" another archiver already uses; omit it to start local-only and +-- attach storage later via archiver_add_storage +CREATE FUNCTION pgautofailover.register_archiver + ( + archivername text, hostname text, + storagepath text DEFAULT NULL, + basebackuppolicyid bigint DEFAULT NULL, + autoregister bool DEFAULT true, + maxresidentreplay int DEFAULT 1, + rcloneconfigname text DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_archiverid bigint; + resolved_policyid bigint; +BEGIN + resolved_policyid := coalesce( + basebackuppolicyid, + (SELECT p.basebackuppolicyid + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default')); + + INSERT INTO pgautofailover.archiver + (archivername, hostname, basebackuppolicyid, + autoregister, maxresidentreplay) + VALUES (archivername, hostname, resolved_policyid, + autoregister, maxresidentreplay) + RETURNING archiverid INTO new_archiverid; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, storagepath) + VALUES (new_archiverid, 'local', storagepath); + + IF rcloneconfigname IS NOT NULL THEN + PERFORM pgautofailover.archiver_add_storage(new_archiverid, rcloneconfigname); + END IF; + + RETURN new_archiverid; +END; +$$; + +comment on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) + is 'register a new Archiver process identity, with its mandatory local storage target'; + +grant execute on function + pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) + to autoctl_node; + +-- named, shareable rclone config objects -- see rclone_config above for +-- what belongs in `config` (architecture only, never credentials) +CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) + RETURNS bigint -- rcloneconfigid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.rclone_config (name, config) + VALUES (name, config) + RETURNING rcloneconfigid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_rclone_config(text,text) + is 'register a named, shareable rclone remote configuration'; + +grant execute on function pgautofailover.create_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_rclone_config(name text, config text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.rclone_config AS rc + SET config = set_rclone_config.config + WHERE rc.name = set_rclone_config.name; + + IF NOT FOUND THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', name; + END IF; +END; +$$; + +comment on function pgautofailover.set_rclone_config(text,text) + is 'update the content of an existing named rclone configuration -- every archiver referencing it picks up the change'; + +grant execute on function pgautofailover.set_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_rclone_config(name text) + RETURNS pgautofailover.rclone_config LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.rclone_config AS rc + WHERE rc.name = get_rclone_config.name; +$$; + +comment on function pgautofailover.get_rclone_config(text) + is 'fetch a named rclone configuration''s raw content'; + +grant execute on function pgautofailover.get_rclone_config(text) + to autoctl_node; + +-- attaches an archiver to an existing, already-named rclone_config row +-- (the sharing path -- several archivers' archiver_storage rows can +-- reference the same rcloneconfigid at once, edit the config once via +-- set_rclone_config and every referencing archiver picks it up) +CREATE FUNCTION pgautofailover.archiver_add_storage + (archiverid bigint, rcloneconfigname text) + RETURNS bigint -- archiverstorageid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + resolved_rcloneconfigid bigint; + new_id bigint; +BEGIN + SELECT rc.rcloneconfigid INTO resolved_rcloneconfigid + FROM pgautofailover.rclone_config rc + WHERE rc.name = rcloneconfigname; + + IF resolved_rcloneconfigid IS NULL THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', rcloneconfigname; + END IF; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, rcloneconfigid) + VALUES (archiverid, 'rclone', resolved_rcloneconfigid) + RETURNING archiverstorageid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.archiver_add_storage(bigint,text) + is 'attach an additional rclone storage target to an archiver, referencing an existing named rclone_config'; + +grant execute on function pgautofailover.archiver_add_storage(bigint,text) + to autoctl_node; + +-- detaches only; the referenced rclone_config row is untouched and +-- keeps serving any other archiver still referencing it +CREATE FUNCTION pgautofailover.archiver_remove_storage(archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_storage AS a_s + WHERE a_s.archiverstorageid = archiver_remove_storage.archiverstorageid + AND a_s.storagemethod <> 'local'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_storage % does not exist, or is the mandatory local target', + archiverstorageid; + END IF; +END; +$$; + +comment on function pgautofailover.archiver_remove_storage(bigint) + is 'detach a non-local storage target from an archiver (the local target cannot be removed)'; + +grant execute on function pgautofailover.archiver_remove_storage(bigint) + to autoctl_node; + +-- fans out to one CREATE of a pgautofailover.node row (haspgdata = +-- false) per group currently in formationid +-- Parameters are prefixed in_* here (unlike this file's usual +-- function-qualified-reference convention): ON CONFLICT's own target +-- column list can't be schema/function-qualified at all (that syntax +-- only accepts bare column names or ON CONSTRAINT), so a same-named +-- parameter would still be genuinely ambiguous there even when every +-- other clause in this function could disambiguate it. +CREATE FUNCTION pgautofailover.archiver_add_formation + (in_archiverid bigint, in_formationid text) + RETURNS SETOF bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + grp record; + new_nodeid bigint; +BEGIN + INSERT INTO pgautofailover.archiver_formation (archiverid, formationid) + VALUES (in_archiverid, in_formationid) + ON CONFLICT (archiverid, formationid) DO NOTHING; + + FOR grp IN + SELECT DISTINCT n.groupid + FROM pgautofailover.node n + WHERE n.formationid = in_formationid + LOOP + -- nodeport = 0 is a permanent sentinel, not an M1 stopgap: an + -- ARCHIVING row has no postmaster of its own to be reachable on, + -- so nodehost:nodeport isn't a connectable address here the way + -- it is for every haspgdata row -- see node_nodehost_nodeport_ + -- haspgdata_idx's own comment, which is exactly why that unique + -- index is scoped to haspgdata rows only. reportedstate starts at + -- 'wait_standby', same as any freshly-registered node -- it only + -- reaches 'archiving' once a real keeper's pg_receivewal is + -- actually running (no service_archiver process exists yet at + -- this milestone). + INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata, candidatepriority, + replicationquorum) + VALUES (in_formationid, grp.groupid, + 'archiver-' || in_archiverid || '-' || grp.groupid, + (SELECT a.hostname FROM pgautofailover.archiver a + WHERE a.archiverid = in_archiverid), + 0, + 'wait_standby', 'wait_standby', false, 0, false) + RETURNING nodeid INTO new_nodeid; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, nodeid) + VALUES (in_archiverid, 'wal-receiver', + '', new_nodeid); + + RETURN NEXT new_nodeid; + END LOOP; + + RETURN; +END; +$$; + +comment on function pgautofailover.archiver_add_formation(bigint,text) + is 'attach an archiver to every group of a formation, creating one lightweight ARCHIVING node row per group'; + +grant execute on function pgautofailover.archiver_add_formation(bigint,text) + to autoctl_node; + +-- Deleting the node row is enough: archiver_node.nodeid's own +-- ON DELETE CASCADE removes the matching wal-receiver archiver_node row +-- automatically (see that column's own comment). +CREATE FUNCTION pgautofailover.archiver_remove_formation + (archiverid bigint, formationid text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.node n + WHERE n.formationid = archiver_remove_formation.formationid + AND n.nodeid IN (SELECT an.nodeid + FROM pgautofailover.archiver_node an + WHERE an.archiverid = archiver_remove_formation.archiverid + AND an.kind = 'wal-receiver'); + + DELETE FROM pgautofailover.archiver_formation af + WHERE af.archiverid = archiver_remove_formation.archiverid + AND af.formationid = archiver_remove_formation.formationid; +END; +$$; + +comment on function pgautofailover.archiver_remove_formation(bigint,text) + is 'detach an archiver from a formation, removing its ARCHIVING node row in every group'; + +grant execute on function pgautofailover.archiver_remove_formation(bigint,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified, even inside an +-- expression like coalesce(groupid, -1)) forces this naming here. +CREATE FUNCTION pgautofailover.set_archiver_policy + ( + in_formationid text, in_groupid int DEFAULT NULL, + in_archiverquorum int DEFAULT NULL, + in_basebackuppolicyid bigint DEFAULT NULL, + in_replicationquorumeligible bool DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.archiver_policy + (formationid, groupid, archiverquorum, + basebackuppolicyid, replicationquorumeligible) + VALUES (in_formationid, in_groupid, + coalesce(in_archiverquorum, 1), + in_basebackuppolicyid, + coalesce(in_replicationquorumeligible, false)) + ON CONFLICT (formationid, (coalesce(groupid, -1))) DO UPDATE + SET archiverquorum = coalesce(EXCLUDED.archiverquorum, + pgautofailover.archiver_policy.archiverquorum), + basebackuppolicyid = coalesce(EXCLUDED.basebackuppolicyid, + pgautofailover.archiver_policy.basebackuppolicyid), + replicationquorumeligible = coalesce(EXCLUDED.replicationquorumeligible, + pgautofailover.archiver_policy.replicationquorumeligible); +END; +$$; + +comment on function pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + is 'set (or override) archiver_quorum/basebackup policy/replication-quorum eligibility for a formation, or one of its groups'; + +grant execute on function + pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + to autoctl_node; + +-- resolves group-specific override first, then the formation-wide +-- (groupid IS NULL) default, then this schema's own hardcoded defaults. +-- Deliberately plpgsql, not a single SQL query: an earlier draft tried to +-- express the three-way fallback as one UNION ALL ... LIMIT 1 query, but +-- UNION ALL has no ordering guarantee across its branches, so LIMIT 1 +-- could just as easily return the formation-wide or hardcoded default +-- even when a group-specific override exists. Sequential SELECT INTO ... +-- IF FOUND is unambiguous. +CREATE FUNCTION pgautofailover.get_archiver_policy(formationid text, groupid int) + RETURNS TABLE (archiverquorum int, basebackuppolicyid bigint, + replicationquorumeligible bool) + LANGUAGE plpgsql STABLE +AS $$ +BEGIN + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid = get_archiver_policy.groupid; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid IS NULL; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT 1, p.basebackuppolicyid, false + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default'; +END; +$$; + +comment on function pgautofailover.get_archiver_policy(text,int) + is 'resolve archiver policy for (formation, group): group override, else formation default, else this schema''s own defaults'; + +grant execute on function pgautofailover.get_archiver_policy(text,int) + to autoctl_node; + +-- the archive_command confirmation check: true iff at least +-- archiver_quorum distinct archivers have durably reported %f +CREATE FUNCTION pgautofailover.wal_archived + (formationid text, groupid int, walfilename text) + RETURNS bool + LANGUAGE sql STABLE +AS $$ + SELECT count(DISTINCT aw.archiverid) >= + (SELECT archiverquorum + FROM pgautofailover.get_archiver_policy(wal_archived.formationid, + wal_archived.groupid)) + FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = wal_archived.formationid + AND aw.groupid = wal_archived.groupid + AND aw.walfilename = wal_archived.walfilename; +$$; + +comment on function pgautofailover.wal_archived(text,int,text) + is 'archive_command confirmation check: has segment %f already landed durably on archiver_quorum archiver(s)?'; + +grant execute on function pgautofailover.wal_archived(text,int,text) + to autoctl_node; + +-- inserts into archiver_wal (idempotent on conflict) +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_wal_received + (in_nodeid bigint, in_walfilename text, in_lsn pg_lsn) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + target record; +BEGIN + SELECT n.formationid, n.groupid, an.archiverid + INTO target + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.nodeid = in_nodeid + AND an.kind = 'wal-receiver'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'node % is not an ARCHIVING wal-receiver node', in_nodeid; + END IF; + + INSERT INTO pgautofailover.archiver_wal + (formationid, groupid, walfilename, archiverid, lsn) + VALUES (target.formationid, target.groupid, in_walfilename, target.archiverid, in_lsn) + ON CONFLICT (formationid, groupid, walfilename, archiverid) DO NOTHING; +END; +$$; + +comment on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + is 'reports a WAL segment durably captured by an ARCHIVING node'; + +grant execute on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_started + ( + archiverid bigint, formationid text, groupid int, + label text, timeline int, startlsn pg_lsn, + source pgautofailover.basebackup_source, + replaymode pgautofailover.basebackup_replay_mode DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup + (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, storagelocation, status) + VALUES (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, '', 'in_progress') + RETURNING basebackupid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + is 'records the start of a new base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_completed + (basebackupid bigint, endlsn pg_lsn, sizebytes bigint, storagelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup AS bb + SET endlsn = report_basebackup_completed.endlsn, + sizebytes = report_basebackup_completed.sizebytes, + storagelocation = report_basebackup_completed.storagelocation, + status = 'complete', + period = tstzrange(lower(bb.period), now()) + WHERE bb.basebackupid = report_basebackup_completed.basebackupid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; +END; +$$; + +comment on function pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + is 'records the successful completion of a base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + to autoctl_node; + +-- marks the basebackup row deleted (never a real DELETE), then prunes +-- any archiver_wal rows this group no longer needs to retain +CREATE FUNCTION pgautofailover.report_basebackup_deleted(basebackupid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + bb record; +BEGIN + UPDATE pgautofailover.basebackup AS b + SET status = 'deleted', deletedat = now() + WHERE b.basebackupid = report_basebackup_deleted.basebackupid + RETURNING b.formationid, b.groupid INTO bb; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; + + PERFORM pgautofailover.prune_archiver_wal(bb.formationid, bb.groupid); +END; +$$; + +comment on function pgautofailover.report_basebackup_deleted(bigint) + is 'marks a base backup deleted (retains history) and prunes any archiver_wal rows no group backup needs anymore'; + +grant execute on function pgautofailover.report_basebackup_deleted(bigint) + to autoctl_node; + +-- deletes every archiver_wal row for (formationid, groupid) older than +-- the earliest still-'complete' basebackup's startlsn, across every +-- archiver holding a copy. When no 'complete' backup remains for this +-- group, nothing is pruned -- there is no anchor point to replay forward +-- from, so every captured segment is still needed. +CREATE FUNCTION pgautofailover.prune_archiver_wal(formationid text, groupid int) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + oldest_startlsn pg_lsn; + deleted_count bigint; +BEGIN + SELECT min(b.startlsn) INTO oldest_startlsn + FROM pgautofailover.basebackup b + WHERE b.formationid = prune_archiver_wal.formationid + AND b.groupid = prune_archiver_wal.groupid + AND b.status = 'complete'; + + IF oldest_startlsn IS NULL THEN + RETURN 0; + END IF; + + WITH deleted AS ( + DELETE FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = prune_archiver_wal.formationid + AND aw.groupid = prune_archiver_wal.groupid + AND aw.lsn < oldest_startlsn + RETURNING 1 + ) + SELECT count(*) INTO deleted_count FROM deleted; + + RETURN deleted_count; +END; +$$; + +comment on function pgautofailover.prune_archiver_wal(text,int) + is 'deletes archiver_wal rows for (formation, group) older than the oldest still-complete base backup''s startlsn'; + +grant execute on function pgautofailover.prune_archiver_wal(text,int) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_basebackup_synced + (in_basebackupid bigint, in_archiverstorageid bigint, in_remotelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.basebackup_storage + (basebackupid, archiverstorageid, syncedat, remotelocation) + VALUES (in_basebackupid, in_archiverstorageid, now(), in_remotelocation) + ON CONFLICT (basebackupid, archiverstorageid) DO UPDATE + SET syncedat = now(), + remotelocation = EXCLUDED.remotelocation; +END; +$$; + +comment on function pgautofailover.report_basebackup_synced(bigint,bigint,text) + is 'records a successful cold-storage sync of a base backup to one storage target'; + +grant execute on function + pgautofailover.report_basebackup_synced(bigint,bigint,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_remote_deleted + (basebackupid bigint, archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_storage AS bs + SET deletedat = now() + WHERE bs.basebackupid = report_basebackup_remote_deleted.basebackupid + AND bs.archiverstorageid = report_basebackup_remote_deleted.archiverstorageid; +END; +$$; + +comment on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + is 'records that a base backup''s remote copy on one storage target has been pruned'; + +grant execute on function + pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + to autoctl_node; + +-- filters status = 'complete' only +CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE +AS $$ + SELECT * FROM pgautofailover.basebackup b + WHERE b.formationid = get_latest_basebackup.formationid + AND b.groupid = get_latest_basebackup.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC + LIMIT 1; +$$; + +comment on function pgautofailover.get_latest_basebackup(text,int) + is 'fetch the most recent complete base backup for (formation, group)'; + +grant execute on function pgautofailover.get_latest_basebackup(text,int) + to autoctl_node; + +-- for kind = 'warm-standby': raises if the owning archiver is already at +-- its maxresidentreplay cap +CREATE FUNCTION pgautofailover.create_archiver_node + ( + archiverid bigint, + kind pgautofailover.archiver_node_kind, + pgdata text, + hostname text DEFAULT NULL, + nodeid bigint DEFAULT NULL, -- required iff kind = 'wal-receiver' + formationid text DEFAULT NULL, -- required iff kind = 'warm-standby' + groupid int DEFAULT NULL, -- required iff kind = 'warm-standby' + cadence pgautofailover.archiver_node_cadence DEFAULT NULL, + nodecluster text DEFAULT NULL, -- only for 'warm-standby' + cadence = 'continuous' + pitrstatus pgautofailover.pitr_status DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + residentcount int; + maxresident int; + new_id bigint; +BEGIN + IF kind = 'warm-standby' THEN + SELECT a.maxresidentreplay INTO maxresident + FROM pgautofailover.archiver a + WHERE a.archiverid = create_archiver_node.archiverid; + + SELECT count(*) INTO residentcount + FROM pgautofailover.archiver_node an + WHERE an.archiverid = create_archiver_node.archiverid + AND an.kind = 'warm-standby'; + + IF residentcount >= maxresident THEN + RAISE EXCEPTION + 'archiver % is already at its maxresidentreplay cap (%)', + archiverid, maxresident; + END IF; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + VALUES (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + RETURNING archivernodeid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + is 'registers a concrete Postgres instance an archiver hosts, derives, or is otherwise associated with'; + +grant execute on function + pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + to autoctl_node; + +CREATE FUNCTION pgautofailover.remove_archiver_node(archivernodeid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_node an + WHERE an.archivernodeid = remove_archiver_node.archivernodeid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist', archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.remove_archiver_node(bigint) + is 'removes an archiver_node row'; + +grant execute on function pgautofailover.remove_archiver_node(bigint) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_archiver_node_pitr_status + (archivernodeid bigint, pitrstatus pgautofailover.pitr_status) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.archiver_node AS an + SET pitrstatus = set_archiver_node_pitr_status.pitrstatus + WHERE an.archivernodeid = set_archiver_node_pitr_status.archivernodeid + AND an.kind = 'pitr'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist, or is not kind = pitr', + archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + is 'updates a PITR archiver_node''s lifecycle status'; + +grant execute on function + pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + to autoctl_node; + +-- pushed by the local pg_autoctl pitr CLI immediately after acting +-- locally -- never blocks or gates the local action on this succeeding +CREATE FUNCTION pgautofailover.report_pitr_status + ( + archivernodeid bigint, operation pgautofailover.pitr_operation, + requestedspec jsonb, + observedlsn pg_lsn, observedtimestamp timestamptz, + observedpausestate text, note text DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_history + (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note) + VALUES (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note); +END; +$$; + +comment on function pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + is 'records one PITR operation''s outcome -- a best-effort report, never gating the local action it follows'; + +grant execute on function + pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.pitr_queue_command + (in_archivernodeid bigint, in_command pgautofailover.pitr_command, + in_commandspec jsonb DEFAULT NULL) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_pending_command + (archivernodeid, command, commandspec) + VALUES (in_archivernodeid, in_command, in_commandspec) + ON CONFLICT (archivernodeid) DO UPDATE + SET command = EXCLUDED.command, + commandspec = EXCLUDED.commandspec, + queuedat = now(); +END; +$$; + +comment on function pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + is 'queues a PITR command for a monitor-mediated (kind = pitr, pg_autoctl node run) agent to pick up'; + +grant execute on function + pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + to autoctl_node; + +-- returns the pending command and resets the queue slot to 'none' in the +-- same call -- an agent polling this never processes the same command twice +-- Reads the pending command, then clears it, as two separate statements: +-- UPDATE ... RETURNING always reflects the row *after* the update is +-- applied, so folding the reset into the same RETURNING clause that reads +-- the command would always report back the very 'none' this function just +-- set, never the command that was actually queued. FOR UPDATE locks the +-- row across both statements, so a concurrent caller for the same +-- archivernodeid still can't observe or consume the same command twice. +CREATE FUNCTION pgautofailover.pitr_next_command(in_archivernodeid bigint) + RETURNS pgautofailover.pitr_command LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + next_command pgautofailover.pitr_command; +BEGIN + SELECT pc.command INTO next_command + FROM pgautofailover.pitr_pending_command pc + WHERE pc.archivernodeid = in_archivernodeid + FOR UPDATE; + + IF next_command IS NULL OR next_command = 'none' THEN + RETURN 'none'; + END IF; + + UPDATE pgautofailover.pitr_pending_command AS pc + SET command = 'none', commandspec = NULL + WHERE pc.archivernodeid = in_archivernodeid; + + RETURN next_command; +END; +$$; + +comment on function pgautofailover.pitr_next_command(bigint) + is 'pops and clears the next queued PITR command for an agent to act on'; + +grant execute on function pgautofailover.pitr_next_command(bigint) + to autoctl_node; diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 8545bf38c..54da1fdfd 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -41,7 +41,8 @@ CREATE TYPE pgautofailover.replication_state 'report_lsn', 'fast_forward', 'join_secondary', - 'dropped' + 'dropped', + 'archiving' ); CREATE TABLE pgautofailover.formation @@ -135,10 +136,18 @@ CREATE TABLE pgautofailover.node pg_versionstring text, citus_version text, + -- true for every ordinary Postgres node (its own PGDATA, promotable); + -- false only for an ARCHIVING membership row (a pg_receivewal client, + -- no PGDATA, no postmaster to manage). See archiving-disaster-recovery + -- design: this single boolean is what candidate_priority enforcement, + -- keeper_ensure_current_state's liveness check, and the FAST_FORWARD + -- source-selection branch all key off, instead of a third node-kind + -- value -- a cascading follower is still haspgdata = true, and a + -- future proxy never becomes a pgautofailover.node row at all. + haspgdata bool NOT NULL DEFAULT true, + -- node names must be unique in a given formation UNIQUE (formationid, nodename), - -- any nodehost:port can only be a unique node in the system - UNIQUE (nodehost, nodeport), -- -- The EXCLUDE constraint only allows the same sysidentifier for all the -- nodes in the same group. The system_identifier is a property that is @@ -176,6 +185,17 @@ CREATE TABLE pgautofailover.node -- we expect few rows and lots of UPDATE, let's benefit from HOT WITH (fillfactor = 25); +-- any nodehost:port can only be a unique real Postgres node in the +-- system -- scoped to haspgdata rows only: an ARCHIVING membership row +-- has no listening postmaster of its own to be unique about (nodehost is +-- its owning archiver's hostname, nodeport is the 0 sentinel -- see +-- haspgdata's own comment above), and the same archiver legitimately +-- backs one row per group it serves, all sharing that same (nodehost, 0) +-- pair. +CREATE UNIQUE INDEX node_nodehost_nodeport_haspgdata_idx + ON pgautofailover.node (nodehost, nodeport) + WHERE haspgdata; + -- Mirrors group_state_machine.h's MonitorFSMSection: which of the three -- real control-flow regions of the monitor's declarative dispatch table -- (MonitorFSM[] in group_state_machine.c) a rule belongs to. See dump_fsm() @@ -1292,6 +1312,1237 @@ comment on function pgautofailover.formation_settings(text) is 'get the current replication settings a formation'; -- +-- +-- Archiving & Disaster Recovery: schema for the Archiver process identity, +-- ARCHIVING node memberships, base-backup policy/history, and PITR. +-- See ~/dev/temp/archiving-disaster-recovery.md for the full design. +-- +-- Milestone 1 (schema + monitor API only): every function here is plain +-- plpgsql/SQL, callable directly with no service_archiver process running +-- -- the pgaftest coverage for this milestone exercises these functions +-- via direct SQL calls against a plain cluster. +-- + +CREATE TYPE pgautofailover.storage_method + AS ENUM ('local', 'rclone'); + +CREATE TYPE pgautofailover.basebackup_source + AS ENUM ('live', 'replay'); + +CREATE TYPE pgautofailover.basebackup_replay_mode + AS ENUM ('volatile', 'persistent'); + +CREATE TYPE pgautofailover.basebackup_cache + AS ENUM ('local', 'none'); + +CREATE TYPE pgautofailover.basebackup_status + AS ENUM ('in_progress', 'complete', 'failed', 'deleted'); + -- 'deleted' is what makes basebackup a full history rather + -- than just a live catalog + +-- shared or per-archiver base-backup production/retention policy +CREATE TABLE pgautofailover.basebackup_policy + ( + basebackuppolicyid bigserial PRIMARY KEY, + policyname text UNIQUE, + + source pgautofailover.basebackup_source + NOT NULL DEFAULT 'replay', + replaymode pgautofailover.basebackup_replay_mode + DEFAULT 'volatile', + cache pgautofailover.basebackup_cache + NOT NULL DEFAULT 'local', + + -- strong, ready-to-use-as-is defaults -- nightly, 3 days retention + frequency interval NOT NULL DEFAULT '24 hours', + maxcount int NOT NULL DEFAULT 3, + maxage interval NOT NULL DEFAULT '3 days', + onpromotion bool NOT NULL DEFAULT true, + + -- backpressure: cap on simultaneous base-backup production jobs, + -- per archiver, per referencing policy + concurrency int NOT NULL DEFAULT 1, + + CHECK (source <> 'replay' OR replaymode IS NOT NULL), + CHECK (concurrency >= 1) + ); + +INSERT INTO pgautofailover.basebackup_policy (policyname) VALUES ('default'); + +-- the physical Archiver entity: one row per archiver host/process +CREATE TABLE pgautofailover.archiver + ( + archiverid bigserial PRIMARY KEY, + archivername text NOT NULL, + hostname text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now(), + + -- NULL by default, same convention as pgautofailover.node.region; + -- unused until the cascading-replication design ships + region text, + + basebackuppolicyid bigint NOT NULL + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + + autoregister bool NOT NULL DEFAULT true, + + -- cap on resident 'warm-standby' archiver_node rows (either cadence) + -- this host is allowed to keep running at once + maxresidentreplay int NOT NULL DEFAULT 1, + + lastreporttime timestamptz, + + UNIQUE (archivername), + CHECK (maxresidentreplay >= 0) + ); + +-- a named, shareable rclone remote configuration -- the literal contents +-- of an rclone config file (real INI format, exactly as rclone itself +-- reads it: https://rclone.org/docs/#config-file). `config` should hold +-- only the non-secret, architectural half of an rclone remote (type, +-- provider, endpoint, region, acl, and a `type = alias` remote baking in +-- the bucket/prefix) -- credentials belong in the archiver process's own +-- environment (RCLONE_CONFIG__), never in this column, which +-- is backed up and readable by anyone with SQL access to the monitor. +CREATE TABLE pgautofailover.rclone_config + ( + rcloneconfigid bigserial PRIMARY KEY, + name text UNIQUE NOT NULL, + config text NOT NULL, + createdat timestamptz NOT NULL DEFAULT now() + ); + +-- 1-N: an archiver's storage targets. Exactly one 'local' row always +-- exists (the mandatory default); adding cloud storage means adding one +-- or more 'rclone' rows, each an independent push target, each +-- referencing a (possibly shared) rclone_config row +CREATE TABLE pgautofailover.archiver_storage + ( + archiverstorageid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + storagemethod pgautofailover.storage_method NOT NULL, + + storagepath text, -- 'local' only: override the default topdir path + rcloneconfigid bigint REFERENCES pgautofailover.rclone_config (rcloneconfigid), + -- 'rclone' only: which named config this target uses + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (storagemethod <> 'local' OR rcloneconfigid IS NULL), + CHECK (storagemethod <> 'rclone' OR rcloneconfigid IS NOT NULL) + ); + +CREATE UNIQUE INDEX archiver_storage_one_local + ON pgautofailover.archiver_storage (archiverid) + WHERE storagemethod = 'local'; + +-- formation-granularity attachment. Only holds explicit rows for the +-- restricted case -- when autoregister is true this table isn't consulted +CREATE TABLE pgautofailover.archiver_formation + ( + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + attachedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (archiverid, formationid) + ); + +-- policy override, resolved formation-default then group-specific; +-- groupid IS NULL means "the formation-wide default for this archiver" +CREATE TABLE pgautofailover.archiver_policy + ( + formationid text NOT NULL REFERENCES pgautofailover.formation (formationid) + ON DELETE CASCADE, + groupid int, + archiverquorum int NOT NULL DEFAULT 1, + basebackuppolicyid bigint + REFERENCES pgautofailover.basebackup_policy (basebackuppolicyid), + replicationquorumeligible bool NOT NULL DEFAULT false + ); + +-- A plain UNIQUE (formationid, groupid) constraint would not actually +-- enforce "at most one formation-wide default row": Postgres treats every +-- NULL groupid as distinct from every other NULL for uniqueness purposes, +-- so two formation-wide rows for the same formation would never conflict. +-- coalesce(groupid, -1) normalizes NULL to a real, comparable value +-- instead -- -1 is safe as a stand-in since groupid is otherwise always +-- >= 0. set_archiver_policy's own ON CONFLICT targets this index. +CREATE UNIQUE INDEX archiver_policy_formation_group_idx + ON pgautofailover.archiver_policy (formationid, coalesce(groupid, -1)); + +-- one row per base backup taken by any archiver -- full history, not just +-- a live catalog: rows are never deleted by retention, only marked +-- status = 'deleted'; get_latest_basebackup filters on status = 'complete' +CREATE TABLE pgautofailover.basebackup + ( + basebackupid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + formationid text NOT NULL, + groupid int NOT NULL, + label text NOT NULL, + timeline int NOT NULL, + startlsn pg_lsn NOT NULL, + endlsn pg_lsn, + + period tstzrange NOT NULL DEFAULT tstzrange(now(), NULL), + + -- snapshot of how this specific backup was produced, independent of + -- whatever basebackup_policy says *now* + source pgautofailover.basebackup_source NOT NULL, + replaymode pgautofailover.basebackup_replay_mode, + + sizebytes bigint, + storagelocation text NOT NULL, -- local path, or object-storage URI + status pgautofailover.basebackup_status + NOT NULL DEFAULT 'in_progress', + deletedat timestamptz + ); + +CREATE INDEX basebackup_group_idx + ON pgautofailover.basebackup (formationid, groupid, lower(period) DESC); + +-- remote-side sync/prune tracking, one row per (basebackup, remote +-- storage target) -- a single backup can sync to several remotes +CREATE TABLE pgautofailover.basebackup_storage + ( + basebackupid bigint NOT NULL REFERENCES pgautofailover.basebackup (basebackupid) + ON DELETE CASCADE, + archiverstorageid bigint NOT NULL REFERENCES pgautofailover.archiver_storage (archiverstorageid) + ON DELETE CASCADE, + + syncedat timestamptz, + remotelocation text, + deletedat timestamptz, + + PRIMARY KEY (basebackupid, archiverstorageid) + ); + +-- one row per (archiver, WAL segment) durably captured -- the real +-- backing store wal_archived() queries. +-- +-- PRIMARY KEY is (formationid, groupid, walfilename, archiverid) -- the +-- hot path is wal_archived()'s lookup across every archiver holding %f +-- for this group, so this ordering makes it a direct index range scan. +-- +-- FILLFACTOR 20: traffic is INSERT + DELETE, never UPDATE, but is +-- continuous and high-throughput -- a low fillfactor spreads rows across +-- more pages, reducing buffer-lock contention between concurrently +-- inserting archivers and easing autovacuum on a table that's never +-- write-quiet. +CREATE TABLE pgautofailover.archiver_wal + ( + formationid text NOT NULL, + groupid int NOT NULL, + walfilename text NOT NULL, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + + lsn pg_lsn NOT NULL, + receivedat timestamptz NOT NULL DEFAULT now(), + + PRIMARY KEY (formationid, groupid, walfilename, archiverid) + ) WITH (fillfactor = 20); + +CREATE TYPE pgautofailover.archiver_node_kind + AS ENUM ('wal-receiver', 'warm-standby', 'pitr'); + -- 'staging' anticipated for a later, not-yet-designed feature + -- (periodic dev/test environments refreshed from the archiver) + +CREATE TYPE pgautofailover.archiver_node_cadence + AS ENUM ('continuous', 'scheduled'); + -- 'manual' considered (operator-driven "advance only when I say so"), + -- not added yet -- same one-value-enum-addition cost as 'staging' + +CREATE TYPE pgautofailover.pitr_status + AS ENUM ('restoring', 'paused', 'registered', 'discarded'); + +-- every concrete Postgres instance an archiver hosts, derives, or is +-- otherwise associated with, beyond the archiver process itself +CREATE TABLE pgautofailover.archiver_node + ( + archivernodeid bigserial PRIMARY KEY, + archiverid bigint NOT NULL REFERENCES pgautofailover.archiver (archiverid) + ON DELETE CASCADE, + kind pgautofailover.archiver_node_kind NOT NULL, + + -- placement, uniform across every kind: NULL = colocated (local file + -- reads, zero network); non-NULL = a separate node (remote fetch) + hostname text, + pgdata text NOT NULL, + + -- 'wal-receiver' only: which ARCHIVING row this instance backs. + -- ON DELETE CASCADE: the ARCHIVING node row can be removed through + -- more than one path (this schema's own archiver_remove_formation, + -- or the ordinary pgautofailover.remove_node() every other node type + -- already goes through) -- cascading here means every path safely + -- cleans up this row too, instead of only the one this schema + -- controls directly. + nodeid bigint REFERENCES pgautofailover.node (nodeid) + ON DELETE CASCADE, + + -- 'warm-standby' only: which group's WAL cache this instance replays + formationid text REFERENCES pgautofailover.formation (formationid), + groupid int, + + -- 'warm-standby' only: continuous (chases the primary continuously, + -- eligible for nodecluster read exposure) or scheduled (advances only + -- at basebackup_policy.frequency's cadence, paused via + -- recovery_target_action = pause in between) + cadence pgautofailover.archiver_node_cadence, + + -- 'warm-standby' + cadence = 'continuous' only: opt-in read-only + -- exposure. Enforced by CHECK, not just CLI convention -- a + -- 'scheduled' instance is stale by up to a full frequency between + -- cycles and must never be reachable as an ordinary read-replica + -- connection string without that caveat + nodecluster text, + + -- 'pitr' only: lifecycle (restoring -> paused -> registered/discarded) + pitrstatus pgautofailover.pitr_status, + + createdat timestamptz NOT NULL DEFAULT now(), + + CHECK (kind <> 'wal-receiver' OR nodeid IS NOT NULL), + CHECK (kind = 'wal-receiver' OR nodeid IS NULL), + CHECK (kind <> 'warm-standby' + OR (formationid IS NOT NULL AND groupid IS NOT NULL AND cadence IS NOT NULL)), + CHECK (kind = 'warm-standby' + OR (formationid IS NULL AND groupid IS NULL AND cadence IS NULL)), + CHECK (nodecluster IS NULL OR (kind = 'warm-standby' AND cadence = 'continuous')), + CHECK (kind = 'pitr' OR pitrstatus IS NULL) + ); + +CREATE TYPE pgautofailover.pitr_operation + AS ENUM ('create', 'status', 'retarget', 'resume', 'promote', + 'register', 'discard'); + +-- every PITR operation, recorded -- not just current status +CREATE TABLE pgautofailover.pitr_history + ( + pitrhistoryid bigserial PRIMARY KEY, + archivernodeid bigint NOT NULL + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + operation pgautofailover.pitr_operation NOT NULL, + occurredat timestamptz NOT NULL DEFAULT now(), + + requestedspec jsonb, -- what was asked for + observedlsn pg_lsn, -- what Postgres actually reported afterward + observedtimestamp timestamptz, + observedpausestate text, -- verbatim: 'not paused'/'pause requested'/'paused' + + note text + ); + +CREATE INDEX pitr_history_node_idx + ON pgautofailover.pitr_history (archivernodeid, occurredat); + +CREATE VIEW pgautofailover.pitr_node_status AS + SELECT n.archivernodeid, n.archiverid, n.hostname, n.pgdata, + n.pitrstatus, h.operation AS lastoperation, + h.observedlsn, h.observedtimestamp, h.observedpausestate, + h.occurredat AS lastupdatedat + FROM pgautofailover.archiver_node n + LEFT JOIN LATERAL ( + SELECT * FROM pgautofailover.pitr_history + WHERE archivernodeid = n.archivernodeid + ORDER BY occurredat DESC LIMIT 1 + ) h ON true + WHERE n.kind = 'pitr'; + +-- opt-in monitor-mediated PITR command queue, for the headless, +-- no-interactive-access deployment shape only (pg_autoctl node run +-- against a node.ini declaring kind = pitr) +CREATE TYPE pgautofailover.pitr_command + AS ENUM ('none', 'retarget', 'pause', 'resume', 'promote', + 'register', 'discard'); + +CREATE TABLE pgautofailover.pitr_pending_command + ( + archivernodeid bigint PRIMARY KEY + REFERENCES pgautofailover.archiver_node (archivernodeid) + ON DELETE CASCADE, + command pgautofailover.pitr_command NOT NULL DEFAULT 'none', + commandspec jsonb, + queuedat timestamptz NOT NULL DEFAULT now() + ); + + +-- +-- Functions +-- + +CREATE FUNCTION pgautofailover.create_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup_policy + (policyname, source, replaymode, cache, + frequency, maxcount, maxage, onpromotion, concurrency) + SELECT policyname, + coalesce((policyspec->>'source')::pgautofailover.basebackup_source, + 'replay'), + coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, + 'volatile'), + coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, + 'local'), + coalesce((policyspec->>'frequency')::interval, '24 hours'), + coalesce((policyspec->>'maxcount')::int, 3), + coalesce((policyspec->>'maxage')::interval, '3 days'), + coalesce((policyspec->>'onpromotion')::bool, true), + coalesce((policyspec->>'concurrency')::int, 1) + RETURNING basebackuppolicyid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_basebackup_policy(text,jsonb) + is 'create a named, shareable base-backup production/retention policy'; + +grant execute on function + pgautofailover.create_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_basebackup_policy + ( + IN policyname text, + IN policyspec jsonb + ) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_policy + SET source = coalesce((policyspec->>'source')::pgautofailover.basebackup_source, source), + replaymode = coalesce((policyspec->>'replaymode')::pgautofailover.basebackup_replay_mode, replaymode), + cache = coalesce((policyspec->>'cache')::pgautofailover.basebackup_cache, cache), + frequency = coalesce((policyspec->>'frequency')::interval, frequency), + maxcount = coalesce((policyspec->>'maxcount')::int, maxcount), + maxage = coalesce((policyspec->>'maxage')::interval, maxage), + onpromotion = coalesce((policyspec->>'onpromotion')::bool, onpromotion), + concurrency = coalesce((policyspec->>'concurrency')::int, concurrency) + WHERE basebackup_policy.policyname = set_basebackup_policy.policyname; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup_policy "%" does not exist', policyname; + END IF; +END; +$$; + +comment on function pgautofailover.set_basebackup_policy(text,jsonb) + is 'update an existing named base-backup production/retention policy'; + +grant execute on function + pgautofailover.set_basebackup_policy(text,jsonb) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.basebackup_policy + WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; +$$; + +comment on function pgautofailover.get_basebackup_policy(text) + is 'fetch a named base-backup production/retention policy'; + +grant execute on function pgautofailover.get_basebackup_policy(text) + to autoctl_node; + +-- creates the physical Archiver entity plus its mandatory 'local' +-- archiver_storage row. basebackuppolicyid NULL resolves to 'default'. +-- rcloneconfigname, when given, also attaches an additional 'rclone' row +-- referencing that existing, already-created rclone_config -- the +-- one-command way to "start a new archiver with the same shared rclone +-- setup" another archiver already uses; omit it to start local-only and +-- attach storage later via archiver_add_storage +CREATE FUNCTION pgautofailover.register_archiver + ( + archivername text, hostname text, + storagepath text DEFAULT NULL, + basebackuppolicyid bigint DEFAULT NULL, + autoregister bool DEFAULT true, + maxresidentreplay int DEFAULT 1, + rcloneconfigname text DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_archiverid bigint; + resolved_policyid bigint; +BEGIN + resolved_policyid := coalesce( + basebackuppolicyid, + (SELECT p.basebackuppolicyid + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default')); + + INSERT INTO pgautofailover.archiver + (archivername, hostname, basebackuppolicyid, + autoregister, maxresidentreplay) + VALUES (archivername, hostname, resolved_policyid, + autoregister, maxresidentreplay) + RETURNING archiverid INTO new_archiverid; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, storagepath) + VALUES (new_archiverid, 'local', storagepath); + + IF rcloneconfigname IS NOT NULL THEN + PERFORM pgautofailover.archiver_add_storage(new_archiverid, rcloneconfigname); + END IF; + + RETURN new_archiverid; +END; +$$; + +comment on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) + is 'register a new Archiver process identity, with its mandatory local storage target'; + +grant execute on function + pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) + to autoctl_node; + +-- named, shareable rclone config objects -- see rclone_config above for +-- what belongs in `config` (architecture only, never credentials) +CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) + RETURNS bigint -- rcloneconfigid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.rclone_config (name, config) + VALUES (name, config) + RETURNING rcloneconfigid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_rclone_config(text,text) + is 'register a named, shareable rclone remote configuration'; + +grant execute on function pgautofailover.create_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_rclone_config(name text, config text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.rclone_config AS rc + SET config = set_rclone_config.config + WHERE rc.name = set_rclone_config.name; + + IF NOT FOUND THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', name; + END IF; +END; +$$; + +comment on function pgautofailover.set_rclone_config(text,text) + is 'update the content of an existing named rclone configuration -- every archiver referencing it picks up the change'; + +grant execute on function pgautofailover.set_rclone_config(text,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.get_rclone_config(name text) + RETURNS pgautofailover.rclone_config LANGUAGE sql STRICT +AS $$ + SELECT * FROM pgautofailover.rclone_config AS rc + WHERE rc.name = get_rclone_config.name; +$$; + +comment on function pgautofailover.get_rclone_config(text) + is 'fetch a named rclone configuration''s raw content'; + +grant execute on function pgautofailover.get_rclone_config(text) + to autoctl_node; + +-- attaches an archiver to an existing, already-named rclone_config row +-- (the sharing path -- several archivers' archiver_storage rows can +-- reference the same rcloneconfigid at once, edit the config once via +-- set_rclone_config and every referencing archiver picks it up) +CREATE FUNCTION pgautofailover.archiver_add_storage + (archiverid bigint, rcloneconfigname text) + RETURNS bigint -- archiverstorageid + LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + resolved_rcloneconfigid bigint; + new_id bigint; +BEGIN + SELECT rc.rcloneconfigid INTO resolved_rcloneconfigid + FROM pgautofailover.rclone_config rc + WHERE rc.name = rcloneconfigname; + + IF resolved_rcloneconfigid IS NULL THEN + RAISE EXCEPTION 'rclone_config "%" does not exist', rcloneconfigname; + END IF; + + INSERT INTO pgautofailover.archiver_storage + (archiverid, storagemethod, rcloneconfigid) + VALUES (archiverid, 'rclone', resolved_rcloneconfigid) + RETURNING archiverstorageid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.archiver_add_storage(bigint,text) + is 'attach an additional rclone storage target to an archiver, referencing an existing named rclone_config'; + +grant execute on function pgautofailover.archiver_add_storage(bigint,text) + to autoctl_node; + +-- detaches only; the referenced rclone_config row is untouched and +-- keeps serving any other archiver still referencing it +CREATE FUNCTION pgautofailover.archiver_remove_storage(archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_storage AS a_s + WHERE a_s.archiverstorageid = archiver_remove_storage.archiverstorageid + AND a_s.storagemethod <> 'local'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_storage % does not exist, or is the mandatory local target', + archiverstorageid; + END IF; +END; +$$; + +comment on function pgautofailover.archiver_remove_storage(bigint) + is 'detach a non-local storage target from an archiver (the local target cannot be removed)'; + +grant execute on function pgautofailover.archiver_remove_storage(bigint) + to autoctl_node; + +-- fans out to one CREATE of a pgautofailover.node row (haspgdata = +-- false) per group currently in formationid +-- Parameters are prefixed in_* here (unlike this file's usual +-- function-qualified-reference convention): ON CONFLICT's own target +-- column list can't be schema/function-qualified at all (that syntax +-- only accepts bare column names or ON CONSTRAINT), so a same-named +-- parameter would still be genuinely ambiguous there even when every +-- other clause in this function could disambiguate it. +CREATE FUNCTION pgautofailover.archiver_add_formation + (in_archiverid bigint, in_formationid text) + RETURNS SETOF bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + grp record; + new_nodeid bigint; +BEGIN + INSERT INTO pgautofailover.archiver_formation (archiverid, formationid) + VALUES (in_archiverid, in_formationid) + ON CONFLICT (archiverid, formationid) DO NOTHING; + + FOR grp IN + SELECT DISTINCT n.groupid + FROM pgautofailover.node n + WHERE n.formationid = in_formationid + LOOP + -- nodeport = 0 is a permanent sentinel, not an M1 stopgap: an + -- ARCHIVING row has no postmaster of its own to be reachable on, + -- so nodehost:nodeport isn't a connectable address here the way + -- it is for every haspgdata row -- see node_nodehost_nodeport_ + -- haspgdata_idx's own comment, which is exactly why that unique + -- index is scoped to haspgdata rows only. reportedstate starts at + -- 'wait_standby', same as any freshly-registered node -- it only + -- reaches 'archiving' once a real keeper's pg_receivewal is + -- actually running (no service_archiver process exists yet at + -- this milestone). + INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata, candidatepriority, + replicationquorum) + VALUES (in_formationid, grp.groupid, + 'archiver-' || in_archiverid || '-' || grp.groupid, + (SELECT a.hostname FROM pgautofailover.archiver a + WHERE a.archiverid = in_archiverid), + 0, + 'wait_standby', 'wait_standby', false, 0, false) + RETURNING nodeid INTO new_nodeid; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, nodeid) + VALUES (in_archiverid, 'wal-receiver', + '', new_nodeid); + + RETURN NEXT new_nodeid; + END LOOP; + + RETURN; +END; +$$; + +comment on function pgautofailover.archiver_add_formation(bigint,text) + is 'attach an archiver to every group of a formation, creating one lightweight ARCHIVING node row per group'; + +grant execute on function pgautofailover.archiver_add_formation(bigint,text) + to autoctl_node; + +-- Deleting the node row is enough: archiver_node.nodeid's own +-- ON DELETE CASCADE removes the matching wal-receiver archiver_node row +-- automatically (see that column's own comment). +CREATE FUNCTION pgautofailover.archiver_remove_formation + (archiverid bigint, formationid text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.node n + WHERE n.formationid = archiver_remove_formation.formationid + AND n.nodeid IN (SELECT an.nodeid + FROM pgautofailover.archiver_node an + WHERE an.archiverid = archiver_remove_formation.archiverid + AND an.kind = 'wal-receiver'); + + DELETE FROM pgautofailover.archiver_formation af + WHERE af.archiverid = archiver_remove_formation.archiverid + AND af.formationid = archiver_remove_formation.formationid; +END; +$$; + +comment on function pgautofailover.archiver_remove_formation(bigint,text) + is 'detach an archiver from a formation, removing its ARCHIVING node row in every group'; + +grant execute on function pgautofailover.archiver_remove_formation(bigint,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified, even inside an +-- expression like coalesce(groupid, -1)) forces this naming here. +CREATE FUNCTION pgautofailover.set_archiver_policy + ( + in_formationid text, in_groupid int DEFAULT NULL, + in_archiverquorum int DEFAULT NULL, + in_basebackuppolicyid bigint DEFAULT NULL, + in_replicationquorumeligible bool DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.archiver_policy + (formationid, groupid, archiverquorum, + basebackuppolicyid, replicationquorumeligible) + VALUES (in_formationid, in_groupid, + coalesce(in_archiverquorum, 1), + in_basebackuppolicyid, + coalesce(in_replicationquorumeligible, false)) + ON CONFLICT (formationid, (coalesce(groupid, -1))) DO UPDATE + SET archiverquorum = coalesce(EXCLUDED.archiverquorum, + pgautofailover.archiver_policy.archiverquorum), + basebackuppolicyid = coalesce(EXCLUDED.basebackuppolicyid, + pgautofailover.archiver_policy.basebackuppolicyid), + replicationquorumeligible = coalesce(EXCLUDED.replicationquorumeligible, + pgautofailover.archiver_policy.replicationquorumeligible); +END; +$$; + +comment on function pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + is 'set (or override) archiver_quorum/basebackup policy/replication-quorum eligibility for a formation, or one of its groups'; + +grant execute on function + pgautofailover.set_archiver_policy(text,int,int,bigint,bool) + to autoctl_node; + +-- resolves group-specific override first, then the formation-wide +-- (groupid IS NULL) default, then this schema's own hardcoded defaults. +-- Deliberately plpgsql, not a single SQL query: an earlier draft tried to +-- express the three-way fallback as one UNION ALL ... LIMIT 1 query, but +-- UNION ALL has no ordering guarantee across its branches, so LIMIT 1 +-- could just as easily return the formation-wide or hardcoded default +-- even when a group-specific override exists. Sequential SELECT INTO ... +-- IF FOUND is unambiguous. +CREATE FUNCTION pgautofailover.get_archiver_policy(formationid text, groupid int) + RETURNS TABLE (archiverquorum int, basebackuppolicyid bigint, + replicationquorumeligible bool) + LANGUAGE plpgsql STABLE +AS $$ +BEGIN + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid = get_archiver_policy.groupid; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT ap.archiverquorum, ap.basebackuppolicyid, ap.replicationquorumeligible + FROM pgautofailover.archiver_policy ap + WHERE ap.formationid = get_archiver_policy.formationid + AND ap.groupid IS NULL; + + IF FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT 1, p.basebackuppolicyid, false + FROM pgautofailover.basebackup_policy p + WHERE p.policyname = 'default'; +END; +$$; + +comment on function pgautofailover.get_archiver_policy(text,int) + is 'resolve archiver policy for (formation, group): group override, else formation default, else this schema''s own defaults'; + +grant execute on function pgautofailover.get_archiver_policy(text,int) + to autoctl_node; + +-- the archive_command confirmation check: true iff at least +-- archiver_quorum distinct archivers have durably reported %f +CREATE FUNCTION pgautofailover.wal_archived + (formationid text, groupid int, walfilename text) + RETURNS bool + LANGUAGE sql STABLE +AS $$ + SELECT count(DISTINCT aw.archiverid) >= + (SELECT archiverquorum + FROM pgautofailover.get_archiver_policy(wal_archived.formationid, + wal_archived.groupid)) + FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = wal_archived.formationid + AND aw.groupid = wal_archived.groupid + AND aw.walfilename = wal_archived.walfilename; +$$; + +comment on function pgautofailover.wal_archived(text,int,text) + is 'archive_command confirmation check: has segment %f already landed durably on archiver_quorum archiver(s)?'; + +grant execute on function pgautofailover.wal_archived(text,int,text) + to autoctl_node; + +-- inserts into archiver_wal (idempotent on conflict) +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_wal_received + (in_nodeid bigint, in_walfilename text, in_lsn pg_lsn) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + target record; +BEGIN + SELECT n.formationid, n.groupid, an.archiverid + INTO target + FROM pgautofailover.archiver_node an + JOIN pgautofailover.node n ON n.nodeid = an.nodeid + WHERE an.nodeid = in_nodeid + AND an.kind = 'wal-receiver'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'node % is not an ARCHIVING wal-receiver node', in_nodeid; + END IF; + + INSERT INTO pgautofailover.archiver_wal + (formationid, groupid, walfilename, archiverid, lsn) + VALUES (target.formationid, target.groupid, in_walfilename, target.archiverid, in_lsn) + ON CONFLICT (formationid, groupid, walfilename, archiverid) DO NOTHING; +END; +$$; + +comment on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + is 'reports a WAL segment durably captured by an ARCHIVING node'; + +grant execute on function pgautofailover.report_wal_received(bigint,text,pg_lsn) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_started + ( + archiverid bigint, formationid text, groupid int, + label text, timeline int, startlsn pg_lsn, + source pgautofailover.basebackup_source, + replaymode pgautofailover.basebackup_replay_mode DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + new_id bigint; +BEGIN + INSERT INTO pgautofailover.basebackup + (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, storagelocation, status) + VALUES (archiverid, formationid, groupid, label, timeline, startlsn, + source, replaymode, '', 'in_progress') + RETURNING basebackupid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + is 'records the start of a new base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_started + (bigint,text,int,text,int,pg_lsn,pgautofailover.basebackup_source,pgautofailover.basebackup_replay_mode) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_completed + (basebackupid bigint, endlsn pg_lsn, sizebytes bigint, storagelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup AS bb + SET endlsn = report_basebackup_completed.endlsn, + sizebytes = report_basebackup_completed.sizebytes, + storagelocation = report_basebackup_completed.storagelocation, + status = 'complete', + period = tstzrange(lower(bb.period), now()) + WHERE bb.basebackupid = report_basebackup_completed.basebackupid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; +END; +$$; + +comment on function pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + is 'records the successful completion of a base-backup production job'; + +grant execute on function + pgautofailover.report_basebackup_completed(bigint,pg_lsn,bigint,text) + to autoctl_node; + +-- marks the basebackup row deleted (never a real DELETE), then prunes +-- any archiver_wal rows this group no longer needs to retain +CREATE FUNCTION pgautofailover.report_basebackup_deleted(basebackupid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + bb record; +BEGIN + UPDATE pgautofailover.basebackup AS b + SET status = 'deleted', deletedat = now() + WHERE b.basebackupid = report_basebackup_deleted.basebackupid + RETURNING b.formationid, b.groupid INTO bb; + + IF NOT FOUND THEN + RAISE EXCEPTION 'basebackup % does not exist', basebackupid; + END IF; + + PERFORM pgautofailover.prune_archiver_wal(bb.formationid, bb.groupid); +END; +$$; + +comment on function pgautofailover.report_basebackup_deleted(bigint) + is 'marks a base backup deleted (retains history) and prunes any archiver_wal rows no group backup needs anymore'; + +grant execute on function pgautofailover.report_basebackup_deleted(bigint) + to autoctl_node; + +-- deletes every archiver_wal row for (formationid, groupid) older than +-- the earliest still-'complete' basebackup's startlsn, across every +-- archiver holding a copy. When no 'complete' backup remains for this +-- group, nothing is pruned -- there is no anchor point to replay forward +-- from, so every captured segment is still needed. +CREATE FUNCTION pgautofailover.prune_archiver_wal(formationid text, groupid int) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + oldest_startlsn pg_lsn; + deleted_count bigint; +BEGIN + SELECT min(b.startlsn) INTO oldest_startlsn + FROM pgautofailover.basebackup b + WHERE b.formationid = prune_archiver_wal.formationid + AND b.groupid = prune_archiver_wal.groupid + AND b.status = 'complete'; + + IF oldest_startlsn IS NULL THEN + RETURN 0; + END IF; + + WITH deleted AS ( + DELETE FROM pgautofailover.archiver_wal aw + WHERE aw.formationid = prune_archiver_wal.formationid + AND aw.groupid = prune_archiver_wal.groupid + AND aw.lsn < oldest_startlsn + RETURNING 1 + ) + SELECT count(*) INTO deleted_count FROM deleted; + + RETURN deleted_count; +END; +$$; + +comment on function pgautofailover.prune_archiver_wal(text,int) + is 'deletes archiver_wal rows for (formation, group) older than the oldest still-complete base backup''s startlsn'; + +grant execute on function pgautofailover.prune_archiver_wal(text,int) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.report_basebackup_synced + (in_basebackupid bigint, in_archiverstorageid bigint, in_remotelocation text) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.basebackup_storage + (basebackupid, archiverstorageid, syncedat, remotelocation) + VALUES (in_basebackupid, in_archiverstorageid, now(), in_remotelocation) + ON CONFLICT (basebackupid, archiverstorageid) DO UPDATE + SET syncedat = now(), + remotelocation = EXCLUDED.remotelocation; +END; +$$; + +comment on function pgautofailover.report_basebackup_synced(bigint,bigint,text) + is 'records a successful cold-storage sync of a base backup to one storage target'; + +grant execute on function + pgautofailover.report_basebackup_synced(bigint,bigint,text) + to autoctl_node; + +CREATE FUNCTION pgautofailover.report_basebackup_remote_deleted + (basebackupid bigint, archiverstorageid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.basebackup_storage AS bs + SET deletedat = now() + WHERE bs.basebackupid = report_basebackup_remote_deleted.basebackupid + AND bs.archiverstorageid = report_basebackup_remote_deleted.archiverstorageid; +END; +$$; + +comment on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + is 'records that a base backup''s remote copy on one storage target has been pruned'; + +grant execute on function + pgautofailover.report_basebackup_remote_deleted(bigint,bigint) + to autoctl_node; + +-- filters status = 'complete' only +CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE +AS $$ + SELECT * FROM pgautofailover.basebackup b + WHERE b.formationid = get_latest_basebackup.formationid + AND b.groupid = get_latest_basebackup.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC + LIMIT 1; +$$; + +comment on function pgautofailover.get_latest_basebackup(text,int) + is 'fetch the most recent complete base backup for (formation, group)'; + +grant execute on function pgautofailover.get_latest_basebackup(text,int) + to autoctl_node; + +-- for kind = 'warm-standby': raises if the owning archiver is already at +-- its maxresidentreplay cap +CREATE FUNCTION pgautofailover.create_archiver_node + ( + archiverid bigint, + kind pgautofailover.archiver_node_kind, + pgdata text, + hostname text DEFAULT NULL, + nodeid bigint DEFAULT NULL, -- required iff kind = 'wal-receiver' + formationid text DEFAULT NULL, -- required iff kind = 'warm-standby' + groupid int DEFAULT NULL, -- required iff kind = 'warm-standby' + cadence pgautofailover.archiver_node_cadence DEFAULT NULL, + nodecluster text DEFAULT NULL, -- only for 'warm-standby' + cadence = 'continuous' + pitrstatus pgautofailover.pitr_status DEFAULT NULL + ) + RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + residentcount int; + maxresident int; + new_id bigint; +BEGIN + IF kind = 'warm-standby' THEN + SELECT a.maxresidentreplay INTO maxresident + FROM pgautofailover.archiver a + WHERE a.archiverid = create_archiver_node.archiverid; + + SELECT count(*) INTO residentcount + FROM pgautofailover.archiver_node an + WHERE an.archiverid = create_archiver_node.archiverid + AND an.kind = 'warm-standby'; + + IF residentcount >= maxresident THEN + RAISE EXCEPTION + 'archiver % is already at its maxresidentreplay cap (%)', + archiverid, maxresident; + END IF; + END IF; + + INSERT INTO pgautofailover.archiver_node + (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + VALUES (archiverid, kind, pgdata, hostname, nodeid, + formationid, groupid, cadence, nodecluster, pitrstatus) + RETURNING archivernodeid INTO new_id; + + RETURN new_id; +END; +$$; + +comment on function pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + is 'registers a concrete Postgres instance an archiver hosts, derives, or is otherwise associated with'; + +grant execute on function + pgautofailover.create_archiver_node + (bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,int, + pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) + to autoctl_node; + +CREATE FUNCTION pgautofailover.remove_archiver_node(archivernodeid bigint) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM pgautofailover.archiver_node an + WHERE an.archivernodeid = remove_archiver_node.archivernodeid; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist', archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.remove_archiver_node(bigint) + is 'removes an archiver_node row'; + +grant execute on function pgautofailover.remove_archiver_node(bigint) + to autoctl_node; + +CREATE FUNCTION pgautofailover.set_archiver_node_pitr_status + (archivernodeid bigint, pitrstatus pgautofailover.pitr_status) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + UPDATE pgautofailover.archiver_node AS an + SET pitrstatus = set_archiver_node_pitr_status.pitrstatus + WHERE an.archivernodeid = set_archiver_node_pitr_status.archivernodeid + AND an.kind = 'pitr'; + + IF NOT FOUND THEN + RAISE EXCEPTION 'archiver_node % does not exist, or is not kind = pitr', + archivernodeid; + END IF; +END; +$$; + +comment on function pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + is 'updates a PITR archiver_node''s lifecycle status'; + +grant execute on function + pgautofailover.set_archiver_node_pitr_status(bigint,pgautofailover.pitr_status) + to autoctl_node; + +-- pushed by the local pg_autoctl pitr CLI immediately after acting +-- locally -- never blocks or gates the local action on this succeeding +CREATE FUNCTION pgautofailover.report_pitr_status + ( + archivernodeid bigint, operation pgautofailover.pitr_operation, + requestedspec jsonb, + observedlsn pg_lsn, observedtimestamp timestamptz, + observedpausestate text, note text DEFAULT NULL + ) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_history + (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note) + VALUES (archivernodeid, operation, requestedspec, + observedlsn, observedtimestamp, observedpausestate, note); +END; +$$; + +comment on function pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + is 'records one PITR operation''s outcome -- a best-effort report, never gating the local action it follows'; + +grant execute on function + pgautofailover.report_pitr_status + (bigint,pgautofailover.pitr_operation,jsonb,pg_lsn,timestamptz,text,text) + to autoctl_node; + +-- in_* parameters: see archiver_add_formation's own comment on why an ON +-- CONFLICT target list (which can't be qualified) forces this naming here. +CREATE FUNCTION pgautofailover.pitr_queue_command + (in_archivernodeid bigint, in_command pgautofailover.pitr_command, + in_commandspec jsonb DEFAULT NULL) + RETURNS void LANGUAGE plpgsql SECURITY DEFINER +AS $$ +BEGIN + INSERT INTO pgautofailover.pitr_pending_command + (archivernodeid, command, commandspec) + VALUES (in_archivernodeid, in_command, in_commandspec) + ON CONFLICT (archivernodeid) DO UPDATE + SET command = EXCLUDED.command, + commandspec = EXCLUDED.commandspec, + queuedat = now(); +END; +$$; + +comment on function pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + is 'queues a PITR command for a monitor-mediated (kind = pitr, pg_autoctl node run) agent to pick up'; + +grant execute on function + pgautofailover.pitr_queue_command(bigint,pgautofailover.pitr_command,jsonb) + to autoctl_node; + +-- returns the pending command and resets the queue slot to 'none' in the +-- same call -- an agent polling this never processes the same command twice +-- Reads the pending command, then clears it, as two separate statements: +-- UPDATE ... RETURNING always reflects the row *after* the update is +-- applied, so folding the reset into the same RETURNING clause that reads +-- the command would always report back the very 'none' this function just +-- set, never the command that was actually queued. FOR UPDATE locks the +-- row across both statements, so a concurrent caller for the same +-- archivernodeid still can't observe or consume the same command twice. +CREATE FUNCTION pgautofailover.pitr_next_command(in_archivernodeid bigint) + RETURNS pgautofailover.pitr_command LANGUAGE plpgsql SECURITY DEFINER +AS $$ +DECLARE + next_command pgautofailover.pitr_command; +BEGIN + SELECT pc.command INTO next_command + FROM pgautofailover.pitr_pending_command pc + WHERE pc.archivernodeid = in_archivernodeid + FOR UPDATE; + + IF next_command IS NULL OR next_command = 'none' THEN + RETURN 'none'; + END IF; + + UPDATE pgautofailover.pitr_pending_command AS pc + SET command = 'none', commandspec = NULL + WHERE pc.archivernodeid = in_archivernodeid; + + RETURN next_command; +END; +$$; + +comment on function pgautofailover.pitr_next_command(bigint) + is 'pops and clears the next queued PITR command for an agent to act on'; + +grant execute on function pgautofailover.pitr_next_command(bigint) + to autoctl_node; + -- Testing-only functions, not granted to autoctl_node: they let -- regression/isolation tests hold the monitor's own LockFormation()/ -- LockNodeGroup() locks explicitly, and simulate a health-check-worker diff --git a/src/monitor/regress_schedule b/src/monitor/regress_schedule index 4d2ca7bdb..60551f7c9 100644 --- a/src/monitor/regress_schedule +++ b/src/monitor/regress_schedule @@ -46,6 +46,7 @@ test: lock_and_fetch_migration test: timeline_fork_detection test: failover_candidate_leaves_secondary test: cluster_init_failover_rule_attribution +test: archiving_schema test: dummy_update test: drop_extension test: upgrade diff --git a/src/monitor/sql/archiving_schema.sql b/src/monitor/sql/archiving_schema.sql new file mode 100644 index 000000000..7dd270b79 --- /dev/null +++ b/src/monitor/sql/archiving_schema.sql @@ -0,0 +1,189 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. + +\x on + +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); + +-- ── register_archiver ──────────────────────────────────────────────────── + +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset + +SELECT archiverid, archivername, hostname, basebackuppolicyid, autoregister, + maxresidentreplay + FROM pgautofailover.archiver; + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── + +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset + +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── + +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); + +-- ── base backup lifecycle ─────────────────────────────────────────────────── + +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset + +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; + +-- ── rclone_config + archiver_storage ───────────────────────────────────── + +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset + +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset + +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset + +SELECT pgautofailover.archiver_remove_storage(:local_storageid); + +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── + +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset + +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); + +-- ── PITR lifecycle ─────────────────────────────────────────────────────── + +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── + +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +SELECT pgautofailover.pitr_next_command(:pitrnodeid); + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; From 68429b9e3c5cf4c7a1398f9ae3056bdb2fd9c61a Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 01:45:00 +0200 Subject: [PATCH 02/55] monitor: add ARCHIVING state to the FSM (M2, monitor side only) Adds monitor-side (SQL FSM + C) support for the ARCHIVING replication state, so an ARCHIVING node row (haspgdata = false, created by M1's archiver_add_formation()) is driven through the same node_active() protocol as an ordinary node instead of being stuck at wait_standby forever. No keeper-side/service_archiver work yet -- this is groundwork, verified via node_active() calls made directly against the monitor. - ReplicationState gains REPLICATION_STATE_ARCHIVING (C) / 'archiving' already existed on the SQL enum from M1. - AutoFailoverNode gains hasPgData, populated via TupleToAutoFailoverNode. Looked up by name (SPI_fnumber), not the file's usual hardcoded Anum_ constant: this function is also called against a "RETURNING node.*" tuple descriptor whose physical column order diverges from the explicit SELECT list's logical order once pg_versionnum/pg_version/ pg_versionstring/citus_version are in the mix, so a hardcoded ordinal would silently read the wrong (and wrongly-typed) column for that caller. - MonitorFSM[]: pos 307/309/315/317/319 (report_lsn/wait_standby, primary converged -> secondary/catchingup) gain an explicit hasPgData = TRUE restriction, paired with 5 new hasPgData = FALSE mirror rows (pos 394-398) assigning ARCHIVING instead -- appended after the existing MS-failover cluster since the ordinary rows are numbered with no room between them for 5 more, and the hasPgData split makes their relative order irrelevant to first-match-wins. Pos 367's MS-failover fan-out row (and BuildCandidateList's own C-side secondaryStates list) now also admits ARCHIVING, pulling it into report_lsn during elections exactly like SECONDARY/CATCHINGUP. - system_identifier_is_null_at_init_only loosened to also allow a NULL sysidentifier while reportedstate is 'archiving' or 'report_lsn': an ARCHIVING row never gets a real one. The 2.2--2.3 migration mirror casts the column to text instead of the literals to the enum, since this script's own earlier ADD VALUE 'archiving' and this constraint run in the same ALTER EXTENSION UPDATE transaction and Postgres refuses to create new instances of a not-yet-committed enum value. - keeper_fsm_edges.sql's own "expect zero rows" comment updated: 8 rows are now expected there, a real and currently correct gap -- the monitor side landed first, with no service_archiver/KeeperFSM[] support yet to report ARCHIVING or drive pg_receivewal (next milestone). Verified against a hand-run node_active() scenario (register primary + secondary, converge to primary/secondary, attach an archiver, confirm wait_standby -> archiving instead of catchingup, steady-state archiving stays archiving, replication_quorum = true fans out apply_settings to the primary exactly like an ordinary quorum standby, and rule_pos attribution points at the new rows) in addition to the full regress (20/20) + isolation (6/6) suites and a real 2.2 -> 2.3 extension upgrade. --- src/monitor/expected/candidate_count_gate.out | 40 ++--- .../expected/check_fsm_reachability.out | 4 +- src/monitor/expected/fsm.out | 109 +++++++++++-- src/monitor/expected/keeper_fsm_edges.out | 25 ++- src/monitor/expected/stale_primary_report.out | 38 ++--- src/monitor/group_state_machine.c | 147 ++++++++++++++++-- src/monitor/node_metadata.c | 17 ++ src/monitor/node_metadata.h | 11 +- src/monitor/pgautofailover--2.2--2.3.sql | 46 ++++++ src/monitor/pgautofailover.sql | 13 +- src/monitor/replication_state.c | 5 + src/monitor/replication_state.h | 3 +- src/monitor/sql/keeper_fsm_edges.sql | 11 +- 13 files changed, 384 insertions(+), 85 deletions(-) diff --git a/src/monitor/expected/candidate_count_gate.out b/src/monitor/expected/candidate_count_gate.out index 9cfa786d7..f2abbeb9c 100644 --- a/src/monitor/expected/candidate_count_gate.out +++ b/src/monitor/expected/candidate_count_gate.out @@ -279,112 +279,112 @@ RESET pgautofailover.startup_grace_period; -- neither is a stable value to pin in this file's own expected output. SELECT reportedstate, goalstate, rule_pos, rule_section, description FROM pgautofailover.last_events('ccg_test', count => 100); --[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------- reportedstate | init goalstate | single rule_pos | 209 rule_section | early_checks description | alone in group, candidate-eligible -> single --[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | single rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "single" --[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "wait_standby" --[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary rule_pos | 401 rule_section | primary_node description | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "wait_primary" --[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 315 rule_section | reporting_node description | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "catchingup" --[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary rule_pos | 321 rule_section | reporting_node description | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 22 "ccg_s1" (ccg_s1:5432): "secondary" --[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary rule_pos | 411 rule_section | primary_node description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | primary rule_pos | rule_section | description | New state is reported by node 21 "ccg_p" (ccg_p:5432): "primary" --[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "wait_standby" --[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | apply_settings rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "catchingup" --[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 23 "ccg_s2" (ccg_s2:5432): "secondary" --[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) --[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) diff --git a/src/monitor/expected/check_fsm_reachability.out b/src/monitor/expected/check_fsm_reachability.out index 467daebb8..04031b3b2 100644 --- a/src/monitor/expected/check_fsm_reachability.out +++ b/src/monitor/expected/check_fsm_reachability.out @@ -18,7 +18,7 @@ SELECT count(*) AS total_edge_count FROM pgautofailover.dump_fsm_edges(); total_edge_count ------------------ - 177 + 182 (1 row) -- pos 301 ("converged secondary, reportedTLI not an ancestor of reference @@ -72,7 +72,7 @@ SELECT count(*) AS missing_with_empty_keeper_edges FROM pgautofailover.check_fsm_reachability('[]'::jsonb); missing_with_empty_keeper_edges --------------------------------- - 177 + 182 (1 row) -- Providing exactly pos 301's own edge, plus one of pos 343's two edges diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index f0040eb11..bf1c41d0c 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -417,7 +417,7 @@ section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | wait_primary, join_primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | isHealthy=true candidate_node_conditions | group_conditions | @@ -432,7 +432,7 @@ section_path | reporting_node.from_context active_node_current_state | report_lsn other_node_current_state | primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | isHealthy=true candidate_node_conditions | group_conditions | @@ -477,7 +477,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | wait_primary, join_primary candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -492,7 +492,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | -active_node_conditions | replicationQuorum=true +active_node_conditions | replicationQuorum=true, hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -507,7 +507,7 @@ section_path | reporting_node.from_context active_node_current_state | wait_standby other_node_current_state | primary candidate_node_current_state | -active_node_conditions | replicationQuorum=false +active_node_conditions | replicationQuorum=false, hasPgData=true other_node_conditions | candidate_node_conditions | group_conditions | @@ -864,7 +864,7 @@ comment | MS-failover: activeNode in report_lsn, failover c pos | 367 section | reporting_node section_path | reporting_node.ms_failover.candidate_fanout -active_node_current_state | secondary, catchingup +active_node_current_state | secondary, catchingup, archiving other_node_current_state | candidate_node_current_state | active_node_conditions | @@ -874,7 +874,7 @@ group_conditions | inMSFailoverCluster=true active_node_assigned_state | report_lsn other_node_assigned_state | has_extra_action | f -comment | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +comment | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) -[ RECORD 58 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 369 section | reporting_node @@ -1071,6 +1071,81 @@ other_node_assigned_state | maintenance has_extra_action | f comment | nodesCount>2, primary unhealthy, converged prepare_maintenance -> primary maintenance -[ RECORD 71 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 394 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 307: report_lsn, primary converged wait/join_primary, healthy -> archiving +-[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 395 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | report_lsn +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | isHealthy=true +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 309: report_lsn, primary converged primary, healthy -> archiving +-[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 396 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | wait_primary, join_primary +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving +-[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 397 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=true, hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | apply_settings +has_extra_action | f +comment | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings +-[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 398 +section | reporting_node +section_path | reporting_node.from_context +active_node_current_state | wait_standby +other_node_current_state | primary +candidate_node_current_state | +active_node_conditions | replicationQuorum=false, hasPgData=false +other_node_conditions | +candidate_node_conditions | +group_conditions | +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving +-[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node section_path | primary_node @@ -1085,7 +1160,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node section_path | primary_node @@ -1100,7 +1175,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 73 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node section_path | primary_node @@ -1115,7 +1190,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node section_path | primary_node @@ -1130,7 +1205,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 75 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node section_path | primary_node @@ -1145,7 +1220,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node section_path | primary_node @@ -1160,7 +1235,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 82 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node section_path | primary_node @@ -1175,7 +1250,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 83 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node section_path | primary_node @@ -1190,7 +1265,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 84 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node section_path | primary_node @@ -1205,7 +1280,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 85 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node section_path | primary_node @@ -1220,7 +1295,7 @@ active_node_assigned_state | other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup --[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 86 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node section_path | primary_node diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 90060dbeb..3b908a2a9 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -175,8 +175,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- Expected result: empty. Every MonitorFSM[] rule currently has a matching --- KeeperFSM[] row for every current_state it can assign a transition from. +-- Expected result: the 8 rows from pos 367/396/397/398's own archiver- +-- related edges (reportedState/goalState = ARCHIVING) -- a real, currently +-- expected gap: the monitor side of the ARCHIVING state (Archiving & +-- Disaster Recovery design, milestone 2) landed first, on its own, with no +-- corresponding KeeperFSM[] rows yet (no service_archiver process exists +-- to report ARCHIVING or drive pg_receivewal at this milestone either) -- +-- see the Build order in ~/dev/temp/archiving-disaster-recovery.md. +-- Every other MonitorFSM[] rule still has a matching KeeperFSM[] row for +-- every current_state it can assign a transition from. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -191,9 +198,17 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | n | current_state | assigned_state | comment -------+---+---------------+----------------+--------- -(0 rows) + rule | n | current_state | assigned_state | comment +------+---+---------------+----------------+------------------------------------------------------------------------------------------------------------------- + 367 | 1 | | report_lsn | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) + 367 | 1 | archiving | report_lsn | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) + 396 | 1 | | archiving | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving + 396 | 1 | wait_standby | archiving | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving + 397 | 1 | | archiving | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings + 397 | 1 | wait_standby | archiving | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings + 398 | 1 | | archiving | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving + 398 | 1 | wait_standby | archiving | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving +(8 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the diff --git a/src/monitor/expected/stale_primary_report.out b/src/monitor/expected/stale_primary_report.out index cc40f4d87..c9f7456d3 100644 --- a/src/monitor/expected/stale_primary_report.out +++ b/src/monitor/expected/stale_primary_report.out @@ -319,112 +319,112 @@ reportedstate | secondary -- neither is a stable value to pin in this file's own expected output. SELECT reportedstate, goalstate, rule_pos, rule_section, description FROM pgautofailover.last_events('spr_test', count => 100); --[ RECORD 1 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 1 ]-+---------------------------------------------------------------------------------------------- reportedstate | init goalstate | single rule_pos | 209 rule_section | early_checks description | alone in group, candidate-eligible -> single --[ RECORD 2 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 2 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | single rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "single" --[ RECORD 3 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 3 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "wait_standby" --[ RECORD 4 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 4 ]-+---------------------------------------------------------------------------------------------- reportedstate | single goalstate | wait_primary rule_pos | 401 rule_section | primary_node description | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 5 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 5 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | wait_primary rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "wait_primary" --[ RECORD 6 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 6 ]-+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 315 rule_section | reporting_node description | wait_standby, primary converged wait/join_primary -> catchingup --[ RECORD 7 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 7 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "catchingup" --[ RECORD 8 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 8 ]-+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | secondary rule_pos | 321 rule_section | reporting_node description | caught up, same TLI as primary, within sync threshold -> secondary --[ RECORD 9 ]-+-------------------------------------------------------------------------------------------- +-[ RECORD 9 ]-+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "secondary" --[ RECORD 10 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 10 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_primary goalstate | primary rule_pos | 411 rule_section | primary_node description | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 11 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 11 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | primary rule_pos | rule_section | description | New state is reported by node 18 "spr_p" (spr_p:5432): "primary" --[ RECORD 12 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 12 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | wait_standby rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "wait_standby" --[ RECORD 13 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 13 ]+---------------------------------------------------------------------------------------------- reportedstate | wait_standby goalstate | catchingup rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 14 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 14 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | apply_settings rule_pos | 317 rule_section | reporting_node description | wait_standby (quorum member), primary converged primary -> catchingup + apply_settings --[ RECORD 15 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 15 ]+---------------------------------------------------------------------------------------------- reportedstate | catchingup goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "catchingup" --[ RECORD 16 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 16 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | catchingup rule_pos | rule_section | description | New state is reported by node 20 "spr_s2" (spr_s2:5432): "secondary" --[ RECORD 17 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 17 ]+---------------------------------------------------------------------------------------------- reportedstate | primary goalstate | secondary rule_pos | rule_section | description | New state is reported by node 19 "spr_s1" (spr_s1:5432): "primary" --[ RECORD 18 ]+-------------------------------------------------------------------------------------------- +-[ RECORD 18 ]+---------------------------------------------------------------------------------------------- reportedstate | secondary goalstate | report_lsn rule_pos | 367 rule_section | reporting_node -description | MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn (1 of 4) +description | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 33310e12d..52ee5dd7e 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -487,6 +487,7 @@ typedef struct NodeStatus bool isCitusWorkerGroup; bool replicationQuorum; bool isComparableToReferenceTli; + bool hasPgData; } NodeStatus; typedef struct NodeStatusPattern @@ -510,6 +511,12 @@ typedef struct NodeStatusPattern BoolPattern replicationQuorum; BoolPattern isComparableToReferenceTli; BoolPattern unreachableFromDemoteTimeout; + + /* + * true for every ordinary Postgres node; false only for an ARCHIVING + * membership row. See AutoFailoverNode.hasPgData's own comment. + */ + BoolPattern hasPgData; } NodeStatusPattern; static void @@ -535,6 +542,7 @@ BuildNodeStatus(GroupStateContext *ctx, AutoFailoverNode *node, NodeStatus *stat status->candidateEligible = node->candidatePriority > 0; status->isCitusWorkerGroup = IsCitusFormation(ctx->formation) && node->groupId > 0; status->replicationQuorum = node->replicationQuorum; + status->hasPgData = node->hasPgData; } @@ -667,7 +675,8 @@ NodeMatchesPattern(const NodeStatus *status, const NodeStatusPattern *pattern) BoolMatchesPattern(status->isComparableToReferenceTli, pattern->isComparableToReferenceTli) && BoolMatchesPattern(unreachableFromDemoteTimeout, - pattern->unreachableFromDemoteTimeout); + pattern->unreachableFromDemoteTimeout) && + BoolMatchesPattern(status->hasPgData, pattern->hasPgData); } @@ -2720,26 +2729,36 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy -> draining/maintenance + MS-failover cascade" }, - /* report_lsn, primary converged wait/join_primary, healthy */ + /* + * report_lsn, primary converged wait/join_primary, healthy -- hasPgData + * = BOOL_TRUE restricts this to ordinary nodes now that pos 394 (below, + * in the archiver mirror cluster appended after pos 393) is the + * hasPgData = BOOL_FALSE sibling assigning ARCHIVING instead of + * SECONDARY; the two are mutually exclusive on hasPgData alone, so + * their relative order doesn't matter. + */ { .pos = 307, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), .comment = "report_lsn, primary converged wait/join_primary, healthy -> secondary" }, - /* report_lsn, primary converged primary, healthy */ + /* report_lsn, primary converged primary, healthy -- see pos 307's own + * comment on hasPgData; pos 395 is this row's archiver mirror. */ { .pos = 309, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_SECONDARY), @@ -2771,39 +2790,51 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "report_lsn or fast_forward, continuing an already-started failover -> " "MS-failover cascade" }, - /* wait_standby, primary converged wait/join_primary */ + /* + * wait_standby, primary converged wait/join_primary -- hasPgData = + * BOOL_TRUE restricts this to ordinary nodes; pos 396 is the + * hasPgData = BOOL_FALSE sibling assigning ARCHIVING (see pos 307's + * own comment on why order between the two doesn't matter). + */ { .pos = 315, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = "wait_standby, primary converged wait/join_primary -> catchingup" }, - /* wait_standby (quorum member), primary converged primary */ + /* wait_standby (quorum member), primary converged primary -- see pos + * 315's own comment on hasPgData; pos 397 is this row's archiver + * mirror. */ { .pos = 317, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), - .replicationQuorum = BOOL_TRUE }, + .replicationQuorum = BOOL_TRUE, + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), .comment = "wait_standby (quorum member), primary converged primary -> " "catchingup + apply_settings" }, - /* wait_standby (not a quorum member), primary converged primary */ + /* wait_standby (not a quorum member), primary converged primary -- see + * pos 315's own comment on hasPgData; pos 398 is this row's archiver + * mirror. */ { .pos = 319, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, MONITOR_FSM_SECTION_FROM_CONTEXT }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), - .replicationQuorum = BOOL_FALSE }, + .replicationQuorum = BOOL_FALSE, + .hasPgData = BOOL_TRUE }, .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_CATCHINGUP), .comment = @@ -3239,15 +3270,17 @@ static const MonitorFSMTransition MonitorFSM[] = { .activeNode = { .statePattern = { .kind = NODE_STATE_TRANSITIONING, .reportedStates = STATES( REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP), + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING), .assignedStates = STATES( REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP) } + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING) } }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_REPORT_LSN), .comment = - "MS-failover fan-out: secondary/catchingup, not yet converged -> report_lsn " - "(1 of 4)" }, + "MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> " + "report_lsn (1 of 4)" }, { .pos = 369, .sectionPath = { @@ -3512,6 +3545,84 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "nodesCount>2, primary unhealthy, converged prepare_maintenance -> " "primary maintenance" }, + /* + * Archiver mirror cluster: the hasPgData = BOOL_FALSE siblings of pos + * 307/309/315/317/319 above, assigning ARCHIVING instead of + * SECONDARY/CATCHINGUP for an ARCHIVING membership row. Appended here + * (still sectionPath'd under REPORTING_NODE/FROM_CONTEXT, like their + * siblings) rather than interleaved next to each one, for the same + * reason the MS-failover cluster above is appended rather than + * renumbered into the ordinary rows: pos 307/309/315/317/319 are + * numbered every 2 with no room between consecutive pairs for 5 more + * rows, and since hasPgData makes each pair mutually exclusive, their + * relative array order doesn't affect first-match-wins correctness -- + * see each of those rows' own comment for the exact pairing. + */ + { .pos = 394, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 307: report_lsn, primary converged " + "wait/join_primary, healthy -> archiving" }, + + { .pos = 395, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY), + .isHealthy = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 309: report_lsn, primary converged " + "primary, healthy -> archiving" }, + + { .pos = 396, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 315: wait_standby, primary converged " + "wait/join_primary -> archiving" }, + + { .pos = 397, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_TRUE, + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .otherNodeAssignedState = GOAL(REPLICATION_STATE_APPLY_SETTINGS), + .comment = "archiver mirror of pos 317: wait_standby (quorum member), " + "primary converged primary -> archiving + apply_settings" }, + + { .pos = 398, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_FROM_CONTEXT + }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), + .replicationQuorum = BOOL_FALSE, + .hasPgData = BOOL_FALSE }, + .primaryNode = { .statePattern = FSM_STATE(REPLICATION_STATE_PRIMARY) }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 319: wait_standby (not a quorum member), " + "primary converged primary -> archiving" }, + /* * --- the PRIMARY_NODE section (sectionPath[0] == * MONITOR_FSM_SECTION_PRIMARY_NODE, pos 401 onward): the declarative @@ -4316,6 +4427,7 @@ NodeStatusPatternConditionsText(const NodeStatusPattern *pattern, bool *isNull) pattern->isComparableToReferenceTli); APPEND_BOOL_CONDITION(&buf, "unreachableFromDemoteTimeout", pattern->unreachableFromDemoteTimeout); + APPEND_BOOL_CONDITION(&buf, "hasPgData", pattern->hasPgData); if (buf.len == 0) { @@ -6242,8 +6354,9 @@ BuildCandidateList(GroupStateContext *ctx, List *nodesGroupList, ListCell *nodeCell = NULL; List *candidateNodesGroupList = NIL; - List *secondaryStates = list_make2_int(REPLICATION_STATE_SECONDARY, - REPLICATION_STATE_CATCHINGUP); + List *secondaryStates = list_make3_int(REPLICATION_STATE_SECONDARY, + REPLICATION_STATE_CATCHINGUP, + REPLICATION_STATE_ARCHIVING); foreach(nodeCell, nodesGroupList) { diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index 2b182ec70..d709eb2ec 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -179,6 +179,22 @@ TupleToAutoFailoverNode(TupleDesc tupleDescriptor, HeapTuple heapTuple) heap_getattr(heapTuple, Anum_pgautofailover_node_replication_stall_since, tupleDescriptor, &stallIsNull); + /* + * haspgdata is looked up by name, not by the Anum_ constant every other + * field here uses: this function is also called against a "RETURNING + * node.*" tuple descriptor (health_check_metadata.c), which reflects + * the table's true physical column order -- pg_versionnum/pg_version/ + * pg_versionstring/citus_version were appended between + * replication_stall_since and haspgdata by an earlier migration but + * were never added to AUTO_FAILOVER_NODE_TABLE_ALL_COLUMNS, so + * haspgdata's physical position (28) and its position in that + * explicit column list (24) genuinely differ. SPI_fnumber resolves the + * real attnum against whichever tupdesc was actually passed in, so + * this works correctly for both callers. + */ + int hasPgDataAttNum = SPI_fnumber(tupleDescriptor, "haspgdata"); + Datum hasPgData = heap_getattr(heapTuple, hasPgDataAttNum, + tupleDescriptor, &isNull); Oid goalStateOid = DatumGetObjectId(goalState); Oid reportedStateOid = DatumGetObjectId(reportedState); @@ -214,6 +230,7 @@ TupleToAutoFailoverNode(TupleDesc tupleDescriptor, HeapTuple heapTuple) regionIsNull ? "" : TextDatumGetCString(region); pgAutoFailoverNode->replicationStallSince = stallIsNull ? 0 : DatumGetTimestampTz(replicationStallSince); + pgAutoFailoverNode->hasPgData = DatumGetBool(hasPgData); return pgAutoFailoverNode; } diff --git a/src/monitor/node_metadata.h b/src/monitor/node_metadata.h index 0d336ca61..db5f5a69f 100644 --- a/src/monitor/node_metadata.h +++ b/src/monitor/node_metadata.h @@ -50,6 +50,7 @@ #define Anum_pgautofailover_node_nodecluster 21 #define Anum_pgautofailover_node_region 22 #define Anum_pgautofailover_node_replication_stall_since 23 +#define Anum_pgautofailover_node_haspgdata 24 #define AUTO_FAILOVER_NODE_TABLE_ALL_COLUMNS \ "formationid, " \ @@ -74,7 +75,8 @@ "replicationquorum, " \ "nodecluster, " \ "region, " \ - "replication_stall_since" + "replication_stall_since, " \ + "haspgdata" #define SELECT_ALL_FROM_AUTO_FAILOVER_NODE_TABLE \ @@ -139,6 +141,13 @@ typedef struct AutoFailoverNode char *nodeCluster; char *region; TimestampTz replicationStallSince; /* 0 = not stalled */ + + /* + * true for every ordinary Postgres node; false only for an ARCHIVING + * membership row (a pg_receivewal client, no PGDATA, no postmaster to + * manage). See pgautofailover.sql's own comment on the haspgdata column. + */ + bool hasPgData; } AutoFailoverNode; diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index f39cfeded..b5f595e8d 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -1971,3 +1971,49 @@ comment on function pgautofailover.pitr_next_command(bigint) grant execute on function pgautofailover.pitr_next_command(bigint) to autoctl_node; + + +-- +-- Archiving & Disaster Recovery, milestone 2: monitor-side FSM support for +-- the ARCHIVING state (group_state_machine.c). Loosen +-- system_identifier_is_null_at_init_only to also allow a NULL sysidentifier +-- in 'archiving' and 'report_lsn': an ARCHIVING row (haspgdata = false) has +-- no PGDATA of its own, ever, so it never acquires a real sysidentifier -- +-- see pgautofailover.sql's own comment on this constraint for why +-- 'report_lsn' is safe to loosen too. +-- +-- reportedstate::text IN (...) here, not reportedstate IN (...): this +-- script's own earlier "ALTER TYPE ... ADD VALUE 'archiving'" added that +-- label in this same transaction (ALTER EXTENSION ... UPDATE runs the whole +-- upgrade script as one transaction), and Postgres refuses to cast a string +-- literal to a not-yet-committed enum value ("unsafe use of new value... +-- must be committed before they can be used") -- casting the *column* +-- to text instead of the literals to the enum sidesteps that restriction +-- entirely, since reportedstate's own stored value is already a valid, +-- fully-committed enum datum by the time this constraint ever evaluates it. +-- pgautofailover.sql's fresh-install CHECK constraint doesn't need this: a +-- freshly CREATE TYPE'd enum has 'archiving' as a member from the start, +-- never added mid-transaction, so the ordinary enum-typed comparison there +-- is unaffected. +-- + +ALTER TABLE pgautofailover.node + DROP CONSTRAINT system_identifier_is_null_at_init_only; + +ALTER TABLE pgautofailover.node + ADD CONSTRAINT system_identifier_is_null_at_init_only + CHECK ( + ( + sysidentifier IS NULL + AND reportedstate::text + IN ( + 'init', + 'wait_standby', + 'catchingup', + 'dropped', + 'archiving', + 'report_lsn' + ) + ) + OR sysidentifier IS NOT NULL + ); diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 54da1fdfd..766b88e4a 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -158,6 +158,15 @@ CREATE TABLE pgautofailover.node -- primary server from scratch, because we have not done pg_ctl initdb -- at the time we call the register_node() function. -- + -- 'archiving' and 'report_lsn' are also allowed here: an ARCHIVING row + -- (haspgdata = false) has no PGDATA of its own, ever, so it never + -- acquires a real sysidentifier -- and 'report_lsn' is a state it + -- legitimately reaches too, pulled into elections the same as + -- SECONDARY/CATCHINGUP (see haspgdata's own comment). Loosening this + -- CHECK to also permit NULL in 'report_lsn' doesn't hide anything for + -- ordinary nodes: by the time a real node ever reaches report_lsn its + -- own bootstrap sequence has long since given it a real sysidentifier. + -- CONSTRAINT system_identifier_is_null_at_init_only CHECK ( ( @@ -167,7 +176,9 @@ CREATE TABLE pgautofailover.node 'init', 'wait_standby', 'catchingup', - 'dropped' + 'dropped', + 'archiving', + 'report_lsn' ) ) OR sysidentifier IS NOT NULL diff --git a/src/monitor/replication_state.c b/src/monitor/replication_state.c index f1191176e..c69d8c181 100644 --- a/src/monitor/replication_state.c +++ b/src/monitor/replication_state.c @@ -256,6 +256,11 @@ ReplicationStateGetName(ReplicationState replicationState) return "dropped"; } + case REPLICATION_STATE_ARCHIVING: + { + return "archiving"; + } + default: { ereport(ERROR, diff --git a/src/monitor/replication_state.h b/src/monitor/replication_state.h index 040aeec8c..060281f04 100644 --- a/src/monitor/replication_state.h +++ b/src/monitor/replication_state.h @@ -40,7 +40,8 @@ typedef enum ReplicationState REPLICATION_STATE_FAST_FORWARD = 18, REPLICATION_STATE_JOIN_SECONDARY = 19, REPLICATION_STATE_DROPPED = 20, - REPLICATION_STATE_UNKNOWN = 21 + REPLICATION_STATE_ARCHIVING = 21, + REPLICATION_STATE_UNKNOWN = 22 } ReplicationState; diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index 276738592..e558871aa 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -66,8 +66,15 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- Expected result: empty. Every MonitorFSM[] rule currently has a matching --- KeeperFSM[] row for every current_state it can assign a transition from. +-- Expected result: the 8 rows from pos 367/396/397/398's own archiver- +-- related edges (reportedState/goalState = ARCHIVING) -- a real, currently +-- expected gap: the monitor side of the ARCHIVING state (Archiving & +-- Disaster Recovery design, milestone 2) landed first, on its own, with no +-- corresponding KeeperFSM[] rows yet (no service_archiver process exists +-- to report ARCHIVING or drive pg_receivewal at this milestone either) -- +-- see the Build order in ~/dev/temp/archiving-disaster-recovery.md. +-- Every other MonitorFSM[] rule still has a matching KeeperFSM[] row for +-- every current_state it can assign a transition from. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos From 095bbbc259e69d01f84c21cde1764ad3925ca38f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 02:14:50 +0200 Subject: [PATCH 03/55] pg_autoctl: add keeper-side ARCHIVING state (M2 continued) Adds the keeper-side counterpart to the monitor-side ARCHIVING FSM support (previous commit): KeeperFSM[] rows for WAIT_STANDBY->ARCHIVING/ARCHIVING->REPORT_LSN/REPORT_LSN->ARCHIVING, each dispatching to a new, archiver-specific transition function (fsm_init_archiver/fsm_archiver_report_lsn/fsm_archiver_follow_new_primary, fsm_transition.c) -- mirroring the existing NODE_KIND_CITUS_* pattern of adding separate functions per node kind rather than branching inside the shared ones (fsm_init_standby, keeper_update_pg_state, keeper_ensure_current_state, keeper_node_active_loop stay untouched). New service_archiver.c launches and tracks the one pg_receivewal child an ARCHIVING node keeps running against its group's current primary -- milestone 2's own "colocated fast path" scope (see the Build order in ~/dev/temp/archiving-disaster-recovery.md): pg_receivewal is a real, unmodified Postgres client talking straight to the real primary's own walsender, so no new wire protocol is needed here at all. Not yet wired into supervisor.c's Service/RestartPolicy machinery or a replication slot -- both noted as follow-ups in that file's own header comment. Also: ARCHIVING_STATE added to NodeState (state.h/state.c) and to nodestate_utils.c's nodestateConnectionType() switch (grouped with the other "Postgres known to be stopped" states, since an ARCHIVING row never has a postmaster of its own -- this switch has no default case by design, so a missing case here would have failed the build). Verified: full regress (20/20) + isolation (6/6) suites still pass, and the monitor/keeper reachability cross-check in keeper_fsm_edges.sql -- which the previous commit deliberately left showing 8 unresolved rows, documented as this milestone's own known gap -- now shows zero rows again, both directions, confirming the two sides agree. Live-checked via `pg_autoctl inspect fsm list --json`, which is also how keeper_fsm_edges.json was regenerated (pretty-printed to match the existing file's own review-friendly formatting, not the CLI's compact default). citus_indent and ci/banned.h.sh both pass (the latter caught a raw strerror()/fprintf(stderr) call in service_archiver.c's own exec- failure path, fixed to the project's own log_fatal(..., "%m") convention already used at the other execv() call sites in this codebase). --- src/bin/pg_autoctl/fsm.c | 30 +++ src/bin/pg_autoctl/fsm.h | 4 + src/bin/pg_autoctl/fsm_transition.c | 71 +++++++ src/bin/pg_autoctl/nodestate_utils.c | 3 + src/bin/pg_autoctl/service_archiver.c | 220 ++++++++++++++++++++++ src/bin/pg_autoctl/service_archiver.h | 23 +++ src/bin/pg_autoctl/state.c | 9 + src/bin/pg_autoctl/state.h | 1 + src/monitor/expected/keeper_fsm_edges.out | 38 ++-- src/monitor/keeper_fsm_edges.json | 12 ++ src/monitor/node_metadata.c | 1 + src/monitor/sql/keeper_fsm_edges.sql | 16 +- 12 files changed, 397 insertions(+), 31 deletions(-) create mode 100644 src/bin/pg_autoctl/service_archiver.c create mode 100644 src/bin/pg_autoctl/service_archiver.h diff --git a/src/bin/pg_autoctl/fsm.c b/src/bin/pg_autoctl/fsm.c index fb66b060d..45a5fd47b 100644 --- a/src/bin/pg_autoctl/fsm.c +++ b/src/bin/pg_autoctl/fsm.c @@ -824,6 +824,36 @@ KeeperFSMTransition KeeperFSM[] = { FSM_PHASE_INIT }, + /* + * Archiving & Disaster Recovery (milestone 2): the ARCHIVING mirror of + * the ordinary standby-init/failover-participation/rejoin rows just + * above and further below (SECONDARY/CATCHINGUP <-> REPORT_LSN) -- an + * ARCHIVING node is only ever assigned these three transitions by the + * monitor (MonitorFSM[] pos 367/396-398, group_state_machine.c), so + * NODE_KIND_ANY carries no ambiguity here despite being the same + * bitmask every ordinary row uses. + */ + { + WAIT_STANDBY_STATE, ARCHIVING_STATE, NODE_KIND_ANY, + "wait_standby to archiving", + &fsm_init_archiver, + FSM_PHASE_INIT + }, + + { + ARCHIVING_STATE, REPORT_LSN_STATE, NODE_KIND_ANY, + "archiving to report_lsn", + &fsm_archiver_report_lsn, + FSM_PHASE_FAILOVER + }, + + { + REPORT_LSN_STATE, ARCHIVING_STATE, NODE_KIND_ANY, + "report_lsn to archiving", + &fsm_archiver_follow_new_primary, + FSM_PHASE_FAILOVER + }, + { DEMOTED_STATE, CATCHINGUP_STATE, NODE_KIND_ANY, COMMENT_DEMOTED_TO_CATCHINGUP, diff --git a/src/bin/pg_autoctl/fsm.h b/src/bin/pg_autoctl/fsm.h index 2ab182fc1..c84f82bcd 100644 --- a/src/bin/pg_autoctl/fsm.h +++ b/src/bin/pg_autoctl/fsm.h @@ -95,6 +95,10 @@ bool fsm_prepare_cascade(Keeper *keeper); bool fsm_follow_new_primary(Keeper *keeper); bool fsm_cleanup_as_primary(Keeper *keeper); +bool fsm_init_archiver(Keeper *keeper); +bool fsm_archiver_report_lsn(Keeper *keeper); +bool fsm_archiver_follow_new_primary(Keeper *keeper); + bool fsm_init_from_standby(Keeper *keeper); bool fsm_drop_node(Keeper *keeper); diff --git a/src/bin/pg_autoctl/fsm_transition.c b/src/bin/pg_autoctl/fsm_transition.c index b0579adfa..2a7cd7d30 100644 --- a/src/bin/pg_autoctl/fsm_transition.c +++ b/src/bin/pg_autoctl/fsm_transition.c @@ -39,6 +39,7 @@ #include "parson.h" #include "pghba.h" #include "primary_standby.h" +#include "service_archiver.h" #include "state.h" #include "timeline_history.h" @@ -1696,3 +1697,73 @@ fsm_drop_node(Keeper *keeper) return unlink_file(config->pathnames.init); } + + +/* + * fsm_init_archiver starts pg_receivewal against the group's current + * primary. Reached from WAIT_STANDBY_STATE once the monitor has assigned + * ARCHIVING as the goal state instead of fsm_init_standby's own + * CATCHINGUP target -- the monitor makes that choice based on this node's + * own haspgdata = false row, see MonitorFSM[] pos 396-398 + * (group_state_machine.c). Unlike fsm_init_standby, there is no local + * Postgres instance to configure as a standby: pg_receivewal is a real, + * unmodified Postgres client that streams straight from the primary's own + * walsender, so no new wire protocol is involved on this node's side + * either (see archiving-disaster-recovery.md's own milestone 2(a) scope). + */ +bool +fsm_init_archiver(Keeper *keeper) +{ + NodeAddress primaryNode = { 0 }; + + /* get the primary node to stream WAL from */ + if (!keeper_get_primary(keeper, &primaryNode)) + { + log_error("Failed to initialize archiver for lack of a primary node, " + "see above for details"); + return false; + } + + return service_archiver_start_pgreceivewal(keeper, &primaryNode); +} + + +/* + * fsm_archiver_report_lsn stops pg_receivewal: the group's primary is + * presumed gone (an election is starting), so the upstream this archiver + * was streaming from is no longer trustworthy to keep querying. Mirrors + * fsm_report_lsn's own "disconnect from current source" half without any + * of its real-Postgres recovery-config/restart machinery, which doesn't + * apply here -- an ARCHIVING row has no PGDATA to reconfigure (see + * haspgdata's own design comment, pgautofailover.sql). + */ +bool +fsm_archiver_report_lsn(Keeper *keeper) +{ + return service_archiver_stop_pgreceivewal(); +} + + +/* + * fsm_archiver_follow_new_primary re-points pg_receivewal at the group's + * newly elected primary -- the archiver's own mirror of + * fsm_follow_new_primary, without that function's live-Postgres-standby + * machinery: pg_receivewal has no "replaying, not caught up yet" + * continuum to wait out (see haspgdata's own design comment), just a + * fresh connection to make. + */ +bool +fsm_archiver_follow_new_primary(Keeper *keeper) +{ + NodeAddress primaryNode = { 0 }; + + /* get the newly elected primary node to stream WAL from */ + if (!keeper_get_primary(keeper, &primaryNode)) + { + log_error("Failed to follow new primary for lack of a primary node, " + "see above for details"); + return false; + } + + return service_archiver_start_pgreceivewal(keeper, &primaryNode); +} diff --git a/src/bin/pg_autoctl/nodestate_utils.c b/src/bin/pg_autoctl/nodestate_utils.c index 114cc65d5..b0854fc42 100644 --- a/src/bin/pg_autoctl/nodestate_utils.c +++ b/src/bin/pg_autoctl/nodestate_utils.c @@ -458,6 +458,9 @@ nodestateConnectionType(CurrentNodeState *nodeState) case DEMOTE_TIMEOUT_STATE: case DRAINING_STATE: case MAINTENANCE_STATE: + + /* an ARCHIVING row has no PGDATA/postmaster of its own, ever */ + case ARCHIVING_STATE: { return "none"; } diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c new file mode 100644 index 000000000..8a6ea896a --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver.c @@ -0,0 +1,220 @@ +/* + * src/bin/pg_autoctl/service_archiver.c + * Archiving & Disaster Recovery: supervision of the pg_receivewal child + * process an ARCHIVING node keeps running against its group's current + * primary. + * + * Milestone 2's own scope, per the Build order in + * ~/dev/temp/archiving-disaster-recovery.md: the colocated fast path only. + * pg_receivewal is a real, unmodified Postgres client talking straight to + * the real primary's own walsender -- no new wire protocol needed here at + * all. This file only launches and tracks that one child process; it does + * not yet integrate with supervisor.c's Service/RestartPolicy machinery + * (a liveness check happens on each FSM tick instead, via + * service_archiver_pgreceivewal_is_running(), the same "is it alive" + * check the design doc's own ARCHIVING FSM section describes for + * keeper_ensure_current_state) -- and does not yet use a replication slot + * (WAL retention across a pg_receivewal restart is a follow-up). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "service_archiver.h" + +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "signals.h" + +/* + * One pg_receivewal child per archiver process, matching milestone 2's own + * single-membership scope (see this file's own comment) -- a future + * milestone generalizing to several (formation, group) memberships per + * archiver will need one pid per membership instead of this one global. + */ +static pid_t pgReceivewalPid = -1; + + +/* + * service_archiver_pgreceivewal_is_running returns true iff the tracked + * pg_receivewal child is still alive. waitpid(WNOHANG) both checks and + * reaps: called on every FSM tick, so a child that exited between ticks is + * reaped promptly rather than lingering as a zombie. + */ +bool +service_archiver_pgreceivewal_is_running(void) +{ + if (pgReceivewalPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(pgReceivewalPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == pgReceivewalPid) + { + log_warn("pg_receivewal (pid %d) exited with status %d", + pgReceivewalPid, status); + } + else + { + log_warn("Failed to check on pg_receivewal (pid %d): %m", + pgReceivewalPid); + } + + pgReceivewalPid = -1; + return false; +} + + +/* + * service_archiver_stop_pgreceivewal stops the tracked pg_receivewal child, + * if any. Idempotent: a no-op when nothing is tracked or the child has + * already exited on its own. + */ +bool +service_archiver_stop_pgreceivewal(void) +{ + if (!service_archiver_pgreceivewal_is_running()) + { + return true; + } + + log_info("Stopping pg_receivewal (pid %d)", pgReceivewalPid); + + if (kill(pgReceivewalPid, SIGTERM) != 0 && errno != ESRCH) + { + log_error("Failed to send SIGTERM to pg_receivewal (pid %d): %m", + pgReceivewalPid); + return false; + } + + int status = 0; + + if (waitpid(pgReceivewalPid, &status, 0) == -1 && errno != ECHILD) + { + log_error("Failed to wait for pg_receivewal (pid %d) to stop: %m", + pgReceivewalPid); + pgReceivewalPid = -1; + return false; + } + + pgReceivewalPid = -1; + return true; +} + + +/* + * service_archiver_start_pgreceivewal starts pg_receivewal against the + * given primary node, writing captured WAL into the archiver's own local + * storage directory (config->pgSetup.pgdata -- an ARCHIVING node's config + * reuses the same field an ordinary node uses for its real PGDATA, see + * this project's own cli_create_archiver, since it plays the same "this + * node's local root directory" role here without ever holding a real + * Postgres cluster). Idempotent: stops any previously-tracked child first, + * exactly like fsm_init_standby's own upstream reuse pattern. + */ +bool +service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) +{ + KeeperConfig *config = &(keeper->config); + + if (!service_archiver_stop_pgreceivewal()) + { + /* errors have already been logged */ + return false; + } + + char pgReceivewalPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, + "pg_receivewal", + pgReceivewalPath); + + if (!file_exists(pgReceivewalPath)) + { + log_error("Failed to find pg_receivewal at \"%s\"", pgReceivewalPath); + return false; + } + + /* + * Create-if-missing only -- never ensure_empty_dir(), which rmtree()s + * first: this directory holds already-captured WAL across restarts, + * the whole point of running an archiver. + */ + if (!directory_exists(config->pgSetup.pgdata) && + mkdir(config->pgSetup.pgdata, 0700) != 0) + { + log_error("Failed to create archiver WAL directory \"%s\": %m", + config->pgSetup.pgdata); + return false; + } + + /* + * A plain key/value conninfo string: trust/no-password authentication, + * matching every other pgaftest docker environment this milestone is + * validated against. A real deployment's --ssl/password handling is a + * follow-up, mirroring pg_basebackup()'s own PGPASSWORD-env dance + * (pgctl.c) once an archiver config carries a replication password. + */ + char primaryConnInfo[MAXCONNINFO] = { 0 }; + + sformat(primaryConnInfo, sizeof(primaryConnInfo), + "host=%s port=%d user=%s application_name=%s", + primaryNode->host, primaryNode->port, + PG_AUTOCTL_REPLICA_USERNAME, config->name); + + log_info("Starting pg_receivewal against %s:%d, writing to \"%s\"", + primaryNode->host, primaryNode->port, config->pgSetup.pgdata); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_receivewal: %m"); + return false; + } + + if (pid == 0) + { + /* child process: replace ourselves with pg_receivewal */ + char *args[8]; + int argsIndex = 0; + + args[argsIndex++] = pgReceivewalPath; + args[argsIndex++] = "-w"; + args[argsIndex++] = "-d"; + args[argsIndex++] = primaryConnInfo; + args[argsIndex++] = "-D"; + args[argsIndex++] = config->pgSetup.pgdata; + args[argsIndex++] = "--no-sync"; + args[argsIndex] = NULL; + + execv(pgReceivewalPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgReceivewalPath); + _exit(127); + } + + /* parent process: track the child, keep running our own loop */ + pgReceivewalPid = pid; + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver.h b/src/bin/pg_autoctl/service_archiver.h new file mode 100644 index 000000000..501f142f9 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_autoctl/service_archiver.h + * Archiving & Disaster Recovery: supervision of the pg_receivewal child + * process an ARCHIVING node keeps running against its group's current + * primary. See ~/dev/temp/archiving-disaster-recovery.md for the design + * this implements milestone 2 of. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_H +#define SERVICE_ARCHIVER_H + +#include "keeper.h" +#include "pgsql.h" + +bool service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode); +bool service_archiver_stop_pgreceivewal(void); +bool service_archiver_pgreceivewal_is_running(void); + +#endif /* SERVICE_ARCHIVER_H */ diff --git a/src/bin/pg_autoctl/state.c b/src/bin/pg_autoctl/state.c index c4df76266..4da841519 100644 --- a/src/bin/pg_autoctl/state.c +++ b/src/bin/pg_autoctl/state.c @@ -484,6 +484,11 @@ NodeStateToString(NodeState s) return "dropped"; } + case ARCHIVING_STATE: + { + return "archiving"; + } + case ANY_STATE: { return "#any state#"; @@ -592,6 +597,10 @@ NodeStateFromString(const char *str) { return DROPPED_STATE; } + else if (strcmp(str, "archiving") == 0) + { + return ARCHIVING_STATE; + } else { log_fatal("Failed to parse state string \"%s\"", str); diff --git a/src/bin/pg_autoctl/state.h b/src/bin/pg_autoctl/state.h index 92243347d..04dc02ffd 100644 --- a/src/bin/pg_autoctl/state.h +++ b/src/bin/pg_autoctl/state.h @@ -52,6 +52,7 @@ typedef enum FAST_FORWARD_STATE, JOIN_SECONDARY_STATE, DROPPED_STATE, + ARCHIVING_STATE, /* Allow some wildcard-matching transitions (from ANY state to) */ ANY_STATE = 128 diff --git a/src/monitor/expected/keeper_fsm_edges.out b/src/monitor/expected/keeper_fsm_edges.out index 3b908a2a9..20eebeb31 100644 --- a/src/monitor/expected/keeper_fsm_edges.out +++ b/src/monitor/expected/keeper_fsm_edges.out @@ -54,6 +54,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; apply_settings | demote_timeout apply_settings | demoted apply_settings | join_primary + archiving | report_lsn catchingup | single catchingup | demote_timeout catchingup | demoted @@ -126,6 +127,7 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; report_lsn | prepare_promotion report_lsn | fast_forward report_lsn | join_secondary + report_lsn | archiving secondary | single secondary | demote_timeout secondary | demoted @@ -154,7 +156,8 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; wait_primary | join_primary wait_primary | apply_settings wait_standby | catchingup -(108 rows) + wait_standby | archiving +(111 rows) -- Step 2a: monitor -> keeper direction -- every pgautofailover.dump_fsm_edges() -- edge the keeper_fsm_edges table above has no matching row for. A @@ -175,15 +178,13 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- Expected result: the 8 rows from pos 367/396/397/398's own archiver- --- related edges (reportedState/goalState = ARCHIVING) -- a real, currently --- expected gap: the monitor side of the ARCHIVING state (Archiving & --- Disaster Recovery design, milestone 2) landed first, on its own, with no --- corresponding KeeperFSM[] rows yet (no service_archiver process exists --- to report ARCHIVING or drive pg_receivewal at this milestone either) -- --- see the Build order in ~/dev/temp/archiving-disaster-recovery.md. --- Every other MonitorFSM[] rule still has a matching KeeperFSM[] row for --- every current_state it can assign a transition from. +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from +-- -- including the pos 367/396/397/398 archiver-related edges (Archiving & +-- Disaster Recovery design, milestone 2): KeeperFSM[]'s own +-- WAIT_STANDBY_STATE/ARCHIVING_STATE/REPORT_LSN_STATE rows +-- (fsm_init_archiver/fsm_archiver_report_lsn/fsm_archiver_follow_new_primary, +-- fsm.c/fsm_transition.c) close this milestone's own gap. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos @@ -198,17 +199,9 @@ SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.commen (e.pos, e.assigned_state, f.comment) ) ORDER BY e.pos, e.assigned_state, e.current_state NULLS FIRST; - rule | n | current_state | assigned_state | comment -------+---+---------------+----------------+------------------------------------------------------------------------------------------------------------------- - 367 | 1 | | report_lsn | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) - 367 | 1 | archiving | report_lsn | MS-failover fan-out: secondary/catchingup/archiving, not yet converged -> report_lsn (1 of 4) - 396 | 1 | | archiving | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving - 396 | 1 | wait_standby | archiving | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving - 397 | 1 | | archiving | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings - 397 | 1 | wait_standby | archiving | archiver mirror of pos 317: wait_standby (quorum member), primary converged primary -> archiving + apply_settings - 398 | 1 | | archiving | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving - 398 | 1 | wait_standby | archiving | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving -(8 rows) + rule | n | current_state | assigned_state | comment +------+---+---------------+----------------+--------- +(0 rows) -- Step 2b: keeper -> monitor direction, the reverse gap -- every keeper -- edge that no MonitorFSM[] row can ever produce (dump_fsm_edges() is the @@ -259,11 +252,12 @@ SELECT k.current_state, k.assigned_state report_lsn | prepare_promotion report_lsn | fast_forward report_lsn | join_secondary + report_lsn | archiving secondary | wait_standby secondary | maintenance secondary | wait_maintenance wait_primary | join_primary wait_primary | apply_settings -(23 rows) +(24 rows) DROP TABLE keeper_fsm_edges; diff --git a/src/monitor/keeper_fsm_edges.json b/src/monitor/keeper_fsm_edges.json index 20ab42ac9..9a81aea70 100644 --- a/src/monitor/keeper_fsm_edges.json +++ b/src/monitor/keeper_fsm_edges.json @@ -307,6 +307,18 @@ "current": "wait_standby", "assigned": "catchingup" }, + { + "current": "wait_standby", + "assigned": "archiving" + }, + { + "current": "archiving", + "assigned": "report_lsn" + }, + { + "current": "report_lsn", + "assigned": "archiving" + }, { "current": "demoted", "assigned": "catchingup" diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index d709eb2ec..e3f0988b4 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -179,6 +179,7 @@ TupleToAutoFailoverNode(TupleDesc tupleDescriptor, HeapTuple heapTuple) heap_getattr(heapTuple, Anum_pgautofailover_node_replication_stall_since, tupleDescriptor, &stallIsNull); + /* * haspgdata is looked up by name, not by the Anum_ constant every other * field here uses: this function is also called against a "RETURNING diff --git a/src/monitor/sql/keeper_fsm_edges.sql b/src/monitor/sql/keeper_fsm_edges.sql index e558871aa..d24d7cfb5 100644 --- a/src/monitor/sql/keeper_fsm_edges.sql +++ b/src/monitor/sql/keeper_fsm_edges.sql @@ -66,15 +66,13 @@ SELECT * FROM keeper_fsm_edges ORDER BY current_state, assigned_state; -- having to count its own detail rows by hand. NULLS FIRST puts each rule's -- summary row right before its own detail rows, as a header. -- --- Expected result: the 8 rows from pos 367/396/397/398's own archiver- --- related edges (reportedState/goalState = ARCHIVING) -- a real, currently --- expected gap: the monitor side of the ARCHIVING state (Archiving & --- Disaster Recovery design, milestone 2) landed first, on its own, with no --- corresponding KeeperFSM[] rows yet (no service_archiver process exists --- to report ARCHIVING or drive pg_receivewal at this milestone either) -- --- see the Build order in ~/dev/temp/archiving-disaster-recovery.md. --- Every other MonitorFSM[] rule still has a matching KeeperFSM[] row for --- every current_state it can assign a transition from. +-- Expected result: empty. Every MonitorFSM[] rule currently has a matching +-- KeeperFSM[] row for every current_state it can assign a transition from +-- -- including the pos 367/396/397/398 archiver-related edges (Archiving & +-- Disaster Recovery design, milestone 2): KeeperFSM[]'s own +-- WAIT_STANDBY_STATE/ARCHIVING_STATE/REPORT_LSN_STATE rows +-- (fsm_init_archiver/fsm_archiver_report_lsn/fsm_archiver_follow_new_primary, +-- fsm.c/fsm_transition.c) close this milestone's own gap. SELECT e.pos AS rule, count(*) AS n, e.current_state, e.assigned_state, f.comment FROM pgautofailover.dump_fsm_edges() e JOIN pgautofailover.fsm f ON f.pos = e.pos From eebc11782cc5dc6a9f8f59ef1b8f4a32c19dc5a5 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 02:39:02 +0200 Subject: [PATCH 04/55] pg_autoctl: add create archiver CLI + service_archiver reporting loop (M2 continued) Adds NODE_KIND_ARCHIVER as a real PgInstanceKind (pgsetup.h/pgsetup.c, name<->enum both directions), two monitor RPC wrapper functions (monitor_register_archiver/monitor_archiver_add_formation, monitor.c, calling M1's own register_archiver()/archiver_add_formation() plpgsql functions -- not the ordinary C register_node() RPC, since an Archiver is a process identity, not a (formation, group) membership by itself), and `pg_autoctl create archiver` (cli_create_node.c): a deliberately minimal, hand-rolled getopts (not the shared cli_create_node_getopts every ordinary node kind uses, since that parser's defaults assume a real PostgresSetup an archiver never has) that registers with the monitor, writes a KeeperConfig + initial state file (WAIT_STANDBY_STATE, mirroring archiver_add_formation()'s own starting point), and with --run hands off to service_archiver_loop() (previous commit). Verified live against the real monitor RPC layer (not just static review): registration, formation attachment, and config/state file writing all confirmed end-to-end against a real running monitor extension instance, including two real bugs the empirical run caught that manual review missed -- - config_find_pg_ctl() unconditionally clears pgSetup.pg_ctl before searching, silently discarding a caller-supplied --pgctl value; fixed by only calling it when pg_ctl is still empty (also added the missing --pgctl flag itself -- this dev machine has two pg_ctl on PATH and needs it to disambiguate, a real scenario, not a test-only one). - keeper_config_write_file() requires pg_autoctl.role set (validated against KEEPER_ROLE, not defaulted on write) -- config.role was never populated, since this path deliberately skips keeper_config_init()'s ordinary defaults (Postgres-specific probing that doesn't apply here). `--run`'s actual pg_receivewal launch is still unverified against a real streaming primary -- needs a real replication-configured Postgres pair, which is exactly what the next step (pgaftest wiring) provides. citus_indent and ci/banned.h.sh both pass. --- src/bin/common/pgsetup.c | 10 +- src/bin/common/pgsetup.h | 1 + src/bin/pg_autoctl/cli_common.h | 1 + src/bin/pg_autoctl/cli_create_node.c | 322 ++++++++++++++++++++++++++ src/bin/pg_autoctl/cli_root.c | 1 + src/bin/pg_autoctl/monitor.c | 94 ++++++++ src/bin/pg_autoctl/monitor.h | 4 + src/bin/pg_autoctl/service_archiver.c | 76 ++++++ src/bin/pg_autoctl/service_archiver.h | 2 + 9 files changed, 509 insertions(+), 2 deletions(-) diff --git a/src/bin/common/pgsetup.c b/src/bin/common/pgsetup.c index 175f02072..486e94c44 100644 --- a/src/bin/common/pgsetup.c +++ b/src/bin/common/pgsetup.c @@ -1496,10 +1496,11 @@ nodeKindFromString(const char *nodeKind) NODE_KIND_UNKNOWN, NODE_KIND_STANDALONE, NODE_KIND_CITUS_COORDINATOR, - NODE_KIND_CITUS_WORKER + NODE_KIND_CITUS_WORKER, + NODE_KIND_ARCHIVER }; char *kindList[] = { - "", "unknown", "standalone", "coordinator", "worker", NULL + "", "unknown", "standalone", "coordinator", "worker", "archiver", NULL }; for (int listIndex = 0; kindList[listIndex] != NULL; listIndex++) @@ -1546,6 +1547,11 @@ nodeKindToString(PgInstanceKind kind) return "worker"; } + case NODE_KIND_ARCHIVER: + { + return "archiver"; + } + default: { log_fatal("nodeKindToString: unknown node kind %d", kind); diff --git a/src/bin/common/pgsetup.h b/src/bin/common/pgsetup.h index f8afcfa18..242946f7f 100644 --- a/src/bin/common/pgsetup.h +++ b/src/bin/common/pgsetup.h @@ -131,6 +131,7 @@ typedef enum PgInstanceKind NODE_KIND_STANDALONE = 1, NODE_KIND_CITUS_COORDINATOR = 2, NODE_KIND_CITUS_WORKER = 4, + NODE_KIND_ARCHIVER = 8, NODE_KIND_ANY = 0xff } PgInstanceKind; diff --git a/src/bin/pg_autoctl/cli_common.h b/src/bin/pg_autoctl/cli_common.h index 45bfca490..348b17cb1 100644 --- a/src/bin/pg_autoctl/cli_common.h +++ b/src/bin/pg_autoctl/cli_common.h @@ -92,6 +92,7 @@ extern CommandLine create_monitor_command; extern CommandLine create_postgres_command; extern CommandLine create_coordinator_command; extern CommandLine create_worker_command; +extern CommandLine create_archiver_command; extern CommandLine activate_node_command; /* cli_drop_node.c */ diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 39b3f4aab..0578a1f21 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -36,10 +36,12 @@ #include "pghba.h" #include "pidfile.h" #include "primary_standby.h" +#include "service_archiver.h" #include "service_keeper.h" #include "service_keeper_init.h" #include "service_monitor.h" #include "service_monitor_init.h" +#include "signals.h" #include "string_utils.h" /* @@ -62,6 +64,9 @@ static void cli_activate_node(int argc, char **argv); static int cli_create_monitor_getopts(int argc, char **argv); static void cli_create_monitor(int argc, char **argv); +static int cli_create_archiver_getopts(int argc, char **argv); +static void cli_create_archiver(int argc, char **argv); + static void check_hostname(const char *hostname); CommandLine create_monitor_command = @@ -1294,6 +1299,323 @@ cli_create_monitor(int argc, char **argv) } +/* + * cli_create_archiver_getopts parses `pg_autoctl create archiver`'s own + * command line options -- deliberately not cli_create_node_getopts (used by + * every ordinary node kind): that shared parser and the KeeperConfig + * defaults it applies assume a real PostgresSetup (pgport, pghost, a real + * PGDATA to validate), none of which apply to an archiver (see haspgdata's + * own design comment, pgautofailover.sql). Milestone 2's own minimal flag + * set, matching the design doc's own Quickstart: --pgdata --monitor + * --hostname --name --formation --run. + */ +static int +cli_create_archiver_getopts(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0, errors = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "pgctl", required_argument, NULL, 'C' }, + { "monitor", required_argument, NULL, 'm' }, + { "hostname", required_argument, NULL, 'n' }, + { "name", required_argument, NULL, 'a' }, + { "formation", required_argument, NULL, 'f' }, + { "run", no_argument, NULL, 'x' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "D:C:m:n:a:f:xVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'C': + { + strlcpy(options.pgSetup.pg_ctl, optarg, MAXPGPATH); + log_trace("--pgctl %s", options.pgSetup.pg_ctl); + break; + } + + case 'm': + { + if (!validate_connection_string(optarg)) + { + log_fatal("Failed to parse --monitor connection string, " + "see above for details."); + exit(EXIT_CODE_BAD_ARGS); + } + strlcpy(options.monitor_pguri, optarg, MAXCONNINFO); + log_trace("--monitor %s", options.monitor_pguri); + break; + } + + case 'n': + { + strlcpy(options.hostname, optarg, _POSIX_HOST_NAME_MAX); + log_trace("--hostname %s", options.hostname); + break; + } + + case 'a': + { + strlcpy(options.name, optarg, _POSIX_HOST_NAME_MAX); + log_trace("--name %s", options.name); + break; + } + + case 'f': + { + strlcpy(options.formation, optarg, NAMEDATALEN); + log_trace("--formation %s", options.formation); + break; + } + + case 'x': + { + createAndRun = true; + log_trace("--run"); + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + exit(EXIT_CODE_QUIT); + } + + case 'v': + { + log_set_level(LOG_INFO); + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + } + + default: + { + ++errors; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(options.pgSetup.pgdata)) + { + log_fatal("Failed to get value for --pgdata"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(options.monitor_pguri)) + { + log_fatal("Failed to get value for --monitor"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(options.formation)) + { + strlcpy(options.formation, "default", NAMEDATALEN); + } + + options.pgSetup.pgKind = NODE_KIND_ARCHIVER; + strlcpy(options.nodeKind, "archiver", NAMEDATALEN); + + keeperOptions = options; + + return optind; +} + + +/* + * cli_create_archiver implements `pg_autoctl create archiver`: registers a + * new Archiver identity and attaches it to a formation via M1's own + * register_archiver()/archiver_add_formation() plpgsql functions (not the + * ordinary C register_node() RPC every other node kind goes through -- an + * Archiver is a process identity, not a (formation, group) membership by + * itself, see pgautofailover.sql's own comment on that function), writes a + * KeeperConfig + initial state file, and with --run hands off to + * service_archiver_loop() (service_archiver.c) -- deliberately not + * service_keeper_init()/keeper_node_active_loop(), which assume a real + * Postgres instance an ARCHIVING node never has. + */ +static void +cli_create_archiver(int argc, char **argv) +{ + pid_t pid = 0; + Keeper keeper = { 0 }; + KeeperConfig *config = &(keeper.config); + + keeper.config = keeperOptions; + + if (!check_or_discover_hostname(config)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!keeper_config_set_pathnames_from_pgdata(&config->pathnames, + config->pgSetup.pgdata)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (read_pidfile(config->pathnames.pid, &pid)) + { + log_fatal("pg_autoctl is already running with pid %d", pid); + exit(EXIT_CODE_BAD_STATE); + } + + if (IS_EMPTY_STRING_BUFFER(config->pgSetup.pg_ctl) && + !config_find_pg_ctl(&(config->pgSetup))) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!directory_exists(config->pgSetup.pgdata)) + { + if (pg_mkdir_p(config->pgSetup.pgdata, 0700) != 0) + { + log_fatal("Failed to create archiver directory \"%s\": %m", + config->pgSetup.pgdata); + exit(EXIT_CODE_BAD_ARGS); + } + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, config->monitor_pguri)) + { + /* errors have already been logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + keeper.monitor = monitor; + + int64_t archiverId = 0; + int64_t archiverNodeId = 0; + + char *archiverName = + IS_EMPTY_STRING_BUFFER(config->name) ? config->hostname : config->name; + + if (!monitor_register_archiver(&monitor, archiverName, config->hostname, + &archiverId)) + { + log_fatal("Failed to register archiver \"%s\" on the monitor, " + "see above for details", archiverName); + exit(EXIT_CODE_MONITOR); + } + + if (!monitor_archiver_add_formation(&monitor, archiverId, + config->formation, &archiverNodeId)) + { + log_fatal("Failed to attach archiver \"%s\" to formation \"%s\", " + "see above for details", archiverName, config->formation); + exit(EXIT_CODE_MONITOR); + } + + log_info("Registered archiver \"%s\" (id %" PRId64 ") for formation " + "\"%s\", ARCHIVING node id %" + PRId64, + archiverName, archiverId, config->formation, archiverNodeId); + + strlcpy(config->role, KEEPER_ROLE, sizeof(config->role)); + config->groupId = 0; + config->network_partition_timeout = NETWORK_PARTITION_TIMEOUT; + config->listen_notifications_timeout = PG_AUTOCTL_LISTEN_NOTIFICATIONS_TIMEOUT; + + if (!keeper_config_write_file(config)) + { + log_fatal("Failed to write archiver configuration file \"%s\", " + "see above for details", config->pathnames.config); + exit(EXIT_CODE_BAD_CONFIG); + } + + /* + * The ARCHIVING node row starts at (goalstate, reportedstate) = + * (wait_standby, wait_standby) -- see archiver_add_formation()'s own + * comment, pgautofailover.sql -- so our own local state mirrors that + * starting point exactly, same as an ordinary node's INIT_STATE. + */ + keeper_state_init(&(keeper.state)); + keeper.state.current_node_id = archiverNodeId; + keeper.state.current_group = 0; + keeper.state.current_role = WAIT_STANDBY_STATE; + keeper.state.assigned_role = WAIT_STANDBY_STATE; + + if (!keeper_store_state(&keeper)) + { + log_fatal("Failed to write archiver state file \"%s\", " + "see above for details", config->pathnames.state); + exit(EXIT_CODE_BAD_STATE); + } + + if (createAndRun) + { + if (!create_pidfile(config->pathnames.pid, getpid())) + { + log_fatal("Failed to write archiver pid file \"%s\"", + config->pathnames.pid); + exit(EXIT_CODE_BAD_STATE); + } + + (void) set_signal_handlers(false); + + if (!service_archiver_loop(&keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + } +} + + +CommandLine create_archiver_command = + make_command( + "archiver", + "Initialize a pg_auto_failover archiver node", + " [ --pgdata --pgctl --monitor --hostname --name --formation ] ", + " --pgdata path to the archiver's local data/cache directory\n" + " --pgctl path to pg_ctl (used to locate pg_receivewal)\n" + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --hostname hostname by which the archiver is reachable\n" + " --name archiver name (default: derived from hostname)\n" + " --formation formation to attach to (default: \"default\")\n" + " --run create node then run pg_autoctl service\n", + cli_create_archiver_getopts, + cli_create_archiver); + + /* * check_or_discover_hostname checks given --hostname or attempt to discover a * suitable default value for the current node when it's not been provided on diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 211c64aab..51f0ef3a7 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -31,6 +31,7 @@ CommandLine *create_subcommands[] = { &create_postgres_command, &create_coordinator_command, &create_worker_command, + &create_archiver_command, &create_formation_command, NULL }; diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index f24c34643..e0adca208 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -868,6 +868,100 @@ monitor_get_most_advanced_standby(Monitor *monitor, * The node ID and group ID selected by the monitor, as well as the goal * state, are set in assignedState, which must not be NULL. */ + + +/* + * monitor_register_archiver calls pgautofailover.register_archiver() on the + * monitor -- the Archiving & Disaster Recovery schema's own registration + * entry point (see ~/dev/temp/archiving-disaster-recovery.md), a plain + * plpgsql function rather than the C register_node() RPC every ordinary + * node kind goes through: an Archiver is a process identity, not a + * (formation, group) membership by itself (see that function's own comment, + * pgautofailover.sql). + */ +bool +monitor_register_archiver(Monitor *monitor, char *name, char *hostname, + int64_t *archiverId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + const char *sql = + "SELECT * FROM pgautofailover.register_archiver($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, TEXTOID }; + const char *paramValues[2] = { name, hostname }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to register archiver \"%s\" on the monitor", + name); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to register archiver \"%s\" on the monitor " + "because it returned an unexpected result, " + "see previous lines for details", name); + return false; + } + + *archiverId = context.bigint; + + return true; +} + + +/* + * monitor_archiver_add_formation calls pgautofailover.archiver_add_formation() + * on the monitor, attaching an already-registered archiver to every group of + * the given formation. Returns the nodeid of the ARCHIVING membership row + * created for group 0 -- the only group this milestone's own single-group + * scope (the "budget setup") ever attaches to; a Citus formation with more + * than one group would need every returned nodeid, not just the first. + */ +bool +monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, + char *formation, int64_t *archiverNodeId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + const char *sql = + "SELECT * FROM pgautofailover.archiver_add_formation($1, $2) LIMIT 1"; + int paramCount = 2; + Oid paramTypes[2] = { INT8OID, TEXTOID }; + IntString archiverIdString = intToString(archiverId); + const char *paramValues[2] = { archiverIdString.strValue, formation }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to attach archiver %" PRId64 " to formation \"%s\" " + "on the monitor", archiverId, + formation); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to attach archiver %" PRId64 " to formation \"%s\" " + "on the monitor because it returned an unexpected result, " + "see previous lines for details", + archiverId, formation); + return false; + } + + *archiverNodeId = context.bigint; + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 3b0fbe770..963dbcbb1 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -154,6 +154,10 @@ bool monitor_print_other_nodes_as_json(Monitor *monitor, bool monitor_get_primary(Monitor *monitor, char *formation, int groupId, NodeAddress *node); +bool monitor_register_archiver(Monitor *monitor, char *name, char *hostname, + int64_t *archiverId); +bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, + char *formation, int64_t *archiverNodeId); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index 8a6ea896a..9328b2d20 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -32,7 +32,9 @@ #include "defaults.h" #include "file_utils.h" +#include "fsm.h" #include "log.h" +#include "monitor.h" #include "signals.h" /* @@ -218,3 +220,77 @@ service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) return true; } + + +/* + * service_archiver_loop is the archiver's own node_active() reporting loop + * -- deliberately not keeper_node_active_loop (service_keeper.c): that + * function's own per-tick keeper_update_pg_state()/keeper_ensure_current_ + * state() calls assume a real Postgres instance with a real PGDATA to + * inspect, which an ARCHIVING node never has (see haspgdata's own design + * comment). This loop reuses everything that IS kind-agnostic -- + * keeper_load_state()/keeper_store_state(), keeper_node_active() (the + * monitor RPC wrapper itself only ever reads Keeper's in-memory fields, + * never touches real Postgres), and keeper_fsm_reach_assigned_state() + * dispatching through the very same KeeperFSM[] table -- while replacing + * the two Postgres-specific calls with nothing at all: an ARCHIVING row's + * only "is it running" check is service_archiver_pgreceivewal_is_running(), + * consulted by the FSM transition functions themselves + * (fsm_init_archiver et al., fsm_transition.c), not by this loop. + * + * Milestone 2's own single-membership scope (see this file's own header + * comment): one archiver, one (formation, group) row, reported here + * directly rather than iterating a list the monitor refreshes. + */ +bool +service_archiver_loop(Keeper *keeper) +{ + KeeperStateData *keeperState = &(keeper->state); + + log_info("pg_autoctl archiver service is starting"); + + while (!asked_to_stop && !asked_to_stop_fast && !asked_to_quit) + { + MonitorAssignedState assignedState = { 0 }; + + if (!keeper_load_state(keeper)) + { + log_error("Failed to read archiver state file, retrying..."); + } + else if (keeper_node_active(keeper, false, &assignedState)) + { + keeperState->assigned_role = assignedState.state; + + if (keeperState->current_role != keeperState->assigned_role) + { + if (keeper_fsm_reach_assigned_state(keeper)) + { + (void) keeper_store_state(keeper); + } + else + { + log_error("Failed to reach assigned state \"%s\", " + "retrying...", + NodeStateToString(keeperState->assigned_role)); + } + } + } + else + { + log_warn("Failed to contact the monitor, retrying..."); + } + + if (asked_to_stop || asked_to_stop_fast || asked_to_quit) + { + break; + } + + sleep(PG_AUTOCTL_KEEPER_SLEEP_TIME); + } + + (void) service_archiver_stop_pgreceivewal(); + + log_info("pg_autoctl archiver service is stopping"); + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver.h b/src/bin/pg_autoctl/service_archiver.h index 501f142f9..860eccc2d 100644 --- a/src/bin/pg_autoctl/service_archiver.h +++ b/src/bin/pg_autoctl/service_archiver.h @@ -20,4 +20,6 @@ bool service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNod bool service_archiver_stop_pgreceivewal(void); bool service_archiver_pgreceivewal_is_running(void); +bool service_archiver_loop(Keeper *keeper); + #endif /* SERVICE_ARCHIVER_H */ From 6204a213ea091f438cfc8852f0e6ef9cf3c954fb Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 13:23:32 +0200 Subject: [PATCH 05/55] monitor: add SECURITY DEFINER to get_latest_basebackup() autoctl_node was only ever granted EXECUTE on the function, never SELECT on pgautofailover.basebackup itself, matching every other autoctl_node- callable helper that reads a table it has no direct grant on (e.g. archiver_add_formation) -- get_latest_basebackup was the odd one out. Found via a real end-to-end test of `pg_autoctl archiver serve` against a live monitor: calling it as autoctl_node failed with "permission denied for table basebackup". --- src/monitor/pgautofailover--2.2--2.3.sql | 8 ++++++-- src/monitor/pgautofailover.sql | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index b5f595e8d..fc1f5948b 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -1758,9 +1758,13 @@ grant execute on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) to autoctl_node; --- filters status = 'complete' only +-- filters status = 'complete' only. SECURITY DEFINER matches every other +-- autoctl_node-callable helper reading a table that role has no direct +-- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only +-- ever granted EXECUTE on the function, never SELECT on pgautofailover. +-- basebackup itself. CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) - RETURNS pgautofailover.basebackup LANGUAGE sql STABLE + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup b WHERE b.formationid = get_latest_basebackup.formationid diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index 766b88e4a..abd19060b 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -2340,9 +2340,13 @@ grant execute on function pgautofailover.report_basebackup_remote_deleted(bigint,bigint) to autoctl_node; --- filters status = 'complete' only +-- filters status = 'complete' only. SECURITY DEFINER matches every other +-- autoctl_node-callable helper reading a table that role has no direct +-- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only +-- ever granted EXECUTE on the function, never SELECT on pgautofailover. +-- basebackup itself. CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) - RETURNS pgautofailover.basebackup LANGUAGE sql STABLE + RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup b WHERE b.formationid = get_latest_basebackup.formationid From e28439c60b151e92182964c5d83faf8f9368d50e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 13:23:53 +0200 Subject: [PATCH 06/55] pg_walsender: new standalone replication-protocol server (M2 continued) The archiver's serving half: a standalone binary (no pg_autoctl/*.c dependency, only src/bin/common/ and src/bin/lib/log/) that speaks enough of the real Postgres replication protocol to serve IDENTIFY_SYSTEM, SHOW, BASE_BACKUP, and a non-standard FETCH_FILE side-channel, backed by an archiver's local WAL cache and base backups instead of a live postmaster. No frontend-linkable server-side protocol library exists anywhere in Postgres (confirmed against pqcomm.c/backend_startup.c/repl_gram.y/ walsender.c, all backend-only) -- this is a genuine reimplementation guided by that source, not a linking exercise. Two pieces are vendored near-verbatim since they're already frontend-safe: vendor/tar.c + pgtar.h (PostgreSQL's own ustar header/checksum logic, src/port/tar.c). Also ships fetch_client.c / `pg_walsender fetch-file`, the client side of the FETCH_FILE side-channel, for use as pg_autoctl's own restore_command. Verified against real, unmodified PostgreSQL client tools: - psql (replication=1): IDENTIFY_SYSTEM, SHOW wal_segment_size - pg_basebackup --format=plain -X none --no-manifest: fetched a real base backup byte-identical to the source, then booted a live Postgres instance from the result - pg_walsender fetch-file: fetched a full 16MB WAL segment byte- identical, plus clean error handling (missing file, path traversal, unknown route) See ~/dev/temp/archiving-disaster-recovery.md for the design this implements milestone 2 of. --- .gitignore | 1 + src/bin/Makefile | 20 +- src/bin/pg_walsender/Makefile | 62 +++ src/bin/pg_walsender/accept_loop.c | 314 ++++++++++++++ src/bin/pg_walsender/accept_loop.h | 29 ++ src/bin/pg_walsender/auth.c | 101 +++++ src/bin/pg_walsender/auth.h | 44 ++ src/bin/pg_walsender/cmd_base_backup.c | 427 +++++++++++++++++++ src/bin/pg_walsender/cmd_base_backup.h | 33 ++ src/bin/pg_walsender/cmd_fetch_file.c | 107 +++++ src/bin/pg_walsender/cmd_fetch_file.h | 36 ++ src/bin/pg_walsender/cmd_identify_system.c | 58 +++ src/bin/pg_walsender/cmd_identify_system.h | 19 + src/bin/pg_walsender/cmd_show.c | 53 +++ src/bin/pg_walsender/cmd_show.h | 17 + src/bin/pg_walsender/defaults.h | 33 ++ src/bin/pg_walsender/fetch_client.c | 262 ++++++++++++ src/bin/pg_walsender/fetch_client.h | 31 ++ src/bin/pg_walsender/framing.c | 456 +++++++++++++++++++++ src/bin/pg_walsender/framing.h | 79 ++++ src/bin/pg_walsender/main.c | 220 ++++++++++ src/bin/pg_walsender/repl_command.c | 115 ++++++ src/bin/pg_walsender/repl_command.h | 60 +++ src/bin/pg_walsender/routes.c | 233 +++++++++++ src/bin/pg_walsender/routes.h | 53 +++ src/bin/pg_walsender/startup.c | 140 +++++++ src/bin/pg_walsender/startup.h | 30 ++ src/bin/pg_walsender/tar_stream.c | 270 ++++++++++++ src/bin/pg_walsender/tar_stream.h | 48 +++ src/bin/pg_walsender/vendor/pgtar.h | 98 +++++ src/bin/pg_walsender/vendor/tar.c | 278 +++++++++++++ src/bin/pg_walsender/walsender.h | 44 ++ 32 files changed, 3764 insertions(+), 7 deletions(-) create mode 100644 src/bin/pg_walsender/Makefile create mode 100644 src/bin/pg_walsender/accept_loop.c create mode 100644 src/bin/pg_walsender/accept_loop.h create mode 100644 src/bin/pg_walsender/auth.c create mode 100644 src/bin/pg_walsender/auth.h create mode 100644 src/bin/pg_walsender/cmd_base_backup.c create mode 100644 src/bin/pg_walsender/cmd_base_backup.h create mode 100644 src/bin/pg_walsender/cmd_fetch_file.c create mode 100644 src/bin/pg_walsender/cmd_fetch_file.h create mode 100644 src/bin/pg_walsender/cmd_identify_system.c create mode 100644 src/bin/pg_walsender/cmd_identify_system.h create mode 100644 src/bin/pg_walsender/cmd_show.c create mode 100644 src/bin/pg_walsender/cmd_show.h create mode 100644 src/bin/pg_walsender/defaults.h create mode 100644 src/bin/pg_walsender/fetch_client.c create mode 100644 src/bin/pg_walsender/fetch_client.h create mode 100644 src/bin/pg_walsender/framing.c create mode 100644 src/bin/pg_walsender/framing.h create mode 100644 src/bin/pg_walsender/main.c create mode 100644 src/bin/pg_walsender/repl_command.c create mode 100644 src/bin/pg_walsender/repl_command.h create mode 100644 src/bin/pg_walsender/routes.c create mode 100644 src/bin/pg_walsender/routes.h create mode 100644 src/bin/pg_walsender/startup.c create mode 100644 src/bin/pg_walsender/startup.h create mode 100644 src/bin/pg_walsender/tar_stream.c create mode 100644 src/bin/pg_walsender/tar_stream.h create mode 100644 src/bin/pg_walsender/vendor/pgtar.h create mode 100644 src/bin/pg_walsender/vendor/tar.c create mode 100644 src/bin/pg_walsender/walsender.h diff --git a/.gitignore b/.gitignore index 83e10cec8..d260eb535 100644 --- a/.gitignore +++ b/.gitignore @@ -56,5 +56,6 @@ valgrind/ src/bin/pgaftest/test_spec_parse.tab.* src/bin/pgaftest/test_spec_parse.output src/bin/pgaftest/pgaftest +src/bin/pg_walsender/pg_walsender run-test.sh tests/tablespaces/__pycache__/ diff --git a/src/bin/Makefile b/src/bin/Makefile index 734eb561f..542b49701 100644 --- a/src/bin/Makefile +++ b/src/bin/Makefile @@ -3,12 +3,13 @@ COMMON_LIB = common/libpgaf_common.a -all: pg_autoctl pgaftest ; +all: pg_autoctl pgaftest pg_walsender ; -# Build the shared archive once, serially, before the two binaries run in -# parallel. Both pg_autoctl and pgaftest include Makefile.common which -# defines compile rules for common/*.c; without this serialisation a -# parallel make -j would race to write the same .o files simultaneously. +# Build the shared archive once, serially, before the binaries run in +# parallel. pg_autoctl, pgaftest, and pg_walsender all include +# Makefile.common which defines compile rules for common/*.c; without this +# serialisation a parallel make -j would race to write the same .o files +# simultaneously. $(COMMON_LIB): $(MAKE) -C common @@ -18,13 +19,18 @@ pg_autoctl: $(COMMON_LIB) pgaftest: $(COMMON_LIB) $(MAKE) -C pgaftest pgaftest +pg_walsender: $(COMMON_LIB) + $(MAKE) -C pg_walsender pg_walsender + clean: $(MAKE) -C common clean $(MAKE) -C pg_autoctl clean $(MAKE) -C pgaftest clean + $(MAKE) -C pg_walsender clean -install: pg_autoctl pgaftest +install: pg_autoctl pgaftest pg_walsender $(MAKE) -C pg_autoctl install $(MAKE) -C pgaftest install + $(MAKE) -C pg_walsender install -.PHONY: all pg_autoctl pgaftest install clean +.PHONY: all pg_autoctl pgaftest pg_walsender install clean diff --git a/src/bin/pg_walsender/Makefile b/src/bin/pg_walsender/Makefile new file mode 100644 index 000000000..669532c49 --- /dev/null +++ b/src/bin/pg_walsender/Makefile @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the PostgreSQL License. +# +# pg_walsender -- the archiver's own replication-protocol server. Standalone +# binary: does NOT link any pg_autoctl/*.c, only src/bin/common/ and +# src/bin/lib/log/, so it can be exec'd and tested independently of +# pg_autoctl (see ~/dev/temp/archiving-disaster-recovery.md and +# src/bin/pg_autoctl/service_archiver.c's own "colocated fast path" note). + +PG_WALSENDER = ./pg_walsender + +SRC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +# Must be set before include so that targets in Makefile.common don't +# become the default goal when included before the all: rule below. +.DEFAULT_GOAL := all + +include $(SRC_DIR)../common/Makefile.common + +override CFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)vendor + +# ----------------------------------------------------------------------- +# Sources that live in this directory +# ----------------------------------------------------------------------- +LOCAL_SRC = main.c accept_loop.c startup.c auth.c framing.c repl_command.c \ + routes.c cmd_identify_system.c cmd_show.c cmd_base_backup.c \ + tar_stream.c cmd_fetch_file.c fetch_client.c + +LOCAL_OBJS = $(patsubst %.c,%.o,$(LOCAL_SRC)) + +# ----------------------------------------------------------------------- +# Vendored PostgreSQL source (see vendor/tar.c's own header comment) -- +# compiled as vendor-%.o to keep it visually distinct from this project's +# own code. +# ----------------------------------------------------------------------- +VENDOR_SRC = tar.c +VENDOR_OBJS = $(patsubst %.c,vendor-%.o,$(VENDOR_SRC)) + +vendor-%.o: $(SRC_DIR)vendor/%.c + @if test ! -d $(DEPDIR); then mkdir -p $(DEPDIR); fi + $(CC) $(CFLAGS) -c -MMD -MP -MF$(DEPDIR)/vendor-$(*F).Po -o $@ $< + +OBJS = $(LOCAL_OBJS) $(VENDOR_OBJS) +OBJS += lib-log.o lib-snprintf.o lib-strerror.o +OBJS += $(COMMON_LIB) + +INCLUDES = $(wildcard $(SRC_DIR)*.h) + +all: $(COMMON_LIB) $(PG_WALSENDER) ; + +$(PG_WALSENDER): $(OBJS) $(INCLUDES) + $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LIBS) -o $@ + +clean: + rm -f $(OBJS) $(PG_WALSENDER) + rm -rf $(DEPDIR) + +install: $(PG_WALSENDER) + install -d $(DESTDIR)$(BINDIR) + install -m 0755 $(PG_WALSENDER) $(DESTDIR)$(BINDIR) + +.PHONY: all clean install diff --git a/src/bin/pg_walsender/accept_loop.c b/src/bin/pg_walsender/accept_loop.c new file mode 100644 index 000000000..4ffae004b --- /dev/null +++ b/src/bin/pg_walsender/accept_loop.c @@ -0,0 +1,314 @@ +/* + * src/bin/pg_walsender/accept_loop.c + * See accept_loop.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "accept_loop.h" +#include "auth.h" +#include "cmd_fetch_file.h" +#include "defaults.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "repl_command.h" +#include "routes.h" +#include "signals.h" +#include "startup.h" + +/* dbname prefix that routes a connection to the FETCH_FILE side-channel + * instead of the normal replication command loop -- see cmd_fetch_file.h */ +#define WS_FETCH_DBNAME_PREFIX "fetch/" + + +static int +create_listen_socket(int port) +{ + int sock = socket(AF_INET, SOCK_STREAM, 0); + + if (sock < 0) + { + log_error("Failed to create the listening socket: %m"); + return -1; + } + + int reuse = 1; + + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons(port); + + if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) != 0) + { + log_error("Failed to bind port %d: %m", port); + close(sock); + return -1; + } + + if (listen(sock, 64) != 0) + { + log_error("Failed to listen on port %d: %m", port); + close(sock); + return -1; + } + + return sock; +} + + +/* + * handle_connection runs the full lifecycle of one accepted connection: + * startup negotiation, routes-based auth, the initial handshake messages a + * real client expects (AuthenticationOk/ParameterStatus/BackendKeyData/ + * ReadyForQuery), and then the simple-query command loop replication + * connections use (see pgsql.c's own comment elsewhere in this project: + * "extended query protocol not supported in a replication connection"). + * Runs entirely inside the forked child; the caller _exit()s right after. + */ +static void +handle_connection(int clientSock, const WsServerConfig *config) +{ + WsStartupParams params; + + if (!ws_startup_negotiate(clientSock, ¶ms)) + { + close(clientSock); + return; + } + + WsRoute *routes = NULL; + int routeCount = 0; + + if (config->routesPath[0] != '\0') + { + if (!routes_load(config->routesPath, &routes, &routeCount)) + { + close(clientSock); + return; + } + } + + bool isFetchMode = (strncmp(params.database, WS_FETCH_DBNAME_PREFIX, + strlen(WS_FETCH_DBNAME_PREFIX)) == 0); + const char *routeKey = isFetchMode + ? params.database + strlen(WS_FETCH_DBNAME_PREFIX) + : params.database; + + const WsRoute *route = NULL; + + if (!ws_authenticate(clientSock, ¶ms, routeKey, routes, routeCount, &route)) + { + routes_free(routes); + close(clientSock); + return; + } + + char title[256]; + + snprintf(title, sizeof(title), "pg_autoctl: walsender %s%s", + isFetchMode ? "fetch " : "", route != NULL ? route->key : routeKey); + set_ps_title(title); + + if (isFetchMode) + { + cmd_fetch_file(clientSock, route); + routes_free(routes); + close(clientSock); + return; + } + + if (!ws_send_authentication_ok(clientSock) || + !ws_send_parameter_status(clientSock, "server_version", WS_SERVER_VERSION) || + !ws_send_parameter_status(clientSock, "client_encoding", "UTF8") || + !ws_send_parameter_status(clientSock, "server_encoding", "UTF8") || + !ws_send_parameter_status(clientSock, "integer_datetimes", "on") || + !ws_send_parameter_status(clientSock, "default_transaction_read_only", "off") || + !ws_send_backend_key_data(clientSock, getpid(), 0) || + !ws_send_ready_for_query(clientSock)) + { + routes_free(routes); + close(clientSock); + return; + } + + for (;;) + { + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(clientSock, &type, &payload, &payloadLen)) + { + free(payload); + break; + } + + if (type == 'X') /* Terminate */ + { + free(payload); + break; + } + + if (type != 'Q') /* Query -- the only message replication + * connections send commands through */ + { + ws_send_error_response(clientSock, "08P01", + "pg_walsender only accepts simple query " + "protocol messages"); + free(payload); + break; + } + + WsCommand cmd; + + if (!repl_command_parse(payload, &cmd)) + { + ws_send_error_response(clientSock, "42601", + "unrecognized replication command"); + } + else + { + ws_dispatch_command(clientSock, &cmd, route, params.database); + } + + free(payload); + + if (!ws_send_ready_for_query(clientSock)) + { + break; + } + } + + routes_free(routes); + close(clientSock); +} + + +bool +ws_accept_loop(const WsServerConfig *config) +{ + int listenSock = create_listen_socket(config->port); + + if (listenSock < 0) + { + return false; + } + + /* + * Auto-reap forked children: SIGCHLD/SIG_IGN is enough here since we + * never need a child's exit status, only that it not linger as a + * zombie -- simpler than an explicit waitpid(WNOHANG) loop. + */ + signal(SIGCHLD, SIG_IGN); + + set_signal_handlers(false); + + log_info("pg_walsender listening on port %d%s%s", + config->port, + config->routesPath[0] != '\0' ? ", routes " : " (no routes file)", + config->routesPath[0] != '\0' ? config->routesPath : ""); + + while (!asked_to_stop && !asked_to_stop_fast) + { + /* + * pqsignal() (signals.c, via postgres_fe.h) installs our handlers + * with SA_RESTART, so a blocking accept() is never interrupted by + * SIGTERM -- it would just keep sleeping through shutdown forever. + * Poll with a short timeout instead, so the loop condition above + * gets re-checked promptly after asked_to_stop is set. + */ + fd_set readSet; + + FD_ZERO(&readSet); + FD_SET(listenSock, &readSet); + + struct timeval timeout = { 1, 0 }; /* 1 second */ + + int selectRet = select(listenSock + 1, &readSet, NULL, NULL, &timeout); + + if (selectRet < 0) + { + if (errno == EINTR) + { + continue; + } + + log_error("select() failed: %m"); + continue; + } + + if (selectRet == 0) + { + /* timed out, no pending connection -- loop back to the + * asked_to_stop check above */ + continue; + } + + struct sockaddr_storage clientAddr; + socklen_t clientAddrLen = sizeof(clientAddr); + + int clientSock = accept(listenSock, + (struct sockaddr *) &clientAddr, + &clientAddrLen); + + if (clientSock < 0) + { + if (errno == EINTR) + { + continue; + } + + log_error("accept() failed: %m"); + continue; + } + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("fork() failed: %m"); + close(clientSock); + continue; + } + + if (pid == 0) + { + /* + * Child: no exec(), just call straight into the connection + * handler -- matches real Postgres's BackendMain() model (see + * the design doc's "Process model" section). + */ + close(listenSock); + handle_connection(clientSock, config); + _exit(0); + } + + /* parent: keep accepting; SIGCHLD/SIG_IGN reaps the child for us */ + close(clientSock); + } + + close(listenSock); + log_info("pg_walsender shutting down"); + + return true; +} diff --git a/src/bin/pg_walsender/accept_loop.h b/src/bin/pg_walsender/accept_loop.h new file mode 100644 index 000000000..c3e5b1f1f --- /dev/null +++ b/src/bin/pg_walsender/accept_loop.h @@ -0,0 +1,29 @@ +/* + * src/bin/pg_walsender/accept_loop.h + * The bare accept loop: socket()/bind()/listen()/accept(), fork() + * per connection with no exec() (matching real Postgres's + * BackendStartup()/BackendMain() model for cheap concurrency -- see + * the design doc's "Process model" section), each forked child running + * the full startup/auth/command-loop for exactly one connection. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_ACCEPT_LOOP_H +#define WS_ACCEPT_LOOP_H + +#include + +#include "postgres_fe.h" + +typedef struct WsServerConfig +{ + int port; + char routesPath[MAXPGPATH]; /* empty: no routing, manual-testing mode */ +} WsServerConfig; + +bool ws_accept_loop(const WsServerConfig *config); + +#endif /* WS_ACCEPT_LOOP_H */ diff --git a/src/bin/pg_walsender/auth.c b/src/bin/pg_walsender/auth.c new file mode 100644 index 000000000..a8cd74198 --- /dev/null +++ b/src/bin/pg_walsender/auth.c @@ -0,0 +1,101 @@ +/* + * src/bin/pg_walsender/auth.c + * See auth.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include + +#include "postgres_fe.h" + +#include "auth.h" +#include "defaults.h" +#include "framing.h" +#include "log.h" + + +static bool +ws_get_peer_ip(int sock, char *ipBuf, size_t ipBufSize) +{ + struct sockaddr_storage addr; + socklen_t addrLen = sizeof(addr); + + if (getpeername(sock, (struct sockaddr *) &addr, &addrLen) != 0) + { + log_error("Failed to getpeername() on the accepted connection: %m"); + return false; + } + + if (getnameinfo((struct sockaddr *) &addr, addrLen, + ipBuf, ipBufSize, NULL, 0, NI_NUMERICHOST) != 0) + { + log_error("Failed to resolve the peer's numeric address: %m"); + return false; + } + + return true; +} + + +bool +ws_authenticate(int sock, const WsStartupParams *params, const char *routeKey, + const WsRoute *routes, int routeCount, + const WsRoute **foundRoute) +{ + *foundRoute = NULL; + + if (strcmp(params->user, PG_AUTOCTL_REPLICA_USERNAME) != 0) + { + log_warn("Rejecting connection for unknown user \"%s\"", params->user); + ws_send_error_response(sock, "28000", + "role is not permitted to connect to pg_walsender"); + return false; + } + + if (routeCount == 0) + { + /* + * No routes file was supplied at all: manual/standalone testing + * mode, accept unconditionally now that the role matched. A real + * deployment always passes --routes (see main.c), so this branch + * never applies to a pg_autoctl-supervised pg_walsender. + */ + return true; + } + + const WsRoute *route = routes_find(routes, routeCount, routeKey); + + if (route == NULL) + { + log_warn("Rejecting connection for unknown route \"%s\"", routeKey); + ws_send_error_response(sock, "3D000", + "unknown formation/group requested as dbname"); + return false; + } + + char peerIP[NI_MAXHOST]; + + if (!ws_get_peer_ip(sock, peerIP, sizeof(peerIP))) + { + ws_send_error_response(sock, "08000", "failed to identify peer address"); + return false; + } + + if (!routes_host_allowed(route, peerIP)) + { + log_warn("Rejecting connection from %s: not in the allowed_hosts list " + "for route \"%s\"", peerIP, route->key); + ws_send_error_response(sock, "28000", + "no pg_hba.conf-equivalent entry for this host"); + return false; + } + + *foundRoute = route; + + return true; +} diff --git a/src/bin/pg_walsender/auth.h b/src/bin/pg_walsender/auth.h new file mode 100644 index 000000000..197ed0a4e --- /dev/null +++ b/src/bin/pg_walsender/auth.h @@ -0,0 +1,44 @@ +/* + * src/bin/pg_walsender/auth.h + * Trust-equivalent authentication, matching this project's existing + * convention: no password/SCRAM infrastructure exists anywhere in + * pg_auto_failover today (pghba.c installs plain "trust" entries for the + * replicator role, defaults.h's REPLICATION_PASSWORD_DEFAULT is NULL). + * pg_walsender mirrors that: accept iff the startup packet's user is the + * replicator role and, when the resolved route carries an allowed_hosts + * list, the peer address matches -- routes.c's allowed_hosts is + * effectively pg_walsender's own pg_hba.conf, since it has no PGDATA of + * its own to carry a real one. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_AUTH_H +#define WS_AUTH_H + +#include + +#include "walsender.h" +#include "routes.h" + +/* + * ws_authenticate checks params against the replicator username and, if + * routes/routeCount is non-empty, against the route matching routeKey and + * its allowed_hosts. routeKey is passed explicitly rather than read from + * params->database because the FETCH_FILE side-channel (see + * cmd_fetch_file.h) reuses this same auth path with a "fetch/" prefix + * stripped off the connection's actual dbname -- the caller (accept_loop.c) + * decides what routeKey means, this function only ever looks it up. On + * success returns true and sets *foundRoute (NULL when routes were not + * supplied at all -- a manual-testing convenience, see main.c's --routes + * option). On failure, an ErrorResponse has already been sent to sock; the + * caller only needs to close the connection. + */ +bool ws_authenticate(int sock, const WsStartupParams *params, + const char *routeKey, + const WsRoute *routes, int routeCount, + const WsRoute **foundRoute); + +#endif /* WS_AUTH_H */ diff --git a/src/bin/pg_walsender/cmd_base_backup.c b/src/bin/pg_walsender/cmd_base_backup.c new file mode 100644 index 000000000..a2851ea86 --- /dev/null +++ b/src/bin/pg_walsender/cmd_base_backup.c @@ -0,0 +1,427 @@ +/* + * src/bin/pg_walsender/cmd_base_backup.c + * See cmd_base_backup.h. + * + * Wire sequence for a successful, synchronous BASE_BACKUP (traced from + * basebackup_copy.c's bbsink_copystream_* callbacks and cross-checked + * against the exact PQgetResult() loop in pg_basebackup.c around its own + * "BASE_BACKUP" psprintf call -- both in + * /Users/dim/dev/PostgreSQL/postgresql): + * + * 1. RowDescription(recptr text, tli int8) + DataRow + CommandComplete + * "SELECT" -- the start position + * 2. RowDescription(spcoid oid, spclocation text, size int8) + + * DataRow(NULL, NULL, NULL) + CommandComplete "SELECT" -- one row, + * the base directory itself (path NULL means "not a tablespace") + * 3. CopyOutResponse(format 0, natts 0) + * 4. CopyData['n', "base.tar\0", "\0"] -- PqBackupMsg_NewArchive + * 5. CopyData['d', ] x N -- PqMsg_CopyData + * 6. CopyDone + * 7. RowDescription(recptr text, tli int8) + DataRow + CommandComplete + * "SELECT" -- the end position + * 8. CommandComplete "BASE_BACKUP" -- EndReplicationCommand + * + * pg_basebackup.c calls PQgetResult() exactly four times for this (steps + * 1, 2, [3-6 consumed internally by ReceiveArchiveStream], 7, 8), and + * explicitly checks step 8's PQresultStatus() == PGRES_COMMAND_OK. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "cmd_base_backup.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "tar_stream.h" + +typedef struct BaseBackupOptions +{ + char label[256]; + bool sendWal; + bool manifestRequested; + bool compressionRequested; + char target[64]; +} BaseBackupOptions; + + +/* + * scan_options tolerantly parses the BASE_BACKUP option list real + * pg_basebackup sends, e.g.: + * LABEL 'pg_basebackup base backup', CHECKPOINT 'fast', TARGET 'client' + * Options this MVP doesn't act on (PROGRESS, CHECKPOINT, WAIT, MAX_RATE, + * TABLESPACE_MAP, VERIFY_CHECKSUMS, MANIFEST_CHECKSUMS) are recognized and + * ignored rather than rejected -- only WAL/MANIFEST/COMPRESSION/a non- + * "client" TARGET actually change behavior (see cmd_base_backup()'s own + * validation right after calling this). + */ +static void +scan_options(const char *raw, BaseBackupOptions *opts) +{ + memset(opts, 0, sizeof(BaseBackupOptions)); + + const char *p = raw; + + while (*p) + { + while (isspace((unsigned char) *p) || *p == ',' || *p == '(' || *p == ')') + { + p++; + } + + if (*p == '\0') + { + break; + } + + const char *keyStart = p; + + while (*p && !isspace((unsigned char) *p) && *p != ',' && *p != ')') + { + p++; + } + + char key[64]; + size_t keyLen = Min((size_t) (p - keyStart), sizeof(key) - 1); + + memcpy(key, keyStart, keyLen); + key[keyLen] = '\0'; + + while (isspace((unsigned char) *p)) + { + p++; + } + + char value[512] = { 0 }; + + if (*p == '\'') + { + p++; + + char *out = value; + char *outEnd = value + sizeof(value) - 1; + + while (*p && !(*p == '\'' && p[1] != '\'')) + { + if (*p == '\'' && p[1] == '\'') + { + if (out < outEnd) + { + *out++ = '\''; + } + p += 2; + continue; + } + + if (out < outEnd) + { + *out++ = *p; + } + + p++; + } + + *out = '\0'; + + if (*p == '\'') + { + p++; + } + } + else if (*p && *p != ',' && *p != ')') + { + const char *valStart = p; + + while (*p && *p != ',' && *p != ')' && !isspace((unsigned char) *p)) + { + p++; + } + + size_t valLen = Min((size_t) (p - valStart), sizeof(value) - 1); + + memcpy(value, valStart, valLen); + value[valLen] = '\0'; + } + + if (strcasecmp(key, "LABEL") == 0) + { + strlcpy(opts->label, value, sizeof(opts->label)); + } + else if (strcasecmp(key, "WAL") == 0) + { + opts->sendWal = true; + } + else if (strcasecmp(key, "MANIFEST") == 0) + { + /* pg_basebackup only ever sends this key when it wants one + * ("yes"/"force-encode"); --no-manifest omits it entirely */ + opts->manifestRequested = true; + } + else if (strcasecmp(key, "TARGET") == 0) + { + strlcpy(opts->target, value, sizeof(opts->target)); + } + else if (strcasecmp(key, "COMPRESSION") == 0) + { + opts->compressionRequested = true; + } + + while (isspace((unsigned char) *p) || *p == ',') + { + p++; + } + } +} + + +/* + * read_backup_label extracts the "START WAL LOCATION" and "START TIMELINE" + * fields real pg_basebackup already wrote into basebackupDir/backup_label + * when the archiver originally took this backup (see cmd_base_backup.h's + * own header comment: do_pg_backup_start() is never called here, this file + * already exists on disk). Returns false (caller falls back to the + * route's own systemid/timeline, "0/0" for the LSN) if the file is + * missing or doesn't parse -- a base backup taken by a later milestone's + * own machinery is expected to always have one. + */ +static bool +read_backup_label(const char *basebackupDir, char *lsnOut, size_t lsnOutSize, + int *timelineOut) +{ + char path[MAXPGPATH]; + + snprintf(path, sizeof(path), "%s/backup_label", basebackupDir); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + bool foundLsn = false; + bool foundTimeline = false; + char *line = contents; + + while (line != NULL && *line != '\0') + { + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + const char *lsnPrefix = "START WAL LOCATION: "; + const char *tliPrefix = "START TIMELINE: "; + + if (strncmp(line, lsnPrefix, strlen(lsnPrefix)) == 0) + { + const char *value = line + strlen(lsnPrefix); + const char *end = value; + + while (*end && !isspace((unsigned char) *end)) + { + end++; + } + + size_t len = Min((size_t) (end - value), lsnOutSize - 1); + + memcpy(lsnOut, value, len); + lsnOut[len] = '\0'; + foundLsn = true; + } + else if (strncmp(line, tliPrefix, strlen(tliPrefix)) == 0) + { + *timelineOut = atoi(line + strlen(tliPrefix)); + foundTimeline = true; + } + + line = (nl != NULL) ? nl + 1 : NULL; + } + + free(contents); + + return foundLsn && foundTimeline; +} + + +typedef struct TarStreamCbContext +{ + int sock; + bool ok; +} TarStreamCbContext; + + +static bool +tar_chunk_cb(void *context, const char *data, size_t len) +{ + TarStreamCbContext *ctx = (TarStreamCbContext *) context; + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'd'); /* PqMsg_CopyData content tag */ + appendBinaryPQExpBuffer(buf, data, len); + + bool ok = !PQExpBufferBroken(buf) && + ws_send_copy_data(ctx->sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + if (!ok) + { + ctx->ok = false; + } + + return ok; +} + + +static bool +send_position_row(int sock, const char *lsn, const char *tli) +{ + WsColumn columns[] = { + { "recptr", WS_TEXTOID, -1 }, + { "tli", WS_INT8OID, 8 }, + }; + + const char *values[] = { lsn, tli }; + + return ws_send_row_description(sock, columns, 2) && + ws_send_data_row(sock, values, 2) && + ws_send_command_complete(sock, "SELECT"); +} + + +void +cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions) +{ + if (route == NULL || route->basebackupDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no base backup configured for this route " + "(the archiver hasn't taken one yet, or this " + "route wasn't given a basebackup directory)"); + return; + } + + BaseBackupOptions opts; + + scan_options(rawOptions, &opts); + + if (opts.sendWal) + { + ws_send_error_response(sock, "0A000", + "WAL-inclusive BASE_BACKUP is not supported " + "yet -- retry with pg_basebackup's -X none"); + return; + } + + if (opts.manifestRequested) + { + ws_send_error_response(sock, "0A000", + "backup manifests are not supported yet -- " + "retry with pg_basebackup's --no-manifest"); + return; + } + + if (opts.compressionRequested) + { + ws_send_error_response(sock, "0A000", + "server-side compression is not supported yet"); + return; + } + + if (opts.target[0] != '\0' && strcasecmp(opts.target, "client") != 0) + { + ws_send_error_response(sock, "0A000", + "only the default client-streaming BASE_BACKUP " + "target is supported"); + return; + } + + char lsn[32] = "0/0"; + int timeline = (route->timeline > 0) ? route->timeline : 1; + + if (!read_backup_label(route->basebackupDir, lsn, sizeof(lsn), &timeline)) + { + log_warn("No parseable backup_label under \"%s\"; reporting a " + "placeholder start position", route->basebackupDir); + } + + char tliStr[16]; + + snprintf(tliStr, sizeof(tliStr), "%d", timeline); + + if (!send_position_row(sock, lsn, tliStr)) + { + return; + } + + WsColumn tsColumns[] = { + { "spcoid", WS_INT4OID, 4 }, + { "spclocation", WS_TEXTOID, -1 }, + { "size", WS_INT8OID, 8 }, + }; + + const char *tsValues[] = { NULL, NULL, NULL }; + + if (!ws_send_row_description(sock, tsColumns, 3) || + !ws_send_data_row(sock, tsValues, 3) || + !ws_send_command_complete(sock, "SELECT")) + { + return; + } + + if (!ws_send_copy_out_response(sock, 0)) + { + return; + } + + { + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'n'); /* PqBackupMsg_NewArchive */ + appendBinaryPQExpBuffer(buf, "base.tar", strlen("base.tar") + 1); + appendBinaryPQExpBuffer(buf, "", 1); /* empty path: not a tablespace */ + + bool ok = !PQExpBufferBroken(buf) && + ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + if (!ok) + { + return; + } + } + + TarStreamCbContext ctx = { sock, true }; + + if (!tar_stream_directory(route->basebackupDir, tar_chunk_cb, &ctx) || !ctx.ok) + { + log_error("Failed to stream base backup tar contents from \"%s\"", + route->basebackupDir); + return; + } + + if (!ws_send_copy_done(sock)) + { + return; + } + + if (!send_position_row(sock, lsn, tliStr)) + { + return; + } + + ws_send_command_complete(sock, "BASE_BACKUP"); +} diff --git a/src/bin/pg_walsender/cmd_base_backup.h b/src/bin/pg_walsender/cmd_base_backup.h new file mode 100644 index 000000000..71dcce70b --- /dev/null +++ b/src/bin/pg_walsender/cmd_base_backup.h @@ -0,0 +1,33 @@ +/* + * src/bin/pg_walsender/cmd_base_backup.h + * BASE_BACKUP: streams route->basebackupDir as a ustar archive over the + * real multiplexed-COPY-stream wire format modern (>= 15) pg_basebackup + * clients expect (traced from + * /Users/dim/dev/PostgreSQL/postgresql's src/backend/backup/ + * basebackup_copy.c and src/bin/pg_basebackup/pg_basebackup.c -- see + * this file's own .c for the exact message sequence, with citations). + * + * MVP scope: a single archive (the base directory itself, no separate + * tablespaces), no server-side compression, no backup manifest, no + * WAL-inclusive backup (`-X none` on the client side) -- each rejected + * up front with a clean ErrorResponse rather than silently ignored. + * do_pg_backup_start()/do_pg_backup_stop() (live-instance, backend-only) + * are never called: the archiver's basebackupDir is already a complete, + * at-rest backup (produced by a real pg_basebackup run against a live + * server -- the "Base backup generation" milestone, not yet + * implemented), so the start/end LSN this command reports comes from + * that backup's own backup_label file, not a live checkpoint. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_BASE_BACKUP_H +#define WS_CMD_BASE_BACKUP_H + +#include "routes.h" + +void cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions); + +#endif /* WS_CMD_BASE_BACKUP_H */ diff --git a/src/bin/pg_walsender/cmd_fetch_file.c b/src/bin/pg_walsender/cmd_fetch_file.c new file mode 100644 index 000000000..f7f33a760 --- /dev/null +++ b/src/bin/pg_walsender/cmd_fetch_file.c @@ -0,0 +1,107 @@ +/* + * src/bin/pg_walsender/cmd_fetch_file.c + * See cmd_fetch_file.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_fetch_file.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + +#define WS_FETCH_FILENAME_MAX 256 + + +/* + * filename_is_safe rejects anything that isn't a bare filename: no path + * separators, no leading dot (rules out "." / ".." / hidden files), not + * empty. WAL segment names and ".history" files are both plain + * [0-9A-F.history]-shaped basenames, never nested paths, so this is not a + * meaningful restriction for real callers -- only for a hostile one trying + * to walk out of walcacheDir. + */ +static bool +filename_is_safe(const char *filename) +{ + if (filename[0] == '\0' || filename[0] == '.') + { + return false; + } + + if (strchr(filename, '/') != NULL || strchr(filename, '\\') != NULL) + { + return false; + } + + return true; +} + + +void +cmd_fetch_file(int sock, const WsRoute *route) +{ + if (!ws_send_authentication_ok(sock)) + { + return; + } + + char filename[WS_FETCH_FILENAME_MAX]; + + if (!ws_read_line(sock, filename, sizeof(filename))) + { + ws_send_error_response(sock, "08P01", + "expected a single filename line after " + "authentication"); + return; + } + + if (!filename_is_safe(filename)) + { + log_warn("Rejecting FETCH_FILE request for unsafe filename \"%s\"", + filename); + ws_send_error_response(sock, "22023", "invalid filename"); + return; + } + + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + char path[MAXPGPATH]; + + snprintf(path, sizeof(path), "%s/%s", route->walcacheDir, filename); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + log_info("FETCH_FILE: \"%s\" not found under \"%s\"", + filename, route->walcacheDir); + ws_send_error_response(sock, "58P01", "requested file not found"); + return; + } + + if (!ws_send_copy_data(sock, contents, (int32_t) fileSize)) + { + log_error("Failed to send \"%s\" (%ld bytes) to a FETCH_FILE client", + filename, fileSize); + } + else + { + log_info("FETCH_FILE: served \"%s\" (%ld bytes) from \"%s\"", + filename, fileSize, route->walcacheDir); + } + + free(contents); +} diff --git a/src/bin/pg_walsender/cmd_fetch_file.h b/src/bin/pg_walsender/cmd_fetch_file.h new file mode 100644 index 000000000..88b2d2d99 --- /dev/null +++ b/src/bin/pg_walsender/cmd_fetch_file.h @@ -0,0 +1,36 @@ +/* + * src/bin/pg_walsender/cmd_fetch_file.h + * FETCH_FILE: a non-standard side-channel, not a replication-protocol + * command, for restore_command-style single-WAL-file fetch (see the + * design doc's own reasoning: restore_command spawns a fresh subprocess + * once per segment, with no persistent session to reuse -- riding the + * replication grammar would add protocol surface no real client ever + * exercises). Reuses the same connection's startup-packet + auth + * machinery (accept_loop.c routes a dbname of the form + * "fetch//" here instead of into the normal + * replication command loop), so it's gated by the same trust/ + * allowed_hosts check, no new auth surface. + * + * Wire shape, deliberately minimal since the only caller is + * fetch_client.c (this project's own code, not a real Postgres tool): + * after AuthenticationOk, the client sends the bare filename as a single + * '\n'-terminated line (ws_read_line, not a real protocol message), and + * the server replies with exactly one message: CopyData carrying the + * raw file bytes on success, or ErrorResponse on failure. Then the + * connection closes -- no CopyOutResponse/CopyDone, this isn't a real + * COPY sub-protocol, just reusing CopyData as a convenient length- + * prefixed binary envelope. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_FETCH_FILE_H +#define WS_CMD_FETCH_FILE_H + +#include "routes.h" + +void cmd_fetch_file(int sock, const WsRoute *route); + +#endif /* WS_CMD_FETCH_FILE_H */ diff --git a/src/bin/pg_walsender/cmd_identify_system.c b/src/bin/pg_walsender/cmd_identify_system.c new file mode 100644 index 000000000..8064d32bb --- /dev/null +++ b/src/bin/pg_walsender/cmd_identify_system.c @@ -0,0 +1,58 @@ +/* + * src/bin/pg_walsender/cmd_identify_system.c + * See cmd_identify_system.h. + * + * systemid/timeline come straight from the route (written by pg_autoctl's + * archiver-serve supervisor from the monitor's own tracked values -- see + * routes.h). xlogpos is reported as "0/0" for now: computing the real + * latest-captured position requires scanning the WAL cache directory, + * which is wired in alongside START_REPLICATION (milestone 2 step 5), + * not required for the protocol handshake itself to be correct. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_identify_system.h" +#include "framing.h" + + +void +cmd_identify_system(int sock, const WsRoute *route, const char *dbname) +{ + WsColumn columns[] = { + { "systemid", WS_TEXTOID, -1 }, + { "timeline", WS_INT4OID, 4 }, + { "xlogpos", WS_TEXTOID, -1 }, + { "dbname", WS_TEXTOID, -1 }, + }; + + char timelineStr[16]; + const char *systemId = (route != NULL && route->systemId[0] != '\0') + ? route->systemId + : "0"; + int timeline = (route != NULL && route->timeline > 0) ? route->timeline : 1; + + snprintf(timelineStr, sizeof(timelineStr), "%d", timeline); + + const char *values[] = { + systemId, + timelineStr, + "0/0", + dbname, + }; + + if (!ws_send_row_description(sock, columns, 4) || + !ws_send_data_row(sock, values, 4) || + !ws_send_command_complete(sock, "IDENTIFY_SYSTEM")) + { + /* the connection is likely dead at this point; the command loop's + * next ws_read_message() will notice and close it */ + return; + } +} diff --git a/src/bin/pg_walsender/cmd_identify_system.h b/src/bin/pg_walsender/cmd_identify_system.h new file mode 100644 index 000000000..62cf8060d --- /dev/null +++ b/src/bin/pg_walsender/cmd_identify_system.h @@ -0,0 +1,19 @@ +/* + * src/bin/pg_walsender/cmd_identify_system.h + * IDENTIFY_SYSTEM: reports systemid/timeline/xlogpos/dbname for the + * resolved route. See cmd_identify_system.c for what's a placeholder in + * this milestone vs. wired to real data. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_IDENTIFY_SYSTEM_H +#define WS_CMD_IDENTIFY_SYSTEM_H + +#include "routes.h" + +void cmd_identify_system(int sock, const WsRoute *route, const char *dbname); + +#endif /* WS_CMD_IDENTIFY_SYSTEM_H */ diff --git a/src/bin/pg_walsender/cmd_show.c b/src/bin/pg_walsender/cmd_show.c new file mode 100644 index 000000000..50ed409ca --- /dev/null +++ b/src/bin/pg_walsender/cmd_show.c @@ -0,0 +1,53 @@ +/* + * src/bin/pg_walsender/cmd_show.c + * See cmd_show.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_show.h" +#include "framing.h" + + +void +cmd_show(int sock, const char *name) +{ + const char *value = NULL; + + if (strcasecmp(name, "wal_segment_size") == 0) + { + /* matches the real default; a non-default segment size would need + * to come from the archived group's own tracked configuration -- + * not wired in yet, see the identify_system placeholder note */ + value = "16MB"; + } + else if (strcasecmp(name, "data_directory_mode") == 0) + { + value = "0700"; + } + + if (value == NULL) + { + ws_send_error_response(sock, "42704", "unrecognized configuration parameter"); + return; + } + + WsColumn columns[] = { + { name, WS_TEXTOID, -1 }, + }; + + const char *values[] = { value }; + + if (!ws_send_row_description(sock, columns, 1) || + !ws_send_data_row(sock, values, 1) || + !ws_send_command_complete(sock, "SHOW")) + { + return; + } +} diff --git a/src/bin/pg_walsender/cmd_show.h b/src/bin/pg_walsender/cmd_show.h new file mode 100644 index 000000000..8e23a404d --- /dev/null +++ b/src/bin/pg_walsender/cmd_show.h @@ -0,0 +1,17 @@ +/* + * src/bin/pg_walsender/cmd_show.h + * SHOW : real pg_basebackup/pg_receivewal only ever query + * wal_segment_size and data_directory_mode (see streamutil.c in the + * Postgres source), so those are the only two GUCs this needs to answer. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_SHOW_H +#define WS_CMD_SHOW_H + +void cmd_show(int sock, const char *name); + +#endif /* WS_CMD_SHOW_H */ diff --git a/src/bin/pg_walsender/defaults.h b/src/bin/pg_walsender/defaults.h new file mode 100644 index 000000000..a2e48bc32 --- /dev/null +++ b/src/bin/pg_walsender/defaults.h @@ -0,0 +1,33 @@ +/* + * src/bin/pg_walsender/defaults.h + * A handful of constants pg_walsender needs that would otherwise come + * from pg_autoctl/defaults.h -- duplicated rather than included, since + * pg_walsender is deliberately a standalone binary that does not link + * any of pg_autoctl's own sources (see + * ~/dev/temp/archiving-disaster-recovery.md). Keep + * PG_AUTOCTL_REPLICA_USERNAME in sync with pg_autoctl/defaults.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_DEFAULTS_H +#define WS_DEFAULTS_H + +#define PG_AUTOCTL_REPLICA_USERNAME "pgautofailover_replicator" + +#define WS_DEFAULT_PORT 6543 + +/* + * Reported as the "server_version" startup parameter so that real libpq + * clients (pg_basebackup, pg_receivewal) compute a sane PQserverVersion(). + * MVP: a fixed, reasonably-current value; wiring this to the archived + * group's actual tracked pg_version (see the Postgres/Citus version + * tracking prerequisite, milestone 0) is a follow-up, not required for the + * protocol to function. + */ +#define WS_SERVER_VERSION "16.4" +#define WS_SERVER_VERSION_NUM 160004 + +#endif /* WS_DEFAULTS_H */ diff --git a/src/bin/pg_walsender/fetch_client.c b/src/bin/pg_walsender/fetch_client.c new file mode 100644 index 000000000..a141d4fb1 --- /dev/null +++ b/src/bin/pg_walsender/fetch_client.c @@ -0,0 +1,262 @@ +/* + * src/bin/pg_walsender/fetch_client.c + * See fetch_client.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "fetch_client.h" +#include "defaults.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + + +static int +connect_to(const char *host, int port) +{ + char portStr[16]; + + snprintf(portStr, sizeof(portStr), "%d", port); + + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *res = NULL; + int rc = getaddrinfo(host, portStr, &hints, &res); + + if (rc != 0) + { + log_error("Failed to resolve \"%s\": %s", host, gai_strerror(rc)); + return -1; + } + + int sock = -1; + + for (struct addrinfo *rp = res; rp != NULL; rp = rp->ai_next) + { + sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + + if (sock < 0) + { + continue; + } + + if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) + { + break; + } + + close(sock); + sock = -1; + } + + freeaddrinfo(res); + + if (sock < 0) + { + log_error("Failed to connect to %s:%d: %m", host, port); + } + + return sock; +} + + +static bool +send_startup_message(int sock, const char *database) +{ + PQExpBuffer buf = createPQExpBuffer(); + int32_t version = htonl(196608); /* protocol 3.0 */ + + appendBinaryPQExpBuffer(buf, (const char *) &version, 4); + + appendBinaryPQExpBuffer(buf, "user", strlen("user") + 1); + appendBinaryPQExpBuffer(buf, PG_AUTOCTL_REPLICA_USERNAME, + strlen(PG_AUTOCTL_REPLICA_USERNAME) + 1); + + appendBinaryPQExpBuffer(buf, "database", strlen("database") + 1); + appendBinaryPQExpBuffer(buf, database, strlen(database) + 1); + + appendPQExpBufferChar(buf, '\0'); /* terminates the parameter list */ + + int32_t totalLen = htonl(buf->len + 4); + bool ok = !PQExpBufferBroken(buf) && + ws_write_bytes(sock, &totalLen, 4) && + ws_write_bytes(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static void +extract_error_message(const char *payload, int32_t payloadLen, + char *out, size_t outSize) +{ + out[0] = '\0'; + + const char *p = payload; + const char *end = payload + payloadLen; + + while (p < end && *p != '\0') + { + char code = *p++; + const char *value = p; + + while (p < end && *p != '\0') + { + p++; + } + + if (code == 'M') + { + size_t len = Min((size_t) (p - value), outSize - 1); + + memcpy(out, value, len); + out[len] = '\0'; + } + + if (p < end) + { + p++; /* skip this field's NUL terminator */ + } + } +} + + +int +ws_fetch_file_client(const char *host, int port, const char *routeKey, + const char *filename, const char *outputPath) +{ + int sock = connect_to(host, port); + + if (sock < 0) + { + return 1; + } + + char database[512]; + + snprintf(database, sizeof(database), "fetch/%s", routeKey); + + if (!send_startup_message(sock, database)) + { + log_error("Failed to send the startup packet to %s:%d: %m", host, port); + close(sock); + return 1; + } + + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + log_error("Failed to read the authentication response from %s:%d", + host, port); + free(payload); + close(sock); + return 1; + } + + if (type == 'E') + { + char message[512]; + + extract_error_message(payload, payloadLen, message, sizeof(message)); + log_error("Authentication failed: %s", message); + free(payload); + close(sock); + return 1; + } + + free(payload); + + if (type != 'R') + { + log_error("Unexpected message type '%c' from %s:%d (expected " + "AuthenticationOk)", type, host, port); + close(sock); + return 1; + } + + char line[300]; + + snprintf(line, sizeof(line), "%s\n", filename); + + if (!ws_write_bytes(sock, line, strlen(line))) + { + log_error("Failed to send the filename request to %s:%d: %m", host, port); + close(sock); + return 1; + } + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + log_error("Failed to read the file response from %s:%d", host, port); + free(payload); + close(sock); + return 1; + } + + if (type == 'E') + { + char message[512]; + + extract_error_message(payload, payloadLen, message, sizeof(message)); + log_error("Failed to fetch \"%s\": %s", filename, message); + free(payload); + close(sock); + return 1; + } + + if (type != 'd') + { + log_error("Unexpected message type '%c' from %s:%d (expected CopyData)", + type, host, port); + free(payload); + close(sock); + return 1; + } + + close(sock); + + char tmpPath[MAXPGPATH]; + + snprintf(tmpPath, sizeof(tmpPath), "%s.pg_walsender_fetch_tmp", outputPath); + + if (!write_file(payload, payloadLen, tmpPath)) + { + log_error("Failed to write \"%s\": %m", tmpPath); + free(payload); + return 1; + } + + free(payload); + + if (rename(tmpPath, outputPath) != 0) + { + log_error("Failed to rename \"%s\" to \"%s\": %m", tmpPath, outputPath); + return 1; + } + + log_info("Fetched \"%s\" (%d bytes) to \"%s\"", filename, payloadLen, outputPath); + + return 0; +} diff --git a/src/bin/pg_walsender/fetch_client.h b/src/bin/pg_walsender/fetch_client.h new file mode 100644 index 000000000..995cc6c68 --- /dev/null +++ b/src/bin/pg_walsender/fetch_client.h @@ -0,0 +1,31 @@ +/* + * src/bin/pg_walsender/fetch_client.h + * The client side of the FETCH_FILE side-channel (cmd_fetch_file.h) -- + * the only caller of that protocol, matching its header comment ("the + * only caller here is pg_autoctl's own restore_command wrapper, which + * this project fully controls end to end"). Exposed as `pg_walsender + * fetch-file ...` (see main.c) so pg_autoctl's restore_command can shell + * out to it directly, the same way it already shells out to real + * pg_receivewal/pg_basebackup elsewhere in this project. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_FETCH_CLIENT_H +#define WS_FETCH_CLIENT_H + +/* + * Connects to host:port, requests filename for routeKey ("/ + * "), and writes the result to outputPath (via a same-directory + * temp file + rename, so a killed/interrupted fetch never leaves a + * partial file at outputPath). Returns 0 on success, 1 on any failure + * (connection, auth, missing file, short write) -- always with a + * human-readable message already logged, matching restore_command's own + * "non-zero means retry me" contract. + */ +int ws_fetch_file_client(const char *host, int port, const char *routeKey, + const char *filename, const char *outputPath); + +#endif /* WS_FETCH_CLIENT_H */ diff --git a/src/bin/pg_walsender/framing.c b/src/bin/pg_walsender/framing.c new file mode 100644 index 000000000..2e2b886b7 --- /dev/null +++ b/src/bin/pg_walsender/framing.c @@ -0,0 +1,456 @@ +/* + * src/bin/pg_walsender/framing.c + * See framing.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pqexpbuffer.h" + +#include "framing.h" +#include "log.h" + +/* startup-packet body larger than this is rejected outright as malformed */ +#define WS_MAX_STARTUP_PACKET_SIZE 10000 + +/* an ordinary post-startup message body larger than this is rejected */ +#define WS_MAX_MESSAGE_SIZE (64 * 1024 * 1024) + +#define SSL_REQUEST_CODE 80877103 +#define GSS_REQUEST_CODE 80877104 +#define CANCEL_REQUEST_CODE 80877102 + + +bool +ws_read_bytes(int sock, void *buf, size_t len) +{ + char *ptr = (char *) buf; + size_t remaining = len; + + while (remaining > 0) + { + ssize_t n = read(sock, ptr, remaining); + + if (n < 0) + { + if (errno == EINTR) + { + continue; + } + return false; + } + + if (n == 0) + { + /* peer closed the connection */ + return false; + } + + ptr += n; + remaining -= n; + } + + return true; +} + + +bool +ws_write_bytes(int sock, const void *buf, size_t len) +{ + const char *ptr = (const char *) buf; + size_t remaining = len; + + while (remaining > 0) + { + ssize_t n = write(sock, ptr, remaining); + + if (n < 0) + { + if (errno == EINTR) + { + continue; + } + return false; + } + + ptr += n; + remaining -= n; + } + + return true; +} + + +bool +ws_write_raw_byte(int sock, char c) +{ + return ws_write_bytes(sock, &c, 1); +} + + +bool +ws_read_line(int sock, char *line, size_t maxLen) +{ + size_t n = 0; + + while (n < maxLen - 1) + { + char c; + + if (!ws_read_bytes(sock, &c, 1)) + { + return false; + } + + if (c == '\n') + { + line[n] = '\0'; + return true; + } + + line[n++] = c; + } + + return false; /* line too long */ +} + + +bool +ws_read_startup_payload(int sock, char **payload, int32_t *payloadLen) +{ + unsigned char lenBuf[4]; + + *payload = NULL; + *payloadLen = 0; + + if (!ws_read_bytes(sock, lenBuf, 4)) + { + return false; + } + + int32_t len = ((int32_t) lenBuf[0] << 24) | ((int32_t) lenBuf[1] << 16) | + ((int32_t) lenBuf[2] << 8) | (int32_t) lenBuf[3]; + + if (len < 4 || len > WS_MAX_STARTUP_PACKET_SIZE) + { + log_error("Received an invalid startup packet length: %d", len); + return false; + } + + int32_t bodyLen = len - 4; + char *buf = (char *) malloc(bodyLen + 1); + + if (buf == NULL) + { + log_error("Failed to allocate %d bytes for a startup packet: %m", bodyLen); + return false; + } + + if (bodyLen > 0 && !ws_read_bytes(sock, buf, bodyLen)) + { + free(buf); + return false; + } + + buf[bodyLen] = '\0'; + + *payload = buf; + *payloadLen = bodyLen; + + return true; +} + + +bool +ws_read_message(int sock, char *type, char **payload, int32_t *payloadLen) +{ + *payload = NULL; + *payloadLen = 0; + + if (!ws_read_bytes(sock, type, 1)) + { + return false; + } + + unsigned char lenBuf[4]; + + if (!ws_read_bytes(sock, lenBuf, 4)) + { + return false; + } + + int32_t len = ((int32_t) lenBuf[0] << 24) | ((int32_t) lenBuf[1] << 16) | + ((int32_t) lenBuf[2] << 8) | (int32_t) lenBuf[3]; + + if (len < 4 || len > WS_MAX_MESSAGE_SIZE) + { + log_error("Received an invalid message length %d for message type '%c'", + len, *type); + return false; + } + + int32_t bodyLen = len - 4; + char *buf = (char *) malloc(bodyLen + 1); + + if (buf == NULL) + { + log_error("Failed to allocate %d bytes for a protocol message: %m", bodyLen); + return false; + } + + if (bodyLen > 0 && !ws_read_bytes(sock, buf, bodyLen)) + { + free(buf); + return false; + } + + buf[bodyLen] = '\0'; + + *payload = buf; + *payloadLen = bodyLen; + + return true; +} + + +bool +ws_send_message(int sock, char type, const char *data, int32_t dataLen) +{ + char header[5]; + int32_t netLen = htonl(dataLen + 4); + + header[0] = type; + memcpy(header + 1, &netLen, 4); + + if (!ws_write_bytes(sock, header, 5)) + { + return false; + } + + if (dataLen > 0 && !ws_write_bytes(sock, data, dataLen)) + { + return false; + } + + return true; +} + + +bool +ws_send_authentication_ok(int sock) +{ + int32_t zero = 0; + + return ws_send_message(sock, 'R', (const char *) &zero, 4); +} + + +bool +ws_send_parameter_status(int sock, const char *name, const char *value) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendBinaryPQExpBuffer(buf, name, strlen(name) + 1); + appendBinaryPQExpBuffer(buf, value, strlen(value) + 1); + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'S', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_backend_key_data(int sock, int32_t pid, int32_t secret) +{ + char data[8]; + int32_t netPid = htonl(pid); + int32_t netSecret = htonl(secret); + + memcpy(data, &netPid, 4); + memcpy(data + 4, &netSecret, 4); + + return ws_send_message(sock, 'K', data, 8); +} + + +bool +ws_send_ready_for_query(int sock) +{ + char status = 'I'; + + return ws_send_message(sock, 'Z', &status, 1); +} + + +bool +ws_send_error_response(int sock, const char *sqlstate, const char *message) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'S'); + appendBinaryPQExpBuffer(buf, "ERROR", strlen("ERROR") + 1); + + appendPQExpBufferChar(buf, 'C'); + appendBinaryPQExpBuffer(buf, sqlstate, strlen(sqlstate) + 1); + + appendPQExpBufferChar(buf, 'M'); + appendBinaryPQExpBuffer(buf, message, strlen(message) + 1); + + appendPQExpBufferChar(buf, '\0'); /* terminates the field list */ + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'E', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + log_debug("walsender: sent ErrorResponse %s: %s", sqlstate, message); + + return ok; +} + + +bool +ws_send_command_complete(int sock, const char *tag) +{ + return ws_send_message(sock, 'C', tag, strlen(tag) + 1); +} + + +bool +ws_send_row_description(int sock, const WsColumn *columns, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + int32_t zero32 = 0; + int16_t zero16 = 0; + int32_t typeOid = htonl(columns[i].typeOid); + int16_t typeLen = htons(columns[i].typeLen); + int32_t typeMod = htonl(-1); + int16_t format = 0; /* text */ + + appendBinaryPQExpBuffer(buf, columns[i].name, strlen(columns[i].name) + 1); + appendBinaryPQExpBuffer(buf, (const char *) &zero32, 4); /* table Oid */ + appendBinaryPQExpBuffer(buf, (const char *) &zero16, 2); /* column attnum */ + appendBinaryPQExpBuffer(buf, (const char *) &typeOid, 4); + appendBinaryPQExpBuffer(buf, (const char *) &typeLen, 2); + appendBinaryPQExpBuffer(buf, (const char *) &typeMod, 4); + appendBinaryPQExpBuffer(buf, (const char *) &format, 2); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'T', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_data_row(int sock, const char **values, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + if (values[i] == NULL) + { + int32_t neg1 = htonl(-1); + + appendBinaryPQExpBuffer(buf, (const char *) &neg1, 4); + } + else + { + int32_t len = htonl((int32_t) strlen(values[i])); + + appendBinaryPQExpBuffer(buf, (const char *) &len, 4); + appendBinaryPQExpBuffer(buf, values[i], strlen(values[i])); + } + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'D', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static bool +ws_send_copy_response(int sock, char type, int ncols) +{ + PQExpBuffer buf = createPQExpBuffer(); + + /* overall format code: 0 (textual) -- we only ever send raw bytes, not + * a real column, so this is a formality real clients don't inspect for + * a CopyBoth/CopyOut stream driven by BASE_BACKUP/START_REPLICATION */ + appendPQExpBufferChar(buf, 0); + + int16_t n = htons((int16_t) ncols); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 2); + + for (int i = 0; i < ncols; i++) + { + int16_t fmt = 0; + + appendBinaryPQExpBuffer(buf, (const char *) &fmt, 2); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, type, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +bool +ws_send_copy_out_response(int sock, int ncols) +{ + return ws_send_copy_response(sock, 'H', ncols); +} + + +bool +ws_send_copy_both_response(int sock, int ncols) +{ + return ws_send_copy_response(sock, 'W', ncols); +} + + +bool +ws_send_copy_data(int sock, const char *data, int32_t dataLen) +{ + return ws_send_message(sock, 'd', data, dataLen); +} + + +bool +ws_send_copy_done(int sock) +{ + return ws_send_message(sock, 'c', NULL, 0); +} diff --git a/src/bin/pg_walsender/framing.h b/src/bin/pg_walsender/framing.h new file mode 100644 index 000000000..8a1cd38a5 --- /dev/null +++ b/src/bin/pg_walsender/framing.h @@ -0,0 +1,79 @@ +/* + * src/bin/pg_walsender/framing.h + * Hand-written wire-level primitives for the Postgres frontend/backend + * protocol's server side: message read/write, CopyData framing, + * RowDescription/DataRow, ReadyForQuery, ErrorResponse. This is the + * pqcomm.c + pqformat.c equivalent -- no reusable library exists for + * this anywhere in Postgres (see walsender.h's own header comment), so + * it's hand-rolled directly from the documented wire format. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_FRAMING_H +#define WS_FRAMING_H + +#include +#include +#include + +/* well-known type Oids used in the RowDescriptions we hand back */ +#define WS_TEXTOID 25 +#define WS_INT4OID 23 +#define WS_INT8OID 20 + +typedef struct WsColumn +{ + const char *name; + int32_t typeOid; + int16_t typeLen; /* -1 for varlena types such as text */ +} WsColumn; + +/* raw byte I/O, EINTR-safe, short-read/short-write safe */ +bool ws_read_bytes(int sock, void *buf, size_t len); +bool ws_write_bytes(int sock, const void *buf, size_t len); + +/* + * ws_read_line reads a single '\n'-terminated line (the '\n' consumed but + * not included in *line), up to maxLen-1 bytes, NUL-terminated. Used only + * by the FETCH_FILE side-channel (cmd_fetch_file.c) for its one-shot + * "filename\n" request -- not part of the real Postgres wire protocol, + * deliberately as simple as the exchange it serves. + */ +bool ws_read_line(int sock, char *line, size_t maxLen); + +/* + * Startup-phase framing: before authentication, messages have no leading + * type byte (StartupMessage, SSLRequest, GSSENCRequest, CancelRequest are + * all just a length-prefixed body). + */ +bool ws_read_startup_payload(int sock, char **payload, int32_t *payloadLen); +bool ws_write_raw_byte(int sock, char c); + +/* + * Post-startup framing: 1-byte type + int32 length (length includes + * itself, matching the real protocol) + payload. ws_read_message + * NUL-terminates the returned payload for convenience (Query message + * bodies are C strings); callers that need the raw length still get it. + */ +bool ws_read_message(int sock, char *type, char **payload, int32_t *payloadLen); +bool ws_send_message(int sock, char type, const char *data, int32_t dataLen); + +bool ws_send_authentication_ok(int sock); +bool ws_send_parameter_status(int sock, const char *name, const char *value); +bool ws_send_backend_key_data(int sock, int32_t pid, int32_t secret); +bool ws_send_ready_for_query(int sock); +bool ws_send_error_response(int sock, const char *sqlstate, const char *message); +bool ws_send_command_complete(int sock, const char *tag); + +bool ws_send_row_description(int sock, const WsColumn *columns, int ncols); +bool ws_send_data_row(int sock, const char **values, int ncols); + +bool ws_send_copy_out_response(int sock, int ncols); +bool ws_send_copy_both_response(int sock, int ncols); +bool ws_send_copy_data(int sock, const char *data, int32_t dataLen); +bool ws_send_copy_done(int sock); + +#endif /* WS_FRAMING_H */ diff --git a/src/bin/pg_walsender/main.c b/src/bin/pg_walsender/main.c new file mode 100644 index 000000000..7876b4f43 --- /dev/null +++ b/src/bin/pg_walsender/main.c @@ -0,0 +1,220 @@ +/* + * src/bin/pg_walsender/main.c + * Entry point for pg_walsender. Two modes, dispatched on argv[1]: + * + * pg_walsender --port [--routes ] + * Runs the accept loop (see accept_loop.h). Exec'd by pg_autoctl's + * `archiver serve` supervisor (service_archiver_serve.c), but fully + * runnable and testable on its own against real psql/pg_basebackup/ + * pg_receivewal. + * + * pg_walsender fetch-file --host --port

--route / + * --filename --output + * Runs the FETCH_FILE client (fetch_client.h) once and exits -- + * pg_autoctl's restore_command shells out to this, the same way it + * already shells out to real pg_receivewal/pg_basebackup elsewhere + * in this project. + * + * Standalone binary (see the Makefile's own header comment) -- links + * neither of these modes against pg_autoctl's own sources. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "lock_utils.h" + +#include "accept_loop.h" +#include "defaults.h" +#include "fetch_client.h" +#include "file_utils.h" +#include "log.h" + +/* + * Globals required by shared common/ sources (file_utils.c's + * init_ps_buffer/set_ps_title in particular) -- pg_walsender owns these + * stub definitions itself, exactly like pgaftest's main.c does, since it + * doesn't link pg_autoctl's own main.c. + */ +char pg_autoctl_argv0[MAXPGPATH] = "pg_walsender"; +char pg_autoctl_program[MAXPGPATH] = "pg_walsender"; +int pgconnect_timeout = 2; +char *ps_buffer; +size_t ps_buffer_size; +size_t last_status_len; +Semaphore log_semaphore = { 0 }; + + +static void +usage(const char *argv0) +{ + fprintf(stderr, + "Usage: %s --port [--routes ]\n" + " %s fetch-file --host --port

--route / " + "--filename --output \n\n" + " --port port to listen on (server mode default: %d)\n" + " --routes path to the routes INI file mapping " + "\"/\" to\n" + " { walcache, basebackup, allowed_hosts } -- " + "omit only for manual\n" + " standalone testing (accepts any dbname, no " + "host restriction)\n" + " fetch-file one-shot FETCH_FILE client, for use as a " + "restore_command\n", + argv0, argv0, WS_DEFAULT_PORT); +} + + +static int +main_fetch_file(int argc, char **argv) +{ + char host[256] = { 0 }; + int port = WS_DEFAULT_PORT; + char route[256] = { 0 }; + char filename[256] = { 0 }; + char output[MAXPGPATH] = { 0 }; + + static struct option longOptions[] = { + { "host", required_argument, NULL, 'H' }, + { "port", required_argument, NULL, 'p' }, + { "route", required_argument, NULL, 'r' }, + { "filename", required_argument, NULL, 'f' }, + { "output", required_argument, NULL, 'o' }, + { NULL, 0, NULL, 0 } + }; + + int c; + + while ((c = getopt_long(argc, argv, "H:p:r:f:o:", longOptions, NULL)) != -1) + { + switch (c) + { + case 'H': + { + strlcpy(host, optarg, sizeof(host)); + break; + } + + case 'p': + { + port = atoi(optarg); + break; + } + + case 'r': + { + strlcpy(route, optarg, sizeof(route)); + break; + } + + case 'f': + { + strlcpy(filename, optarg, sizeof(filename)); + break; + } + + case 'o': + { + strlcpy(output, optarg, sizeof(output)); + break; + } + + default: + { + usage(argv[0]); + return 1; + } + } + } + + if (host[0] == '\0' || route[0] == '\0' || filename[0] == '\0' || + output[0] == '\0') + { + fprintf(stderr, "fetch-file: --host, --route, --filename, and " + "--output are all required\n"); + usage(argv[0]); + return 1; + } + + return ws_fetch_file_client(host, port, route, filename, output); +} + + +int +main(int argc, char **argv) +{ + strlcpy(pg_autoctl_program, argv[0], sizeof(pg_autoctl_program)); + init_ps_buffer(argc, argv); + + log_set_level(LOG_INFO); + + if (argc >= 2 && strcmp(argv[1], "fetch-file") == 0) + { + /* shift argv so getopt_long in main_fetch_file() skips "fetch-file" */ + return main_fetch_file(argc - 1, argv + 1); + } + + WsServerConfig config = { 0 }; + + config.port = WS_DEFAULT_PORT; + + static struct option longOptions[] = { + { "port", required_argument, NULL, 'p' }, + { "routes", required_argument, NULL, 'r' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + int c; + + while ((c = getopt_long(argc, argv, "p:r:h", longOptions, NULL)) != -1) + { + switch (c) + { + case 'p': + { + config.port = atoi(optarg); + break; + } + + case 'r': + { + strlcpy(config.routesPath, optarg, sizeof(config.routesPath)); + break; + } + + case 'h': + { + usage(argv[0]); + return 0; + } + + default: + { + usage(argv[0]); + return 1; + } + } + } + + if (config.port <= 0 || config.port > 65535) + { + log_fatal("Invalid --port value"); + return 1; + } + + if (!ws_accept_loop(&config)) + { + return 1; + } + + return 0; +} diff --git a/src/bin/pg_walsender/repl_command.c b/src/bin/pg_walsender/repl_command.c new file mode 100644 index 000000000..be7bb5205 --- /dev/null +++ b/src/bin/pg_walsender/repl_command.c @@ -0,0 +1,115 @@ +/* + * src/bin/pg_walsender/repl_command.c + * See repl_command.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include + +#include "postgres_fe.h" + +#include "repl_command.h" +#include "cmd_base_backup.h" +#include "cmd_identify_system.h" +#include "cmd_show.h" +#include "framing.h" + + +static const char * +skip_whitespace(const char *p) +{ + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') + { + p++; + } + + return p; +} + + +static void +rtrim(char *s) +{ + size_t n = strlen(s); + + while (n > 0 && + (s[n - 1] == ' ' || s[n - 1] == '\t' || s[n - 1] == '\n' || + s[n - 1] == '\r' || s[n - 1] == ';')) + { + s[--n] = '\0'; + } +} + + +bool +repl_command_parse(const char *query, WsCommand *cmd) +{ + memset(cmd, 0, sizeof(WsCommand)); + + const char *p = skip_whitespace(query); + + if (strncasecmp(p, "IDENTIFY_SYSTEM", strlen("IDENTIFY_SYSTEM")) == 0) + { + cmd->type = WS_CMD_IDENTIFY_SYSTEM; + return true; + } + + if (strncasecmp(p, "SHOW", strlen("SHOW")) == 0 && isspace((unsigned char) p[4])) + { + p = skip_whitespace(p + 4); + strlcpy(cmd->showName, p, sizeof(cmd->showName)); + rtrim(cmd->showName); + cmd->type = WS_CMD_SHOW; + return true; + } + + if (strncasecmp(p, "BASE_BACKUP", strlen("BASE_BACKUP")) == 0) + { + p = skip_whitespace(p + strlen("BASE_BACKUP")); + strlcpy(cmd->rawOptions, p, sizeof(cmd->rawOptions)); + rtrim(cmd->rawOptions); + cmd->type = WS_CMD_BASE_BACKUP; + return true; + } + + cmd->type = WS_CMD_UNKNOWN; + return false; +} + + +void +ws_dispatch_command(int sock, const WsCommand *cmd, + const WsRoute *route, const char *dbname) +{ + switch (cmd->type) + { + case WS_CMD_IDENTIFY_SYSTEM: + { + cmd_identify_system(sock, route, dbname); + break; + } + + case WS_CMD_SHOW: + { + cmd_show(sock, cmd->showName); + break; + } + + case WS_CMD_BASE_BACKUP: + { + cmd_base_backup(sock, route, cmd->rawOptions); + break; + } + + default: + { + ws_send_error_response(sock, "42601", "unsupported replication command"); + break; + } + } +} diff --git a/src/bin/pg_walsender/repl_command.h b/src/bin/pg_walsender/repl_command.h new file mode 100644 index 000000000..c45dd36fe --- /dev/null +++ b/src/bin/pg_walsender/repl_command.h @@ -0,0 +1,60 @@ +/* + * src/bin/pg_walsender/repl_command.h + * Parses the Query-message command strings real replication clients send + * (e.g. "IDENTIFY_SYSTEM", "SHOW wal_segment_size") and dispatches to the + * matching cmd_*.c handler. This is repl_gram.y/repl_scanner.l's + * equivalent, hand-rolled: the real grammar is backend-locked (bison + * output building backend Node types via palloc, see the design + * research), and the fixed ~7-command surface this project needs doesn't + * justify vendoring bison/flex infrastructure for it -- plain C + * tokenizing is enough. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_REPL_COMMAND_H +#define WS_REPL_COMMAND_H + +#include + +#include "routes.h" + +typedef enum WsCommandType +{ + WS_CMD_IDENTIFY_SYSTEM, + WS_CMD_SHOW, + WS_CMD_BASE_BACKUP, + WS_CMD_UNKNOWN +} WsCommandType; + +typedef struct WsCommand +{ + WsCommandType type; + char showName[NAMEDATALEN]; /* WS_CMD_SHOW only */ + char rawOptions[1024]; /* WS_CMD_BASE_BACKUP only: the "(...)" or + * trailing-token option list verbatim, + * parsed by cmd_base_backup.c itself */ +} WsCommand; + +/* + * repl_command_parse fills *cmd from the given Query-message string. + * Returns false (cmd->type == WS_CMD_UNKNOWN) for anything not yet + * recognized -- the caller sends the ErrorResponse, this function doesn't + * touch the socket. + */ +bool repl_command_parse(const char *query, WsCommand *cmd); + +/* + * ws_dispatch_command runs cmd against the connection's resolved route + * (NULL in manual-testing mode, see auth.h) and the dbname the client + * originally requested (always set, even without a route -- see + * startup.c), sending whatever RowDescription/DataRow/CommandComplete or + * ErrorResponse the command produces. Never sends ReadyForQuery -- the + * caller's command loop does that once per Query message, uniformly. + */ +void ws_dispatch_command(int sock, const WsCommand *cmd, + const WsRoute *route, const char *dbname); + +#endif /* WS_REPL_COMMAND_H */ diff --git a/src/bin/pg_walsender/routes.c b/src/bin/pg_walsender/routes.c new file mode 100644 index 000000000..539802624 --- /dev/null +++ b/src/bin/pg_walsender/routes.c @@ -0,0 +1,233 @@ +/* + * src/bin/pg_walsender/routes.c + * See routes.h. Deliberately built on the low-level, dynamic-section + * ini.h API (ini_load/ini_section_count/...) rather than this project's + * own ini_file.c wrapper: ini_file.c's IniOption model assumes a fixed, + * compile-time-known set of section/key names, which doesn't fit a file + * whose sections are one per archived (formation, group) -- unknown in + * advance. ini.h's lower-level, enumerable API is exactly the right + * shape and is already vendored into this project (src/bin/lib/libs/ + * ini.h, compiled into libpgaf_common.a via common/ini_implementation.c). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "ini.h" + +#include "routes.h" +#include "file_utils.h" +#include "log.h" + + +bool +routes_load(const char *path, WsRoute **routesOut, int *countOut) +{ + *routesOut = NULL; + *countOut = 0; + + char *contents = NULL; + long fileSize = 0; + + if (!read_file(path, &contents, &fileSize)) + { + log_error("Failed to read routes file \"%s\"", path); + return false; + } + + ini_t *ini = ini_load(contents, NULL); + + free(contents); + + if (ini == NULL) + { + log_error("Failed to parse routes file \"%s\"", path); + return false; + } + + int sectionCount = ini_section_count(ini); + + /* section 0 is ini.h's implicit global section: never a real route */ + WsRoute *routes = (WsRoute *) calloc(sectionCount, sizeof(WsRoute)); + + if (routes == NULL && sectionCount > 0) + { + log_error("Failed to allocate memory for %d routes", sectionCount); + ini_destroy(ini); + return false; + } + + int n = 0; + + for (int s = 0; s < sectionCount; s++) + { + const char *name = ini_section_name(ini, s); + + if (name == NULL || name[0] == '\0') + { + continue; /* the global section */ + } + + WsRoute *route = &routes[n]; + + memset(route, 0, sizeof(WsRoute)); + strlcpy(route->key, name, sizeof(route->key)); + + int propCount = ini_property_count(ini, s); + + for (int p = 0; p < propCount; p++) + { + const char *rawPropName = ini_property_name(ini, s, p); + const char *propValue = ini_property_value(ini, s, p); + + if (rawPropName == NULL || propValue == NULL) + { + continue; + } + + /* + * ini.h's own parser (src/bin/lib/libs/ini.h's ini_load) trims + * whitespace around the value but NOT trailing whitespace + * between a key and '=' -- "walcache = /path" parses the key + * as "walcache " with a trailing space. Trim defensively here + * rather than relying on every routes file being written with + * no space before '='. + */ + char propName[128]; + + strlcpy(propName, rawPropName, sizeof(propName)); + + size_t nameLen = strlen(propName); + + while (nameLen > 0 && isspace((unsigned char) propName[nameLen - 1])) + { + propName[--nameLen] = '\0'; + } + + if (strcmp(propName, "walcache") == 0) + { + strlcpy(route->walcacheDir, propValue, sizeof(route->walcacheDir)); + } + else if (strcmp(propName, "basebackup") == 0) + { + strlcpy(route->basebackupDir, propValue, sizeof(route->basebackupDir)); + } + else if (strcmp(propName, "allowed_hosts") == 0) + { + strlcpy(route->allowedHosts, propValue, sizeof(route->allowedHosts)); + } + else if (strcmp(propName, "systemid") == 0) + { + strlcpy(route->systemId, propValue, sizeof(route->systemId)); + } + else if (strcmp(propName, "timeline") == 0) + { + route->timeline = atoi(propValue); + } + else + { + log_warn("Ignoring unknown routes file key \"%s\" in section [%s]", + propName, name); + } + } + + n++; + } + + ini_destroy(ini); + + *routesOut = routes; + *countOut = n; + + return true; +} + + +void +routes_free(WsRoute *routes) +{ + free(routes); +} + + +const WsRoute * +routes_find(const WsRoute *routes, int count, const char *key) +{ + for (int i = 0; i < count; i++) + { + if (strcmp(routes[i].key, key) == 0) + { + return &routes[i]; + } + } + + return NULL; +} + + +bool +routes_host_allowed(const WsRoute *route, const char *peerIP) +{ + if (route->allowedHosts[0] == '\0') + { + return true; /* no restriction configured for this route */ + } + + char list[sizeof(route->allowedHosts)]; + + strlcpy(list, route->allowedHosts, sizeof(list)); + + char *saveptr = NULL; + + for (char *tok = strtok_r(list, ",", &saveptr); + tok != NULL; + tok = strtok_r(NULL, ",", &saveptr)) + { + while (*tok == ' ' || *tok == '\t') + { + tok++; + } + + if (strcmp(tok, peerIP) == 0) + { + return true; + } + + /* also resolve hostnames in the allow-list and compare addresses */ + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + + struct addrinfo *res = NULL; + + if (getaddrinfo(tok, NULL, &hints, &res) == 0) + { + for (struct addrinfo *rp = res; rp != NULL; rp = rp->ai_next) + { + char resolved[NI_MAXHOST]; + + if (getnameinfo(rp->ai_addr, rp->ai_addrlen, + resolved, sizeof(resolved), + NULL, 0, NI_NUMERICHOST) == 0 && + strcmp(resolved, peerIP) == 0) + { + freeaddrinfo(res); + return true; + } + } + + freeaddrinfo(res); + } + } + + return false; +} diff --git a/src/bin/pg_walsender/routes.h b/src/bin/pg_walsender/routes.h new file mode 100644 index 000000000..50ec272ae --- /dev/null +++ b/src/bin/pg_walsender/routes.h @@ -0,0 +1,53 @@ +/* + * src/bin/pg_walsender/routes.h + * The archiver's own "pg_hba.conf" equivalent: a small INI file, one + * section per "/" this archiver serves, mapping the + * incoming connection's dbname to a WAL-cache directory, a base-backup + * directory, and an optional allowed-hosts list. Written and periodically + * refreshed by pg_autoctl's archiver-serve supervisor + * (service_archiver_serve.c) from the monitor's archiver_node/basebackup + * rows; pg_walsender itself never talks to the monitor (see the + * "Routing" section of ~/dev/temp/archiving-disaster-recovery.md's + * implementation plan). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_ROUTES_H +#define WS_ROUTES_H + +#include + +#include "postgres_fe.h" + +typedef struct WsRoute +{ + char key[NAMEDATALEN + 16]; /* "/", matches dbname */ + char walcacheDir[MAXPGPATH]; + char basebackupDir[MAXPGPATH]; + char allowedHosts[1024]; /* comma-separated, empty = unrestricted */ + char systemId[32]; /* decimal uint64, as text; "" = unknown */ + int timeline; /* 0 = unknown */ +} WsRoute; + +/* + * routes_load parses the routes file at path into a freshly malloc'ed + * array. Returns true with *routesOut and *countOut set (possibly count + * == 0 for an empty file) on success, false on a missing/malformed file. + */ +bool routes_load(const char *path, WsRoute **routesOut, int *countOut); +void routes_free(WsRoute *routes); + +const WsRoute * routes_find(const WsRoute *routes, int count, const char *key); + +/* + * routes_host_allowed checks peerIP (a numeric address string, as returned + * by getnameinfo(..., NI_NUMERICHOST)) against route->allowedHosts, which + * may contain either numeric addresses or hostnames (resolved via DNS at + * check time). An empty allowedHosts list means "no restriction." + */ +bool routes_host_allowed(const WsRoute *route, const char *peerIP); + +#endif /* WS_ROUTES_H */ diff --git a/src/bin/pg_walsender/startup.c b/src/bin/pg_walsender/startup.c new file mode 100644 index 000000000..348c3e7ca --- /dev/null +++ b/src/bin/pg_walsender/startup.c @@ -0,0 +1,140 @@ +/* + * src/bin/pg_walsender/startup.c + * See startup.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "startup.h" +#include "framing.h" +#include "log.h" + +#define SSL_REQUEST_CODE 80877103 +#define GSS_REQUEST_CODE 80877104 +#define CANCEL_REQUEST_CODE 80877102 + + +bool +ws_startup_negotiate(int sock, WsStartupParams *params) +{ + memset(params, 0, sizeof(WsStartupParams)); + + for (;;) + { + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_startup_payload(sock, &payload, &payloadLen)) + { + free(payload); + return false; + } + + if (payloadLen < 4) + { + log_error("Received a malformed startup packet (%d bytes)", payloadLen); + free(payload); + return false; + } + + int32_t code; + + memcpy(&code, payload, 4); + code = ntohl(code); + + if (code == SSL_REQUEST_CODE || code == GSS_REQUEST_CODE) + { + free(payload); + + /* + * MVP: no SSL/GSS support yet (see the design doc's Auth + * section) -- decline, real libpq's default sslmode=prefer + * falls back to plaintext automatically on 'N'. + */ + if (!ws_write_raw_byte(sock, 'N')) + { + return false; + } + + continue; + } + + if (code == CANCEL_REQUEST_CODE) + { + log_debug("Ignoring a CancelRequest on a walsender connection"); + free(payload); + return false; + } + + if ((code >> 16) != 3) + { + log_error("Unsupported startup protocol version 0x%08x", code); + free(payload); + return false; + } + + /* parse the NUL-separated key/value pairs following the version code */ + const char *ptr = payload + 4; + const char *end = payload + payloadLen; + + while (ptr < end && *ptr != '\0') + { + const char *key = ptr; + + ptr += strlen(ptr) + 1; + + if (ptr >= end) + { + break; + } + + const char *value = ptr; + + ptr += strlen(ptr) + 1; + + if (strcmp(key, "user") == 0) + { + strlcpy(params->user, value, sizeof(params->user)); + } + else if (strcmp(key, "database") == 0) + { + strlcpy(params->database, value, sizeof(params->database)); + } + else if (strcmp(key, "application_name") == 0) + { + strlcpy(params->applicationName, value, sizeof(params->applicationName)); + } + else if (strcmp(key, "replication") == 0) + { + params->replication = (strcmp(value, "1") == 0 || + strcasecmp(value, "true") == 0 || + strcasecmp(value, "database") == 0); + } + } + + free(payload); + + /* + * A real replication connection always carries "database" too when + * replication=database is used (that's how pg_basebackup connects); + * a bare replication=1/true connection (pg_receivewal's style) may + * not set "database" at all. Default it to the "user" so downstream + * routing always has *something* to look up rather than an empty + * key -- callers that require a real "/" key + * still get a clean "unknown route" ErrorResponse from auth.c. + */ + if (params->database[0] == '\0') + { + strlcpy(params->database, params->user, sizeof(params->database)); + } + + return true; + } +} diff --git a/src/bin/pg_walsender/startup.h b/src/bin/pg_walsender/startup.h new file mode 100644 index 000000000..d1c4477c5 --- /dev/null +++ b/src/bin/pg_walsender/startup.h @@ -0,0 +1,30 @@ +/* + * src/bin/pg_walsender/startup.h + * Startup-packet negotiation: SSL/GSS decline, protocol version check, + * and StartupMessage key/value parsing. Structurally mirrors real + * Postgres's ProcessStartupPacket() (backend_startup.c), reimplemented + * frontend-only -- that function is backend-locked (palloc/List/ereport, + * see the design research in ~/dev/temp/archiving-disaster-recovery.md's + * companion investigation), not something we can call into directly. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_STARTUP_H +#define WS_STARTUP_H + +#include + +#include "walsender.h" + +/* + * ws_startup_negotiate reads (and answers) SSLRequest/GSSENCRequest + * probes until the client sends a real StartupMessage, then parses it into + * *params. Returns false on any protocol error or if the client gives up + * (socket already unusable at that point; caller should just close it). + */ +bool ws_startup_negotiate(int sock, WsStartupParams *params); + +#endif /* WS_STARTUP_H */ diff --git a/src/bin/pg_walsender/tar_stream.c b/src/bin/pg_walsender/tar_stream.c new file mode 100644 index 000000000..689553395 --- /dev/null +++ b/src/bin/pg_walsender/tar_stream.c @@ -0,0 +1,270 @@ +/* + * src/bin/pg_walsender/tar_stream.c + * See tar_stream.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "pgtar.h" + +#include "tar_stream.h" +#include "log.h" + +/* matches basebackup.c's own TAR_NUM_TERMINATION_BLOCKS */ +#define TAR_NUM_TERMINATION_BLOCKS 2 + +#define TAR_READ_CHUNK_SIZE (64 * 1024) + +typedef struct TarWalkState +{ + TarChunkCallback callback; + void *context; + bool ok; +} TarWalkState; + + +static bool +emit(TarWalkState *state, const char *data, size_t len) +{ + if (!state->ok) + { + return false; + } + + if (!state->callback(state->context, data, len)) + { + state->ok = false; + } + + return state->ok; +} + + +static bool +emit_header(TarWalkState *state, const char *memberName, + const char *linkTarget, struct stat *st) +{ + char header[TAR_BLOCK_SIZE]; + + enum tarError rc = tarCreateHeader(header, memberName, linkTarget, + st->st_size, st->st_mode, + st->st_uid, st->st_gid, st->st_mtime); + + if (rc != TAR_OK) + { + log_error("Failed to build a tar header for \"%s\": %s", memberName, + rc == TAR_NAME_TOO_LONG + ? "file name too long for tar format" + : "symbolic link target too long for tar format"); + return false; + } + + return emit(state, header, TAR_BLOCK_SIZE); +} + + +static bool +emit_file_contents(TarWalkState *state, const char *path, off_t size) +{ + FILE *file = fopen(path, "rb"); + + if (file == NULL) + { + log_error("Failed to open \"%s\": %m", path); + return false; + } + + char buffer[TAR_READ_CHUNK_SIZE]; + off_t remaining = size; + + while (remaining > 0) + { + size_t want = (size_t) Min(remaining, (off_t) sizeof(buffer)); + size_t got = fread(buffer, 1, want, file); + + if (got == 0) + { + log_error("Short read on \"%s\" while building a base backup tar " + "stream (file changed size mid-read?)", path); + fclose(file); + return false; + } + + if (!emit(state, buffer, got)) + { + fclose(file); + return false; + } + + remaining -= (off_t) got; + } + + fclose(file); + + size_t pad = tarPaddingBytesRequired((size_t) size); + + if (pad > 0) + { + char zeros[TAR_BLOCK_SIZE] = { 0 }; + + if (!emit(state, zeros, pad)) + { + return false; + } + } + + return true; +} + + +static bool +walk_directory(TarWalkState *state, const char *rootDir, const char *relDir) +{ + char fullDir[MAXPGPATH]; + + if (relDir[0] == '\0') + { + strlcpy(fullDir, rootDir, sizeof(fullDir)); + } + else + { + snprintf(fullDir, sizeof(fullDir), "%s/%s", rootDir, relDir); + } + + DIR *dir = opendir(fullDir); + + if (dir == NULL) + { + log_error("Failed to open directory \"%s\": %m", fullDir); + return false; + } + + struct dirent *entry; + + while (state->ok && (entry = readdir(dir)) != NULL) + { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + { + continue; + } + + char fullPath[MAXPGPATH]; + char relPath[MAXPGPATH]; + + snprintf(fullPath, sizeof(fullPath), "%s/%s", fullDir, entry->d_name); + + if (relDir[0] == '\0') + { + strlcpy(relPath, entry->d_name, sizeof(relPath)); + } + else + { + snprintf(relPath, sizeof(relPath), "%s/%s", relDir, entry->d_name); + } + + struct stat st; + + if (lstat(fullPath, &st) != 0) + { + log_error("Failed to stat \"%s\": %m", fullPath); + state->ok = false; + break; + } + + if (S_ISLNK(st.st_mode)) + { + char linkTarget[MAXPGPATH]; + ssize_t len = readlink(fullPath, linkTarget, sizeof(linkTarget) - 1); + + if (len < 0) + { + log_error("Failed to read symbolic link \"%s\": %m", fullPath); + state->ok = false; + break; + } + + linkTarget[len] = '\0'; + + /* + * A symlink to a directory (Postgres uses this for tablespace + * links under pg_tblspc/) is written as a directory entry with + * a link target, matching tarCreateHeader()'s own convention + * (see its S_ISDIR/linktarget handling) -- but we don't + * recurse through it: multi-tablespace archives are a later + * milestone (see this file's own header comment), a symlink + * here is emitted as a bare tar entry, not expanded. + */ + if (!emit_header(state, relPath, linkTarget, &st)) + { + state->ok = false; + break; + } + + continue; + } + + if (S_ISDIR(st.st_mode)) + { + if (!emit_header(state, relPath, NULL, &st)) + { + state->ok = false; + break; + } + + if (!walk_directory(state, rootDir, relPath)) + { + state->ok = false; + break; + } + + continue; + } + + if (!S_ISREG(st.st_mode)) + { + /* skip anything else (sockets, fifos, device files) */ + continue; + } + + if (!emit_header(state, relPath, NULL, &st)) + { + state->ok = false; + break; + } + + if (!emit_file_contents(state, fullPath, st.st_size)) + { + state->ok = false; + break; + } + } + + closedir(dir); + + return state->ok; +} + + +bool +tar_stream_directory(const char *rootDir, TarChunkCallback callback, void *context) +{ + TarWalkState state = { callback, context, true }; + + if (!walk_directory(&state, rootDir, "")) + { + return false; + } + + char zeros[TAR_BLOCK_SIZE * TAR_NUM_TERMINATION_BLOCKS] = { 0 }; + + return emit(&state, zeros, sizeof(zeros)); +} diff --git a/src/bin/pg_walsender/tar_stream.h b/src/bin/pg_walsender/tar_stream.h new file mode 100644 index 000000000..5c4fcce6f --- /dev/null +++ b/src/bin/pg_walsender/tar_stream.h @@ -0,0 +1,48 @@ +/* + * src/bin/pg_walsender/tar_stream.h + * Walks a directory tree and emits it as a ustar-format byte stream via a + * callback, chunked for CopyData framing. Reproduces the shape of + * basebackup.c's sendDir()/sendFile()/_tarWriteHeader() pattern -- not + * linked (backend-only, tied to the bbsink sink-chain and palloc/ + * ereport), but the tar-header math itself comes straight from the + * vendored vendor/tar.c (tarCreateHeader(), the real Postgres source + * both basebackup.c and pg_basebackup itself build on). + * + * Deliberately simpler than basebackup.c's own sendDir(): this walks an + * already-complete, static backup directory (produced by a real + * pg_basebackup run against a live server -- see the "Base backup + * generation" milestone, not yet implemented), so none of basebackup.c's + * live-PGDATA special-casing (skipping pg_wal/pg_stat_tmp/postmaster + * files, injecting a synthesized backup_label, tracking WAL positions + * mid-walk) applies -- the directory is tarred up exactly as it sits on + * disk. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_TAR_STREAM_H +#define WS_TAR_STREAM_H + +#include +#include + +/* + * Called with successive chunks of the tar byte stream (header blocks, + * file content, padding, and the final end-of-archive zero blocks all flow + * through this same callback) -- return false to abort the walk early + * (e.g. the client disconnected mid-stream). + */ +typedef bool (*TarChunkCallback) (void *context, const char *data, size_t len); + +/* + * tar_stream_directory walks rootDir recursively and invokes callback with + * the resulting ustar byte stream, including the standard two-zero-block + * end-of-archive marker. Tar member names are rootDir-relative, with no + * leading "./" (matching real Postgres's own convention -- see + * basebackup.c's sendDir()). + */ +bool tar_stream_directory(const char *rootDir, TarChunkCallback callback, void *context); + +#endif /* WS_TAR_STREAM_H */ diff --git a/src/bin/pg_walsender/vendor/pgtar.h b/src/bin/pg_walsender/vendor/pgtar.h new file mode 100644 index 000000000..9f80583b5 --- /dev/null +++ b/src/bin/pg_walsender/vendor/pgtar.h @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * pgtar.h + * Functions for manipulating tarfile datastructures (vendor/tar.c) + * + * Vendored from PostgreSQL's src/include/pgtar.h (checked against + * /Users/dim/dev/PostgreSQL/postgresql; logic unchanged, reformatted to + * this project's own brace style via citus_indent) -- pure ustar-format + * constants and one inline helper, no backend dependency, genuinely + * reusable as-is (same PostgreSQL License). pg_walsender's own + * tar_stream.c builds the BASE_BACKUP tar stream on top of + * tarCreateHeader() from vendor/tar.c, the same way basebackup.c's + * _tarWriteHeader() does. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/pgtar.h + * + *------------------------------------------------------------------------- + */ +#ifndef PG_TAR_H +#define PG_TAR_H + +#define TAR_BLOCK_SIZE 512 + +enum tarError +{ + TAR_OK = 0, + TAR_NAME_TOO_LONG, + TAR_SYMLINK_TOO_LONG, +}; + +/* + * Offsets of fields within a 512-byte tar header. + * + * "tar number" values should be generated using print_tar_number() and can be + * read using read_tar_number(). Fields that contain strings are generally + * both filled and read using strlcpy(). + * + * The value for the checksum field can be computed using tarChecksum(). + * + * Some fields are not used by PostgreSQL; see tarCreateHeader(). + */ +enum tarHeaderOffset +{ + TAR_OFFSET_NAME = 0, /* 100 byte string */ + TAR_OFFSET_MODE = 100, /* 8 byte tar number, excludes S_IFMT */ + TAR_OFFSET_UID = 108, /* 8 byte tar number */ + TAR_OFFSET_GID = 116, /* 8 byte tar number */ + TAR_OFFSET_SIZE = 124, /* 8 byte tar number */ + TAR_OFFSET_MTIME = 136, /* 12 byte tar number */ + TAR_OFFSET_CHECKSUM = 148, /* 8 byte tar number */ + TAR_OFFSET_TYPEFLAG = 156, /* 1 byte file type, see TAR_FILETYPE_* */ + TAR_OFFSET_LINKNAME = 157, /* 100 byte string */ + TAR_OFFSET_MAGIC = 257, /* "ustar" with terminating zero byte */ + TAR_OFFSET_VERSION = 263, /* "00" */ + TAR_OFFSET_UNAME = 265, /* 32 byte string */ + TAR_OFFSET_GNAME = 297, /* 32 byte string */ + TAR_OFFSET_DEVMAJOR = 329, /* 8 byte tar number */ + TAR_OFFSET_DEVMINOR = 337, /* 8 byte tar number */ + TAR_OFFSET_PREFIX = 345, /* 155 byte string */ + /* last 12 bytes of the 512-byte block are unassigned */ +}; + +/* See POSIX (not all the standard file type codes are listed here) */ +enum tarFileType +{ + TAR_FILETYPE_PLAIN = '0', + TAR_FILETYPE_PLAIN_OLD = '\0', /* backwards compatibility, per POSIX */ + TAR_FILETYPE_SYMLINK = '2', + TAR_FILETYPE_DIRECTORY = '5', + TAR_FILETYPE_PAX_EXTENDED = 'x', + TAR_FILETYPE_PAX_EXTENDED_GLOBAL = 'g', +}; + +extern enum tarError tarCreateHeader(char *h, const char *filename, + const char *linktarget, pgoff_t size, + mode_t mode, uid_t uid, gid_t gid, + time_t mtime); +extern uint64 read_tar_number(const char *s, int len); +extern void print_tar_number(char *s, int len, uint64 val); +extern int tarChecksum(const char *header); +extern bool isValidTarHeader(const char *header); + +/* + * Compute the number of padding bytes required for an entry in a tar + * archive. We must pad out to a multiple of TAR_BLOCK_SIZE. Since that's + * a power of 2, we can use TYPEALIGN(). + */ +static inline size_t +tarPaddingBytesRequired(size_t len) +{ + return TYPEALIGN(TAR_BLOCK_SIZE, len) - len; +} + + +#endif diff --git a/src/bin/pg_walsender/vendor/tar.c b/src/bin/pg_walsender/vendor/tar.c new file mode 100644 index 000000000..8049b4995 --- /dev/null +++ b/src/bin/pg_walsender/vendor/tar.c @@ -0,0 +1,278 @@ +/* + * vendor/tar.c + * Vendored from PostgreSQL's src/port/tar.c (checked against + * /Users/dim/dev/PostgreSQL/postgresql; logic unchanged, reformatted to + * this project's own brace style via citus_indent) -- ustar header + * construction/checksum logic, pure C with no backend dependency (only + * c.h/pgtar.h), already proven frontend-safe since it's what + * pg_basebackup's own client-side tar handling and the backend's + * basebackup.c both build on. pg_walsender's tar_stream.c uses + * tarCreateHeader()/tarPaddingBytesRequired() directly rather than + * re-deriving the ustar byte layout by hand. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/port/tar.c + * + */ + +#include "c.h" + +#include + +#include "pgtar.h" + +/* + * Print a numeric field in a tar header. The field starts at *s and is of + * length len; val is the value to be written. + * + * Per POSIX, the way to write a number is in octal with leading zeroes and + * one trailing space (or NUL, but we use space) at the end of the specified + * field width. + * + * However, the given value may not fit in the available space in octal form. + * If that's true, we use the GNU extension of writing \200 followed by the + * number in base-256 form (ie, stored in binary MSB-first). (Note: here we + * support only non-negative numbers, so we don't worry about the GNU rules + * for handling negative numbers.) + */ +void +print_tar_number(char *s, int len, uint64 val) +{ + if (val < (((uint64) 1) << ((len - 1) * 3))) + { + /* Use octal with trailing space */ + s[--len] = ' '; + while (len) + { + s[--len] = (val & 7) + '0'; + val >>= 3; + } + } + else + { + /* Use base-256 with leading \200 */ + s[0] = '\200'; + while (len > 1) + { + s[--len] = (val & 255); + val >>= 8; + } + } +} + + +/* + * Read a numeric field in a tar header. The field starts at *s and is of + * length len. + * + * The POSIX-approved format for a number is octal, ending with a space or + * NUL. However, for values that don't fit, we recognize the GNU extension + * of \200 followed by the number in base-256 form (ie, stored in binary + * MSB-first). (Note: here we support only non-negative numbers, so we don't + * worry about the GNU rules for handling negative numbers.) + */ +uint64 +read_tar_number(const char *s, int len) +{ + uint64 result = 0; + + if (*s == '\200') + { + /* base-256 */ + while (--len) + { + result <<= 8; + result |= (unsigned char) (*++s); + } + } + else + { + /* octal */ + while (len-- && *s >= '0' && *s <= '7') + { + result <<= 3; + result |= (*s - '0'); + s++; + } + } + return result; +} + + +/* + * Calculate the tar checksum for a header. The header is assumed to always + * be 512 bytes, per the tar standard. + */ +int +tarChecksum(const char *header) +{ + int i, + sum; + + /* + * Per POSIX, the checksum is the simple sum of all bytes in the header, + * treating the bytes as unsigned, and treating the checksum field (at + * offset TAR_OFFSET_CHECKSUM) as though it contained 8 spaces. + */ + sum = 8 * ' '; /* presumed value for checksum field */ + for (i = 0; i < TAR_BLOCK_SIZE; i++) + { + if (i < TAR_OFFSET_CHECKSUM || i >= TAR_OFFSET_CHECKSUM + 8) + { + sum += 0xFF & header[i]; + } + } + return sum; +} + + +/* + * Check validity of a tar header (assumed to be 512 bytes long). + * We verify the checksum and the magic number / version. + */ +bool +isValidTarHeader(const char *header) +{ + int sum; + int chk = tarChecksum(header); + + sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8); + + if (sum != chk) + { + return false; + } + + /* POSIX tar format */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 && + memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0) + { + return true; + } + + /* GNU tar format */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar \0", 8) == 0) + { + return true; + } + + /* not-quite-POSIX format written by pre-9.3 pg_dump */ + if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0) + { + return true; + } + + return false; +} + + +/* + * Fill in the buffer pointed to by h with a tar format header. This buffer + * must always have space for 512 characters, which is a requirement of + * the tar format. + */ +enum tarError +tarCreateHeader(char *h, const char *filename, const char *linktarget, + pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime) +{ + if (strlen(filename) > 99) + { + return TAR_NAME_TOO_LONG; + } + + if (linktarget && strlen(linktarget) > 99) + { + return TAR_SYMLINK_TOO_LONG; + } + + memset(h, 0, TAR_BLOCK_SIZE); + + /* Name 100 */ + strlcpy(&h[TAR_OFFSET_NAME], filename, 100); + if (linktarget != NULL || S_ISDIR(mode)) + { + /* + * We only support symbolic links to directories, and this is + * indicated in the tar format by adding a slash at the end of the + * name, the same as for regular directories. + */ + int flen = strlen(filename); + + flen = Min(flen, 99); + h[flen] = '/'; + h[flen + 1] = '\0'; + } + + /* Mode 8 - this doesn't include the file type bits (S_IFMT) */ + print_tar_number(&h[TAR_OFFSET_MODE], 8, (mode & 07777)); + + /* User ID 8 */ + print_tar_number(&h[TAR_OFFSET_UID], 8, uid); + + /* Group 8 */ + print_tar_number(&h[TAR_OFFSET_GID], 8, gid); + + /* File size 12 */ + if (linktarget != NULL || S_ISDIR(mode)) + { + /* Symbolic link or directory has size zero */ + print_tar_number(&h[TAR_OFFSET_SIZE], 12, 0); + } + else + { + print_tar_number(&h[TAR_OFFSET_SIZE], 12, size); + } + + /* Mod Time 12 */ + print_tar_number(&h[TAR_OFFSET_MTIME], 12, mtime); + + /* Checksum 8 cannot be calculated until we've filled all other fields */ + + if (linktarget != NULL) + { + /* Type - Symbolic link */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_SYMLINK; + + /* Link Name 100 */ + strlcpy(&h[TAR_OFFSET_LINKNAME], linktarget, 100); + } + else if (S_ISDIR(mode)) + { + /* Type - directory */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_DIRECTORY; + } + else + { + /* Type - regular file */ + h[TAR_OFFSET_TYPEFLAG] = TAR_FILETYPE_PLAIN; + } + + /* Magic 6 */ + strcpy(&h[TAR_OFFSET_MAGIC], "ustar"); + + /* Version 2 */ + memcpy(&h[TAR_OFFSET_VERSION], "00", 2); + + /* User 32 */ + /* XXX: Do we need to care about setting correct username? */ + strlcpy(&h[TAR_OFFSET_UNAME], "postgres", 32); + + /* Group 32 */ + /* XXX: Do we need to care about setting correct group name? */ + strlcpy(&h[TAR_OFFSET_GNAME], "postgres", 32); + + /* Major Dev 8 */ + print_tar_number(&h[TAR_OFFSET_DEVMAJOR], 8, 0); + + /* Minor Dev 8 */ + print_tar_number(&h[TAR_OFFSET_DEVMINOR], 8, 0); + + /* Prefix 155 - not used, leave as nulls */ + + /* Finally, compute and insert the checksum */ + print_tar_number(&h[TAR_OFFSET_CHECKSUM], 8, tarChecksum(h)); + + return TAR_OK; +} diff --git a/src/bin/pg_walsender/walsender.h b/src/bin/pg_walsender/walsender.h new file mode 100644 index 000000000..d7cd5dd50 --- /dev/null +++ b/src/bin/pg_walsender/walsender.h @@ -0,0 +1,44 @@ +/* + * src/bin/pg_walsender/walsender.h + * Shared types for pg_walsender, the archiver's own replication-protocol + * server (see ~/dev/temp/archiving-disaster-recovery.md, "Process model" + * and "Build order" milestone 2). Reimplements the wire-level surface of + * the real Postgres walsender well enough to serve IDENTIFY_SYSTEM, SHOW, + * and (later milestones) BASE_BACKUP/START_REPLICATION/TIMELINE_HISTORY + * to unmodified pg_basebackup/pg_receivewal clients, backed by an + * archiver's local WAL cache and base backups instead of a live + * postmaster. No frontend-linkable server-side protocol library exists + * anywhere in Postgres (confirmed against + * /Users/dim/dev/PostgreSQL/postgresql's pqcomm.c/backend_startup.c/ + * repl_gram.y/walsender.c, all backend-only) -- this is a genuine + * reimplementation guided by that source, not a linking exercise. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_WALSENDER_H +#define WS_WALSENDER_H + +#include + +#include "postgres_fe.h" + +/* one entry per "/" the archiver serves, see routes.h */ +typedef struct WsRoute WsRoute; + +/* + * Parsed StartupMessage contents we care about. "database" doubles as our + * routing key ("/", see the design doc's own worked + * process-title example, "pg_autoctl: walsender default/0"). + */ +typedef struct WsStartupParams +{ + char user[NAMEDATALEN]; + char database[NAMEDATALEN + 16]; /* "/", may exceed a bare NAMEDATALEN */ + char applicationName[NAMEDATALEN]; + bool replication; +} WsStartupParams; + +#endif /* WS_WALSENDER_H */ From 454ea8147675d0e035298affa9f9a52af19d1447 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 13:24:16 +0200 Subject: [PATCH 07/55] pg_autoctl: wire archiver serve to pg_walsender (M2 continued) `pg_autoctl archiver serve` (cli_archiver.c) is the supervisor verb that execs pg_walsender as a persistent child (service_archiver_serve.c), mirroring exactly how service_archiver.c already execs real pg_receivewal for the outbound WAL-capture direction -- same pattern, new direction. Keeps pg_walsender's routes file ("/" -> { walcache, basebackup }) current, refreshed periodically and on SIGHUP. The routes file is built from *local* config (formation/groupId/pgSetup. pgdata), not a monitor round-trip: archiver_add_formation()'s own SQL inserts the new archiver_node row's pgdata as an empty string, since the monitor has no way to know an archiver's local WAL cache path -- that's inherently archiver-host-local information. The one genuinely monitor- tracked piece is the latest base backup's storage location (monitor_get_latest_basebackup_location, new in monitor.c). KeeperConfig gains archiverId/archiverIdStr (keeper_config.h/.c) so a later, separate `archiver serve` invocation can identify itself to the monitor -- ini_file.c's INI_INT_T is a plain int, too narrow for a bigserial id, so this follows citusRoleStr/citusRole's existing string- plus-parsed-value pattern in the same struct. Verified against a real, freshly-created cluster (create monitor -> create postgres -> create archiver -> archiver serve): archiverId persists and round-trips correctly, the routes file is generated correctly from live monitor state, pg_walsender starts and serves real clients through it, and SIGTERM shuts the whole thing down cleanly. --- src/bin/pg_autoctl/cli_archiver.c | 245 +++++++++++++++ src/bin/pg_autoctl/cli_archiver.h | 23 ++ src/bin/pg_autoctl/cli_create_node.c | 11 + src/bin/pg_autoctl/cli_root.c | 2 + src/bin/pg_autoctl/defaults.h | 4 + src/bin/pg_autoctl/keeper_config.c | 18 ++ src/bin/pg_autoctl/keeper_config.h | 14 + src/bin/pg_autoctl/monitor.c | 67 ++++ src/bin/pg_autoctl/monitor.h | 4 + src/bin/pg_autoctl/service_archiver_serve.c | 322 ++++++++++++++++++++ src/bin/pg_autoctl/service_archiver_serve.h | 31 ++ 11 files changed, 741 insertions(+) create mode 100644 src/bin/pg_autoctl/cli_archiver.c create mode 100644 src/bin/pg_autoctl/cli_archiver.h create mode 100644 src/bin/pg_autoctl/service_archiver_serve.c create mode 100644 src/bin/pg_autoctl/service_archiver_serve.h diff --git a/src/bin/pg_autoctl/cli_archiver.c b/src/bin/pg_autoctl/cli_archiver.c new file mode 100644 index 000000000..638673ba4 --- /dev/null +++ b/src/bin/pg_autoctl/cli_archiver.c @@ -0,0 +1,245 @@ +/* + * src/bin/pg_autoctl/cli_archiver.c + * See cli_archiver.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cli_archiver.h" +#include "cli_common.h" +#include "commandline.h" +#include "defaults.h" +#include "file_utils.h" +#include "keeper.h" +#include "keeper_config.h" +#include "log.h" +#include "monitor.h" +#include "pidfile.h" +#include "service_archiver_serve.h" +#include "signals.h" +#include "string_utils.h" + +static int cli_archiver_serve_getopts(int argc, char **argv); +static void cli_archiver_serve(int argc, char **argv); + +/* set by --port; 0 means "use PG_AUTOCTL_ARCHIVER_SERVE_PORT" */ +static int archiverServePortOption = 0; + + +static int +cli_archiver_serve_getopts(int argc, char **argv) +{ + KeeperConfig options = { 0 }; + int c, option_index = 0; + int verboseCount = 0; + + static struct option long_options[] = { + { "pgdata", required_argument, NULL, 'D' }, + { "port", required_argument, NULL, 'p' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "D:p:Vvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'D': + { + strlcpy(options.pgSetup.pgdata, optarg, MAXPGPATH); + log_trace("--pgdata %s", options.pgSetup.pgdata); + break; + } + + case 'p': + { + if (!stringToInt(optarg, &archiverServePortOption) || + archiverServePortOption <= 0 || + archiverServePortOption > 65535) + { + log_fatal("Failed to parse --port value \"%s\"", optarg); + exit(EXIT_CODE_BAD_ARGS); + } + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + break; + } + + case 'v': + { + ++verboseCount; + switch (verboseCount) + { + case 1: + { + log_set_level(LOG_INFO); + break; + } + + case 2: + { + log_set_level(LOG_DEBUG); + break; + } + + default: + { + log_set_level(LOG_TRACE); + break; + } + } + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + break; + } + + default: + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + break; + } + } + } + + (void) prepare_keeper_options(&options); + + keeperOptions = options; + + return optind; +} + + +/* + * cli_archiver_serve implements `pg_autoctl archiver serve`: loads the + * archiver's own config/state (already written by `pg_autoctl create + * archiver`), connects to the monitor, and runs + * service_archiver_serve_loop() -- exec'ing pg_walsender and keeping its + * routes file current. See service_archiver_serve.h. + */ +static void +cli_archiver_serve(int argc, char **argv) +{ + Keeper keeper = { 0 }; + + keeper.config = keeperOptions; + + /* + * An archiver's pgdata is its local WAL-cache root, never a real + * Postgres instance (see service_archiver.c's own header comment) -- + * both flags must tolerate that, matching cli_create_archiver's own + * choice not to run pg_setup_init's real-instance checks at all. + */ + bool missingPgdataIsOk = true; + bool pgIsNotRunningIsOk = true; + bool monitorDisabledIsOk = false; + + if (!keeper_config_read_file(&(keeper.config), + missingPgdataIsOk, + pgIsNotRunningIsOk, + monitorDisabledIsOk)) + { + log_fatal("Failed to read the archiver configuration file \"%s\", " + "see above for details", keeper.config.pathnames.config); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (strcmp(keeper.config.nodeKind, "archiver") != 0) + { + log_fatal("\"%s\" is not an archiver's configuration file " + "(pg_autoctl.nodekind is \"%s\", expected \"archiver\")", + keeper.config.pathnames.config, keeper.config.nodeKind); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (keeper.config.archiverId <= 0) + { + log_fatal("This archiver's configuration file has no archiver_id " + "recorded -- it may predate `pg_autoctl archiver serve` " + "support; re-create the archiver with `pg_autoctl create " + "archiver` to pick it up"); + exit(EXIT_CODE_BAD_CONFIG); + } + + if (!keeper_load_state(&keeper)) + { + log_fatal("Failed to read the archiver state file \"%s\", " + "see above for details", keeper.config.pathnames.state); + exit(EXIT_CODE_BAD_STATE); + } + + if (!monitor_init(&(keeper.monitor), keeper.config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (archiverServePortOption > 0) + { + service_archiver_serve_set_port(archiverServePortOption); + } + + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver serve"); + + if (!create_pidfile(keeper.config.pathnames.pid, getpid())) + { + log_fatal("Failed to write archiver pid file \"%s\"", + keeper.config.pathnames.pid); + exit(EXIT_CODE_BAD_STATE); + } + + if (!service_archiver_serve_loop(&keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } +} + + +CommandLine archiver_serve_command = + make_command( + "serve", + "Start serving this archiver's captured WAL and base backups", + " [ --pgdata --port ] ", + " --pgdata path to the archiver's local data/cache directory\n" + " --port port for pg_walsender to listen on " + "(default: 6543)\n", + cli_archiver_serve_getopts, + cli_archiver_serve); + +CommandLine *archiver_subcommands[] = { + &archiver_serve_command, + NULL +}; + +CommandLine archiver_commands = + make_command_set("archiver", + "Manage a pg_auto_failover archiver node", NULL, NULL, + NULL, archiver_subcommands); diff --git a/src/bin/pg_autoctl/cli_archiver.h b/src/bin/pg_autoctl/cli_archiver.h new file mode 100644 index 000000000..0db95b09c --- /dev/null +++ b/src/bin/pg_autoctl/cli_archiver.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_autoctl/cli_archiver.h + * pg_autoctl archiver -- the archiver's own command group. Only `serve` + * is implemented this milestone; the other CLI-reference subverbs + * (add-storage, remove-storage, backup, prefetch, ...) belong to later + * milestones and stay unregistered until then, per + * ~/dev/temp/archiving-disaster-recovery.md's Build order. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef CLI_ARCHIVER_H +#define CLI_ARCHIVER_H + +#include "commandline.h" + +extern CommandLine archiver_serve_command; +extern CommandLine *archiver_subcommands[]; +extern CommandLine archiver_commands; + +#endif /* CLI_ARCHIVER_H */ diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 0578a1f21..82449eab0 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -1555,6 +1555,17 @@ cli_create_archiver(int argc, char **argv) config->network_partition_timeout = NETWORK_PARTITION_TIMEOUT; config->listen_notifications_timeout = PG_AUTOCTL_LISTEN_NOTIFICATIONS_TIMEOUT; + /* + * Persist the archiver's own archiverid (distinct from archiverNodeId + * below, which is this specific ARCHIVING membership's nodeid) so that + * `pg_autoctl archiver serve`'s supervisor loop can identify itself to + * the monitor on a later, separate invocation -- see keeper_config.h's + * own comment on archiverIdStr/archiverId. + */ + config->archiverId = archiverId; + sformat(config->archiverIdStr, sizeof(config->archiverIdStr), + "%" PRId64, archiverId); + if (!keeper_config_write_file(config)) { log_fatal("Failed to write archiver configuration file \"%s\", " diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 51f0ef3a7..242597597 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -8,6 +8,7 @@ * */ +#include "cli_archiver.h" #include "cli_common.h" #include "cli_do_root.h" #include "cli_inspect.h" @@ -109,6 +110,7 @@ CommandLine *root_subcommands[] = { &internal_commands, &do_compat_commands, &node_commands, + &archiver_commands, &service_run_command, &watch_command, &service_stop_command, diff --git a/src/bin/pg_autoctl/defaults.h b/src/bin/pg_autoctl/defaults.h index 80448667b..6797df257 100644 --- a/src/bin/pg_autoctl/defaults.h +++ b/src/bin/pg_autoctl/defaults.h @@ -226,6 +226,10 @@ #define PG_AUTOCTL_HEALTH_PASSWORD "pgautofailover_monitor" #define PG_AUTOCTL_REPLICA_USERNAME "pgautofailover_replicator" +/* default port pg_walsender listens on, started via `pg_autoctl archiver + * serve` -- matches src/bin/pg_walsender/defaults.h's own WS_DEFAULT_PORT */ +#define PG_AUTOCTL_ARCHIVER_SERVE_PORT 6543 + #define PG_AUTOCTL_MONITOR_DBNAME "pg_auto_failover" #define PG_AUTOCTL_MONITOR_EXTENSION_NAME "pgautofailover" #define PG_AUTOCTL_MONITOR_DBOWNER "autoctl" diff --git a/src/bin/pg_autoctl/keeper_config.c b/src/bin/pg_autoctl/keeper_config.c index 7621ff530..9b97aed69 100644 --- a/src/bin/pg_autoctl/keeper_config.c +++ b/src/bin/pg_autoctl/keeper_config.c @@ -61,6 +61,11 @@ make_strbuf_option("pg_autoctl", "nodekind", NULL, false, NAMEDATALEN, \ config->nodeKind) +#define OPTION_AUTOCTL_ARCHIVER_ID(config) \ + make_strbuf_option_default("pg_autoctl", "archiver_id", NULL, false, \ + INTSTRING_MAX_DIGITS, \ + config->archiverIdStr, "") + #define OPTION_POSTGRESQL_PGDATA(config) \ make_strbuf_option("postgresql", "pgdata", "pgdata", true, MAXPGPATH, \ config->pgSetup.pgdata) @@ -227,6 +232,7 @@ OPTION_AUTOCTL_HOSTNAME(config), \ OPTION_AUTOCTL_NODENAME(config), \ OPTION_AUTOCTL_NODEKIND(config), \ + OPTION_AUTOCTL_ARCHIVER_ID(config), \ OPTION_POSTGRESQL_PGDATA(config), \ OPTION_POSTGRESQL_PG_CTL(config), \ OPTION_POSTGRESQL_USERNAME(config), \ @@ -517,6 +523,18 @@ keeper_config_read_file_skip_pgsetup(KeeperConfig *config, return false; } + /* parse archiverIdStr (see keeper_config.h's own comment) into archiverId */ + if (IS_EMPTY_STRING_BUFFER(config->archiverIdStr)) + { + config->archiverId = 0; + } + else if (!stringToInt64(config->archiverIdStr, &(config->archiverId))) + { + log_error("Failed to parse pg_autoctl.archiver_id \"%s\" as a number", + config->archiverIdStr); + return false; + } + return true; } diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 0c71a65e0..04eb3d612 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -17,6 +17,7 @@ #include "defaults.h" #include "pgctl.h" #include "pgsql.h" +#include "string_utils.h" /* * We support "primary" and "secondary" roles in Citus, when Citus support is @@ -47,6 +48,19 @@ typedef struct KeeperConfig char hostname[_POSIX_HOST_NAME_MAX]; char nodeKind[NAMEDATALEN]; + /* + * The archiver's own archiverid (distinct from keeper.state. + * current_node_id, which holds the ARCHIVING membership row's nodeid -- + * see cli_create_archiver's own comment). Only meaningful when nodeKind + * is "archiver"; 0 otherwise. archiverIdStr is the ini-persisted form + * (ini_file.c's INI_INT_T only supports a plain int, too narrow for a + * bigserial id -- same string-plus-parsed-value pattern citusRoleStr/ + * citusRole already use in this struct), archiverId is parsed from it + * once at config-read time. + */ + char archiverIdStr[INTSTRING_MAX_DIGITS]; + int64_t archiverId; + /* PostgreSQL setup */ PostgresSetup pgSetup; diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index e0adca208..194941a31 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -962,6 +962,73 @@ monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, } +/* + * monitor_get_latest_basebackup_location calls + * pgautofailover.get_latest_basebackup(formationId, groupId) and returns + * its storagelocation column. *found is set to false (not an error) when + * the archiver hasn't taken a base backup for this group yet -- the "Base + * backup generation" milestone this depends on hasn't landed, so every + * caller of this function must already tolerate that. + */ +bool +monitor_get_latest_basebackup_location(Monitor *monitor, + const char *formationId, int groupId, + char *storageLocation, size_t size, + bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + + /* + * get_latest_basebackup() is not SETOF: called with no matching + * backup, it still produces one row, with every output column + * (including storagelocation) NULL -- not zero rows. Filtering on + * "IS NOT NULL" here, rather than trying to detect that NULL + * composite downstream, is what makes context.ntuples == 0 below + * an accurate "no backup yet" signal. + */ + "SELECT storagelocation " + " FROM pgautofailover.get_latest_basebackup($1, $2) " + " WHERE storagelocation IS NOT NULL"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_STRING, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to get the latest base backup location from the " + "monitor for \"%s\"/%d", formationId, groupId); + return false; + } + + if (context.ntuples == 0) + { + /* no base backup taken yet for this group -- not an error */ + return true; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the latest base backup location returned " + "by the monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + strlcpy(storageLocation, context.strVal, size); + free(context.strVal); + *found = true; + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 963dbcbb1..ee5389ae3 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -158,6 +158,10 @@ bool monitor_register_archiver(Monitor *monitor, char *name, char *hostname, int64_t *archiverId); bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, char *formation, int64_t *archiverNodeId); +bool monitor_get_latest_basebackup_location(Monitor *monitor, + const char *formationId, int groupId, + char *storageLocation, size_t size, + bool *found); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c new file mode 100644 index 000000000..0359e1031 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -0,0 +1,322 @@ +/* + * src/bin/pg_autoctl/service_archiver_serve.c + * See service_archiver_serve.h. + * + * Milestone 2's own scope, per the Build order in + * ~/dev/temp/archiving-disaster-recovery.md: pg_walsender is exec'd exactly + * once per archiver process, matching service_archiver.c's own single- + * membership scope for pg_receivewal -- a future milestone generalizing to + * several (formation, group) memberships per archiver needs this file and + * service_archiver.c to grow the same "one child per membership" model + * together, not independently. + * + * The routes file this writes is deliberately built from *local* config + * (config->formation/groupId/pgSetup.pgdata), not a monitor round-trip: + * archiver_add_formation()'s own SQL (pgautofailover.sql) inserts the new + * archiver_node row's pgdata as an empty string -- the monitor has no way + * to know an archiver's local WAL cache path, that's inherently + * archiver-host-local information never sent to it. The one thing genuinely + * worth asking the monitor is the latest base backup's storage location + * (monitor_get_latest_basebackup_location), which is real, monitor-tracked + * state once the "Base backup generation" milestone lands. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include + +#include "service_archiver_serve.h" + +#include "cli_root.h" /* pg_autoctl_program */ +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "signals.h" + +/* how often service_archiver_serve_loop() re-checks pg_walsender's + * liveness and refreshes the routes file, in seconds */ +#define ARCHIVER_SERVE_TICK_SECONDS 1 +#define ARCHIVER_SERVE_ROUTES_REFRESH_TICKS 30 + +/* + * One pg_walsender child per archiver process, matching service_archiver. + * c's own single-membership scope (see this file's own header comment). + */ +static pid_t pgWalsenderPid = -1; +static int archiverServePort = 0; + + +void +service_archiver_serve_set_port(int port) +{ + archiverServePort = port; +} + + +static void +service_archiver_serve_routes_path(KeeperConfig *config, char *dest) +{ + path_in_same_directory(config->pathnames.config, + "archiver-routes.ini", dest); +} + + +bool +service_archiver_serve_walsender_is_running(void) +{ + if (pgWalsenderPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(pgWalsenderPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == pgWalsenderPid) + { + log_warn("pg_walsender (pid %d) exited", pgWalsenderPid); + } + else if (ret == -1 && errno != ECHILD) + { + log_warn("Failed to waitpid() on pg_walsender (pid %d): %m", pgWalsenderPid); + } + + pgWalsenderPid = -1; + return false; +} + + +bool +service_archiver_serve_stop_walsender(void) +{ + if (pgWalsenderPid <= 0) + { + return true; + } + + log_info("Stopping pg_walsender (pid %d)", pgWalsenderPid); + + if (kill(pgWalsenderPid, SIGTERM) != 0 && errno != ESRCH) + { + log_error("Failed to send SIGTERM to pg_walsender (pid %d): %m", + pgWalsenderPid); + return false; + } + + int status = 0; + + if (waitpid(pgWalsenderPid, &status, 0) == -1 && errno != ECHILD) + { + log_error("Failed to waitpid() on pg_walsender (pid %d): %m", + pgWalsenderPid); + pgWalsenderPid = -1; + return false; + } + + pgWalsenderPid = -1; + return true; +} + + +bool +service_archiver_serve_start_walsender(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + + if (!service_archiver_serve_stop_walsender()) + { + /* errors have already been logged */ + return false; + } + + char pgWalsenderPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(pg_autoctl_program, "pg_walsender", pgWalsenderPath); + + if (!file_exists(pgWalsenderPath)) + { + log_error("Failed to find pg_walsender at \"%s\"", pgWalsenderPath); + return false; + } + + char routesPath[MAXPGPATH] = { 0 }; + + service_archiver_serve_routes_path(config, routesPath); + + int port = archiverServePort > 0 ? archiverServePort : PG_AUTOCTL_ARCHIVER_SERVE_PORT; + char portStr[16] = { 0 }; + + sformat(portStr, sizeof(portStr), "%d", port); + + log_info("Starting pg_walsender on port %d, routes \"%s\"", port, routesPath); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_walsender: %m"); + return false; + } + + if (pid == 0) + { + /* child process: replace ourselves with pg_walsender */ + char *args[6]; + int argsIndex = 0; + + args[argsIndex++] = pgWalsenderPath; + args[argsIndex++] = "--port"; + args[argsIndex++] = portStr; + args[argsIndex++] = "--routes"; + args[argsIndex++] = routesPath; + args[argsIndex] = NULL; + + execv(pgWalsenderPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgWalsenderPath); + _exit(127); + } + + /* parent process: track the child, keep running our own loop */ + pgWalsenderPid = pid; + + return true; +} + + +bool +service_archiver_serve_refresh_routes(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + + char basebackupLocation[MAXPGPATH] = { 0 }; + bool found = false; + + if (!monitor_get_latest_basebackup_location(&(keeper->monitor), + config->formation, + config->groupId, + basebackupLocation, + sizeof(basebackupLocation), + &found)) + { + log_warn("Failed to fetch the latest base backup location from the " + "monitor; the routes file will omit it for now"); + found = false; + } + + char routesPath[MAXPGPATH] = { 0 }; + + service_archiver_serve_routes_path(config, routesPath); + + char tmpPath[MAXPGPATH] = { 0 }; + + sformat(tmpPath, sizeof(tmpPath), "%s.tmp", routesPath); + + FILE *fileStream = fopen_with_umask(tmpPath, "w", FOPEN_FLAGS_W, 0644); + + if (fileStream == NULL) + { + /* errors have already been logged */ + return false; + } + + fformat(fileStream, "[%s/%d]\n", config->formation, config->groupId); + fformat(fileStream, "walcache = %s\n", config->pgSetup.pgdata); + + if (found) + { + fformat(fileStream, "basebackup = %s\n", basebackupLocation); + } + + if (fclose(fileStream) == EOF) + { + log_error("Failed to write file \"%s\": %m", tmpPath); + return false; + } + + if (rename(tmpPath, routesPath) != 0) + { + log_error("Failed to rename \"%s\" to \"%s\": %m", tmpPath, routesPath); + return false; + } + + log_debug("Refreshed archiver routes file \"%s\"", routesPath); + + return true; +} + + +bool +service_archiver_serve_loop(Keeper *keeper) +{ + log_info("pg_autoctl archiver serve: archiver %" PRId64 ", formation " + "\"%s\", group %d", + keeper->config.archiverId, keeper->config.formation, + keeper->config.groupId); + + if (!service_archiver_serve_refresh_routes(keeper)) + { + log_warn("Failed to write the initial routes file; pg_walsender " + "will start without one route resolved yet"); + } + + if (!service_archiver_serve_start_walsender(keeper)) + { + log_fatal("Failed to start pg_walsender, see above for details"); + return false; + } + + int tickCount = 0; + + for (;;) + { + if (asked_to_stop || asked_to_stop_fast || asked_to_quit) + { + break; + } + + if (asked_to_reload) + { + asked_to_reload = 0; + (void) service_archiver_serve_refresh_routes(keeper); + } + + if (!service_archiver_serve_walsender_is_running()) + { + log_warn("pg_walsender is not running anymore, restarting it"); + + if (!service_archiver_serve_start_walsender(keeper)) + { + log_error("Failed to restart pg_walsender, will retry on " + "the next tick"); + } + } + + if (tickCount > 0 && + tickCount % ARCHIVER_SERVE_ROUTES_REFRESH_TICKS == 0) + { + (void) service_archiver_serve_refresh_routes(keeper); + } + + sleep(ARCHIVER_SERVE_TICK_SECONDS); + ++tickCount; + } + + (void) service_archiver_serve_stop_walsender(); + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver_serve.h b/src/bin/pg_autoctl/service_archiver_serve.h new file mode 100644 index 000000000..7f801745a --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_serve.h @@ -0,0 +1,31 @@ +/* + * src/bin/pg_autoctl/service_archiver_serve.h + * Archiving & Disaster Recovery: supervision of the pg_walsender child + * process an archiver runs to serve its captured WAL and base backups to + * downstream consumers (warm standbies, PITR nodes, `create postgres + * --from-archiver` rebuilds) -- the inbound counterpart to + * service_archiver.c's outbound pg_receivewal supervision. See + * ~/dev/temp/archiving-disaster-recovery.md and + * src/bin/pg_walsender/walsender.h for the protocol this serves. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_SERVE_H +#define SERVICE_ARCHIVER_SERVE_H + +#include "keeper.h" + +void service_archiver_serve_set_port(int port); + +bool service_archiver_serve_start_walsender(Keeper *keeper); +bool service_archiver_serve_stop_walsender(void); +bool service_archiver_serve_walsender_is_running(void); + +bool service_archiver_serve_refresh_routes(Keeper *keeper); + +bool service_archiver_serve_loop(Keeper *keeper); + +#endif /* SERVICE_ARCHIVER_SERVE_H */ From 796e0e86b91baae4cc39b603685d132688f13b7d Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 13:41:34 +0200 Subject: [PATCH 08/55] pg_walsender: TIMELINE_HISTORY, START_REPLICATION, replication slots (M2c) Completes milestone 2's command surface: - TIMELINE_HISTORY : serves a ".history" file straight out of the WAL cache directory (RowDescription/DataRow, no COPY involved -- traced from walsender.c's own SendTimeLineHistory()). - CREATE_REPLICATION_SLOT / READ_REPLICATION_SLOT (physical only): a slot is a bookkeeping marker file under the WAL cache directory, not a real Postgres slot -- there's no live server to hold one. Not yet wired into WAL-retention enforcement (prune_archiver_wal()'s job). - START_REPLICATION [SLOT ] TIMELINE : streams raw WAL bytes straight from the WAL cache directory. Deliberately does NOT vendor xlogreader.c: real walsender's own WalSndSegmentOpen just opens a path computed from TLI+segno and streams bytes -- no WAL *record* decoding is needed to serve a byte range, only the offset/segment bookkeeping this file does directly. Handles the actively-growing (".partial") segment case by polling, matching pg_receivewal's own producer on the other end of this same protocol. - wal_dir_scan.c: shared helper -- finds the newest fully-captured WAL segment and derives its boundary LSN from the filename (XLogFileName format, fixed 16MB segments). Used by START_REPLICATION's default position, CREATE_REPLICATION_SLOT's consistent_point, and improves IDENTIFY_SYSTEM's xlogpos (previously a "0/0" placeholder). One correctness fix alongside: IDENTIFY_SYSTEM's dbname column must be NULL for a plain replication=1/true connection (pg_receivewal's style) -- only replication=database (pg_basebackup's style) gets a real dbname back. Always returning a value broke real pg_receivewal outright ("replication connection using slot ... is unexpectedly database specific"), caught by this milestone's own end-to-end testing, not by any narrower unit check. Verified against real, unmodified PostgreSQL client tools: - psql: TIMELINE_HISTORY round-trips real file content - psql: CREATE_REPLICATION_SLOT / READ_REPLICATION_SLOT round-trip consistent_point/restart_lsn correctly, including the "slot doesn't exist" all-NULL-row case - pg_receivewal -S --endpos=...: streamed a full 16MB WAL segment byte-identical to the source via START_REPLICATION --- src/bin/pg_walsender/Makefile | 4 +- src/bin/pg_walsender/accept_loop.c | 3 +- src/bin/pg_walsender/cmd_identify_system.c | 26 +- src/bin/pg_walsender/cmd_replication_slot.c | 297 +++++++++++++++ src/bin/pg_walsender/cmd_replication_slot.h | 23 ++ src/bin/pg_walsender/cmd_start_replication.c | 371 +++++++++++++++++++ src/bin/pg_walsender/cmd_start_replication.h | 26 ++ src/bin/pg_walsender/cmd_timeline_history.c | 71 ++++ src/bin/pg_walsender/cmd_timeline_history.h | 23 ++ src/bin/pg_walsender/repl_command.c | 64 ++++ src/bin/pg_walsender/repl_command.h | 9 + src/bin/pg_walsender/startup.c | 3 +- src/bin/pg_walsender/wal_dir_scan.c | 113 ++++++ src/bin/pg_walsender/wal_dir_scan.h | 45 +++ src/bin/pg_walsender/walsender.h | 10 + 15 files changed, 1079 insertions(+), 9 deletions(-) create mode 100644 src/bin/pg_walsender/cmd_replication_slot.c create mode 100644 src/bin/pg_walsender/cmd_replication_slot.h create mode 100644 src/bin/pg_walsender/cmd_start_replication.c create mode 100644 src/bin/pg_walsender/cmd_start_replication.h create mode 100644 src/bin/pg_walsender/cmd_timeline_history.c create mode 100644 src/bin/pg_walsender/cmd_timeline_history.h create mode 100644 src/bin/pg_walsender/wal_dir_scan.c create mode 100644 src/bin/pg_walsender/wal_dir_scan.h diff --git a/src/bin/pg_walsender/Makefile b/src/bin/pg_walsender/Makefile index 669532c49..79598500d 100644 --- a/src/bin/pg_walsender/Makefile +++ b/src/bin/pg_walsender/Makefile @@ -24,7 +24,9 @@ override CFLAGS += -I$(SRC_DIR) -I$(SRC_DIR)vendor # ----------------------------------------------------------------------- LOCAL_SRC = main.c accept_loop.c startup.c auth.c framing.c repl_command.c \ routes.c cmd_identify_system.c cmd_show.c cmd_base_backup.c \ - tar_stream.c cmd_fetch_file.c fetch_client.c + tar_stream.c cmd_fetch_file.c fetch_client.c \ + cmd_timeline_history.c cmd_replication_slot.c \ + cmd_start_replication.c wal_dir_scan.c LOCAL_OBJS = $(patsubst %.c,%.o,$(LOCAL_SRC)) diff --git a/src/bin/pg_walsender/accept_loop.c b/src/bin/pg_walsender/accept_loop.c index 4ffae004b..d753bfb5c 100644 --- a/src/bin/pg_walsender/accept_loop.c +++ b/src/bin/pg_walsender/accept_loop.c @@ -188,7 +188,8 @@ handle_connection(int clientSock, const WsServerConfig *config) } else { - ws_dispatch_command(clientSock, &cmd, route, params.database); + ws_dispatch_command(clientSock, &cmd, route, + params.replicationDatabase ? params.database : NULL); } free(payload); diff --git a/src/bin/pg_walsender/cmd_identify_system.c b/src/bin/pg_walsender/cmd_identify_system.c index 8064d32bb..425f208d0 100644 --- a/src/bin/pg_walsender/cmd_identify_system.c +++ b/src/bin/pg_walsender/cmd_identify_system.c @@ -2,12 +2,13 @@ * src/bin/pg_walsender/cmd_identify_system.c * See cmd_identify_system.h. * - * systemid/timeline come straight from the route (written by pg_autoctl's + * systemid comes straight from the route (written by pg_autoctl's * archiver-serve supervisor from the monitor's own tracked values -- see - * routes.h). xlogpos is reported as "0/0" for now: computing the real - * latest-captured position requires scanning the WAL cache directory, - * which is wired in alongside START_REPLICATION (milestone 2 step 5), - * not required for the protocol handshake itself to be correct. + * routes.h). timeline/xlogpos prefer the newest fully-captured WAL + * segment's own boundary (wal_dir_scan.h, filename-derived, not a + * parsed WAL record position) when the WAL cache has one, falling back + * to the route's static timeline and "0/0" when it doesn't (a brand + * new archiver with nothing captured yet). * * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the PostgreSQL License. @@ -20,6 +21,7 @@ #include "cmd_identify_system.h" #include "framing.h" +#include "wal_dir_scan.h" void @@ -33,17 +35,29 @@ cmd_identify_system(int sock, const WsRoute *route, const char *dbname) }; char timelineStr[16]; + char xlogpos[32] = "0/0"; const char *systemId = (route != NULL && route->systemId[0] != '\0') ? route->systemId : "0"; int timeline = (route != NULL && route->timeline > 0) ? route->timeline : 1; + if (route != NULL && route->walcacheDir[0] != '\0') + { + uint32_t foundTimeline; + + if (wal_dir_find_latest(route->walcacheDir, &foundTimeline, + xlogpos, sizeof(xlogpos))) + { + timeline = (int) foundTimeline; + } + } + snprintf(timelineStr, sizeof(timelineStr), "%d", timeline); const char *values[] = { systemId, timelineStr, - "0/0", + xlogpos, dbname, }; diff --git a/src/bin/pg_walsender/cmd_replication_slot.c b/src/bin/pg_walsender/cmd_replication_slot.c new file mode 100644 index 000000000..9d0661a44 --- /dev/null +++ b/src/bin/pg_walsender/cmd_replication_slot.c @@ -0,0 +1,297 @@ +/* + * src/bin/pg_walsender/cmd_replication_slot.c + * See cmd_replication_slot.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "cmd_replication_slot.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "wal_dir_scan.h" + +#define WS_SLOT_NAME_MAX 64 + + +/* + * parse_slot_name reads a possibly-quoted identifier (matching real + * Postgres's AppendQuotedIdentifier on the client side -- unquoted for a + * simple lowercase name, double-quoted otherwise) from the front of *p, + * advancing *p past it. + */ +static bool +parse_slot_name(const char **p, char *nameOut, size_t nameOutSize) +{ + const char *s = *p; + + while (isspace((unsigned char) *s)) + { + s++; + } + + if (*s == '"') + { + s++; + + char *out = nameOut; + char *outEnd = nameOut + nameOutSize - 1; + + while (*s && *s != '"') + { + if (out < outEnd) + { + *out++ = *s; + } + s++; + } + + if (*s != '"') + { + return false; + } + + *out = '\0'; + s++; + } + else + { + const char *start = s; + + while (*s && !isspace((unsigned char) *s)) + { + s++; + } + + size_t len = Min((size_t) (s - start), nameOutSize - 1); + + memcpy(nameOut, start, len); + nameOut[len] = '\0'; + } + + *p = s; + + return nameOut[0] != '\0'; +} + + +static bool +slot_name_is_safe(const char *name) +{ + if (name[0] == '\0') + { + return false; + } + + for (const char *p = name; *p; p++) + { + if (!(isalnum((unsigned char) *p) || *p == '_' || *p == '-')) + { + return false; + } + } + + return true; +} + + +static void +slot_marker_path(const WsRoute *route, const char *slotName, char *dest, size_t destSize) +{ + snprintf(dest, destSize, "%s/.slot_%s", route->walcacheDir, slotName); +} + + +void +cmd_create_replication_slot(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = rawArgs; + char slotName[WS_SLOT_NAME_MAX]; + + if (!parse_slot_name(&p, slotName, sizeof(slotName)) || !slot_name_is_safe(slotName)) + { + ws_send_error_response(sock, "22023", "invalid or missing slot name"); + return; + } + + bool sawPhysical = false; + bool sawLogical = false; + char word[64]; + + while (*p) + { + while (*p && (isspace((unsigned char) *p) || *p == ',' || *p == '(' || *p == ')')) + { + p++; + } + + if (!*p) + { + break; + } + + const char *start = p; + + while (*p && !isspace((unsigned char) *p) && *p != ',' && + *p != '(' && *p != ')') + { + p++; + } + + size_t len = Min((size_t) (p - start), sizeof(word) - 1); + + memcpy(word, start, len); + word[len] = '\0'; + + if (strcasecmp(word, "PHYSICAL") == 0) + { + sawPhysical = true; + } + else if (strcasecmp(word, "LOGICAL") == 0) + { + sawLogical = true; + } + + /* TEMPORARY and RESERVE_WAL are accepted but not enforced yet -- + * see this file's own header comment on retention */ + } + + if (sawLogical || !sawPhysical) + { + ws_send_error_response(sock, "0A000", + "only physical replication slots are supported"); + return; + } + + char consistentPoint[32] = "0/0"; + uint32_t timeline; + + (void) wal_dir_find_latest(route->walcacheDir, &timeline, consistentPoint, + sizeof(consistentPoint)); + + char path[MAXPGPATH]; + + slot_marker_path(route, slotName, path, sizeof(path)); + + char contents[128]; + + snprintf(contents, sizeof(contents), "restart_lsn=%s\n", consistentPoint); + + if (!write_file(contents, strlen(contents), path)) + { + log_error("Failed to write replication slot marker \"%s\"", path); + ws_send_error_response(sock, "58030", "failed to persist the replication slot"); + return; + } + + WsColumn columns[] = { + { "slot_name", WS_TEXTOID, -1 }, + { "consistent_point", WS_TEXTOID, -1 }, + { "snapshot_name", WS_TEXTOID, -1 }, + { "output_plugin", WS_TEXTOID, -1 }, + }; + + const char *values[] = { slotName, consistentPoint, NULL, NULL }; + + if (ws_send_row_description(sock, columns, 4) && + ws_send_data_row(sock, values, 4)) + { + ws_send_command_complete(sock, "CREATE_REPLICATION_SLOT"); + } +} + + +void +cmd_read_replication_slot(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = rawArgs; + char slotName[WS_SLOT_NAME_MAX]; + + if (!parse_slot_name(&p, slotName, sizeof(slotName)) || !slot_name_is_safe(slotName)) + { + ws_send_error_response(sock, "22023", "invalid or missing slot name"); + return; + } + + char path[MAXPGPATH]; + + slot_marker_path(route, slotName, path, sizeof(path)); + + char *contents = NULL; + long fileSize = 0; + + WsColumn columns[] = { + { "slot_type", WS_TEXTOID, -1 }, + { "restart_lsn", WS_TEXTOID, -1 }, + { "restart_tli", WS_INT8OID, 8 }, + }; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + /* matches real Postgres: slot doesn't exist -> one all-NULL row, + * not an ErrorResponse -- the client checks PQgetisnull() itself */ + const char *nullValues[] = { NULL, NULL, NULL }; + + if (ws_send_row_description(sock, columns, 3) && + ws_send_data_row(sock, nullValues, 3)) + { + ws_send_command_complete(sock, "READ_REPLICATION_SLOT"); + } + + return; + } + + char restartLsn[32] = "0/0"; + const char *prefix = "restart_lsn="; + char *line = strstr(contents, prefix); + + if (line != NULL) + { + line += strlen(prefix); + + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + strlcpy(restartLsn, line, sizeof(restartLsn)); + } + + free(contents); + + uint32_t timeline = (route->timeline > 0) ? (uint32_t) route->timeline : 1; + char timelineStr[16]; + + snprintf(timelineStr, sizeof(timelineStr), "%u", timeline); + + const char *values[] = { "physical", restartLsn, timelineStr }; + + if (ws_send_row_description(sock, columns, 3) && + ws_send_data_row(sock, values, 3)) + { + ws_send_command_complete(sock, "READ_REPLICATION_SLOT"); + } +} diff --git a/src/bin/pg_walsender/cmd_replication_slot.h b/src/bin/pg_walsender/cmd_replication_slot.h new file mode 100644 index 000000000..f48ddb0a1 --- /dev/null +++ b/src/bin/pg_walsender/cmd_replication_slot.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_walsender/cmd_replication_slot.h + * CREATE_REPLICATION_SLOT / READ_REPLICATION_SLOT, physical slots only + * (matching the design doc's own scope). A slot here is a bookkeeping + * marker file under the route's WAL cache directory -- not a real + * Postgres slot on a live server (there's no live server), and not yet + * wired into any WAL-retention enforcement (that's the prune/retention + * milestone's job, see prune_archiver_wal() in the SQL schema). + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_REPLICATION_SLOT_H +#define WS_CMD_REPLICATION_SLOT_H + +#include "routes.h" + +void cmd_create_replication_slot(int sock, const WsRoute *route, const char *rawArgs); +void cmd_read_replication_slot(int sock, const WsRoute *route, const char *rawArgs); + +#endif /* WS_CMD_REPLICATION_SLOT_H */ diff --git a/src/bin/pg_walsender/cmd_start_replication.c b/src/bin/pg_walsender/cmd_start_replication.c new file mode 100644 index 000000000..da72b621b --- /dev/null +++ b/src/bin/pg_walsender/cmd_start_replication.c @@ -0,0 +1,371 @@ +/* + * src/bin/pg_walsender/cmd_start_replication.c + * See cmd_start_replication.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "port/pg_bswap.h" +#include "pqexpbuffer.h" + +#include "cmd_start_replication.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" +#include "signals.h" +#include "wal_dir_scan.h" + +#define WS_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define WS_STREAM_CHUNK_SIZE (32 * 1024) +#define WS_KEEPALIVE_INTERVAL_SEC 5 +#define WS_POLL_INTERVAL_USEC (200 * 1000) + + +static void +append_int64(PQExpBuffer buf, int64_t v) +{ + uint64_t n = pg_hton64((uint64_t) v); + + appendBinaryPQExpBuffer(buf, (const char *) &n, 8); +} + + +static bool +send_xlogdata(int sock, uint64_t dataStart, uint64_t walEnd, + const char *data, size_t len) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'w'); /* PqReplMsg_WALData */ + append_int64(buf, (int64_t) dataStart); + append_int64(buf, (int64_t) walEnd); + append_int64(buf, (int64_t) 0); /* sendTime, not load-bearing here */ + appendBinaryPQExpBuffer(buf, data, len); + + bool ok = !PQExpBufferBroken(buf) && ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +static bool +send_keepalive(int sock, uint64_t walEnd) +{ + PQExpBuffer buf = createPQExpBuffer(); + + appendPQExpBufferChar(buf, 'k'); /* PqReplMsg_Keepalive */ + append_int64(buf, (int64_t) walEnd); + append_int64(buf, (int64_t) 0); /* sendTime */ + appendPQExpBufferChar(buf, 0); /* replyRequested = false */ + + bool ok = !PQExpBufferBroken(buf) && ws_send_copy_data(sock, buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + +/* + * wait_for_more_data_or_client waits up to WS_POLL_INTERVAL_USEC for + * either more WAL bytes to become available or a message from the client, + * draining (and ignoring the content of) any standby status update the + * client sends meanwhile -- this project has no cascading/retention logic + * that needs to react to it yet. Returns false when the client has + * disconnected/terminated or we've been asked to stop, in which case the + * caller should end the stream. + */ +static bool +wait_for_more_data_or_client(int sock, uint64_t currentLsn, time_t *lastKeepalive) +{ + if (asked_to_stop || asked_to_stop_fast) + { + return false; + } + + fd_set readSet; + + FD_ZERO(&readSet); + FD_SET(sock, &readSet); + + struct timeval timeout = { 0, WS_POLL_INTERVAL_USEC }; + + int selectRet = select(sock + 1, &readSet, NULL, NULL, &timeout); + + if (selectRet < 0 && errno != EINTR) + { + return false; + } + + if (selectRet > 0 && FD_ISSET(sock, &readSet)) + { + char type; + char *payload = NULL; + int32_t payloadLen = 0; + + if (!ws_read_message(sock, &type, &payload, &payloadLen)) + { + free(payload); + return false; /* client disconnected */ + } + + free(payload); + + if (type == 'X' || type == 'c') /* Terminate or CopyDone */ + { + return false; + } + + /* 'd' CopyData: a standby status update / hot-standby feedback we + * don't act on yet -- already consumed above, nothing more to do */ + } + + time_t now = time(NULL); + + if (now - *lastKeepalive >= WS_KEEPALIVE_INTERVAL_SEC) + { + if (!send_keepalive(sock, currentLsn)) + { + return false; + } + + *lastKeepalive = now; + } + + return true; +} + + +static bool +parse_lsn(const char *s, uint64_t *lsn, const char **endptr) +{ + char *afterHi; + unsigned long hi = strtoul(s, &afterHi, 16); + + if (afterHi == s || *afterHi != '/') + { + return false; + } + + char *afterLo; + unsigned long lo = strtoul(afterHi + 1, &afterLo, 16); + + if (afterLo == afterHi + 1) + { + return false; + } + + *lsn = ((uint64_t) hi << 32) | (uint32_t) lo; + *endptr = afterLo; + + return true; +} + + +static const char * +skip_ws(const char *p) +{ + while (isspace((unsigned char) *p)) + { + p++; + } + + return p; +} + + +void +cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + const char *p = skip_ws(rawArgs); + + if (strncasecmp(p, "SLOT", 4) == 0 && isspace((unsigned char) p[4])) + { + p = skip_ws(p + 4); + + /* consume a possibly-quoted slot name, positioning is unaffected + * by which slot (if any) was named -- see this file's own header + * comment on why no real slot-based retention exists yet */ + if (*p == '"') + { + p++; + while (*p && *p != '"') + { + p++; + } + if (*p == '"') + { + p++; + } + } + else + { + while (*p && !isspace((unsigned char) *p)) + { + p++; + } + } + + p = skip_ws(p); + } + + if (strncasecmp(p, "PHYSICAL", 8) == 0 && + (isspace((unsigned char) p[8]) || p[8] == '\0')) + { + p = skip_ws(p + 8); + } + + uint64_t startLsn; + const char *after; + + if (!parse_lsn(p, &startLsn, &after)) + { + ws_send_error_response(sock, "22023", "invalid or missing start LSN"); + return; + } + + p = skip_ws(after); + + uint32_t timeline = (route->timeline > 0) ? (uint32_t) route->timeline : 1; + + if (strncasecmp(p, "TIMELINE", 8) == 0) + { + p = skip_ws(p + 8); + timeline = (uint32_t) strtoul(p, NULL, 10); + } + + if (!ws_send_copy_both_response(sock, 0)) + { + return; + } + + log_info("START_REPLICATION: streaming from %X/%08X on timeline %u " + "from \"%s\"", + (uint32_t) (startLsn >> 32), (uint32_t) startLsn, timeline, + route->walcacheDir); + + uint64_t segno = startLsn / WS_WAL_SEGMENT_SIZE; + uint64_t offset = startLsn % WS_WAL_SEGMENT_SIZE; + uint64_t currentLsn = startLsn; + time_t lastKeepalive = time(NULL); + + for (;;) + { + if (asked_to_stop || asked_to_stop_fast) + { + break; + } + + char filename[32]; + + wal_segment_filename(timeline, segno, filename, sizeof(filename)); + + char completePath[MAXPGPATH]; + + snprintf(completePath, sizeof(completePath), "%s/%s", + route->walcacheDir, filename); + + bool isComplete = file_exists(completePath); + + char partialPath[MAXPGPATH]; + + snprintf(partialPath, sizeof(partialPath), "%s.partial", completePath); + + const char *readPath = isComplete ? completePath : partialPath; + + if (!isComplete && !file_exists(partialPath)) + { + /* nothing captured for this segment yet -- wait for it */ + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + FILE *file = fopen(readPath, "rb"); + + if (file == NULL) + { + log_warn("Failed to open \"%s\": %m (will retry)", readPath); + + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + if (fseeko(file, (off_t) offset, SEEK_SET) != 0) + { + log_error("Failed to seek to offset %" PRIu64 " in \"%s\": %m", + offset, readPath); + fclose(file); + break; + } + + char buffer[WS_STREAM_CHUNK_SIZE]; + size_t got = fread(buffer, 1, sizeof(buffer), file); + + fclose(file); + + if (got == 0) + { + if (isComplete) + { + /* fully drained this now-complete segment: move on */ + segno++; + offset = 0; + continue; + } + + if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) + { + break; + } + + continue; + } + + if (!send_xlogdata(sock, currentLsn, currentLsn + got, buffer, got)) + { + break; /* client gone */ + } + + currentLsn += got; + offset += got; + + if (offset >= WS_WAL_SEGMENT_SIZE) + { + segno++; + offset = 0; + } + } + + (void) ws_send_copy_done(sock); + + log_info("START_REPLICATION: stream ended at %X/%08X", + (uint32_t) (currentLsn >> 32), (uint32_t) currentLsn); +} diff --git a/src/bin/pg_walsender/cmd_start_replication.h b/src/bin/pg_walsender/cmd_start_replication.h new file mode 100644 index 000000000..de151d950 --- /dev/null +++ b/src/bin/pg_walsender/cmd_start_replication.h @@ -0,0 +1,26 @@ +/* + * src/bin/pg_walsender/cmd_start_replication.h + * START_REPLICATION [SLOT ] TIMELINE : streams WAL + * bytes straight out of the route's WAL cache directory, physical-only. + * + * Deliberately does NOT vendor xlogreader.c for this: real walsender's + * own WalSndSegmentOpen (walsender.c) just computes a path from TLI+segno + * and opens it -- streaming raw bytes needs no WAL *record* decoding at + * all, only byte-range bookkeeping this file does directly. xlogreader.c + * would only earn its keep here for validating record boundaries, not + * required for a client (a real pg_receivewal) that already does its own + * validation on the bytes it receives. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_START_REPLICATION_H +#define WS_CMD_START_REPLICATION_H + +#include "routes.h" + +void cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs); + +#endif /* WS_CMD_START_REPLICATION_H */ diff --git a/src/bin/pg_walsender/cmd_timeline_history.c b/src/bin/pg_walsender/cmd_timeline_history.c new file mode 100644 index 000000000..b5b0edc25 --- /dev/null +++ b/src/bin/pg_walsender/cmd_timeline_history.c @@ -0,0 +1,71 @@ +/* + * src/bin/pg_walsender/cmd_timeline_history.c + * See cmd_timeline_history.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "postgres_fe.h" + +#include "cmd_timeline_history.h" +#include "file_utils.h" +#include "framing.h" +#include "log.h" + +/* matches xlog_internal.h's own MAXFNAMELEN (backend-only header, not + * pulled in here) -- "%08X.history" is always exactly 17 bytes + NUL */ +#define WS_MAXFNAMELEN 64 + + +void +cmd_timeline_history(int sock, const WsRoute *route, int timeline) +{ + if (route == NULL || route->walcacheDir[0] == '\0') + { + ws_send_error_response(sock, "58P01", + "no WAL cache directory configured for this route"); + return; + } + + /* matches real Postgres's TLHistoryFileName() macro exactly */ + char filename[WS_MAXFNAMELEN]; + + snprintf(filename, sizeof(filename), "%08X.history", timeline); + + char path[MAXPGPATH]; + + snprintf(path, sizeof(path), "%s/%s", route->walcacheDir, filename); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + /* matches real walsender.c: no history file for this timeline is + * an ERROR there too, not a soft "empty" fallback */ + log_info("TIMELINE_HISTORY: \"%s\" not found under \"%s\"", + filename, route->walcacheDir); + ws_send_error_response(sock, "58P01", + "requested timeline history file not found"); + return; + } + + WsColumn columns[] = { + { "filename", WS_TEXTOID, -1 }, + { "content", WS_TEXTOID, -1 }, + }; + + const char *values[] = { filename, contents }; + + if (ws_send_row_description(sock, columns, 2) && + ws_send_data_row(sock, values, 2)) + { + ws_send_command_complete(sock, "TIMELINE_HISTORY"); + } + + free(contents); +} diff --git a/src/bin/pg_walsender/cmd_timeline_history.h b/src/bin/pg_walsender/cmd_timeline_history.h new file mode 100644 index 000000000..4e93ae251 --- /dev/null +++ b/src/bin/pg_walsender/cmd_timeline_history.h @@ -0,0 +1,23 @@ +/* + * src/bin/pg_walsender/cmd_timeline_history.h + * TIMELINE_HISTORY : serves a ".history" file straight out of + * the route's WAL cache directory. Traced from walsender.c's own + * SendTimeLineHistory() (backend, not linked -- see walsender.h's own + * header comment): a single RowDescription(filename text, content text) + * + one DataRow + CommandComplete, no COPY involved. Genuinely just a + * flat-file read; the only real-instance-shaped input is which timeline + * was asked for. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_CMD_TIMELINE_HISTORY_H +#define WS_CMD_TIMELINE_HISTORY_H + +#include "routes.h" + +void cmd_timeline_history(int sock, const WsRoute *route, int timeline); + +#endif /* WS_CMD_TIMELINE_HISTORY_H */ diff --git a/src/bin/pg_walsender/repl_command.c b/src/bin/pg_walsender/repl_command.c index be7bb5205..4db7fe859 100644 --- a/src/bin/pg_walsender/repl_command.c +++ b/src/bin/pg_walsender/repl_command.c @@ -16,7 +16,10 @@ #include "repl_command.h" #include "cmd_base_backup.h" #include "cmd_identify_system.h" +#include "cmd_replication_slot.h" #include "cmd_show.h" +#include "cmd_start_replication.h" +#include "cmd_timeline_history.h" #include "framing.h" @@ -77,6 +80,43 @@ repl_command_parse(const char *query, WsCommand *cmd) return true; } + if (strncasecmp(p, "TIMELINE_HISTORY", strlen("TIMELINE_HISTORY")) == 0) + { + p = skip_whitespace(p + strlen("TIMELINE_HISTORY")); + cmd->timeline = atoi(p); + cmd->type = WS_CMD_TIMELINE_HISTORY; + return true; + } + + if (strncasecmp(p, "CREATE_REPLICATION_SLOT", + strlen("CREATE_REPLICATION_SLOT")) == 0) + { + p = skip_whitespace(p + strlen("CREATE_REPLICATION_SLOT")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_CREATE_REPLICATION_SLOT; + return true; + } + + if (strncasecmp(p, "READ_REPLICATION_SLOT", + strlen("READ_REPLICATION_SLOT")) == 0) + { + p = skip_whitespace(p + strlen("READ_REPLICATION_SLOT")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_READ_REPLICATION_SLOT; + return true; + } + + if (strncasecmp(p, "START_REPLICATION", strlen("START_REPLICATION")) == 0) + { + p = skip_whitespace(p + strlen("START_REPLICATION")); + strlcpy(cmd->rawArgs, p, sizeof(cmd->rawArgs)); + rtrim(cmd->rawArgs); + cmd->type = WS_CMD_START_REPLICATION; + return true; + } + cmd->type = WS_CMD_UNKNOWN; return false; } @@ -106,6 +146,30 @@ ws_dispatch_command(int sock, const WsCommand *cmd, break; } + case WS_CMD_TIMELINE_HISTORY: + { + cmd_timeline_history(sock, route, cmd->timeline); + break; + } + + case WS_CMD_CREATE_REPLICATION_SLOT: + { + cmd_create_replication_slot(sock, route, cmd->rawArgs); + break; + } + + case WS_CMD_READ_REPLICATION_SLOT: + { + cmd_read_replication_slot(sock, route, cmd->rawArgs); + break; + } + + case WS_CMD_START_REPLICATION: + { + cmd_start_replication(sock, route, cmd->rawArgs); + break; + } + default: { ws_send_error_response(sock, "42601", "unsupported replication command"); diff --git a/src/bin/pg_walsender/repl_command.h b/src/bin/pg_walsender/repl_command.h index c45dd36fe..fe53e5964 100644 --- a/src/bin/pg_walsender/repl_command.h +++ b/src/bin/pg_walsender/repl_command.h @@ -26,6 +26,10 @@ typedef enum WsCommandType WS_CMD_IDENTIFY_SYSTEM, WS_CMD_SHOW, WS_CMD_BASE_BACKUP, + WS_CMD_TIMELINE_HISTORY, + WS_CMD_CREATE_REPLICATION_SLOT, + WS_CMD_READ_REPLICATION_SLOT, + WS_CMD_START_REPLICATION, WS_CMD_UNKNOWN } WsCommandType; @@ -36,6 +40,11 @@ typedef struct WsCommand char rawOptions[1024]; /* WS_CMD_BASE_BACKUP only: the "(...)" or * trailing-token option list verbatim, * parsed by cmd_base_backup.c itself */ + int timeline; /* WS_CMD_TIMELINE_HISTORY only */ + char rawArgs[512]; /* WS_CMD_{CREATE,READ}_REPLICATION_SLOT / + * WS_CMD_START_REPLICATION: everything + * after the keyword, verbatim, parsed by + * each command's own cmd_*.c */ } WsCommand; /* diff --git a/src/bin/pg_walsender/startup.c b/src/bin/pg_walsender/startup.c index 348c3e7ca..341a31fdc 100644 --- a/src/bin/pg_walsender/startup.c +++ b/src/bin/pg_walsender/startup.c @@ -113,9 +113,10 @@ ws_startup_negotiate(int sock, WsStartupParams *params) } else if (strcmp(key, "replication") == 0) { + params->replicationDatabase = (strcasecmp(value, "database") == 0); params->replication = (strcmp(value, "1") == 0 || strcasecmp(value, "true") == 0 || - strcasecmp(value, "database") == 0); + params->replicationDatabase); } } diff --git a/src/bin/pg_walsender/wal_dir_scan.c b/src/bin/pg_walsender/wal_dir_scan.c new file mode 100644 index 000000000..2ebf26bbc --- /dev/null +++ b/src/bin/pg_walsender/wal_dir_scan.c @@ -0,0 +1,113 @@ +/* + * src/bin/pg_walsender/wal_dir_scan.c + * See wal_dir_scan.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include + +#include "postgres_fe.h" + +#include "wal_dir_scan.h" + +/* default WAL segment size (16MB), matching cmd_show.c's own + * "SHOW wal_segment_size" -> "16MB" answer */ +#define WS_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define WS_XLOG_SEGMENTS_PER_XLOGID (UINT64CONST(0x100000000) / WS_WAL_SEGMENT_SIZE) + +#define WS_WAL_FNAME_LEN 24 + + +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != WS_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +void +wal_segment_filename(uint32_t timeline, uint64_t segno, char *dest, size_t destSize) +{ + uint32_t logId = (uint32_t) (segno / WS_XLOG_SEGMENTS_PER_XLOGID); + uint32_t seg = (uint32_t) (segno % WS_XLOG_SEGMENTS_PER_XLOGID); + + snprintf(dest, destSize, "%08X%08X%08X", timeline, logId, seg); +} + + +bool +wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char best[WS_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (!is_wal_segment_filename(entry->d_name)) + { + continue; + } + + if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + { + strlcpy(best, entry->d_name, sizeof(best)); + } + } + + closedir(dir); + + if (best[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(tliHex, best, 8); + memcpy(logIdHex, best + 8, 8); + memcpy(segHex, best + 16, 8); + + uint32_t tli = (uint32_t) strtoul(tliHex, NULL, 16); + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * WS_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t endOfSegment = (segno + 1) * WS_WAL_SEGMENT_SIZE; + + *timeline = tli; + snprintf(endLsn, endLsnSize, "%X/%08X", + (uint32_t) (endOfSegment >> 32), (uint32_t) (endOfSegment & 0xFFFFFFFF)); + + return true; +} diff --git a/src/bin/pg_walsender/wal_dir_scan.h b/src/bin/pg_walsender/wal_dir_scan.h new file mode 100644 index 000000000..1e5382c18 --- /dev/null +++ b/src/bin/pg_walsender/wal_dir_scan.h @@ -0,0 +1,45 @@ +/* + * src/bin/pg_walsender/wal_dir_scan.h + * Finds the newest fully-captured (non-.partial) WAL segment in an + * archiver's WAL cache directory and derives its boundary LSNs from the + * segment filename alone (standard 24-hex-digit XLogFileName format, + * assuming the fixed 16MB default segment size this project's own SHOW + * wal_segment_size already reports -- see cmd_show.c). + * + * This is a segment-boundary approximation, not a real-record-level + * position: it doesn't parse WAL contents, just the filename. Good + * enough for CREATE_REPLICATION_SLOT's consistent_point and + * IDENTIFY_SYSTEM's xlogpos; START_REPLICATION's actual segment + * streaming (wal_segment_source.c) reads the real bytes. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef WS_WAL_DIR_SCAN_H +#define WS_WAL_DIR_SCAN_H + +#include +#include + +/* + * wal_dir_find_latest scans walcacheDir for the highest-numbered complete + * WAL segment (24 hex chars, no ".partial" suffix). On success, returns + * true with *timeline set and endLsn filled with that segment's end-of- + * segment LSN (formatted "%X/%08X", matching pg_lsn's own text form) -- + * the natural "resume from here" position once this segment is fully + * captured. Returns false (not an error, *timeline and *endLsn untouched) + * if the directory has no WAL segments yet. + */ +bool wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize); + +/* + * wal_segment_filename formats a filename the same way real Postgres does + * (XLogFileName), for a given timeline and 0-based segment number. + */ +void wal_segment_filename(uint32_t timeline, uint64_t segno, + char *dest, size_t destSize); + +#endif /* WS_WAL_DIR_SCAN_H */ diff --git a/src/bin/pg_walsender/walsender.h b/src/bin/pg_walsender/walsender.h index d7cd5dd50..6eecfe10b 100644 --- a/src/bin/pg_walsender/walsender.h +++ b/src/bin/pg_walsender/walsender.h @@ -39,6 +39,16 @@ typedef struct WsStartupParams char database[NAMEDATALEN + 16]; /* "/", may exceed a bare NAMEDATALEN */ char applicationName[NAMEDATALEN]; bool replication; + + /* + * True only when the client's startup packet set replication=database + * (pg_basebackup's style) rather than a plain replication=1/true + * (pg_receivewal's style). IDENTIFY_SYSTEM's own dbname column must be + * NULL for the latter -- real pg_receivewal fatals out ("unexpectedly + * database specific") if it isn't, since a non-NULL dbname is its + * signal that the connection was accidentally database-qualified. + */ + bool replicationDatabase; } WsStartupParams; #endif /* WS_WALSENDER_H */ From c564a5aa334bddac86ef9116d6392ad44abdddc9 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 13:53:36 +0200 Subject: [PATCH 09/55] pg_autoctl: run support for kind = archiver (M3) (and `pg_autoctl stop`) now supervises an archiver's two halves together -- WAL capture (service_archiver.c's service_archiver_loop, outbound pg_receivewal against the primary) and serving (service_archiver_ serve.c's service_archiver_serve_loop, inbound pg_walsender) -- as two real supervisor.c Service[] entries under one restart-on-crash process tree, the same way start_keeper() already supervises postgres + node-active together for an ordinary node (service_archiver_run.c). Dispatched from cli_service. c's cli_keeper_run(), which already reaches an archiver's config file via the existing role=keeper path; branches on nodeKind before the Postgres- instance-specific local_postgres_init()/start_keeper() calls, which don't apply to an archiver. Two real bugs surfaced by actually running the full archiver process tree end-to-end for the first time this session (service_archiver_loop's own monitor-reporting loop was never previously exercised against a live monitor for more than a few ticks): - keeper->postgres.currentLSN was never initialized for an archiver (it has no real Postgres instance to query it from, so keeper_update_pg_ state() -- the only place that ever set it -- is never called on this path). node_active()'s own pg_lsn parameter rejected the resulting empty string outright. Fixed by seeding it to "0/0" once, matching keeper_update_pg_state()'s own placeholder before a real reading exists; an archiver's actual capture progress is tracked separately via archiver_wal, not through this per-node report. - An ordinary node's own get_other_nodes()/current_state listings now legitimately include ARCHIVING rows with nodeport = 0 (a deliberate sentinel, see archiver_add_formation()'s own SQL comment: no postmaster to be reachable on) -- but monitor.c's node-parsing helpers treated a parsed port of exactly 0 as an unconditional error, so any ordinary primary/secondary in a formation with an archiver attached would fail its own node-active loop entirely. Relaxed the two multi- node-listing parsers to only reject a genuine parse failure, not the value 0 itself; left the single-node lookup (which can never legitimately return an archiver, candidate_priority = 0 excludes it) unchanged. Verified against a real, freshly-created cluster (monitor + primary + archiver): `pg_autoctl run --pgdata archiver1` starts both services cleanly, the FSM transitions wait_standby -> archiving and real pg_receivewal starts against the primary, pg_walsender serves real clients through it, and `pg_autoctl stop` cascades a graceful shutdown through both services and their own child processes (pg_walsender, pg_receivewal) with no orphans left behind. Also re-verified the primary node's own node-active loop, previously broken by the port=0 regression, now runs cleanly with an archiver attached to its formation. --- src/bin/pg_autoctl/cli_service.c | 20 +++ src/bin/pg_autoctl/monitor.c | 15 +- src/bin/pg_autoctl/service_archiver.c | 14 ++ src/bin/pg_autoctl/service_archiver_run.c | 174 ++++++++++++++++++++++ src/bin/pg_autoctl/service_archiver_run.h | 28 ++++ src/bin/pg_autoctl/supervisor.h | 5 + 6 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 src/bin/pg_autoctl/service_archiver_run.c create mode 100644 src/bin/pg_autoctl/service_archiver_run.h diff --git a/src/bin/pg_autoctl/cli_service.c b/src/bin/pg_autoctl/cli_service.c index 33a95ce8f..b50962a46 100644 --- a/src/bin/pg_autoctl/cli_service.c +++ b/src/bin/pg_autoctl/cli_service.c @@ -26,6 +26,7 @@ #include "monitor.h" #include "monitor_config.h" #include "pidfile.h" +#include "service_archiver_run.h" #include "service_keeper.h" #include "service_monitor.h" #include "signals.h" @@ -203,6 +204,25 @@ cli_keeper_run(int argc, char **argv) pgsql_finish(&(monitor->pgsql)); } + /* + * An archiver has no real Postgres instance of its own (see + * service_archiver.c's own comment on config->pgSetup.pgdata's reused + * meaning for an ARCHIVING node) -- local_postgres_init()/start_keeper() + * both assume one, so branch to start_archiver() instead, milestone 3's + * own `pg_autoctl run` support (service_archiver_run.c). + */ + if (strcmp(config->nodeKind, "archiver") == 0) + { + if (!start_archiver(&keeper)) + { + log_fatal("Failed to start pg_autoctl archiver service, " + "see above for details"); + exit(EXIT_CODE_INTERNAL_ERROR); + } + + return; + } + /* initialize our local Postgres instance representation */ (void) local_postgres_init(postgres, pgSetup); diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 194941a31..2b07e35bc 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -2376,7 +2376,15 @@ parseNode(PGresult *result, int rowNumber, NodeAddress *node) value = PQgetvalue(result, rowNumber, 3); - if (!stringToInt(value, &node->port) || node->port == 0) + /* + * nodeport = 0 is a real, intentional value for an ARCHIVING row (see + * pgautofailover.sql's own comment on archiver_add_formation()'s + * INSERT): it has no postmaster of its own to be reachable on. This + * function parses whole-formation node listings (get_nodes/ + * get_other_nodes) that legitimately include those rows now, so a + * parsed zero is not an error -- only a genuine parse failure is. + */ + if (!stringToInt(value, &node->port)) { log_error("Invalid port number \"%s\" returned by monitor", value); return false; @@ -2766,8 +2774,9 @@ parseCurrentNodeState(PGresult *result, int rowNumber, value = PQgetvalue(result, rowNumber, 3); - if (!stringToInt(value, &(nodeState->node.port)) || - nodeState->node.port == 0) + /* nodeport = 0 is a real, intentional value for an ARCHIVING row -- see + * the sibling comment on this same check in parseNode() above */ + if (!stringToInt(value, &(nodeState->node.port))) { log_error("Invalid port number \"%s\" returned by monitor", value); ++errors; diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index 9328b2d20..50fade5cb 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -249,6 +249,20 @@ service_archiver_loop(Keeper *keeper) log_info("pg_autoctl archiver service is starting"); + /* + * An archiver never calls keeper_update_pg_state() -- there's no real + * Postgres instance to query (see haspgdata's own design comment) -- + * so keeper->postgres.currentLSN is otherwise left at its zero-valued + * empty string for the lifetime of this process. keeper_node_active() + * always sends it as one of node_active()'s own parameters, and the + * monitor-side pg_lsn column rejects an empty string outright ("invalid + * input syntax for type pg_lsn"). "0/0" is the same placeholder + * keeper_update_pg_state() itself defaults to before it has a real + * reading; an archiver's own WAL-capture progress is tracked + * separately via archiver_wal, not through this per-node report. + */ + strlcpy(keeper->postgres.currentLSN, "0/0", sizeof(keeper->postgres.currentLSN)); + while (!asked_to_stop && !asked_to_stop_fast && !asked_to_quit) { MonitorAssignedState assignedState = { 0 }; diff --git a/src/bin/pg_autoctl/service_archiver_run.c b/src/bin/pg_autoctl/service_archiver_run.c new file mode 100644 index 000000000..4f48f60de --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_run.c @@ -0,0 +1,174 @@ +/* + * src/bin/pg_autoctl/service_archiver_run.c + * See service_archiver_run.h. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include + +#include "service_archiver_run.h" + +#include "cli_root.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "service_archiver.h" +#include "service_archiver_serve.h" +#include "signals.h" +#include "supervisor.h" + + +/* + * service_archiver_capture_start forks a child that runs + * service_archiver_loop() (service_archiver.c) -- the outbound WAL-capture + * half, supervising pg_receivewal against the group's primary. No exec(): + * this project's own binary already implements the loop, matching + * service_keeper_start()'s sibling shape for an ordinary node minus the + * execv() re-exec (that one replaces the process image to get a fresh + * "node-active"-titled process; forking straight into the loop function is + * simpler and just as correct here). + */ +static bool +service_archiver_capture_start(void *context, pid_t *pid) +{ + Keeper *keeper = (Keeper *) context; + + fflush(stdout); + fflush(stderr); + + pid_t fpid = fork(); + + switch (fpid) + { + case -1: + { + log_error("Failed to fork the archiver capture process"); + return false; + } + + case 0: + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver capture"); + + /* + * Re-connect: the parent's own keeper->monitor connection is + * not fork-safe to share, and may already have been closed by + * the caller (cli_service.c's cli_keeper_run finishes its own + * connection before starting services) -- each supervised + * child establishes its own, exactly like a freshly exec'd + * process would. + */ + if (!monitor_init(&(keeper->monitor), keeper->config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (!service_archiver_loop(keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + + exit(EXIT_CODE_QUIT); + } + + default: + { + log_debug("pg_autoctl archiver capture process started in " + "subprocess %d", fpid); + *pid = fpid; + return true; + } + } +} + + +/* + * service_archiver_serve_start_service forks a child that runs + * service_archiver_serve_loop() (service_archiver_serve.c) -- the inbound + * serving half, exec'ing and supervising pg_walsender. Named with a + * "_service" suffix to avoid colliding with service_archiver_serve.c's own + * service_archiver_serve_start_walsender(), a different function one level + * down (that one starts pg_walsender itself; this one starts the loop that + * in turn starts and monitors pg_walsender). + */ +static bool +service_archiver_serve_start_service(void *context, pid_t *pid) +{ + Keeper *keeper = (Keeper *) context; + + fflush(stdout); + fflush(stderr); + + pid_t fpid = fork(); + + switch (fpid) + { + case -1: + { + log_error("Failed to fork the archiver serve process"); + return false; + } + + case 0: + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver serve"); + + /* see service_archiver_capture_start()'s own comment on why + * each supervised child re-connects independently */ + if (!monitor_init(&(keeper->monitor), keeper->config.monitor_pguri)) + { + log_fatal("Failed to contact the monitor, see above for details"); + exit(EXIT_CODE_MONITOR); + } + + if (!service_archiver_serve_loop(keeper)) + { + exit(EXIT_CODE_INTERNAL_ERROR); + } + + exit(EXIT_CODE_QUIT); + } + + default: + { + log_debug("pg_autoctl archiver serve process started in " + "subprocess %d", fpid); + *pid = fpid; + return true; + } + } +} + + +bool +start_archiver(Keeper *keeper) +{ + const char *pidfile = keeper->config.pathnames.pid; + + Service subprocesses[] = { + { + SERVICE_NAME_ARCHIVER_CAPTURE, + RP_PERMANENT, + -1, + &service_archiver_capture_start, + (void *) keeper + }, + { + SERVICE_NAME_ARCHIVER_SERVE, + RP_PERMANENT, + -1, + &service_archiver_serve_start_service, + (void *) keeper + } + }; + + int subprocessesCount = sizeof(subprocesses) / sizeof(subprocesses[0]); + + return supervisor_start(subprocesses, subprocessesCount, pidfile); +} diff --git a/src/bin/pg_autoctl/service_archiver_run.h b/src/bin/pg_autoctl/service_archiver_run.h new file mode 100644 index 000000000..12d9a9324 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_run.h @@ -0,0 +1,28 @@ +/* + * src/bin/pg_autoctl/service_archiver_run.h + * Archiving & Disaster Recovery: `pg_autoctl run` support for + * kind = archiver (milestone 3's own build-order line). Supervises the + * archiver's two halves -- WAL capture (service_archiver.c's + * service_archiver_loop, outbound pg_receivewal against the primary) + * and serving (service_archiver_serve.c's service_archiver_serve_loop, + * inbound pg_walsender) -- as two real supervisor.c Service[] entries + * under one supervised process tree, restart-on-crash, the same way + * start_keeper() already supervises postgres + node-active together for + * an ordinary node. Replaces needing two separately-managed processes + * (`create archiver --run` for capture, `archiver serve` for serving) + * with the one unified entry point operators already expect from + * `pg_autoctl run`. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_RUN_H +#define SERVICE_ARCHIVER_RUN_H + +#include "keeper.h" + +bool start_archiver(Keeper *keeper); + +#endif /* SERVICE_ARCHIVER_RUN_H */ diff --git a/src/bin/pg_autoctl/supervisor.h b/src/bin/pg_autoctl/supervisor.h index 0cddf99d7..540f9dae4 100644 --- a/src/bin/pg_autoctl/supervisor.h +++ b/src/bin/pg_autoctl/supervisor.h @@ -26,6 +26,11 @@ #define SERVICE_NAME_KEEPER "node-active" #define SERVICE_NAME_MONITOR "listener" +/* an archiver's two halves, supervised together by start_archiver() + * (service_archiver_run.c) -- see that file's own header comment */ +#define SERVICE_NAME_ARCHIVER_CAPTURE "archiver-capture" +#define SERVICE_NAME_ARCHIVER_SERVE "archiver-serve" + /* * At pg_autoctl create time we use a transient service to initialize our local * node. When using the --run option, the transient service is terminated and From 8cd6733b33838256577381e415d29ac1a3aab3f5 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 14:19:30 +0200 Subject: [PATCH 10/55] pg_autoctl: report captured WAL segments to the monitor (M4) service_archiver.c gains service_archiver_report_captured_wal(), called once per node_active tick from service_archiver_loop(): it scans the archiver's local WAL cache directory for segments pg_receivewal has completed (i.e. no longer ".partial") since the last one reported, and calls the new monitor_report_wal_received() (monitor.c/.h) -- a thin wrapper around the already-existing pgautofailover.report_wal_received() SQL function -- for each one. This is what actually populates archiver_wal and makes wal_archived() return true; until now nothing in the codebase ever called that SQL function. Also fixes a liveness gap this uncovered: pg_receivewal was only ever (re)started from the FSM transition functions that move a node *into* ARCHIVING_STATE (fsm_init_archiver, fsm_archiver_follow_new_primary). An archiver process restarted while already ARCHIVING (or one whose pg_receivewal child died on its own) had nothing to bring it back up, despite this file's own header comment already describing that as the design. service_archiver_loop() now checks service_archiver_pgreceivewal_is_running() every tick and restarts it when needed, exactly matching that comment. Verified end-to-end against a real monitor + primary + archiver: forced WAL switches on the primary, confirmed archiver_wal gets populated with the correct end-of-segment LSNs and wal_archived() correctly reflects archiver_quorum, confirmed the liveness restart itself by killing and restarting the archiver process while already ARCHIVING. Full SQL regression schedule (src/monitor, 20/20) still passes. Dockerfile: copy pg_walsender into the "run" stage image alongside pg_autoctl -- needed for any archiver node in a pgaftest Docker environment, and a prerequisite for M4's own pgaftest specs. --- Dockerfile | 1 + src/bin/pg_autoctl/monitor.c | 33 ++++ src/bin/pg_autoctl/monitor.h | 2 + src/bin/pg_autoctl/service_archiver.c | 216 ++++++++++++++++++++++++++ src/bin/pg_autoctl/service_archiver.h | 2 + 5 files changed, 254 insertions(+) diff --git a/Dockerfile b/Dockerfile index 116d00961..75293188b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,6 +120,7 @@ COPY --from=build /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \ COPY --from=build /usr/share/postgresql/${PGVERSION}/extension/pgautofailover* \ /usr/share/postgresql/${PGVERSION}/extension/ COPY --from=build /usr/local/bin/pg_autoctl /usr/local/bin/ +COPY --from=build /usr/local/bin/pg_walsender /usr/local/bin/ RUN mkdir -p /var/lib/postgres \ && chown -R docker /var/lib/postgres diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 2b07e35bc..a674b76e3 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -1029,6 +1029,39 @@ monitor_get_latest_basebackup_location(Monitor *monitor, } +/* + * monitor_report_wal_received calls pgautofailover.report_wal_received() + * to record that nodeId (the ARCHIVING membership's own nodeid, not the + * archiver's archiverid) has durably captured walFileName up to lsn. + * Idempotent on the monitor side (ON CONFLICT DO NOTHING), so callers are + * free to re-report an already-known segment without checking first -- + * see service_archiver.c's own use of this. + */ +bool +monitor_report_wal_received(Monitor *monitor, int64_t nodeId, + const char *walFileName, const char *lsn) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_wal_received($1, $2, $3)"; + int paramCount = 3; + Oid paramTypes[3] = { INT8OID, TEXTOID, LSNOID }; + IntString nodeIdString = intToString(nodeId); + const char *paramValues[3] = { nodeIdString.strValue, walFileName, lsn }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report WAL file \"%s\" received for node %" + PRId64, walFileName, nodeId); + return false; + } + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index ee5389ae3..2b935d3bf 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -162,6 +162,8 @@ bool monitor_get_latest_basebackup_location(Monitor *monitor, const char *formationId, int groupId, char *storageLocation, size_t size, bool *found); +bool monitor_report_wal_received(Monitor *monitor, int64_t nodeId, + const char *walFileName, const char *lsn); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index 50fade5cb..1b3192f72 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -21,6 +21,8 @@ * */ +#include +#include #include #include #include @@ -37,6 +39,26 @@ #include "monitor.h" #include "signals.h" +/* + * WAL segment filename layout, duplicated from pg_walsender/wal_dir_scan.c: + * pg_autoctl doesn't link that standalone binary's code (see this project's + * Makefile split), so the ~15-line segno/LSN arithmetic is small enough to + * repeat here rather than share. + */ +#define ARCHIVER_WAL_FNAME_LEN 24 +#define ARCHIVER_WAL_SEGMENT_SIZE ((uint64_t) 0x1000000) +#define ARCHIVER_XLOG_SEGMENTS_PER_XLOGID \ + (((uint64_t) 0x100000000) / ARCHIVER_WAL_SEGMENT_SIZE) + +/* + * Last WAL filename already reported to the monitor, so each tick only + * reports newly-appeared segments instead of re-scanning and re-reporting + * the whole cache directory every time (the monitor-side insert is + * idempotent, ON CONFLICT DO NOTHING, but that's a fallback for restarts, + * not meant to be relied on every tick). + */ +static char lastReportedWalFileName[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + /* * One pg_receivewal child per archiver process, matching milestone 2's own * single-membership scope (see this file's own comment) -- a future @@ -222,6 +244,170 @@ service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) } +/* + * is_wal_segment_filename returns true iff name has the shape of a real WAL + * segment file (24 hex digits) -- this also naturally excludes pg_receivewal's + * own ".partial" in-progress file, since it's longer than 24 chars. + */ +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != ARCHIVER_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +/* + * wal_filename_compare is a qsort() comparator over an array of char*, + * ordering WAL segment filenames the same way their fixed-width hex names + * already sort lexicographically (== numerically, oldest to newest). + */ +static int +wal_filename_compare(const void *a, const void *b) +{ + const char *nameA = *(const char *const *) a; + const char *nameB = *(const char *const *) b; + + return strcmp(nameA, nameB); +} + + +/* + * wal_segment_end_lsn computes the LSN just past the end of the WAL segment + * named walFileName -- what report_wal_received() records as "captured up + * to", matching pg_walsender/wal_dir_scan.c's own wal_dir_find_latest() + * arithmetic for the same filename layout. + */ +static void +wal_segment_end_lsn(const char *walFileName, char *lsn, size_t lsnSize) +{ + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(logIdHex, walFileName + 8, 8); + memcpy(segHex, walFileName + 16, 8); + + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * ARCHIVER_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t endOfSegment = (segno + 1) * ARCHIVER_WAL_SEGMENT_SIZE; + + snprintf(lsn, lsnSize, "%X/%08X", + (uint32_t) (endOfSegment >> 32), + (uint32_t) (endOfSegment & 0xFFFFFFFF)); +} + + +/* + * service_archiver_report_captured_wal scans the archiver's local WAL cache + * directory for segments pg_receivewal has completed (i.e. no longer + * ".partial") since the last-reported filename, and reports each one to the + * monitor via monitor_report_wal_received() -- the mechanism backing + * archiver_wal/wal_archived(), so archive_command callers elsewhere in the + * cluster can learn when a segment has landed durably on quorum archivers. + * + * Reports oldest-to-newest and only advances lastReportedWalFileName past a + * segment once its report has actually succeeded, so a monitor hiccup + * retries that segment (and anything after it) on the next tick instead of + * silently skipping it. + */ +bool +service_archiver_report_captured_wal(Keeper *keeper) +{ + const char *walcacheDir = keeper->config.pgSetup.pgdata; + + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + /* nothing captured yet -- not an error */ + return true; + } + + char **names = NULL; + int count = 0; + int capacity = 0; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (!is_wal_segment_filename(entry->d_name)) + { + continue; + } + + if (strcmp(entry->d_name, lastReportedWalFileName) <= 0) + { + continue; + } + + if (count == capacity) + { + capacity = capacity == 0 ? 16 : capacity * 2; + names = realloc(names, capacity * sizeof(char *)); + } + + names[count++] = strdup(entry->d_name); + } + + closedir(dir); + + if (count == 0) + { + return true; + } + + qsort(names, count, sizeof(char *), wal_filename_compare); + + bool success = true; + + for (int i = 0; i < count; i++) + { + if (success) + { + char lsn[PG_LSN_MAXLENGTH] = { 0 }; + + wal_segment_end_lsn(names[i], lsn, sizeof(lsn)); + + if (monitor_report_wal_received(&(keeper->monitor), + keeper->state.current_node_id, + names[i], lsn)) + { + strlcpy(lastReportedWalFileName, names[i], + sizeof(lastReportedWalFileName)); + } + else + { + log_error("Failed to report WAL file \"%s\" to the monitor", + names[i]); + success = false; + } + } + + free(names[i]); + } + + free(names); + + return success; +} + + /* * service_archiver_loop is the archiver's own node_active() reporting loop * -- deliberately not keeper_node_active_loop (service_keeper.c): that @@ -288,6 +474,36 @@ service_archiver_loop(Keeper *keeper) NodeStateToString(keeperState->assigned_role)); } } + + /* + * Liveness check: a state transition only (re)starts + * pg_receivewal at the moment current_role becomes + * ARCHIVING_STATE (fsm_init_archiver/fsm_archiver_follow_new_ + * primary, fsm_transition.c) -- it does not run again on later + * ticks where current_role and assigned_role already agree. + * Without this check, a pg_receivewal that dies (or an archiver + * process that gets restarted while already ARCHIVING) would + * stay down forever instead of being noticed and restarted here, + * exactly the "is it running" check this loop's own header + * comment describes. + */ + if (keeperState->current_role == ARCHIVING_STATE && + !service_archiver_pgreceivewal_is_running()) + { + NodeAddress primaryNode = { 0 }; + + if (!keeper_get_primary(keeper, &primaryNode) || + !service_archiver_start_pgreceivewal(keeper, &primaryNode)) + { + log_error("Failed to restart pg_receivewal, retrying..."); + } + } + + if (!service_archiver_report_captured_wal(keeper)) + { + log_warn("Failed to report newly captured WAL segments to " + "the monitor, will retry"); + } } else { diff --git a/src/bin/pg_autoctl/service_archiver.h b/src/bin/pg_autoctl/service_archiver.h index 860eccc2d..afb209b70 100644 --- a/src/bin/pg_autoctl/service_archiver.h +++ b/src/bin/pg_autoctl/service_archiver.h @@ -20,6 +20,8 @@ bool service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNod bool service_archiver_stop_pgreceivewal(void); bool service_archiver_pgreceivewal_is_running(void); +bool service_archiver_report_captured_wal(Keeper *keeper); + bool service_archiver_loop(Keeper *keeper); #endif /* SERVICE_ARCHIVER_H */ From 55abeecf69604e783577b5df3046621ca01ea9cb Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 14:29:21 +0200 Subject: [PATCH 11/55] pgaftest: support kind=archiver in .pgaf cluster specs Adds the "archiver" node kind to pgaftest's own DSL, needed to write any .pgaf spec that includes an ARCHIVING node: - test_spec_scan.l/.y: new "archiver" keyword (T_ARCHIVER), usable the same way "coordinator"/"worker" already are: `archiver1 archiver` inside a formation{} block. - compose_gen.c: writes kind = archiver into the node's .ini, and links service_archiver.c into pgaftest's own SHARED_SRCS (Makefile) so the binary can drive an archiver node the same way it already drives postgres/coordinator/worker ones. - nodespec.c (pg_autoctl, not pgaftest): teaches `pg_autoctl node run ` -- the one command every pgaftest container actually execs -- to recognize kind = archiver and build the right `pg_autoctl create archiver` argv. An archiver's own getopts is deliberately minimal (no --pgport/--ssl-*/--auth/...), so it gets its own argv branch rather than falling through into the generic postgres-flags path every other kind shares. Also fixes a real bug in cli_indent.c's print_node() found while writing the first archiver spec and round-tripping it through `pgaftest indent`: the node-kind-to-keyword switch only had cases for coordinator and worker, so indenting a spec containing an archiver node silently dropped the "archiver" keyword on write-back, turning it into a plain postgres node. Added the missing NODE_KIND_ARCHIVER case. test_spec_parse.c/.h and test_spec_scan.c are bison/flex output, regenerated from the .y/.l changes above. --- src/bin/pg_autoctl/nodespec.c | 56 +- src/bin/pgaftest/Makefile | 2 +- src/bin/pgaftest/cli_indent.c | 4 + src/bin/pgaftest/compose_gen.c | 6 + src/bin/pgaftest/test_spec_parse.c | 1693 ++++++++++++++-------------- src/bin/pgaftest/test_spec_parse.h | 400 +++---- src/bin/pgaftest/test_spec_parse.y | 6 +- src/bin/pgaftest/test_spec_scan.c | 1627 +++++++++++++------------- src/bin/pgaftest/test_spec_scan.l | 1 + 9 files changed, 1944 insertions(+), 1851 deletions(-) diff --git a/src/bin/pg_autoctl/nodespec.c b/src/bin/pg_autoctl/nodespec.c index d57655d2d..fb1e78165 100644 --- a/src/bin/pg_autoctl/nodespec.c +++ b/src/bin/pg_autoctl/nodespec.c @@ -203,10 +203,14 @@ nodespec_read(const char *path, NodeSpec *spec) { spec->kind = NODE_KIND_CITUS_WORKER; } + else if (strcmp(kindStr, "archiver") == 0) + { + spec->kind = NODE_KIND_ARCHIVER; + } else { log_error("Unknown node kind \"%s\" in \"%s\"; " - "expected: monitor, postgres, coordinator, worker", + "expected: monitor, postgres, coordinator, worker, archiver", kindStr, path); return false; } @@ -556,6 +560,12 @@ nodespec_create_argv(const NodeSpec *spec, break; } + case NODE_KIND_ARCHIVER: + { + PUSH("archiver"); + break; + } + default: { PUSH("postgres"); @@ -563,6 +573,50 @@ nodespec_create_argv(const NodeSpec *spec, } } + /* + * An archiver's own getopts (cli_create_archiver_getopts, + * cli_create_node.c) is deliberately minimal -- no --pgport, --ssl-*, + * --auth, --pg-hba-lan, --candidate-priority, ... -- none of which + * apply to a node with no real PostgresSetup (see haspgdata's own + * design comment, pgautofailover.sql). Building its own argv here + * rather than falling through into the rest of this function (which + * assumes every kind accepts the full postgres flag set) avoids + * "unrecognized option" failures on every one of those. + */ + if (spec->kind == NODE_KIND_ARCHIVER) + { + PUSH("--pgdata"); + PUSH(spec->pgdata); + + if (!IS_EMPTY_STRING_BUFFER(spec->name)) + { + PUSH("--name"); + PUSH(spec->name); + } + + if (!IS_EMPTY_STRING_BUFFER(spec->hostname)) + { + PUSH("--hostname"); + PUSH(spec->hostname); + } + + PUSH("--monitor"); + PUSH(spec->monitor_pguri); + + if (!IS_EMPTY_STRING_BUFFER(spec->formation) && + strcmp(spec->formation, "default") != 0) + { + PUSH("--formation"); + PUSH(spec->formation); + } + + PUSH("--run"); + + args[i] = NULL; + + return i; + } + PUSH("--pgdata"); PUSH(spec->pgdata); diff --git a/src/bin/pgaftest/Makefile b/src/bin/pgaftest/Makefile index 6564f559f..fbf6c0ed0 100644 --- a/src/bin/pgaftest/Makefile +++ b/src/bin/pgaftest/Makefile @@ -32,7 +32,7 @@ SHARED_SRCS = cli_common.c config.c coordinator.c fsm.c fsm_transition.c \ fsm_transition_citus.c keeper.c keeper_config.c keeper_pg_init.c \ monitor.c monitor_config.c monitor_pg_init.c \ nodespec.c nodestate_utils.c pghba.c primary_standby.c \ - service_keeper.c service_keeper_init.c service_monitor.c \ + service_archiver.c service_keeper.c service_keeper_init.c service_monitor.c \ service_monitor_init.c service_postgres.c service_postgres_ctl.c \ state.c step_socket.c supervisor.c systemd_config.c timeline_history.c diff --git a/src/bin/pgaftest/cli_indent.c b/src/bin/pgaftest/cli_indent.c index 3732652d5..5f69fd44e 100644 --- a/src/bin/pgaftest/cli_indent.c +++ b/src/bin/pgaftest/cli_indent.c @@ -415,6 +415,10 @@ print_node(FILE *out, const TestNode *n, int baseIndent) strlcpy(kindbuf, "worker", sizeof(kindbuf)); } } + else if (n->kind == NODE_KIND_ARCHIVER) + { + strlcpy(kindbuf, "archiver", sizeof(kindbuf)); + } const char *kind = kindbuf; #define ADD(k, v) do { props[pc].kw = (k); strlcpy(props[pc].val, (v), \ diff --git a/src/bin/pgaftest/compose_gen.c b/src/bin/pgaftest/compose_gen.c index 12acb35eb..fa558f5c9 100644 --- a/src/bin/pgaftest/compose_gen.c +++ b/src/bin/pgaftest/compose_gen.c @@ -1631,6 +1631,12 @@ compose_gen_write_node_ini(const TestCluster *cluster, break; } + case NODE_KIND_ARCHIVER: + { + kindStr = "archiver"; + break; + } + default: { kindStr = "postgres"; diff --git a/src/bin/pgaftest/test_spec_parse.c b/src/bin/pgaftest/test_spec_parse.c index 153815756..bc99f37b0 100644 --- a/src/bin/pgaftest/test_spec_parse.c +++ b/src/bin/pgaftest/test_spec_parse.c @@ -85,105 +85,106 @@ T_NUM_SYNC = 274, T_COORDINATOR = 275, T_WORKER = 276, - T_ASYNC = 277, - T_NO_MONITOR = 278, - T_SUSPENDED = 279, - T_LAUNCH = 280, - T_CREATE = 281, - T_DEFERRED = 282, - T_IMMEDIATE = 283, - T_FALSE = 284, - T_TRUE = 285, - T_INITIALLY = 286, - T_VOLUME = 287, - T_LISTEN = 288, - T_CITUS_SECONDARY = 289, - T_CANDIDATE_PRIORITY = 290, - T_PORT = 291, - T_PASSWORD = 292, - T_MONITOR_PASSWORD = 293, - T_CITUS_CLUSTER_NAME = 294, - T_DEBIAN_CLUSTER = 295, - T_REPLICATION_QUORUM = 296, - T_REPLICATION_PASSWORD = 297, - T_EXTENSION_VERSION = 298, - T_BIND_SOURCE = 299, - T_LEGACY_STARTUP = 300, - T_REGION = 301, - T_NODEINI = 302, - T_FS_INIT = 303, - T_FS_SINGLE = 304, - T_FS_PRIMARY = 305, - T_FS_WAIT_PRIMARY = 306, - T_FS_WAIT_STANDBY = 307, - T_FS_DEMOTED = 308, - T_FS_DEMOTE_TIMEOUT = 309, - T_FS_DRAINING = 310, - T_FS_SECONDARY = 311, - T_FS_CATCHINGUP = 312, - T_FS_PREP_PROMOTION = 313, - T_FS_STOP_REPLICATION = 314, - T_FS_MAINTENANCE = 315, - T_FS_JOIN_PRIMARY = 316, - T_FS_APPLY_SETTINGS = 317, - T_FS_PREPARE_MAINTENANCE = 318, - T_FS_WAIT_MAINTENANCE = 319, - T_FS_REPORT_LSN = 320, - T_FS_FAST_FORWARD = 321, - T_FS_JOIN_SECONDARY = 322, - T_FS_DROPPED = 323, - T_EXEC = 324, - T_EXEC_FAILS = 325, - T_RUN = 326, - T_PG_AUTOCTL = 327, - T_WAIT = 328, - T_UNTIL = 329, - T_TIMEOUT = 330, - T_AND = 331, - T_IS = 332, - T_WITH = 333, - T_REPLAYS = 334, - T_ASSERT = 335, - T_SQL = 336, - T_EXPECT = 337, - T_ERROR = 338, - T_PROMOTE = 339, - T_PERFORM = 340, - T_FAILOVER = 341, - T_NETWORK = 342, - T_DISCONNECT = 343, - T_CONNECT = 344, - T_SLEEP = 345, - T_COMPOSE = 346, - T_DOWN = 347, - T_START = 348, - T_STOP = 349, - T_STOPPED = 350, - T_KILL = 351, - T_INJECT = 352, - T_STATE = 353, - T_ASSIGNED_STATE = 354, - T_IN = 355, - T_GROUP = 356, - T_LBRACE = 357, - T_RBRACE = 358, - T_COMMA = 359, - T_POSTGRES = 360, - T_STAYS = 361, - T_WHILE = 362, - T_THROUGH = 363, - T_SET = 364, - T_GET = 365, - T_FSM = 366, - T_LOGS = 367, - T_NOT = 368, - T_CONTAINS = 369, - T_MATCHES = 370, - T_INTEGER = 371, - T_IDENT = 372, - T_STRING = 373, - T_BLOCK = 374, - T_SHELL_ARGS = 375 + T_ARCHIVER = 277, + T_ASYNC = 278, + T_NO_MONITOR = 279, + T_SUSPENDED = 280, + T_LAUNCH = 281, + T_CREATE = 282, + T_DEFERRED = 283, + T_IMMEDIATE = 284, + T_FALSE = 285, + T_TRUE = 286, + T_INITIALLY = 287, + T_VOLUME = 288, + T_LISTEN = 289, + T_CITUS_SECONDARY = 290, + T_CANDIDATE_PRIORITY = 291, + T_PORT = 292, + T_PASSWORD = 293, + T_MONITOR_PASSWORD = 294, + T_CITUS_CLUSTER_NAME = 295, + T_DEBIAN_CLUSTER = 296, + T_REPLICATION_QUORUM = 297, + T_REPLICATION_PASSWORD = 298, + T_EXTENSION_VERSION = 299, + T_BIND_SOURCE = 300, + T_LEGACY_STARTUP = 301, + T_REGION = 302, + T_NODEINI = 303, + T_FS_INIT = 304, + T_FS_SINGLE = 305, + T_FS_PRIMARY = 306, + T_FS_WAIT_PRIMARY = 307, + T_FS_WAIT_STANDBY = 308, + T_FS_DEMOTED = 309, + T_FS_DEMOTE_TIMEOUT = 310, + T_FS_DRAINING = 311, + T_FS_SECONDARY = 312, + T_FS_CATCHINGUP = 313, + T_FS_PREP_PROMOTION = 314, + T_FS_STOP_REPLICATION = 315, + T_FS_MAINTENANCE = 316, + T_FS_JOIN_PRIMARY = 317, + T_FS_APPLY_SETTINGS = 318, + T_FS_PREPARE_MAINTENANCE = 319, + T_FS_WAIT_MAINTENANCE = 320, + T_FS_REPORT_LSN = 321, + T_FS_FAST_FORWARD = 322, + T_FS_JOIN_SECONDARY = 323, + T_FS_DROPPED = 324, + T_EXEC = 325, + T_EXEC_FAILS = 326, + T_RUN = 327, + T_PG_AUTOCTL = 328, + T_WAIT = 329, + T_UNTIL = 330, + T_TIMEOUT = 331, + T_AND = 332, + T_IS = 333, + T_WITH = 334, + T_REPLAYS = 335, + T_ASSERT = 336, + T_SQL = 337, + T_EXPECT = 338, + T_ERROR = 339, + T_PROMOTE = 340, + T_PERFORM = 341, + T_FAILOVER = 342, + T_NETWORK = 343, + T_DISCONNECT = 344, + T_CONNECT = 345, + T_SLEEP = 346, + T_COMPOSE = 347, + T_DOWN = 348, + T_START = 349, + T_STOP = 350, + T_STOPPED = 351, + T_KILL = 352, + T_INJECT = 353, + T_STATE = 354, + T_ASSIGNED_STATE = 355, + T_IN = 356, + T_GROUP = 357, + T_LBRACE = 358, + T_RBRACE = 359, + T_COMMA = 360, + T_POSTGRES = 361, + T_STAYS = 362, + T_WHILE = 363, + T_THROUGH = 364, + T_SET = 365, + T_GET = 366, + T_FSM = 367, + T_LOGS = 368, + T_NOT = 369, + T_CONTAINS = 370, + T_MATCHES = 371, + T_INTEGER = 372, + T_IDENT = 373, + T_STRING = 374, + T_BLOCK = 375, + T_SHELL_ARGS = 376 }; #endif /* Tokens. */ @@ -206,105 +207,106 @@ #define T_NUM_SYNC 274 #define T_COORDINATOR 275 #define T_WORKER 276 -#define T_ASYNC 277 -#define T_NO_MONITOR 278 -#define T_SUSPENDED 279 -#define T_LAUNCH 280 -#define T_CREATE 281 -#define T_DEFERRED 282 -#define T_IMMEDIATE 283 -#define T_FALSE 284 -#define T_TRUE 285 -#define T_INITIALLY 286 -#define T_VOLUME 287 -#define T_LISTEN 288 -#define T_CITUS_SECONDARY 289 -#define T_CANDIDATE_PRIORITY 290 -#define T_PORT 291 -#define T_PASSWORD 292 -#define T_MONITOR_PASSWORD 293 -#define T_CITUS_CLUSTER_NAME 294 -#define T_DEBIAN_CLUSTER 295 -#define T_REPLICATION_QUORUM 296 -#define T_REPLICATION_PASSWORD 297 -#define T_EXTENSION_VERSION 298 -#define T_BIND_SOURCE 299 -#define T_LEGACY_STARTUP 300 -#define T_REGION 301 -#define T_NODEINI 302 -#define T_FS_INIT 303 -#define T_FS_SINGLE 304 -#define T_FS_PRIMARY 305 -#define T_FS_WAIT_PRIMARY 306 -#define T_FS_WAIT_STANDBY 307 -#define T_FS_DEMOTED 308 -#define T_FS_DEMOTE_TIMEOUT 309 -#define T_FS_DRAINING 310 -#define T_FS_SECONDARY 311 -#define T_FS_CATCHINGUP 312 -#define T_FS_PREP_PROMOTION 313 -#define T_FS_STOP_REPLICATION 314 -#define T_FS_MAINTENANCE 315 -#define T_FS_JOIN_PRIMARY 316 -#define T_FS_APPLY_SETTINGS 317 -#define T_FS_PREPARE_MAINTENANCE 318 -#define T_FS_WAIT_MAINTENANCE 319 -#define T_FS_REPORT_LSN 320 -#define T_FS_FAST_FORWARD 321 -#define T_FS_JOIN_SECONDARY 322 -#define T_FS_DROPPED 323 -#define T_EXEC 324 -#define T_EXEC_FAILS 325 -#define T_RUN 326 -#define T_PG_AUTOCTL 327 -#define T_WAIT 328 -#define T_UNTIL 329 -#define T_TIMEOUT 330 -#define T_AND 331 -#define T_IS 332 -#define T_WITH 333 -#define T_REPLAYS 334 -#define T_ASSERT 335 -#define T_SQL 336 -#define T_EXPECT 337 -#define T_ERROR 338 -#define T_PROMOTE 339 -#define T_PERFORM 340 -#define T_FAILOVER 341 -#define T_NETWORK 342 -#define T_DISCONNECT 343 -#define T_CONNECT 344 -#define T_SLEEP 345 -#define T_COMPOSE 346 -#define T_DOWN 347 -#define T_START 348 -#define T_STOP 349 -#define T_STOPPED 350 -#define T_KILL 351 -#define T_INJECT 352 -#define T_STATE 353 -#define T_ASSIGNED_STATE 354 -#define T_IN 355 -#define T_GROUP 356 -#define T_LBRACE 357 -#define T_RBRACE 358 -#define T_COMMA 359 -#define T_POSTGRES 360 -#define T_STAYS 361 -#define T_WHILE 362 -#define T_THROUGH 363 -#define T_SET 364 -#define T_GET 365 -#define T_FSM 366 -#define T_LOGS 367 -#define T_NOT 368 -#define T_CONTAINS 369 -#define T_MATCHES 370 -#define T_INTEGER 371 -#define T_IDENT 372 -#define T_STRING 373 -#define T_BLOCK 374 -#define T_SHELL_ARGS 375 +#define T_ARCHIVER 277 +#define T_ASYNC 278 +#define T_NO_MONITOR 279 +#define T_SUSPENDED 280 +#define T_LAUNCH 281 +#define T_CREATE 282 +#define T_DEFERRED 283 +#define T_IMMEDIATE 284 +#define T_FALSE 285 +#define T_TRUE 286 +#define T_INITIALLY 287 +#define T_VOLUME 288 +#define T_LISTEN 289 +#define T_CITUS_SECONDARY 290 +#define T_CANDIDATE_PRIORITY 291 +#define T_PORT 292 +#define T_PASSWORD 293 +#define T_MONITOR_PASSWORD 294 +#define T_CITUS_CLUSTER_NAME 295 +#define T_DEBIAN_CLUSTER 296 +#define T_REPLICATION_QUORUM 297 +#define T_REPLICATION_PASSWORD 298 +#define T_EXTENSION_VERSION 299 +#define T_BIND_SOURCE 300 +#define T_LEGACY_STARTUP 301 +#define T_REGION 302 +#define T_NODEINI 303 +#define T_FS_INIT 304 +#define T_FS_SINGLE 305 +#define T_FS_PRIMARY 306 +#define T_FS_WAIT_PRIMARY 307 +#define T_FS_WAIT_STANDBY 308 +#define T_FS_DEMOTED 309 +#define T_FS_DEMOTE_TIMEOUT 310 +#define T_FS_DRAINING 311 +#define T_FS_SECONDARY 312 +#define T_FS_CATCHINGUP 313 +#define T_FS_PREP_PROMOTION 314 +#define T_FS_STOP_REPLICATION 315 +#define T_FS_MAINTENANCE 316 +#define T_FS_JOIN_PRIMARY 317 +#define T_FS_APPLY_SETTINGS 318 +#define T_FS_PREPARE_MAINTENANCE 319 +#define T_FS_WAIT_MAINTENANCE 320 +#define T_FS_REPORT_LSN 321 +#define T_FS_FAST_FORWARD 322 +#define T_FS_JOIN_SECONDARY 323 +#define T_FS_DROPPED 324 +#define T_EXEC 325 +#define T_EXEC_FAILS 326 +#define T_RUN 327 +#define T_PG_AUTOCTL 328 +#define T_WAIT 329 +#define T_UNTIL 330 +#define T_TIMEOUT 331 +#define T_AND 332 +#define T_IS 333 +#define T_WITH 334 +#define T_REPLAYS 335 +#define T_ASSERT 336 +#define T_SQL 337 +#define T_EXPECT 338 +#define T_ERROR 339 +#define T_PROMOTE 340 +#define T_PERFORM 341 +#define T_FAILOVER 342 +#define T_NETWORK 343 +#define T_DISCONNECT 344 +#define T_CONNECT 345 +#define T_SLEEP 346 +#define T_COMPOSE 347 +#define T_DOWN 348 +#define T_START 349 +#define T_STOP 350 +#define T_STOPPED 351 +#define T_KILL 352 +#define T_INJECT 353 +#define T_STATE 354 +#define T_ASSIGNED_STATE 355 +#define T_IN 356 +#define T_GROUP 357 +#define T_LBRACE 358 +#define T_RBRACE 359 +#define T_COMMA 360 +#define T_POSTGRES 361 +#define T_STAYS 362 +#define T_WHILE 363 +#define T_THROUGH 364 +#define T_SET 365 +#define T_GET 366 +#define T_FSM 367 +#define T_LOGS 368 +#define T_NOT 369 +#define T_CONTAINS 370 +#define T_MATCHES 371 +#define T_INTEGER 372 +#define T_IDENT 373 +#define T_STRING 374 +#define T_BLOCK 375 +#define T_SHELL_ARGS 376 @@ -483,7 +485,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 193 of yacc.c. */ -#line 487 "test_spec_parse.c" +#line 489 "test_spec_parse.c" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 @@ -496,7 +498,7 @@ typedef union YYSTYPE /* Line 216 of yacc.c. */ -#line 500 "test_spec_parse.c" +#line 502 "test_spec_parse.c" #ifdef short # undef short @@ -711,20 +713,20 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 21 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 620 +#define YYLAST 634 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 121 +#define YYNTOKENS 122 /* YYNNTS -- Number of nonterminals. */ #define YYNNTS 65 /* YYNRULES -- Number of rules. */ -#define YYNRULES 214 +#define YYNRULES 215 /* YYNRULES -- Number of states. */ -#define YYNSTATES 355 +#define YYNSTATES 356 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 375 +#define YYMAXUTOK 376 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -769,7 +771,7 @@ static const yytype_uint8 yytranslate[] = 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120 + 115, 116, 117, 118, 119, 120, 121 }; #if YYDEBUG @@ -783,95 +785,95 @@ static const yytype_uint16 yyprhs[] = 83, 86, 89, 92, 95, 98, 101, 102, 109, 110, 113, 115, 117, 119, 121, 123, 125, 128, 131, 132, 135, 137, 139, 140, 141, 146, 147, 155, 156, 159, - 161, 163, 165, 167, 169, 171, 174, 177, 182, 185, - 187, 189, 191, 194, 197, 200, 203, 206, 209, 212, - 215, 218, 221, 224, 227, 230, 233, 237, 241, 244, - 247, 251, 255, 256, 259, 261, 263, 265, 267, 269, + 161, 163, 165, 167, 169, 171, 173, 176, 179, 184, + 187, 189, 191, 193, 196, 199, 202, 205, 208, 211, + 214, 217, 220, 223, 226, 229, 232, 235, 239, 243, + 246, 249, 253, 257, 258, 261, 263, 265, 267, 269, 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, - 291, 295, 298, 302, 305, 309, 312, 316, 319, 321, - 323, 325, 330, 335, 337, 341, 342, 345, 347, 349, - 353, 357, 358, 368, 369, 379, 387, 395, 401, 408, - 414, 421, 423, 425, 429, 433, 434, 437, 440, 445, - 446, 449, 453, 460, 467, 474, 481, 485, 488, 491, - 495, 499, 502, 504, 508, 511, 516, 522, 530, 534, - 538, 544, 550, 553, 556, 560, 564, 568, 573, 577, - 581, 585, 586, 592, 598, 602, 607, 613, 618, 624, - 627, 628, 631, 633, 635, 637, 639, 641, 643, 645, + 291, 293, 297, 300, 304, 307, 311, 314, 318, 321, + 323, 325, 327, 332, 337, 339, 343, 344, 347, 349, + 351, 355, 359, 360, 370, 371, 381, 389, 397, 403, + 410, 416, 423, 425, 427, 431, 435, 436, 439, 442, + 447, 448, 451, 455, 462, 469, 476, 483, 487, 490, + 493, 497, 501, 504, 506, 510, 513, 518, 524, 532, + 536, 540, 546, 552, 555, 558, 562, 566, 570, 575, + 579, 583, 587, 588, 594, 600, 604, 609, 615, 620, + 626, 629, 630, 633, 635, 637, 639, 641, 643, 645, 647, 649, 651, 653, 655, 657, 659, 661, 663, 665, - 667, 669, 671, 673, 675 + 667, 669, 671, 673, 675, 677 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { - 122, 0, -1, 123, -1, 122, 123, -1, 124, -1, - 146, -1, 147, -1, 148, -1, 182, -1, -1, 3, - 102, 125, 126, 103, -1, -1, 126, 127, -1, 128, - -1, 129, -1, 131, -1, 132, -1, 130, -1, 133, - -1, 44, -1, 45, -1, 4, -1, 4, 40, 117, - -1, 4, 14, 117, -1, 4, 36, 116, -1, 4, - 37, 118, -1, 4, 117, 25, 27, -1, 4, 117, - 31, 95, -1, 4, 117, 25, 27, 37, 118, -1, - 13, 118, -1, 13, 117, -1, 43, 117, -1, 43, - 118, -1, 15, 117, -1, 16, 117, -1, 17, 117, - -1, -1, 18, 134, 135, 102, 138, 103, -1, -1, - 135, 137, -1, 117, -1, 118, -1, 16, -1, 4, - -1, 5, -1, 136, -1, 19, 116, -1, 56, 29, - -1, -1, 138, 141, -1, 117, -1, 4, -1, -1, - -1, 139, 140, 142, 144, -1, -1, 5, 117, 140, - 143, 102, 144, 103, -1, -1, 144, 145, -1, 20, - -1, 21, -1, 22, -1, 23, -1, 24, -1, 27, - -1, 25, 27, -1, 26, 27, -1, 26, 76, 25, - 27, -1, 25, 28, -1, 28, -1, 33, -1, 34, - -1, 35, 116, -1, 46, 117, -1, 46, 118, -1, - 101, 116, -1, 36, 116, -1, 39, 117, -1, 40, - 117, -1, 15, 117, -1, 16, 117, -1, 17, 117, - -1, 41, 30, -1, 41, 29, -1, 42, 118, -1, - 38, 118, -1, 32, 117, 117, -1, 32, 117, 118, - -1, 8, 149, -1, 9, 149, -1, 10, 185, 149, - -1, 102, 150, 103, -1, -1, 150, 151, -1, 152, - -1, 158, -1, 165, -1, 166, -1, 167, -1, 168, - -1, 170, -1, 171, -1, 173, -1, 174, -1, 175, - -1, 176, -1, 179, -1, 180, -1, 181, -1, 172, - -1, 69, 117, 120, -1, 69, 117, -1, 70, 117, - 120, -1, 70, 117, -1, 71, 117, 120, -1, 71, - 117, -1, 72, 117, 120, -1, 72, 117, -1, 72, - -1, 12, -1, 77, -1, 117, 98, 153, 184, -1, - 117, 98, 153, 117, -1, 154, -1, 155, 76, 154, - -1, -1, 108, 157, -1, 184, -1, 117, -1, 157, - 104, 184, -1, 157, 104, 117, -1, -1, 73, 74, - 117, 98, 153, 184, 159, 156, 164, -1, -1, 73, - 74, 117, 98, 153, 117, 160, 156, 164, -1, 73, - 74, 117, 99, 153, 184, 164, -1, 73, 74, 117, - 99, 153, 117, 164, -1, 73, 74, 117, 95, 164, - -1, 73, 74, 117, 79, 117, 164, -1, 73, 74, - 161, 162, 164, -1, 73, 74, 154, 76, 155, 164, - -1, 184, -1, 117, -1, 161, 104, 184, -1, 161, - 104, 117, -1, -1, 100, 163, -1, 101, 116, -1, - 163, 104, 101, 116, -1, -1, 75, 116, -1, 78, - 75, 116, -1, 80, 117, 98, 153, 184, 164, -1, - 80, 117, 98, 153, 117, 164, -1, 80, 117, 99, - 153, 184, 164, -1, 80, 117, 99, 153, 117, 164, - -1, 81, 117, 119, -1, 82, 119, -1, 82, 83, - -1, 82, 83, 117, -1, 82, 83, 116, -1, 84, - 169, -1, 117, -1, 169, 104, 117, -1, 85, 86, - -1, 85, 86, 101, 116, -1, 85, 86, 100, 18, - 117, -1, 85, 86, 100, 18, 117, 101, 116, -1, - 87, 88, 117, -1, 87, 89, 117, -1, 47, 109, - 117, 117, 117, -1, 47, 110, 117, 117, 117, -1, - 90, 116, -1, 91, 92, -1, 91, 93, 117, -1, - 91, 94, 117, -1, 91, 96, 117, -1, 91, 97, - 117, 120, -1, 94, 105, 139, -1, 93, 105, 139, - -1, 111, 10, 139, -1, -1, 107, 178, 102, 150, - 103, -1, 80, 139, 106, 184, 177, -1, 109, 117, - 117, -1, 112, 117, 114, 118, -1, 112, 117, 113, - 114, 118, -1, 112, 117, 115, 118, -1, 112, 117, - 113, 115, 118, -1, 11, 183, -1, -1, 183, 185, - -1, 48, -1, 49, -1, 50, -1, 51, -1, 52, + 123, 0, -1, 124, -1, 123, 124, -1, 125, -1, + 147, -1, 148, -1, 149, -1, 183, -1, -1, 3, + 103, 126, 127, 104, -1, -1, 127, 128, -1, 129, + -1, 130, -1, 132, -1, 133, -1, 131, -1, 134, + -1, 45, -1, 46, -1, 4, -1, 4, 41, 118, + -1, 4, 14, 118, -1, 4, 37, 117, -1, 4, + 38, 119, -1, 4, 118, 26, 28, -1, 4, 118, + 32, 96, -1, 4, 118, 26, 28, 38, 119, -1, + 13, 119, -1, 13, 118, -1, 44, 118, -1, 44, + 119, -1, 15, 118, -1, 16, 118, -1, 17, 118, + -1, -1, 18, 135, 136, 103, 139, 104, -1, -1, + 136, 138, -1, 118, -1, 119, -1, 16, -1, 4, + -1, 5, -1, 137, -1, 19, 117, -1, 57, 30, + -1, -1, 139, 142, -1, 118, -1, 4, -1, -1, + -1, 140, 141, 143, 145, -1, -1, 5, 118, 141, + 144, 103, 145, 104, -1, -1, 145, 146, -1, 20, + -1, 21, -1, 22, -1, 23, -1, 24, -1, 25, + -1, 28, -1, 26, 28, -1, 27, 28, -1, 27, + 77, 26, 28, -1, 26, 29, -1, 29, -1, 34, + -1, 35, -1, 36, 117, -1, 47, 118, -1, 47, + 119, -1, 102, 117, -1, 37, 117, -1, 40, 118, + -1, 41, 118, -1, 15, 118, -1, 16, 118, -1, + 17, 118, -1, 42, 31, -1, 42, 30, -1, 43, + 119, -1, 39, 119, -1, 33, 118, 118, -1, 33, + 118, 119, -1, 8, 150, -1, 9, 150, -1, 10, + 186, 150, -1, 103, 151, 104, -1, -1, 151, 152, + -1, 153, -1, 159, -1, 166, -1, 167, -1, 168, + -1, 169, -1, 171, -1, 172, -1, 174, -1, 175, + -1, 176, -1, 177, -1, 180, -1, 181, -1, 182, + -1, 173, -1, 70, 118, 121, -1, 70, 118, -1, + 71, 118, 121, -1, 71, 118, -1, 72, 118, 121, + -1, 72, 118, -1, 73, 118, 121, -1, 73, 118, + -1, 73, -1, 12, -1, 78, -1, 118, 99, 154, + 185, -1, 118, 99, 154, 118, -1, 155, -1, 156, + 77, 155, -1, -1, 109, 158, -1, 185, -1, 118, + -1, 158, 105, 185, -1, 158, 105, 118, -1, -1, + 74, 75, 118, 99, 154, 185, 160, 157, 165, -1, + -1, 74, 75, 118, 99, 154, 118, 161, 157, 165, + -1, 74, 75, 118, 100, 154, 185, 165, -1, 74, + 75, 118, 100, 154, 118, 165, -1, 74, 75, 118, + 96, 165, -1, 74, 75, 118, 80, 118, 165, -1, + 74, 75, 162, 163, 165, -1, 74, 75, 155, 77, + 156, 165, -1, 185, -1, 118, -1, 162, 105, 185, + -1, 162, 105, 118, -1, -1, 101, 164, -1, 102, + 117, -1, 164, 105, 102, 117, -1, -1, 76, 117, + -1, 79, 76, 117, -1, 81, 118, 99, 154, 185, + 165, -1, 81, 118, 99, 154, 118, 165, -1, 81, + 118, 100, 154, 185, 165, -1, 81, 118, 100, 154, + 118, 165, -1, 82, 118, 120, -1, 83, 120, -1, + 83, 84, -1, 83, 84, 118, -1, 83, 84, 117, + -1, 85, 170, -1, 118, -1, 170, 105, 118, -1, + 86, 87, -1, 86, 87, 102, 117, -1, 86, 87, + 101, 18, 118, -1, 86, 87, 101, 18, 118, 102, + 117, -1, 88, 89, 118, -1, 88, 90, 118, -1, + 48, 110, 118, 118, 118, -1, 48, 111, 118, 118, + 118, -1, 91, 117, -1, 92, 93, -1, 92, 94, + 118, -1, 92, 95, 118, -1, 92, 97, 118, -1, + 92, 98, 118, 121, -1, 95, 106, 140, -1, 94, + 106, 140, -1, 112, 10, 140, -1, -1, 108, 179, + 103, 151, 104, -1, 81, 140, 107, 185, 178, -1, + 110, 118, 118, -1, 113, 118, 115, 119, -1, 113, + 118, 114, 115, 119, -1, 113, 118, 116, 119, -1, + 113, 118, 114, 116, 119, -1, 11, 184, -1, -1, + 184, 186, -1, 49, -1, 50, -1, 51, -1, 52, -1, 53, -1, 54, -1, 55, -1, 56, -1, 57, -1, 58, -1, 59, -1, 60, -1, 61, -1, 62, -1, 63, -1, 64, -1, 65, -1, 66, -1, 67, - -1, 68, -1, 117, -1, 118, -1 + -1, 68, -1, 69, -1, 118, -1, 119, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ @@ -883,22 +885,22 @@ static const yytype_uint16 yyrline[] = 337, 347, 353, 363, 373, 379, 390, 389, 406, 408, 417, 418, 419, 420, 421, 425, 430, 434, 440, 442, 461, 462, 471, 488, 487, 495, 494, 502, 504, 508, - 513, 518, 522, 526, 530, 536, 541, 545, 550, 554, - 558, 562, 566, 570, 575, 580, 584, 588, 594, 600, - 605, 610, 615, 619, 623, 629, 635, 649, 670, 677, - 688, 706, 721, 724, 732, 733, 734, 735, 736, 737, - 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, - 761, 768, 774, 781, 787, 794, 800, 808, 814, 841, - 841, 852, 867, 885, 886, 901, 903, 907, 915, 923, - 930, 942, 941, 953, 952, 963, 972, 981, 995, 1003, - 1017, 1032, 1038, 1045, 1051, 1064, 1066, 1070, 1075, 1083, - 1084, 1085, 1096, 1104, 1112, 1120, 1138, 1153, 1160, 1164, - 1170, 1183, 1191, 1199, 1220, 1227, 1234, 1242, 1258, 1264, - 1285, 1293, 1308, 1322, 1326, 1332, 1338, 1364, 1398, 1404, - 1425, 1442, 1442, 1447, 1466, 1491, 1500, 1509, 1518, 1534, - 1537, 1539, 1561, 1562, 1563, 1564, 1565, 1566, 1567, 1568, - 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, - 1579, 1580, 1581, 1589, 1590 + 513, 518, 522, 526, 530, 534, 540, 545, 549, 554, + 558, 562, 566, 570, 574, 579, 584, 588, 592, 598, + 604, 609, 614, 619, 623, 627, 633, 639, 653, 674, + 681, 692, 710, 725, 728, 736, 737, 738, 739, 740, + 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, + 751, 765, 772, 778, 785, 791, 798, 804, 812, 818, + 845, 845, 856, 871, 889, 890, 905, 907, 911, 919, + 927, 934, 946, 945, 957, 956, 967, 976, 985, 999, + 1007, 1021, 1036, 1042, 1049, 1055, 1068, 1070, 1074, 1079, + 1087, 1088, 1089, 1100, 1108, 1116, 1124, 1142, 1157, 1164, + 1168, 1174, 1187, 1195, 1203, 1224, 1231, 1238, 1246, 1262, + 1268, 1289, 1297, 1312, 1326, 1330, 1336, 1342, 1368, 1402, + 1408, 1429, 1446, 1446, 1451, 1470, 1495, 1504, 1513, 1522, + 1538, 1541, 1543, 1565, 1566, 1567, 1568, 1569, 1570, 1571, + 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, + 1582, 1583, 1584, 1585, 1593, 1594 }; #endif @@ -911,8 +913,8 @@ static const char *const yytname[] = "T_CITUS_COORDINATOR", "T_CITUS_WORKER", "T_SETUP", "T_TEARDOWN", "T_STEP", "T_SEQUENCE", "T_EQUALS", "T_IMAGE", "T_IMAGE_TARGET", "T_SSL", "T_AUTH", "T_AUTH_METHOD", "T_FORMATION", "T_NUM_SYNC", "T_COORDINATOR", - "T_WORKER", "T_ASYNC", "T_NO_MONITOR", "T_SUSPENDED", "T_LAUNCH", - "T_CREATE", "T_DEFERRED", "T_IMMEDIATE", "T_FALSE", "T_TRUE", + "T_WORKER", "T_ARCHIVER", "T_ASYNC", "T_NO_MONITOR", "T_SUSPENDED", + "T_LAUNCH", "T_CREATE", "T_DEFERRED", "T_IMMEDIATE", "T_FALSE", "T_TRUE", "T_INITIALLY", "T_VOLUME", "T_LISTEN", "T_CITUS_SECONDARY", "T_CANDIDATE_PRIORITY", "T_PORT", "T_PASSWORD", "T_MONITOR_PASSWORD", "T_CITUS_CLUSTER_NAME", "T_DEBIAN_CLUSTER", "T_REPLICATION_QUORUM", @@ -967,35 +969,35 @@ static const yytype_uint16 yytoknum[] = 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375 + 375, 376 }; # endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint8 yyr1[] = { - 0, 121, 122, 122, 123, 123, 123, 123, 123, 125, - 124, 126, 126, 127, 127, 127, 127, 127, 127, 127, - 127, 128, 128, 128, 128, 128, 128, 128, 128, 129, - 129, 130, 130, 131, 132, 132, 134, 133, 135, 135, - 136, 136, 136, 136, 136, 137, 137, 137, 138, 138, - 139, 139, 140, 142, 141, 143, 141, 144, 144, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 145, 145, - 145, 145, 145, 145, 145, 145, 145, 145, 146, 147, - 148, 149, 150, 150, 151, 151, 151, 151, 151, 151, - 151, 151, 151, 151, 151, 151, 151, 151, 151, 151, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 153, - 153, 154, 154, 155, 155, 156, 156, 157, 157, 157, - 157, 159, 158, 160, 158, 158, 158, 158, 158, 158, - 158, 161, 161, 161, 161, 162, 162, 163, 163, 164, - 164, 164, 165, 165, 165, 165, 166, 167, 167, 167, - 167, 168, 169, 169, 170, 170, 170, 170, 171, 171, - 172, 172, 173, 174, 174, 174, 174, 174, 175, 175, - 176, 178, 177, 179, 180, 181, 181, 181, 181, 182, - 183, 183, 184, 184, 184, 184, 184, 184, 184, 184, - 184, 184, 184, 184, 184, 184, 184, 184, 184, 184, - 184, 184, 184, 185, 185 + 0, 122, 123, 123, 124, 124, 124, 124, 124, 126, + 125, 127, 127, 128, 128, 128, 128, 128, 128, 128, + 128, 129, 129, 129, 129, 129, 129, 129, 129, 130, + 130, 131, 131, 132, 133, 133, 135, 134, 136, 136, + 137, 137, 137, 137, 137, 138, 138, 138, 139, 139, + 140, 140, 141, 143, 142, 144, 142, 145, 145, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 147, + 148, 149, 150, 151, 151, 152, 152, 152, 152, 152, + 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, + 152, 153, 153, 153, 153, 153, 153, 153, 153, 153, + 154, 154, 155, 155, 156, 156, 157, 157, 158, 158, + 158, 158, 160, 159, 161, 159, 159, 159, 159, 159, + 159, 159, 162, 162, 162, 162, 163, 163, 164, 164, + 165, 165, 165, 166, 166, 166, 166, 167, 168, 168, + 168, 168, 169, 170, 170, 171, 171, 171, 171, 172, + 172, 173, 173, 174, 175, 175, 175, 175, 175, 176, + 176, 177, 179, 178, 180, 181, 182, 182, 182, 182, + 183, 184, 184, 185, 185, 185, 185, 185, 185, 185, + 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, + 185, 185, 185, 185, 186, 186 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1007,22 +1009,22 @@ static const yytype_uint8 yyr2[] = 2, 2, 2, 2, 2, 2, 0, 6, 0, 2, 1, 1, 1, 1, 1, 1, 2, 2, 0, 2, 1, 1, 0, 0, 4, 0, 7, 0, 2, 1, - 1, 1, 1, 1, 1, 2, 2, 4, 2, 1, - 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, - 3, 3, 0, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 2, 4, 2, + 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 3, 3, 2, + 2, 3, 3, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 3, 2, 3, 2, 3, 2, 3, 2, 1, 1, - 1, 4, 4, 1, 3, 0, 2, 1, 1, 3, - 3, 0, 9, 0, 9, 7, 7, 5, 6, 5, - 6, 1, 1, 3, 3, 0, 2, 2, 4, 0, - 2, 3, 6, 6, 6, 6, 3, 2, 2, 3, - 3, 2, 1, 3, 2, 4, 5, 7, 3, 3, - 5, 5, 2, 2, 3, 3, 3, 4, 3, 3, - 3, 0, 5, 5, 3, 4, 5, 4, 5, 2, - 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 2, 3, 2, 3, 2, 3, 2, 1, + 1, 1, 4, 4, 1, 3, 0, 2, 1, 1, + 3, 3, 0, 9, 0, 9, 7, 7, 5, 6, + 5, 6, 1, 1, 3, 3, 0, 2, 2, 4, + 0, 2, 3, 6, 6, 6, 6, 3, 2, 2, + 3, 3, 2, 1, 3, 2, 4, 5, 7, 3, + 3, 5, 5, 2, 2, 3, 3, 3, 4, 3, + 3, 3, 0, 5, 5, 3, 4, 5, 4, 5, + 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -1030,42 +1032,42 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint8 yydefact[] = { - 0, 0, 0, 0, 0, 190, 0, 2, 4, 5, - 6, 7, 8, 9, 92, 88, 89, 213, 214, 0, - 189, 1, 3, 11, 0, 90, 191, 0, 0, 0, - 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 91, 0, 0, 0, 93, 94, - 95, 96, 97, 98, 99, 100, 101, 109, 102, 103, - 104, 105, 106, 107, 108, 21, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 191, 0, 2, 4, 5, + 6, 7, 8, 9, 93, 89, 90, 214, 215, 0, + 190, 1, 3, 11, 0, 91, 192, 0, 0, 0, + 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 92, 0, 0, 0, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 110, 103, 104, + 105, 106, 107, 108, 109, 21, 0, 0, 0, 0, 36, 0, 19, 20, 10, 12, 13, 14, 17, 15, - 16, 18, 0, 0, 111, 113, 115, 117, 0, 51, - 50, 0, 0, 158, 157, 162, 161, 164, 0, 0, - 172, 173, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 18, 0, 0, 112, 114, 116, 118, 0, 51, + 50, 0, 0, 159, 158, 163, 162, 165, 0, 0, + 173, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 29, 33, 34, - 35, 38, 31, 32, 0, 0, 110, 112, 114, 116, - 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, - 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, - 212, 142, 0, 145, 141, 0, 0, 0, 156, 160, - 159, 0, 0, 0, 168, 169, 174, 175, 176, 0, - 50, 179, 178, 184, 180, 0, 0, 0, 23, 24, - 25, 22, 0, 0, 0, 0, 0, 0, 149, 0, - 0, 0, 0, 0, 149, 119, 120, 0, 0, 0, - 163, 0, 165, 177, 0, 0, 185, 187, 26, 27, + 35, 38, 31, 32, 0, 0, 111, 113, 115, 117, + 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, + 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, + 213, 143, 0, 146, 142, 0, 0, 0, 157, 161, + 160, 0, 0, 0, 169, 170, 175, 176, 177, 0, + 50, 180, 179, 185, 181, 0, 0, 0, 23, 24, + 25, 22, 0, 0, 0, 0, 0, 0, 150, 0, + 0, 0, 0, 0, 150, 120, 121, 0, 0, 0, + 164, 0, 166, 178, 0, 0, 186, 188, 26, 27, 43, 44, 42, 0, 0, 48, 40, 41, 45, 39, - 170, 171, 149, 0, 0, 137, 0, 0, 0, 123, - 149, 0, 146, 144, 143, 139, 149, 149, 149, 149, - 181, 183, 166, 186, 188, 0, 46, 47, 0, 138, - 150, 0, 133, 131, 149, 149, 0, 0, 140, 147, - 0, 153, 152, 155, 154, 0, 0, 28, 0, 37, - 52, 49, 151, 125, 125, 136, 135, 0, 124, 0, - 92, 167, 52, 53, 0, 149, 149, 122, 121, 148, - 0, 55, 57, 128, 126, 127, 134, 132, 182, 0, + 171, 172, 150, 0, 0, 138, 0, 0, 0, 124, + 150, 0, 147, 145, 144, 140, 150, 150, 150, 150, + 182, 184, 167, 187, 189, 0, 46, 47, 0, 139, + 151, 0, 134, 132, 150, 150, 0, 0, 141, 148, + 0, 154, 153, 156, 155, 0, 0, 28, 0, 37, + 52, 49, 152, 126, 126, 137, 136, 0, 125, 0, + 93, 168, 52, 53, 0, 150, 150, 123, 122, 149, + 0, 55, 57, 129, 127, 128, 135, 133, 183, 0, 54, 0, 57, 0, 0, 0, 59, 60, 61, 62, - 63, 0, 0, 64, 69, 0, 70, 71, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 58, 130, 129, - 0, 79, 80, 81, 65, 68, 66, 0, 0, 72, - 76, 85, 77, 78, 83, 82, 84, 73, 74, 75, - 56, 0, 86, 87, 67 + 63, 64, 0, 0, 65, 70, 0, 71, 72, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 58, 131, + 130, 0, 80, 81, 82, 66, 69, 67, 0, 0, + 73, 77, 86, 78, 79, 84, 83, 85, 74, 75, + 76, 56, 0, 87, 88, 68 }; /* YYDEFGOTO[NTERM-NUM]. */ @@ -1073,7 +1075,7 @@ static const yytype_int16 yydefgoto[] = { -1, 6, 7, 8, 23, 27, 75, 76, 77, 78, 79, 80, 81, 121, 184, 218, 219, 248, 91, 283, - 271, 292, 299, 300, 327, 9, 10, 11, 15, 24, + 271, 292, 299, 300, 328, 9, 10, 11, 15, 24, 48, 49, 197, 152, 230, 285, 294, 50, 274, 273, 153, 194, 232, 225, 51, 52, 53, 54, 96, 55, 56, 57, 58, 59, 60, 61, 241, 265, 62, 63, @@ -1082,238 +1084,240 @@ static const yytype_int16 yydefgoto[] = /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -180 +#define YYPACT_NINF -179 static const yytype_int16 yypact[] = { - 77, -81, -74, -74, -36, -180, 39, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -74, - -36, -180, -180, -180, 442, -180, -180, 9, -17, -59, - -43, -40, -23, 30, -1, -4, -64, 0, 36, 7, - 26, -21, 21, 45, -180, 43, 160, 61, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -7, -20, 69, 94, 95, - -180, -18, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, 96, 98, 118, 119, 120, 121, 140, -180, - 3, 137, 125, -11, -180, -180, 143, 8, 128, 129, - -180, -180, 131, 132, 133, 134, 6, 6, 135, 6, - -24, 136, 138, 161, 139, 31, -180, -180, -180, -180, - -180, -180, -180, -180, 163, 164, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -61, 179, -69, -180, 2, 2, 552, -180, -180, - -180, 165, 265, 168, -180, -180, -180, -180, -180, 188, - -180, -180, -180, -180, -180, 10, 167, 191, -180, -180, - -180, -180, 283, 216, 1, 195, 196, 197, -32, 2, - 2, 198, 215, 169, -32, -180, -180, 210, 239, 211, - -180, 200, -180, -180, 201, 202, -180, -180, 284, -180, - -180, -180, -180, 206, 294, -180, -180, -180, -180, -180, - -180, -180, -32, 208, 250, -180, 280, 309, 228, -180, - -15, 233, 246, -180, -180, -180, -32, -32, -32, -32, - -180, -180, 251, -180, -180, 235, -180, -180, 4, -180, - -180, 238, 275, 279, -32, -32, 2, 198, -180, -180, - 277, -180, -180, -180, -180, 278, 263, -180, 264, -180, - -180, -180, -180, 274, 274, -180, -180, 350, -180, 267, - -180, -180, -180, -180, 379, -32, -32, -180, -180, -180, - 487, -180, -180, -180, 281, -180, -180, -180, -180, 282, - 141, 420, -180, 269, 270, 271, -180, -180, -180, -180, - -180, 104, -12, -180, -180, 272, -180, -180, 276, 303, - 273, 304, 305, 142, 302, 67, 307, -180, -180, -180, - 113, -180, -180, -180, -180, -180, -180, 365, 92, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, 366, -180, -180, -180 + 47, -64, -54, -54, -51, -179, 85, -179, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -54, + -51, -179, -179, -179, 455, -179, -179, 8, -33, -21, + -18, -5, 0, -16, -1, 10, -69, 19, 39, -3, + -52, -22, 25, 26, -179, 20, 129, 37, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -179, -179, -179, -179, -4, -29, 38, 45, 55, + -179, -27, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -179, 66, 67, 36, 65, 71, 77, 153, -179, + 2, 92, 80, -10, -179, -179, 118, 14, 106, 107, + -179, -179, 108, 110, 133, 134, 5, 5, 135, 5, + -86, 136, 138, 139, 141, 16, -179, -179, -179, -179, + -179, -179, -179, -179, 142, 143, -179, -179, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -53, 180, -70, -179, 6, 6, 565, -179, -179, + -179, 144, 245, 147, -179, -179, -179, -179, -179, 145, + -179, -179, -179, -179, -179, -12, 146, 148, -179, -179, + -179, -179, 240, 173, 3, 152, 175, 176, -59, 6, + 6, 177, 194, 181, -59, -179, -179, 223, 251, 189, + -179, 203, -179, -179, 179, 204, -179, -179, 284, -179, + -179, -179, -179, 207, 295, -179, -179, -179, -179, -179, + -179, -179, -59, 209, 252, -179, 293, 321, 228, -179, + -15, 212, 225, -179, -179, -179, -59, -59, -59, -59, + -179, -179, 229, -179, -179, 213, -179, -179, 1, -179, + -179, 216, 257, 258, -59, -59, 6, 177, -179, -179, + 234, -179, -179, -179, -179, 235, 220, -179, 221, -179, + -179, -179, -179, 231, 231, -179, -179, 363, -179, 246, + -179, -179, -179, -179, 391, -59, -59, -179, -179, -179, + 500, -179, -179, -179, 259, -179, -179, -179, -179, 262, + 154, 433, -179, 248, 249, 250, -179, -179, -179, -179, + -179, -179, 81, -14, -179, -179, 273, -179, -179, 275, + 276, 277, 279, 280, 94, 281, 15, 278, -179, -179, + -179, 125, -179, -179, -179, -179, -179, -179, 368, 17, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -179, 371, -179, -179, -179 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -180, -180, 388, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -105, 114, - -180, -180, -180, 93, -180, -180, -180, -180, 13, 144, - -180, -180, -145, -179, -180, 151, -180, -180, -180, -180, - -180, -180, -180, -171, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -157, 428 + -179, -179, 395, -179, -179, -179, -179, -179, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -105, 120, + -179, -179, -179, 101, -179, -179, -179, -179, 13, 124, + -179, -179, -145, -178, -179, 131, -179, -179, -179, -179, + -179, -179, -179, -156, -179, -179, -179, -179, -179, -179, + -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, + -179, -179, -179, -157, 386 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -123 +#define YYTABLE_NINF -124 static const yytype_int16 yytable[] = { - 199, 171, 172, 89, 174, 210, 211, 111, 89, 268, - 89, 198, 229, 65, 195, 336, 16, 212, 187, 93, - 213, 13, 66, 235, 67, 68, 69, 70, 14, 112, - 113, 192, 25, 114, 188, 193, 234, 189, 190, 21, - 237, 239, 1, 223, 226, 227, 224, 2, 3, 4, - 5, 249, 71, 72, 73, 94, 182, 214, 84, 258, - 223, 257, 183, 224, 337, 261, 262, 263, 264, 253, - 255, 101, 102, 103, 85, 104, 105, 86, 278, 196, - 1, 17, 18, 275, 276, 2, 3, 4, 5, 175, - 176, 177, 82, 83, 87, 98, 99, 116, 117, 122, - 123, 155, 156, 215, 88, 159, 160, 269, 162, 163, - 115, 277, 74, 92, 296, 297, 90, 95, 216, 217, - 288, 170, 97, 170, 204, 205, 106, 295, 303, 304, - 305, 334, 335, 306, 307, 308, 309, 310, 311, 312, - 313, 314, 100, 270, 329, 315, 316, 317, 318, 319, - 107, 320, 321, 322, 323, 324, 303, 304, 305, 325, - 108, 306, 307, 308, 309, 310, 311, 312, 313, 314, - 109, 344, 345, 315, 316, 317, 318, 319, 110, 320, - 321, 322, 323, 324, 347, 348, 118, 325, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 352, - 353, 119, 120, 124, 326, 125, 350, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 126, 127, - 128, 129, 326, 157, 158, 164, 165, 161, 166, 167, - 168, 169, 173, 178, 179, 191, 181, 151, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 180, - 185, 186, 200, 201, 202, 206, 233, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 203, 207, - 208, 209, 220, 221, 222, 228, 231, 242, 240, 243, - 244, 245, 246, 247, 250, 251, 256, 236, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 259, - 260, -122, 266, 267, 272, -121, 238, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 279, 281, - 280, 282, 284, 289, 302, 301, 331, 332, 333, 338, - 351, 341, 339, 354, 22, 330, 291, 252, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 340, - 346, 342, 343, 349, 290, 286, 254, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 26, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 287, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 149, 150, 28, - 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 29, 30, 31, 32, 33, 0, 0, 0, 0, - 0, 0, 34, 35, 36, 0, 37, 38, 0, 39, - 0, 0, 40, 41, 28, 42, 43, 328, 0, 0, - 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, - 0, 45, 0, 46, 47, 0, 29, 30, 31, 32, - 33, 0, 0, 0, 0, 0, 0, 34, 35, 36, - 0, 37, 38, 0, 39, 0, 0, 40, 41, 0, - 42, 43, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 0, 0, 0, 0, 0, 45, 0, 46, 47, + 199, 171, 172, 89, 174, 89, 268, 210, 211, 89, + 111, 198, 65, 229, 337, 93, 16, 223, 195, 212, + 224, 66, 213, 67, 68, 69, 70, 187, 175, 176, + 177, 192, 25, 112, 113, 193, 234, 114, 235, 13, + 237, 239, 182, 188, 226, 227, 189, 190, 183, 14, + 1, 94, 71, 72, 73, 2, 3, 4, 5, 88, + 214, 223, 257, 338, 224, 100, 249, 17, 18, 253, + 255, 101, 102, 103, 258, 104, 105, 82, 83, 278, + 261, 262, 263, 264, 196, 21, 98, 99, 1, 116, + 117, 122, 123, 2, 3, 4, 5, 84, 275, 276, + 85, 155, 156, 204, 205, 269, 215, 159, 160, 335, + 336, 277, 74, 86, 115, 162, 163, 90, 87, 170, + 288, 216, 217, 170, 345, 346, 97, 295, 92, 296, + 297, 106, 107, 348, 349, 353, 354, 95, 108, 109, + 303, 304, 305, 270, 330, 306, 307, 308, 309, 310, + 311, 312, 313, 314, 315, 110, 118, 126, 316, 317, + 318, 319, 320, 119, 321, 322, 323, 324, 325, 303, + 304, 305, 326, 120, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 124, 125, 127, 316, 317, 318, + 319, 320, 128, 321, 322, 323, 324, 325, 129, 157, + 158, 326, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 161, 164, 165, 166, 327, 167, 351, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 168, 169, 173, 178, 179, 327, 191, 180, 181, + 185, 186, 200, 201, 202, 206, 203, 207, 208, 209, + 220, 151, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 221, 222, 228, 231, 240, 243, 233, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 242, 245, 244, 246, 247, 250, 256, 251, 259, + 260, 266, 267, 272, -123, -122, 279, 281, 280, 282, + 284, 236, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 289, 301, 302, 332, 333, 334, 238, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150 + 150, 339, 340, 341, 352, 350, 342, 343, 344, 355, + 347, 22, 291, 331, 290, 286, 26, 0, 0, 0, + 0, 252, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 0, 0, 0, 0, 0, 0, 254, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 287, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 28, 0, 0, 0, 0, 0, 293, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 29, 30, 31, 32, 33, + 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, + 37, 38, 0, 39, 0, 0, 40, 41, 28, 42, + 43, 329, 0, 0, 0, 0, 0, 0, 0, 44, + 0, 0, 0, 0, 0, 45, 0, 46, 47, 0, + 29, 30, 31, 32, 33, 0, 0, 0, 0, 0, + 0, 34, 35, 36, 0, 37, 38, 0, 39, 0, + 0, 40, 41, 0, 42, 43, 0, 0, 0, 0, + 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, + 45, 0, 46, 47, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150 }; static const yytype_int16 yycheck[] = { - 157, 106, 107, 4, 109, 4, 5, 14, 4, 5, - 4, 156, 191, 4, 12, 27, 3, 16, 79, 83, - 19, 102, 13, 194, 15, 16, 17, 18, 102, 36, - 37, 100, 19, 40, 95, 104, 193, 98, 99, 0, - 197, 198, 3, 75, 189, 190, 78, 8, 9, 10, - 11, 222, 43, 44, 45, 119, 25, 56, 117, 230, - 75, 76, 31, 78, 76, 236, 237, 238, 239, 226, - 227, 92, 93, 94, 117, 96, 97, 117, 257, 77, - 3, 117, 118, 254, 255, 8, 9, 10, 11, 113, - 114, 115, 109, 110, 117, 88, 89, 117, 118, 117, - 118, 98, 99, 102, 74, 116, 117, 103, 100, 101, - 117, 256, 103, 117, 285, 286, 117, 117, 117, 118, - 277, 117, 86, 117, 114, 115, 105, 284, 15, 16, - 17, 27, 28, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 116, 248, 301, 32, 33, 34, 35, 36, - 105, 38, 39, 40, 41, 42, 15, 16, 17, 46, - 117, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 10, 29, 30, 32, 33, 34, 35, 36, 117, 38, - 39, 40, 41, 42, 117, 118, 117, 46, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 117, - 118, 117, 117, 117, 101, 117, 103, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 120, 120, - 120, 120, 101, 106, 119, 117, 117, 104, 117, 117, - 117, 117, 117, 117, 116, 76, 117, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 118, - 117, 117, 117, 18, 116, 118, 117, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 120, 118, - 27, 95, 117, 117, 117, 117, 101, 117, 107, 118, - 118, 37, 116, 29, 116, 75, 98, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 116, - 104, 76, 101, 118, 116, 76, 117, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 101, 116, - 102, 117, 108, 116, 102, 104, 117, 117, 117, 117, - 25, 118, 116, 27, 6, 302, 282, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 116, - 118, 117, 117, 116, 280, 274, 117, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 20, -1, + 157, 106, 107, 4, 109, 4, 5, 4, 5, 4, + 14, 156, 4, 191, 28, 84, 3, 76, 12, 16, + 79, 13, 19, 15, 16, 17, 18, 80, 114, 115, + 116, 101, 19, 37, 38, 105, 193, 41, 194, 103, + 197, 198, 26, 96, 189, 190, 99, 100, 32, 103, + 3, 120, 44, 45, 46, 8, 9, 10, 11, 75, + 57, 76, 77, 77, 79, 117, 222, 118, 119, 226, + 227, 93, 94, 95, 230, 97, 98, 110, 111, 257, + 236, 237, 238, 239, 78, 0, 89, 90, 3, 118, + 119, 118, 119, 8, 9, 10, 11, 118, 254, 255, + 118, 99, 100, 115, 116, 104, 103, 117, 118, 28, + 29, 256, 104, 118, 118, 101, 102, 118, 118, 118, + 277, 118, 119, 118, 30, 31, 87, 284, 118, 285, + 286, 106, 106, 118, 119, 118, 119, 118, 118, 10, + 15, 16, 17, 248, 301, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 118, 118, 121, 33, 34, + 35, 36, 37, 118, 39, 40, 41, 42, 43, 15, + 16, 17, 47, 118, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 118, 118, 121, 33, 34, 35, + 36, 37, 121, 39, 40, 41, 42, 43, 121, 107, + 120, 47, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 105, 118, 118, 118, 102, 118, 104, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 118, 118, 118, 118, 117, 102, 77, 119, 118, + 118, 118, 118, 18, 117, 119, 121, 119, 28, 96, + 118, 118, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 118, 118, 118, 102, 108, 119, 118, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 118, 38, 119, 117, 30, 117, 99, 76, 117, + 105, 102, 119, 117, 77, 77, 102, 117, 103, 118, + 109, 118, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 117, 105, 103, 118, 118, 118, 118, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 118, 117, 117, 26, 117, 119, 118, 118, 28, + 119, 6, 282, 302, 280, 274, 20, -1, -1, -1, + -1, 118, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, -1, -1, -1, -1, -1, -1, 118, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 117, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 47, - -1, -1, -1, -1, -1, -1, 117, -1, -1, -1, + -1, 118, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 48, -1, -1, -1, -1, -1, 118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 69, 70, 71, 72, 73, -1, -1, -1, -1, - -1, -1, 80, 81, 82, -1, 84, 85, -1, 87, - -1, -1, 90, 91, 47, 93, 94, 117, -1, -1, - -1, -1, -1, -1, -1, 103, -1, -1, -1, -1, - -1, 109, -1, 111, 112, -1, 69, 70, 71, 72, - 73, -1, -1, -1, -1, -1, -1, 80, 81, 82, - -1, 84, 85, -1, 87, -1, -1, 90, 91, -1, - 93, 94, -1, -1, -1, -1, -1, -1, -1, -1, - 103, -1, -1, -1, -1, -1, 109, -1, 111, 112, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68 + -1, -1, -1, -1, -1, 70, 71, 72, 73, 74, + -1, -1, -1, -1, -1, -1, 81, 82, 83, -1, + 85, 86, -1, 88, -1, -1, 91, 92, 48, 94, + 95, 118, -1, -1, -1, -1, -1, -1, -1, 104, + -1, -1, -1, -1, -1, 110, -1, 112, 113, -1, + 70, 71, 72, 73, 74, -1, -1, -1, -1, -1, + -1, 81, 82, 83, -1, 85, 86, -1, 88, -1, + -1, 91, 92, -1, 94, 95, -1, -1, -1, -1, + -1, -1, -1, -1, 104, -1, -1, -1, -1, -1, + 110, -1, 112, 113, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 3, 8, 9, 10, 11, 122, 123, 124, 146, - 147, 148, 182, 102, 102, 149, 149, 117, 118, 185, - 183, 0, 123, 125, 150, 149, 185, 126, 47, 69, - 70, 71, 72, 73, 80, 81, 82, 84, 85, 87, - 90, 91, 93, 94, 103, 109, 111, 112, 151, 152, - 158, 165, 166, 167, 168, 170, 171, 172, 173, 174, - 175, 176, 179, 180, 181, 4, 13, 15, 16, 17, - 18, 43, 44, 45, 103, 127, 128, 129, 130, 131, - 132, 133, 109, 110, 117, 117, 117, 117, 74, 4, - 117, 139, 117, 83, 119, 117, 169, 86, 88, 89, - 116, 92, 93, 94, 96, 97, 105, 105, 117, 10, - 117, 14, 36, 37, 40, 117, 117, 118, 117, 117, - 117, 134, 117, 118, 117, 117, 120, 120, 120, 120, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 117, 154, 161, 184, 98, 99, 106, 119, 116, - 117, 104, 100, 101, 117, 117, 117, 117, 117, 117, - 117, 139, 139, 117, 139, 113, 114, 115, 117, 116, - 118, 117, 25, 31, 135, 117, 117, 79, 95, 98, - 99, 76, 100, 104, 162, 12, 77, 153, 153, 184, - 117, 18, 116, 120, 114, 115, 118, 118, 27, 95, - 4, 5, 16, 19, 56, 102, 117, 118, 136, 137, - 117, 117, 117, 75, 78, 164, 153, 153, 117, 154, - 155, 101, 163, 117, 184, 164, 117, 184, 117, 184, - 107, 177, 117, 118, 118, 37, 116, 29, 138, 164, - 116, 75, 117, 184, 117, 184, 98, 76, 164, 116, - 104, 164, 164, 164, 164, 178, 101, 118, 5, 103, - 139, 141, 116, 160, 159, 164, 164, 153, 154, 101, - 102, 116, 117, 140, 108, 156, 156, 117, 184, 116, - 150, 140, 142, 117, 157, 184, 164, 164, 103, 143, - 144, 104, 102, 15, 16, 17, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 32, 33, 34, 35, 36, - 38, 39, 40, 41, 42, 46, 101, 145, 117, 184, - 144, 117, 117, 117, 27, 28, 27, 76, 117, 116, - 116, 118, 117, 117, 29, 30, 118, 117, 118, 116, - 103, 25, 117, 118, 27 + 0, 3, 8, 9, 10, 11, 123, 124, 125, 147, + 148, 149, 183, 103, 103, 150, 150, 118, 119, 186, + 184, 0, 124, 126, 151, 150, 186, 127, 48, 70, + 71, 72, 73, 74, 81, 82, 83, 85, 86, 88, + 91, 92, 94, 95, 104, 110, 112, 113, 152, 153, + 159, 166, 167, 168, 169, 171, 172, 173, 174, 175, + 176, 177, 180, 181, 182, 4, 13, 15, 16, 17, + 18, 44, 45, 46, 104, 128, 129, 130, 131, 132, + 133, 134, 110, 111, 118, 118, 118, 118, 75, 4, + 118, 140, 118, 84, 120, 118, 170, 87, 89, 90, + 117, 93, 94, 95, 97, 98, 106, 106, 118, 10, + 118, 14, 37, 38, 41, 118, 118, 119, 118, 118, + 118, 135, 118, 119, 118, 118, 121, 121, 121, 121, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 118, 155, 162, 185, 99, 100, 107, 120, 117, + 118, 105, 101, 102, 118, 118, 118, 118, 118, 118, + 118, 140, 140, 118, 140, 114, 115, 116, 118, 117, + 119, 118, 26, 32, 136, 118, 118, 80, 96, 99, + 100, 77, 101, 105, 163, 12, 78, 154, 154, 185, + 118, 18, 117, 121, 115, 116, 119, 119, 28, 96, + 4, 5, 16, 19, 57, 103, 118, 119, 137, 138, + 118, 118, 118, 76, 79, 165, 154, 154, 118, 155, + 156, 102, 164, 118, 185, 165, 118, 185, 118, 185, + 108, 178, 118, 119, 119, 38, 117, 30, 139, 165, + 117, 76, 118, 185, 118, 185, 99, 77, 165, 117, + 105, 165, 165, 165, 165, 179, 102, 119, 5, 104, + 140, 142, 117, 161, 160, 165, 165, 154, 155, 102, + 103, 117, 118, 141, 109, 157, 157, 118, 185, 117, + 151, 141, 143, 118, 158, 185, 165, 165, 104, 144, + 145, 105, 103, 15, 16, 17, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 33, 34, 35, 36, + 37, 39, 40, 41, 42, 43, 47, 102, 146, 118, + 185, 145, 118, 118, 118, 28, 29, 28, 77, 118, + 117, 117, 119, 118, 118, 30, 31, 119, 118, 119, + 117, 104, 26, 118, 119, 28 }; #define yyerrok (yyerrstatus = 0) @@ -2413,45 +2417,45 @@ yyparse () case 61: #line 519 "test_spec_parse.y" { - current_node->replicationQuorum = false; + current_node->kind = NODE_KIND_ARCHIVER; ;} break; case 62: #line 523 "test_spec_parse.y" { - current_node->noMonitor = true; + current_node->replicationQuorum = false; ;} break; case 63: #line 527 "test_spec_parse.y" { - current_node->suspended = true; + current_node->noMonitor = true; ;} break; case 64: #line 531 "test_spec_parse.y" { - /* bare "deferred" = create and launch deferred (both gates) */ - current_node->createDeferred = true; - current_node->launchDeferred = true; + current_node->suspended = true; ;} break; case 65: -#line 537 "test_spec_parse.y" +#line 535 "test_spec_parse.y" { - /* "launch deferred" alone = run-deferred only, create immediate */ + /* bare "deferred" = create and launch deferred (both gates) */ + current_node->createDeferred = true; current_node->launchDeferred = true; ;} break; case 66: -#line 542 "test_spec_parse.y" +#line 541 "test_spec_parse.y" { - current_node->createDeferred = true; + /* "launch deferred" alone = run-deferred only, create immediate */ + current_node->launchDeferred = true; ;} break; @@ -2459,14 +2463,14 @@ yyparse () #line 546 "test_spec_parse.y" { current_node->createDeferred = true; - current_node->launchDeferred = true; ;} break; case 68: -#line 551 "test_spec_parse.y" +#line 550 "test_spec_parse.y" { - current_node->launchDeferred = false; + current_node->createDeferred = true; + current_node->launchDeferred = true; ;} break; @@ -2480,34 +2484,33 @@ yyparse () case 70: #line 559 "test_spec_parse.y" { - current_node->listen = true; + current_node->launchDeferred = false; ;} break; case 71: #line 563 "test_spec_parse.y" { - current_node->citusSecondary = true; + current_node->listen = true; ;} break; case 72: #line 567 "test_spec_parse.y" { - current_node->candidatePriority = (yyvsp[(2) - (2)].ival); + current_node->citusSecondary = true; ;} break; case 73: #line 571 "test_spec_parse.y" { - strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); - free((yyvsp[(2) - (2)].str)); + current_node->candidatePriority = (yyvsp[(2) - (2)].ival); ;} break; case 74: -#line 576 "test_spec_parse.y" +#line 575 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); @@ -2515,55 +2518,55 @@ yyparse () break; case 75: -#line 581 "test_spec_parse.y" +#line 580 "test_spec_parse.y" { - current_node->group = (yyvsp[(2) - (2)].ival); + strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); + free((yyvsp[(2) - (2)].str)); ;} break; case 76: #line 585 "test_spec_parse.y" { - current_node->pgPort = (yyvsp[(2) - (2)].ival); + current_node->group = (yyvsp[(2) - (2)].ival); ;} break; case 77: #line 589 "test_spec_parse.y" { - strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), - sizeof(current_node->citusClusterName)); - free((yyvsp[(2) - (2)].str)); + current_node->pgPort = (yyvsp[(2) - (2)].ival); ;} break; case 78: -#line 595 "test_spec_parse.y" +#line 593 "test_spec_parse.y" { - strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), - sizeof(current_node->debianCluster)); + strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), + sizeof(current_node->citusClusterName)); free((yyvsp[(2) - (2)].str)); ;} break; case 79: -#line 601 "test_spec_parse.y" +#line 599 "test_spec_parse.y" { - strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); + strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), + sizeof(current_node->debianCluster)); free((yyvsp[(2) - (2)].str)); ;} break; case 80: -#line 606 "test_spec_parse.y" +#line 605 "test_spec_parse.y" { - strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); + strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); free((yyvsp[(2) - (2)].str)); ;} break; case 81: -#line 611 "test_spec_parse.y" +#line 610 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); @@ -2571,21 +2574,29 @@ yyparse () break; case 82: -#line 616 "test_spec_parse.y" +#line 615 "test_spec_parse.y" { - current_node->replicationQuorum = true; + strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); + free((yyvsp[(2) - (2)].str)); ;} break; case 83: #line 620 "test_spec_parse.y" { - current_node->replicationQuorum = false; + current_node->replicationQuorum = true; ;} break; case 84: #line 624 "test_spec_parse.y" + { + current_node->replicationQuorum = false; + ;} + break; + + case 85: +#line 628 "test_spec_parse.y" { strlcpy(current_node->replicationPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->replicationPassword)); @@ -2593,8 +2604,8 @@ yyparse () ;} break; - case 85: -#line 630 "test_spec_parse.y" + case 86: +#line 634 "test_spec_parse.y" { strlcpy(current_node->monitorPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->monitorPassword)); @@ -2602,8 +2613,8 @@ yyparse () ;} break; - case 86: -#line 636 "test_spec_parse.y" + case 87: +#line 640 "test_spec_parse.y" { /* volume — adds a named Docker volume */ int vi = current_node->volumeCount; @@ -2619,8 +2630,8 @@ yyparse () ;} break; - case 87: -#line 650 "test_spec_parse.y" + case 88: +#line 654 "test_spec_parse.y" { /* volume "/path/with spaces" */ int vi = current_node->volumeCount; @@ -2636,22 +2647,22 @@ yyparse () ;} break; - case 88: -#line 671 "test_spec_parse.y" + case 89: +#line 675 "test_spec_parse.y" { current_spec->setup = (yyvsp[(2) - (2)].step); ;} break; - case 89: -#line 678 "test_spec_parse.y" + case 90: +#line 682 "test_spec_parse.y" { current_spec->teardown = (yyvsp[(2) - (2)].step); ;} break; - case 90: -#line 689 "test_spec_parse.y" + case 91: +#line 693 "test_spec_parse.y" { TestStep *s = (yyvsp[(3) - (3)].step); strncpy(s->name, (yyvsp[(2) - (3)].str), sizeof(s->name) - 1); @@ -2660,8 +2671,8 @@ yyparse () ;} break; - case 91: -#line 707 "test_spec_parse.y" + case 92: +#line 711 "test_spec_parse.y" { /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ for (TestCmd *c = (yyvsp[(2) - (3)].step)->commands; c; c = c->next) @@ -2674,103 +2685,103 @@ yyparse () ;} break; - case 92: -#line 721 "test_spec_parse.y" + case 93: +#line 725 "test_spec_parse.y" { (yyval.step) = make_step(""); ;} break; - case 93: -#line 725 "test_spec_parse.y" + case 94: +#line 729 "test_spec_parse.y" { if ((yyvsp[(2) - (2)].cmd)) append_cmd((yyvsp[(1) - (2)].step), (yyvsp[(2) - (2)].cmd)); (yyval.step) = (yyvsp[(1) - (2)].step); ;} break; - case 94: -#line 732 "test_spec_parse.y" - { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} - break; - case 95: -#line 733 "test_spec_parse.y" +#line 736 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 96: -#line 734 "test_spec_parse.y" +#line 737 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 97: -#line 735 "test_spec_parse.y" +#line 738 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 98: -#line 736 "test_spec_parse.y" +#line 739 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 99: -#line 737 "test_spec_parse.y" +#line 740 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 100: -#line 738 "test_spec_parse.y" +#line 741 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 101: -#line 739 "test_spec_parse.y" +#line 742 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 102: -#line 740 "test_spec_parse.y" +#line 743 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 103: -#line 741 "test_spec_parse.y" +#line 744 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 104: -#line 742 "test_spec_parse.y" +#line 745 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 105: -#line 743 "test_spec_parse.y" +#line 746 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 106: -#line 744 "test_spec_parse.y" +#line 747 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 107: -#line 745 "test_spec_parse.y" +#line 748 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 108: -#line 746 "test_spec_parse.y" +#line 749 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 109: -#line 747 "test_spec_parse.y" +#line 750 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 110: -#line 762 "test_spec_parse.y" +#line 751 "test_spec_parse.y" + { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} + break; + + case 111: +#line 766 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2779,8 +2790,8 @@ yyparse () ;} break; - case 111: -#line 769 "test_spec_parse.y" + case 112: +#line 773 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2788,8 +2799,8 @@ yyparse () ;} break; - case 112: -#line 775 "test_spec_parse.y" + case 113: +#line 779 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2798,8 +2809,8 @@ yyparse () ;} break; - case 113: -#line 782 "test_spec_parse.y" + case 114: +#line 786 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2807,8 +2818,8 @@ yyparse () ;} break; - case 114: -#line 788 "test_spec_parse.y" + case 115: +#line 792 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2817,8 +2828,8 @@ yyparse () ;} break; - case 115: -#line 795 "test_spec_parse.y" + case 116: +#line 799 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2826,8 +2837,8 @@ yyparse () ;} break; - case 116: -#line 801 "test_spec_parse.y" + case 117: +#line 805 "test_spec_parse.y" { /* "pg_autoctl perform failover --formation auth" * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ @@ -2837,8 +2848,8 @@ yyparse () ;} break; - case 117: -#line 809 "test_spec_parse.y" + case 118: +#line 813 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); strlcpy((yyval.cmd)->args, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->args)); @@ -2846,15 +2857,15 @@ yyparse () ;} break; - case 118: -#line 815 "test_spec_parse.y" + case 119: +#line 819 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); ;} break; - case 121: -#line 853 "test_spec_parse.y" + case 122: +#line 857 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2871,8 +2882,8 @@ yyparse () ;} break; - case 122: -#line 868 "test_spec_parse.y" + case 123: +#line 872 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2889,8 +2900,8 @@ yyparse () ;} break; - case 127: -#line 908 "test_spec_parse.y" + case 128: +#line 912 "test_spec_parse.y" { /* current_pass_cmd set by the enclosing wait_cmd rule */ if (current_pass_cmd && @@ -2900,8 +2911,8 @@ yyparse () ;} break; - case 128: -#line 916 "test_spec_parse.y" + case 129: +#line 920 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2911,8 +2922,8 @@ yyparse () ;} break; - case 129: -#line 924 "test_spec_parse.y" + case 130: +#line 928 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2921,8 +2932,8 @@ yyparse () ;} break; - case 130: -#line 931 "test_spec_parse.y" + case 131: +#line 935 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2932,16 +2943,16 @@ yyparse () ;} break; - case 131: -#line 942 "test_spec_parse.y" + case 132: +#line 946 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); ;} break; - case 132: -#line 947 "test_spec_parse.y" + case 133: +#line 951 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2949,16 +2960,16 @@ yyparse () ;} break; - case 133: -#line 953 "test_spec_parse.y" + case 134: +#line 957 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); free((yyvsp[(6) - (6)].str)); ;} break; - case 134: -#line 958 "test_spec_parse.y" + case 135: +#line 962 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2966,8 +2977,8 @@ yyparse () ;} break; - case 135: -#line 964 "test_spec_parse.y" + case 136: +#line 968 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -2978,8 +2989,8 @@ yyparse () ;} break; - case 136: -#line 973 "test_spec_parse.y" + case 137: +#line 977 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -2990,8 +3001,8 @@ yyparse () ;} break; - case 137: -#line 982 "test_spec_parse.y" + case 138: +#line 986 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STOPPED); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3000,8 +3011,8 @@ yyparse () ;} break; - case 138: -#line 996 "test_spec_parse.y" + case 139: +#line 1000 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_LSN); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3011,8 +3022,8 @@ yyparse () ;} break; - case 139: -#line 1004 "test_spec_parse.y" + case 140: +#line 1008 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); @@ -3020,8 +3031,8 @@ yyparse () ;} break; - case 140: -#line 1018 "test_spec_parse.y" + case 141: +#line 1022 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); @@ -3029,8 +3040,8 @@ yyparse () ;} break; - case 141: -#line 1033 "test_spec_parse.y" + case 142: +#line 1037 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3038,8 +3049,8 @@ yyparse () ;} break; - case 142: -#line 1039 "test_spec_parse.y" + case 143: +#line 1043 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3048,8 +3059,8 @@ yyparse () ;} break; - case 143: -#line 1046 "test_spec_parse.y" + case 144: +#line 1050 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3057,8 +3068,8 @@ yyparse () ;} break; - case 144: -#line 1052 "test_spec_parse.y" + case 145: +#line 1056 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3067,39 +3078,39 @@ yyparse () ;} break; - case 147: -#line 1071 "test_spec_parse.y" + case 148: +#line 1075 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(2) - (2)].ival); ;} break; - case 148: -#line 1076 "test_spec_parse.y" + case 149: +#line 1080 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(4) - (4)].ival); ;} break; - case 149: -#line 1083 "test_spec_parse.y" + case 150: +#line 1087 "test_spec_parse.y" { (yyval.ival) = PGAF_TIMEOUT_DEFAULT; ;} break; - case 150: -#line 1084 "test_spec_parse.y" + case 151: +#line 1088 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} break; - case 151: -#line 1085 "test_spec_parse.y" + case 152: +#line 1089 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 152: -#line 1097 "test_spec_parse.y" + case 153: +#line 1101 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3109,8 +3120,8 @@ yyparse () ;} break; - case 153: -#line 1105 "test_spec_parse.y" + case 154: +#line 1109 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3120,8 +3131,8 @@ yyparse () ;} break; - case 154: -#line 1113 "test_spec_parse.y" + case 155: +#line 1117 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3131,8 +3142,8 @@ yyparse () ;} break; - case 155: -#line 1121 "test_spec_parse.y" + case 156: +#line 1125 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3142,8 +3153,8 @@ yyparse () ;} break; - case 156: -#line 1139 "test_spec_parse.y" + case 157: +#line 1143 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SQL); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3152,8 +3163,8 @@ yyparse () ;} break; - case 157: -#line 1154 "test_spec_parse.y" + case 158: +#line 1158 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT); strlcpy((yyval.cmd)->expected, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->expected)); @@ -3162,15 +3173,15 @@ yyparse () ;} break; - case 158: -#line 1161 "test_spec_parse.y" + case 159: +#line 1165 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); ;} break; - case 159: -#line 1165 "test_spec_parse.y" + case 160: +#line 1169 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); strlcpy((yyval.cmd)->state, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->state)); @@ -3178,8 +3189,8 @@ yyparse () ;} break; - case 160: -#line 1171 "test_spec_parse.y" + case 161: +#line 1175 "test_spec_parse.y" { /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); @@ -3187,16 +3198,16 @@ yyparse () ;} break; - case 161: -#line 1184 "test_spec_parse.y" + case 162: +#line 1188 "test_spec_parse.y" { (yyval.cmd) = current_promote_cmd; current_promote_cmd = NULL; ;} break; - case 162: -#line 1192 "test_spec_parse.y" + case 163: +#line 1196 "test_spec_parse.y" { current_promote_cmd = make_cmd(CMD_PROMOTE); current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; @@ -3206,8 +3217,8 @@ yyparse () ;} break; - case 163: -#line 1200 "test_spec_parse.y" + case 164: +#line 1204 "test_spec_parse.y" { if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], @@ -3216,8 +3227,8 @@ yyparse () ;} break; - case 164: -#line 1221 "test_spec_parse.y" + case 165: +#line 1225 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3226,8 +3237,8 @@ yyparse () ;} break; - case 165: -#line 1228 "test_spec_parse.y" + case 166: +#line 1232 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3236,8 +3247,8 @@ yyparse () ;} break; - case 166: -#line 1235 "test_spec_parse.y" + case 167: +#line 1239 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3247,8 +3258,8 @@ yyparse () ;} break; - case 167: -#line 1243 "test_spec_parse.y" + case 168: +#line 1247 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (7)].str), sizeof((yyval.cmd)->service)); @@ -3258,8 +3269,8 @@ yyparse () ;} break; - case 168: -#line 1259 "test_spec_parse.y" + case 169: +#line 1263 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_OFF); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3267,8 +3278,8 @@ yyparse () ;} break; - case 169: -#line 1265 "test_spec_parse.y" + case 170: +#line 1269 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_ON); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3276,8 +3287,8 @@ yyparse () ;} break; - case 170: -#line 1286 "test_spec_parse.y" + case 171: +#line 1290 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_SET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3287,8 +3298,8 @@ yyparse () ;} break; - case 171: -#line 1294 "test_spec_parse.y" + case 172: +#line 1298 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_GET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3298,23 +3309,23 @@ yyparse () ;} break; - case 172: -#line 1309 "test_spec_parse.y" + case 173: +#line 1313 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SLEEP); (yyval.cmd)->timeoutSeconds = (yyvsp[(2) - (2)].ival); ;} break; - case 173: -#line 1323 "test_spec_parse.y" + case 174: +#line 1327 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_DOWN); ;} break; - case 174: -#line 1327 "test_spec_parse.y" + case 175: +#line 1331 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_START); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3322,8 +3333,8 @@ yyparse () ;} break; - case 175: -#line 1333 "test_spec_parse.y" + case 176: +#line 1337 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_STOP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3331,8 +3342,8 @@ yyparse () ;} break; - case 176: -#line 1339 "test_spec_parse.y" + case 177: +#line 1343 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_KILL); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3340,8 +3351,8 @@ yyparse () ;} break; - case 177: -#line 1365 "test_spec_parse.y" + case 178: +#line 1369 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_INJECT); strlcpy((yyval.cmd)->expected, (yyvsp[(3) - (4)].str), sizeof((yyval.cmd)->expected)); /* image */ @@ -3366,8 +3377,8 @@ yyparse () ;} break; - case 178: -#line 1399 "test_spec_parse.y" + case 179: +#line 1403 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STOP_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3375,8 +3386,8 @@ yyparse () ;} break; - case 179: -#line 1405 "test_spec_parse.y" + case 180: +#line 1409 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_START_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3384,8 +3395,8 @@ yyparse () ;} break; - case 180: -#line 1426 "test_spec_parse.y" + case 181: +#line 1430 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FSM_STEP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3393,18 +3404,18 @@ yyparse () ;} break; - case 181: -#line 1442 "test_spec_parse.y" + case 182: +#line 1446 "test_spec_parse.y" { pgaf_next_brace_is_while = 1; ;} break; - case 182: -#line 1443 "test_spec_parse.y" + case 183: +#line 1447 "test_spec_parse.y" { (yyval.step) = (yyvsp[(4) - (5)].step); ;} break; - case 183: -#line 1448 "test_spec_parse.y" + case 184: +#line 1452 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STAYS_WHILE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3414,8 +3425,8 @@ yyparse () ;} break; - case 184: -#line 1467 "test_spec_parse.y" + case 185: +#line 1471 "test_spec_parse.y" { /* only "set monitor " is supported; $2 must be "monitor" */ if (strcmp((yyvsp[(2) - (3)].str), "monitor") != 0) @@ -3430,8 +3441,8 @@ yyparse () ;} break; - case 185: -#line 1492 "test_spec_parse.y" + case 186: +#line 1496 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3442,8 +3453,8 @@ yyparse () ;} break; - case 186: -#line 1501 "test_spec_parse.y" + case 187: +#line 1505 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3454,8 +3465,8 @@ yyparse () ;} break; - case 187: -#line 1510 "test_spec_parse.y" + case 188: +#line 1514 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3466,8 +3477,8 @@ yyparse () ;} break; - case 188: -#line 1519 "test_spec_parse.y" + case 189: +#line 1523 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3478,8 +3489,8 @@ yyparse () ;} break; - case 191: -#line 1540 "test_spec_parse.y" + case 192: +#line 1544 "test_spec_parse.y" { int i = current_spec->sequenceLength; if (i < PGAF_MAX_SEQ) @@ -3493,124 +3504,124 @@ yyparse () ;} break; - case 192: -#line 1561 "test_spec_parse.y" + case 193: +#line 1565 "test_spec_parse.y" { (yyval.str) = "init"; ;} break; - case 193: -#line 1562 "test_spec_parse.y" + case 194: +#line 1566 "test_spec_parse.y" { (yyval.str) = "single"; ;} break; - case 194: -#line 1563 "test_spec_parse.y" + case 195: +#line 1567 "test_spec_parse.y" { (yyval.str) = "primary"; ;} break; - case 195: -#line 1564 "test_spec_parse.y" + case 196: +#line 1568 "test_spec_parse.y" { (yyval.str) = "wait_primary"; ;} break; - case 196: -#line 1565 "test_spec_parse.y" + case 197: +#line 1569 "test_spec_parse.y" { (yyval.str) = "wait_standby"; ;} break; - case 197: -#line 1566 "test_spec_parse.y" + case 198: +#line 1570 "test_spec_parse.y" { (yyval.str) = "demoted"; ;} break; - case 198: -#line 1567 "test_spec_parse.y" + case 199: +#line 1571 "test_spec_parse.y" { (yyval.str) = "demote_timeout"; ;} break; - case 199: -#line 1568 "test_spec_parse.y" + case 200: +#line 1572 "test_spec_parse.y" { (yyval.str) = "draining"; ;} break; - case 200: -#line 1569 "test_spec_parse.y" + case 201: +#line 1573 "test_spec_parse.y" { (yyval.str) = "secondary"; ;} break; - case 201: -#line 1570 "test_spec_parse.y" + case 202: +#line 1574 "test_spec_parse.y" { (yyval.str) = "catchingup"; ;} break; - case 202: -#line 1571 "test_spec_parse.y" + case 203: +#line 1575 "test_spec_parse.y" { (yyval.str) = "prepare_promotion"; ;} break; - case 203: -#line 1572 "test_spec_parse.y" + case 204: +#line 1576 "test_spec_parse.y" { (yyval.str) = "stop_replication"; ;} break; - case 204: -#line 1573 "test_spec_parse.y" + case 205: +#line 1577 "test_spec_parse.y" { (yyval.str) = "maintenance"; ;} break; - case 205: -#line 1574 "test_spec_parse.y" + case 206: +#line 1578 "test_spec_parse.y" { (yyval.str) = "join_primary"; ;} break; - case 206: -#line 1575 "test_spec_parse.y" + case 207: +#line 1579 "test_spec_parse.y" { (yyval.str) = "apply_settings"; ;} break; - case 207: -#line 1576 "test_spec_parse.y" + case 208: +#line 1580 "test_spec_parse.y" { (yyval.str) = "prepare_maintenance"; ;} break; - case 208: -#line 1577 "test_spec_parse.y" + case 209: +#line 1581 "test_spec_parse.y" { (yyval.str) = "wait_maintenance"; ;} break; - case 209: -#line 1578 "test_spec_parse.y" + case 210: +#line 1582 "test_spec_parse.y" { (yyval.str) = "report_lsn"; ;} break; - case 210: -#line 1579 "test_spec_parse.y" + case 211: +#line 1583 "test_spec_parse.y" { (yyval.str) = "fast_forward"; ;} break; - case 211: -#line 1580 "test_spec_parse.y" + case 212: +#line 1584 "test_spec_parse.y" { (yyval.str) = "join_secondary"; ;} break; - case 212: -#line 1581 "test_spec_parse.y" + case 213: +#line 1585 "test_spec_parse.y" { (yyval.str) = "dropped"; ;} break; - case 213: -#line 1589 "test_spec_parse.y" + case 214: +#line 1593 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 214: -#line 1590 "test_spec_parse.y" + case 215: +#line 1594 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; /* Line 1267 of yacc.c. */ -#line 3614 "test_spec_parse.c" +#line 3625 "test_spec_parse.c" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); @@ -3824,7 +3835,7 @@ yyparse () } -#line 1593 "test_spec_parse.y" +#line 1597 "test_spec_parse.y" /* ----------------------------------------------------------------------- diff --git a/src/bin/pgaftest/test_spec_parse.h b/src/bin/pgaftest/test_spec_parse.h index 0f520e315..d0ad03e5d 100644 --- a/src/bin/pgaftest/test_spec_parse.h +++ b/src/bin/pgaftest/test_spec_parse.h @@ -58,105 +58,106 @@ T_NUM_SYNC = 274, T_COORDINATOR = 275, T_WORKER = 276, - T_ASYNC = 277, - T_NO_MONITOR = 278, - T_SUSPENDED = 279, - T_LAUNCH = 280, - T_CREATE = 281, - T_DEFERRED = 282, - T_IMMEDIATE = 283, - T_FALSE = 284, - T_TRUE = 285, - T_INITIALLY = 286, - T_VOLUME = 287, - T_LISTEN = 288, - T_CITUS_SECONDARY = 289, - T_CANDIDATE_PRIORITY = 290, - T_PORT = 291, - T_PASSWORD = 292, - T_MONITOR_PASSWORD = 293, - T_CITUS_CLUSTER_NAME = 294, - T_DEBIAN_CLUSTER = 295, - T_REPLICATION_QUORUM = 296, - T_REPLICATION_PASSWORD = 297, - T_EXTENSION_VERSION = 298, - T_BIND_SOURCE = 299, - T_LEGACY_STARTUP = 300, - T_REGION = 301, - T_NODEINI = 302, - T_FS_INIT = 303, - T_FS_SINGLE = 304, - T_FS_PRIMARY = 305, - T_FS_WAIT_PRIMARY = 306, - T_FS_WAIT_STANDBY = 307, - T_FS_DEMOTED = 308, - T_FS_DEMOTE_TIMEOUT = 309, - T_FS_DRAINING = 310, - T_FS_SECONDARY = 311, - T_FS_CATCHINGUP = 312, - T_FS_PREP_PROMOTION = 313, - T_FS_STOP_REPLICATION = 314, - T_FS_MAINTENANCE = 315, - T_FS_JOIN_PRIMARY = 316, - T_FS_APPLY_SETTINGS = 317, - T_FS_PREPARE_MAINTENANCE = 318, - T_FS_WAIT_MAINTENANCE = 319, - T_FS_REPORT_LSN = 320, - T_FS_FAST_FORWARD = 321, - T_FS_JOIN_SECONDARY = 322, - T_FS_DROPPED = 323, - T_EXEC = 324, - T_EXEC_FAILS = 325, - T_RUN = 326, - T_PG_AUTOCTL = 327, - T_WAIT = 328, - T_UNTIL = 329, - T_TIMEOUT = 330, - T_AND = 331, - T_IS = 332, - T_WITH = 333, - T_REPLAYS = 334, - T_ASSERT = 335, - T_SQL = 336, - T_EXPECT = 337, - T_ERROR = 338, - T_PROMOTE = 339, - T_PERFORM = 340, - T_FAILOVER = 341, - T_NETWORK = 342, - T_DISCONNECT = 343, - T_CONNECT = 344, - T_SLEEP = 345, - T_COMPOSE = 346, - T_DOWN = 347, - T_START = 348, - T_STOP = 349, - T_STOPPED = 350, - T_KILL = 351, - T_INJECT = 352, - T_STATE = 353, - T_ASSIGNED_STATE = 354, - T_IN = 355, - T_GROUP = 356, - T_LBRACE = 357, - T_RBRACE = 358, - T_COMMA = 359, - T_POSTGRES = 360, - T_STAYS = 361, - T_WHILE = 362, - T_THROUGH = 363, - T_SET = 364, - T_GET = 365, - T_FSM = 366, - T_LOGS = 367, - T_NOT = 368, - T_CONTAINS = 369, - T_MATCHES = 370, - T_INTEGER = 371, - T_IDENT = 372, - T_STRING = 373, - T_BLOCK = 374, - T_SHELL_ARGS = 375 + T_ARCHIVER = 277, + T_ASYNC = 278, + T_NO_MONITOR = 279, + T_SUSPENDED = 280, + T_LAUNCH = 281, + T_CREATE = 282, + T_DEFERRED = 283, + T_IMMEDIATE = 284, + T_FALSE = 285, + T_TRUE = 286, + T_INITIALLY = 287, + T_VOLUME = 288, + T_LISTEN = 289, + T_CITUS_SECONDARY = 290, + T_CANDIDATE_PRIORITY = 291, + T_PORT = 292, + T_PASSWORD = 293, + T_MONITOR_PASSWORD = 294, + T_CITUS_CLUSTER_NAME = 295, + T_DEBIAN_CLUSTER = 296, + T_REPLICATION_QUORUM = 297, + T_REPLICATION_PASSWORD = 298, + T_EXTENSION_VERSION = 299, + T_BIND_SOURCE = 300, + T_LEGACY_STARTUP = 301, + T_REGION = 302, + T_NODEINI = 303, + T_FS_INIT = 304, + T_FS_SINGLE = 305, + T_FS_PRIMARY = 306, + T_FS_WAIT_PRIMARY = 307, + T_FS_WAIT_STANDBY = 308, + T_FS_DEMOTED = 309, + T_FS_DEMOTE_TIMEOUT = 310, + T_FS_DRAINING = 311, + T_FS_SECONDARY = 312, + T_FS_CATCHINGUP = 313, + T_FS_PREP_PROMOTION = 314, + T_FS_STOP_REPLICATION = 315, + T_FS_MAINTENANCE = 316, + T_FS_JOIN_PRIMARY = 317, + T_FS_APPLY_SETTINGS = 318, + T_FS_PREPARE_MAINTENANCE = 319, + T_FS_WAIT_MAINTENANCE = 320, + T_FS_REPORT_LSN = 321, + T_FS_FAST_FORWARD = 322, + T_FS_JOIN_SECONDARY = 323, + T_FS_DROPPED = 324, + T_EXEC = 325, + T_EXEC_FAILS = 326, + T_RUN = 327, + T_PG_AUTOCTL = 328, + T_WAIT = 329, + T_UNTIL = 330, + T_TIMEOUT = 331, + T_AND = 332, + T_IS = 333, + T_WITH = 334, + T_REPLAYS = 335, + T_ASSERT = 336, + T_SQL = 337, + T_EXPECT = 338, + T_ERROR = 339, + T_PROMOTE = 340, + T_PERFORM = 341, + T_FAILOVER = 342, + T_NETWORK = 343, + T_DISCONNECT = 344, + T_CONNECT = 345, + T_SLEEP = 346, + T_COMPOSE = 347, + T_DOWN = 348, + T_START = 349, + T_STOP = 350, + T_STOPPED = 351, + T_KILL = 352, + T_INJECT = 353, + T_STATE = 354, + T_ASSIGNED_STATE = 355, + T_IN = 356, + T_GROUP = 357, + T_LBRACE = 358, + T_RBRACE = 359, + T_COMMA = 360, + T_POSTGRES = 361, + T_STAYS = 362, + T_WHILE = 363, + T_THROUGH = 364, + T_SET = 365, + T_GET = 366, + T_FSM = 367, + T_LOGS = 368, + T_NOT = 369, + T_CONTAINS = 370, + T_MATCHES = 371, + T_INTEGER = 372, + T_IDENT = 373, + T_STRING = 374, + T_BLOCK = 375, + T_SHELL_ARGS = 376 }; #endif /* Tokens. */ @@ -179,105 +180,106 @@ #define T_NUM_SYNC 274 #define T_COORDINATOR 275 #define T_WORKER 276 -#define T_ASYNC 277 -#define T_NO_MONITOR 278 -#define T_SUSPENDED 279 -#define T_LAUNCH 280 -#define T_CREATE 281 -#define T_DEFERRED 282 -#define T_IMMEDIATE 283 -#define T_FALSE 284 -#define T_TRUE 285 -#define T_INITIALLY 286 -#define T_VOLUME 287 -#define T_LISTEN 288 -#define T_CITUS_SECONDARY 289 -#define T_CANDIDATE_PRIORITY 290 -#define T_PORT 291 -#define T_PASSWORD 292 -#define T_MONITOR_PASSWORD 293 -#define T_CITUS_CLUSTER_NAME 294 -#define T_DEBIAN_CLUSTER 295 -#define T_REPLICATION_QUORUM 296 -#define T_REPLICATION_PASSWORD 297 -#define T_EXTENSION_VERSION 298 -#define T_BIND_SOURCE 299 -#define T_LEGACY_STARTUP 300 -#define T_REGION 301 -#define T_NODEINI 302 -#define T_FS_INIT 303 -#define T_FS_SINGLE 304 -#define T_FS_PRIMARY 305 -#define T_FS_WAIT_PRIMARY 306 -#define T_FS_WAIT_STANDBY 307 -#define T_FS_DEMOTED 308 -#define T_FS_DEMOTE_TIMEOUT 309 -#define T_FS_DRAINING 310 -#define T_FS_SECONDARY 311 -#define T_FS_CATCHINGUP 312 -#define T_FS_PREP_PROMOTION 313 -#define T_FS_STOP_REPLICATION 314 -#define T_FS_MAINTENANCE 315 -#define T_FS_JOIN_PRIMARY 316 -#define T_FS_APPLY_SETTINGS 317 -#define T_FS_PREPARE_MAINTENANCE 318 -#define T_FS_WAIT_MAINTENANCE 319 -#define T_FS_REPORT_LSN 320 -#define T_FS_FAST_FORWARD 321 -#define T_FS_JOIN_SECONDARY 322 -#define T_FS_DROPPED 323 -#define T_EXEC 324 -#define T_EXEC_FAILS 325 -#define T_RUN 326 -#define T_PG_AUTOCTL 327 -#define T_WAIT 328 -#define T_UNTIL 329 -#define T_TIMEOUT 330 -#define T_AND 331 -#define T_IS 332 -#define T_WITH 333 -#define T_REPLAYS 334 -#define T_ASSERT 335 -#define T_SQL 336 -#define T_EXPECT 337 -#define T_ERROR 338 -#define T_PROMOTE 339 -#define T_PERFORM 340 -#define T_FAILOVER 341 -#define T_NETWORK 342 -#define T_DISCONNECT 343 -#define T_CONNECT 344 -#define T_SLEEP 345 -#define T_COMPOSE 346 -#define T_DOWN 347 -#define T_START 348 -#define T_STOP 349 -#define T_STOPPED 350 -#define T_KILL 351 -#define T_INJECT 352 -#define T_STATE 353 -#define T_ASSIGNED_STATE 354 -#define T_IN 355 -#define T_GROUP 356 -#define T_LBRACE 357 -#define T_RBRACE 358 -#define T_COMMA 359 -#define T_POSTGRES 360 -#define T_STAYS 361 -#define T_WHILE 362 -#define T_THROUGH 363 -#define T_SET 364 -#define T_GET 365 -#define T_FSM 366 -#define T_LOGS 367 -#define T_NOT 368 -#define T_CONTAINS 369 -#define T_MATCHES 370 -#define T_INTEGER 371 -#define T_IDENT 372 -#define T_STRING 373 -#define T_BLOCK 374 -#define T_SHELL_ARGS 375 +#define T_ARCHIVER 277 +#define T_ASYNC 278 +#define T_NO_MONITOR 279 +#define T_SUSPENDED 280 +#define T_LAUNCH 281 +#define T_CREATE 282 +#define T_DEFERRED 283 +#define T_IMMEDIATE 284 +#define T_FALSE 285 +#define T_TRUE 286 +#define T_INITIALLY 287 +#define T_VOLUME 288 +#define T_LISTEN 289 +#define T_CITUS_SECONDARY 290 +#define T_CANDIDATE_PRIORITY 291 +#define T_PORT 292 +#define T_PASSWORD 293 +#define T_MONITOR_PASSWORD 294 +#define T_CITUS_CLUSTER_NAME 295 +#define T_DEBIAN_CLUSTER 296 +#define T_REPLICATION_QUORUM 297 +#define T_REPLICATION_PASSWORD 298 +#define T_EXTENSION_VERSION 299 +#define T_BIND_SOURCE 300 +#define T_LEGACY_STARTUP 301 +#define T_REGION 302 +#define T_NODEINI 303 +#define T_FS_INIT 304 +#define T_FS_SINGLE 305 +#define T_FS_PRIMARY 306 +#define T_FS_WAIT_PRIMARY 307 +#define T_FS_WAIT_STANDBY 308 +#define T_FS_DEMOTED 309 +#define T_FS_DEMOTE_TIMEOUT 310 +#define T_FS_DRAINING 311 +#define T_FS_SECONDARY 312 +#define T_FS_CATCHINGUP 313 +#define T_FS_PREP_PROMOTION 314 +#define T_FS_STOP_REPLICATION 315 +#define T_FS_MAINTENANCE 316 +#define T_FS_JOIN_PRIMARY 317 +#define T_FS_APPLY_SETTINGS 318 +#define T_FS_PREPARE_MAINTENANCE 319 +#define T_FS_WAIT_MAINTENANCE 320 +#define T_FS_REPORT_LSN 321 +#define T_FS_FAST_FORWARD 322 +#define T_FS_JOIN_SECONDARY 323 +#define T_FS_DROPPED 324 +#define T_EXEC 325 +#define T_EXEC_FAILS 326 +#define T_RUN 327 +#define T_PG_AUTOCTL 328 +#define T_WAIT 329 +#define T_UNTIL 330 +#define T_TIMEOUT 331 +#define T_AND 332 +#define T_IS 333 +#define T_WITH 334 +#define T_REPLAYS 335 +#define T_ASSERT 336 +#define T_SQL 337 +#define T_EXPECT 338 +#define T_ERROR 339 +#define T_PROMOTE 340 +#define T_PERFORM 341 +#define T_FAILOVER 342 +#define T_NETWORK 343 +#define T_DISCONNECT 344 +#define T_CONNECT 345 +#define T_SLEEP 346 +#define T_COMPOSE 347 +#define T_DOWN 348 +#define T_START 349 +#define T_STOP 350 +#define T_STOPPED 351 +#define T_KILL 352 +#define T_INJECT 353 +#define T_STATE 354 +#define T_ASSIGNED_STATE 355 +#define T_IN 356 +#define T_GROUP 357 +#define T_LBRACE 358 +#define T_RBRACE 359 +#define T_COMMA 360 +#define T_POSTGRES 361 +#define T_STAYS 362 +#define T_WHILE 363 +#define T_THROUGH 364 +#define T_SET 365 +#define T_GET 366 +#define T_FSM 367 +#define T_LOGS 368 +#define T_NOT 369 +#define T_CONTAINS 370 +#define T_MATCHES 371 +#define T_INTEGER 372 +#define T_IDENT 373 +#define T_STRING 374 +#define T_BLOCK 375 +#define T_SHELL_ARGS 376 @@ -292,7 +294,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 1529 of yacc.c. */ -#line 296 "test_spec_parse.h" +#line 298 "test_spec_parse.h" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 diff --git a/src/bin/pgaftest/test_spec_parse.y b/src/bin/pgaftest/test_spec_parse.y index 956a60711..2670bcb5a 100644 --- a/src/bin/pgaftest/test_spec_parse.y +++ b/src/bin/pgaftest/test_spec_parse.y @@ -156,7 +156,7 @@ static TestNode *current_node = NULL; /* ---- Cluster-body tokens ---- */ %token T_IMAGE T_IMAGE_TARGET T_SSL T_AUTH T_AUTH_METHOD T_FORMATION T_NUM_SYNC -%token T_COORDINATOR T_WORKER T_ASYNC T_NO_MONITOR T_SUSPENDED +%token T_COORDINATOR T_WORKER T_ARCHIVER T_ASYNC T_NO_MONITOR T_SUSPENDED %token T_LAUNCH T_CREATE T_DEFERRED T_IMMEDIATE T_FALSE T_TRUE T_INITIALLY T_VOLUME %token T_LISTEN T_CITUS_SECONDARY T_CANDIDATE_PRIORITY T_PORT T_PASSWORD T_MONITOR_PASSWORD %token T_CITUS_CLUSTER_NAME T_DEBIAN_CLUSTER T_REPLICATION_QUORUM T_REPLICATION_PASSWORD @@ -515,6 +515,10 @@ node_opt: current_node->kind = NODE_KIND_CITUS_WORKER; current_spec->cluster.withCitus = true; } + | T_ARCHIVER + { + current_node->kind = NODE_KIND_ARCHIVER; + } | T_ASYNC { current_node->replicationQuorum = false; diff --git a/src/bin/pgaftest/test_spec_scan.c b/src/bin/pgaftest/test_spec_scan.c index 815753697..8e5d33bb0 100644 --- a/src/bin/pgaftest/test_spec_scan.c +++ b/src/bin/pgaftest/test_spec_scan.c @@ -356,8 +356,8 @@ static void yynoreturn yy_fatal_error ( const char* msg ); (yy_hold_char) = *yy_cp; \ *yy_cp = '\0'; \ (yy_c_buf_p) = yy_cp; -#define YY_NUM_RULES 163 -#define YY_END_OF_BUFFER 164 +#define YY_NUM_RULES 164 +#define YY_END_OF_BUFFER 165 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -365,146 +365,146 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[1252] = +static const flex_int16_t yy_accept[1259] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 18, 17, 16, 18, 1, 12, 11, 13, 13, - 13, 13, 13, 13, 15, 163, 21, 20, 163, 19, - 62, 61, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 64, 65, 102, 101, 163, 100, 138, 153, 137, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 155, - 156, 159, 158, 160, 161, 162, 17, 0, 14, 1, + 165, 18, 17, 16, 18, 1, 12, 11, 13, 13, + 13, 13, 13, 13, 15, 164, 21, 20, 164, 19, + 63, 62, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 65, 66, 103, 102, 164, 101, 139, 154, 138, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 156, + 157, 160, 159, 161, 162, 163, 17, 0, 14, 1, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, - 21, 0, 63, 19, 62, 62, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 102, 0, 154, 100, 153, - 153, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 129, 135, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 159, 158, 161, 13, 13, 13, - - 13, 13, 13, 13, 13, 44, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 25, 99, 99, 99, 99, - 99, 99, 134, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 140, 147, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 150, 157, 157, 157, 157, 157, 157, 157, - 157, 105, 157, 146, 157, 157, 112, 157, 157, 157, - - 157, 157, 157, 157, 157, 157, 13, 13, 13, 4, - 13, 13, 9, 13, 99, 99, 27, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 66, 99, 99, 99, 99, - 99, 99, 99, 30, 99, 99, 53, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 43, 99, 99, 99, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 124, 157, 157, 157, 104, 157, 157, 157, 157, 157, - 66, 157, 157, 128, 149, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 141, 126, 157, 157, 157, 107, - 157, 136, 13, 13, 13, 13, 7, 13, 99, 33, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 42, 99, 99, 99, 52, 24, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 114, 157, - 157, 157, 157, 157, 157, 133, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 121, 125, 130, 142, 157, 157, - 157, 157, 157, 108, 157, 157, 143, 13, 13, 13, - 13, 13, 99, 99, 99, 99, 99, 99, 99, 99, - 39, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 38, 99, 48, - 99, 99, 99, 99, 99, 99, 99, 51, 99, 99, - 99, 67, 99, 99, 99, 99, 47, 99, 99, 99, - 99, 99, 99, 32, 157, 157, 111, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 113, 157, - 157, 157, 157, 148, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 67, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 2, 3, 13, 13, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 73, 99, 98, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 22, - 99, 99, 99, 99, 68, 99, 99, 99, 99, 99, - 99, 46, 99, 99, 99, 99, 99, 99, 99, 157, - 157, 157, 157, 157, 122, 120, 157, 157, 157, 73, - 157, 157, 98, 157, 157, 157, 157, 157, 157, 157, - - 157, 157, 157, 152, 118, 123, 157, 116, 157, 157, - 157, 68, 115, 109, 157, 157, 157, 157, 157, 127, - 145, 110, 157, 157, 157, 157, 157, 157, 13, 13, - 10, 8, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 40, 99, 99, 76, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 29, 36, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 157, 157, - 157, 157, 157, 151, 157, 157, 157, 76, 157, 117, - 157, 157, 157, 157, 157, 157, 157, 157, 0, 157, - - 139, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 28, 99, 41, 45, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 77, 99, 99, 35, 99, 99, 99, 99, 99, 99, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 28, 157, 157, 157, 157, 157, 0, 157, 157, - 157, 157, 157, 157, 157, 77, 157, 157, 157, 157, - 157, 157, 157, 157, 13, 13, 99, 99, 99, 99, - - 99, 78, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 34, - 99, 99, 99, 99, 99, 93, 92, 99, 99, 99, - 99, 99, 99, 99, 99, 157, 157, 157, 157, 78, - 157, 157, 119, 103, 157, 157, 157, 157, 157, 157, - 157, 0, 106, 157, 157, 157, 157, 93, 92, 157, - 157, 157, 157, 157, 157, 157, 157, 13, 13, 99, - 99, 26, 59, 99, 99, 99, 31, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 83, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - - 99, 99, 99, 99, 157, 157, 157, 157, 157, 157, - 157, 157, 157, 157, 157, 157, 83, 0, 157, 157, - 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, - 13, 6, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 95, 94, 23, 85, 99, 84, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 70, 72, - 99, 69, 71, 157, 157, 157, 157, 157, 157, 95, - 94, 85, 157, 84, 157, 0, 157, 157, 157, 157, - 157, 157, 157, 70, 72, 157, 69, 71, 13, 99, - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - - 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 157, 157, 157, 157, 157, 157, 157, 157, - 0, 157, 157, 157, 157, 157, 157, 157, 157, 13, - 87, 86, 99, 99, 99, 55, 75, 74, 99, 97, - 96, 60, 99, 99, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 87, 86, 131, 157, 75, 74, 97, - 96, 0, 157, 157, 157, 157, 157, 157, 157, 157, - 13, 99, 99, 49, 99, 99, 99, 99, 99, 99, - 99, 99, 99, 99, 99, 99, 157, 144, 157, 157, - 157, 157, 157, 157, 157, 157, 13, 99, 99, 99, - - 37, 99, 99, 99, 99, 99, 99, 82, 81, 91, - 90, 157, 157, 157, 157, 157, 82, 81, 91, 90, - 5, 99, 99, 58, 99, 80, 99, 79, 99, 99, - 157, 157, 80, 157, 79, 50, 54, 99, 99, 99, - 56, 132, 157, 157, 89, 88, 99, 89, 88, 57, - 0 + 21, 0, 64, 19, 63, 63, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 103, 0, 155, 101, + 154, 154, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 130, + 136, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 160, 159, 162, 13, 13, + + 13, 13, 13, 13, 13, 13, 45, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 25, 100, 100, + 100, 100, 100, 100, 135, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 141, 148, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 151, 158, 158, 158, 158, 158, + 158, 158, 158, 106, 158, 147, 158, 158, 113, 158, + + 158, 158, 158, 158, 158, 158, 158, 158, 13, 13, + 13, 4, 13, 13, 9, 13, 100, 100, 100, 27, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 67, 100, + 100, 100, 100, 100, 100, 100, 30, 100, 100, 54, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 44, + 100, 100, 100, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 125, 158, 158, 158, 105, 158, 158, + 158, 158, 158, 67, 158, 158, 129, 150, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + + 158, 158, 158, 158, 158, 158, 158, 142, 127, 158, + 158, 158, 108, 158, 137, 13, 13, 13, 13, 7, + 13, 100, 100, 34, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 43, 100, 100, + 100, 53, 24, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 115, 158, 158, 158, 158, 158, 158, 134, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + + 158, 158, 158, 158, 158, 158, 158, 158, 122, 126, + 131, 143, 158, 158, 158, 158, 158, 109, 158, 158, + 144, 13, 13, 13, 13, 13, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 40, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 39, 100, 49, 100, 100, 100, 100, 100, + 100, 100, 52, 100, 100, 100, 68, 100, 100, 100, + 100, 48, 100, 100, 100, 100, 100, 100, 32, 158, + 158, 112, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 114, 158, 158, 158, 158, 149, 158, + + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 68, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 13, + 13, 2, 3, 13, 13, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 74, + 100, 99, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 22, 100, 100, 100, 100, + 69, 100, 100, 100, 100, 100, 100, 47, 100, 100, + 100, 100, 100, 100, 100, 158, 158, 158, 158, 158, + 123, 121, 158, 158, 158, 74, 158, 158, 99, 158, + + 158, 158, 158, 158, 158, 158, 158, 158, 158, 153, + 119, 124, 158, 117, 158, 158, 158, 69, 116, 110, + 158, 158, 158, 158, 158, 128, 146, 111, 158, 158, + 158, 158, 158, 158, 13, 13, 10, 8, 100, 100, + 33, 100, 100, 100, 100, 100, 100, 100, 100, 41, + 100, 100, 77, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 29, 37, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 158, 158, 158, 158, 158, + 152, 158, 158, 158, 77, 158, 118, 158, 158, 158, + + 158, 158, 158, 158, 158, 0, 158, 140, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 13, 13, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 28, 100, + 42, 46, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 78, 100, 100, + 36, 100, 100, 100, 100, 100, 100, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 28, 158, + 158, 158, 158, 158, 0, 158, 158, 158, 158, 158, + 158, 158, 78, 158, 158, 158, 158, 158, 158, 158, + + 158, 13, 13, 100, 100, 100, 100, 100, 79, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 35, 100, 100, 100, + 100, 100, 94, 93, 100, 100, 100, 100, 100, 100, + 100, 100, 158, 158, 158, 158, 79, 158, 158, 120, + 104, 158, 158, 158, 158, 158, 158, 158, 0, 107, + 158, 158, 158, 158, 94, 93, 158, 158, 158, 158, + 158, 158, 158, 158, 13, 13, 100, 100, 26, 60, + 100, 100, 100, 31, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 84, 100, 100, 100, + + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 84, 0, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 13, 6, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 96, 95, + 23, 86, 100, 85, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 71, 73, 100, 70, 72, + 158, 158, 158, 158, 158, 158, 96, 95, 86, 158, + 85, 158, 0, 158, 158, 158, 158, 158, 158, 158, + 71, 73, 158, 70, 72, 13, 100, 100, 100, 100, + + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 158, + 158, 158, 158, 158, 158, 158, 158, 0, 158, 158, + 158, 158, 158, 158, 158, 158, 13, 88, 87, 100, + 100, 100, 56, 76, 75, 100, 98, 97, 61, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 88, 87, 132, 158, 76, 75, 98, 97, 0, 158, + 158, 158, 158, 158, 158, 158, 158, 13, 100, 100, + 50, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 158, 145, 158, 158, 158, 158, 158, + + 158, 158, 158, 13, 100, 100, 100, 38, 100, 100, + 100, 100, 100, 100, 83, 82, 92, 91, 158, 158, + 158, 158, 158, 83, 82, 92, 91, 5, 100, 100, + 59, 100, 81, 100, 80, 100, 100, 158, 158, 81, + 158, 80, 51, 55, 100, 100, 100, 57, 133, 158, + 158, 90, 89, 100, 90, 89, 58, 0 } ; static const YY_CHAR yy_ec[256] = @@ -547,293 +547,295 @@ static const YY_CHAR yy_meta[41] = 4, 4, 4, 4, 4, 4, 4, 4, 1, 1 } ; -static const flex_int16_t yy_base[1265] = +static const flex_int16_t yy_base[1272] = { 0, - 0, 0, 40, 0, 80, 0, 119, 121, 1344, 1343, - 1345, 1348, 123, 1348, 1339, 0, 117, 1348, 0, 106, - 1315, 1314, 112, 1323, 1348, 1348, 130, 1348, 1335, 0, - 124, 1348, 0, 108, 1317, 124, 123, 1301, 125, 1306, - 117, 1308, 143, 134, 130, 145, 1317, 145, 1303, 1305, - 146, 1348, 1348, 164, 1348, 1327, 0, 1348, 138, 1348, - 0, 153, 155, 153, 155, 173, 171, 161, 1303, 1308, - 1301, 1314, 172, 189, 176, 186, 174, 1300, 177, 1348, - 1348, 0, 1324, 1348, 0, 1348, 210, 1320, 1348, 0, - 206, 1348, 0, 1291, 1289, 1295, 1304, 191, 1302, 1305, - - 221, 1313, 1348, 0, 217, 1348, 0, 1300, 1287, 1277, - 1281, 1286, 195, 1279, 1283, 1292, 215, 199, 1276, 207, - 1277, 1279, 217, 1284, 1283, 1270, 1283, 1270, 1279, 1273, - 189, 1273, 1266, 1266, 215, 215, 1280, 1268, 1269, 1265, - 1260, 1257, 1265, 1267, 1257, 238, 1282, 1348, 0, 236, - 1348, 0, 1269, 1256, 1252, 219, 224, 1257, 1250, 1245, - 233, 1249, 235, 233, 1248, 1252, 1244, 1248, 234, 0, - 1253, 1249, 1253, 236, 1239, 237, 1239, 1239, 1256, 1236, - 244, 1238, 1239, 243, 1238, 1246, 1238, 249, 1231, 1235, - 1227, 1237, 1236, 1224, 0, 1254, 0, 1221, 1222, 1231, - - 1234, 1217, 1216, 1220, 1217, 0, 1222, 1219, 1224, 1227, - 1226, 1226, 1207, 1209, 1225, 1216, 1219, 1208, 1213, 1205, - 1215, 1200, 1198, 1204, 1195, 1208, 1209, 1193, 1198, 1197, - 1209, 1189, 1194, 1198, 1193, 1200, 1209, 1184, 1182, 1185, - 1187, 1190, 246, 1183, 1190, 0, 1180, 1179, 1189, 1172, - 1172, 1180, 0, 1178, 257, 1185, 1185, 1171, 251, 1171, - 1182, 1170, 1174, 1166, 1166, 1177, 1174, 1166, 1157, 1163, - 0, 0, 1154, 1154, 1168, 1158, 1159, 1151, 1155, 1165, - 1144, 1161, 0, 1146, 1158, 1162, 1142, 1145, 1147, 1146, - 255, 0, 1143, 0, 1150, 1151, 0, 254, 1139, 1138, - - 1138, 1147, 1142, 1130, 1137, 1140, 1128, 1126, 1125, 0, - 1139, 1127, 0, 1138, 1116, 1137, 1144, 1143, 1128, 1128, - 1116, 1130, 1113, 1131, 1113, 1110, 1115, 1112, 1113, 1121, - 273, 1124, 1108, 1118, 1118, 1112, 280, 1117, 1116, 1113, - 1097, 1096, 1100, 0, 1095, 1090, 0, 1111, 1110, 1095, - 1100, 1090, 1093, 1094, 281, 1100, 0, 1091, 282, 1098, - 1077, 1083, 1093, 1090, 1090, 1082, 1091, 1094, 1074, 1078, - 0, 1078, 1075, 1072, 1094, 1085, 1072, 283, 1085, 1069, - 0, 1081, 289, 0, 0, 1063, 1074, 1066, 1071, 1070, - 1063, 1056, 1069, 1074, 1073, 1058, 1071, 1053, 1056, 1057, - - 1052, 1047, 1061, 1046, 0, 290, 1043, 1048, 1050, 291, - 1056, 0, 1065, 1054, 1043, 1043, 0, 1041, 292, 0, - 1042, 1035, 1049, 1043, 1056, 1041, 1044, 1034, 1029, 1041, - 1036, 1039, 1024, 0, 1036, 1035, 1020, 0, 1044, 1029, - 1036, 277, 279, 1028, 1010, 1020, 1028, 1017, 1017, 1005, - 1014, 1010, 1009, 1012, 1022, 1004, 1019, 1017, 1003, 1002, - 1014, 1004, 1012, 286, 288, 998, 308, 995, 1000, 1009, - 1003, 992, 1007, 1000, 1003, 993, 997, 1000, 0, 998, - 983, 980, 995, 994, 979, 0, 978, 293, 294, 992, - 991, 977, 980, 979, 974, 971, 972, 971, 970, 967, - - 961, 965, 980, 978, 0, 0, 0, 0, 964, 963, - 975, 972, 957, 0, 298, 302, 0, 297, 959, 958, - 972, 951, 954, 953, 966, 955, 968, 954, 313, 953, - 0, 971, 960, 324, 950, 959, 953, 946, 945, 950, - 938, 956, 944, 937, 949, 935, 947, 0, 956, 0, - 936, 931, 939, 933, 928, 940, 919, 0, 942, 327, - 941, 0, 936, 935, 935, 934, 0, 936, 918, 915, - 933, 915, 912, 0, 912, 911, 0, 924, 927, 913, - 921, 905, 910, 330, 909, 908, 917, 919, 0, 914, - 903, 902, 907, 0, 897, 909, 895, 907, 897, 891, - - 898, 899, 900, 893, 890, 899, 898, 877, 896, 881, - 331, 898, 0, 893, 892, 892, 887, 874, 892, 874, - 871, 889, 871, 868, 872, 871, 0, 0, 880, 870, - 878, 877, 861, 859, 859, 871, 865, 871, 874, 871, - 869, 852, 851, 0, 863, 0, 854, 850, 849, 851, - 864, 844, 851, 853, 858, 851, 856, 839, 856, 861, - 835, 851, 849, 338, 0, 832, 839, 838, 831, 832, - 831, 0, 841, 836, 835, 842, 833, 832, 839, 834, - 832, 829, 812, 131, 0, 0, 140, 203, 227, 0, - 255, 262, 0, 279, 276, 305, 311, 320, 327, 334, - - 330, 337, 340, 0, 0, 0, 353, 0, 342, 327, - 352, 0, 0, 0, 336, 337, 332, 335, 337, 0, - 0, 0, 345, 346, 355, 348, 349, 358, 345, 343, - 0, 0, 342, 343, 356, 347, 361, 346, 347, 366, - 350, 359, 0, 363, 364, 0, 360, 352, 353, 363, - 360, 374, 355, 368, 367, 370, 369, 365, 372, 371, - 373, 0, 0, 376, 377, 382, 375, 376, 371, 385, - 386, 395, 386, 388, 388, 389, 391, 391, 386, 387, - 413, 404, 389, 0, 402, 403, 410, 0, 402, 0, - 392, 393, 403, 405, 404, 407, 406, 408, 434, 406, - - 0, 414, 415, 410, 413, 408, 422, 423, 422, 424, - 424, 425, 427, 427, 424, 432, 424, 425, 431, 444, - 453, 433, 431, 436, 437, 432, 442, 443, 462, 457, - 458, 0, 453, 0, 0, 460, 448, 462, 450, 464, - 463, 466, 450, 468, 452, 470, 454, 458, 460, 461, - 0, 467, 468, 0, 458, 478, 476, 461, 481, 479, - 464, 465, 467, 492, 472, 476, 477, 471, 473, 492, - 493, 0, 494, 482, 496, 484, 496, 492, 489, 501, - 485, 503, 487, 492, 493, 0, 499, 500, 490, 510, - 508, 493, 513, 511, 512, 512, 509, 510, 516, 516, - - 506, 0, 503, 510, 507, 507, 522, 523, 507, 512, - 513, 527, 515, 530, 517, 532, 519, 533, 520, 0, - 531, 526, 533, 528, 530, 0, 0, 542, 543, 542, - 530, 547, 545, 533, 550, 544, 545, 535, 540, 0, - 552, 553, 0, 0, 541, 542, 543, 558, 545, 560, - 560, 548, 0, 558, 553, 560, 555, 0, 0, 568, - 569, 568, 556, 573, 571, 559, 576, 570, 562, 567, - 568, 0, 0, 565, 579, 581, 0, 566, 572, 573, - 584, 586, 587, 572, 568, 593, 570, 595, 577, 0, - 579, 585, 587, 587, 589, 608, 603, 604, 592, 582, - - 583, 595, 585, 586, 598, 599, 613, 597, 601, 602, - 614, 615, 595, 620, 597, 622, 0, 609, 611, 613, - 613, 615, 628, 629, 617, 607, 608, 620, 610, 611, - 623, 0, 631, 632, 631, 623, 641, 638, 623, 624, - 628, 0, 0, 0, 0, 629, 0, 630, 628, 627, - 631, 637, 633, 639, 639, 637, 638, 658, 0, 0, - 659, 0, 0, 654, 655, 643, 655, 644, 645, 0, - 0, 0, 649, 0, 650, 648, 650, 656, 652, 658, - 654, 655, 675, 0, 0, 676, 0, 0, 677, 660, - 661, 666, 687, 665, 666, 665, 666, 668, 663, 664, - - 674, 676, 687, 673, 689, 675, 695, 676, 689, 690, - 686, 687, 683, 684, 699, 690, 686, 687, 683, 684, - 703, 706, 692, 708, 694, 706, 707, 703, 704, 699, - 0, 0, 702, 707, 697, 0, 0, 0, 714, 0, - 0, 0, 706, 711, 717, 713, 719, 710, 715, 716, - 717, 730, 731, 0, 0, 0, 717, 0, 0, 0, - 0, 728, 723, 729, 725, 731, 726, 727, 740, 741, - 730, 737, 746, 0, 733, 745, 749, 736, 751, 738, - 735, 737, 742, 743, 753, 754, 751, 1348, 760, 747, - 762, 749, 751, 752, 762, 763, 751, 750, 758, 758, - - 0, 759, 760, 761, 762, 754, 757, 0, 0, 0, - 0, 759, 766, 767, 768, 769, 0, 0, 0, 0, - 0, 759, 780, 0, 783, 0, 784, 0, 773, 776, - 765, 788, 0, 789, 0, 0, 0, 788, 789, 777, - 0, 0, 791, 792, 0, 0, 794, 0, 0, 0, - 1348, 811, 815, 819, 823, 822, 827, 831, 830, 835, - 839, 838, 843, 847 + 0, 0, 40, 0, 80, 0, 119, 121, 1351, 1350, + 1352, 1355, 123, 1355, 1346, 0, 117, 1355, 0, 106, + 1322, 1321, 112, 1330, 1355, 1355, 130, 1355, 1342, 0, + 124, 1355, 0, 108, 1324, 124, 123, 1308, 129, 1313, + 121, 1315, 144, 136, 125, 137, 1324, 145, 1310, 1312, + 146, 1355, 1355, 167, 1355, 1334, 0, 1355, 161, 1355, + 0, 153, 147, 166, 150, 158, 158, 164, 1310, 1315, + 1308, 1321, 174, 181, 180, 185, 183, 1307, 199, 1355, + 1355, 0, 1331, 1355, 0, 1355, 204, 1327, 1355, 0, + 207, 1355, 0, 1298, 1296, 1302, 1311, 192, 1309, 1312, + + 222, 1320, 1355, 0, 218, 1355, 0, 1307, 1294, 1306, + 1283, 1287, 1292, 184, 1285, 1289, 1298, 214, 209, 1282, + 203, 1283, 1285, 217, 1290, 1289, 1276, 1289, 1276, 1285, + 1279, 224, 1279, 1272, 1272, 216, 216, 1286, 1274, 1275, + 1271, 1266, 1263, 1271, 1273, 1263, 240, 1288, 1355, 0, + 237, 1355, 0, 1275, 1262, 1258, 220, 225, 1263, 1256, + 1251, 234, 1255, 236, 234, 1254, 1258, 1250, 1254, 235, + 0, 1259, 1255, 1259, 237, 1245, 238, 1245, 1245, 1262, + 1242, 245, 1244, 1245, 167, 1244, 1252, 1244, 246, 1237, + 1241, 1233, 1243, 1242, 1230, 0, 1260, 0, 1227, 1228, + + 1237, 1240, 1223, 1222, 1226, 1223, 0, 1228, 1231, 1224, + 1229, 1232, 1231, 1231, 1212, 1214, 1230, 1221, 1224, 1213, + 1218, 1210, 1220, 1205, 1203, 1209, 1200, 1213, 1214, 1198, + 1203, 1202, 1214, 1194, 1199, 1203, 1198, 1205, 1214, 1189, + 1187, 1190, 1192, 1195, 247, 1188, 1195, 0, 1185, 1184, + 1194, 1177, 1177, 1185, 0, 1183, 258, 1190, 1190, 1176, + 250, 1176, 1187, 1175, 1179, 1171, 1171, 1182, 1179, 1171, + 1162, 1168, 0, 0, 1159, 1159, 1173, 1163, 1164, 1156, + 1160, 1170, 1149, 1166, 0, 1151, 1163, 1167, 1147, 1150, + 1152, 1151, 253, 0, 1148, 0, 1155, 1156, 0, 251, + + 1144, 1143, 1143, 1152, 1147, 1135, 1142, 1145, 1133, 1131, + 1130, 0, 1144, 1132, 0, 1143, 1121, 1136, 1141, 1148, + 1147, 1132, 1132, 1120, 1134, 1117, 1135, 1117, 1114, 1119, + 1116, 1117, 1125, 277, 1128, 1112, 1122, 1122, 1116, 278, + 1121, 1120, 1117, 1101, 1100, 1104, 0, 1099, 1094, 0, + 1115, 1114, 1099, 1104, 1094, 1097, 1098, 279, 1104, 0, + 1095, 280, 1102, 1081, 1087, 1097, 1094, 1094, 1086, 1095, + 1098, 1078, 1082, 0, 1082, 1079, 1076, 1098, 1089, 1076, + 286, 1089, 1073, 0, 1085, 287, 0, 0, 1067, 1078, + 1070, 1075, 1074, 1067, 1060, 1073, 1078, 1077, 1062, 1075, + + 1057, 1060, 1061, 1056, 1051, 1065, 1050, 0, 288, 1047, + 1052, 1054, 289, 1060, 0, 1069, 1058, 1047, 1047, 0, + 1045, 290, 1037, 0, 1045, 1038, 1052, 1046, 1059, 1044, + 1047, 1037, 1032, 1044, 1039, 1042, 1027, 0, 1039, 1038, + 1023, 0, 1047, 1032, 1039, 275, 277, 1031, 1013, 1023, + 1031, 1020, 1020, 1008, 1017, 1013, 1012, 1015, 1025, 1007, + 1022, 1020, 1006, 1005, 1017, 1007, 1015, 284, 286, 1001, + 306, 998, 1003, 1012, 1006, 995, 1010, 1003, 1006, 996, + 1000, 1003, 0, 1001, 986, 983, 998, 997, 982, 0, + 981, 291, 292, 995, 994, 980, 983, 982, 977, 974, + + 975, 974, 973, 970, 964, 968, 983, 981, 0, 0, + 0, 0, 967, 966, 978, 975, 960, 0, 296, 300, + 0, 295, 962, 961, 975, 954, 957, 956, 969, 968, + 957, 970, 956, 311, 955, 0, 973, 962, 322, 952, + 961, 955, 948, 947, 952, 940, 958, 946, 939, 951, + 937, 949, 0, 958, 0, 938, 933, 941, 935, 930, + 942, 921, 0, 944, 325, 943, 0, 938, 937, 937, + 936, 0, 938, 920, 917, 935, 917, 914, 0, 914, + 913, 0, 926, 929, 915, 923, 907, 912, 328, 911, + 910, 919, 921, 0, 916, 905, 904, 909, 0, 899, + + 911, 897, 909, 899, 893, 900, 901, 902, 895, 892, + 901, 900, 879, 898, 883, 329, 900, 0, 895, 894, + 894, 889, 876, 894, 876, 873, 891, 873, 870, 874, + 873, 0, 0, 882, 872, 880, 879, 865, 862, 860, + 860, 872, 866, 872, 875, 872, 870, 853, 852, 0, + 864, 0, 855, 851, 850, 852, 865, 845, 852, 854, + 859, 852, 857, 840, 857, 862, 836, 852, 850, 336, + 0, 833, 840, 839, 832, 833, 832, 0, 842, 837, + 835, 839, 830, 122, 247, 250, 261, 288, 301, 320, + 0, 0, 314, 314, 315, 0, 332, 331, 0, 330, + + 322, 323, 324, 328, 335, 342, 337, 344, 347, 0, + 0, 0, 360, 0, 349, 334, 359, 0, 0, 0, + 343, 344, 339, 342, 344, 0, 0, 0, 352, 353, + 362, 355, 356, 365, 352, 350, 0, 0, 349, 350, + 0, 363, 354, 368, 353, 354, 373, 357, 366, 0, + 370, 371, 0, 367, 359, 360, 370, 367, 381, 362, + 375, 374, 377, 376, 372, 379, 378, 380, 0, 0, + 383, 384, 389, 382, 383, 378, 392, 393, 402, 393, + 395, 395, 396, 398, 398, 393, 394, 420, 411, 396, + 0, 409, 410, 417, 0, 409, 0, 399, 400, 410, + + 412, 411, 414, 413, 415, 441, 413, 0, 421, 422, + 417, 420, 415, 429, 430, 429, 431, 431, 432, 434, + 434, 431, 439, 431, 432, 438, 451, 460, 440, 438, + 443, 444, 439, 449, 450, 469, 464, 465, 0, 460, + 0, 0, 467, 455, 469, 457, 471, 470, 473, 457, + 475, 459, 477, 461, 465, 467, 468, 0, 474, 475, + 0, 465, 485, 483, 468, 488, 486, 471, 472, 474, + 499, 479, 483, 484, 478, 480, 499, 500, 0, 501, + 489, 503, 491, 503, 499, 496, 508, 492, 510, 494, + 499, 500, 0, 506, 507, 497, 517, 515, 500, 520, + + 518, 519, 519, 516, 517, 523, 523, 513, 0, 510, + 517, 514, 514, 529, 530, 514, 519, 520, 534, 522, + 537, 524, 539, 526, 540, 527, 0, 538, 533, 540, + 535, 537, 0, 0, 549, 550, 549, 537, 554, 552, + 540, 557, 551, 552, 542, 547, 0, 559, 560, 0, + 0, 548, 549, 550, 565, 552, 567, 567, 555, 0, + 565, 560, 567, 562, 0, 0, 575, 576, 575, 563, + 580, 578, 566, 583, 577, 569, 574, 575, 0, 0, + 572, 586, 588, 0, 573, 579, 580, 591, 593, 594, + 579, 575, 600, 577, 602, 584, 0, 586, 592, 594, + + 594, 596, 615, 610, 611, 599, 589, 590, 602, 592, + 593, 605, 606, 620, 604, 608, 609, 621, 622, 602, + 627, 604, 629, 0, 616, 618, 620, 620, 622, 635, + 636, 624, 614, 615, 627, 617, 618, 630, 0, 638, + 639, 638, 630, 648, 645, 630, 631, 635, 0, 0, + 0, 0, 636, 0, 637, 635, 634, 638, 644, 640, + 646, 646, 644, 645, 665, 0, 0, 666, 0, 0, + 661, 662, 650, 662, 651, 652, 0, 0, 0, 656, + 0, 657, 655, 657, 663, 659, 665, 661, 662, 682, + 0, 0, 683, 0, 0, 684, 667, 668, 673, 694, + + 672, 673, 672, 673, 675, 670, 671, 681, 683, 694, + 680, 696, 682, 702, 683, 696, 697, 693, 694, 690, + 691, 706, 697, 693, 694, 690, 691, 710, 713, 699, + 715, 701, 713, 714, 710, 711, 706, 0, 0, 709, + 714, 704, 0, 0, 0, 721, 0, 0, 0, 713, + 718, 724, 720, 726, 717, 722, 723, 724, 737, 738, + 0, 0, 0, 724, 0, 0, 0, 0, 735, 730, + 736, 732, 738, 733, 734, 747, 748, 737, 744, 753, + 0, 740, 752, 756, 743, 758, 745, 742, 744, 749, + 750, 760, 761, 758, 1355, 767, 754, 769, 756, 758, + + 759, 769, 770, 758, 757, 765, 765, 0, 766, 767, + 768, 769, 761, 764, 0, 0, 0, 0, 766, 773, + 774, 775, 776, 0, 0, 0, 0, 0, 766, 787, + 0, 790, 0, 791, 0, 780, 783, 772, 795, 0, + 796, 0, 0, 0, 795, 796, 784, 0, 0, 798, + 799, 0, 0, 801, 0, 0, 0, 1355, 818, 822, + 826, 830, 829, 834, 838, 837, 842, 846, 845, 850, + 854 } ; -static const flex_int16_t yy_def[1265] = +static const flex_int16_t yy_def[1272] = { 0, - 1251, 1, 1251, 3, 1251, 5, 1252, 1252, 1253, 1253, - 1251, 1251, 1251, 1251, 1254, 1255, 1251, 1251, 1256, 1256, - 1256, 1256, 1256, 1256, 1251, 1251, 1251, 1251, 1257, 1258, - 1251, 1251, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1251, 1251, 1251, 1251, 1260, 1261, 1251, 1251, 1251, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, - 1251, 1263, 1251, 1251, 1264, 1251, 1251, 1254, 1251, 1255, - 1251, 1251, 1256, 1256, 1256, 1256, 1256, 1256, 1256, 1256, - - 1251, 1257, 1251, 1258, 1251, 1251, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1251, 1260, 1251, 1261, 1251, - 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1263, 1251, 1264, 1256, 1256, 1256, - - 1256, 1256, 1256, 1256, 1256, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1256, 1256, - 1256, 1256, 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1256, 1256, 1256, 1256, 1256, 1256, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1256, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1256, 1256, 1256, 1256, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, - - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1256, 1256, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1256, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1251, 1262, 1262, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1256, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1262, 1251, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1259, 1259, 1259, 1259, 1259, 1262, 1251, 1262, 1262, - 1262, 1262, 1262, 1262, 1262, 1262, 1256, 1259, 1259, 1259, - - 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1259, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, 1262, - 1256, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, 1259, - 1262, 1262, 1262, 1262, 1262, 1259, 1259, 1259, 1259, 1259, - 1259, 1262, 1262, 1262, 1259, 1259, 1259, 1262, 1262, 1259, - 0, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251 + 1258, 1, 1258, 3, 1258, 5, 1259, 1259, 1260, 1260, + 1258, 1258, 1258, 1258, 1261, 1262, 1258, 1258, 1263, 1263, + 1263, 1263, 1263, 1263, 1258, 1258, 1258, 1258, 1264, 1265, + 1258, 1258, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1258, 1258, 1258, 1258, 1267, 1268, 1258, 1258, 1258, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, + 1258, 1270, 1258, 1258, 1271, 1258, 1258, 1261, 1258, 1262, + 1258, 1258, 1263, 1263, 1263, 1263, 1263, 1263, 1263, 1263, + + 1258, 1264, 1258, 1265, 1258, 1258, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1258, 1267, 1258, 1268, + 1258, 1258, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1270, 1258, 1271, 1263, 1263, + + 1263, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1263, + 1263, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1263, 1263, 1263, 1263, 1263, + 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, + 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1263, 1263, 1263, 1263, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1263, 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + + 1269, 1263, 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1263, 1263, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1263, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, 1266, 1266, + + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1269, 1258, 1269, 1269, 1269, 1269, 1269, + + 1269, 1269, 1269, 1263, 1266, 1266, 1266, 1266, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, + 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, + 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, + 1269, 1269, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, + 1269, 1266, 1266, 1266, 1269, 1269, 1266, 0, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258 } ; -static const flex_int16_t yy_nxt[1389] = +static const flex_int16_t yy_nxt[1396] = { 0, 12, 13, 14, 13, 15, 16, 12, 12, 17, 18, 19, 19, 19, 19, 19, 20, 19, 19, 19, 19, @@ -849,44 +851,44 @@ static const flex_int16_t yy_nxt[1389] = 61, 68, 69, 70, 71, 72, 73, 61, 74, 61, 75, 76, 77, 78, 61, 79, 61, 61, 80, 81, 83, 84, 83, 84, 87, 91, 87, 94, 92, 98, - 95, 101, 105, 101, 108, 106, 109, 113, 120, 110, - 117, 111, 123, 124, 99, 114, 150, 129, 92, 151, - 783, 115, 121, 118, 116, 106, 126, 131, 133, 144, - 127, 130, 137, 132, 128, 146, 138, 146, 156, 151, - 158, 784, 134, 145, 159, 135, 139, 140, 141, 153, - 160, 154, 157, 161, 155, 162, 164, 169, 167, 175, - 192, 163, 170, 182, 189, 190, 235, 193, 194, 176, - - 165, 168, 177, 184, 166, 236, 178, 185, 179, 183, - 186, 87, 219, 87, 91, 187, 180, 92, 188, 181, - 202, 211, 101, 203, 101, 105, 220, 212, 106, 216, - 226, 222, 240, 217, 242, 785, 241, 92, 223, 146, - 218, 146, 227, 243, 150, 256, 263, 151, 106, 258, - 259, 257, 266, 282, 268, 274, 275, 279, 293, 786, - 264, 288, 298, 267, 269, 289, 299, 151, 280, 283, - 351, 290, 787, 352, 362, 294, 300, 367, 363, 397, - 435, 788, 398, 368, 402, 436, 403, 442, 459, 464, - 483, 404, 443, 460, 465, 484, 488, 509, 515, 523, - - 789, 489, 510, 516, 524, 544, 790, 546, 545, 461, - 547, 568, 625, 571, 569, 575, 572, 570, 511, 573, - 576, 595, 597, 619, 596, 598, 620, 622, 637, 621, - 623, 642, 626, 624, 667, 791, 643, 688, 715, 668, - 644, 792, 689, 716, 638, 764, 690, 793, 794, 795, - 765, 796, 797, 798, 799, 799, 799, 800, 801, 802, - 804, 805, 806, 807, 803, 808, 809, 810, 811, 812, - 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, + 95, 101, 105, 101, 108, 106, 109, 114, 110, 111, + 118, 112, 121, 784, 99, 115, 124, 125, 92, 130, + 134, 116, 132, 119, 117, 106, 122, 127, 133, 145, + 157, 128, 138, 131, 135, 129, 139, 136, 147, 151, + 147, 165, 152, 146, 158, 168, 140, 141, 142, 154, + 163, 155, 295, 159, 156, 166, 164, 160, 169, 167, + 170, 176, 152, 161, 178, 171, 162, 183, 179, 296, + + 180, 177, 185, 190, 191, 87, 186, 87, 181, 187, + 213, 182, 193, 184, 188, 91, 214, 189, 92, 194, + 195, 203, 221, 101, 204, 101, 105, 224, 218, 106, + 228, 237, 219, 242, 225, 244, 222, 243, 92, 220, + 238, 147, 229, 147, 245, 151, 258, 265, 152, 106, + 260, 261, 259, 268, 284, 270, 276, 277, 281, 300, + 785, 266, 290, 301, 269, 271, 291, 786, 152, 282, + 285, 354, 292, 302, 355, 365, 370, 400, 787, 366, + 401, 405, 371, 406, 439, 446, 463, 468, 407, 440, + 447, 464, 469, 487, 492, 513, 519, 527, 488, 493, + + 514, 520, 528, 549, 788, 551, 550, 465, 552, 573, + 630, 576, 574, 580, 577, 575, 515, 578, 581, 600, + 602, 624, 601, 603, 625, 627, 643, 626, 628, 648, + 631, 629, 673, 789, 649, 694, 721, 674, 650, 790, + 695, 722, 644, 771, 696, 791, 792, 793, 772, 794, + 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, + 805, 806, 806, 806, 807, 808, 809, 811, 812, 813, + 814, 810, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, - 843, 844, 846, 848, 845, 847, 849, 850, 851, 852, - 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, + 843, 844, 845, 846, 847, 848, 849, 850, 851, 853, + 855, 852, 854, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, - 873, 874, 875, 876, 877, 799, 799, 799, 879, 880, - 882, 884, 881, 883, 885, 886, 887, 888, 889, 890, - 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, - 901, 902, 903, 904, 905, 906, 878, 907, 908, 909, - 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, + 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, + 883, 884, 806, 806, 806, 886, 887, 889, 891, 888, + 890, 892, 893, 894, 895, 896, 897, 898, 899, 900, + 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, + 911, 912, 913, 885, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, @@ -924,72 +926,74 @@ static const flex_int16_t yy_nxt[1389] = 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, - 1250, 82, 82, 82, 82, 85, 85, 85, 85, 88, - 88, 88, 88, 90, 90, 93, 90, 102, 102, 102, - 102, 104, 104, 107, 104, 147, 147, 147, 147, 149, - 149, 152, 149, 195, 782, 781, 195, 197, 197, 780, - 197, 779, 778, 777, 776, 775, 774, 773, 772, 771, - 770, 769, 768, 767, 766, 763, 762, 761, 760, 759, - 758, 757, 756, 755, 754, 753, 752, 751, 750, 749, - 748, 747, 746, 745, 744, 743, 742, 741, 740, 739, - 738, 737, 736, 735, 734, 733, 732, 731, 730, 729, - - 728, 727, 726, 725, 724, 723, 722, 721, 720, 719, - 718, 717, 714, 713, 712, 711, 710, 709, 708, 707, - 706, 705, 704, 703, 702, 701, 700, 699, 698, 697, - 696, 695, 694, 693, 692, 691, 687, 686, 685, 684, - 683, 682, 681, 680, 679, 678, 677, 676, 675, 674, - 673, 672, 671, 670, 669, 666, 665, 664, 663, 662, - 661, 660, 659, 658, 657, 656, 655, 654, 653, 652, - 651, 650, 649, 648, 647, 646, 645, 641, 640, 639, - 636, 635, 634, 633, 632, 631, 630, 629, 628, 627, - 618, 617, 616, 615, 614, 613, 612, 611, 610, 609, - - 608, 607, 606, 605, 604, 603, 602, 601, 600, 599, - 594, 593, 592, 591, 590, 589, 588, 587, 586, 585, - 584, 583, 582, 581, 580, 579, 578, 577, 574, 567, - 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, - 556, 555, 554, 553, 552, 551, 550, 549, 548, 543, - 542, 541, 540, 539, 538, 537, 536, 535, 534, 533, - 532, 531, 530, 529, 528, 527, 526, 525, 522, 521, - 520, 519, 518, 517, 514, 513, 512, 508, 507, 506, - 505, 504, 503, 502, 501, 500, 499, 498, 497, 496, - 495, 494, 493, 492, 491, 490, 487, 486, 485, 482, - - 481, 480, 479, 478, 477, 476, 475, 474, 473, 472, - 471, 470, 469, 468, 467, 466, 463, 462, 458, 457, - 456, 455, 454, 453, 452, 451, 450, 449, 448, 447, - 446, 445, 444, 441, 440, 439, 438, 437, 434, 433, - 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, - 422, 421, 420, 419, 418, 417, 416, 415, 414, 413, - 412, 411, 410, 409, 408, 407, 406, 405, 401, 400, - 399, 396, 395, 394, 393, 392, 391, 390, 389, 388, - 387, 386, 385, 384, 383, 382, 381, 380, 379, 378, - 377, 376, 375, 374, 373, 372, 371, 370, 369, 366, - - 365, 364, 361, 360, 359, 358, 357, 356, 355, 354, - 353, 350, 349, 348, 347, 346, 345, 344, 343, 342, - 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, - 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, - 321, 320, 319, 318, 317, 316, 315, 314, 313, 312, - 311, 310, 309, 308, 307, 196, 306, 305, 304, 303, - 302, 301, 297, 296, 295, 292, 291, 287, 286, 285, - 284, 281, 278, 277, 276, 273, 272, 271, 270, 265, - 262, 261, 260, 255, 254, 253, 148, 252, 251, 250, - 249, 248, 247, 246, 245, 244, 239, 238, 237, 234, - - 233, 232, 231, 230, 229, 228, 225, 224, 221, 215, - 214, 213, 210, 209, 208, 207, 206, 103, 205, 204, - 201, 200, 199, 198, 89, 196, 191, 174, 173, 172, - 171, 148, 143, 142, 136, 125, 122, 119, 112, 103, - 100, 97, 96, 89, 1251, 86, 86, 11, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251 + 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 82, 82, + 82, 82, 85, 85, 85, 85, 88, 88, 88, 88, + 90, 90, 93, 90, 102, 102, 102, 102, 104, 104, + 107, 104, 148, 148, 148, 148, 150, 150, 153, 150, + 196, 783, 782, 196, 198, 198, 781, 198, 780, 779, + 778, 777, 776, 775, 774, 773, 770, 769, 768, 767, + 766, 765, 764, 763, 762, 761, 760, 759, 758, 757, + 756, 755, 754, 753, 752, 751, 750, 749, 748, 747, + 746, 745, 744, 743, 742, 741, 740, 739, 738, 737, + + 736, 735, 734, 733, 732, 731, 730, 729, 728, 727, + 726, 725, 724, 723, 720, 719, 718, 717, 716, 715, + 714, 713, 712, 711, 710, 709, 708, 707, 706, 705, + 704, 703, 702, 701, 700, 699, 698, 697, 693, 692, + 691, 690, 689, 688, 687, 686, 685, 684, 683, 682, + 681, 680, 679, 678, 677, 676, 675, 672, 671, 670, + 669, 668, 667, 666, 665, 664, 663, 662, 661, 660, + 659, 658, 657, 656, 655, 654, 653, 652, 651, 647, + 646, 645, 642, 641, 640, 639, 638, 637, 636, 635, + 634, 633, 632, 623, 622, 621, 620, 619, 618, 617, + + 616, 615, 614, 613, 612, 611, 610, 609, 608, 607, + 606, 605, 604, 599, 598, 597, 596, 595, 594, 593, + 592, 591, 590, 589, 588, 587, 586, 585, 584, 583, + 582, 579, 572, 571, 570, 569, 568, 567, 566, 565, + 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, + 554, 553, 548, 547, 546, 545, 544, 543, 542, 541, + 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, + 530, 529, 526, 525, 524, 523, 522, 521, 518, 517, + 516, 512, 511, 510, 509, 508, 507, 506, 505, 504, + 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, + + 491, 490, 489, 486, 485, 484, 483, 482, 481, 480, + 479, 478, 477, 476, 475, 474, 473, 472, 471, 470, + 467, 466, 462, 461, 460, 459, 458, 457, 456, 455, + 454, 453, 452, 451, 450, 449, 448, 445, 444, 443, + 442, 441, 438, 437, 436, 435, 434, 433, 432, 431, + 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, + 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, + 410, 409, 408, 404, 403, 402, 399, 398, 397, 396, + 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, + 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, + + 375, 374, 373, 372, 369, 368, 367, 364, 363, 362, + 361, 360, 359, 358, 357, 356, 353, 352, 351, 350, + 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, + 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, + 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, + 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, + 309, 197, 308, 307, 306, 305, 304, 303, 299, 298, + 297, 294, 293, 289, 288, 287, 286, 283, 280, 279, + 278, 275, 274, 273, 272, 267, 264, 263, 262, 257, + 256, 255, 149, 254, 253, 252, 251, 250, 249, 248, + + 247, 246, 241, 240, 239, 236, 235, 234, 233, 232, + 231, 230, 227, 226, 223, 217, 216, 215, 212, 211, + 210, 209, 208, 207, 103, 206, 205, 202, 201, 200, + 199, 89, 197, 192, 175, 174, 173, 172, 149, 144, + 143, 137, 126, 123, 120, 113, 103, 100, 97, 96, + 89, 1258, 86, 86, 11, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258 + } ; -static const flex_int16_t yy_chk[1389] = +static const flex_int16_t yy_chk[1396] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1005,144 +1009,146 @@ static const flex_int16_t yy_chk[1389] = 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 7, 7, 8, 8, 13, 17, 13, 20, 17, 23, - 20, 27, 31, 27, 34, 31, 34, 36, 39, 34, - 37, 34, 41, 41, 23, 36, 59, 44, 17, 59, - 684, 36, 39, 37, 36, 31, 43, 45, 46, 51, - 43, 44, 48, 45, 43, 54, 48, 54, 63, 59, - 64, 687, 46, 51, 64, 46, 48, 48, 48, 62, - 64, 62, 63, 64, 62, 65, 66, 68, 67, 73, - 79, 65, 68, 75, 77, 77, 131, 79, 79, 73, - - 66, 67, 74, 76, 66, 131, 74, 76, 74, 75, - 76, 87, 118, 87, 91, 76, 74, 91, 76, 74, - 98, 113, 101, 98, 101, 105, 118, 113, 105, 117, - 123, 120, 135, 117, 136, 688, 135, 91, 120, 146, - 117, 146, 123, 136, 150, 156, 161, 150, 105, 157, - 157, 156, 163, 176, 164, 169, 169, 174, 184, 689, - 161, 181, 188, 163, 164, 181, 188, 150, 174, 176, - 243, 181, 691, 243, 255, 184, 188, 259, 255, 291, - 331, 692, 291, 259, 298, 331, 298, 337, 355, 359, - 378, 298, 337, 355, 359, 378, 383, 406, 410, 419, - - 694, 383, 406, 410, 419, 442, 695, 443, 442, 355, - 443, 464, 518, 465, 464, 467, 465, 464, 406, 465, - 467, 488, 489, 515, 488, 489, 515, 516, 529, 515, - 516, 534, 518, 516, 560, 696, 534, 584, 611, 560, - 534, 697, 584, 611, 529, 664, 584, 698, 699, 700, - 664, 701, 702, 703, 707, 707, 707, 709, 710, 711, - 715, 716, 717, 718, 711, 719, 723, 724, 725, 726, - 727, 728, 729, 730, 733, 734, 735, 736, 737, 738, - 739, 740, 741, 742, 744, 745, 747, 748, 749, 750, - 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, - - 761, 764, 765, 766, 764, 765, 767, 768, 769, 770, - 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, - 781, 782, 783, 785, 786, 787, 789, 791, 792, 793, - 794, 795, 796, 797, 798, 799, 799, 799, 800, 802, - 803, 804, 802, 803, 805, 806, 807, 808, 809, 810, - 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, - 821, 822, 823, 824, 825, 826, 799, 827, 828, 829, - 830, 831, 833, 836, 837, 838, 839, 840, 841, 842, - 843, 844, 845, 846, 847, 848, 849, 850, 852, 853, - 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, - - 865, 866, 867, 868, 869, 870, 871, 873, 874, 875, - 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, - 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, - 897, 898, 899, 900, 901, 903, 904, 905, 906, 907, - 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, - 918, 919, 921, 922, 923, 924, 925, 928, 929, 930, - 931, 932, 933, 934, 935, 936, 937, 938, 939, 941, - 942, 945, 946, 947, 948, 949, 950, 951, 952, 954, - 955, 956, 957, 960, 961, 962, 963, 964, 965, 966, - 967, 968, 969, 970, 971, 974, 975, 976, 978, 979, - - 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, - 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, + 20, 27, 31, 27, 34, 31, 34, 36, 34, 34, + 37, 34, 39, 684, 23, 36, 41, 41, 17, 44, + 46, 36, 45, 37, 36, 31, 39, 43, 45, 51, + 63, 43, 48, 44, 46, 43, 48, 46, 54, 59, + 54, 66, 59, 51, 63, 67, 48, 48, 48, 62, + 65, 62, 185, 64, 62, 66, 65, 64, 67, 66, + 68, 73, 59, 64, 74, 68, 64, 75, 74, 185, + + 74, 73, 76, 77, 77, 87, 76, 87, 74, 76, + 114, 74, 79, 75, 76, 91, 114, 76, 91, 79, + 79, 98, 119, 101, 98, 101, 105, 121, 118, 105, + 124, 132, 118, 136, 121, 137, 119, 136, 91, 118, + 132, 147, 124, 147, 137, 151, 157, 162, 151, 105, + 158, 158, 157, 164, 177, 165, 170, 170, 175, 189, + 685, 162, 182, 189, 164, 165, 182, 686, 151, 175, + 177, 245, 182, 189, 245, 257, 261, 293, 687, 257, + 293, 300, 261, 300, 334, 340, 358, 362, 300, 334, + 340, 358, 362, 381, 386, 409, 413, 422, 381, 386, + + 409, 413, 422, 446, 688, 447, 446, 358, 447, 468, + 522, 469, 468, 471, 469, 468, 409, 469, 471, 492, + 493, 519, 492, 493, 519, 520, 534, 519, 520, 539, + 522, 520, 565, 689, 539, 589, 616, 565, 539, 690, + 589, 616, 534, 670, 589, 693, 694, 695, 670, 697, + 698, 700, 701, 702, 703, 704, 705, 706, 707, 708, + 709, 713, 713, 713, 715, 716, 717, 721, 722, 723, + 724, 717, 725, 729, 730, 731, 732, 733, 734, 735, + 736, 739, 740, 742, 743, 744, 745, 746, 747, 748, + 749, 751, 752, 754, 755, 756, 757, 758, 759, 760, + + 761, 762, 763, 764, 765, 766, 767, 768, 771, 772, + 773, 771, 772, 774, 775, 776, 777, 778, 779, 780, + 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, + 792, 793, 794, 796, 798, 799, 800, 801, 802, 803, + 804, 805, 806, 806, 806, 807, 809, 810, 811, 809, + 810, 812, 813, 814, 815, 816, 817, 818, 819, 820, + 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, + 831, 832, 833, 806, 834, 835, 836, 837, 838, 840, + 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, + 853, 854, 855, 856, 857, 859, 860, 862, 863, 864, + + 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, + 875, 876, 877, 878, 880, 881, 882, 883, 884, 885, + 886, 887, 888, 889, 890, 891, 892, 894, 895, 896, + 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, + 907, 908, 910, 911, 912, 913, 914, 915, 916, 917, + 918, 919, 920, 921, 922, 923, 924, 925, 926, 928, + 929, 930, 931, 932, 935, 936, 937, 938, 939, 940, + 941, 942, 943, 944, 945, 946, 948, 949, 952, 953, + 954, 955, 956, 957, 958, 959, 961, 962, 963, 964, + 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, + + 977, 978, 981, 982, 983, 985, 986, 987, 988, 989, + 990, 991, 992, 993, 994, 995, 996, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, - 1011, 1012, 1013, 1014, 1015, 1016, 1018, 1019, 1020, 1021, - 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, - 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1046, - 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1055, 1056, - 1057, 1058, 1061, 1064, 1065, 1066, 1067, 1068, 1069, 1073, - 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1086, - 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, + 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, + 1021, 1022, 1023, 1025, 1026, 1027, 1028, 1029, 1030, 1031, + 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1040, 1041, 1042, + 1043, 1044, 1045, 1046, 1047, 1048, 1053, 1055, 1056, 1057, + 1058, 1059, 1060, 1061, 1062, 1062, 1063, 1064, 1065, 1068, + 1071, 1072, 1073, 1074, 1075, 1076, 1080, 1082, 1083, 1084, + 1085, 1086, 1087, 1088, 1089, 1090, 1093, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, - 1129, 1130, 1133, 1134, 1135, 1139, 1143, 1144, 1145, 1146, - 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1157, 1162, 1163, - 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, - 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, - 1185, 1186, 1187, 1189, 1190, 1191, 1192, 1193, 1194, 1195, - 1196, 1197, 1198, 1199, 1200, 1202, 1203, 1204, 1205, 1206, - 1207, 1212, 1213, 1214, 1215, 1216, 1222, 1223, 1225, 1227, - - 1229, 1230, 1231, 1232, 1234, 1238, 1239, 1240, 1243, 1244, - 1247, 1252, 1252, 1252, 1252, 1253, 1253, 1253, 1253, 1254, - 1254, 1254, 1254, 1255, 1255, 1256, 1255, 1257, 1257, 1257, - 1257, 1258, 1258, 1259, 1258, 1260, 1260, 1260, 1260, 1261, - 1261, 1262, 1261, 1263, 683, 682, 1263, 1264, 1264, 681, - 1264, 680, 679, 678, 677, 676, 675, 674, 673, 671, - 670, 669, 668, 667, 666, 663, 662, 661, 660, 659, - 658, 657, 656, 655, 654, 653, 652, 651, 650, 649, - 648, 647, 645, 643, 642, 641, 640, 639, 638, 637, - 636, 635, 634, 633, 632, 631, 630, 629, 626, 625, - - 624, 623, 622, 621, 620, 619, 618, 617, 616, 615, - 614, 612, 610, 609, 608, 607, 606, 605, 604, 603, - 602, 601, 600, 599, 598, 597, 596, 595, 593, 592, - 591, 590, 588, 587, 586, 585, 583, 582, 581, 580, - 579, 578, 576, 575, 573, 572, 571, 570, 569, 568, - 566, 565, 564, 563, 561, 559, 557, 556, 555, 554, - 553, 552, 551, 549, 547, 546, 545, 544, 543, 542, - 541, 540, 539, 538, 537, 536, 535, 533, 532, 530, - 528, 527, 526, 525, 524, 523, 522, 521, 520, 519, - 513, 512, 511, 510, 509, 504, 503, 502, 501, 500, - - 499, 498, 497, 496, 495, 494, 493, 492, 491, 490, - 487, 485, 484, 483, 482, 481, 480, 478, 477, 476, - 475, 474, 473, 472, 471, 470, 469, 468, 466, 463, - 462, 461, 460, 459, 458, 457, 456, 455, 454, 453, - 452, 451, 450, 449, 448, 447, 446, 445, 444, 441, - 440, 439, 437, 436, 435, 433, 432, 431, 430, 429, - 428, 427, 426, 425, 424, 423, 422, 421, 418, 416, - 415, 414, 413, 411, 409, 408, 407, 404, 403, 402, - 401, 400, 399, 398, 397, 396, 395, 394, 393, 392, - 391, 390, 389, 388, 387, 386, 382, 380, 379, 377, - - 376, 375, 374, 373, 372, 370, 369, 368, 367, 366, - 365, 364, 363, 362, 361, 360, 358, 356, 354, 353, - 352, 351, 350, 349, 348, 346, 345, 343, 342, 341, - 340, 339, 338, 336, 335, 334, 333, 332, 330, 329, - 328, 327, 326, 325, 324, 323, 322, 321, 320, 319, - 318, 317, 316, 315, 314, 312, 311, 309, 308, 307, - 306, 305, 304, 303, 302, 301, 300, 299, 296, 295, - 293, 290, 289, 288, 287, 286, 285, 284, 282, 281, - 280, 279, 278, 277, 276, 275, 274, 273, 270, 269, - 268, 267, 266, 265, 264, 263, 262, 261, 260, 258, - - 257, 256, 254, 252, 251, 250, 249, 248, 247, 245, - 244, 242, 241, 240, 239, 238, 237, 236, 235, 234, - 233, 232, 231, 230, 229, 228, 227, 226, 225, 224, - 223, 222, 221, 220, 219, 218, 217, 216, 215, 214, - 213, 212, 211, 210, 209, 208, 207, 205, 204, 203, - 202, 201, 200, 199, 198, 196, 194, 193, 192, 191, - 190, 189, 187, 186, 185, 183, 182, 180, 179, 178, - 177, 175, 173, 172, 171, 168, 167, 166, 165, 162, - 160, 159, 158, 155, 154, 153, 147, 145, 144, 143, - 142, 141, 140, 139, 138, 137, 134, 133, 132, 130, - - 129, 128, 127, 126, 125, 124, 122, 121, 119, 116, - 115, 114, 112, 111, 110, 109, 108, 102, 100, 99, - 97, 96, 95, 94, 88, 83, 78, 72, 71, 70, - 69, 56, 50, 49, 47, 42, 40, 38, 35, 29, - 24, 22, 21, 15, 11, 10, 9, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251, - 1251, 1251, 1251, 1251, 1251, 1251, 1251, 1251 + 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1140, + 1141, 1142, 1146, 1150, 1151, 1152, 1153, 1154, 1155, 1156, + 1157, 1158, 1159, 1160, 1164, 1169, 1170, 1171, 1172, 1173, + 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1182, 1183, 1184, + 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, + 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, + 1206, 1207, 1209, 1210, 1211, 1212, 1213, 1214, 1219, 1220, + + 1221, 1222, 1223, 1229, 1230, 1232, 1234, 1236, 1237, 1238, + 1239, 1241, 1245, 1246, 1247, 1250, 1251, 1254, 1259, 1259, + 1259, 1259, 1260, 1260, 1260, 1260, 1261, 1261, 1261, 1261, + 1262, 1262, 1263, 1262, 1264, 1264, 1264, 1264, 1265, 1265, + 1266, 1265, 1267, 1267, 1267, 1267, 1268, 1268, 1269, 1268, + 1270, 683, 682, 1270, 1271, 1271, 681, 1271, 680, 679, + 677, 676, 675, 674, 673, 672, 669, 668, 667, 666, + 665, 664, 663, 662, 661, 660, 659, 658, 657, 656, + 655, 654, 653, 651, 649, 648, 647, 646, 645, 644, + 643, 642, 641, 640, 639, 638, 637, 636, 635, 634, + + 631, 630, 629, 628, 627, 626, 625, 624, 623, 622, + 621, 620, 619, 617, 615, 614, 613, 612, 611, 610, + 609, 608, 607, 606, 605, 604, 603, 602, 601, 600, + 598, 597, 596, 595, 593, 592, 591, 590, 588, 587, + 586, 585, 584, 583, 581, 580, 578, 577, 576, 575, + 574, 573, 571, 570, 569, 568, 566, 564, 562, 561, + 560, 559, 558, 557, 556, 554, 552, 551, 550, 549, + 548, 547, 546, 545, 544, 543, 542, 541, 540, 538, + 537, 535, 533, 532, 531, 530, 529, 528, 527, 526, + 525, 524, 523, 517, 516, 515, 514, 513, 508, 507, + + 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, + 496, 495, 494, 491, 489, 488, 487, 486, 485, 484, + 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, + 472, 470, 467, 466, 465, 464, 463, 462, 461, 460, + 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, + 449, 448, 445, 444, 443, 441, 440, 439, 437, 436, + 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, + 425, 423, 421, 419, 418, 417, 416, 414, 412, 411, + 410, 407, 406, 405, 404, 403, 402, 401, 400, 399, + 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, + + 385, 383, 382, 380, 379, 378, 377, 376, 375, 373, + 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, + 361, 359, 357, 356, 355, 354, 353, 352, 351, 349, + 348, 346, 345, 344, 343, 342, 341, 339, 338, 337, + 336, 335, 333, 332, 331, 330, 329, 328, 327, 326, + 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, + 314, 313, 311, 310, 309, 308, 307, 306, 305, 304, + 303, 302, 301, 298, 297, 295, 292, 291, 290, 289, + 288, 287, 286, 284, 283, 282, 281, 280, 279, 278, + 277, 276, 275, 272, 271, 270, 269, 268, 267, 266, + + 265, 264, 263, 262, 260, 259, 258, 256, 254, 253, + 252, 251, 250, 249, 247, 246, 244, 243, 242, 241, + 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, + 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, + 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, + 210, 209, 208, 206, 205, 204, 203, 202, 201, 200, + 199, 197, 195, 194, 193, 192, 191, 190, 188, 187, + 186, 184, 183, 181, 180, 179, 178, 176, 174, 173, + 172, 169, 168, 167, 166, 163, 161, 160, 159, 156, + 155, 154, 148, 146, 145, 144, 143, 142, 141, 140, + + 139, 138, 135, 134, 133, 131, 130, 129, 128, 127, + 126, 125, 123, 122, 120, 117, 116, 115, 113, 112, + 111, 110, 109, 108, 102, 100, 99, 97, 96, 95, + 94, 88, 83, 78, 72, 71, 70, 69, 56, 50, + 49, 47, 42, 40, 38, 35, 29, 24, 22, 21, + 15, 11, 10, 9, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, + 1258, 1258, 1258, 1258, 1258 + } ; static yy_state_type yy_last_accepting_state; @@ -1237,9 +1243,9 @@ static char *pgaf_strdup(const char *s) * call flex's static input() function. */ static void pgaf_read_raw_block(void); -#line 1240 "test_spec_scan.c" +#line 1246 "test_spec_scan.c" -#line 1242 "test_spec_scan.c" +#line 1248 "test_spec_scan.c" #define INITIAL 0 #define CLUSTER_BODY 1 @@ -1461,7 +1467,7 @@ YY_DECL #line 89 "test_spec_scan.l" -#line 1464 "test_spec_scan.c" +#line 1470 "test_spec_scan.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1488,13 +1494,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1259 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_current_state != 1251 ); + while ( yy_current_state != 1258 ); yy_cp = (yy_last_accepting_cpos); yy_current_state = (yy_last_accepting_state); @@ -1709,177 +1715,182 @@ YY_RULE_SETUP case 33: YY_RULE_SETUP #line 164 "test_spec_scan.l" -{ return T_ASYNC; } +{ return T_ARCHIVER; } YY_BREAK case 34: YY_RULE_SETUP #line 165 "test_spec_scan.l" -{ return T_NO_MONITOR; } +{ return T_ASYNC; } YY_BREAK case 35: YY_RULE_SETUP #line 166 "test_spec_scan.l" -{ return T_SUSPENDED; } +{ return T_NO_MONITOR; } YY_BREAK case 36: YY_RULE_SETUP #line 167 "test_spec_scan.l" -{ return T_PASSWORD; } +{ return T_SUSPENDED; } YY_BREAK case 37: YY_RULE_SETUP #line 168 "test_spec_scan.l" -{ return T_MONITOR_PASSWORD; } +{ return T_PASSWORD; } YY_BREAK case 38: YY_RULE_SETUP #line 169 "test_spec_scan.l" -{ return T_LAUNCH; } +{ return T_MONITOR_PASSWORD; } YY_BREAK case 39: YY_RULE_SETUP #line 170 "test_spec_scan.l" -{ return T_CREATE; } +{ return T_LAUNCH; } YY_BREAK case 40: YY_RULE_SETUP #line 171 "test_spec_scan.l" -{ return T_DEFERRED; } +{ return T_CREATE; } YY_BREAK case 41: YY_RULE_SETUP #line 172 "test_spec_scan.l" -{ return T_IMMEDIATE; } +{ return T_DEFERRED; } YY_BREAK case 42: YY_RULE_SETUP #line 173 "test_spec_scan.l" -{ return T_FALSE; } +{ return T_IMMEDIATE; } YY_BREAK case 43: YY_RULE_SETUP #line 174 "test_spec_scan.l" -{ return T_TRUE; } +{ return T_FALSE; } YY_BREAK case 44: YY_RULE_SETUP #line 175 "test_spec_scan.l" -{ return T_AND; } +{ return T_TRUE; } YY_BREAK case 45: YY_RULE_SETUP #line 176 "test_spec_scan.l" -{ return T_INITIALLY; } +{ return T_AND; } YY_BREAK case 46: YY_RULE_SETUP #line 177 "test_spec_scan.l" -{ return T_STOPPED; } +{ return T_INITIALLY; } YY_BREAK case 47: YY_RULE_SETUP #line 178 "test_spec_scan.l" -{ return T_VOLUME; } +{ return T_STOPPED; } YY_BREAK case 48: YY_RULE_SETUP #line 179 "test_spec_scan.l" -{ return T_LISTEN; } +{ return T_VOLUME; } YY_BREAK case 49: YY_RULE_SETUP #line 180 "test_spec_scan.l" -{ return T_CITUS_SECONDARY; } +{ return T_LISTEN; } YY_BREAK case 50: YY_RULE_SETUP #line 181 "test_spec_scan.l" -{ return T_CANDIDATE_PRIORITY; } +{ return T_CITUS_SECONDARY; } YY_BREAK case 51: YY_RULE_SETUP #line 182 "test_spec_scan.l" -{ return T_REGION; } +{ return T_CANDIDATE_PRIORITY; } YY_BREAK case 52: YY_RULE_SETUP #line 183 "test_spec_scan.l" -{ return T_GROUP; } +{ return T_REGION; } YY_BREAK case 53: YY_RULE_SETUP #line 184 "test_spec_scan.l" -{ return T_PORT; } +{ return T_GROUP; } YY_BREAK case 54: YY_RULE_SETUP #line 185 "test_spec_scan.l" -{ return T_CITUS_CLUSTER_NAME; } +{ return T_PORT; } YY_BREAK case 55: YY_RULE_SETUP #line 186 "test_spec_scan.l" -{ return T_DEBIAN_CLUSTER; } +{ return T_CITUS_CLUSTER_NAME; } YY_BREAK case 56: YY_RULE_SETUP #line 187 "test_spec_scan.l" -{ return T_REPLICATION_QUORUM; } +{ return T_DEBIAN_CLUSTER; } YY_BREAK case 57: YY_RULE_SETUP #line 188 "test_spec_scan.l" -{ return T_REPLICATION_PASSWORD; } +{ return T_REPLICATION_QUORUM; } YY_BREAK case 58: YY_RULE_SETUP #line 189 "test_spec_scan.l" -{ return T_EXTENSION_VERSION; } +{ return T_REPLICATION_PASSWORD; } YY_BREAK case 59: YY_RULE_SETUP #line 190 "test_spec_scan.l" -{ return T_BIND_SOURCE; } +{ return T_EXTENSION_VERSION; } YY_BREAK case 60: YY_RULE_SETUP #line 191 "test_spec_scan.l" -{ return T_LEGACY_STARTUP; } +{ return T_BIND_SOURCE; } YY_BREAK case 61: YY_RULE_SETUP -#line 193 "test_spec_scan.l" -{ return T_EQUALS; } +#line 192 "test_spec_scan.l" +{ return T_LEGACY_STARTUP; } YY_BREAK case 62: YY_RULE_SETUP -#line 195 "test_spec_scan.l" +#line 194 "test_spec_scan.l" +{ return T_EQUALS; } + YY_BREAK +case 63: +YY_RULE_SETUP +#line 196 "test_spec_scan.l" { yylval.ival = atoi(yytext); return T_INTEGER; } YY_BREAK -case 63: -/* rule 63 can match eol */ +case 64: +/* rule 64 can match eol */ YY_RULE_SETUP -#line 200 "test_spec_scan.l" +#line 201 "test_spec_scan.l" { yytext[yyleng - 1] = '\0'; yylval.str = pgaf_strdup(yytext + 1); return T_STRING; } YY_BREAK -case 64: +case 65: YY_RULE_SETUP -#line 206 "test_spec_scan.l" +#line 207 "test_spec_scan.l" { pgaf_cluster_depth++; return T_LBRACE; } YY_BREAK -case 65: +case 66: YY_RULE_SETUP -#line 211 "test_spec_scan.l" +#line 212 "test_spec_scan.l" { pgaf_cluster_depth--; if (pgaf_cluster_depth == 0) @@ -1887,25 +1898,20 @@ YY_RULE_SETUP return T_RBRACE; } YY_BREAK -case 66: -YY_RULE_SETUP -#line 218 "test_spec_scan.l" -{ return T_FS_INIT; } - YY_BREAK case 67: YY_RULE_SETUP #line 219 "test_spec_scan.l" -{ return T_FS_SINGLE; } +{ return T_FS_INIT; } YY_BREAK case 68: YY_RULE_SETUP #line 220 "test_spec_scan.l" -{ return T_FS_PRIMARY; } +{ return T_FS_SINGLE; } YY_BREAK case 69: YY_RULE_SETUP #line 221 "test_spec_scan.l" -{ return T_FS_WAIT_PRIMARY; } +{ return T_FS_PRIMARY; } YY_BREAK case 70: YY_RULE_SETUP @@ -1915,7 +1921,7 @@ YY_RULE_SETUP case 71: YY_RULE_SETUP #line 223 "test_spec_scan.l" -{ return T_FS_WAIT_STANDBY; } +{ return T_FS_WAIT_PRIMARY; } YY_BREAK case 72: YY_RULE_SETUP @@ -1925,12 +1931,12 @@ YY_RULE_SETUP case 73: YY_RULE_SETUP #line 225 "test_spec_scan.l" -{ return T_FS_DEMOTED; } +{ return T_FS_WAIT_STANDBY; } YY_BREAK case 74: YY_RULE_SETUP #line 226 "test_spec_scan.l" -{ return T_FS_DEMOTE_TIMEOUT; } +{ return T_FS_DEMOTED; } YY_BREAK case 75: YY_RULE_SETUP @@ -1940,22 +1946,22 @@ YY_RULE_SETUP case 76: YY_RULE_SETUP #line 228 "test_spec_scan.l" -{ return T_FS_DRAINING; } +{ return T_FS_DEMOTE_TIMEOUT; } YY_BREAK case 77: YY_RULE_SETUP #line 229 "test_spec_scan.l" -{ return T_FS_SECONDARY; } +{ return T_FS_DRAINING; } YY_BREAK case 78: YY_RULE_SETUP #line 230 "test_spec_scan.l" -{ return T_FS_CATCHINGUP; } +{ return T_FS_SECONDARY; } YY_BREAK case 79: YY_RULE_SETUP #line 231 "test_spec_scan.l" -{ return T_FS_PREP_PROMOTION; } +{ return T_FS_CATCHINGUP; } YY_BREAK case 80: YY_RULE_SETUP @@ -1965,7 +1971,7 @@ YY_RULE_SETUP case 81: YY_RULE_SETUP #line 233 "test_spec_scan.l" -{ return T_FS_STOP_REPLICATION; } +{ return T_FS_PREP_PROMOTION; } YY_BREAK case 82: YY_RULE_SETUP @@ -1975,12 +1981,12 @@ YY_RULE_SETUP case 83: YY_RULE_SETUP #line 235 "test_spec_scan.l" -{ return T_FS_MAINTENANCE; } +{ return T_FS_STOP_REPLICATION; } YY_BREAK case 84: YY_RULE_SETUP #line 236 "test_spec_scan.l" -{ return T_FS_JOIN_PRIMARY; } +{ return T_FS_MAINTENANCE; } YY_BREAK case 85: YY_RULE_SETUP @@ -1990,7 +1996,7 @@ YY_RULE_SETUP case 86: YY_RULE_SETUP #line 238 "test_spec_scan.l" -{ return T_FS_APPLY_SETTINGS; } +{ return T_FS_JOIN_PRIMARY; } YY_BREAK case 87: YY_RULE_SETUP @@ -2000,7 +2006,7 @@ YY_RULE_SETUP case 88: YY_RULE_SETUP #line 240 "test_spec_scan.l" -{ return T_FS_PREPARE_MAINTENANCE; } +{ return T_FS_APPLY_SETTINGS; } YY_BREAK case 89: YY_RULE_SETUP @@ -2010,7 +2016,7 @@ YY_RULE_SETUP case 90: YY_RULE_SETUP #line 242 "test_spec_scan.l" -{ return T_FS_WAIT_MAINTENANCE; } +{ return T_FS_PREPARE_MAINTENANCE; } YY_BREAK case 91: YY_RULE_SETUP @@ -2020,7 +2026,7 @@ YY_RULE_SETUP case 92: YY_RULE_SETUP #line 244 "test_spec_scan.l" -{ return T_FS_REPORT_LSN; } +{ return T_FS_WAIT_MAINTENANCE; } YY_BREAK case 93: YY_RULE_SETUP @@ -2030,7 +2036,7 @@ YY_RULE_SETUP case 94: YY_RULE_SETUP #line 246 "test_spec_scan.l" -{ return T_FS_FAST_FORWARD; } +{ return T_FS_REPORT_LSN; } YY_BREAK case 95: YY_RULE_SETUP @@ -2040,7 +2046,7 @@ YY_RULE_SETUP case 96: YY_RULE_SETUP #line 248 "test_spec_scan.l" -{ return T_FS_JOIN_SECONDARY; } +{ return T_FS_FAST_FORWARD; } YY_BREAK case 97: YY_RULE_SETUP @@ -2050,244 +2056,244 @@ YY_RULE_SETUP case 98: YY_RULE_SETUP #line 250 "test_spec_scan.l" -{ return T_FS_DROPPED; } +{ return T_FS_JOIN_SECONDARY; } YY_BREAK case 99: YY_RULE_SETUP -#line 252 "test_spec_scan.l" +#line 251 "test_spec_scan.l" +{ return T_FS_DROPPED; } + YY_BREAK +case 100: +YY_RULE_SETUP +#line 253 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); return T_IDENT; } YY_BREAK -case 100: -YY_RULE_SETUP -#line 257 "test_spec_scan.l" -{ /* comment */ } - YY_BREAK case 101: -/* rule 101 can match eol */ YY_RULE_SETUP #line 258 "test_spec_scan.l" -{ pgaf_line_number++; } +{ /* comment */ } YY_BREAK case 102: +/* rule 102 can match eol */ YY_RULE_SETUP #line 259 "test_spec_scan.l" -{ /* whitespace */ } +{ pgaf_line_number++; } YY_BREAK case 103: YY_RULE_SETUP -#line 261 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } +#line 260 "test_spec_scan.l" +{ /* whitespace */ } YY_BREAK case 104: YY_RULE_SETUP #line 262 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_EXEC; } +{ BEGIN(EXEC_ARGS); return T_EXEC_FAILS; } YY_BREAK case 105: YY_RULE_SETUP #line 263 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_RUN; } +{ BEGIN(EXEC_ARGS); return T_EXEC; } YY_BREAK case 106: YY_RULE_SETUP #line 264 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } +{ BEGIN(EXEC_ARGS); return T_RUN; } YY_BREAK case 107: YY_RULE_SETUP -#line 266 "test_spec_scan.l" -{ return T_WAIT; } +#line 265 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_PG_AUTOCTL; } YY_BREAK case 108: YY_RULE_SETUP #line 267 "test_spec_scan.l" -{ return T_UNTIL; } +{ return T_WAIT; } YY_BREAK case 109: YY_RULE_SETUP #line 268 "test_spec_scan.l" -{ return T_REPLAYS; } +{ return T_UNTIL; } YY_BREAK case 110: YY_RULE_SETUP #line 269 "test_spec_scan.l" -{ return T_TIMEOUT; } +{ return T_REPLAYS; } YY_BREAK case 111: YY_RULE_SETUP #line 270 "test_spec_scan.l" -{ return T_ASSERT; } +{ return T_TIMEOUT; } YY_BREAK case 112: YY_RULE_SETUP #line 271 "test_spec_scan.l" -{ return T_SQL; } +{ return T_ASSERT; } YY_BREAK case 113: YY_RULE_SETUP #line 272 "test_spec_scan.l" -{ return T_EXPECT; } +{ return T_SQL; } YY_BREAK case 114: YY_RULE_SETUP #line 273 "test_spec_scan.l" -{ return T_ERROR; } +{ return T_EXPECT; } YY_BREAK case 115: YY_RULE_SETUP #line 274 "test_spec_scan.l" -{ return T_PROMOTE; } +{ return T_ERROR; } YY_BREAK case 116: YY_RULE_SETUP #line 275 "test_spec_scan.l" -{ return T_PERFORM; } +{ return T_PROMOTE; } YY_BREAK case 117: YY_RULE_SETUP #line 276 "test_spec_scan.l" -{ return T_FAILOVER; } +{ return T_PERFORM; } YY_BREAK case 118: YY_RULE_SETUP #line 277 "test_spec_scan.l" -{ return T_NETWORK; } +{ return T_FAILOVER; } YY_BREAK case 119: YY_RULE_SETUP #line 278 "test_spec_scan.l" -{ return T_DISCONNECT; } +{ return T_NETWORK; } YY_BREAK case 120: YY_RULE_SETUP #line 279 "test_spec_scan.l" -{ return T_CONNECT; } +{ return T_DISCONNECT; } YY_BREAK case 121: YY_RULE_SETUP #line 280 "test_spec_scan.l" -{ return T_SLEEP; } +{ return T_CONNECT; } YY_BREAK case 122: YY_RULE_SETUP #line 281 "test_spec_scan.l" -{ return T_COMPOSE; } +{ return T_SLEEP; } YY_BREAK case 123: YY_RULE_SETUP #line 282 "test_spec_scan.l" -{ return T_NODEINI; } +{ return T_COMPOSE; } YY_BREAK case 124: YY_RULE_SETUP #line 283 "test_spec_scan.l" -{ return T_DOWN; } +{ return T_NODEINI; } YY_BREAK case 125: YY_RULE_SETUP #line 284 "test_spec_scan.l" -{ return T_START; } +{ return T_DOWN; } YY_BREAK case 126: YY_RULE_SETUP #line 285 "test_spec_scan.l" -{ return T_STOP; } +{ return T_START; } YY_BREAK case 127: YY_RULE_SETUP #line 286 "test_spec_scan.l" -{ return T_STOPPED; } +{ return T_STOP; } YY_BREAK case 128: YY_RULE_SETUP #line 287 "test_spec_scan.l" -{ return T_KILL; } +{ return T_STOPPED; } YY_BREAK case 129: YY_RULE_SETUP #line 288 "test_spec_scan.l" -{ return T_IN; } +{ return T_KILL; } YY_BREAK case 130: YY_RULE_SETUP #line 289 "test_spec_scan.l" -{ return T_STATE; } +{ return T_IN; } YY_BREAK case 131: YY_RULE_SETUP #line 290 "test_spec_scan.l" -{ return T_ASSIGNED_STATE; } +{ return T_STATE; } YY_BREAK case 132: YY_RULE_SETUP #line 291 "test_spec_scan.l" -{ return T_CANDIDATE_PRIORITY; } +{ return T_ASSIGNED_STATE; } YY_BREAK case 133: YY_RULE_SETUP #line 292 "test_spec_scan.l" -{ return T_GROUP; } +{ return T_CANDIDATE_PRIORITY; } YY_BREAK case 134: YY_RULE_SETUP #line 293 "test_spec_scan.l" -{ return T_AND; } +{ return T_GROUP; } YY_BREAK case 135: YY_RULE_SETUP #line 294 "test_spec_scan.l" -{ return T_IS; } +{ return T_AND; } YY_BREAK case 136: YY_RULE_SETUP #line 295 "test_spec_scan.l" -{ return T_WITH; } +{ return T_IS; } YY_BREAK case 137: YY_RULE_SETUP #line 296 "test_spec_scan.l" -{ return T_EQUALS; } +{ return T_WITH; } YY_BREAK case 138: YY_RULE_SETUP #line 297 "test_spec_scan.l" -{ return T_COMMA; } +{ return T_EQUALS; } YY_BREAK case 139: YY_RULE_SETUP #line 298 "test_spec_scan.l" -{ return T_POSTGRES; } +{ return T_COMMA; } YY_BREAK case 140: YY_RULE_SETUP #line 299 "test_spec_scan.l" -{ return T_FSM; } +{ return T_POSTGRES; } YY_BREAK case 141: YY_RULE_SETUP #line 300 "test_spec_scan.l" -{ return T_STEP; } +{ return T_FSM; } YY_BREAK case 142: YY_RULE_SETUP #line 301 "test_spec_scan.l" -{ return T_STAYS; } +{ return T_STEP; } YY_BREAK case 143: YY_RULE_SETUP #line 302 "test_spec_scan.l" -{ return T_WHILE; } +{ return T_STAYS; } YY_BREAK case 144: -/* rule 144 can match eol */ YY_RULE_SETUP #line 303 "test_spec_scan.l" -{ return T_THROUGH; } +{ return T_WHILE; } YY_BREAK case 145: +/* rule 145 can match eol */ YY_RULE_SETUP #line 304 "test_spec_scan.l" { return T_THROUGH; } @@ -2295,59 +2301,64 @@ YY_RULE_SETUP case 146: YY_RULE_SETUP #line 305 "test_spec_scan.l" -{ return T_SET; } +{ return T_THROUGH; } YY_BREAK case 147: YY_RULE_SETUP #line 306 "test_spec_scan.l" -{ return T_GET; } +{ return T_SET; } YY_BREAK case 148: YY_RULE_SETUP #line 307 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_INJECT; } +{ return T_GET; } YY_BREAK case 149: YY_RULE_SETUP #line 308 "test_spec_scan.l" -{ return T_LOGS; } +{ BEGIN(EXEC_ARGS); return T_INJECT; } YY_BREAK case 150: YY_RULE_SETUP #line 309 "test_spec_scan.l" -{ return T_NOT; } +{ return T_LOGS; } YY_BREAK case 151: YY_RULE_SETUP #line 310 "test_spec_scan.l" -{ return T_CONTAINS; } +{ return T_NOT; } YY_BREAK case 152: YY_RULE_SETUP #line 311 "test_spec_scan.l" -{ return T_MATCHES; } +{ return T_CONTAINS; } YY_BREAK case 153: YY_RULE_SETUP -#line 313 "test_spec_scan.l" +#line 312 "test_spec_scan.l" +{ return T_MATCHES; } + YY_BREAK +case 154: +YY_RULE_SETUP +#line 314 "test_spec_scan.l" { yylval.ival = atoi(yytext); return T_INTEGER; } YY_BREAK -case 154: -/* rule 154 can match eol */ +case 155: +/* rule 155 can match eol */ YY_RULE_SETUP -#line 318 "test_spec_scan.l" +#line 319 "test_spec_scan.l" { yytext[yyleng - 1] = '\0'; yylval.str = pgaf_strdup(yytext + 1); return T_STRING; } YY_BREAK -case 155: +case 156: YY_RULE_SETUP -#line 324 "test_spec_scan.l" +#line 325 "test_spec_scan.l" { if (pgaf_next_brace_is_while) { pgaf_next_brace_is_while = 0; @@ -2358,9 +2369,9 @@ YY_RULE_SETUP return T_BLOCK; } YY_BREAK -case 156: +case 157: YY_RULE_SETUP -#line 334 "test_spec_scan.l" +#line 335 "test_spec_scan.l" { if (pgaf_step_brace_depth > 0) { pgaf_step_brace_depth--; @@ -2371,40 +2382,40 @@ YY_RULE_SETUP return T_RBRACE; } YY_BREAK -case 157: +case 158: YY_RULE_SETUP -#line 344 "test_spec_scan.l" +#line 345 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); return T_IDENT; } YY_BREAK -case 158: +case 159: YY_RULE_SETUP -#line 349 "test_spec_scan.l" +#line 350 "test_spec_scan.l" { /* skip whitespace before service name */ } YY_BREAK -case 159: +case 160: YY_RULE_SETUP -#line 351 "test_spec_scan.l" +#line 352 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); BEGIN(EXEC_ARGS_REST); return T_IDENT; } YY_BREAK -case 160: -/* rule 160 can match eol */ +case 161: +/* rule 161 can match eol */ YY_RULE_SETUP -#line 357 "test_spec_scan.l" +#line 358 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 161: +case 162: YY_RULE_SETUP -#line 362 "test_spec_scan.l" +#line 363 "test_spec_scan.l" { char *p = yytext; while (*p == ' ' || *p == '\t') p++; @@ -2413,21 +2424,21 @@ YY_RULE_SETUP return T_SHELL_ARGS; } YY_BREAK -case 162: -/* rule 162 can match eol */ +case 163: +/* rule 163 can match eol */ YY_RULE_SETUP -#line 370 "test_spec_scan.l" +#line 371 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 163: +case 164: YY_RULE_SETUP -#line 375 "test_spec_scan.l" +#line 376 "test_spec_scan.l" ECHO; YY_BREAK -#line 2430 "test_spec_scan.c" +#line 2441 "test_spec_scan.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(CLUSTER_BODY): case YY_STATE_EOF(STEP_BODY): @@ -2729,7 +2740,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1259 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2757,11 +2768,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1252 ) + if ( yy_current_state >= 1259 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 1251); + yy_is_jam = (yy_current_state == 1258); return yy_is_jam ? 0 : yy_current_state; } @@ -3400,7 +3411,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 375 "test_spec_scan.l" +#line 376 "test_spec_scan.l" static void diff --git a/src/bin/pgaftest/test_spec_scan.l b/src/bin/pgaftest/test_spec_scan.l index c456617fe..1ad9d10e7 100644 --- a/src/bin/pgaftest/test_spec_scan.l +++ b/src/bin/pgaftest/test_spec_scan.l @@ -161,6 +161,7 @@ static void pgaf_read_raw_block(void); "coordinator" { return T_COORDINATOR; } "worker" { return T_WORKER; } +"archiver" { return T_ARCHIVER; } "async" { return T_ASYNC; } "no-monitor" { return T_NO_MONITOR; } "suspended" { return T_SUSPENDED; } From cf2c99fd3351ab3ea4bfe4453a307ae6b7e22eb1 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 14:29:33 +0200 Subject: [PATCH 12/55] tests: add archiver_wal_capture.pgaf (M4 WAL-capture + failover) First pgaftest spec exercising an ARCHIVING node, covering the two things Milestone 4 adds: - test_001/test_002: forcing WAL segment switches on the primary gets each completed segment reported to the monitor (service_archiver_report_captured_wal(), service_archiver.c) and reflected by pgautofailover.wal_archived() -- the archive_command confirmation check nothing populated before this milestone. - test_002 also exercises the liveness fix in service_archiver_loop(): killing and restarting the archiver process while it's already ARCHIVING must bring pg_receivewal back up on its own, not just on the FSM transition that first enters that state. - test_003: fails node1 over to node2 and confirms the archiver passes through REPORT_LSN_STATE and back to ARCHIVING_STATE (following the new primary), and that segments recorded before the failover are still there afterwards. Segment filenames are asserted directly (a fresh cluster deterministically starts WAL at 000000010000000000000001, and each pg_switch_wal() on an idle test database advances exactly one segment) since autoctl_node has no direct SELECT on archiver_wal -- wal_archived() is the only accessor it can call. Registered in tests/tap/schedule and tests/tap/schedules/node.sch, alongside the other node-lifecycle/FSM specs. --- tests/tap/schedule | 1 + tests/tap/schedules/node.sch | 1 + tests/tap/specs/archiver_wal_capture.pgaf | 111 ++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 tests/tap/specs/archiver_wal_capture.pgaf diff --git a/tests/tap/schedule b/tests/tap/schedule index 3b29df16a..619da99b8 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -31,6 +31,7 @@ fast_forward demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock +archiver_wal_capture keeper_fsm_gap_209_wait_maintenance keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 1c9ed4fef..2c6f99f5c 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -18,3 +18,4 @@ replication_stall_3dc demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect +archiver_wal_capture diff --git a/tests/tap/specs/archiver_wal_capture.pgaf b/tests/tap/specs/archiver_wal_capture.pgaf new file mode 100644 index 000000000..520cf82af --- /dev/null +++ b/tests/tap/specs/archiver_wal_capture.pgaf @@ -0,0 +1,111 @@ +# Archiving & Disaster Recovery, Milestone 4: WAL-capture correctness and +# failover continuity for an ARCHIVING node. +# +# Covers the mechanism added in service_archiver.c's +# service_archiver_report_captured_wal(): once pg_receivewal completes a WAL +# segment (no longer ".partial"), the archiver reports it to the monitor via +# pgautofailover.report_wal_received(), which is what actually populates +# pgautofailover.archiver_wal and makes wal_archived() -- the archive_command +# confirmation check -- return true. Before this milestone nothing in the +# codebase ever called that SQL function, so archiver_wal stayed permanently +# empty no matter how much WAL an archiver captured. +# +# Segment filenames are deterministic: a freshly initialized primary starts +# WAL at timeline 1, segment 000000010000000000000001, and each +# pg_switch_wal() call on an otherwise idle test database advances exactly +# one segment (confirmed against a real cluster while developing this spec). +# The autoctl_node role has no direct SELECT on archiver_wal (see +# report_wal_received()'s own SECURITY DEFINER indirection in +# pgautofailover.sql) -- wal_archived() is the one function it can call to +# observe that table's contents, so every check here goes through it rather +# than a raw SELECT. +# +# Predecessor: none -- this is the first archiver-kind pgaftest spec. + +cluster { + monitor + formation { + node1 + node2 + archiver1 archiver + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: force two WAL segment switches on the primary and confirm both +# land durably in archiver_wal (archiver_quorum defaults to 1, and +# there is exactly one archiver here, so wal_archived() flips to +# true as soon as service_archiver_report_captured_wal()'s next +# tick reports the segment). +# + +step test_001_capture_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { SELECT pg_switch_wal(); } + # PG_AUTOCTL_KEEPER_SLEEP_TIME is 1s; this margin covers a slow CI runner. + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000001'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000002'); } + expect { t } +} + +# +# test_002: kill and restart the archiver process while it is already +# ARCHIVING (persisted state, no FSM transition on restart). This +# exercises service_archiver_loop()'s own liveness check: without +# it, pg_receivewal never comes back up after a restart, because +# it is otherwise only (re)started from fsm_init_archiver / +# fsm_archiver_follow_new_primary -- the transition *into* +# ARCHIVING_STATE, which does not run again once current_role and +# assigned_role already agree. +# + +step test_002_archiver_restart_liveness { + compose stop archiver1 + wait until archiver1 stopped timeout 60s + compose start archiver1 + wait until archiver1 state is archiving timeout 60s + sql node1 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } +} + +# +# test_003: fail node1 (primary) over to node2. Every ARCHIVING row in the +# group is expected to pass through REPORT_LSN_STATE during the +# election (fsm_archiver_report_lsn stops pg_receivewal against the +# now-untrustworthy old primary) and back to ARCHIVING_STATE once +# node2 is confirmed primary (fsm_archiver_follow_new_primary +# re-points pg_receivewal at it) -- the same election phases every +# other node kind goes through, applied to an archiver for the +# first time here. Segments captured before the failover must stay +# recorded: report_wal_received() never deletes archiver_wal rows. +# + +step test_003_failover_continuity { + compose stop node1 + wait until node1 stopped timeout 60s + wait until node2 state is primary + passing through wait_primary + timeout 120s + wait until archiver1 state is archiving + passing through report_lsn + timeout 120s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000001'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } +} From 3cc6b86dd9a41c602215a03323c2f2e02dce95c0 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 14:52:37 +0200 Subject: [PATCH 13/55] pg_autoctl: base backup generation, live source (M5) First half of Milestone 5 per the design doc's own Build order ("live first, then replay/volatile"). New file service_archiver_basebackup.c adds service_archiver_maybe_generate_basebackup(), called once per tick from service_archiver_loop() alongside the M4 WAL-report/liveness calls. Trigger scope for this pass: bootstrap only -- a group with zero existing base backups (monitor_get_latest_basebackup_location() reports not-found) gets one immediately. Scheduled/timeline-change/retention triggers need basebackup_policy wired through the CLI first, a later milestone. Target selection follows the design doc's `live` precedence, minus its warm-standby tier (nothing to select from yet, also later): the first healthy secondary in the group (monitor_get_nodes(), skipping port == 0 ARCHIVING rows), falling back to the primary when none exists. Generation itself is a one-shot forked child (basebackupPid, tracked the same way service_archiver.c tracks pgReceivewalPid) rather than a persistent service, so a potentially long-running pg_basebackup can't stall the main loop's own node_active()/WAL-report tick. The child execs the real, unmodified pg_basebackup client with --wal-method=none -- this backup is deliberately not self-consistent on its own, since the archiver's already-running WAL capture is what supplies the WAL needed to reach consistency on replay -- then reads the resulting backup_label for the authoritative start LSN/timeline and reports both start and completion to the monitor via two new wrappers, monitor_report_basebackup_started()/_completed() (monitor.c/.h), calling the SQL functions M1's schema already shipped but nothing had called yet. endlsn is best-effort: a live read of the source's current WAL (or last-replayed, if the source is a standby) position right after the backup finishes: not Postgres's own internal stop-backup LSN (not observable from a plain CLI wrapper around pg_basebackup), but a reasonable upper bound, and never fatal to the backup itself if that one query fails. Verified end-to-end against a real monitor + primary + archiver: the bootstrap backup fires automatically, archiver_wal / basebackup rows land correctly (source = 'live', status = 'complete', a real endlsn distinct from startlsn), and the resulting directory passes pg_verifybackup. Full SQL regression schedule (src/monitor, 20/20) still passes. --- src/bin/pg_autoctl/monitor.c | 93 +++ src/bin/pg_autoctl/monitor.h | 10 + src/bin/pg_autoctl/service_archiver.c | 6 + .../pg_autoctl/service_archiver_basebackup.c | 563 ++++++++++++++++++ .../pg_autoctl/service_archiver_basebackup.h | 19 + src/bin/pgaftest/Makefile | 3 +- 6 files changed, 693 insertions(+), 1 deletion(-) create mode 100644 src/bin/pg_autoctl/service_archiver_basebackup.c create mode 100644 src/bin/pg_autoctl/service_archiver_basebackup.h diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index a674b76e3..d97af9f66 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -1062,6 +1062,99 @@ monitor_report_wal_received(Monitor *monitor, int64_t nodeId, } +/* + * monitor_report_basebackup_started calls + * pgautofailover.report_basebackup_started() to record the start of a new + * base-backup production job and returns its basebackupid, needed by the + * matching monitor_report_basebackup_completed() call once the backup + * finishes. source is hardcoded to 'live' and replaymode to NULL -- + * Milestone 5's own first pass, "live" only (see + * service_archiver_basebackup.c's own header comment); 'replay' is a + * follow-up that will need both as real parameters. + */ +bool +monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, + const char *formationId, int groupId, + const char *label, int timeline, + const char *startLsn, + int64_t *basebackupId) +{ + PGSQL *pgsql = &monitor->pgsql; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + const char *sql = + "SELECT pgautofailover.report_basebackup_started(" + "$1, $2, $3, $4, $5, $6, 'live'::pgautofailover.basebackup_source)"; + int paramCount = 6; + Oid paramTypes[6] = { + INT8OID, TEXTOID, INT4OID, TEXTOID, INT4OID, LSNOID + }; + IntString archiverIdString = intToString(archiverId); + IntString groupIdString = intToString(groupId); + IntString timelineString = intToString(timeline); + const char *paramValues[6] = { + archiverIdString.strValue, formationId, groupIdString.strValue, + label, timelineString.strValue, startLsn + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to report the start of base backup \"%s\" to " + "the monitor", label); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to report the start of base backup \"%s\" to " + "the monitor because it returned an unexpected result, " + "see previous lines for details", label); + return false; + } + + *basebackupId = context.bigint; + + return true; +} + + +/* + * monitor_report_basebackup_completed calls + * pgautofailover.report_basebackup_completed() to record the successful + * completion of a base-backup production job previously created with + * monitor_report_basebackup_started(). + */ +bool +monitor_report_basebackup_completed(Monitor *monitor, int64_t basebackupId, + const char *endLsn, int64_t sizeBytes, + const char *storageLocation) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_basebackup_completed($1, $2, $3, $4)"; + int paramCount = 4; + Oid paramTypes[4] = { INT8OID, LSNOID, INT8OID, TEXTOID }; + IntString basebackupIdString = intToString(basebackupId); + IntString sizeBytesString = intToString(sizeBytes); + const char *paramValues[4] = { + basebackupIdString.strValue, endLsn, sizeBytesString.strValue, + storageLocation + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report base backup %" PRId64 " as completed " + "to the monitor", basebackupId); + return false; + } + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 2b935d3bf..3c992385d 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -164,6 +164,16 @@ bool monitor_get_latest_basebackup_location(Monitor *monitor, bool *found); bool monitor_report_wal_received(Monitor *monitor, int64_t nodeId, const char *walFileName, const char *lsn); +bool monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, + const char *formationId, int groupId, + const char *label, int timeline, + const char *startLsn, + int64_t *basebackupId); +bool monitor_report_basebackup_completed(Monitor *monitor, + int64_t basebackupId, + const char *endLsn, + int64_t sizeBytes, + const char *storageLocation); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index 1b3192f72..ab06d5c77 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -37,6 +37,7 @@ #include "fsm.h" #include "log.h" #include "monitor.h" +#include "service_archiver_basebackup.h" #include "signals.h" /* @@ -504,6 +505,11 @@ service_archiver_loop(Keeper *keeper) log_warn("Failed to report newly captured WAL segments to " "the monitor, will retry"); } + + if (!service_archiver_maybe_generate_basebackup(keeper)) + { + log_warn("Failed to generate a base backup, will retry"); + } } else { diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c new file mode 100644 index 000000000..0438273d4 --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -0,0 +1,563 @@ +/* + * src/bin/pg_autoctl/service_archiver_basebackup.c + * Archiving & Disaster Recovery: base backup generation, `live` source + * only (Milestone 5's own first half, per the Build order in + * ~/dev/temp/archiving-disaster-recovery.md: "live first, then + * replay/volatile"). + * + * Trigger scope for this pass: bootstrap only -- a group with zero + * existing base backups gets one immediately, sourced live. Scheduled/ + * timeline-change/retention-driven triggers all need basebackup_policy + * wired through the CLI first (a later milestone); get_archiver_policy() + * already resolves a default policy row today, but nothing here reads its + * frequency yet. + * + * Target selection follows the design doc's own `live` precedence, minus + * its warm-standby tier (a later milestone, nothing to select from yet): + * the first healthy secondary in the group, falling back to the primary + * when none exists. "Healthy" here just means "reachable via + * pgautofailover.get_nodes()", not "least-loaded" -- picking between + * several healthy secondaries by load is a refinement, not required for + * base-backup generation to work at all. + * + * Base backup generation itself is a one-shot forked child (tracked via + * basebackupPid, the same pattern service_archiver.c uses for + * pgReceivewalPid), not a persistent supervised service: it runs to + * completion and exits, so it must not block service_archiver_loop()'s own + * per-tick node_active()/WAL-report cycle for however long pg_basebackup + * takes. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "service_archiver_basebackup.h" + +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "pgsql.h" +#include "signals.h" + +/* + * One base backup generation child at a time, mirroring + * service_archiver.c's own pgReceivewalPid tracking pattern. + */ +static pid_t basebackupPid = -1; + +/* accumulator for directory_size()'s nftw() callback -- nftw() has no + * user-data parameter, so this has to be file-scope */ +static uint64_t directorySizeAccumulator = 0; + + +static bool +basebackup_child_is_running(void) +{ + if (basebackupPid <= 0) + { + return false; + } + + int status = 0; + pid_t ret = waitpid(basebackupPid, &status, WNOHANG); + + if (ret == 0) + { + /* still running */ + return true; + } + + if (ret == basebackupPid) + { + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + { + log_warn("Base backup generation process (pid %d) exited " + "with status %d", basebackupPid, status); + } + } + else + { + log_warn("Failed to check on base backup generation process " + "(pid %d): %m", basebackupPid); + } + + basebackupPid = -1; + return false; +} + + +/* + * select_basebackup_source picks the `live` target: the first healthy + * secondary in the group, falling back to the primary when none exists. + * Rows with port == 0 are ARCHIVING memberships (this node's own row among + * them, per the port == 0 sentinel documented in pgautofailover.sql) -- + * never a valid pg_basebackup source, so they are skipped outright. + */ +static bool +select_basebackup_source(Keeper *keeper, NodeAddress *source) +{ + NodeAddressArray nodeArray = { 0 }; + + if (!monitor_get_nodes(&(keeper->monitor), + keeper->config.formation, + keeper->config.groupId, + &nodeArray)) + { + /* errors already logged */ + return false; + } + + NodeAddress *primary = NULL; + + for (int i = 0; i < nodeArray.count; i++) + { + NodeAddress *node = &(nodeArray.nodes[i]); + + if (node->port == 0) + { + continue; + } + + if (node->isPrimary) + { + primary = node; + continue; + } + + *source = *node; + return true; + } + + if (primary != NULL) + { + *source = *primary; + return true; + } + + return false; +} + + +/* + * read_basebackup_label extracts "START WAL LOCATION" and "START TIMELINE" + * from a just-completed pg_basebackup's own backup_label file -- the + * authoritative start position, matching pg_walsender/cmd_base_backup.c's + * own read_backup_label() (duplicated rather than shared: pg_autoctl + * doesn't link that standalone binary's code, see this project's Makefile + * split). + */ +static bool +read_basebackup_label(const char *backupDir, char *lsnOut, size_t lsnOutSize, + int *timelineOut) +{ + char path[MAXPGPATH]; + + sformat(path, sizeof(path), "%s/backup_label", backupDir); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + bool foundLsn = false; + bool foundTimeline = false; + char *line = contents; + + while (line != NULL && *line != '\0') + { + char *nl = strchr(line, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + const char *lsnPrefix = "START WAL LOCATION: "; + const char *tliPrefix = "START TIMELINE: "; + + if (strncmp(line, lsnPrefix, strlen(lsnPrefix)) == 0) + { + const char *value = line + strlen(lsnPrefix); + const char *end = value; + + while (*end && !isspace((unsigned char) *end)) + { + end++; + } + + size_t len = Min((size_t) (end - value), lsnOutSize - 1); + + memcpy(lsnOut, value, len); + lsnOut[len] = '\0'; + foundLsn = true; + } + else if (strncmp(line, tliPrefix, strlen(tliPrefix)) == 0) + { + *timelineOut = atoi(line + strlen(tliPrefix)); + foundTimeline = true; + } + + line = (nl != NULL) ? nl + 1 : NULL; + } + + free(contents); + + return foundLsn && foundTimeline; +} + + +/* + * query_wal_position runs a single ad hoc query against sourceConnInfo, + * used right after a base backup finishes to capture the source's current + * WAL write position (primary) or replay position (standby) -- recorded as + * the backup's endlsn. Not the exact internal stop-backup LSN real + * pg_basebackup computes server-side (not observable from a plain CLI + * wrapper around it), but a reasonable upper bound: "WAL up to at least + * this point must be replayed to reach consistency." + */ +static bool +query_wal_position(const char *sourceConnInfo, bool isPrimary, + char *lsn, size_t lsnSize) +{ + PGSQL client = { 0 }; + + if (!pgsql_init(&client, (char *) sourceConnInfo, PGSQL_CONN_UPSTREAM)) + { + return false; + } + + const char *sql = isPrimary + ? "SELECT pg_current_wal_lsn()::text" + : "SELECT pg_last_wal_replay_lsn()::text"; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_STRING, false }; + + bool result = pgsql_execute_with_params(&client, sql, 0, NULL, NULL, + &context, &parseSingleValueResult); + + PQfinish(client.connection); + + if (!result || !context.parsedOk || context.strVal == NULL) + { + return false; + } + + strlcpy(lsn, context.strVal, lsnSize); + free(context.strVal); + + return true; +} + + +static int +accumulate_file_size(const char *path, const struct stat *sb, + int typeflag, struct FTW *ftwbuf) +{ + if (typeflag == FTW_F) + { + directorySizeAccumulator += (uint64_t) sb->st_size; + } + + return 0; +} + + +/* + * directory_size adds up the apparent size of every regular file under + * dirPath. Best effort: sizebytes is informational only (nothing in the + * monitor schema's own logic -- prune_archiver_wal() included -- reads it + * back), so a failure here is not worth failing an otherwise-successful + * base backup over. + */ +static uint64_t +directory_size(const char *dirPath) +{ + directorySizeAccumulator = 0; + + (void) nftw(dirPath, accumulate_file_size, 16, FTW_PHYS); + + return directorySizeAccumulator; +} + + +/* + * run_pg_basebackup execs the real, unmodified pg_basebackup client + * against source, writing into backupDir. --wal-method=none: this backup + * is deliberately not self-consistent on its own -- the archiver's already + * -running WAL capture (service_archiver.c) is what supplies the WAL + * needed to reach consistency on replay, so bundling a second independent + * copy of it into every single backup would be pure waste. + */ +static bool +run_pg_basebackup(KeeperConfig *config, NodeAddress *source, + const char *backupDir, const char *label) +{ + char pgBasebackupPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, "pg_basebackup", + pgBasebackupPath); + + if (!file_exists(pgBasebackupPath)) + { + log_error("Failed to find pg_basebackup at \"%s\"", pgBasebackupPath); + return false; + } + + log_info("Generating a live base backup from %s:%d into \"%s\"", + source->host, source->port, backupDir); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork pg_basebackup: %m"); + return false; + } + + if (pid == 0) + { + char portStr[NAMEDATALEN]; + + sformat(portStr, sizeof(portStr), "%d", source->port); + + char *args[16]; + int argsIndex = 0; + + args[argsIndex++] = pgBasebackupPath; + args[argsIndex++] = "-h"; + args[argsIndex++] = source->host; + args[argsIndex++] = "-p"; + args[argsIndex++] = portStr; + args[argsIndex++] = "-U"; + args[argsIndex++] = PG_AUTOCTL_REPLICA_USERNAME; + args[argsIndex++] = "-D"; + args[argsIndex++] = (char *) backupDir; + args[argsIndex++] = "--format=plain"; + args[argsIndex++] = "--wal-method=none"; + args[argsIndex++] = "--checkpoint=fast"; + args[argsIndex++] = "--label"; + args[argsIndex++] = (char *) label; + args[argsIndex++] = "--no-password"; + args[argsIndex] = NULL; + + execv(pgBasebackupPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", pgBasebackupPath); + _exit(127); + } + + int status = 0; + + if (waitpid(pid, &status, 0) == -1) + { + log_error("Failed to wait for pg_basebackup (pid %d): %m", pid); + return false; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + { + log_error("pg_basebackup failed while generating base backup \"%s\"", + label); + return false; + } + + return true; +} + + +/* + * generate_basebackup is the forked child's own body: run pg_basebackup to + * completion, then report start and completion to the monitor from the + * authoritative backup_label it wrote. Runs in its own process, with its + * own monitor connection (the parent's keeper->monitor is not fork-safe to + * share, exactly as service_archiver_run.c's own supervised children + * already document). + */ +static bool +generate_basebackup(Keeper *keeper, NodeAddress *source, + const char *backupDir, const char *label) +{ + KeeperConfig *config = &(keeper->config); + + if (!run_pg_basebackup(config, source, backupDir, label)) + { + return false; + } + + char startLsn[PG_LSN_MAXLENGTH] = { 0 }; + int timeline = 1; + + if (!read_basebackup_label(backupDir, startLsn, sizeof(startLsn), + &timeline)) + { + log_error("Failed to read backup_label from \"%s\" after " + "pg_basebackup completed", backupDir); + return false; + } + + if (!monitor_init(&(keeper->monitor), config->monitor_pguri)) + { + log_error("Failed to contact the monitor to report base backup " + "\"%s\"", label); + return false; + } + + int64_t basebackupId = 0; + + if (!monitor_report_basebackup_started(&(keeper->monitor), + config->archiverId, + config->formation, + config->groupId, + label, timeline, startLsn, + &basebackupId)) + { + /* errors already logged */ + return false; + } + + /* + * dbname is otherwise unknown here -- an ARCHIVING node has no real + * PostgresSetup of its own to read one from (haspgdata's own design + * comment). DEFAULT_DATABASE_NAME ("postgres") is what every ordinary + * node defaults its own --dbname to (cli_create_node.c), and is always + * present regardless of that default, so it is a safe target for a + * plain read-only SQL query. + */ + char sourceConnInfo[MAXCONNINFO] = { 0 }; + + sformat(sourceConnInfo, sizeof(sourceConnInfo), + "host=%s port=%d user=%s dbname=%s application_name=%s", + source->host, source->port, + PG_AUTOCTL_REPLICA_USERNAME, DEFAULT_DATABASE_NAME, config->name); + + char endLsn[PG_LSN_MAXLENGTH] = { 0 }; + + if (!query_wal_position(sourceConnInfo, source->isPrimary, + endLsn, sizeof(endLsn))) + { + /* not fatal: the backup itself succeeded, only this one piece of + * informational metadata is missing -- fall back to the start + * position rather than failing an otherwise-successful backup */ + strlcpy(endLsn, startLsn, sizeof(endLsn)); + } + + uint64_t sizeBytes = directory_size(backupDir); + + return monitor_report_basebackup_completed(&(keeper->monitor), + basebackupId, endLsn, + (int64_t) sizeBytes, + backupDir); +} + + +/* + * service_archiver_maybe_generate_basebackup checks, once per + * service_archiver_loop() tick, whether this group has zero existing base + * backups yet and -- if so, and no generation is already in flight -- forks + * a child to produce one. See this file's own header comment for the full + * scope of this first pass (bootstrap trigger, live source only). + */ +bool +service_archiver_maybe_generate_basebackup(Keeper *keeper) +{ + if (basebackup_child_is_running()) + { + return true; + } + + KeeperConfig *config = &(keeper->config); + bool found = false; + char storageLocation[MAXPGPATH] = { 0 }; + + if (!monitor_get_latest_basebackup_location(&(keeper->monitor), + config->formation, + config->groupId, + storageLocation, + sizeof(storageLocation), + &found)) + { + /* errors already logged */ + return false; + } + + if (found) + { + /* bootstrap already satisfied; scheduled/retention-driven triggers + * are a follow-up milestone, see this file's own header comment */ + return true; + } + + NodeAddress source = { 0 }; + + if (!select_basebackup_source(keeper, &source)) + { + log_warn("No eligible node to source a live base backup from yet, " + "will retry"); + return true; + } + + char backupsDir[MAXPGPATH] = { 0 }; + + sformat(backupsDir, sizeof(backupsDir), "%s/basebackups", + config->pgSetup.pgdata); + + if (!directory_exists(backupsDir) && mkdir(backupsDir, 0700) != 0) + { + log_error("Failed to create \"%s\": %m", backupsDir); + return false; + } + + time_t now = time(NULL); + struct tm nowUTC = { 0 }; + + gmtime_r(&now, &nowUTC); + + char label[NAMEDATALEN] = { 0 }; + + strftime(label, sizeof(label), "basebackup-%Y%m%dT%H%M%SZ", &nowUTC); + + char backupDir[MAXPGPATH] = { 0 }; + + sformat(backupDir, sizeof(backupDir), "%s/%s", backupsDir, label); + + fflush(stdout); + fflush(stderr); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork the base backup generation process: %m"); + return false; + } + + if (pid == 0) + { + (void) set_signal_handlers(false); + (void) set_ps_title("pg_autoctl: archiver basebackup"); + + bool ok = generate_basebackup(keeper, &source, backupDir, label); + + exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); + } + + log_debug("pg_autoctl archiver basebackup process started in " + "subprocess %d", pid); + basebackupPid = pid; + + return true; +} diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.h b/src/bin/pg_autoctl/service_archiver_basebackup.h new file mode 100644 index 000000000..2313a8f1e --- /dev/null +++ b/src/bin/pg_autoctl/service_archiver_basebackup.h @@ -0,0 +1,19 @@ +/* + * src/bin/pg_autoctl/service_archiver_basebackup.h + * Archiving & Disaster Recovery: base backup generation, `live` source + * only (Milestone 5's own first half). See service_archiver_basebackup.c + * for the full scope note. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef SERVICE_ARCHIVER_BASEBACKUP_H +#define SERVICE_ARCHIVER_BASEBACKUP_H + +#include "keeper.h" + +bool service_archiver_maybe_generate_basebackup(Keeper *keeper); + +#endif /* SERVICE_ARCHIVER_BASEBACKUP_H */ diff --git a/src/bin/pgaftest/Makefile b/src/bin/pgaftest/Makefile index fbf6c0ed0..19a2af41a 100644 --- a/src/bin/pgaftest/Makefile +++ b/src/bin/pgaftest/Makefile @@ -32,7 +32,8 @@ SHARED_SRCS = cli_common.c config.c coordinator.c fsm.c fsm_transition.c \ fsm_transition_citus.c keeper.c keeper_config.c keeper_pg_init.c \ monitor.c monitor_config.c monitor_pg_init.c \ nodespec.c nodestate_utils.c pghba.c primary_standby.c \ - service_archiver.c service_keeper.c service_keeper_init.c service_monitor.c \ + service_archiver.c service_archiver_basebackup.c \ + service_keeper.c service_keeper_init.c service_monitor.c \ service_monitor_init.c service_postgres.c service_postgres_ctl.c \ state.c step_socket.c supervisor.c systemd_config.c timeline_history.c From 71f4b78269f2a22fc4ef957200a93e5ca53f92ca Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 15:45:27 +0200 Subject: [PATCH 14/55] pg_autoctl: base backup generation, replay/volatile source (M5) Second half of Milestone 5 ("live first, then replay/volatile" per the design doc's Build order). service_archiver_maybe_generate_basebackup() now takes a bootstrap `live` backup as before, then -- on the very next tick -- exercises the `replay`/`volatile` pipeline exactly once: extract the last retained backup into a throwaway staging directory, point its recovery at this archiver's own already-captured WAL (restore_command + recovery.signal, entirely local, no network round trip), let it replay forward and promote once it runs out of locally-captured segments, then pg_basebackup it over loopback and discard the staging instance -- 'volatile' means nothing survives between cycles. Real frequency-driven scheduling (basebackup_policy's own frequency/ onpromotion/retention, resolved through get_archiver_policy()/ get_basebackup_policy()) is a deliberate follow-up, not built here: the milestone-defining new capability is the replay mechanism itself, not a general scheduler (matching the design doc's own build order, which lists warm standby's scheduling machinery as a later milestone). monitor_report_basebackup_started() (added in the `live`-only commit) now takes real source/replaymode parameters instead of a hardcoded 'live', and monitor_get_latest_basebackup_location() is renamed to monitor_get_latest_basebackup_info() and returns the latest backup's source alongside its storage location -- what the trigger above uses to tell "only the bootstrap has run" from "the replay exercise is already done". Getting a working staging instance up took two real, load-bearing fixes along the way: - pg_ctl start, invoked here through both a hand-rolled fork()/execl() and this project's own run_program() helper, reproducibly misparsed its own arguments in this exact process tree (deep in a supervised archiver's own fork chain) even though byte-identical argv worked fine in every standalone reproduction attempted. Root cause not fully isolated; worked around by execing the real "postgres" binary directly instead of going through pg_ctl at all -- the same fork()/execv() pattern already used for pg_receivewal and pg_basebackup in this codebase, with readiness confirmed by polling a real SQL connection rather than relying on pg_ctl's own "-w". - recovery_target_lsn set to "the end of the latest complete segment" is not actually a reachable record boundary on a mostly-idle source (a renamed, "complete" segment file is always its full fixed size regardless of how much of it is real WAL) -- recovery correctly refused to pause there ("recovery ended before configured recovery target was reached"). Replaying to "everything locally available" and letting Postgres promote on its own sidesteps needing a precise target at all, which a `volatile`, discard-after-use snapshot never actually needed in the first place. Verified end-to-end against a real monitor + primary + archiver, from a cold start through both the live bootstrap and the replay follow-up: basebackup rows land correctly (source/replaymode/status all correct, real distinct startlsn/endlsn across the sequence), and both resulting directories pass pg_verifybackup. Full SQL regression schedule (src/monitor) passed 20/20 twice earlier against this same unchanged schema in this session; a later re-run hit an apparent local pg_regress/DROP DATABASE environment hang (ProcSignalBarrier) unrelated to any change in this commit -- no .sql files are touched here. --- src/bin/pg_autoctl/defaults.h | 5 + src/bin/pg_autoctl/monitor.c | 101 +++- src/bin/pg_autoctl/monitor.h | 11 +- .../pg_autoctl/service_archiver_basebackup.c | 559 +++++++++++++++--- src/bin/pg_autoctl/service_archiver_serve.c | 17 +- 5 files changed, 583 insertions(+), 110 deletions(-) diff --git a/src/bin/pg_autoctl/defaults.h b/src/bin/pg_autoctl/defaults.h index 6797df257..a473661f6 100644 --- a/src/bin/pg_autoctl/defaults.h +++ b/src/bin/pg_autoctl/defaults.h @@ -230,6 +230,11 @@ * serve` -- matches src/bin/pg_walsender/defaults.h's own WS_DEFAULT_PORT */ #define PG_AUTOCTL_ARCHIVER_SERVE_PORT 6543 +/* port the archiver's own throwaway replay-mode staging Postgres instance + * listens on, loopback only -- see service_archiver_basebackup.c's own + * replay/volatile implementation */ +#define PG_AUTOCTL_ARCHIVER_REPLAY_PORT 6899 + #define PG_AUTOCTL_MONITOR_DBNAME "pg_auto_failover" #define PG_AUTOCTL_MONITOR_EXTENSION_NAME "pgautofailover" #define PG_AUTOCTL_MONITOR_DBOWNER "autoctl" diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index d97af9f66..791c51c7e 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -963,18 +963,63 @@ monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, /* - * monitor_get_latest_basebackup_location calls + * BasebackupInfoParseContext/parseBasebackupInfo parse the two columns + * monitor_get_latest_basebackup_info() needs out of a single-row result -- + * SingleValueResultContext only carries one column, not enough here. + */ +typedef struct BasebackupInfoParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + bool parsedOk; + int ntuples; + char *storageLocation; + char *source; +} BasebackupInfoParseContext; + + +static void +parseBasebackupInfo(void *ctx, PGresult *result) +{ + BasebackupInfoParseContext *context = (BasebackupInfoParseContext *) ctx; + + context->ntuples = PQntuples(result); + + if (context->ntuples != 1) + { + /* zero rows is a valid "no backup yet" signal, not a parse error */ + context->parsedOk = (context->ntuples == 0); + return; + } + + char *storageLocation = PQgetvalue(result, 0, 0); + char *source = PQgetvalue(result, 0, 1); + + context->storageLocation = strdup(storageLocation); + context->source = strdup(source); + + context->parsedOk = + context->storageLocation != NULL && context->source != NULL; + + if (!context->parsedOk) + { + log_error(ALLOCATION_FAILED_ERROR); + } +} + + +/* + * monitor_get_latest_basebackup_info calls * pgautofailover.get_latest_basebackup(formationId, groupId) and returns - * its storagelocation column. *found is set to false (not an error) when - * the archiver hasn't taken a base backup for this group yet -- the "Base - * backup generation" milestone this depends on hasn't landed, so every - * caller of this function must already tolerate that. + * its storagelocation and source columns. *found is set to false (not an + * error) when the archiver hasn't taken a base backup for this group yet -- + * every caller must already tolerate that. */ bool -monitor_get_latest_basebackup_location(Monitor *monitor, - const char *formationId, int groupId, - char *storageLocation, size_t size, - bool *found) +monitor_get_latest_basebackup_info(Monitor *monitor, + const char *formationId, int groupId, + char *storageLocation, size_t storageLocationSize, + char *source, size_t sourceSize, + bool *found) { PGSQL *pgsql = &monitor->pgsql; const char *sql = @@ -987,22 +1032,22 @@ monitor_get_latest_basebackup_location(Monitor *monitor, * composite downstream, is what makes context.ntuples == 0 below * an accurate "no backup yet" signal. */ - "SELECT storagelocation " + "SELECT storagelocation, source::text " " FROM pgautofailover.get_latest_basebackup($1, $2) " " WHERE storagelocation IS NOT NULL"; int paramCount = 2; Oid paramTypes[2] = { TEXTOID, INT4OID }; IntString groupIdString = intToString(groupId); const char *paramValues[2] = { formationId, groupIdString.strValue }; - SingleValueResultContext context = { { 0 }, PGSQL_RESULT_STRING, false }; + BasebackupInfoParseContext context = { { 0 }, false, 0, NULL, NULL }; *found = false; if (!pgsql_execute_with_params(pgsql, sql, paramCount, paramTypes, paramValues, - &context, &parseSingleValueResult)) + &context, &parseBasebackupInfo)) { - log_error("Failed to get the latest base backup location from the " + log_error("Failed to get the latest base backup info from the " "monitor for \"%s\"/%d", formationId, groupId); return false; } @@ -1015,14 +1060,16 @@ monitor_get_latest_basebackup_location(Monitor *monitor, if (!context.parsedOk) { - log_error("Failed to parse the latest base backup location returned " + log_error("Failed to parse the latest base backup info returned " "by the monitor for \"%s\"/%d, see above for details", formationId, groupId); return false; } - strlcpy(storageLocation, context.strVal, size); - free(context.strVal); + strlcpy(storageLocation, context.storageLocation, storageLocationSize); + strlcpy(source, context.source, sourceSize); + free(context.storageLocation); + free(context.source); *found = true; return true; @@ -1067,33 +1114,35 @@ monitor_report_wal_received(Monitor *monitor, int64_t nodeId, * pgautofailover.report_basebackup_started() to record the start of a new * base-backup production job and returns its basebackupid, needed by the * matching monitor_report_basebackup_completed() call once the backup - * finishes. source is hardcoded to 'live' and replaymode to NULL -- - * Milestone 5's own first pass, "live" only (see - * service_archiver_basebackup.c's own header comment); 'replay' is a - * follow-up that will need both as real parameters. + * finishes. source is one of "live"/"replay" (basebackup_source's own + * labels); replaymode is required when source is "replay" ("volatile"/ + * "persistent"), NULL otherwise -- pass NULL for a "live" backup. */ bool monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, const char *formationId, int groupId, const char *label, int timeline, const char *startLsn, + const char *source, const char *replaymode, int64_t *basebackupId) { PGSQL *pgsql = &monitor->pgsql; SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; const char *sql = "SELECT pgautofailover.report_basebackup_started(" - "$1, $2, $3, $4, $5, $6, 'live'::pgautofailover.basebackup_source)"; - int paramCount = 6; - Oid paramTypes[6] = { - INT8OID, TEXTOID, INT4OID, TEXTOID, INT4OID, LSNOID + "$1, $2, $3, $4, $5, $6, " + "$7::pgautofailover.basebackup_source, " + "$8::pgautofailover.basebackup_replay_mode)"; + int paramCount = 8; + Oid paramTypes[8] = { + INT8OID, TEXTOID, INT4OID, TEXTOID, INT4OID, LSNOID, TEXTOID, TEXTOID }; IntString archiverIdString = intToString(archiverId); IntString groupIdString = intToString(groupId); IntString timelineString = intToString(timeline); - const char *paramValues[6] = { + const char *paramValues[8] = { archiverIdString.strValue, formationId, groupIdString.strValue, - label, timelineString.strValue, startLsn + label, timelineString.strValue, startLsn, source, replaymode }; if (!pgsql_execute_with_params(pgsql, sql, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 3c992385d..00b595411 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -158,16 +158,19 @@ bool monitor_register_archiver(Monitor *monitor, char *name, char *hostname, int64_t *archiverId); bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, char *formation, int64_t *archiverNodeId); -bool monitor_get_latest_basebackup_location(Monitor *monitor, - const char *formationId, int groupId, - char *storageLocation, size_t size, - bool *found); +bool monitor_get_latest_basebackup_info(Monitor *monitor, + const char *formationId, int groupId, + char *storageLocation, size_t storageLocationSize, + char *source, size_t sourceSize, + bool *found); bool monitor_report_wal_received(Monitor *monitor, int64_t nodeId, const char *walFileName, const char *lsn); bool monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, const char *formationId, int groupId, const char *label, int timeline, const char *startLsn, + const char *source, + const char *replaymode, int64_t *basebackupId); bool monitor_report_basebackup_completed(Monitor *monitor, int64_t basebackupId, diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c index 0438273d4..aee864027 100644 --- a/src/bin/pg_autoctl/service_archiver_basebackup.c +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -1,31 +1,51 @@ /* * src/bin/pg_autoctl/service_archiver_basebackup.c - * Archiving & Disaster Recovery: base backup generation, `live` source - * only (Milestone 5's own first half, per the Build order in + * Archiving & Disaster Recovery: base backup generation, both `live` and + * `replay`/`volatile` sources (Milestone 5, per the Build order in * ~/dev/temp/archiving-disaster-recovery.md: "live first, then - * replay/volatile"). + * replay/volatile"). `replay`/`persistent` is a later milestone -- + * that mode keeps its staging instance resident as a `warm-standby` + * `archiver_node` row, which doesn't exist until Milestone 6. * - * Trigger scope for this pass: bootstrap only -- a group with zero - * existing base backups gets one immediately, sourced live. Scheduled/ - * timeline-change/retention-driven triggers all need basebackup_policy - * wired through the CLI first (a later milestone); get_archiver_policy() - * already resolves a default policy row today, but nothing here reads its - * frequency yet. + * Trigger scope for this pass: bootstrap, then exactly one replay-sourced + * backup to exercise that pipeline once -- both hardcoded here, not read + * from basebackup_policy. A group with zero existing base backups gets one + * immediately, sourced live (matching the design doc's own bootstrap rule: + * nothing to replay from yet on the first run). Once that lands, the very + * next tick takes exactly one more, this time sourced replay/volatile, and + * after that this file goes quiet for the group. Real frequency-driven + * scheduling (basebackup_policy's own `source`/`replaymode`/`frequency`/ + * `onpromotion`/retention fields, resolved through `get_archiver_policy()`/ + * `get_basebackup_policy()`) needs that policy wired through the CLI + * first -- out of scope here. This is a deliberate scope cut, not an + * oversight: the milestone-defining new capability is the replay mechanism + * itself (extract, replay, promote, snapshot, discard), not a general + * scheduler -- see the design doc's own build order, which lists "warm + * standby" and its `advance`/scheduling machinery as later milestones. * - * Target selection follows the design doc's own `live` precedence, minus - * its warm-standby tier (a later milestone, nothing to select from yet): - * the first healthy secondary in the group, falling back to the primary - * when none exists. "Healthy" here just means "reachable via + * Target selection ('live') follows the design doc's own precedence, + * minus its warm-standby tier (a later milestone, nothing to select from + * yet): the first healthy secondary in the group, falling back to the + * primary when none exists. "Healthy" here just means "reachable via * pgautofailover.get_nodes()", not "least-loaded" -- picking between * several healthy secondaries by load is a refinement, not required for * base-backup generation to work at all. * + * Target selection ('replay') is entirely local: a throwaway staging + * Postgres instance, extracted from the last retained base backup and + * replayed forward using this archiver's own already-captured WAL (no + * network round trip to any live node at all) until it promotes on its + * own (see write_replay_recovery_config()'s own comment on why this + * targets "everything locally available" rather than a specific LSN), + * then sourced via pg_basebackup over loopback and discarded ('volatile': + * no persistent archiver_node row, nothing left running or on disk between + * cycles). + * * Base backup generation itself is a one-shot forked child (tracked via * basebackupPid, the same pattern service_archiver.c uses for * pgReceivewalPid), not a persistent supervised service: it runs to * completion and exits, so it must not block service_archiver_loop()'s own - * per-tick node_active()/WAL-report cycle for however long pg_basebackup - * takes. + * per-tick node_active()/WAL-report cycle for however long it takes. * * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the PostgreSQL License. @@ -41,6 +61,8 @@ #include #include +#include "postgres_fe.h" + #include "service_archiver_basebackup.h" #include "defaults.h" @@ -48,6 +70,7 @@ #include "log.h" #include "monitor.h" #include "pgsql.h" +#include "runprogram.h" #include "signals.h" /* @@ -60,6 +83,10 @@ static pid_t basebackupPid = -1; * user-data parameter, so this has to be file-scope */ static uint64_t directorySizeAccumulator = 0; +/* how long to wait for the replay staging instance to finish replaying + * available WAL and promote before giving up on this cycle */ +#define ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS 60 + static bool basebackup_child_is_running(void) @@ -221,21 +248,21 @@ read_basebackup_label(const char *backupDir, char *lsnOut, size_t lsnOutSize, /* - * query_wal_position runs a single ad hoc query against sourceConnInfo, - * used right after a base backup finishes to capture the source's current - * WAL write position (primary) or replay position (standby) -- recorded as - * the backup's endlsn. Not the exact internal stop-backup LSN real - * pg_basebackup computes server-side (not observable from a plain CLI - * wrapper around it), but a reasonable upper bound: "WAL up to at least - * this point must be replayed to reach consistency." + * query_wal_position runs a single ad hoc query against connInfo, used + * right after a base backup finishes to capture the source's current WAL + * write position (primary) or replay position (standby/staging instance) + * -- recorded as the backup's endlsn. Not the exact internal stop-backup + * LSN real pg_basebackup computes server-side (not observable from a plain + * CLI wrapper around it), but a reasonable upper bound: "WAL up to at + * least this point must be replayed to reach consistency." */ static bool -query_wal_position(const char *sourceConnInfo, bool isPrimary, +query_wal_position(const char *connInfo, bool isPrimary, char *lsn, size_t lsnSize) { PGSQL client = { 0 }; - if (!pgsql_init(&client, (char *) sourceConnInfo, PGSQL_CONN_UPSTREAM)) + if (!pgsql_init(&client, (char *) connInfo, PGSQL_CONN_UPSTREAM)) { return false; } @@ -296,10 +323,11 @@ directory_size(const char *dirPath) /* * run_pg_basebackup execs the real, unmodified pg_basebackup client * against source, writing into backupDir. --wal-method=none: this backup - * is deliberately not self-consistent on its own -- the archiver's already - * -running WAL capture (service_archiver.c) is what supplies the WAL - * needed to reach consistency on replay, so bundling a second independent - * copy of it into every single backup would be pure waste. + * is deliberately not self-consistent on its own -- for a `live` backup, + * the archiver's already-running WAL capture (service_archiver.c) is what + * supplies the WAL needed to reach consistency on replay; for a `replay` + * backup, the source is itself already paused at a known-consistent LSN, + * so there is nothing further to bundle either way. */ static bool run_pg_basebackup(KeeperConfig *config, NodeAddress *source, @@ -316,8 +344,8 @@ run_pg_basebackup(KeeperConfig *config, NodeAddress *source, return false; } - log_info("Generating a live base backup from %s:%d into \"%s\"", - source->host, source->port, backupDir); + log_info("Generating base backup \"%s\" from %s:%d into \"%s\"", + label, source->host, source->port, backupDir); pid_t pid = fork(); @@ -380,24 +408,18 @@ run_pg_basebackup(KeeperConfig *config, NodeAddress *source, /* - * generate_basebackup is the forked child's own body: run pg_basebackup to - * completion, then report start and completion to the monitor from the - * authoritative backup_label it wrote. Runs in its own process, with its - * own monitor connection (the parent's keeper->monitor is not fork-safe to - * share, exactly as service_archiver_run.c's own supervised children - * already document). + * report_basebackup reads backupDir's own backup_label for the + * authoritative start position, then reports both the start and + * completion of this base backup to the monitor. Shared by the live and + * replay paths; source/replaymode is the one thing that differs. */ static bool -generate_basebackup(Keeper *keeper, NodeAddress *source, - const char *backupDir, const char *label) +report_basebackup(Keeper *keeper, NodeAddress *endLsnSource, + const char *backupDir, const char *label, + const char *source, const char *replaymode) { KeeperConfig *config = &(keeper->config); - if (!run_pg_basebackup(config, source, backupDir, label)) - { - return false; - } - char startLsn[PG_LSN_MAXLENGTH] = { 0 }; int timeline = 1; @@ -423,6 +445,7 @@ generate_basebackup(Keeper *keeper, NodeAddress *source, config->formation, config->groupId, label, timeline, startLsn, + source, replaymode, &basebackupId)) { /* errors already logged */ @@ -435,18 +458,19 @@ generate_basebackup(Keeper *keeper, NodeAddress *source, * comment). DEFAULT_DATABASE_NAME ("postgres") is what every ordinary * node defaults its own --dbname to (cli_create_node.c), and is always * present regardless of that default, so it is a safe target for a - * plain read-only SQL query. + * plain read-only SQL query -- true of the replay staging instance too, + * copied verbatim from a `live` backup of an ordinary node. */ - char sourceConnInfo[MAXCONNINFO] = { 0 }; + char connInfo[MAXCONNINFO] = { 0 }; - sformat(sourceConnInfo, sizeof(sourceConnInfo), + sformat(connInfo, sizeof(connInfo), "host=%s port=%d user=%s dbname=%s application_name=%s", - source->host, source->port, + endLsnSource->host, endLsnSource->port, PG_AUTOCTL_REPLICA_USERNAME, DEFAULT_DATABASE_NAME, config->name); char endLsn[PG_LSN_MAXLENGTH] = { 0 }; - if (!query_wal_position(sourceConnInfo, source->isPrimary, + if (!query_wal_position(connInfo, endLsnSource->isPrimary, endLsn, sizeof(endLsn))) { /* not fatal: the backup itself succeeded, only this one piece of @@ -464,12 +488,374 @@ generate_basebackup(Keeper *keeper, NodeAddress *source, } +/* + * generate_live_basebackup is the forked child's own body for a `live` + * backup: run pg_basebackup against source to completion, then report it. + * Runs in its own process, with its own monitor connection (the parent's + * keeper->monitor is not fork-safe to share, exactly as + * service_archiver_run.c's own supervised children already document). + */ +static bool +generate_live_basebackup(Keeper *keeper, NodeAddress *source, + const char *backupDir, const char *label) +{ + if (!run_pg_basebackup(&(keeper->config), source, backupDir, label)) + { + return false; + } + + return report_basebackup(keeper, source, backupDir, label, "live", NULL); +} + + +/* + * copy_directory_tree shells out to `cp -R -p` (POSIX-portable across this + * project's actual dev/CI targets, unlike GNU cp's `-a`) to seed the + * replay staging directory from the last retained base backup. No + * existing recursive-copy helper exists in this codebase to reuse, and + * reimplementing one (special files, symlinks, permissions) is a much + * larger and riskier undertaking than reusing a battle-tested system + * utility -- the same reasoning this project already applies to + * pg_basebackup/pg_receivewal/pg_ctl themselves. Uses run_program() + * (runprogram.h), this project's own subprocess helper, rather than a + * hand-rolled fork()/exec(): matches every other external-program call in + * this codebase, and captures stderr for the error message below. + */ +static bool +copy_directory_tree(const char *sourceDir, const char *destDir) +{ + char cpPath[MAXPGPATH] = { 0 }; + + if (!search_path_first("cp", cpPath, LOG_ERROR)) + { + log_error("Failed to find \"cp\" in PATH"); + return false; + } + + Program program = run_program(cpPath, "-R", "-p", sourceDir, destDir, NULL); + bool success = program.returnCode == 0; + + if (!success) + { + log_error("cp -R -p \"%s\" \"%s\" failed: %s", + sourceDir, destDir, + program.stdErr != NULL ? program.stdErr : ""); + } + + free_program(&program); + + return success; +} + + +/* + * write_replay_recovery_config points the staging instance's recovery at + * this archiver's own local WAL cache (the colocated fast path -- no + * network round trip needed, matching service_archiver.c's own philosophy). + * No recovery_target_lsn: an idle-ish source produces mostly-zero-padded + * segments (a "complete", renamed segment file is always its full fixed + * size regardless of how much of it is real WAL), so "the end of the + * latest complete segment" is not actually a reachable record boundary -- + * recovery correctly refuses to pause at a target that doesn't correspond + * to any real record, and errors out instead ("recovery ended before + * configured recovery target was reached"). Instead, this replays every + * available locally-captured record and lets Postgres promote once + * restore_command runs out of segments to fetch -- for a snapshot that + * gets pg_basebackup'd and discarded immediately after (this is + * `volatile`: nothing persists between cycles), a promoted instance is + * exactly as usable a source as a paused one; only a `persistent` replica + * kept resident between cycles (a later milestone) would need the more + * precise pause-at-target-LSN behavior the design doc describes for + * `pg_autoctl warm-standby advance`. + * + * recovery.signal, not standby.signal: this is a one-shot archive recovery + * of already-captured WAL, not open-ended standby streaming. + */ +static bool +write_replay_recovery_config(const char *stagingDir, const char *walcacheDir) +{ + /* + * recovery.signal is what actually puts Postgres into archive recovery + * at startup (PG12+): without it, a data directory that still has + * backup_label is treated as an ordinary crash-recovery restart, which + * fails outright since the copied backup's pg_wal has no local WAL to + * replay from ("could not locate required checkpoint record") -- + * restore_command is only ever consulted once recovery.signal (or + * standby.signal) says this is a recovery in the first place. + */ + char signalPath[MAXPGPATH] = { 0 }; + + sformat(signalPath, sizeof(signalPath), "%s/recovery.signal", stagingDir); + + if (!write_file("", 0, signalPath)) + { + return false; + } + + char confPath[MAXPGPATH] = { 0 }; + + sformat(confPath, sizeof(confPath), "%s/postgresql.auto.conf", stagingDir); + + char conf[BUFSIZE] = { 0 }; + + sformat(conf, sizeof(conf), + "\n" + "# added by pg_autoctl's archiver replay/volatile base backup generation\n" + "restore_command = 'cp \"%s/%%f\" \"%%p\"'\n", + walcacheDir); + + return append_to_file(conf, strlen(conf), confPath); +} + + +/* + * pid of the currently-running replay staging instance, if any -- tracked + * the same way service_archiver.c tracks pgReceivewalPid, so + * stop_staging_postgres() knows what to signal. + */ +static pid_t stagingPostgresPid = -1; + + +/* + * start_staging_postgres execs the real "postgres" binary directly against + * stagingDir, loopback-only, on PG_AUTOCTL_ARCHIVER_REPLAY_PORT -- the same + * fork()/execv() pattern already used for pg_receivewal + * (service_archiver.c) and pg_basebackup (run_pg_basebackup(), this file), + * rather than going through pg_ctl: readiness is confirmed by + * wait_for_replay_pause()'s own connection-retry loop below, so pg_ctl's + * own "-w" startup wait buys nothing here, and this sidesteps it -- and the + * SQL-connection-based readiness check this needs anyway. + */ +static bool +start_staging_postgres(KeeperConfig *config, const char *stagingDir) +{ + char postgresPath[MAXPGPATH] = { 0 }; + + path_in_same_directory(config->pgSetup.pg_ctl, "postgres", postgresPath); + + if (!file_exists(postgresPath)) + { + log_error("Failed to find postgres at \"%s\"", postgresPath); + return false; + } + + char portStr[NAMEDATALEN] = { 0 }; + + sformat(portStr, sizeof(portStr), "%d", PG_AUTOCTL_ARCHIVER_REPLAY_PORT); + + pid_t pid = fork(); + + if (pid == -1) + { + log_error("Failed to fork postgres: %m"); + return false; + } + + if (pid == 0) + { + char *args[8]; + int argsIndex = 0; + + args[argsIndex++] = postgresPath; + args[argsIndex++] = "-D"; + args[argsIndex++] = (char *) stagingDir; + args[argsIndex++] = "-p"; + args[argsIndex++] = portStr; + args[argsIndex++] = "-h"; + args[argsIndex++] = "127.0.0.1"; + args[argsIndex] = NULL; + + execv(postgresPath, args); + + /* execv only returns on failure */ + log_fatal("execv(\"%s\"): %m", postgresPath); + _exit(127); + } + + stagingPostgresPid = pid; + + return true; +} + + +/* + * stop_staging_postgres stops the replay staging instance. Best effort: + * called during cleanup, including on failure paths where the instance may + * or may not have actually started. + */ +static void +stop_staging_postgres(void) +{ + if (stagingPostgresPid <= 0) + { + return; + } + + if (kill(stagingPostgresPid, SIGTERM) != 0 && errno != ESRCH) + { + log_warn("Failed to send SIGTERM to the replay staging instance " + "(pid %d): %m", stagingPostgresPid); + } + + int status = 0; + + if (waitpid(stagingPostgresPid, &status, 0) == -1 && errno != ECHILD) + { + log_warn("Failed to wait for the replay staging instance " + "(pid %d) to stop: %m", stagingPostgresPid); + } + + stagingPostgresPid = -1; +} + + +/* + * wait_for_replay_promotion connects to the staging instance (retrying: it + * takes a moment after fork()/execv() to start accepting connections) and + * polls pg_is_in_recovery() until it reports false -- Postgres promotes on + * its own once restore_command runs out of segments to fetch (see + * write_replay_recovery_config()'s own comment on why this replays to "no + * more locally-captured WAL" rather than a specific target LSN) -- or + * timeoutSeconds elapses. + */ +static bool +wait_for_replay_promotion(const char *connInfo, int timeoutSeconds) +{ + time_t deadline = time(NULL) + timeoutSeconds; + bool promoted = false; + + while (!promoted && time(NULL) < deadline) + { + PGSQL client = { 0 }; + + if (pgsql_init(&client, (char *) connInfo, PGSQL_CONN_UPSTREAM)) + { + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BOOL, false }; + const char *sql = "SELECT pg_is_in_recovery()"; + + if (pgsql_execute_with_params(&client, sql, 0, NULL, NULL, + &context, &parseSingleValueResult) && + context.parsedOk) + { + promoted = !context.boolVal; + } + + PQfinish(client.connection); + } + + if (!promoted) + { + sleep(1); + } + } + + return promoted; +} + + +/* + * generate_replay_basebackup is the forked child's own body for a + * `replay`/`volatile` backup: extract the last retained base backup into a + * fresh staging directory, replay this archiver's own locally-captured WAL + * forward until it promotes (see write_replay_recovery_config()'s own + * comment for why this targets "everything locally available" rather than + * a specific LSN), snapshot the promoted instance via pg_basebackup over + * loopback, report it, then stop and discard the staging instance -- + * 'volatile' means nothing survives between cycles, each one replays the + * whole gap since the last retained backup again. + */ +static bool +generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, + const char *backupDir, const char *label) +{ + KeeperConfig *config = &(keeper->config); + + char stagingDir[MAXPGPATH] = { 0 }; + + sformat(stagingDir, sizeof(stagingDir), "%s/replay-staging", + config->pgSetup.pgdata); + + if (directory_exists(stagingDir) && !rmtree(stagingDir, true)) + { + log_error("Failed to remove leftover replay staging directory " + "\"%s\" from a previous cycle", stagingDir); + return false; + } + + log_info("Generating a replay base backup, extracting \"%s\" into \"%s\"", + sourceBackupDir, stagingDir); + + if (!copy_directory_tree(sourceBackupDir, stagingDir)) + { + return false; + } + + if (!write_replay_recovery_config(stagingDir, config->pgSetup.pgdata)) + { + log_error("Failed to write replay recovery configuration in \"%s\"", + stagingDir); + return false; + } + + if (!start_staging_postgres(config, stagingDir)) + { + return false; + } + + char stagingConnInfo[MAXCONNINFO] = { 0 }; + + sformat(stagingConnInfo, sizeof(stagingConnInfo), + "host=127.0.0.1 port=%d user=%s dbname=%s application_name=%s", + PG_AUTOCTL_ARCHIVER_REPLAY_PORT, + PG_AUTOCTL_REPLICA_USERNAME, DEFAULT_DATABASE_NAME, config->name); + + bool ok = wait_for_replay_promotion(stagingConnInfo, + ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); + + if (!ok) + { + log_error("Replay staging instance at \"%s\" failed to replay " + "available WAL and promote within %d seconds", + stagingDir, ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); + } + else + { + NodeAddress stagingNode = { 0 }; + + strlcpy(stagingNode.host, "127.0.0.1", sizeof(stagingNode.host)); + stagingNode.port = PG_AUTOCTL_ARCHIVER_REPLAY_PORT; + + /* promoted by the time wait_for_replay_promotion() returns true -- + * report_basebackup()'s own endlsn query needs to know to use + * pg_current_wal_lsn(), not pg_last_wal_replay_lsn() (NULL outside + * recovery) */ + stagingNode.isPrimary = true; + + ok = run_pg_basebackup(config, &stagingNode, backupDir, label) && + report_basebackup(keeper, &stagingNode, backupDir, label, + "replay", "volatile"); + } + + stop_staging_postgres(); + + /* volatile: discard the staging instance unconditionally, success or not */ + if (!rmtree(stagingDir, true)) + { + log_warn("Failed to remove replay staging directory \"%s\" after " + "use, will be overwritten on the next cycle", stagingDir); + } + + return ok; +} + + /* * service_archiver_maybe_generate_basebackup checks, once per - * service_archiver_loop() tick, whether this group has zero existing base - * backups yet and -- if so, and no generation is already in flight -- forks - * a child to produce one. See this file's own header comment for the full - * scope of this first pass (bootstrap trigger, live source only). + * service_archiver_loop() tick, whether a base backup generation is due + * for this group and -- if so, and no generation is already in flight -- + * forks a child to produce one. See this file's own header comment for + * the full trigger scope of this pass (bootstrap live, then exactly one + * replay/volatile backup to exercise that pipeline). */ bool service_archiver_maybe_generate_basebackup(Keeper *keeper) @@ -482,31 +868,26 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) KeeperConfig *config = &(keeper->config); bool found = false; char storageLocation[MAXPGPATH] = { 0 }; - - if (!monitor_get_latest_basebackup_location(&(keeper->monitor), - config->formation, - config->groupId, - storageLocation, - sizeof(storageLocation), - &found)) + char latestSource[NAMEDATALEN] = { 0 }; + + if (!monitor_get_latest_basebackup_info(&(keeper->monitor), + config->formation, + config->groupId, + storageLocation, + sizeof(storageLocation), + latestSource, + sizeof(latestSource), + &found)) { /* errors already logged */ return false; } - if (found) - { - /* bootstrap already satisfied; scheduled/retention-driven triggers - * are a follow-up milestone, see this file's own header comment */ - return true; - } - - NodeAddress source = { 0 }; - - if (!select_basebackup_source(keeper, &source)) + if (found && strcmp(latestSource, "replay") == 0) { - log_warn("No eligible node to source a live base backup from yet, " - "will retry"); + /* both the bootstrap live backup and this pass's own one-time + * replay exercise are done; real scheduling is a follow-up, see + * this file's own header comment */ return true; } @@ -528,12 +909,41 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) char label[NAMEDATALEN] = { 0 }; - strftime(label, sizeof(label), "basebackup-%Y%m%dT%H%M%SZ", &nowUTC); + strftime(label, sizeof(label), + found ? "basebackup-replay-%Y%m%dT%H%M%SZ" + : "basebackup-%Y%m%dT%H%M%SZ", + &nowUTC); char backupDir[MAXPGPATH] = { 0 }; sformat(backupDir, sizeof(backupDir), "%s/%s", backupsDir, label); + /* + * sourceBackupDir must be captured now, in the parent, into a + * fixed-size buffer the forked child can safely read after fork(): + * storageLocation itself is a local, stack-allocated array, still + * valid across fork() (the child gets its own copy of the whole + * address space), so this is really just documenting that fact. + */ + char sourceBackupDir[MAXPGPATH] = { 0 }; + + strlcpy(sourceBackupDir, storageLocation, sizeof(sourceBackupDir)); + + NodeAddress liveSource = { 0 }; + bool haveLiveSource = false; + + if (!found) + { + if (!select_basebackup_source(keeper, &liveSource)) + { + log_warn("No eligible node to source a live base backup from " + "yet, will retry"); + return true; + } + + haveLiveSource = true; + } + fflush(stdout); fflush(stderr); @@ -550,7 +960,10 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) (void) set_signal_handlers(false); (void) set_ps_title("pg_autoctl: archiver basebackup"); - bool ok = generate_basebackup(keeper, &source, backupDir, label); + bool ok = haveLiveSource + ? generate_live_basebackup(keeper, &liveSource, backupDir, label) + : generate_replay_basebackup(keeper, sourceBackupDir, + backupDir, label); exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); } diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c index 0359e1031..13b66eafe 100644 --- a/src/bin/pg_autoctl/service_archiver_serve.c +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -17,7 +17,7 @@ * to know an archiver's local WAL cache path, that's inherently * archiver-host-local information never sent to it. The one thing genuinely * worth asking the monitor is the latest base backup's storage location - * (monitor_get_latest_basebackup_location), which is real, monitor-tracked + * (monitor_get_latest_basebackup_info), which is real, monitor-tracked * state once the "Base backup generation" milestone lands. * * Copyright (c) Microsoft Corporation. All rights reserved. @@ -204,14 +204,17 @@ service_archiver_serve_refresh_routes(Keeper *keeper) KeeperConfig *config = &(keeper->config); char basebackupLocation[MAXPGPATH] = { 0 }; + char basebackupSource[NAMEDATALEN] = { 0 }; bool found = false; - if (!monitor_get_latest_basebackup_location(&(keeper->monitor), - config->formation, - config->groupId, - basebackupLocation, - sizeof(basebackupLocation), - &found)) + if (!monitor_get_latest_basebackup_info(&(keeper->monitor), + config->formation, + config->groupId, + basebackupLocation, + sizeof(basebackupLocation), + basebackupSource, + sizeof(basebackupSource), + &found)) { log_warn("Failed to fetch the latest base backup location from the " "monitor; the routes file will omit it for now"); From c566ae438dadae57fc246c13ab353408401432b9 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 15:48:32 +0200 Subject: [PATCH 15/55] tests: add archiver_basebackup_generation.pgaf (M5) pgaftest coverage for Milestone 5's own base backup generation (both live and replay/volatile): brings up a monitor + primary + archiver, then waits for both the bootstrap live backup and the one-time replay/volatile follow-up to land, checking the group's final pgautofailover.get_latest_basebackup() row (source = 'replay', replaymode = 'volatile', status = 'complete'). No explicit trigger step is needed here, unlike archiver_wal_capture.pgaf's pg_switch_wal() calls -- both backups fire on their own within a couple of service_archiver_loop() ticks of the archiver starting. That also makes the intermediate 'live'-only state unsafe to assert on directly (this pass's own trigger logic produces at most one live and one replay backup before going quiet for the group, a couple of ticks apart, with nothing in this spec's control over exactly when to look) -- only the final state, once both have landed, is deterministic. Registered in tests/tap/schedule and tests/tap/schedules/node.sch, alongside archiver_wal_capture.pgaf. --- tests/tap/schedule | 1 + tests/tap/schedules/node.sch | 1 + .../specs/archiver_basebackup_generation.pgaf | 54 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 tests/tap/specs/archiver_basebackup_generation.pgaf diff --git a/tests/tap/schedule b/tests/tap/schedule index 619da99b8..9083e4280 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -32,6 +32,7 @@ demote_timeout_wait_primary_deadlock wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock archiver_wal_capture +archiver_basebackup_generation keeper_fsm_gap_209_wait_maintenance keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index 2c6f99f5c..ac8ee1cc4 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -19,3 +19,4 @@ demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect archiver_wal_capture +archiver_basebackup_generation diff --git a/tests/tap/specs/archiver_basebackup_generation.pgaf b/tests/tap/specs/archiver_basebackup_generation.pgaf new file mode 100644 index 000000000..c5497776e --- /dev/null +++ b/tests/tap/specs/archiver_basebackup_generation.pgaf @@ -0,0 +1,54 @@ +# Archiving & Disaster Recovery, Milestone 5: base backup generation, +# `live` source then `replay`/`volatile`. +# +# Covers service_archiver_maybe_generate_basebackup() (service_archiver_ +# basebackup.c): a group with no base backups yet gets one immediately, +# sourced live (pg_basebackup run directly against a real node); once that +# lands, the very next tick exercises replay/volatile once -- extract that +# live backup into a throwaway staging instance, replay this archiver's own +# already-captured WAL forward until it promotes, pg_basebackup it over +# loopback, then discard the staging instance. Both are real, +# monitor-tracked pgautofailover.basebackup rows by the end. +# +# No explicit trigger step is needed here (unlike archiver_wal_capture.pgaf's +# pg_switch_wal() calls): both backups fire on their own, a couple of +# service_archiver_loop() ticks apart, as soon as the archiver starts. +# get_latest_basebackup() only ever reports the single newest row, and this +# pass's own trigger logic produces at most two rows total (live, then +# replay) before going quiet for the group -- so the *final* state is +# deterministic (source = 'replay') even though the intermediate 'live'-only +# state is not something this spec can reliably catch mid-flight. +# +# Predecessor: archiver_wal_capture.pgaf (M4). + +cluster { + monitor + formation { + node1 + archiver1 archiver + } +} + +setup { + wait until primary timeout 60s + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: both the bootstrap live backup and the one-time replay/volatile +# exercise complete on their own; check the final state. +# + +step test_001_replay_backup_lands { + sleep 45s + sql monitor { SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0); } + expect { replay } + sql monitor { SELECT replaymode::text FROM pgautofailover.get_latest_basebackup('default', 0); } + expect { volatile } + sql monitor { SELECT status::text FROM pgautofailover.get_latest_basebackup('default', 0); } + expect { complete } +} From 803ac383b3c17dcddcecb62ead6417163033b007 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 16:24:25 +0200 Subject: [PATCH 16/55] docs: Archiving & Disaster Recovery architecture section + diagram New "Archiving & Disaster Recovery Architecture" section in intro.rst, between "Single Standby Architecture" and "Multiple Standby Architecture" -- an archiver is orthogonal to standby count, so it reads best as the thing you add on top of the simplest case before the doc branches into standby-count variations. New docs/tikz/arch-archiver.tex, rendered to .svg the same way every other architecture diagram in this directory is (latexmk -lualatex + pdftocairo, verified locally): primary + secondary + archiver, with the archiver's own WAL cache / base backups called out, a distinct WAL streaming (pg_receivewal) edge separate from real streaming replication, and the monitor's health-check/WAL-report edges to all three. common.tex gains one new color pair (abox/atxt, MS amber) and one new edge style (wal, dashed) for the archiver box and its WAL-streaming edge -- deliberately not reusing the primary/standby colors, since an archiver is a different kind of entity, not a replica. Terminology: uses "archiver" for the physical entity and "archiving node" for its per-group FSM membership, per-project decision -- avoids colliding with pgautofailover.archiver_node, the broader schema table that also covers warm-standby/pitr instances which don't participate in elections at all. --- docs/intro.rst | 45 +++ docs/tikz/arch-archiver.svg | 529 ++++++++++++++++++++++++++++++++++++ docs/tikz/arch-archiver.tex | 54 ++++ docs/tikz/common.tex | 7 + 4 files changed, 635 insertions(+) create mode 100644 docs/tikz/arch-archiver.svg create mode 100644 docs/tikz/arch-archiver.tex diff --git a/docs/intro.rst b/docs/intro.rst index 15d772baa..c96067f7e 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -36,6 +36,51 @@ setting on the *primary* node. Until the *secondary* is back to being monitored healthy, failover and switchover operations are not allowed, preventing data loss. +.. _archiving_architecture: + +Archiving & Disaster Recovery Architecture +------------------------------------------- + +.. figure:: ./tikz/arch-archiver.svg + :alt: pg_auto_failover Architecture with a primary, a standby, and an archiver + + pg_auto_failover architecture with a primary, a standby, and an archiver + +An **archiver** is a separate physical entity, added on top of any of the +architectures on this page — it applies just as well to the single-standby +setup above as it does to a multi-standby fleet, since it addresses a +different concern: disaster recovery, independent of how many nodes +currently participate in the failover quorum. + +Unlike a standby, an archiver holds no copy of the primary's data directory +and never takes writes or reads for the application. It runs its own +`pg_receivewal`__ continuously against the group's current primary, +capturing every WAL segment into a local cache the moment it's generated, +and periodically produces full base backups from that cache. Both are +reported back to the pg_auto_failover Monitor, the same way a standby +reports its own replication state — so the Monitor can tell an operator, +or a client library, when a given segment has landed durably on enough +archivers to be considered safe (``archiver_quorum``), and where the most +recent base backup lives. + +__ https://www.postgresql.org/docs/current/app-pgreceivewal.html + +The pg_auto_failover Monitor tracks an archiver's participation in a group +as its own node, in the ``archiving`` **archiving node** state — reported +and monitored the same way ``primary``/``secondary`` are, but never a +candidate for promotion or failover: an archiving node holds no +`PGDATA`__ of its own, so there is nothing to promote it *to*. + +__ https://www.postgresql.org/docs/current/app-initdb.html + +Because the archiver keeps a complete, continuously updated copy of the +group's WAL stream and periodic base backups independent of any single +standby, it serves two purposes beyond ordinary high availability: a new +node can be provisioned straight from an archiver's cache instead of +placing extra load on a live primary or secondary, and a group that has +lost every other node still has everything needed to rebuild from scratch, +as long as the archiver itself survived. + Multiple Standby Architecture ----------------------------- diff --git a/docs/tikz/arch-archiver.svg b/docs/tikz/arch-archiver.svg new file mode 100644 index 000000000..d54c3476b --- /dev/null +++ b/docs/tikz/arch-archiver.svg @@ -0,0 +1,529 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-archiver.tex b/docs/tikz/arch-archiver.tex new file mode 100644 index 000000000..ad9790957 --- /dev/null +++ b/docs/tikz/arch-archiver.tex @@ -0,0 +1,54 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% \draw [help lines] (-11,0) grid (11,24); + + \node (p) at (0,20) [primary] + {\textbf{\Large Primary}}; + \node (s) at (0,12) [standby] + {\textbf{\Large Secondary}}; + \node (app) at (-8,16) [app] {\textbf{Application}}; + + \node (a) at (9,20) [archiver] + {\textbf{\normalsize Archiver} + \nodepart{second} + \textbf{\large ARCHIVING} + \nodepart[align=left]{third} + \texttt{WAL cache} \\ + \texttt{Base backups} + }; + + \node (m) at (9,12) [monitor] {\textbf{Monitor}}; + + \path (app.north east) edge [sql,out=90,in=180] node {SQL} (p) + (app.south east) edge [sqlf,out=-90,in=180] node[below] {SQL (fallback)} (s) + (p) edge [sr] + node[left] {Streaming} + node [right] {Replication} (s) + (p.east) edge [wal] node[above,pos=0.38] {WAL streaming} + node[below,pos=0.38] {(\texttt{pg\_receivewal})} (a.west) + (a.south) edge [hc] node[right] {WAL reports} (m.north) + (m.west) edge [hc,out=180,in=-70] node[below,sloped] {Health checks} (s.east) + (m.east) edge [hc,out=20,in=-20,looseness=1.6] node[right] {Health checks} (p.east); +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/common.tex b/docs/tikz/common.tex index c9c9624d1..fd414cb2d 100644 --- a/docs/tikz/common.tex +++ b/docs/tikz/common.tex @@ -12,6 +12,9 @@ \definecolor{async}{HTML}{EBEFF5} % very light grey +\definecolor{abox}{HTML}{FFB900} % MS amber -- cold storage, distinct from primary/standby +\definecolor{atxt}{HTML}{2F2F2F} % off-black + \tikzstyle{app}=[circle,thick, text=aptxt,draw=apbox,fill=white, line width=0.25em,minimum size=4cm] @@ -29,6 +32,9 @@ \tikzstyle{standby}=[mpnode,text=stxt,draw=white, rectangle split part fill={sbox,sbox,white}] +\tikzstyle{archiver}=[mpnode,text=atxt,draw=white, + rectangle split part fill={abox,abox,white}] + \tikzstyle{monitor}=[node,text=mtxt,draw=mbox,fill=mbox] \tikzstyle{citusnode}=[rectangle split,rectangle split parts=2, @@ -45,6 +51,7 @@ \tikzstyle{sql}=[->,color=pbox,text=stxt,line width=0.15em] \tikzstyle{sqlf}=[->,color=sbox,text=stxt,line width=0.15em,loosely dashed] \tikzstyle{sr}=[>->,color=stxt,text=stxt,line width=0.15em] +\tikzstyle{wal}=[>->,color=abox,text=atxt,line width=0.15em,densely dashed] \tikzstyle{hc}=[<->,color=mbox,text=mtxt,line width=0.15em,dotted] \tikzstyle{hcmid}=[color=mbox,text=mtxt,line width=0.15em,dotted] \tikzstyle{cw}=[<->,color=stxt,text=stxt,line width=0.1em] From c505ea4abfaa47b4e92bd18872abc51a82d33e3d Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 16:32:52 +0200 Subject: [PATCH 17/55] pg_autoctl: report an archiver's real captured-WAL LSN keeper->postgres.currentLSN was set to "0/0" once at service_archiver_ loop() startup and never updated again -- an archiving node's own reportedlsn in pgautofailover.node stayed at that placeholder forever, no matter how much WAL it had actually captured. This mattered more than it looked: pgautofailover.get_most_advanced_ standby() -- the query fast-forward uses to pick a WAL source during a failover election -- has no kind-based exclusion at all, and an archiving node already passes through REPORT_LSN_STATE during an election exactly like any other node (ARCHIVING_STATE -> REPORT_LSN_STATE, fsm.c). A "0/0" reportedlsn was the only thing keeping an archiver from ever being ranked as a candidate WAL source. service_archiver_update_current_lsn() now scans the local WAL cache for the newest complete segment each tick and updates currentLSN to its end LSN before keeper_node_active() reports it -- verified against a real cluster: after two pg_switch_wal() calls, the archiver's own pgautofailover.node.reportedlsn row tracks the primary's position almost exactly (0/A000000 vs. the primary's own 0/A000060). --- src/bin/pg_autoctl/service_archiver.c | 77 ++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index ab06d5c77..d49e55197 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -409,6 +409,68 @@ service_archiver_report_captured_wal(Keeper *keeper) } +/* + * service_archiver_update_current_lsn scans walcacheDir for the newest + * complete (non-".partial") WAL segment and updates keeper->postgres. + * currentLSN to the LSN just past its end -- the real, local, self- + * consistent "how far have I actually captured" position, reported to the + * monitor by keeper_node_active() the same way every other node kind + * reports its own currentLSN. + * + * This is what makes an archiving node a real, rankable candidate for + * pgautofailover.get_most_advanced_standby() during a failover election: + * that query already has no kind-based exclusion and already considers any + * node reporting REPORT_LSN_STATE (an archiving node passes through it + * during elections, see ARCHIVING_STATE -> REPORT_LSN_STATE in fsm.c) -- + * the only thing that ever kept an archiver from being selected was this + * value staying "0/0" forever. Falls back to "0/0" itself when nothing has + * been captured yet, matching keeper_update_pg_state()'s own default + * before it has a real reading. + */ +static void +service_archiver_update_current_lsn(Keeper *keeper) +{ + const char *walcacheDir = keeper->config.pgSetup.pgdata; + + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + strlcpy(keeper->postgres.currentLSN, "0/0", + sizeof(keeper->postgres.currentLSN)); + return; + } + + char best[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (!is_wal_segment_filename(entry->d_name)) + { + continue; + } + + if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + { + strlcpy(best, entry->d_name, sizeof(best)); + } + } + + closedir(dir); + + if (best[0] == '\0') + { + strlcpy(keeper->postgres.currentLSN, "0/0", + sizeof(keeper->postgres.currentLSN)); + return; + } + + wal_segment_end_lsn(best, keeper->postgres.currentLSN, + sizeof(keeper->postgres.currentLSN)); +} + + /* * service_archiver_loop is the archiver's own node_active() reporting loop * -- deliberately not keeper_node_active_loop (service_keeper.c): that @@ -439,14 +501,11 @@ service_archiver_loop(Keeper *keeper) /* * An archiver never calls keeper_update_pg_state() -- there's no real * Postgres instance to query (see haspgdata's own design comment) -- - * so keeper->postgres.currentLSN is otherwise left at its zero-valued - * empty string for the lifetime of this process. keeper_node_active() - * always sends it as one of node_active()'s own parameters, and the - * monitor-side pg_lsn column rejects an empty string outright ("invalid - * input syntax for type pg_lsn"). "0/0" is the same placeholder - * keeper_update_pg_state() itself defaults to before it has a real - * reading; an archiver's own WAL-capture progress is tracked - * separately via archiver_wal, not through this per-node report. + * so keeper->postgres.currentLSN needs its own source of truth here. + * keeper_node_active() always sends it as one of node_active()'s own + * parameters, and the monitor-side pg_lsn column rejects an empty + * string outright ("invalid input syntax for type pg_lsn"), so it must + * hold a valid value even before the first tick's own scan runs. */ strlcpy(keeper->postgres.currentLSN, "0/0", sizeof(keeper->postgres.currentLSN)); @@ -454,6 +513,8 @@ service_archiver_loop(Keeper *keeper) { MonitorAssignedState assignedState = { 0 }; + (void) service_archiver_update_current_lsn(keeper); + if (!keeper_load_state(keeper)) { log_error("Failed to read archiver state file, retrying..."); From 033e4f615119f54e0d988b42055eaa38f6eb9c35 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 17:00:38 +0200 Subject: [PATCH 18/55] pg_walsender/pg_autoctl: make a real standby able to fast-forward from an archiver Confirms (and builds out) the reframing from the previous commit: an archiving node already passes through REPORT_LSN_STATE during elections and get_most_advanced_standby() has no kind-based exclusion, so once its currentLSN is real, fast-forward's existing streaming-replication code path can already select and target one -- no new restore_command plumbing needed. Four real gaps stood between that and actually working, found and fixed by testing a genuine, unmodified Postgres standby against a real archiver end to end (not just pg_receivewal, which never exercises any of these): - pg_walsender routing is dbname-based (formation/group as dbname), but a real standby's own walreceiver never forwards the operator's dbname for a physical replication connection -- it always sends the literal "replication", confirmed against a real standby. accept_loop.c now falls back to the single configured route when it sees that sentinel, matching this milestone's own one-membership-per-archiver scope; a multi-route archiver (later milestone) needs a different mechanism (e.g. application_name, which real walreceiver does forward). - IDENTIFY_SYSTEM's systemid always fell back to the "unknown" placeholder "0" because nothing ever populated route->systemId: service_archiver_ serve.c's own routes-file writer never wrote a systemid key, even though routes.c already knew how to parse one. A real standby rejects a mismatched system identifier outright ("database system identifier differs between the primary and standby"). Fixed with a new monitor RPC, monitor_get_group_system_identifier() (pgautofailover. get_group_system_identifier(), new SQL function in both pgautofailover.sql and the 2.2--2.3 migration -- an archiving node has no sysidentifier of its own, but every other node in its group shares the same one), wired into the routes-file refresh. - cmd_start_replication.c read raw fread() bytes from a ".partial" segment without knowing where pg_receivewal's actually-written data ends -- pg_receivewal pre-allocates the full segment size up front (matching real Postgres's own WAL file pre-allocation), so reading past the real tail returns zeros indistinguishable from real content at the byte level. Sending that tail as WAL data is exactly what a real standby's recovery logic flags as "invalid record length ... got 0", and on seeing it, terminates its own walreceiver outright rather than treating it as "nothing new yet, retry" -- with no automatic reconnection afterward. Fixed by trimming any trailing zero run before ever sending a ".partial" chunk (self-correcting: an in-progress boundary just gets re-read next tick instead of shipped early). - get_most_advanced_standby() returns an ARCHIVING row's real nodeport, which is the port == 0 sentinel (no postmaster of its own), not the archiver's actual pg_walsender serve port -- the monitor has no column for that (archiver-host-local information, same reasoning service_ archiver_serve.c's own routes file exists for). keeper_get_most_ advanced_standby() now resolves a port == 0 candidate to PG_AUTOCTL_ARCHIVER_SERVE_PORT, matching this milestone's single- well-known-port scope. Verified end-to-end: a real pg_basebackup-seeded standby, given nothing but an ordinary primary_conninfo pointing at the archiver's serve port, completed backup recovery, reached consistent recovery state, streamed live via START_REPLICATION, stayed connected indefinitely (pg_stat_wal_ receiver: status = streaming), and correctly applied newly-written data (a table created and populated on the real primary afterward) -- with zero restore_command, zero new replication-source machinery, and zero changes to fsm_fast_forward's own selection logic beyond the port fix above. --- src/bin/pg_autoctl/keeper.c | 17 +++++ src/bin/pg_autoctl/monitor.c | 65 ++++++++++++++++++++ src/bin/pg_autoctl/monitor.h | 4 ++ src/bin/pg_autoctl/service_archiver_serve.c | 19 ++++++ src/bin/pg_walsender/accept_loop.c | 30 +++++++++ src/bin/pg_walsender/cmd_start_replication.c | 36 +++++++++++ src/monitor/pgautofailover--2.2--2.3.sql | 27 ++++++++ src/monitor/pgautofailover.sql | 27 ++++++++ 8 files changed, 225 insertions(+) diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index c4461f894..13aa014c4 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -3327,6 +3327,23 @@ keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *upstreamNode, return false; } + /* + * port == 0 is the ARCHIVING row sentinel documented in + * pgautofailover.sql ("an ARCHIVING row has no postmaster of its + * own") -- get_most_advanced_standby() returns it verbatim from + * pgautofailover.node, which has no column for an archiver's real + * pg_walsender serve port (archiver-host-local information the + * monitor is never told, matching service_archiver_serve.c's own + * routes-file rationale). This milestone's own scope is one + * archiver on the well-known default serve port, so resolving it + * here is enough; a configurable-port archiver is a follow-up that + * would need the monitor to actually track it. + */ + if (*found && upstreamNode->port == 0) + { + upstreamNode->port = PG_AUTOCTL_ARCHIVER_SERVE_PORT; + } + return true; } else diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 791c51c7e..46d62b25d 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -1076,6 +1076,71 @@ monitor_get_latest_basebackup_info(Monitor *monitor, } +/* + * monitor_get_group_system_identifier calls + * pgautofailover.get_group_system_identifier(formationId, groupId) -- + * needed by an archiving node (which has no real Postgres instance of its + * own to report one) to serve a correct IDENTIFY_SYSTEM response, so a real + * standby streaming from it doesn't reject the connection with "database + * system identifier differs". *found is false (not an error) when no other + * node in the group has reported one yet. + */ +bool +monitor_get_group_system_identifier(Monitor *monitor, + const char *formationId, int groupId, + uint64_t *systemIdentifier, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + + /* + * COALESCE to 0, the same "unset" sentinel this column already uses + * elsewhere (node_metadata.c/pgautofailover.sql): the SQL function + * itself is a plain scalar, not SETOF, so a no-match query still + * produces one row with a NULL value rather than zero rows -- + * PGSQL_RESULT_BIGINT's own parser treats a NULL value as a parse + * failure, which "not reported yet" is not. + */ + const char *sql = + "SELECT coalesce(" + "pgautofailover.get_group_system_identifier($1, $2), 0)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to get the system identifier for \"%s\"/%d " + "from the monitor", formationId, groupId); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the system identifier returned by the " + "monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + if (context.bigint == 0) + { + /* no node in the group has reported one yet -- not an error */ + return true; + } + + *systemIdentifier = context.bigint; + *found = true; + + return true; +} + + /* * monitor_report_wal_received calls pgautofailover.report_wal_received() * to record that nodeId (the ARCHIVING membership's own nodeid, not the diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 00b595411..39a0ee758 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -163,6 +163,10 @@ bool monitor_get_latest_basebackup_info(Monitor *monitor, char *storageLocation, size_t storageLocationSize, char *source, size_t sourceSize, bool *found); +bool monitor_get_group_system_identifier(Monitor *monitor, + const char *formationId, int groupId, + uint64_t *systemIdentifier, + bool *found); bool monitor_report_wal_received(Monitor *monitor, int64_t nodeId, const char *walFileName, const char *lsn); bool monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c index 13b66eafe..9de744cbb 100644 --- a/src/bin/pg_autoctl/service_archiver_serve.c +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -221,6 +221,20 @@ service_archiver_serve_refresh_routes(Keeper *keeper) found = false; } + uint64_t systemIdentifier = 0; + bool foundSystemIdentifier = false; + + if (!monitor_get_group_system_identifier(&(keeper->monitor), + config->formation, + config->groupId, + &systemIdentifier, + &foundSystemIdentifier)) + { + log_warn("Failed to fetch the group's system identifier from the " + "monitor; the routes file will omit it for now"); + foundSystemIdentifier = false; + } + char routesPath[MAXPGPATH] = { 0 }; service_archiver_serve_routes_path(config, routesPath); @@ -245,6 +259,11 @@ service_archiver_serve_refresh_routes(Keeper *keeper) fformat(fileStream, "basebackup = %s\n", basebackupLocation); } + if (foundSystemIdentifier) + { + fformat(fileStream, "systemid = %" PRIu64 "\n", systemIdentifier); + } + if (fclose(fileStream) == EOF) { log_error("Failed to write file \"%s\": %m", tmpPath); diff --git a/src/bin/pg_walsender/accept_loop.c b/src/bin/pg_walsender/accept_loop.c index d753bfb5c..8ae91b57f 100644 --- a/src/bin/pg_walsender/accept_loop.c +++ b/src/bin/pg_walsender/accept_loop.c @@ -35,6 +35,17 @@ * instead of the normal replication command loop -- see cmd_fetch_file.h */ #define WS_FETCH_DBNAME_PREFIX "fetch/" +/* + * A real, unmodified Postgres standby's own internal walreceiver process + * (primary_conninfo-driven physical replication) always sends this literal + * string as its startup packet's dbname -- confirmed against a real + * standby: it does not forward whatever dbname the operator wrote into + * primary_conninfo the way a generic libpq client (psql, pg_receivewal, + * this project's own FETCH_FILE client) does. See the routeKey fallback + * below. + */ +#define WS_REAL_WALRECEIVER_DBNAME "replication" + static int create_listen_socket(int port) @@ -114,6 +125,25 @@ handle_connection(int clientSock, const WsServerConfig *config) ? params.database + strlen(WS_FETCH_DBNAME_PREFIX) : params.database; + /* + * dbname-based routing cannot work for a real walreceiver connection + * (see WS_REAL_WALRECEIVER_DBNAME's own comment) -- fall back to the + * single configured route unambiguously, matching this milestone's own + * one-membership-per-archiver scope. Multiple routes with a real + * walreceiver connecting is left as a clean auth rejection (routeKey + * stays "replication", which never matches a real route.key) rather + * than guessing; a multi-route archiver needs a different mechanism + * for a real standby to identify its route (e.g. application_name, + * which real walreceiver does forward from primary_conninfo, unlike + * dbname) -- a later milestone's problem, not this one's. + */ + if (!isFetchMode && + strcmp(routeKey, WS_REAL_WALRECEIVER_DBNAME) == 0 && + routeCount == 1) + { + routeKey = routes[0].key; + } + const WsRoute *route = NULL; if (!ws_authenticate(clientSock, ¶ms, routeKey, routes, routeCount, &route)) diff --git a/src/bin/pg_walsender/cmd_start_replication.c b/src/bin/pg_walsender/cmd_start_replication.c index da72b621b..cb7cae147 100644 --- a/src/bin/pg_walsender/cmd_start_replication.c +++ b/src/bin/pg_walsender/cmd_start_replication.c @@ -149,6 +149,37 @@ wait_for_more_data_or_client(int sock, uint64_t currentLsn, time_t *lastKeepaliv } +/* + * trim_trailing_zeros returns the length of buffer with any trailing run of + * zero bytes removed. A ".partial" segment is pre-allocated to its full + * WS_WAL_SEGMENT_SIZE by pg_receivewal the moment it's created (matching + * real Postgres's own WAL file pre-allocation, XLogFileInitInternal) -- + * unlike a real primary's own walsender, which only ever knows about bytes + * it has actually flushed, a plain fread() from a ".partial" file cannot + * tell real WAL content apart from the not-yet-written tail, which reads + * back as zeros. Sending that tail as if it were real WAL data is exactly + * what a real standby's own recovery logic detects as "invalid record + * length ... got 0" -- and, on that response, terminates its walreceiver + * outright rather than treating it as "no more data yet, retry" (which is + * pg_receivewal's own polling behavior, so it never noticed). + * + * Trimming any trailing zero run before ever sending it means an in- + * progress chunk boundary is re-read (and re-trimmed) on the next + * iteration rather than shipped as real data -- self-correcting, at worst + * a few bytes of redundant re-reads per tick, never sent out early. + */ +static size_t +trim_trailing_zeros(const char *buffer, size_t len) +{ + while (len > 0 && buffer[len - 1] == 0) + { + len--; + } + + return len; +} + + static bool parse_lsn(const char *s, uint64_t *lsn, const char **endptr) { @@ -331,6 +362,11 @@ cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) fclose(file); + if (!isComplete) + { + got = trim_trailing_zeros(buffer, got); + } + if (got == 0) { if (isComplete) diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index fc1f5948b..cc8531954 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -1780,6 +1780,33 @@ comment on function pgautofailover.get_latest_basebackup(text,int) grant execute on function pgautofailover.get_latest_basebackup(text,int) to autoctl_node; +-- an archiving node has no sysidentifier of its own (haspgdata = false, +-- see that column's own comment): it never runs a real Postgres instance +-- to report one. Every other node in the group shares the same physical +-- cluster's identifier, so any one of them answers for the whole group -- +-- needed by pg_walsender's own IDENTIFY_SYSTEM response (cmd_identify_ +-- system.c) so a real standby streaming from the archiver doesn't reject +-- it with "database system identifier differs between the primary and +-- standby". +CREATE FUNCTION pgautofailover.get_group_system_identifier + (formationid text, groupid int) + RETURNS bigint LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT sysidentifier + FROM pgautofailover.node + WHERE node.formationid = get_group_system_identifier.formationid + AND node.groupid = get_group_system_identifier.groupid + AND sysidentifier IS NOT NULL + AND sysidentifier != 0 + LIMIT 1; +$$; + +comment on function pgautofailover.get_group_system_identifier(text,int) + is 'the Postgres system identifier shared by every node in a group, for an archiving node (which has none of its own) to serve via IDENTIFY_SYSTEM'; + +grant execute on function pgautofailover.get_group_system_identifier(text,int) + to autoctl_node; + -- for kind = 'warm-standby': raises if the owning archiver is already at -- its maxresidentreplay cap CREATE FUNCTION pgautofailover.create_archiver_node diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index abd19060b..a0a98884d 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -2362,6 +2362,33 @@ comment on function pgautofailover.get_latest_basebackup(text,int) grant execute on function pgautofailover.get_latest_basebackup(text,int) to autoctl_node; +-- an archiving node has no sysidentifier of its own (haspgdata = false, +-- see that column's own comment): it never runs a real Postgres instance +-- to report one. Every other node in the group shares the same physical +-- cluster's identifier, so any one of them answers for the whole group -- +-- needed by pg_walsender's own IDENTIFY_SYSTEM response (cmd_identify_ +-- system.c) so a real standby streaming from the archiver doesn't reject +-- it with "database system identifier differs between the primary and +-- standby". +CREATE FUNCTION pgautofailover.get_group_system_identifier + (formationid text, groupid int) + RETURNS bigint LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT sysidentifier + FROM pgautofailover.node + WHERE node.formationid = get_group_system_identifier.formationid + AND node.groupid = get_group_system_identifier.groupid + AND sysidentifier IS NOT NULL + AND sysidentifier != 0 + LIMIT 1; +$$; + +comment on function pgautofailover.get_group_system_identifier(text,int) + is 'the Postgres system identifier shared by every node in a group, for an archiving node (which has none of its own) to serve via IDENTIFY_SYSTEM'; + +grant execute on function pgautofailover.get_group_system_identifier(text,int) + to autoctl_node; + -- for kind = 'warm-standby': raises if the owning archiver is already at -- its maxresidentreplay cap CREATE FUNCTION pgautofailover.create_archiver_node From 2e8ead06625e39a9eaad768fd50de828243256bc Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 17:52:29 +0200 Subject: [PATCH 19/55] pg_autoctl: implement create postgres --from-archiver Bootstraps a brand new node from a registered archiver's base backup plus captured WAL instead of the group's live primary -- the disaster- recovery case: rebuild after every live standby (or even the primary) is gone, with only the archiver left standing. Verified end to end against a real cluster: `create postgres --from-archiver` completed pg_basebackup from the archiver, replayed WAL, and settled into a genuinely healthy "secondary" (pg_stat_wal_receiver: status = streaming), matching reportedlsn against the real primary once it re-parented there. New plumbing: - KeeperConfig.fromArchiver (keeper_config.h) plus the `--from-archiver` CLI flag on `create postgres` (cli_create_node.c, cli_common.c) -- runtime-only, same as createAndRun, since reach_initial_state() runs in the same `pg_autoctl create` invocation that parses it. - pgautofailover.get_archiver_node() (pgautofailover.sql, the 2.2--2.3 migration) plus its monitor_get_archiver_node()/keeper_get_archiver_ node() C wrappers (monitor.c, keeper.c): finds the ARCHIVING row for (formation, group) directly. Deliberately not get_most_advanced_ standby() -- that function filters on reportedstate = 'report_lsn', a transient state an archiving node only visits during a FAST_FORWARD election, never during its normal steady-state 'archiving' operation, so it can never find an idle archiver outside of an election. - fsm_init_standby() (fsm_transition.c) branches on config->fromArchiver to resolve the archiver via the above instead of keeper_get_primary(), and passes an empty replication slot name -- pg_walsender has no slot-based retention in this milestone (cmd_start_replication.c's own header comment), so standby_init_database's pre-flight replication- slot check must be skipped rather than asked to verify a slot that will never exist, matching that function's own existing "initialising from another standby, no primary yet" precedent. Four further real, narrow gaps stood between that and actually working, each found by running the real `pg_basebackup`/`pg_autoctl` code paths end to end rather than by inspection: - pg_walsender's BASE_BACKUP had no manifest support (documented scope cut, cmd_base_backup.c), but PG13+ pg_basebackup requests one by default -- ReplicationSource.noManifest (pgsql.h) plus pg_basebackup() passing --no-manifest when set (pgctl.c) works around it for an archiver-sourced clone specifically, without touching a real primary's own backup path. - pgctl_identify_system() (pgctl.c) built its replication connection string with no dbname at all, relying on real pg_basebackup's and real walreceiver's own respective "default unset dbname to the literal 'replication'" behaviors -- neither of which this is: it's pg_auto_ failover's own raw libpq connection, which has no such default and instead falls back to plain libpq's own "dbname = username" rule (fe-connect.c), a route pg_walsender's routes file was never going to have an entry for. Passing "replication" explicitly matches what every other replication client already sends on the wire, and is a no-op against a real primary (which ignores dbname for replication=true connections regardless). - A "replay" base backup (basebackup_replay_mode, milestone 5) promotes a throwaway extracted copy to make it self-consistent, which genuinely puts it on a *later* timeline than whatever the archiver's own walcache has actually captured (which only ever advances on the real primary's timeline) -- serving that pairing breaks a real pg_basebackup's own timeline consistency check once it reaches its background WAL-streaming step ("starting timeline N is not present in the server", comparing the backup's own timeline against IDENTIFY_SYSTEM's). Fixed at the source: pgautofailover.get_latest_basebackup() grew an optional preferred_source filter (both SQL files), and service_archiver_serve.c's routes refresh now asks for 'live' specifically -- a live-sourced backup always shares the walcache's timeline by construction. A second, independent, defense-in-depth check (walcache_current_timeline(), comparing the walcache's own newest captured segment's embedded timeline against whatever's about to be advertised) keeps the routes file from ever serving a mismatched pairing even if that invariant is ever violated by a future backup mode. monitor_get_latest_basebackup_ info() also grew a timeline out-param, threaded into the routes file's own (previously unpopulated) "timeline" key -- already parsed by routes.c, never written by anyone until now. - cmd_start_replication.c ended a stream with bare CopyDone and nothing else. A real, long-lived streaming client (real walreceiver, via primary_conninfo) never triggers the gap because it never decides to stop on its own -- which is exactly why this went unnoticed through all of the earlier fast-forward-from-archiver verification. But pg_basebackup's --wal-method=stream background WAL receiver does decide to stop, once it reaches its own target LSN, and real receive- log.c's ReceiveXlogStream only accepts that as a *successful* stop when it can read a matching CommandComplete afterward (matching real walsender.c's own WalSndDone, which sends exactly that on controlled shutdown) -- without it, the client falls through to "unexpected termination of replication stream" and exits non-zero even though nothing was actually wrong on the wire. Fixed by sending a CommandComplete tagged "COPY" right after CopyDone. --- src/bin/common/pgctl.c | 27 +++- src/bin/common/pgsql.h | 11 ++ src/bin/pg_autoctl/cli_common.c | 9 ++ src/bin/pg_autoctl/cli_create_node.c | 6 +- src/bin/pg_autoctl/fsm_transition.c | 56 +++++++- src/bin/pg_autoctl/keeper.c | 46 ++++++ src/bin/pg_autoctl/keeper.h | 2 + src/bin/pg_autoctl/keeper_config.h | 11 ++ src/bin/pg_autoctl/monitor.c | 134 +++++++++++++++--- src/bin/pg_autoctl/monitor.h | 5 + src/bin/pg_autoctl/service_archiver.c | 2 +- .../pg_autoctl/service_archiver_basebackup.c | 43 +++--- src/bin/pg_autoctl/service_archiver_serve.c | 109 ++++++++++++++ src/bin/pg_walsender/cmd_start_replication.c | 18 +++ src/monitor/pgautofailover--2.2--2.3.sql | 66 ++++++++- src/monitor/pgautofailover.sql | 66 ++++++++- 16 files changed, 555 insertions(+), 56 deletions(-) diff --git a/src/bin/common/pgctl.c b/src/bin/common/pgctl.c index a8e5cad28..372ee1ee6 100644 --- a/src/bin/common/pgctl.c +++ b/src/bin/common/pgctl.c @@ -1264,7 +1264,8 @@ pg_basebackup(const char *pgdata, NodeAddress *primaryNode = &(replicationSource->primaryNode); char primaryConnInfo[MAXCONNINFO] = { 0 }; - char *args[18]; /* enough for all pg_basebackup flags incl. --checkpoint=fast */ + char *args[20]; /* enough for all pg_basebackup flags incl. --checkpoint=fast + * and --no-manifest */ int argsIndex = 0; char command[BUFSIZE]; @@ -1339,6 +1340,12 @@ pg_basebackup(const char *pgdata, args[argsIndex++] = replicationSource->slotName; } + /* see ReplicationSource.noManifest's own comment, pgsql.h */ + if (replicationSource->noManifest) + { + args[argsIndex++] = "--no-manifest"; + } + args[argsIndex] = NULL; /* @@ -2743,12 +2750,28 @@ pgctl_identify_system(ReplicationSource *replicationSource) char primaryConnInfoReplication[MAXCONNINFO] = { 0 }; PGSQL replicationClient = { 0 }; + /* + * Real Postgres ignores dbname for a replication=true connection (see + * libpqrcv_connect's own comment, libpqwalreceiver.c: "The database + * name is ignored by the server in replication mode, but specify + * 'replication' for .pgpass lookup"), so this is a no-op against a real + * primary. It is NOT a no-op against pg_walsender: unlike real + * walreceiver/pg_basebackup, which both default an unset dbname to the + * literal "replication" themselves (walreceiver hardcodes it; + * pg_basebackup's own GetConnection() does too), this is our own raw + * libpq connection with no such default applied for us -- leaving + * dbname unset here falls through to plain libpq's *own* default + * instead (dbname = the connection's user name, fe-connect.c), which + * pg_walsender's routes file was never going to have an entry for. + * Passing it explicitly matches what every other replication client + * already sends on the wire. + */ if (!prepare_primary_conninfo(primaryConnInfo, MAXCONNINFO, primaryNode->host, primaryNode->port, replicationSource->userName, - NULL, /* no database */ + "replication", replicationSource->password, replicationSource->applicationName, replicationSource->sslOptions, diff --git a/src/bin/common/pgsql.h b/src/bin/common/pgsql.h index f27c6f6e0..5edc42fcc 100644 --- a/src/bin/common/pgsql.h +++ b/src/bin/common/pgsql.h @@ -267,6 +267,17 @@ typedef struct ReplicationSource * for themselves when to promote, should leave this false. */ bool pauseAtRecoveryTarget; + + /* + * pg_walsender's BASE_BACKUP doesn't implement backup manifests yet + * (~/dev/temp/archiving-disaster-recovery.md's own documented scope for + * this milestone), which a real pg_basebackup requests by default from + * PG13+ -- set for an archiver-sourced base backup (create postgres + * --from-archiver) so pg_basebackup() knows to pass --no-manifest; + * false (the default) for a real primary/standby upstream, which does + * support manifests and should keep getting one. + */ + bool noManifest; SSLOptions sslOptions; IdentifySystem system; } ReplicationSource; diff --git a/src/bin/pg_autoctl/cli_common.c b/src/bin/pg_autoctl/cli_common.c index 709504b7c..360c47f56 100644 --- a/src/bin/pg_autoctl/cli_common.c +++ b/src/bin/pg_autoctl/cli_common.c @@ -99,6 +99,7 @@ cli_common_keeper_getopts(int argc, char **argv, /* force some non-zero default values */ LocalOptionConfig.monitorDisabled = false; + LocalOptionConfig.fromArchiver = false; LocalOptionConfig.groupId = -1; LocalOptionConfig.network_partition_timeout = -1; LocalOptionConfig.prepare_promotion_catchup = -1; @@ -471,6 +472,14 @@ cli_common_keeper_getopts(int argc, char **argv, break; } + case 'K': + { + /* { "from-archiver", no_argument, NULL, 'K' }, */ + LocalOptionConfig.fromArchiver = true; + log_trace("--from-archiver"); + break; + } + case 's': { /* { "ssl-self-signed", no_argument, NULL, 's' }, */ diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 82449eab0..95a196a3e 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -107,7 +107,8 @@ CommandLine create_postgres_command = KEEPER_CLI_SSL_OPTIONS " --candidate-priority priority of the node to be promoted to become primary\n" " --replication-quorum true if node participates in write quorum\n" - " --maximum-backup-rate maximum transfer rate of data transferred from the server during initial sync\n", + " --maximum-backup-rate maximum transfer rate of data transferred from the server during initial sync\n" + " --from-archiver bootstrap from a registered archiver's base backup and WAL cache\n", cli_create_postgres_getopts, cli_create_postgres); @@ -357,12 +358,13 @@ cli_create_postgres_getopts(int argc, char **argv) { "ssl-crl-file", required_argument, &ssl_flag, SSL_CRL_FILE_FLAG }, { "server-cert", required_argument, &ssl_flag, SSL_SERVER_CRT_FLAG }, { "server-key", required_argument, &ssl_flag, SSL_SERVER_KEY_FLAG }, + { "from-archiver", no_argument, NULL, 'K' }, { NULL, 0, NULL, 0 } }; int optind = cli_create_node_getopts(argc, argv, long_options, - "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RGVvqhP:r:xsN", + "C:D:H:p:l:U:A:SLd:a:n:f:m:MI:W:w:RGVvqhP:r:xsNK", &options); /* publish our option parsing in the global variable */ diff --git a/src/bin/pg_autoctl/fsm_transition.c b/src/bin/pg_autoctl/fsm_transition.c index 2a7cd7d30..58bb16864 100644 --- a/src/bin/pg_autoctl/fsm_transition.c +++ b/src/bin/pg_autoctl/fsm_transition.c @@ -925,20 +925,64 @@ fsm_init_standby(Keeper *keeper) NodeAddress *primaryNode = NULL; + /* + * `pg_autoctl create postgres --from-archiver`: bootstrap from a + * registered archiver's base backup + WAL cache instead of the group's + * live primary -- the disaster-recovery case this flag exists for. The + * archiver serves the same real replication protocol a live primary + * does (pg_walsender), so standby_init_replication_source/ + * standby_init_database below don't need to know the difference, except + * for one: pg_walsender has no slot-based retention in this milestone + * (see cmd_start_replication.c's own header comment), so we mustn't ask + * standby_init_database to first verify a replication slot exists on + * the archiver -- it never will. Passing an empty slot name here + * matches standby_init_database's own existing "initialising from + * another standby, no primary yet" precedent (see that function's + * comment on needsReplicationSlot). + */ + const char *slotName = config->replication_slot_name; - /* get the primary node to follow */ - if (!keeper_get_primary(keeper, &(postgres->replicationSource.primaryNode))) + if (config->fromArchiver) { - log_error("Failed to initialize standby for lack of a primary node, " - "see above for details"); - return false; + NodeAddress archiverNode = { 0 }; + bool found = false; + + if (!keeper_get_archiver_node(keeper, &archiverNode, &found)) + { + log_error("Failed to initialize standby from an archiver, " + "see above for details"); + return false; + } + + if (!found) + { + log_error("Failed to initialize standby from an archiver: " + "no archiver is registered for formation \"%s\" " + "group %d", config->formation, + keeper->state.current_group); + return false; + } + + postgres->replicationSource.primaryNode = archiverNode; + postgres->replicationSource.noManifest = true; + slotName = ""; + } + else + { + /* get the primary node to follow */ + if (!keeper_get_primary(keeper, &(postgres->replicationSource.primaryNode))) + { + log_error("Failed to initialize standby for lack of a primary node, " + "see above for details"); + return false; + } } if (!standby_init_replication_source(postgres, primaryNode, PG_AUTOCTL_REPLICA_USERNAME, config->replication_password, - config->replication_slot_name, + slotName, config->maximum_backup_rate, config->backupDirectory, NULL, /* no targetLSN */ diff --git a/src/bin/pg_autoctl/keeper.c b/src/bin/pg_autoctl/keeper.c index 13aa014c4..ddcdd4ee5 100644 --- a/src/bin/pg_autoctl/keeper.c +++ b/src/bin/pg_autoctl/keeper.c @@ -3395,6 +3395,52 @@ keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *upstreamNode, } +/* + * keeper_get_archiver_node fetches the ARCHIVING node registered for our + * (formation, group), for `create postgres --from-archiver` to bootstrap + * from -- deliberately not keeper_get_most_advanced_standby's election + * machinery (see monitor_get_archiver_node's own comment for why that + * function can't find an archiver outside of an election). Monitor-only: + * a brand new node discovering an archiver to rebuild from is exactly the + * disaster-recovery case --disable-monitor's manually-populated otherNodes + * list isn't meant to serve. + */ +bool +keeper_get_archiver_node(Keeper *keeper, NodeAddress *archiverNode, bool *found) +{ + KeeperConfig *config = &(keeper->config); + int groupId = keeper->state.current_group; + + if (config->monitorDisabled) + { + log_error("Failed to find an archiver to bootstrap from: " + "--from-archiver requires a monitor"); + return false; + } + + Monitor *monitor = &(keeper->monitor); + + if (!monitor_get_archiver_node(monitor, + config->formation, + groupId, + archiverNode, + found)) + { + log_error("Failed to get the archiver node from the monitor, " + "see above for details"); + return false; + } + + /* see keeper_get_most_advanced_standby's own comment on this sentinel */ + if (*found && archiverNode->port == 0) + { + archiverNode->port = PG_AUTOCTL_ARCHIVER_SERVE_PORT; + } + + return true; +} + + /* * keeper_pg_autoctl_get_version_from_disk calls pg_autoctl version --json and * parses the output to fill-in the keeper version. diff --git a/src/bin/pg_autoctl/keeper.h b/src/bin/pg_autoctl/keeper.h index d136998e7..2fdcd92d2 100644 --- a/src/bin/pg_autoctl/keeper.h +++ b/src/bin/pg_autoctl/keeper.h @@ -125,6 +125,8 @@ bool keeper_read_nodes_from_file(Keeper *keeper, NodeAddressArray *nodesArray); bool keeper_get_primary(Keeper *keeper, NodeAddress *primaryNode); bool keeper_get_most_advanced_standby(Keeper *keeper, NodeAddress *primaryNode, bool *found); +bool keeper_get_archiver_node(Keeper *keeper, NodeAddress *archiverNode, + bool *found); bool keeper_pg_autoctl_get_version_from_disk(Keeper *keeper, diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 04eb3d612..1e17fed1c 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -89,6 +89,17 @@ typedef struct KeeperConfig /* allow data loss during a perform failover operation */ bool allowDataLoss; + + /* + * `pg_autoctl create postgres --from-archiver`: bootstrap this standby + * from a registered archiver's base backup + WAL cache instead of from + * the group's live primary -- the disaster-recovery case where no live + * standby (or even primary) is left to clone from. Runtime-only, same + * as createAndRun (cli_common.c): only meaningful for the single + * in-process reach_initial_state() call `create postgres` itself makes, + * never persisted to the ini file. + */ + bool fromArchiver; } KeeperConfig; #define PG_AUTOCTL_MONITOR_IS_DISABLED(config) \ diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index 46d62b25d..a1f468197 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -848,6 +848,80 @@ monitor_get_most_advanced_standby(Monitor *monitor, } +/* + * monitor_get_archiver_node finds the ARCHIVING node for (formation, group), + * for a client-side bootstrap (create postgres --from-archiver) rather than + * an election: unlike monitor_get_most_advanced_standby, this doesn't filter + * on reportedstate = 'report_lsn' (a transient election-only state), since + * an archiver sits in its normal 'archiving' state outside of elections. + */ +bool +monitor_get_archiver_node(Monitor *monitor, + char *formation, int groupId, + NodeAddress *node, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT * FROM pgautofailover.get_archiver_node($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + const char *paramValues[2]; + + /* we expect zero or one entry */ + NodeAddressArray nodeArray = { 0 }; + NodeAddressArrayParseContext parseContext = { { 0 }, &nodeArray, false }; + + IntString groupIdString = intToString(groupId); + + paramValues[0] = formation; + paramValues[1] = groupIdString.strValue; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, parseNodeArray)) + { + log_error( + "Failed to get the archiver node in the HA group " + "from the monitor while running \"%s\" with " + "formation \"%s\" and group ID %d", + sql, formation, groupId); + return false; + } + + if (!parseContext.parsedOK) + { + log_error( + "Failed to get the archiver node from the monitor " + "while running \"%s\" with formation \"%s\" and group ID %d " + "because it returned an unexpected result. " + "See previous line for details.", + sql, formation, groupId); + return false; + } + + if (nodeArray.count == 0) + { + *found = false; + return true; + } + + /* copy the node we retrieved in the expected place */ + node->nodeId = nodeArray.nodes[0].nodeId; + strlcpy(node->name, nodeArray.nodes[0].name, _POSIX_HOST_NAME_MAX); + strlcpy(node->host, nodeArray.nodes[0].host, _POSIX_HOST_NAME_MAX); + node->port = nodeArray.nodes[0].port; + strlcpy(node->lsn, nodeArray.nodes[0].lsn, PG_LSN_MAXLENGTH); + node->isPrimary = nodeArray.nodes[0].isPrimary; + + log_debug("The archiver node for %s/%d is node " NODE_FORMAT, + formation, groupId, node->nodeId, node->name, + node->host, node->port); + + *found = true; + return true; +} + + /* * monitor_register_node performs the initial registration of a node with the * monitor in the given formation. @@ -974,6 +1048,7 @@ typedef struct BasebackupInfoParseContext int ntuples; char *storageLocation; char *source; + int timeline; } BasebackupInfoParseContext; @@ -993,9 +1068,11 @@ parseBasebackupInfo(void *ctx, PGresult *result) char *storageLocation = PQgetvalue(result, 0, 0); char *source = PQgetvalue(result, 0, 1); + char *timeline = PQgetvalue(result, 0, 2); context->storageLocation = strdup(storageLocation); context->source = strdup(source); + context->timeline = strtol(timeline, NULL, 10); context->parsedOk = context->storageLocation != NULL && context->source != NULL; @@ -1010,15 +1087,30 @@ parseBasebackupInfo(void *ctx, PGresult *result) /* * monitor_get_latest_basebackup_info calls * pgautofailover.get_latest_basebackup(formationId, groupId) and returns - * its storagelocation and source columns. *found is set to false (not an - * error) when the archiver hasn't taken a base backup for this group yet -- - * every caller must already tolerate that. + * its storagelocation, source, and timeline columns. *found is set to false + * (not an error) when the archiver hasn't taken a base backup for this + * group yet -- every caller must already tolerate that. + * + * timeline matters beyond metadata: pg_walsender's BASE_BACKUP response + * reads it straight out of the served backup_label (cmd_base_backup.c's + * own read_backup_label), and a real pg_basebackup's own background WAL + * streaming then requests exactly that timeline back via START_REPLICATION + * -- for a "replay" base backup (basebackup_replay_mode), which promotes a + * throwaway extracted copy to make it self-consistent, that's genuinely a + * *later* timeline than what the archiver's own captured WAL cache holds + * (which only ever advances on the real primary's timeline). Passing it + * through into the routes file's own "timeline" key (already parsed by + * routes.c, previously never written by anyone) is what lets pg_walsender + * serve a START_REPLICATION request consistent with whichever backup it + * just described, instead of always defaulting to timeline 1. */ bool monitor_get_latest_basebackup_info(Monitor *monitor, const char *formationId, int groupId, + const char *preferredSource, char *storageLocation, size_t storageLocationSize, char *source, size_t sourceSize, + int *timeline, bool *found) { PGSQL *pgsql = &monitor->pgsql; @@ -1031,15 +1123,22 @@ monitor_get_latest_basebackup_info(Monitor *monitor, * "IS NOT NULL" here, rather than trying to detect that NULL * composite downstream, is what makes context.ntuples == 0 below * an accurate "no backup yet" signal. + * + * preferredSource is passed as text plus an explicit cast (matching + * this file's own established pattern for enum parameters, e.g. + * monitor_register_node's use of ::pgautofailover.replication_ + * state below) rather than a raw enum OID -- NULL means "any", the + * function's own default. */ - "SELECT storagelocation, source::text " - " FROM pgautofailover.get_latest_basebackup($1, $2) " + "SELECT storagelocation, source::text, timeline " + " FROM pgautofailover.get_latest_basebackup(" + " $1, $2, $3::pgautofailover.basebackup_source) " " WHERE storagelocation IS NOT NULL"; - int paramCount = 2; - Oid paramTypes[2] = { TEXTOID, INT4OID }; + int paramCount = 3; + Oid paramTypes[3] = { TEXTOID, INT4OID, TEXTOID }; IntString groupIdString = intToString(groupId); - const char *paramValues[2] = { formationId, groupIdString.strValue }; - BasebackupInfoParseContext context = { { 0 }, false, 0, NULL, NULL }; + const char *paramValues[3] = { formationId, groupIdString.strValue, preferredSource }; + BasebackupInfoParseContext context = { { 0 }, false, 0, NULL, NULL, 0 }; *found = false; @@ -1068,6 +1167,7 @@ monitor_get_latest_basebackup_info(Monitor *monitor, strlcpy(storageLocation, context.storageLocation, storageLocationSize); strlcpy(source, context.source, sourceSize); + *timeline = context.timeline; free(context.storageLocation); free(context.source); *found = true; @@ -1116,15 +1216,15 @@ monitor_get_group_system_identifier(Monitor *monitor, &context, &parseSingleValueResult)) { log_error("Failed to get the system identifier for \"%s\"/%d " - "from the monitor", formationId, groupId); + "from the monitor", formationId, groupId); return false; } if (!context.parsedOk) { log_error("Failed to parse the system identifier returned by the " - "monitor for \"%s\"/%d, see above for details", - formationId, groupId); + "monitor for \"%s\"/%d, see above for details", + formationId, groupId); return false; } @@ -1166,7 +1266,7 @@ monitor_report_wal_received(Monitor *monitor, int64_t nodeId, NULL, NULL)) { log_error("Failed to report WAL file \"%s\" received for node %" - PRId64, walFileName, nodeId); + PRId64, walFileName, nodeId); return false; } @@ -1215,15 +1315,15 @@ monitor_report_basebackup_started(Monitor *monitor, int64_t archiverId, &context, &parseSingleValueResult)) { log_error("Failed to report the start of base backup \"%s\" to " - "the monitor", label); + "the monitor", label); return false; } if (!context.parsedOk) { log_error("Failed to report the start of base backup \"%s\" to " - "the monitor because it returned an unexpected result, " - "see previous lines for details", label); + "the monitor because it returned an unexpected result, " + "see previous lines for details", label); return false; } @@ -1261,7 +1361,7 @@ monitor_report_basebackup_completed(Monitor *monitor, int64_t basebackupId, NULL, NULL)) { log_error("Failed to report base backup %" PRId64 " as completed " - "to the monitor", basebackupId); + "to the monitor", basebackupId); return false; } diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index 39a0ee758..bb5384f96 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -160,8 +160,10 @@ bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, char *formation, int64_t *archiverNodeId); bool monitor_get_latest_basebackup_info(Monitor *monitor, const char *formationId, int groupId, + const char *preferredSource, char *storageLocation, size_t storageLocationSize, char *source, size_t sourceSize, + int *timeline, bool *found); bool monitor_get_group_system_identifier(Monitor *monitor, const char *formationId, int groupId, @@ -187,6 +189,9 @@ bool monitor_get_most_advanced_standby(Monitor *monitor, char *formation, int groupId, int64_t callerNodeId, NodeAddress *node, bool *found); +bool monitor_get_archiver_node(Monitor *monitor, + char *formation, int groupId, + NodeAddress *node, bool *found); bool monitor_register_node(Monitor *monitor, char *formation, char *name, diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index d49e55197..63f6d7070 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -467,7 +467,7 @@ service_archiver_update_current_lsn(Keeper *keeper) } wal_segment_end_lsn(best, keeper->postgres.currentLSN, - sizeof(keeper->postgres.currentLSN)); + sizeof(keeper->postgres.currentLSN)); } diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c index aee864027..5dd439c2b 100644 --- a/src/bin/pg_autoctl/service_archiver_basebackup.c +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -268,8 +268,8 @@ query_wal_position(const char *connInfo, bool isPrimary, } const char *sql = isPrimary - ? "SELECT pg_current_wal_lsn()::text" - : "SELECT pg_last_wal_replay_lsn()::text"; + ? "SELECT pg_current_wal_lsn()::text" + : "SELECT pg_last_wal_replay_lsn()::text"; SingleValueResultContext context = { { 0 }, PGSQL_RESULT_STRING, false }; bool result = pgsql_execute_with_params(&client, sql, 0, NULL, NULL, @@ -399,7 +399,7 @@ run_pg_basebackup(KeeperConfig *config, NodeAddress *source, if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { log_error("pg_basebackup failed while generating base backup \"%s\"", - label); + label); return false; } @@ -427,14 +427,14 @@ report_basebackup(Keeper *keeper, NodeAddress *endLsnSource, &timeline)) { log_error("Failed to read backup_label from \"%s\" after " - "pg_basebackup completed", backupDir); + "pg_basebackup completed", backupDir); return false; } if (!monitor_init(&(keeper->monitor), config->monitor_pguri)) { log_error("Failed to contact the monitor to report base backup " - "\"%s\"", label); + "\"%s\"", label); return false; } @@ -538,8 +538,8 @@ copy_directory_tree(const char *sourceDir, const char *destDir) if (!success) { log_error("cp -R -p \"%s\" \"%s\" failed: %s", - sourceDir, destDir, - program.stdErr != NULL ? program.stdErr : ""); + sourceDir, destDir, + program.stdErr != NULL ? program.stdErr : ""); } free_program(&program); @@ -694,7 +694,7 @@ stop_staging_postgres(void) if (kill(stagingPostgresPid, SIGTERM) != 0 && errno != ESRCH) { log_warn("Failed to send SIGTERM to the replay staging instance " - "(pid %d): %m", stagingPostgresPid); + "(pid %d): %m", stagingPostgresPid); } int status = 0; @@ -702,7 +702,7 @@ stop_staging_postgres(void) if (waitpid(stagingPostgresPid, &status, 0) == -1 && errno != ECHILD) { log_warn("Failed to wait for the replay staging instance " - "(pid %d) to stop: %m", stagingPostgresPid); + "(pid %d) to stop: %m", stagingPostgresPid); } stagingPostgresPid = -1; @@ -778,7 +778,7 @@ generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, if (directory_exists(stagingDir) && !rmtree(stagingDir, true)) { log_error("Failed to remove leftover replay staging directory " - "\"%s\" from a previous cycle", stagingDir); + "\"%s\" from a previous cycle", stagingDir); return false; } @@ -793,7 +793,7 @@ generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, if (!write_replay_recovery_config(stagingDir, config->pgSetup.pgdata)) { log_error("Failed to write replay recovery configuration in \"%s\"", - stagingDir); + stagingDir); return false; } @@ -815,8 +815,8 @@ generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, if (!ok) { log_error("Replay staging instance at \"%s\" failed to replay " - "available WAL and promote within %d seconds", - stagingDir, ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); + "available WAL and promote within %d seconds", + stagingDir, ARCHIVER_REPLAY_PROMOTE_TIMEOUT_SECONDS); } else { @@ -869,14 +869,17 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) bool found = false; char storageLocation[MAXPGPATH] = { 0 }; char latestSource[NAMEDATALEN] = { 0 }; + int latestTimeline = 0; if (!monitor_get_latest_basebackup_info(&(keeper->monitor), config->formation, config->groupId, + NULL, /* any source */ storageLocation, sizeof(storageLocation), latestSource, sizeof(latestSource), + &latestTimeline, &found)) { /* errors already logged */ @@ -910,9 +913,9 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) char label[NAMEDATALEN] = { 0 }; strftime(label, sizeof(label), - found ? "basebackup-replay-%Y%m%dT%H%M%SZ" - : "basebackup-%Y%m%dT%H%M%SZ", - &nowUTC); + found ? "basebackup-replay-%Y%m%dT%H%M%SZ" + : "basebackup-%Y%m%dT%H%M%SZ", + &nowUTC); char backupDir[MAXPGPATH] = { 0 }; @@ -961,15 +964,15 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) (void) set_ps_title("pg_autoctl: archiver basebackup"); bool ok = haveLiveSource - ? generate_live_basebackup(keeper, &liveSource, backupDir, label) - : generate_replay_basebackup(keeper, sourceBackupDir, - backupDir, label); + ? generate_live_basebackup(keeper, &liveSource, backupDir, label) + : generate_replay_basebackup(keeper, sourceBackupDir, + backupDir, label); exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); } log_debug("pg_autoctl archiver basebackup process started in " - "subprocess %d", pid); + "subprocess %d", pid); basebackupPid = pid; return true; diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c index 9de744cbb..bcf1deebf 100644 --- a/src/bin/pg_autoctl/service_archiver_serve.c +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -25,6 +25,8 @@ * */ +#include +#include #include #include #include @@ -43,6 +45,10 @@ /* how often service_archiver_serve_loop() re-checks pg_walsender's * liveness and refreshes the routes file, in seconds */ #define ARCHIVER_SERVE_TICK_SECONDS 1 + +/* matches service_archiver.c's own ARCHIVER_WAL_FNAME_LEN: a real WAL + * segment filename is 24 hex digits (8 TLI + 8 logId + 8 seg) */ +#define ARCHIVER_SERVE_WAL_FNAME_LEN 24 #define ARCHIVER_SERVE_ROUTES_REFRESH_TICKS 30 /* @@ -198,6 +204,65 @@ service_archiver_serve_start_walsender(Keeper *keeper) } +/* + * walcache_current_timeline scans walcacheDir for the newest captured WAL + * segment (same 24-hex-digit filename shape and sort order as service_ + * archiver.c's own is_wal_segment_filename/wal_filename_compare, matching + * pg_walsender/wal_dir_scan.c's own wal_dir_find_latest arithmetic for the + * same layout) and returns its embedded timeline (the filename's first 8 + * hex digits). Returns false (not an error) when the walcache has no + * complete segment yet -- too early to know, not "timeline 0". + */ +static bool +walcache_current_timeline(const char *walcacheDir, int *timeline) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char best[ARCHIVER_SERVE_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + size_t len = strlen(entry->d_name); + bool isWalSegment = (len == ARCHIVER_SERVE_WAL_FNAME_LEN); + + for (size_t i = 0; isWalSegment && i < len; i++) + { + isWalSegment = isxdigit((unsigned char) entry->d_name[i]); + } + + if (!isWalSegment) + { + continue; + } + + if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + { + strlcpy(best, entry->d_name, sizeof(best)); + } + } + + closedir(dir); + + if (best[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + + memcpy(tliHex, best, 8); + *timeline = (int) strtol(tliHex, NULL, 16); + + return true; +} + + bool service_archiver_serve_refresh_routes(Keeper *keeper) { @@ -205,15 +270,36 @@ service_archiver_serve_refresh_routes(Keeper *keeper) char basebackupLocation[MAXPGPATH] = { 0 }; char basebackupSource[NAMEDATALEN] = { 0 }; + int basebackupTimeline = 0; bool found = false; + /* + * preferredSource = "live": a "replay" base backup (basebackup_replay_ + * mode) promotes a throwaway extracted copy to make it self- + * consistent, which genuinely puts it on a *later* timeline than + * whatever the walcache itself has captured (which only ever advances + * on the real primary's own timeline). A real pg_basebackup rejects + * that combination outright once it reaches its own background WAL + * streaming step ("starting timeline N is not present in the server", + * receivelog.c comparing the backup's own timeline against IDENTIFY_ + * SYSTEM's -- and IDENTIFY_SYSTEM itself correctly reports the + * walcache's real captured timeline, see cmd_identify_system.c). A + * "live" backup is taken directly from the actively-followed primary, + * so it always shares the walcache's timeline by construction -- ask + * for one specifically rather than "whatever is newest regardless of + * type", which would otherwise serve an unusable pairing as soon as a + * newer 'replay' backup exists (get_latest_basebackup's own comment, + * pgautofailover.sql). + */ if (!monitor_get_latest_basebackup_info(&(keeper->monitor), config->formation, config->groupId, + "live", basebackupLocation, sizeof(basebackupLocation), basebackupSource, sizeof(basebackupSource), + &basebackupTimeline, &found)) { log_warn("Failed to fetch the latest base backup location from the " @@ -221,6 +307,28 @@ service_archiver_serve_refresh_routes(Keeper *keeper) found = false; } + /* + * Defense in depth against the same mismatch, in case a future + * 'live'-sourced backup mode is ever added that doesn't actually + * guarantee walcache-timeline compatibility: never advertise a pairing + * we can independently tell apart, even though preferredSource = + * "live" above should already make this unreachable today. + */ + if (found) + { + int walcacheTimeline = 0; + + if (walcache_current_timeline(config->pgSetup.pgdata, &walcacheTimeline) && + walcacheTimeline != basebackupTimeline) + { + log_warn("The latest base backup is on timeline %d, but the " + "walcache is capturing timeline %d; omitting the base " + "backup from the routes file until they match", + basebackupTimeline, walcacheTimeline); + found = false; + } + } + uint64_t systemIdentifier = 0; bool foundSystemIdentifier = false; @@ -257,6 +365,7 @@ service_archiver_serve_refresh_routes(Keeper *keeper) if (found) { fformat(fileStream, "basebackup = %s\n", basebackupLocation); + fformat(fileStream, "timeline = %d\n", basebackupTimeline); } if (foundSystemIdentifier) diff --git a/src/bin/pg_walsender/cmd_start_replication.c b/src/bin/pg_walsender/cmd_start_replication.c index cb7cae147..d6c44f828 100644 --- a/src/bin/pg_walsender/cmd_start_replication.c +++ b/src/bin/pg_walsender/cmd_start_replication.c @@ -402,6 +402,24 @@ cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) (void) ws_send_copy_done(sock); + /* + * Real walsender.c's own controlled-shutdown path (WalSndDone) follows + * CopyDone with a CommandComplete tagged "COPY" before returning to + * the command loop -- required protocol, not optional decoration: a + * real client's receivelog.c (ReceiveXlogStream) only accepts an + * ended stream as a *successful* stop when it can read a matching + * PGRES_COMMAND_OK result afterward; without it, a client that decided + * on its own to stop here (e.g. pg_basebackup's --wal-method=stream + * background receiver, once it reaches its target LSN) falls through + * to "unexpected termination of replication stream" and exits + * non-zero, even though nothing on the wire was actually wrong. A + * genuinely long-lived streaming client (real walreceiver, primary_ + * conninfo) never triggers this path at all -- it never decides to + * stop on its own -- which is why this went unnoticed until a real + * pg_basebackup was tested end to end. + */ + (void) ws_send_command_complete(sock, "COPY"); + log_info("START_REPLICATION: stream ended at %X/%08X", (uint32_t) (currentLsn >> 32), (uint32_t) currentLsn); } diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index cc8531954..1ca777511 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -1763,21 +1763,42 @@ grant execute on function -- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only -- ever granted EXECUTE on the function, never SELECT on pgautofailover. -- basebackup itself. -CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) +-- +-- preferred_source (default NULL, meaning "any") exists for service_ +-- archiver_serve.c's own routes-file refresh: a 'replay' backup promotes a +-- throwaway extracted copy, which genuinely puts it on a *later* timeline +-- than whatever the archiver's own walcache has actually captured (which +-- only ever advances on the real primary's timeline) -- serving that pair +-- together breaks a real pg_basebackup's own timeline consistency check +-- (receivelog.c). Since a 'live' backup is taken directly from the +-- actively-followed primary, it always shares the walcache's timeline by +-- construction; passing preferred_source = 'live' is how the routes +-- refresh asks for one specifically, rather than "whatever is newest +-- regardless of type". +CREATE FUNCTION pgautofailover.get_latest_basebackup + ( + formationid text, + groupid int, + preferred_source pgautofailover.basebackup_source default NULL + ) RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup b WHERE b.formationid = get_latest_basebackup.formationid AND b.groupid = get_latest_basebackup.groupid AND b.status = 'complete' + AND (get_latest_basebackup.preferred_source IS NULL + OR b.source = get_latest_basebackup.preferred_source) ORDER BY lower(b.period) DESC LIMIT 1; $$; -comment on function pgautofailover.get_latest_basebackup(text,int) - is 'fetch the most recent complete base backup for (formation, group)'; +comment on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + is 'fetch the most recent complete base backup for (formation, group), optionally filtered to one source'; -grant execute on function pgautofailover.get_latest_basebackup(text,int) +grant execute on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) to autoctl_node; -- an archiving node has no sysidentifier of its own (haspgdata = false, @@ -1807,6 +1828,43 @@ comment on function pgautofailover.get_group_system_identifier(text,int) grant execute on function pgautofailover.get_group_system_identifier(text,int) to autoctl_node; +-- `create postgres --from-archiver` needs the ARCHIVING row itself, not +-- get_most_advanced_standby()'s election-only pool: that function filters +-- on reportedstate = 'report_lsn', a transient state a group's ARCHIVING +-- node only visits during a FAST_FORWARD election, never during its normal +-- steady-state operation (reportedstate = 'archiving'). node_port is the +-- port == 0 sentinel documented on get_most_advanced_standby's own C +-- caller (keeper_get_most_advanced_standby, keeper.c) -- resolving it to +-- the archiver's real pg_walsender serve port is this milestone's C +-- caller's job too, same pattern. +CREATE FUNCTION pgautofailover.get_archiver_node + ( + IN formationid text default 'default', + IN groupid int default 0, + OUT node_id bigint, + OUT node_name text, + OUT node_host text, + OUT node_port int, + OUT node_lsn pg_lsn, + OUT node_is_primary bool + ) +RETURNS SETOF record LANGUAGE SQL STRICT +AS $$ + select nodeid, nodename, nodehost, nodeport, reportedlsn, false + from pgautofailover.node + where formationid = $1 + and groupid = $2 + and reportedstate = 'archiving' + order by nodeid + limit 1; +$$; + +comment on function pgautofailover.get_archiver_node(text,int) + is 'fetch the ARCHIVING node for (formation, group), for create postgres --from-archiver to bootstrap from'; + +grant execute on function pgautofailover.get_archiver_node(text,int) + to autoctl_node; + -- for kind = 'warm-standby': raises if the owning archiver is already at -- its maxresidentreplay cap CREATE FUNCTION pgautofailover.create_archiver_node diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index a0a98884d..f2b3a7d98 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -2345,21 +2345,42 @@ grant execute on function -- SELECT grant on (e.g. archiver_add_formation) -- autoctl_node is only -- ever granted EXECUTE on the function, never SELECT on pgautofailover. -- basebackup itself. -CREATE FUNCTION pgautofailover.get_latest_basebackup(formationid text, groupid int) +-- +-- preferred_source (default NULL, meaning "any") exists for service_ +-- archiver_serve.c's own routes-file refresh: a 'replay' backup promotes a +-- throwaway extracted copy, which genuinely puts it on a *later* timeline +-- than whatever the archiver's own walcache has actually captured (which +-- only ever advances on the real primary's timeline) -- serving that pair +-- together breaks a real pg_basebackup's own timeline consistency check +-- (receivelog.c). Since a 'live' backup is taken directly from the +-- actively-followed primary, it always shares the walcache's timeline by +-- construction; passing preferred_source = 'live' is how the routes +-- refresh asks for one specifically, rather than "whatever is newest +-- regardless of type". +CREATE FUNCTION pgautofailover.get_latest_basebackup + ( + formationid text, + groupid int, + preferred_source pgautofailover.basebackup_source default NULL + ) RETURNS pgautofailover.basebackup LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup b WHERE b.formationid = get_latest_basebackup.formationid AND b.groupid = get_latest_basebackup.groupid AND b.status = 'complete' + AND (get_latest_basebackup.preferred_source IS NULL + OR b.source = get_latest_basebackup.preferred_source) ORDER BY lower(b.period) DESC LIMIT 1; $$; -comment on function pgautofailover.get_latest_basebackup(text,int) - is 'fetch the most recent complete base backup for (formation, group)'; +comment on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) + is 'fetch the most recent complete base backup for (formation, group), optionally filtered to one source'; -grant execute on function pgautofailover.get_latest_basebackup(text,int) +grant execute on function pgautofailover.get_latest_basebackup + (text,int,pgautofailover.basebackup_source) to autoctl_node; -- an archiving node has no sysidentifier of its own (haspgdata = false, @@ -2389,6 +2410,43 @@ comment on function pgautofailover.get_group_system_identifier(text,int) grant execute on function pgautofailover.get_group_system_identifier(text,int) to autoctl_node; +-- `create postgres --from-archiver` needs the ARCHIVING row itself, not +-- get_most_advanced_standby()'s election-only pool: that function filters +-- on reportedstate = 'report_lsn', a transient state a group's ARCHIVING +-- node only visits during a FAST_FORWARD election, never during its normal +-- steady-state operation (reportedstate = 'archiving'). node_port is the +-- port == 0 sentinel documented on get_most_advanced_standby's own C +-- caller (keeper_get_most_advanced_standby, keeper.c) -- resolving it to +-- the archiver's real pg_walsender serve port is this milestone's C +-- caller's job too, same pattern. +CREATE FUNCTION pgautofailover.get_archiver_node + ( + IN formationid text default 'default', + IN groupid int default 0, + OUT node_id bigint, + OUT node_name text, + OUT node_host text, + OUT node_port int, + OUT node_lsn pg_lsn, + OUT node_is_primary bool + ) +RETURNS SETOF record LANGUAGE SQL STRICT +AS $$ + select nodeid, nodename, nodehost, nodeport, reportedlsn, false + from pgautofailover.node + where formationid = $1 + and groupid = $2 + and reportedstate = 'archiving' + order by nodeid + limit 1; +$$; + +comment on function pgautofailover.get_archiver_node(text,int) + is 'fetch the ARCHIVING node for (formation, group), for create postgres --from-archiver to bootstrap from'; + +grant execute on function pgautofailover.get_archiver_node(text,int) + to autoctl_node; + -- for kind = 'warm-standby': raises if the owning archiver is already at -- its maxresidentreplay cap CREATE FUNCTION pgautofailover.create_archiver_node From 473595745a4f0e457c223c6f0babc7118c1f5359 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Tue, 4 Aug 2026 18:04:55 +0200 Subject: [PATCH 20/55] pgaftest: spec for --from-archiver bootstrap + fast-forward-from-archiver Adds archiver_bootstrap_and_fast_forward.pgaf, the disaster-recovery scenario this whole investigation was driven by: a primary, an archiver, and a secondary that's created via `pg_autoctl create postgres --from-archiver` (not from the live primary) after the archiver's first live base backup is ready, then a FAST_FORWARD election where the archiver is the only node with the WAL the winning candidate is missing. node2 is declared `create and launch deferred`: the normal ini-driven node bring-up (`pg_autoctl node start`) has no hook for a custom flag like --from-archiver (NodeSpec/nodespec.c carries no such field -- fromArchiver lives only in KeeperConfig, populated exclusively by cli_create_node.c's own direct CLI parsing), so test_001 `exec`s into node2's own container and runs `pg_autoctl create postgres --from-archiver` by hand, then backgrounds `pg_autoctl run` the same way debug_citus_worker_switchover.pgaf backgrounds a long-lived process (`bash -c "nohup ... &"` -- a foreground `pg_autoctl run` would hang `docker compose exec -T` forever otherwise). test_002-004 engineer a real WAL gap rather than relying on race timing: stop node2 so it can't stream from node1 anymore, generate more WAL on the primary and give the archiver (still capturing independently via pg_receivewal) time to land it, kill the primary, then bring node2 back -- at that point the archiver is strictly ahead of node2 and is the only viable FAST_FORWARD WAL source. The final row-count check on node2 post-promotion confirms real WAL bytes were fetched and applied, not just that the FSM passed through the right state label. Verified: `pgaftest show spec`/`show compose` parse this spec cleanly (exit 0) and `pgaftest indent` round-trips it losslessly, confirming the DSL usage (deferred node declaration, exec/nohup backgrounding, multi-state `passing through` clause) is syntactically valid against the real grammar. Could not run it against a live docker compose cluster in this session: `make -f Makefile.docker build-pg17` fails fetching ghcr.io/hapostgres/pg_auto_failover/pgaf-base (401 Unauthorized, no registry credentials available here), and no local base image is cached to build from instead. Every C-level behavior this spec exercises (--from-archiver's own bootstrap, and fast-forward sourcing WAL from an archiver) was independently verified working end-to-end by hand against a real cluster in the two preceding commits on this branch. --- tests/tap/schedule | 1 + .../archiver_bootstrap_and_fast_forward.pgaf | 124 ++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf diff --git a/tests/tap/schedule b/tests/tap/schedule index 9083e4280..aa6d8b3dc 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -33,6 +33,7 @@ wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock archiver_wal_capture archiver_basebackup_generation +archiver_bootstrap_and_fast_forward keeper_fsm_gap_209_wait_maintenance keeper_fsm_gap_211_wait_maintenance keeper_fsm_gap_209_wait_standby diff --git a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf new file mode 100644 index 000000000..db7013a45 --- /dev/null +++ b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf @@ -0,0 +1,124 @@ +# Archiving & Disaster Recovery: `pg_autoctl create postgres --from-archiver` +# (bootstrap a brand new standby from the archiver's own base backup + WAL +# cache instead of the group's live primary), followed by a FAST_FORWARD +# election where the archiver is the only node with the WAL the winning +# candidate is missing. +# +# node2 is declared `create and launch deferred` (compose_gen.c's normal +# per-node command is `pg_autoctl node run `, which just spin-polls +# the ini forever while deferred) so its container starts but never runs +# the normal, ini-driven `create postgres` -- that path has no hook for +# custom flags like --from-archiver (see nodespec.c: NodeSpec has no +# fromArchiver field, only KeeperConfig does, populated exclusively by +# cli_create_node.c's own direct CLI parsing). test_001 instead `exec`s +# into node2's own container and runs `pg_autoctl create postgres +# --from-archiver` by hand, then backgrounds `pg_autoctl run` the same way +# debug_citus_worker_switchover.pgaf backgrounds a long-lived command +# (`bash -c "nohup ... &"` -- `docker compose exec -T` blocks until its +# argv exits, so a foreground `pg_autoctl run` would hang the test step +# forever without this). +# +# test_002/003/004 engineer an actual WAL gap rather than relying on race +# timing: node2 is stopped (so it can't stream anything further from +# node1), more WAL is generated and given time to land in the archiver's +# walcache, *then* node1 is killed and node2 brought back -- at that +# point node2 is the only live standby-kind candidate, the archiver is +# strictly ahead of it, and get_most_advanced_standby() (already proven +# in this milestone's own manual testing to resolve an ARCHIVING row's +# port == 0 sentinel to the archiver's real serve port, keeper.c) selects +# the archiver as fast_forward's WAL source. test_004's final row-count +# check on node2 (post-promotion) confirms real WAL bytes were actually +# fetched and applied, not just that the FSM passed through the right +# state label. +# +# Predecessor: archiver_basebackup_generation.pgaf (M5, base backup +# generation this spec's test_001 depends on already being ready). + +cluster { + monitor + formation { + node1 + archiver1 archiver + node2 create and launch deferred + } +} + +setup { + wait until node1 state is single timeout 60s + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: wait for the archiver's live base backup (get_latest_basebackup's +# preferred_source = 'live' overload -- see pgautofailover.sql -- +# is what service_archiver_serve.c's own routes refresh asks for; +# a plain 'live' check here is the same thing this spec can +# observe from the monitor side), then bootstrap node2 from it. +# + +step test_001_bootstrap_secondary_from_archiver { + sleep 30s + sql monitor { + SELECT source::text, status::text + FROM pgautofailover.get_latest_basebackup('default', 0, 'live'); + } + expect { live|complete } + exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --ssl-self-signed --name node2 --hostname node2 --from-archiver + exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" + wait until node2 state is secondary + passing through catchingup + timeout 90s +} + +# +# test_002: stop node2 so it can no longer stream from node1, then generate +# more WAL on the primary and give the archiver (still capturing +# independently via pg_receivewal) time to land it. node2 now +# knows nothing about this WAL; the archiver does. +# + +step test_002_stop_secondary_and_advance_primary { + compose stop node2 + wait until node2 stopped timeout 60s + sql node1 { CREATE TABLE archiver_ff_probe(a int); } + sql node1 { + INSERT INTO archiver_ff_probe SELECT generate_series(1, 1000); + } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s +} + +# +# test_003: kill the primary. node2 is still stopped, so at this instant the +# archiver is the only node in the group with any of test_002's +# WAL -- node1 is gone, node2 never received it. +# + +step test_003_kill_primary_leaving_archiver_only { + compose kill node1 + wait until node1 assigned-state = draining timeout 120s +} + +# +# test_004: bring node2 back. It reports in behind the archiver, the monitor +# assigns fast_forward with the archiver as WAL source, node2 +# fetches the missing WAL from it (standby_fetch_missing_wal, +# already proven against a real archiver during this milestone's +# own development), and promotes. The row count on the other side +# confirms the fetched WAL was real and got applied, not just that +# the FSM label passed through fast_forward. +# + +step test_004_bring_back_secondary_and_fast_forward { + compose start node2 + wait until node2 state is primary + passing through report_lsn, fast_forward, prepare_promotion, wait_primary + timeout 180s + sql node2 { SELECT count(*) FROM archiver_ff_probe; } + expect { 1000 } +} From 29bf82b0b56192c67763780355357f8b082ed835 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 00:38:06 +0200 Subject: [PATCH 21/55] monitor: fix missing SECURITY DEFINER on wal_archived() wal_archived() is a plain LANGUAGE sql function (not SECURITY DEFINER), so it runs under the caller's own privileges. autoctl_node never got a direct SELECT grant on pgautofailover.archiver_wal: the blanket `GRANT SELECT ON ALL TABLES IN SCHEMA pgautofailover TO autoctl_node` only covers tables that already existed when that statement ran, and archiver_wal (like every other table in the M1 archiving schema) was created after it. Confirmed live: calling wal_archived() as autoctl_node (the role node_active() actually uses) failed with "permission denied for table archiver_wal". get_latest_basebackup() had this exact same bug, already fixed the same way (SECURITY DEFINER) in a prior commit -- apply the same fix here. --- src/monitor/pgautofailover--2.2--2.3.sql | 2 +- src/monitor/pgautofailover.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index 1ca777511..b9b6e6b14 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -1538,7 +1538,7 @@ grant execute on function pgautofailover.get_archiver_policy(text,int) CREATE FUNCTION pgautofailover.wal_archived (formationid text, groupid int, walfilename text) RETURNS bool - LANGUAGE sql STABLE + LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT count(DISTINCT aw.archiverid) >= (SELECT archiverquorum diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index f2b3a7d98..ce76ffef0 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -2120,7 +2120,7 @@ grant execute on function pgautofailover.get_archiver_policy(text,int) CREATE FUNCTION pgautofailover.wal_archived (formationid text, groupid int, walfilename text) RETURNS bool - LANGUAGE sql STABLE + LANGUAGE sql STABLE SECURITY DEFINER AS $$ SELECT count(DISTINCT aw.archiverid) >= (SELECT archiverquorum From 438998b6e81a8eea0249ac51f87a6fe2558b3681 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 01:54:46 +0200 Subject: [PATCH 22/55] monitor: don't let an archiver block a lone primary reaching PRIMARY BuildForPrimaryNodeNodeActiveContext() counted every other node in the group toward replicationQuorumCount/secondaryNodesCount/ secondaryQuorumNodesCount, including ARCHIVING rows -- which are never real Postgres secondaries and can never report SECONDARY. In a formation with only a primary and an archiver, that miscount let the archiver's own bootstrap WAIT_STANDBY reading trip anyOtherNodeWaitingStandby (pos 401) and bump the primary off SINGLE, while secondaryQuorumNodesCount could then never legitimately reach zero -- so the primary got stuck between SINGLE and PRIMARY forever. Skip ARCHIVING (hasPgData=false) rows in that loop, matching the hasPgData-based exclusion this file's own REPORTING_NODE section already applies for a different purpose. Also adds the archiver-mirror FSM rows (pos 394/396/399) their own SINGLE|WAIT_PRIMARY|JOIN_PRIMARY match set, since a primary attached only to an archiver legitimately stays SINGLE the whole time instead of ever reaching WAIT_PRIMARY. --- src/monitor/expected/fsm.out | 45 +++++++---- src/monitor/group_state_machine.c | 123 ++++++++++++++++++++++++------ 2 files changed, 131 insertions(+), 37 deletions(-) diff --git a/src/monitor/expected/fsm.out b/src/monitor/expected/fsm.out index bf1c41d0c..75d7899cb 100644 --- a/src/monitor/expected/fsm.out +++ b/src/monitor/expected/fsm.out @@ -852,7 +852,7 @@ section_path | reporting_node.ms_failover.candidate_join active_node_current_state | report_lsn other_node_current_state | candidate_node_current_state | -active_node_conditions | +active_node_conditions | hasPgData=true other_node_conditions | candidate_node_conditions | isReadyToStreamWAL=true group_conditions | candidatePromotionInProgress=true @@ -1075,7 +1075,7 @@ pos | 394 section | reporting_node section_path | reporting_node.from_context active_node_current_state | report_lsn -other_node_current_state | wait_primary, join_primary +other_node_current_state | single, wait_primary, join_primary candidate_node_current_state | active_node_conditions | hasPgData=false other_node_conditions | isHealthy=true @@ -1084,7 +1084,7 @@ group_conditions | active_node_assigned_state | archiving other_node_assigned_state | has_extra_action | f -comment | archiver mirror of pos 307: report_lsn, primary converged wait/join_primary, healthy -> archiving +comment | archiver mirror of pos 307: report_lsn, primary converged single/wait/join_primary, healthy -> archiving -[ RECORD 72 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 395 section | reporting_node @@ -1105,7 +1105,7 @@ pos | 396 section | reporting_node section_path | reporting_node.from_context active_node_current_state | wait_standby -other_node_current_state | wait_primary, join_primary +other_node_current_state | single, wait_primary, join_primary candidate_node_current_state | active_node_conditions | hasPgData=false other_node_conditions | @@ -1114,7 +1114,7 @@ group_conditions | active_node_assigned_state | archiving other_node_assigned_state | has_extra_action | f -comment | archiver mirror of pos 315: wait_standby, primary converged wait/join_primary -> archiving +comment | archiver mirror of pos 315: wait_standby, primary converged single/wait/join_primary -> archiving -[ RECORD 74 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 397 section | reporting_node @@ -1146,6 +1146,21 @@ other_node_assigned_state | has_extra_action | f comment | archiver mirror of pos 319: wait_standby (not a quorum member), primary converged primary -> archiving -[ RECORD 76 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +pos | 399 +section | reporting_node +section_path | reporting_node.ms_failover.candidate_join +active_node_current_state | report_lsn +other_node_current_state | +candidate_node_current_state | +active_node_conditions | hasPgData=false +other_node_conditions | +candidate_node_conditions | isReadyToStreamWAL=true +group_conditions | candidatePromotionInProgress=true +active_node_assigned_state | archiving +other_node_assigned_state | +has_extra_action | f +comment | archiver mirror of pos 365: MS-failover, activeNode in report_lsn, failover candidate ready to stream WAL -> archiving (no join_secondary detour -- see pos 365's own comment) +-[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 401 section | primary_node section_path | primary_node @@ -1160,7 +1175,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | has_extra_action | f comment | primary alone, another node reached wait_standby -> wait_primary --[ RECORD 77 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 403 section | primary_node section_path | primary_node @@ -1175,7 +1190,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, zero secondaries -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 78 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 405 section | primary_node section_path | primary_node @@ -1190,7 +1205,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | all nodes async, >=1 secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 79 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 407 section | primary_node section_path | primary_node @@ -1205,7 +1220,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys=0 -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 80 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 409 section | primary_node section_path | primary_node @@ -1220,7 +1235,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/apply_settings, no quorum secondaries, no failover in progress, number_sync_standbys>0 -> primary (block writes) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 81 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 82 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 411 section | primary_node section_path | primary_node @@ -1235,7 +1250,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | wait_primary, >=1 quorum secondary -> primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 82 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 83 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 413 section | primary_node section_path | primary_node @@ -1250,7 +1265,7 @@ active_node_assigned_state | wait_primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, both zero -> wait_primary (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 83 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 84 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 415 section | primary_node section_path | primary_node @@ -1265,7 +1280,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, number_sync_standbys != 0 -> primary (1 of 2 disjuncts) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 84 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 85 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 417 section | primary_node section_path | primary_node @@ -1280,7 +1295,7 @@ active_node_assigned_state | primary other_node_assigned_state | catchingup has_extra_action | f comment | apply_settings, sync_standbys=0 but >=1 quorum secondary -> primary (2 of 2) (+ unhealthy-secondary fan-out to catchingup) --[ RECORD 85 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 86 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 419 section | primary_node section_path | primary_node @@ -1295,7 +1310,7 @@ active_node_assigned_state | other_node_assigned_state | catchingup has_extra_action | f comment | converged primary/wait_primary/apply_settings, no other condition applies -> no-op besides the unhealthy-secondary fan-out to catchingup --[ RECORD 86 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +-[ RECORD 87 ]---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- pos | 421 section | primary_node section_path | primary_node diff --git a/src/monitor/group_state_machine.c b/src/monitor/group_state_machine.c index 52ee5dd7e..9e751f429 100644 --- a/src/monitor/group_state_machine.c +++ b/src/monitor/group_state_machine.c @@ -292,6 +292,27 @@ static const NodeStatePattern FSM_WAIT_OR_JOIN_PRIMARY = { REPLICATION_STATE_JOIN_PRIMARY), }; +/* + * FSM_WAIT_OR_JOIN_PRIMARY plus SINGLE -- used only by the archiver mirror + * rows (pos 394/396), never by their ordinary hasPgData=true siblings (pos + * 307/315): a real secondary joining a lone primary always first bumps that + * primary from SINGLE to WAIT_PRIMARY (pos 401, "primary alone, another node + * reached wait_standby"), so pos 307/315 never actually need to match SINGLE + * themselves. An archiver attaching to a lone primary is different -- since + * BuildForPrimaryNodeNodeActiveContext excludes archiver rows from ever + * triggering that same pos 401 bump (an archiver isn't a quorum-eligible + * node kind, see that function's own comment), the primary legitimately + * stays SINGLE the entire time the archiver is only being watched by it. + * Without SINGLE in this set, an archiver attached to a genuinely + * single-node formation could never leave WAIT_STANDBY/REPORT_LSN at all. + */ +static const NodeStatePattern FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY = { + .kind = NODE_STATE_STABLE, + .reportedStates = STATES(REPLICATION_STATE_SINGLE, + REPLICATION_STATE_WAIT_PRIMARY, + REPLICATION_STATE_JOIN_PRIMARY), +}; + /* * the "primary role" states MONITOR_FSM_SECTION_PRIMARY_NODE's own rows * match against -- a different three-element set from @@ -1904,12 +1925,26 @@ BuildFromContextNodeActiveContext(GroupStateContext *ctx, AutoFailoverNode *prim /* * BuildForPrimaryNodeNodeActiveContext computes every fact SectionPrimaryNode - * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other node in - * the primary's group, using the same OtherNodeIsDueForCatchingUp() test - * OtherNodesDueForCatchingUp() (above) uses for its own fan-out, to derive - * the group-level counts (replicationQuorumCount, secondaryNodesCount, - * secondaryQuorumNodesCount) and the anyOtherNodeWaitingStandby flag those - * rows match against. + * (MonitorFSM[]'s pos 401-421 rows) needs: it loops over every other *real* + * (hasPgData) node in the primary's group, using the same OtherNodeIsDueFor + * CatchingUp() test OtherNodesDueForCatchingUp() (above) uses for its own + * fan-out, to derive the group-level counts (replicationQuorumCount, + * secondaryNodesCount, secondaryQuorumNodesCount) and the anyOtherNode + * WaitingStandby flag those rows match against. + * + * An ARCHIVING node is skipped entirely here (see the hasPgData check inside + * the loop below): it is never a real Postgres secondary participating in + * synchronous-replication quorum, and it can never reach reported SECONDARY + * state. Counting it like an ordinary node would let it single-handedly + * block this primary's own SINGLE -> WAIT_PRIMARY -> PRIMARY progression -- + * anyOtherNodeWaitingStandby would fire (pos 401) the moment the archiver's + * own bootstrap briefly passes through WAIT_STANDBY, bumping the primary off + * SINGLE, and it could then never reach PRIMARY since secondaryQuorumNodes + * Count could never legitimately drop to zero via an archiver's own reported + * state. Same hasPgData-based exclusion this file's own REPORTING_NODE + * section already applies for a different purpose (pos 365/399's own + * comment) -- an archiver simply isn't a quorum-eligible node kind, in + * either section. */ static void BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, @@ -1927,11 +1962,10 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, */ List *otherNodesGroupList = AutoFailoverOtherNodesList(primaryNode); - int otherNodesCount = list_length(otherNodesGroupList); - int replicationQuorumCount = otherNodesCount; - int secondaryNodesCount = otherNodesCount; - int secondaryQuorumNodesCount = otherNodesCount; + int replicationQuorumCount = 0; + int secondaryNodesCount = 0; + int secondaryQuorumNodesCount = 0; ListCell *nodeCell = NULL; @@ -1939,6 +1973,16 @@ BuildForPrimaryNodeNodeActiveContext(GroupStateContext *ctx, { AutoFailoverNode *otherNode = (AutoFailoverNode *) lfirst(nodeCell); + if (!otherNode->hasPgData) + { + /* an ARCHIVING row -- see this function's own header comment */ + continue; + } + + ++replicationQuorumCount; + ++secondaryNodesCount; + ++secondaryQuorumNodesCount; + if (OtherNodeIsDueForCatchingUp(ctx, otherNode)) { --secondaryNodesCount; @@ -3221,6 +3265,21 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * MS-failover: candidate ready to stream WAL -> follower joins as secondary */ + + /* + * hasPgData = BOOL_TRUE restricts this to ordinary nodes now that pos + * 399 (in the archiver mirror cluster, below) is the hasPgData = + * BOOL_FALSE sibling assigning ARCHIVING directly instead of the + * intermediate JOIN_SECONDARY -> SECONDARY dance an ARCHIVING row has + * no real Postgres to actually perform (its client-side transition + * function, fsm_checkpoint_and_stop_postgres, unconditionally fails + * for a haspgdata=false node): REPORT_LSN_STATE -> ARCHIVING_STATE is + * already a real, working transition on its own (fsm_archiver_follow_ + * new_primary, exercised by archiver_wal_capture.pgaf's own failover + * test), so there's no need for an archiver to ever pass through + * JOIN_SECONDARY_STATE at all -- unlike SECONDARY, ARCHIVING isn't + * gated on the primary having fully converged first. + */ { .pos = 365, .sectionPath = { MONITOR_FSM_SECTION_REPORTING_NODE, @@ -3228,7 +3287,8 @@ static const MonitorFSMTransition MonitorFSM[] = { MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN }, .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, - .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN) }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_TRUE }, .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_JOIN_SECONDARY), .comment = @@ -3547,16 +3607,20 @@ static const MonitorFSMTransition MonitorFSM[] = { /* * Archiver mirror cluster: the hasPgData = BOOL_FALSE siblings of pos - * 307/309/315/317/319 above, assigning ARCHIVING instead of - * SECONDARY/CATCHINGUP for an ARCHIVING membership row. Appended here - * (still sectionPath'd under REPORTING_NODE/FROM_CONTEXT, like their - * siblings) rather than interleaved next to each one, for the same + * 307/309/315/317/319/365 above, assigning ARCHIVING instead of + * SECONDARY/CATCHINGUP/JOIN_SECONDARY for an ARCHIVING membership row. + * Appended here rather than interleaved next to each one, for the same * reason the MS-failover cluster above is appended rather than - * renumbered into the ordinary rows: pos 307/309/315/317/319 are - * numbered every 2 with no room between consecutive pairs for 5 more + * renumbered into the ordinary rows: pos 307/309/315/317/319/365 are + * numbered every 2 with no room between consecutive pairs for 6 more * rows, and since hasPgData makes each pair mutually exclusive, their * relative array order doesn't affect first-match-wins correctness -- - * see each of those rows' own comment for the exact pairing. + * see each of those rows' own comment for the exact pairing. Pos 399 + * is the one exception to "sectionPath'd under REPORTING_NODE/ + * FROM_CONTEXT, like their siblings": it mirrors pos 365, which lives + * under the MS-failover cluster's own sectionPath, so it must too -- + * sectionPath is what the dispatcher actually matches evaluation + * context against, not physical position in this array. */ { .pos = 394, .sectionPath = { @@ -3565,11 +3629,11 @@ static const MonitorFSMTransition MonitorFSM[] = { }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), .hasPgData = BOOL_FALSE }, - .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY, + .primaryNode = { .statePattern = FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY, .isHealthy = BOOL_TRUE }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), .comment = "archiver mirror of pos 307: report_lsn, primary converged " - "wait/join_primary, healthy -> archiving" }, + "single/wait/join_primary, healthy -> archiving" }, { .pos = 395, .sectionPath = { @@ -3591,10 +3655,10 @@ static const MonitorFSMTransition MonitorFSM[] = { }, .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_WAIT_STANDBY), .hasPgData = BOOL_FALSE }, - .primaryNode = { .statePattern = FSM_WAIT_OR_JOIN_PRIMARY }, + .primaryNode = { .statePattern = FSM_SINGLE_OR_WAIT_OR_JOIN_PRIMARY }, .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), .comment = "archiver mirror of pos 315: wait_standby, primary converged " - "wait/join_primary -> archiving" }, + "single/wait/join_primary -> archiving" }, { .pos = 397, .sectionPath = { @@ -3623,6 +3687,21 @@ static const MonitorFSMTransition MonitorFSM[] = { .comment = "archiver mirror of pos 319: wait_standby (not a quorum member), " "primary converged primary -> archiving" }, + { .pos = 399, + .sectionPath = { + MONITOR_FSM_SECTION_REPORTING_NODE, + MONITOR_FSM_SECTION_MS_FAILOVER, + MONITOR_FSM_SECTION_MS_FAILOVER_CANDIDATE_JOIN + }, + .conditions = { .candidatePromotionInProgress = BOOL_TRUE }, + .activeNode = { .statePattern = FSM_STATE(REPLICATION_STATE_REPORT_LSN), + .hasPgData = BOOL_FALSE }, + .candidateNode = { .isReadyToStreamWAL = BOOL_TRUE }, + .activeNodeAssignedState = GOAL(REPLICATION_STATE_ARCHIVING), + .comment = "archiver mirror of pos 365: MS-failover, activeNode in report_lsn, " + "failover candidate ready to stream WAL -> archiving (no " + "join_secondary detour -- see pos 365's own comment)" }, + /* * --- the PRIMARY_NODE section (sectionPath[0] == * MONITOR_FSM_SECTION_PRIMARY_NODE, pos 401 onward): the declarative From 7f516159296919d0e9ce90f56b727f532b100667 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:03:59 +0200 Subject: [PATCH 23/55] archiver: WAL-capture reliability, telemetry, and base-backup policy (M5) Several pieces of the Archiving & Disaster Recovery milestone, landing together since they build on each other: WAL-capture reliability - service_archiver_start_pgreceivewal() now creates a replication slot for pg_receivewal (pgautofailover_standby_), the same one keeper_create_and_drop_replication_slots() already creates eagerly on any primary for every other node regardless of kind. Without a slot, a pg_receivewal that loses the startup HBA-propagation race restarts from the server's then-current position, silently and permanently skipping whatever WAL existed in between. - pg_walsender's START_REPLICATION now fails loudly ("58P01") instead of waiting forever when asked for a segment that predates this archiver's own captured history and will never arrive. - The archiver's real captured-WAL position is now tracked out of band (a position file, service_archiver_position_path() and friends) so it can cross the fork() boundary between the capture and serve processes -- consumed by cmd_base_backup.c's own end-of-backup position (previously could re-send a stale start position and hang a real pg_basebackup's background WAL streamer forever) and by cmd_identify_system.c indirectly via the routes file's new "position" key. - service_archiver_loop() now sets pgIsRunning = true for the archiver's own keeper state, which the monitor's NodeIsHealthy() unconditionally requires before ever selecting a node as a FAST_FORWARD WAL source. Telemetry - service_archiver_report_storage() reports disk usage/free space to the monitor periodically; monitor_get_archivers() surfaces it (and each archiver's FSM state) to `pg_autoctl watch`'s new archivers section. Base-backup production/retention policy - New SQL: get_basebackup_policy_for_group(), list_basebackups(); get_basebackup_policy() gains SECURITY DEFINER (needed now that `pg_autoctl show basebackup-policy` calls it directly). - service_archiver_basebackup.c's scheduling is now policy-driven instead of the previous hardcoded "bootstrap live, then exactly one replay, then quiet" scope: frequency/source/replaymode/onpromotion read from whichever policy resolves for the group, plus maxcount/maxage retention pruning after each successful backup. The very first backup for a group is always sourced live regardless of policy (nothing to replay from yet). - The replay/volatile staging instance now starts with ssl = off: the copied postgresql.conf/postgresql.auto.conf still carries the source node's own ssl_cert_file/ssl_key_file paths, meaningless here since the archiver has no Postgres SSL certs of its own -- left enabled, the staging instance failed outright at startup. - New CLI: `pg_autoctl create/show/set basebackup-policy`, and `pg_autoctl create archiver --basebackup-policy ` to attach one at creation time via set_archiver_policy(). Verified via a full --no-cache Docker rebuild plus the archiver_wal_ capture, archiver_basebackup_generation, archiver_basebackup_policy, and archiver_bootstrap_and_fast_forward pgaftest specs, all passing. --- src/bin/pg_autoctl/cli_basebackup_policy.c | 436 +++++++++++++ src/bin/pg_autoctl/cli_basebackup_policy.h | 20 + src/bin/pg_autoctl/cli_create_node.c | 82 ++- src/bin/pg_autoctl/cli_get_set_properties.c | 2 + src/bin/pg_autoctl/cli_root.c | 4 + src/bin/pg_autoctl/monitor.c | 600 ++++++++++++++++++ src/bin/pg_autoctl/monitor.h | 107 ++++ src/bin/pg_autoctl/service_archiver.c | 379 ++++++++++- src/bin/pg_autoctl/service_archiver.h | 3 + .../pg_autoctl/service_archiver_basebackup.c | 345 ++++++++-- .../pg_autoctl/service_archiver_basebackup.h | 1 + src/bin/pg_autoctl/service_archiver_serve.c | 29 + src/bin/pg_autoctl/watch.c | 164 ++++- src/bin/pg_autoctl/watch.h | 4 +- src/bin/pg_walsender/cmd_base_backup.c | 241 ++++++- src/bin/pg_walsender/cmd_start_replication.c | 130 ++++ src/bin/pg_walsender/routes.c | 4 + src/bin/pg_walsender/routes.h | 6 + src/monitor/pgautofailover--2.2--2.3.sql | 168 ++++- src/monitor/pgautofailover.sql | 168 ++++- 20 files changed, 2791 insertions(+), 102 deletions(-) create mode 100644 src/bin/pg_autoctl/cli_basebackup_policy.c create mode 100644 src/bin/pg_autoctl/cli_basebackup_policy.h diff --git a/src/bin/pg_autoctl/cli_basebackup_policy.c b/src/bin/pg_autoctl/cli_basebackup_policy.c new file mode 100644 index 000000000..4f000b052 --- /dev/null +++ b/src/bin/pg_autoctl/cli_basebackup_policy.c @@ -0,0 +1,436 @@ +/* + * src/bin/pg_autoctl/cli_basebackup_policy.c + * See cli_basebackup_policy.h. + * + * A basebackup_policy row is a monitor-side object, not tied to any one + * node's local pgdata (unlike `pg_autoctl create archiver`'s own --pgdata- + * rooted config), so these commands connect straight to --monitor, the + * same self-contained shape create_archiver_command already uses, rather + * than resolving a monitor URL through an existing node's config file the + * way the `get`/`set` property commands (cli_get_set_properties.c) do. + * + * --config is a JSON document read from disk and passed straight + * through, as text, to the monitor's own create_basebackup_policy()/set_ + * basebackup_policy() (pgautofailover.sql) -- their own jsonb cast and + * per-field coalesce-to-default/coalesce-to-current-value logic is the one + * and only place this document actually gets validated and applied, so + * there is nothing to duplicate client-side. The document is the flat + * policy body itself (source/replaymode/cache/frequency/maxcount/maxage/ + * onpromotion/concurrency, whichever subset is being set) -- not wrapped + * in the design doc's own illustrative "pgaf-archiver"/"basebackup-policy" + * namespace, since the monitor-side functions this calls don't unwrap one. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#include +#include + +#include "postgres_fe.h" + +#include "parson.h" + +#include "cli_basebackup_policy.h" +#include "cli_common.h" +#include "commandline.h" +#include "defaults.h" +#include "file_utils.h" +#include "log.h" +#include "monitor.h" +#include "string_utils.h" + +typedef struct BasebackupPolicyCLIOptions +{ + char monitorPguri[MAXCONNINFO]; + char policyName[NAMEDATALEN]; + char configFilePath[MAXPGPATH]; +} BasebackupPolicyCLIOptions; + +static BasebackupPolicyCLIOptions basebackupPolicyOptions = { 0 }; + +static int cli_basebackup_policy_getopts(int argc, char **argv, + bool requireConfig); +static int cli_create_basebackup_policy_getopts(int argc, char **argv); +static int cli_show_basebackup_policy_getopts(int argc, char **argv); +static int cli_set_basebackup_policy_getopts(int argc, char **argv); + +static void cli_create_basebackup_policy(int argc, char **argv); +static void cli_show_basebackup_policy(int argc, char **argv); +static void cli_set_basebackup_policy(int argc, char **argv); + +static bool read_json_config_file(const char *path, char *jsonOut, + size_t jsonOutSize); +static void print_basebackup_policy(BasebackupPolicy *policy); + + +/* + * cli_basebackup_policy_getopts parses the option set shared by all three + * commands (--monitor --name --json, plus --config for create/set). Kept + * as one function with a requireConfig switch rather than three near- + * duplicates, matching cli_create_archiver_getopts's own minimal, hand- + * rolled style for this milestone's own archiver-adjacent commands + * (rather than the ordinary-node cli_create_node_getopts, which assumes a + * real PostgresSetup none of these commands have any use for). + */ +static int +cli_basebackup_policy_getopts(int argc, char **argv, bool requireConfig) +{ + int c, option_index = 0, errors = 0; + + static struct option long_options[] = { + { "monitor", required_argument, NULL, 'm' }, + { "name", required_argument, NULL, 'a' }, + { "config", required_argument, NULL, 'c' }, + { "json", no_argument, NULL, 'J' }, + { "version", no_argument, NULL, 'V' }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, 'q' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + optind = 0; + + while ((c = getopt_long(argc, argv, "m:a:c:JVvqh", + long_options, &option_index)) != -1) + { + switch (c) + { + case 'm': + { + if (!validate_connection_string(optarg)) + { + log_fatal("Failed to parse --monitor connection string, " + "see above for details."); + exit(EXIT_CODE_BAD_ARGS); + } + strlcpy(basebackupPolicyOptions.monitorPguri, optarg, + MAXCONNINFO); + log_trace("--monitor %s", basebackupPolicyOptions.monitorPguri); + break; + } + + case 'a': + { + strlcpy(basebackupPolicyOptions.policyName, optarg, + NAMEDATALEN); + log_trace("--name %s", basebackupPolicyOptions.policyName); + break; + } + + case 'c': + { + strlcpy(basebackupPolicyOptions.configFilePath, optarg, + MAXPGPATH); + log_trace("--config %s", basebackupPolicyOptions.configFilePath); + break; + } + + case 'J': + { + outputJSON = true; + break; + } + + case 'V': + { + keeper_cli_print_version(argc, argv); + exit(EXIT_CODE_QUIT); + } + + case 'v': + { + log_set_level(LOG_INFO); + break; + } + + case 'q': + { + log_set_level(LOG_ERROR); + break; + } + + case 'h': + { + commandline_help(stderr); + exit(EXIT_CODE_QUIT); + } + + default: + { + ++errors; + break; + } + } + } + + if (errors > 0) + { + commandline_help(stderr); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.monitorPguri)) + { + log_fatal("Failed to get value for --monitor"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.policyName)) + { + log_fatal("Failed to get value for --name"); + exit(EXIT_CODE_BAD_ARGS); + } + + if (requireConfig && IS_EMPTY_STRING_BUFFER(basebackupPolicyOptions.configFilePath)) + { + log_fatal("Failed to get value for --config"); + exit(EXIT_CODE_BAD_ARGS); + } + + return optind; +} + + +static int +cli_create_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, true); +} + + +static int +cli_set_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, true); +} + + +static int +cli_show_basebackup_policy_getopts(int argc, char **argv) +{ + return cli_basebackup_policy_getopts(argc, argv, false); +} + + +/* + * read_json_config_file reads path's whole contents into jsonOut, for + * pass-through to the monitor's own ::jsonb cast -- no client-side JSON + * parsing/validation, see this file's own header comment on why. + */ +static bool +read_json_config_file(const char *path, char *jsonOut, size_t jsonOutSize) +{ + char *contents = NULL; + long fileSize = 0; + + if (!read_file(path, &contents, &fileSize)) + { + log_error("Failed to read base-backup policy config file \"%s\"", + path); + return false; + } + + strlcpy(jsonOut, contents, jsonOutSize); + free(contents); + + return true; +} + + +/* + * print_basebackup_policy prints a resolved policy either as plain text + * (one "field: value" line each) or, with --json, the same fields as a + * JSON object -- matching cli_get_set_properties.c's own established + * plain/--json duality for monitor-resolved properties. + */ +static void +print_basebackup_policy(BasebackupPolicy *policy) +{ + if (outputJSON) + { + JSON_Value *js = json_value_init_object(); + JSON_Object *jsObj = json_value_get_object(js); + + json_object_set_string(jsObj, "name", policy->policyName); + json_object_set_string(jsObj, "source", policy->source); + json_object_set_string(jsObj, "replaymode", policy->replayMode); + json_object_set_string(jsObj, "cache", policy->cache); + json_object_set_number(jsObj, "frequency-seconds", + (double) policy->frequencySeconds); + json_object_set_number(jsObj, "maxcount", (double) policy->maxCount); + json_object_set_number(jsObj, "maxage-seconds", + (double) policy->maxAgeSeconds); + json_object_set_boolean(jsObj, "onpromotion", policy->onPromotion); + json_object_set_number(jsObj, "concurrency", + (double) policy->concurrency); + + (void) cli_pprint_json(js); + + return; + } + + fformat(stdout, "%12s: %s\n", "name", policy->policyName); + fformat(stdout, "%12s: %s\n", "source", policy->source); + fformat(stdout, "%12s: %s\n", "replaymode", + IS_EMPTY_STRING_BUFFER(policy->replayMode) ? "-" : policy->replayMode); + fformat(stdout, "%12s: %s\n", "cache", policy->cache); + fformat(stdout, "%12s: %d\n", "frequency", policy->frequencySeconds); + fformat(stdout, "%12s: %d\n", "maxcount", policy->maxCount); + fformat(stdout, "%12s: %d\n", "maxage", policy->maxAgeSeconds); + fformat(stdout, "%12s: %s\n", "onpromotion", + policy->onPromotion ? "true" : "false"); + fformat(stdout, "%12s: %d\n", "concurrency", policy->concurrency); +} + + +/* + * cli_create_basebackup_policy implements `pg_autoctl create basebackup- + * policy`. + */ +static void +cli_create_basebackup_policy(int argc, char **argv) +{ + char jsonSpec[BUFSIZE] = { 0 }; + + if (!read_json_config_file(basebackupPolicyOptions.configFilePath, + jsonSpec, sizeof(jsonSpec))) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + int64_t basebackupPolicyId = 0; + + if (!monitor_create_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + jsonSpec, &basebackupPolicyId)) + { + log_fatal("Failed to create base-backup policy \"%s\", see above " + "for details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + log_info("Created base-backup policy \"%s\" (id %" PRId64 ")", + basebackupPolicyOptions.policyName, basebackupPolicyId); +} + + +/* + * cli_show_basebackup_policy implements `pg_autoctl show basebackup- + * policy`. + */ +static void +cli_show_basebackup_policy(int argc, char **argv) +{ + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + BasebackupPolicy policy = { 0 }; + bool found = false; + + if (!monitor_get_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + &policy, &found)) + { + log_fatal("Failed to get base-backup policy \"%s\", see above for " + "details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + if (!found) + { + log_fatal("Base-backup policy \"%s\" does not exist", + basebackupPolicyOptions.policyName); + exit(EXIT_CODE_BAD_ARGS); + } + + print_basebackup_policy(&policy); +} + + +/* + * cli_set_basebackup_policy implements `pg_autoctl set basebackup- + * policy`. + */ +static void +cli_set_basebackup_policy(int argc, char **argv) +{ + char jsonSpec[BUFSIZE] = { 0 }; + + if (!read_json_config_file(basebackupPolicyOptions.configFilePath, + jsonSpec, sizeof(jsonSpec))) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + Monitor monitor = { 0 }; + + if (!monitor_init(&monitor, basebackupPolicyOptions.monitorPguri)) + { + /* errors already logged */ + exit(EXIT_CODE_BAD_ARGS); + } + + if (!monitor_set_basebackup_policy(&monitor, + basebackupPolicyOptions.policyName, + jsonSpec)) + { + log_fatal("Failed to set base-backup policy \"%s\", see above for " + "details", basebackupPolicyOptions.policyName); + exit(EXIT_CODE_MONITOR); + } + + log_info("Updated base-backup policy \"%s\"", + basebackupPolicyOptions.policyName); +} + + +CommandLine create_basebackup_policy_command = + make_command( + "basebackup-policy", + "Create a named base-backup production/retention policy", + " --monitor --name --config ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --config path to a JSON document with the policy body\n", + cli_create_basebackup_policy_getopts, + cli_create_basebackup_policy); + +CommandLine show_basebackup_policy_command = + make_command( + "basebackup-policy", + "Show a named base-backup production/retention policy", + " --monitor --name [ --json ] ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --json output data in the JSON format\n", + cli_show_basebackup_policy_getopts, + cli_show_basebackup_policy); + +CommandLine set_basebackup_policy_command = + make_command( + "basebackup-policy", + "Update a named base-backup production/retention policy", + " --monitor --name --config ", + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --name policy name\n" + " --config path to a JSON document with the fields to change\n", + cli_set_basebackup_policy_getopts, + cli_set_basebackup_policy); diff --git a/src/bin/pg_autoctl/cli_basebackup_policy.h b/src/bin/pg_autoctl/cli_basebackup_policy.h new file mode 100644 index 000000000..73150a9c3 --- /dev/null +++ b/src/bin/pg_autoctl/cli_basebackup_policy.h @@ -0,0 +1,20 @@ +/* + * src/bin/pg_autoctl/cli_basebackup_policy.h + * CLI for pgautofailover.basebackup_policy: create/show/set a named + * base-backup production/retention policy on the monitor. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the PostgreSQL License. + * + */ + +#ifndef CLI_BASEBACKUP_POLICY_H +#define CLI_BASEBACKUP_POLICY_H + +#include "commandline.h" + +extern CommandLine create_basebackup_policy_command; +extern CommandLine show_basebackup_policy_command; +extern CommandLine set_basebackup_policy_command; + +#endif /* CLI_BASEBACKUP_POLICY_H */ diff --git a/src/bin/pg_autoctl/cli_create_node.c b/src/bin/pg_autoctl/cli_create_node.c index 95a196a3e..36eb88399 100644 --- a/src/bin/pg_autoctl/cli_create_node.c +++ b/src/bin/pg_autoctl/cli_create_node.c @@ -67,6 +67,12 @@ static void cli_create_monitor(int argc, char **argv); static int cli_create_archiver_getopts(int argc, char **argv); static void cli_create_archiver(int argc, char **argv); +/* --basebackup-policy on `create archiver`: a policy name to resolve and + * attach via set_archiver_policy(), not part of KeeperConfig/keeperOptions + * -- it's applied once at creation time, never persisted to the archiver's + * own config file (see cli_create_archiver()'s own use of this). */ +static char archiverBasebackupPolicyName[NAMEDATALEN] = { 0 }; + static void check_hostname(const char *hostname); CommandLine create_monitor_command = @@ -1324,6 +1330,7 @@ cli_create_archiver_getopts(int argc, char **argv) { "hostname", required_argument, NULL, 'n' }, { "name", required_argument, NULL, 'a' }, { "formation", required_argument, NULL, 'f' }, + { "basebackup-policy", required_argument, NULL, 'P' }, { "run", no_argument, NULL, 'x' }, { "version", no_argument, NULL, 'V' }, { "verbose", no_argument, NULL, 'v' }, @@ -1334,7 +1341,7 @@ cli_create_archiver_getopts(int argc, char **argv) optind = 0; - while ((c = getopt_long(argc, argv, "D:C:m:n:a:f:xVvqh", + while ((c = getopt_long(argc, argv, "D:C:m:n:a:f:P:xVvqh", long_options, &option_index)) != -1) { switch (c) @@ -1387,6 +1394,13 @@ cli_create_archiver_getopts(int argc, char **argv) break; } + case 'P': + { + strlcpy(archiverBasebackupPolicyName, optarg, NAMEDATALEN); + log_trace("--basebackup-policy %s", archiverBasebackupPolicyName); + break; + } + case 'x': { createAndRun = true; @@ -1552,6 +1566,54 @@ cli_create_archiver(int argc, char **argv) PRId64, archiverName, archiverId, config->formation, archiverNodeId); + /* + * --basebackup-policy resolves a name to its basebackuppolicyid and + * attaches it to this (formation, group) via set_archiver_policy() -- + * a formation/group-level setting (archiver_policy), not per-archiver, + * matching get_archiver_policy()'s own resolution scope: any other + * archiver later added to the same (formation, group) inherits it too. + * archiverQuorum=1, replicationQuorumEligible=false are this schema's + * own hardcoded defaults (get_archiver_policy()'s final fallback tier) + * -- passed through explicitly here since set_archiver_policy() only + * coaleses NULL to "keep existing" for a row that already exists, and + * this may be the first policy ever set for this (formation, group). + */ + if (!IS_EMPTY_STRING_BUFFER(archiverBasebackupPolicyName)) + { + BasebackupPolicy basebackupPolicy = { 0 }; + bool foundPolicy = false; + + if (!monitor_get_basebackup_policy(&monitor, archiverBasebackupPolicyName, + &basebackupPolicy, &foundPolicy)) + { + log_fatal("Failed to resolve base-backup policy \"%s\", see " + "above for details", archiverBasebackupPolicyName); + exit(EXIT_CODE_MONITOR); + } + + if (!foundPolicy) + { + log_fatal("Base-backup policy \"%s\" does not exist", + archiverBasebackupPolicyName); + exit(EXIT_CODE_BAD_ARGS); + } + + if (!monitor_set_archiver_policy(&monitor, config->formation, + -1, /* formation-wide, not one group */ + 1, /* archiverQuorum */ + basebackupPolicy.basebackupPolicyId, + false /* replicationQuorumEligible */)) + { + log_fatal("Failed to attach base-backup policy \"%s\" to " + "formation \"%s\", see above for details", + archiverBasebackupPolicyName, config->formation); + exit(EXIT_CODE_MONITOR); + } + + log_info("Attached base-backup policy \"%s\" to formation \"%s\"", + archiverBasebackupPolicyName, config->formation); + } + strlcpy(config->role, KEEPER_ROLE, sizeof(config->role)); config->groupId = 0; config->network_partition_timeout = NETWORK_PARTITION_TIMEOUT; @@ -1617,14 +1679,16 @@ CommandLine create_archiver_command = make_command( "archiver", "Initialize a pg_auto_failover archiver node", - " [ --pgdata --pgctl --monitor --hostname --name --formation ] ", - " --pgdata path to the archiver's local data/cache directory\n" - " --pgctl path to pg_ctl (used to locate pg_receivewal)\n" - " --monitor pg_auto_failover Monitor Postgres URL\n" - " --hostname hostname by which the archiver is reachable\n" - " --name archiver name (default: derived from hostname)\n" - " --formation formation to attach to (default: \"default\")\n" - " --run create node then run pg_autoctl service\n", + " [ --pgdata --pgctl --monitor --hostname --name --formation --basebackup-policy ] ", + " --pgdata path to the archiver's local data/cache directory\n" + " --pgctl path to pg_ctl (used to locate pg_receivewal)\n" + " --monitor pg_auto_failover Monitor Postgres URL\n" + " --hostname hostname by which the archiver is reachable\n" + " --name archiver name (default: derived from hostname)\n" + " --formation formation to attach to (default: \"default\")\n" + " --basebackup-policy base-backup production/retention policy to attach " + "(default: \"default\")\n" + " --run create node then run pg_autoctl service\n", cli_create_archiver_getopts, cli_create_archiver); diff --git a/src/bin/pg_autoctl/cli_get_set_properties.c b/src/bin/pg_autoctl/cli_get_set_properties.c index f080cf55e..ae152ff4d 100644 --- a/src/bin/pg_autoctl/cli_get_set_properties.c +++ b/src/bin/pg_autoctl/cli_get_set_properties.c @@ -9,6 +9,7 @@ */ #include "parson.h" +#include "cli_basebackup_policy.h" #include "cli_common.h" #include "parsing.h" #include "string_utils.h" @@ -212,6 +213,7 @@ static CommandLine set_formation_command = static CommandLine *set_subcommands[] = { &set_node_command, &set_formation_command, + &set_basebackup_policy_command, NULL }; diff --git a/src/bin/pg_autoctl/cli_root.c b/src/bin/pg_autoctl/cli_root.c index 242597597..5293a47fa 100644 --- a/src/bin/pg_autoctl/cli_root.c +++ b/src/bin/pg_autoctl/cli_root.c @@ -9,6 +9,7 @@ */ #include "cli_archiver.h" +#include "cli_basebackup_policy.h" #include "cli_common.h" #include "cli_do_root.h" #include "cli_inspect.h" @@ -33,6 +34,7 @@ CommandLine *create_subcommands[] = { &create_coordinator_command, &create_worker_command, &create_archiver_command, + &create_basebackup_policy_command, &create_formation_command, NULL }; @@ -50,6 +52,7 @@ CommandLine *show_subcommands_with_debug[] = { &show_standby_names_command, &show_timeline_command, &show_file_command, + &show_basebackup_policy_command, &systemd_cat_service_file_command, NULL }; @@ -67,6 +70,7 @@ CommandLine *show_subcommands[] = { &show_standby_names_command, &show_timeline_command, &show_file_command, + &show_basebackup_policy_command, &systemd_cat_service_file_command, NULL }; diff --git a/src/bin/pg_autoctl/monitor.c b/src/bin/pg_autoctl/monitor.c index a1f468197..fc02223ef 100644 --- a/src/bin/pg_autoctl/monitor.c +++ b/src/bin/pg_autoctl/monitor.c @@ -1036,6 +1036,181 @@ monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, } +/* + * monitor_report_archiver_storage calls pgautofailover.report_archiver_ + * storage() to record this archiver's own disk usage and free space, + * alongside a fresh lastreporttime -- the same periodic heartbeat + * service_archiver_loop() already uses to report captured-WAL LSN. + */ +bool +monitor_report_archiver_storage(Monitor *monitor, int64_t archiverId, + uint64_t usedBytes, uint64_t freeBytes) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.report_archiver_storage($1, $2, $3)"; + int paramCount = 3; + Oid paramTypes[3] = { INT8OID, INT8OID, INT8OID }; + IntString archiverIdString = intToString(archiverId); + IntString usedBytesString = intToString((int64_t) usedBytes); + IntString freeBytesString = intToString((int64_t) freeBytes); + const char *paramValues[3] = { + archiverIdString.strValue, + usedBytesString.strValue, + freeBytesString.strValue + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report storage usage for archiver %" PRId64, + archiverId); + return false; + } + + return true; +} + + +typedef struct ArchiverInfoArrayParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + ArchiverInfoArray *archiversArray; + bool parsedOK; +} ArchiverInfoArrayParseContext; + + +/* + * parseArchiverInfo parses one row of pgautofailover.get_archivers()'s + * result: archiver_id, archiver_name, hostname, used_bytes, free_bytes, + * last_report_time, node_id, reported_state, goal_state. usedbytes/ + * freebytes (NULL until the archiver's first storage report) and the + * node_id/reportedState/goalState side of the LEFT JOIN (NULL if this + * milestone's one-'wal-receiver'-row-per-group assumption isn't met yet, + * see get_archivers()'s own comment) are both optional -- everything else + * is not. + */ +static bool +parseArchiverInfo(PGresult *result, int rowNumber, ArchiverInfo *archiver) +{ + if (PQgetisnull(result, rowNumber, 0) || + PQgetisnull(result, rowNumber, 1) || + PQgetisnull(result, rowNumber, 2)) + { + log_error("archiver_id, archiver_name or hostname returned by " + "the monitor is NULL"); + return false; + } + + char *value = PQgetvalue(result, rowNumber, 0); + + archiver->archiverId = strtol(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 1); + strlcpy(archiver->archiverName, value, _POSIX_HOST_NAME_MAX); + + value = PQgetvalue(result, rowNumber, 2); + strlcpy(archiver->hostname, value, _POSIX_HOST_NAME_MAX); + + archiver->hasStorageStats = + !PQgetisnull(result, rowNumber, 3) && !PQgetisnull(result, rowNumber, 4); + + if (archiver->hasStorageStats) + { + value = PQgetvalue(result, rowNumber, 3); + archiver->usedBytes = strtoull(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 4); + archiver->freeBytes = strtoull(value, NULL, 0); + } + + archiver->hasNode = !PQgetisnull(result, rowNumber, 6); + + if (archiver->hasNode) + { + value = PQgetvalue(result, rowNumber, 6); + archiver->nodeId = strtol(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 7); + archiver->reportedState = NodeStateFromString(value); + + value = PQgetvalue(result, rowNumber, 8); + archiver->goalState = NodeStateFromString(value); + } + + return true; +} + + +static void +parseArchiverInfoArray(void *ctx, PGresult *result) +{ + ArchiverInfoArrayParseContext *context = (ArchiverInfoArrayParseContext *) ctx; + bool parsedOk = true; + + if (PQntuples(result) > ARCHIVER_ARRAY_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only " + "up to %d archivers at the moment", + PQntuples(result), ARCHIVER_ARRAY_MAX_COUNT); + context->parsedOK = false; + return; + } + + context->archiversArray->count = PQntuples(result); + + for (int rowNumber = 0; rowNumber < PQntuples(result); rowNumber++) + { + ArchiverInfo *archiver = &(context->archiversArray->archivers[rowNumber]); + + parsedOk = parsedOk && parseArchiverInfo(result, rowNumber, archiver); + } + + context->parsedOK = parsedOk; +} + + +/* + * monitor_get_archivers calls pgautofailover.get_archivers(formation) and + * returns every archiver attached to that formation, with its storage + * stats and FSM state -- used by `pg_autoctl watch`'s own archivers + * section. + */ +bool +monitor_get_archivers(Monitor *monitor, const char *formation, + ArchiverInfoArray *archiversArray) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = "SELECT * FROM pgautofailover.get_archivers($1)"; + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { formation }; + ArchiverInfoArrayParseContext parseContext = { { 0 }, archiversArray, false }; + + archiversArray->count = 0; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, &parseArchiverInfoArray)) + { + log_error("Failed to get the list of archivers from the monitor " + "for formation \"%s\"", formation); + return false; + } + + if (!parseContext.parsedOK) + { + log_error("Failed to parse the list of archivers returned by the " + "monitor for formation \"%s\", see previous lines for " + "details", formation); + return false; + } + + return true; +} + + /* * BasebackupInfoParseContext/parseBasebackupInfo parse the two columns * monitor_get_latest_basebackup_info() needs out of a single-row result -- @@ -1369,6 +1544,431 @@ monitor_report_basebackup_completed(Monitor *monitor, int64_t basebackupId, } +/* + * monitor_report_basebackup_deleted calls + * pgautofailover.report_basebackup_deleted() to mark a base backup deleted + * (retaining its history row) once service_archiver_basebackup.c's own + * retention pass has actually removed the directory on disk. Cascades on + * the monitor side to prune any archiver_wal rows no remaining backup + * needs anymore (prune_archiver_wal(), that function's own comment). + */ +bool +monitor_report_basebackup_deleted(Monitor *monitor, int64_t basebackupId) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = "SELECT pgautofailover.report_basebackup_deleted($1)"; + int paramCount = 1; + Oid paramTypes[1] = { INT8OID }; + IntString basebackupIdString = intToString(basebackupId); + const char *paramValues[1] = { basebackupIdString.strValue }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to report base backup %" PRId64 " as deleted " + "to the monitor", basebackupId); + return false; + } + + return true; +} + + +typedef struct BasebackupInfoArrayParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + BasebackupInfoArray *backupsArray; + bool parsedOK; +} BasebackupInfoArrayParseContext; + + +static bool +parseBasebackupInfoRow(PGresult *result, int rowNumber, BasebackupInfo *backup) +{ + if (PQgetisnull(result, rowNumber, 0) || + PQgetisnull(result, rowNumber, 2) || + PQgetisnull(result, rowNumber, 3)) + { + log_error("basebackupid, storagelocation, or startedat_epoch " + "returned by the monitor is NULL"); + return false; + } + + char *value = PQgetvalue(result, rowNumber, 0); + + backup->basebackupId = strtoll(value, NULL, 0); + + value = PQgetvalue(result, rowNumber, 1); + strlcpy(backup->label, value, NAMEDATALEN); + + value = PQgetvalue(result, rowNumber, 2); + strlcpy(backup->storageLocation, value, MAXPGPATH); + + value = PQgetvalue(result, rowNumber, 3); + backup->startedAtEpoch = strtoll(value, NULL, 0); + + return true; +} + + +static void +parseBasebackupInfoArray(void *ctx, PGresult *result) +{ + BasebackupInfoArrayParseContext *context = + (BasebackupInfoArrayParseContext *) ctx; + bool parsedOk = true; + + if (PQntuples(result) > BASEBACKUP_ARRAY_MAX_COUNT) + { + log_error("Query returned %d rows, pg_auto_failover supports only " + "up to %d base backups per group at the moment", + PQntuples(result), BASEBACKUP_ARRAY_MAX_COUNT); + context->parsedOK = false; + return; + } + + context->backupsArray->count = PQntuples(result); + + for (int rowNumber = 0; rowNumber < PQntuples(result); rowNumber++) + { + BasebackupInfo *backup = &(context->backupsArray->backups[rowNumber]); + + parsedOk = parsedOk && parseBasebackupInfoRow(result, rowNumber, backup); + } + + context->parsedOK = parsedOk; +} + + +/* + * monitor_list_basebackups calls pgautofailover.list_basebackups(formation, + * group) and returns every complete base backup for that group, newest + * first -- what service_archiver_basebackup.c's own retention pass walks + * to decide what survives maxcount/maxage. + */ +bool +monitor_list_basebackups(Monitor *monitor, + const char *formationId, int groupId, + BasebackupInfoArray *backupsArray) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT * FROM pgautofailover.list_basebackups($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + BasebackupInfoArrayParseContext parseContext = { { 0 }, backupsArray, false }; + + backupsArray->count = 0; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &parseContext, &parseBasebackupInfoArray)) + { + log_error("Failed to list base backups from the monitor for " + "\"%s\"/%d", formationId, groupId); + return false; + } + + if (!parseContext.parsedOK) + { + log_error("Failed to parse the list of base backups returned by " + "the monitor for \"%s\"/%d, see previous lines for " + "details", formationId, groupId); + return false; + } + + return true; +} + + +/* + * BasebackupPolicyParseContext/parseBasebackupPolicy parse the 9-column + * row shape both monitor_get_basebackup_policy_for_group() and monitor_ + * get_basebackup_policy() use -- one via get_basebackup_policy_for_group() + * (resolved for a formation/group), the other via get_basebackup_policy() + * (looked up by name for `pg_autoctl show basebackup-policy`) -- both + * wrapped in the same SELECT column list on the C side so a single parser + * serves either. + */ +typedef struct BasebackupPolicyParseContext +{ + char sqlstate[SQLSTATE_LENGTH]; + BasebackupPolicy *policy; + bool found; + bool parsedOk; +} BasebackupPolicyParseContext; + + +static void +parseBasebackupPolicy(void *ctx, PGresult *result) +{ + BasebackupPolicyParseContext *context = + (BasebackupPolicyParseContext *) ctx; + + int ntuples = PQntuples(result); + + if (ntuples != 1) + { + /* zero rows is a valid "no such policy" signal, not a parse error */ + context->parsedOk = (ntuples == 0); + context->found = false; + return; + } + + if (PQgetisnull(result, 0, 0)) + { + /* get_basebackup_policy_for_group() found no policy to resolve -- + * shouldn't happen given the schema's own 'default' row always + * exists, but treat it as "not found" rather than a parse error */ + context->parsedOk = true; + context->found = false; + return; + } + + BasebackupPolicy *policy = context->policy; + + strlcpy(policy->policyName, PQgetvalue(result, 0, 0), NAMEDATALEN); + strlcpy(policy->source, PQgetvalue(result, 0, 1), NAMEDATALEN); + + strlcpy(policy->replayMode, + PQgetisnull(result, 0, 2) ? "" : PQgetvalue(result, 0, 2), + NAMEDATALEN); + + strlcpy(policy->cache, PQgetvalue(result, 0, 3), NAMEDATALEN); + + policy->frequencySeconds = strtol(PQgetvalue(result, 0, 4), NULL, 0); + policy->maxCount = strtol(PQgetvalue(result, 0, 5), NULL, 0); + policy->maxAgeSeconds = strtol(PQgetvalue(result, 0, 6), NULL, 0); + policy->onPromotion = strcmp(PQgetvalue(result, 0, 7), "t") == 0; + policy->concurrency = strtol(PQgetvalue(result, 0, 8), NULL, 0); + + context->found = true; + context->parsedOk = true; +} + + +/* + * monitor_get_basebackup_policy_for_group calls pgautofailover.get_ + * basebackup_policy_for_group(formation, group) -- the policy service_ + * archiver_basebackup.c's own scheduling/retention pass actually applies, + * already resolved through archiver_policy's group-override / formation- + * default / schema-default fallback chain (get_archiver_policy()'s own + * comment). + */ +bool +monitor_get_basebackup_policy_for_group(Monitor *monitor, + const char *formationId, int groupId, + BasebackupPolicy *policy, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT policyname, source::text, replaymode::text, cache::text, " + " frequency_seconds, maxcount, maxage_seconds, " + " onpromotion, concurrency " + " FROM pgautofailover.get_basebackup_policy_for_group($1, $2)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, INT4OID }; + IntString groupIdString = intToString(groupId); + const char *paramValues[2] = { formationId, groupIdString.strValue }; + BasebackupPolicyParseContext context = { { 0 }, policy, false, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseBasebackupPolicy)) + { + log_error("Failed to get the base-backup policy from the monitor " + "for \"%s\"/%d", formationId, groupId); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse the base-backup policy returned by the " + "monitor for \"%s\"/%d, see above for details", + formationId, groupId); + return false; + } + + *found = context.found; + + return true; +} + + +/* + * monitor_get_basebackup_policy calls pgautofailover.get_basebackup_policy + * (policyname) -- a named policy looked up directly, for `pg_autoctl show + * basebackup-policy`. Wrapped in the same 9-column SELECT list as monitor_ + * get_basebackup_policy_for_group() above so parseBasebackupPolicy() can + * serve both. *found is false (not an error) when no policy has that name. + */ +bool +monitor_get_basebackup_policy(Monitor *monitor, const char *policyName, + BasebackupPolicy *policy, bool *found) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT policyname, source::text, replaymode::text, cache::text, " + " extract(epoch FROM frequency)::int, maxcount, " + " extract(epoch FROM maxage)::int, onpromotion, concurrency " + " FROM pgautofailover.get_basebackup_policy($1)"; + int paramCount = 1; + Oid paramTypes[1] = { TEXTOID }; + const char *paramValues[1] = { policyName }; + BasebackupPolicyParseContext context = { { 0 }, policy, false, false }; + + *found = false; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseBasebackupPolicy)) + { + log_error("Failed to get base-backup policy \"%s\" from the " + "monitor", policyName); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to parse base-backup policy \"%s\" returned by " + "the monitor, see above for details", policyName); + return false; + } + + *found = context.found; + + return true; +} + + +/* + * monitor_create_basebackup_policy calls pgautofailover.create_ + * basebackup_policy(policyname, policyspec) -- policyspec is a raw JSON + * document text, passed straight through to the monitor's own jsonb + * parsing and per-field coalesce-to-default logic (create_basebackup_ + * policy()'s own body, pgautofailover.sql) rather than parsed twice. + */ +bool +monitor_create_basebackup_policy(Monitor *monitor, + const char *policyName, + const char *jsonSpec, + int64_t *basebackupPolicyId) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.create_basebackup_policy($1, $2::jsonb)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, TEXTOID }; + const char *paramValues[2] = { policyName, jsonSpec }; + SingleValueResultContext context = { { 0 }, PGSQL_RESULT_BIGINT, false }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + &context, &parseSingleValueResult)) + { + log_error("Failed to create base-backup policy \"%s\" on the " + "monitor", policyName); + return false; + } + + if (!context.parsedOk) + { + log_error("Failed to create base-backup policy \"%s\" on the " + "monitor because it returned an unexpected result, see " + "previous lines for details", policyName); + return false; + } + + *basebackupPolicyId = context.bigint; + + return true; +} + + +/* + * monitor_set_basebackup_policy calls pgautofailover.set_basebackup_policy + * (policyname, policyspec) to update an existing named policy -- only the + * fields present in the JSON document change (set_basebackup_policy()'s + * own per-field coalesce, pgautofailover.sql), everything else keeps its + * current value. + */ +bool +monitor_set_basebackup_policy(Monitor *monitor, const char *policyName, + const char *jsonSpec) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.set_basebackup_policy($1, $2::jsonb)"; + int paramCount = 2; + Oid paramTypes[2] = { TEXTOID, TEXTOID }; + const char *paramValues[2] = { policyName, jsonSpec }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to set base-backup policy \"%s\" on the monitor", + policyName); + return false; + } + + return true; +} + + +/* + * monitor_set_archiver_policy calls pgautofailover.set_archiver_policy() + * to attach basebackupPolicyId to (formationId, groupId) -- groupId < 0 + * sets the formation-wide default (archiver_policy's own groupid IS NULL + * row, matching set_archiver_policy()'s own in_groupid DEFAULT NULL). + * archiverQuorum <= 0 and replicationQuorumEligible are passed through + * as-is; callers that only want to change the backup policy pass whatever + * this formation/group's own current values already are, since set_ + * archiver_policy() UPSERTs and coalesces NULL to "keep existing" only + * when the row already exists -- a brand new row still needs real values, + * hence no NULL-means-unchanged shortcut is exposed at this C layer. + */ +bool +monitor_set_archiver_policy(Monitor *monitor, + const char *formationId, int groupId, + int archiverQuorum, + int64_t basebackupPolicyId, + bool replicationQuorumEligible) +{ + PGSQL *pgsql = &monitor->pgsql; + const char *sql = + "SELECT pgautofailover.set_archiver_policy($1, $2, $3, $4, $5)"; + int paramCount = 5; + Oid paramTypes[5] = { TEXTOID, INT4OID, INT4OID, INT8OID, BOOLOID }; + IntString groupIdString = intToString(groupId); + IntString archiverQuorumString = intToString(archiverQuorum); + IntString basebackupPolicyIdString = intToString(basebackupPolicyId); + const char *paramValues[5] = { + formationId, + groupId < 0 ? NULL : groupIdString.strValue, + archiverQuorumString.strValue, + basebackupPolicyIdString.strValue, + replicationQuorumEligible ? "true" : "false" + }; + + if (!pgsql_execute_with_params(pgsql, sql, + paramCount, paramTypes, paramValues, + NULL, NULL)) + { + log_error("Failed to set archiver policy for \"%s\"/%d on the " + "monitor", formationId, groupId); + return false; + } + + return true; +} + + bool monitor_register_node(Monitor *monitor, char *formation, char *name, char *host, int port, diff --git a/src/bin/pg_autoctl/monitor.h b/src/bin/pg_autoctl/monitor.h index bb5384f96..ce311790c 100644 --- a/src/bin/pg_autoctl/monitor.h +++ b/src/bin/pg_autoctl/monitor.h @@ -36,6 +36,87 @@ typedef struct MonitorAssignedState bool replicationQuorum; } MonitorAssignedState; +#define ARCHIVER_ARRAY_MAX_COUNT 128 + +/* + * One row per archiver attached to a formation, from pgautofailover. + * get_archivers() -- storage stats and FSM state are both nullable on the + * SQL side (usedbytes/freebytes: NULL until the first report; the node_id/ + * reportedState/goalState side of the LEFT JOIN: NULL if this milestone's + * one-'wal-receiver'-row-per-group assumption isn't met yet), hence the + * separate hasStorageStats/hasNode flags rather than a sentinel value. + */ +typedef struct ArchiverInfo +{ + int64_t archiverId; + char archiverName[_POSIX_HOST_NAME_MAX]; + char hostname[_POSIX_HOST_NAME_MAX]; + + bool hasStorageStats; + uint64_t usedBytes; + uint64_t freeBytes; + + bool hasNode; + int64_t nodeId; + NodeState reportedState; + NodeState goalState; +} ArchiverInfo; + +typedef struct ArchiverInfoArray +{ + int count; + ArchiverInfo archivers[ARCHIVER_ARRAY_MAX_COUNT]; +} ArchiverInfoArray; + +/* + * A base-backup production/retention policy (pgautofailover.basebackup_ + * policy), resolved either by name (monitor_get_basebackup_policy(), `pg_ + * autoctl show basebackup-policy`) or for a (formation, group) pair + * (monitor_get_basebackup_policy_for_group(), what service_archiver_ + * basebackup.c's own scheduling/retention pass actually consumes) -- + * both go through the same SQL-side flattening of the policy's interval + * columns to plain integer seconds (get_basebackup_policy_for_group()'s + * own comment, pgautofailover.sql), so both share this one struct. + * replayMode is empty when source is "live" (basebackup_policy's own + * CHECK constraint: replaymode is NULL unless source = 'replay'). + */ +typedef struct BasebackupPolicy +{ + int64_t basebackupPolicyId; + char policyName[NAMEDATALEN]; + char source[NAMEDATALEN]; + char replayMode[NAMEDATALEN]; + char cache[NAMEDATALEN]; + int frequencySeconds; + int maxCount; + int maxAgeSeconds; + bool onPromotion; + int concurrency; +} BasebackupPolicy; + +#define BASEBACKUP_ARRAY_MAX_COUNT 256 + +/* + * One row per complete base backup for a (formation, group), from + * pgautofailover.list_basebackups() -- just enough for a retention + * decision (age via startedAtEpoch, which of maxcount survives) and to + * act on one once pruned (storageLocation to remove the directory, + * basebackupId to report the deletion). + */ +typedef struct BasebackupInfo +{ + int64_t basebackupId; + char label[NAMEDATALEN]; + char storageLocation[MAXPGPATH]; + int64_t startedAtEpoch; +} BasebackupInfo; + +typedef struct BasebackupInfoArray +{ + int count; + BasebackupInfo backups[BASEBACKUP_ARRAY_MAX_COUNT]; +} BasebackupInfoArray; + typedef struct StateNotification { char message[BUFSIZE]; @@ -158,6 +239,10 @@ bool monitor_register_archiver(Monitor *monitor, char *name, char *hostname, int64_t *archiverId); bool monitor_archiver_add_formation(Monitor *monitor, int64_t archiverId, char *formation, int64_t *archiverNodeId); +bool monitor_report_archiver_storage(Monitor *monitor, int64_t archiverId, + uint64_t usedBytes, uint64_t freeBytes); +bool monitor_get_archivers(Monitor *monitor, const char *formation, + ArchiverInfoArray *archiversArray); bool monitor_get_latest_basebackup_info(Monitor *monitor, const char *formationId, int groupId, const char *preferredSource, @@ -183,6 +268,28 @@ bool monitor_report_basebackup_completed(Monitor *monitor, const char *endLsn, int64_t sizeBytes, const char *storageLocation); +bool monitor_report_basebackup_deleted(Monitor *monitor, int64_t basebackupId); +bool monitor_list_basebackups(Monitor *monitor, + const char *formationId, int groupId, + BasebackupInfoArray *backupsArray); +bool monitor_get_basebackup_policy_for_group(Monitor *monitor, + const char *formationId, + int groupId, + BasebackupPolicy *policy, + bool *found); +bool monitor_get_basebackup_policy(Monitor *monitor, const char *policyName, + BasebackupPolicy *policy, bool *found); +bool monitor_create_basebackup_policy(Monitor *monitor, + const char *policyName, + const char *jsonSpec, + int64_t *basebackupPolicyId); +bool monitor_set_basebackup_policy(Monitor *monitor, const char *policyName, + const char *jsonSpec); +bool monitor_set_archiver_policy(Monitor *monitor, + const char *formationId, int groupId, + int archiverQuorum, + int64_t basebackupPolicyId, + bool replicationQuorumEligible); bool monitor_get_coordinator(Monitor *monitor, char *formation, CoordinatorNodeAddress *coordinatorNodeAddress); bool monitor_get_most_advanced_standby(Monitor *monitor, diff --git a/src/bin/pg_autoctl/service_archiver.c b/src/bin/pg_autoctl/service_archiver.c index 63f6d7070..81bdd844d 100644 --- a/src/bin/pg_autoctl/service_archiver.c +++ b/src/bin/pg_autoctl/service_archiver.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,15 @@ #define ARCHIVER_XLOG_SEGMENTS_PER_XLOGID \ (((uint64_t) 0x100000000) / ARCHIVER_WAL_SEGMENT_SIZE) +/* + * How often service_archiver_loop() reports storage usage, in ticks + * (PG_AUTOCTL_KEEPER_SLEEP_TIME apart, currently 1s each) -- directory_size() + * walks the archiver's whole pgdata (walcache + basebackups, potentially + * many GB across several retained backups), real I/O work unlike the other + * per-tick checks in this loop, so it isn't worth doing every single tick. + */ +#define ARCHIVER_STORAGE_REPORT_TICKS 30 + /* * Last WAL filename already reported to the monitor, so each tick only * reports newly-appeared segments instead of re-scanning and re-reporting @@ -154,6 +164,27 @@ service_archiver_stop_pgreceivewal(void) * node's local root directory" role here without ever holding a real * Postgres cluster). Idempotent: stops any previously-tracked child first, * exactly like fsm_init_standby's own upstream reuse pattern. + * + * Passes -S/--slot, naming the slot exactly the way keeper_create_and_drop_ + * replication_slots()/pgsql_replication_slot_create_and_drop() (keeper.c, + * primary_standby.c, pgsql.c) already name it for an ordinary standby -- + * REPLICATION_SLOT_NAME_DEFAULT + "_" + this archiver's own node id. That + * mechanism runs on every primary-role node regardless of the other node's + * kind (AutoFailoverOtherNodesList() has no hasPgData filter, node_active_ + * protocol.c's get_other_nodes()), eagerly creating and maintaining this + * exact slot on whichever node is currently primary the same way it does + * for every real standby -- nothing on the primary side needs to change for + * this to work. Without a slot, a pg_receivewal whose first connection + * attempt loses the startup HBA-propagation race (a real, observed + * scenario) restarts streaming from the server's then-current position + * instead of resuming, permanently and silently skipping every WAL segment + * in between: report_wal_received() never reports them (they were simply + * never captured), and any consumer later asked to stream from inside that + * gap (e.g. pg_walsender's own START_REPLICATION, cmd_start_replication.c) + * would wait forever for a segment that will never exist. A replication + * slot fixes this the same way it does for a standby: the slot pins a + * restart_lsn at creation time and the server retains WAL back to it + * regardless of how many times the consumer disconnects and reconnects. */ bool service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) @@ -205,8 +236,15 @@ service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) primaryNode->host, primaryNode->port, PG_AUTOCTL_REPLICA_USERNAME, config->name); - log_info("Starting pg_receivewal against %s:%d, writing to \"%s\"", - primaryNode->host, primaryNode->port, config->pgSetup.pgdata); + char slotName[MAXCONNINFO] = { 0 }; + + sformat(slotName, sizeof(slotName), "%s_%d", + REPLICATION_SLOT_NAME_DEFAULT, keeper->state.current_node_id); + + log_info("Starting pg_receivewal against %s:%d, writing to \"%s\", " + "using replication slot \"%s\"", + primaryNode->host, primaryNode->port, config->pgSetup.pgdata, + slotName); pid_t pid = fork(); @@ -219,7 +257,7 @@ service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) if (pid == 0) { /* child process: replace ourselves with pg_receivewal */ - char *args[8]; + char *args[10]; int argsIndex = 0; args[argsIndex++] = pgReceivewalPath; @@ -229,6 +267,8 @@ service_archiver_start_pgreceivewal(Keeper *keeper, NodeAddress *primaryNode) args[argsIndex++] = "-D"; args[argsIndex++] = config->pgSetup.pgdata; args[argsIndex++] = "--no-sync"; + args[argsIndex++] = "-S"; + args[argsIndex++] = slotName; args[argsIndex] = NULL; execv(pgReceivewalPath, args); @@ -294,7 +334,8 @@ wal_filename_compare(const void *a, const void *b) * arithmetic for the same filename layout. */ static void -wal_segment_end_lsn(const char *walFileName, char *lsn, size_t lsnSize) +wal_segment_position_lsn(const char *walFileName, uint64_t offsetInSegment, + char *lsn, size_t lsnSize) { char logIdHex[9] = { 0 }; char segHex[9] = { 0 }; @@ -306,11 +347,69 @@ wal_segment_end_lsn(const char *walFileName, char *lsn, size_t lsnSize) uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); uint64_t segno = (uint64_t) logId * ARCHIVER_XLOG_SEGMENTS_PER_XLOGID + seg; - uint64_t endOfSegment = (segno + 1) * ARCHIVER_WAL_SEGMENT_SIZE; + uint64_t position = segno * ARCHIVER_WAL_SEGMENT_SIZE + offsetInSegment; snprintf(lsn, lsnSize, "%X/%08X", - (uint32_t) (endOfSegment >> 32), - (uint32_t) (endOfSegment & 0xFFFFFFFF)); + (uint32_t) (position >> 32), + (uint32_t) (position & 0xFFFFFFFF)); +} + + +static void +wal_segment_end_lsn(const char *walFileName, char *lsn, size_t lsnSize) +{ + wal_segment_position_lsn(walFileName, ARCHIVER_WAL_SEGMENT_SIZE, lsn, lsnSize); +} + + +/* + * partial_segment_real_length reads a ".partial" WAL segment file (pre- + * allocated to its full ARCHIVER_WAL_SEGMENT_SIZE by pg_receivewal the + * moment it's created, matching real Postgres's own WAL file pre- + * allocation, XLogFileInitInternal) and returns the length of its real + * content, trimming the zero-padded unwritten tail -- same technique + * pg_walsender/cmd_start_replication.c's own trim_trailing_zeros() already + * applies when actually serving one of these files. + * + * Trusting a trailing zero run to mean "unwritten" isn't safe in the + * general case -- a real primary's own WAL segments get recycled (renamed + * and reused rather than freshly zero-filled, so old content can linger + * past the real write position) -- but pg_receivewal itself never + * recycles; every ".partial" file it ever creates is fresh, so this holds + * here specifically. + */ +static bool +partial_segment_real_length(const char *path, uint64_t *length) +{ + FILE *file = fopen(path, "rb"); + + if (file == NULL) + { + return false; + } + + char *buffer = malloc(ARCHIVER_WAL_SEGMENT_SIZE); + + if (buffer == NULL) + { + fclose(file); + return false; + } + + size_t got = fread(buffer, 1, ARCHIVER_WAL_SEGMENT_SIZE, file); + + fclose(file); + + while (got > 0 && buffer[got - 1] == 0) + { + got--; + } + + free(buffer); + + *length = (uint64_t) got; + + return true; } @@ -410,14 +509,126 @@ service_archiver_report_captured_wal(Keeper *keeper) /* - * service_archiver_update_current_lsn scans walcacheDir for the newest - * complete (non-".partial") WAL segment and updates keeper->postgres. - * currentLSN to the LSN just past its end -- the real, local, self- - * consistent "how far have I actually captured" position, reported to the - * monitor by keeper_node_active() the same way every other node kind - * reports its own currentLSN. + * service_archiver_position_path computes the local, host-only file both + * the archiver-capture and archiver-serve processes use to exchange the + * current captured LSN. The two are separate fork()ed processes (see + * service_archiver_run.c's own comment on why each gets an independent + * connection) -- each has its own private copy of the Keeper struct after + * the fork, so keeper->postgres.currentLSN as updated by this file's own + * service_archiver_update_current_lsn() is invisible to the archiver-serve + * process no matter how it's written; only a real, external, re-read-each- + * time channel like this file makes the value cross that boundary. Built + * from config->pathnames.config exactly like service_archiver_serve.c's own + * service_archiver_serve_routes_path(), so both independently-started + * processes compute the identical path from their own (identically loaded) + * config, without needing shared memory or IPC. + */ +static void +service_archiver_position_path(KeeperConfig *config, char *dest) +{ + path_in_same_directory(config->pathnames.config, + "archiver-position", dest); +} + + +/* + * service_archiver_persist_current_lsn writes keeper->postgres.currentLSN to + * the local position file (see service_archiver_position_path's own + * comment), atomically (write-to-tmp then rename, matching service_archiver_ + * serve_refresh_routes()'s own pattern) so a concurrent reader never + * observes a partial write. + */ +static bool +service_archiver_persist_current_lsn(Keeper *keeper) +{ + char path[MAXPGPATH] = { 0 }; + + service_archiver_position_path(&(keeper->config), path); + + char tmpPath[MAXPGPATH] = { 0 }; + + sformat(tmpPath, sizeof(tmpPath), "%s.tmp", path); + + FILE *fileStream = fopen_with_umask(tmpPath, "w", FOPEN_FLAGS_W, 0644); + + if (fileStream == NULL) + { + /* errors have already been logged */ + return false; + } + + fformat(fileStream, "%s\n", keeper->postgres.currentLSN); + + if (fclose(fileStream) == EOF) + { + log_warn("Failed to write file \"%s\": %m", tmpPath); + return false; + } + + if (rename(tmpPath, path) != 0) + { + log_warn("Failed to rename \"%s\" to \"%s\": %m", tmpPath, path); + return false; + } + + return true; +} + + +/* + * service_archiver_read_current_lsn reads back the position file written by + * service_archiver_persist_current_lsn(), for use by the (separate process) + * archiver-serve side. Returns false (lsnOut left untouched) when the file + * doesn't exist yet -- the archiver-capture process hasn't completed its + * first tick -- callers should fall back to "0/0" themselves. + */ +bool +service_archiver_read_current_lsn(KeeperConfig *config, + char *lsnOut, size_t lsnOutSize) +{ + char path[MAXPGPATH] = { 0 }; + + service_archiver_position_path(config, path); + + char *contents = NULL; + long fileSize = 0; + + if (!read_file_if_exists(path, &contents, &fileSize) || contents == NULL) + { + return false; + } + + char *nl = strchr(contents, '\n'); + + if (nl != NULL) + { + *nl = '\0'; + } + + strlcpy(lsnOut, contents, lsnOutSize); + free(contents); + + return lsnOut[0] != '\0'; +} + + +/* + * service_archiver_update_current_lsn scans walcacheDir for the newest WAL + * segment -- complete, or still ".partial" -- and updates keeper->postgres. + * currentLSN to the real, currently-captured position: the full segment + * boundary for a complete one, or the real (zero-tail-trimmed) content + * length within the current ".partial" one when that's the frontier. This + * is the single, out-of-band-maintained source of truth for "how far has + * this archiver actually captured" -- computed here, once, per tick, and + * from here alone: both keeper_node_active()'s own per-tick report to the + * monitor (the same way every other node kind reports its own currentLSN) + * and service_archiver_serve_refresh_routes()'s own routes-file "position" + * key (service_archiver_serve.c) read via service_archiver_read_current_lsn() + * above, rather than each independently re-deriving it by scanning WAL file + * content on their own -- one canonical value, not several that could + * disagree. * - * This is what makes an archiving node a real, rankable candidate for + * This is also what makes an archiving node a real, rankable candidate for * pgautofailover.get_most_advanced_standby() during a failover election: * that query already has no kind-based exclusion and already considers any * node reporting REPORT_LSN_STATE (an archiving node passes through it @@ -441,36 +652,123 @@ service_archiver_update_current_lsn(Keeper *keeper) return; } - char best[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + char bestComplete[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + char bestPartial[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { - if (!is_wal_segment_filename(entry->d_name)) + if (is_wal_segment_filename(entry->d_name)) { + if (bestComplete[0] == '\0' || strcmp(entry->d_name, bestComplete) > 0) + { + strlcpy(bestComplete, entry->d_name, sizeof(bestComplete)); + } + continue; } - if (best[0] == '\0' || strcmp(entry->d_name, best) > 0) + const char *partialSuffix = ".partial"; + size_t nameLen = strlen(entry->d_name); + size_t suffixLen = strlen(partialSuffix); + + if (nameLen == ARCHIVER_WAL_FNAME_LEN + suffixLen && + strcmp(entry->d_name + ARCHIVER_WAL_FNAME_LEN, partialSuffix) == 0) { - strlcpy(best, entry->d_name, sizeof(best)); + char segPart[ARCHIVER_WAL_FNAME_LEN + 1] = { 0 }; + + memcpy(segPart, entry->d_name, ARCHIVER_WAL_FNAME_LEN); + + if (is_wal_segment_filename(segPart) && + (bestPartial[0] == '\0' || strcmp(segPart, bestPartial) > 0)) + { + strlcpy(bestPartial, segPart, sizeof(bestPartial)); + } } } closedir(dir); - if (best[0] == '\0') + /* + * A ".partial" file only ever exists for the segment actively being + * written, always the same as or newer than the newest complete one -- + * whenever it exists at all, it's the real frontier. + */ + if (bestPartial[0] != '\0' && + (bestComplete[0] == '\0' || strcmp(bestPartial, bestComplete) >= 0)) + { + char path[MAXPGPATH]; + uint64_t realLength = 0; + + snprintf(path, sizeof(path), "%s/%s.partial", walcacheDir, bestPartial); + + if (partial_segment_real_length(path, &realLength)) + { + wal_segment_position_lsn(bestPartial, realLength, + keeper->postgres.currentLSN, + sizeof(keeper->postgres.currentLSN)); + return; + } + + /* fall through to the complete segment below on read failure */ + } + + if (bestComplete[0] == '\0') { strlcpy(keeper->postgres.currentLSN, "0/0", sizeof(keeper->postgres.currentLSN)); return; } - wal_segment_end_lsn(best, keeper->postgres.currentLSN, + wal_segment_end_lsn(bestComplete, keeper->postgres.currentLSN, sizeof(keeper->postgres.currentLSN)); } +/* + * service_archiver_report_storage reports this archiver's own disk usage + * (directory_size() over its whole pgdata -- walcache and basebackups + * share the same root, see service_archiver_serve.c's own header comment) + * and free space (statvfs's f_bavail, "available to a non-privileged + * process" -- what actually predicts whether the next base backup or WAL + * segment fits, not f_bfree's superuser-reserved total) to the monitor. + * + * Skips the report outright on a statvfs failure rather than reporting a + * free space of zero: unlike directory_size()'s own "best effort, this is + * informational" stance, a wrong zero here would misleadingly read as + * "completely full" to anything watching (pg_autoctl watch's own archivers + * section). + */ +static bool +service_archiver_report_storage(Keeper *keeper) +{ + KeeperConfig *config = &(keeper->config); + const char *pgdata = config->pgSetup.pgdata; + + uint64_t usedBytes = directory_size(pgdata); + + struct statvfs fsStats = { 0 }; + + if (statvfs(pgdata, &fsStats) != 0) + { + log_warn("Failed to statvfs \"%s\": %m, skipping this storage report", + pgdata); + return false; + } + + uint64_t freeBytes = (uint64_t) fsStats.f_bavail * (uint64_t) fsStats.f_frsize; + + if (!monitor_report_archiver_storage(&(keeper->monitor), config->archiverId, + usedBytes, freeBytes)) + { + log_warn("Failed to report storage usage to the monitor, will retry"); + return false; + } + + return true; +} + + /* * service_archiver_loop is the archiver's own node_active() reporting loop * -- deliberately not keeper_node_active_loop (service_keeper.c): that @@ -509,11 +807,46 @@ service_archiver_loop(Keeper *keeper) */ strlcpy(keeper->postgres.currentLSN, "0/0", sizeof(keeper->postgres.currentLSN)); + int tickCount = 0; + while (!asked_to_stop && !asked_to_stop_fast && !asked_to_quit) { MonitorAssignedState assignedState = { 0 }; (void) service_archiver_update_current_lsn(keeper); + (void) service_archiver_persist_current_lsn(keeper); + + /* + * An archiver never sets postgres.pgIsRunning through the usual + * keeper_update_pg_state() path (there's no real Postgres to + * query, see haspgdata's own design comment) -- it stays at its + * zero-initialized false forever otherwise. That's not just + * cosmetic: the monitor's own NodeIsHealthy() (node_metadata.c) + * unconditionally requires pgIsRunning to be true before ever + * considering a node healthy, in every one of its branches -- + * including group_state_machine.c's own FAST_FORWARD candidate + * selection, which refuses to assign fast_forward against an + * unhealthy WAL source. Without this, an archiver could never + * legitimately serve as a FAST_FORWARD WAL source no matter how + * caught up it was: the monitor would always see it as unhealthy + * and never select it. + * + * Deliberately NOT tied to service_archiver_pgreceivewal_is_ + * running(): that reflects a narrower "is WAL actively being + * captured from a live primary right now" fact, which is + * legitimately false exactly during the window a FAST_FORWARD + * candidate needs the archiver most -- pg_receivewal has nothing + * to stream from once the primary it was following is dead, but + * the WAL this archiver already captured is still there and still + * servable via pg_walsender regardless. pgIsRunning here means + * "this archiver's own keeper service is alive and reporting", + * the same thing a real node's pgIsRunning=true ultimately proves + * about itself -- a crashed or partitioned archiver is still + * caught by the monitor's own separate report-staleness check + * (NodeIsUnhealthy's reportTime/unhealthyTimeoutMs), which + * doesn't depend on this flag at all. + */ + keeper->postgres.pgIsRunning = true; if (!keeper_load_state(keeper)) { @@ -571,6 +904,11 @@ service_archiver_loop(Keeper *keeper) { log_warn("Failed to generate a base backup, will retry"); } + + if (tickCount % ARCHIVER_STORAGE_REPORT_TICKS == 0) + { + (void) service_archiver_report_storage(keeper); + } } else { @@ -583,6 +921,7 @@ service_archiver_loop(Keeper *keeper) } sleep(PG_AUTOCTL_KEEPER_SLEEP_TIME); + ++tickCount; } (void) service_archiver_stop_pgreceivewal(); diff --git a/src/bin/pg_autoctl/service_archiver.h b/src/bin/pg_autoctl/service_archiver.h index afb209b70..522b661ff 100644 --- a/src/bin/pg_autoctl/service_archiver.h +++ b/src/bin/pg_autoctl/service_archiver.h @@ -22,6 +22,9 @@ bool service_archiver_pgreceivewal_is_running(void); bool service_archiver_report_captured_wal(Keeper *keeper); +bool service_archiver_read_current_lsn(KeeperConfig *config, + char *lsnOut, size_t lsnOutSize); + bool service_archiver_loop(Keeper *keeper); #endif /* SERVICE_ARCHIVER_H */ diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c index 5dd439c2b..8f76692f4 100644 --- a/src/bin/pg_autoctl/service_archiver_basebackup.c +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -3,25 +3,34 @@ * Archiving & Disaster Recovery: base backup generation, both `live` and * `replay`/`volatile` sources (Milestone 5, per the Build order in * ~/dev/temp/archiving-disaster-recovery.md: "live first, then - * replay/volatile"). `replay`/`persistent` is a later milestone -- - * that mode keeps its staging instance resident as a `warm-standby` - * `archiver_node` row, which doesn't exist until Milestone 6. + * replay/volatile"), plus policy-driven scheduling and retention -- + * appended to M5 rather than left as a follow-up, so the archiver's own + * base-backup production is a real, bounded resource (frequency-gated, + * count/age-pruned) before Milestones 6/7/8 (warm standby, PITR, cloud + * push) start building on top of it. `replay`/`persistent` is still a + * later milestone -- that mode keeps its staging instance resident as a + * `warm-standby` `archiver_node` row, which doesn't exist until + * Milestone 6. * - * Trigger scope for this pass: bootstrap, then exactly one replay-sourced - * backup to exercise that pipeline once -- both hardcoded here, not read - * from basebackup_policy. A group with zero existing base backups gets one - * immediately, sourced live (matching the design doc's own bootstrap rule: - * nothing to replay from yet on the first run). Once that lands, the very - * next tick takes exactly one more, this time sourced replay/volatile, and - * after that this file goes quiet for the group. Real frequency-driven - * scheduling (basebackup_policy's own `source`/`replaymode`/`frequency`/ - * `onpromotion`/retention fields, resolved through `get_archiver_policy()`/ - * `get_basebackup_policy()`) needs that policy wired through the CLI - * first -- out of scope here. This is a deliberate scope cut, not an - * oversight: the milestone-defining new capability is the replay mechanism - * itself (extract, replay, promote, snapshot, discard), not a general - * scheduler -- see the design doc's own build order, which lists "warm - * standby" and its `advance`/scheduling machinery as later milestones. + * Trigger scope: bootstrap is always `live` (nothing to replay from yet on + * the first run, matching the design doc's own bootstrap rule), every + * backup after that follows basebackup_policy's own `source`/`replaymode` + * (resolved via monitor_get_basebackup_policy_for_group(), which chains + * get_archiver_policy()'s group-override / formation-default / schema- + * default fallback the same way wal_archived()'s own archiver_quorum + * lookup does), gated on `frequency` seconds having elapsed since the + * newest existing backup -- or fired immediately regardless of frequency + * when `onpromotion` is set and the group's primary has changed since the + * last tick that checked (see get_current_primary_node_id()'s own + * comment). After each successful completion, retention prunes anything + * beyond `maxcount` or older than `maxage` (apply_basebackup_retention()): + * removes the directory, then report_basebackup_deleted() on the monitor, + * which cascades to prune_archiver_wal() on its own. `concurrency` is + * read but not enforced: this milestone's own single-membership scope + * (one archiver, one group) already limits this file to one base backup + * production job in flight at a time (basebackup_child_is_running()) -- + * running several concurrently only has meaning once an archiver can serve + * more than one (formation, group) at once, a later milestone's concern. * * Target selection ('live') follows the design doc's own precedence, * minus its warm-standby tier (a later milestone, nothing to select from @@ -307,9 +316,11 @@ accumulate_file_size(const char *path, const struct stat *sb, * dirPath. Best effort: sizebytes is informational only (nothing in the * monitor schema's own logic -- prune_archiver_wal() included -- reads it * back), so a failure here is not worth failing an otherwise-successful - * base backup over. + * base backup over. Exposed (service_archiver_basebackup.h) for service_ + * archiver.c's own periodic storage-usage report, over the archiver's + * whole pgdata rather than just one backup directory. */ -static uint64_t +uint64_t directory_size(const char *dirPath) { directorySizeAccumulator = 0; @@ -488,23 +499,110 @@ report_basebackup(Keeper *keeper, NodeAddress *endLsnSource, } +/* + * apply_basebackup_retention lists every complete base backup for this + * group (newest first, list_basebackups()'s own ordering) and prunes + * whatever policy says shouldn't survive: anything beyond the newest + * maxcount, or older than maxage, whichever fires first for a given + * backup -- a backup can be pruned for either reason independently, not + * only once maxcount is already exceeded. maxcount <= 0 or maxage_seconds + * <= 0 disables that particular rule (there is no real-world policy where + * "keep zero backups" or "expire instantly" is the intended behavior; the + * schema's own CHECK constraints don't allow either as a stored value, + * but this stays defensive against a hand-edited row or a future relaxed + * constraint). + * + * Best effort past the first failure: one backup's directory failing to + * remove (e.g. a permissions issue) does not stop the rest of the list + * from being evaluated -- each one is independent, and the failed one + * simply gets retried on the next cycle since it's still 'complete' and + * still over its own retention rule. + */ +static bool +apply_basebackup_retention(Keeper *keeper, BasebackupPolicy *policy) +{ + KeeperConfig *config = &(keeper->config); + BasebackupInfoArray backups = { 0 }; + + if (!monitor_list_basebackups(&(keeper->monitor), + config->formation, config->groupId, + &backups)) + { + log_warn("Failed to list base backups for retention, will retry " + "on the next cycle"); + return false; + } + + time_t now = time(NULL); + bool success = true; + + for (int i = 0; i < backups.count; i++) + { + BasebackupInfo *backup = &(backups.backups[i]); + + bool beyondMaxCount = policy->maxCount > 0 && i >= policy->maxCount; + bool beyondMaxAge = policy->maxAgeSeconds > 0 && + (now - (time_t) backup->startedAtEpoch) > + policy->maxAgeSeconds; + + if (!beyondMaxCount && !beyondMaxAge) + { + continue; + } + + log_info("Pruning base backup \"%s\" (%s)", + backup->label, + beyondMaxCount ? "beyond maxcount" : "past maxage"); + + if (directory_exists(backup->storageLocation) && + !rmtree(backup->storageLocation, true)) + { + log_warn("Failed to remove base backup directory \"%s\", will " + "retry on the next cycle", backup->storageLocation); + success = false; + continue; + } + + if (!monitor_report_basebackup_deleted(&(keeper->monitor), + backup->basebackupId)) + { + log_warn("Failed to report base backup %" PRId64 " as deleted " + "to the monitor, will retry on the next cycle", + backup->basebackupId); + success = false; + } + } + + return success; +} + + /* * generate_live_basebackup is the forked child's own body for a `live` - * backup: run pg_basebackup against source to completion, then report it. - * Runs in its own process, with its own monitor connection (the parent's - * keeper->monitor is not fork-safe to share, exactly as - * service_archiver_run.c's own supervised children already document). + * backup: run pg_basebackup against source to completion, report it, then + * apply retention. Runs in its own process, with its own monitor + * connection (the parent's keeper->monitor is not fork-safe to share, + * exactly as service_archiver_run.c's own supervised children already + * document). */ static bool generate_live_basebackup(Keeper *keeper, NodeAddress *source, - const char *backupDir, const char *label) + const char *backupDir, const char *label, + BasebackupPolicy *policy) { if (!run_pg_basebackup(&(keeper->config), source, backupDir, label)) { return false; } - return report_basebackup(keeper, source, backupDir, label, "live", NULL); + if (!report_basebackup(keeper, source, backupDir, label, "live", NULL)) + { + return false; + } + + (void) apply_basebackup_retention(keeper, policy); + + return true; } @@ -598,10 +696,23 @@ write_replay_recovery_config(const char *stagingDir, const char *walcacheDir) char conf[BUFSIZE] = { 0 }; + /* + * ssl = off: the copied postgresql.conf/postgresql.auto.conf still + * carries the source node's own ssl_cert_file/ssl_key_file settings + * (typically absolute paths into *that node's* PGDATA, e.g. from + * --ssl-self-signed) -- meaningless here, since this archiver has no + * Postgres SSL certs of its own to begin with. Left enabled, the + * staging instance fails outright at startup ("could not load server + * certificate file ...: No such file or directory"). Safe to disable + * unconditionally: this instance only ever accepts the loopback + * pg_basebackup connection below, for the lifetime of one throwaway + * cycle. + */ sformat(conf, sizeof(conf), "\n" "# added by pg_autoctl's archiver replay/volatile base backup generation\n" - "restore_command = 'cp \"%s/%%f\" \"%%p\"'\n", + "restore_command = 'cp \"%s/%%f\" \"%%p\"'\n" + "ssl = off\n", walcacheDir); return append_to_file(conf, strlen(conf), confPath); @@ -766,7 +877,8 @@ wait_for_replay_promotion(const char *connInfo, int timeoutSeconds) */ static bool generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, - const char *backupDir, const char *label) + const char *backupDir, const char *label, + BasebackupPolicy *policy) { KeeperConfig *config = &(keeper->config); @@ -833,7 +945,12 @@ generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, ok = run_pg_basebackup(config, &stagingNode, backupDir, label) && report_basebackup(keeper, &stagingNode, backupDir, label, - "replay", "volatile"); + "replay", policy->replayMode); + + if (ok) + { + (void) apply_basebackup_retention(keeper, policy); + } } stop_staging_postgres(); @@ -849,13 +966,61 @@ generate_replay_basebackup(Keeper *keeper, const char *sourceBackupDir, } +/* + * lastKnownPrimaryNodeId tracks the group's primary across ticks, purely + * in-memory (reset on archiver restart, same lifetime as basebackupPid/ + * stagingPostgresPid above) -- -1 means "not observed yet", which the + * onpromotion check below treats as "nothing to compare against", not "a + * promotion just happened" (that would misfire a forced backup on this + * process's very first tick). + */ +static int64_t lastKnownPrimaryNodeId = -1; + + +/* + * get_current_primary_node_id finds the group's current primary via the + * same monitor_get_nodes() call select_basebackup_source() already makes + * for its own, different purpose (picking a live source) -- kept as a + * separate round trip rather than sharing state across the two call + * sites, since either can run without the other on a given tick + * (onpromotion is checked unconditionally; select_basebackup_source() only + * runs once a backup already turns out to be due). Returns false (not an + * error) when the group currently has no primary at all (mid-election) -- + * callers should skip the comparison for this tick rather than treat that + * as "no promotion". + */ +static bool +get_current_primary_node_id(Keeper *keeper, int64_t *primaryNodeId) +{ + NodeAddressArray nodeArray = { 0 }; + + if (!monitor_get_nodes(&(keeper->monitor), + keeper->config.formation, + keeper->config.groupId, + &nodeArray)) + { + return false; + } + + for (int i = 0; i < nodeArray.count; i++) + { + if (nodeArray.nodes[i].isPrimary) + { + *primaryNodeId = nodeArray.nodes[i].nodeId; + return true; + } + } + + return false; +} + + /* * service_archiver_maybe_generate_basebackup checks, once per * service_archiver_loop() tick, whether a base backup generation is due * for this group and -- if so, and no generation is already in flight -- * forks a child to produce one. See this file's own header comment for - * the full trigger scope of this pass (bootstrap live, then exactly one - * replay/volatile backup to exercise that pipeline). + * the full policy-driven trigger scope this implements. */ bool service_archiver_maybe_generate_basebackup(Keeper *keeper) @@ -866,31 +1031,82 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) } KeeperConfig *config = &(keeper->config); - bool found = false; - char storageLocation[MAXPGPATH] = { 0 }; - char latestSource[NAMEDATALEN] = { 0 }; - int latestTimeline = 0; - - if (!monitor_get_latest_basebackup_info(&(keeper->monitor), - config->formation, - config->groupId, - NULL, /* any source */ - storageLocation, - sizeof(storageLocation), - latestSource, - sizeof(latestSource), - &latestTimeline, - &found)) + + BasebackupPolicy policy = { 0 }; + bool foundPolicy = false; + + if (!monitor_get_basebackup_policy_for_group(&(keeper->monitor), + config->formation, + config->groupId, + &policy, &foundPolicy)) { /* errors already logged */ return false; } - if (found && strcmp(latestSource, "replay") == 0) + if (!foundPolicy) + { + /* shouldn't happen: the schema's own 'default' policy always + * exists, and get_archiver_policy()'s own three-way fallback + * always resolves to at least that row */ + log_warn("Failed to resolve a base-backup policy for \"%s\"/%d, " + "skipping this cycle", config->formation, config->groupId); + return true; + } + + BasebackupInfoArray backups = { 0 }; + + if (!monitor_list_basebackups(&(keeper->monitor), + config->formation, config->groupId, + &backups)) + { + /* errors already logged */ + return false; + } + + bool bootstrap = (backups.count == 0); + + /* + * Runs every tick regardless of whether a backup is otherwise due, so + * lastKnownPrimaryNodeId always reflects the most recently observed + * primary -- skipping this update on a due-anyway tick would compare + * a future promotion against a stale value from several ticks back + * and misfire. + */ + bool forcedByPromotion = false; + + if (policy.onPromotion) + { + int64_t currentPrimaryNodeId = 0; + + if (get_current_primary_node_id(keeper, ¤tPrimaryNodeId)) + { + if (lastKnownPrimaryNodeId >= 0 && + lastKnownPrimaryNodeId != currentPrimaryNodeId) + { + forcedByPromotion = true; + + log_info("Forcing a new base backup: the group's primary " + "changed (node %" PRId64 " -> node %" PRId64 ")", + lastKnownPrimaryNodeId, currentPrimaryNodeId); + } + + lastKnownPrimaryNodeId = currentPrimaryNodeId; + } + } + + bool due = bootstrap || forcedByPromotion; + + if (!due) + { + time_t now = time(NULL); + time_t elapsed = now - (time_t) backups.backups[0].startedAtEpoch; + + due = elapsed >= (time_t) policy.frequencySeconds; + } + + if (!due) { - /* both the bootstrap live backup and this pass's own one-time - * replay exercise are done; real scheduling is a follow-up, see - * this file's own header comment */ return true; } @@ -905,6 +1121,11 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) return false; } + /* bootstrap is always 'live' -- nothing to replay from yet, matching + * the design doc's own bootstrap rule -- every backup after that + * follows the resolved policy's own source */ + bool useReplay = !bootstrap && strcmp(policy.source, "replay") == 0; + time_t now = time(NULL); struct tm nowUTC = { 0 }; @@ -913,7 +1134,7 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) char label[NAMEDATALEN] = { 0 }; strftime(label, sizeof(label), - found ? "basebackup-replay-%Y%m%dT%H%M%SZ" + useReplay ? "basebackup-replay-%Y%m%dT%H%M%SZ" : "basebackup-%Y%m%dT%H%M%SZ", &nowUTC); @@ -922,20 +1143,23 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) sformat(backupDir, sizeof(backupDir), "%s/%s", backupsDir, label); /* - * sourceBackupDir must be captured now, in the parent, into a - * fixed-size buffer the forked child can safely read after fork(): - * storageLocation itself is a local, stack-allocated array, still - * valid across fork() (the child gets its own copy of the whole - * address space), so this is really just documenting that fact. + * sourceBackupDir/policy must be captured now, in the parent, into + * buffers the forked child can safely read after fork(): both are + * local, stack-allocated, still valid across fork() (the child gets + * its own copy of the whole address space). */ char sourceBackupDir[MAXPGPATH] = { 0 }; - strlcpy(sourceBackupDir, storageLocation, sizeof(sourceBackupDir)); + if (useReplay) + { + strlcpy(sourceBackupDir, backups.backups[0].storageLocation, + sizeof(sourceBackupDir)); + } NodeAddress liveSource = { 0 }; bool haveLiveSource = false; - if (!found) + if (!useReplay) { if (!select_basebackup_source(keeper, &liveSource)) { @@ -964,9 +1188,10 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) (void) set_ps_title("pg_autoctl: archiver basebackup"); bool ok = haveLiveSource - ? generate_live_basebackup(keeper, &liveSource, backupDir, label) + ? generate_live_basebackup(keeper, &liveSource, backupDir, + label, &policy) : generate_replay_basebackup(keeper, sourceBackupDir, - backupDir, label); + backupDir, label, &policy); exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); } diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.h b/src/bin/pg_autoctl/service_archiver_basebackup.h index 2313a8f1e..c005016f6 100644 --- a/src/bin/pg_autoctl/service_archiver_basebackup.h +++ b/src/bin/pg_autoctl/service_archiver_basebackup.h @@ -15,5 +15,6 @@ #include "keeper.h" bool service_archiver_maybe_generate_basebackup(Keeper *keeper); +uint64_t directory_size(const char *dirPath); #endif /* SERVICE_ARCHIVER_BASEBACKUP_H */ diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c index bcf1deebf..d5fff82f7 100644 --- a/src/bin/pg_autoctl/service_archiver_serve.c +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -40,6 +40,7 @@ #include "file_utils.h" #include "log.h" #include "monitor.h" +#include "service_archiver.h" #include "signals.h" /* how often service_archiver_serve_loop() re-checks pg_walsender's @@ -362,6 +363,34 @@ service_archiver_serve_refresh_routes(Keeper *keeper) fformat(fileStream, "[%s/%d]\n", config->formation, config->groupId); fformat(fileStream, "walcache = %s\n", config->pgSetup.pgdata); + /* + * The single, out-of-band-maintained "how far have I actually + * captured" value -- see service_archiver_update_current_lsn()'s own + * comment (service_archiver.c) for why pg_walsender should read this + * rather than re-derive it by scanning WAL file content itself: + * cmd_base_backup.c's own end-of-backup position, and cmd_identify_ + * system.c's own xlogpos, both prefer this route-file value when + * present, falling back to their own (WAL-cache-scanning) logic only + * when it's missing -- an older archiver-serve binary talking to a + * newer routes file, or vice versa, during a rolling upgrade. + * + * Read via service_archiver_read_current_lsn() rather than keeper-> + * postgres.currentLSN directly: this process (archiver-serve) and the + * one that actually maintains that value (archiver-capture, + * service_archiver.c) are separate fork()ed processes (service_ + * archiver_run.c) with independent copies of the Keeper struct after + * the fork -- keeper->postgres.currentLSN in *this* process's memory + * is permanently frozen at whatever it was at fork time (typically + * empty/"0/0"), never updated by the sibling process's own writes. + * The position file is the real, re-read-every-refresh channel that + * actually crosses that boundary. + */ + char currentLSN[PG_LSN_MAXLENGTH] = "0/0"; + + (void) service_archiver_read_current_lsn(config, currentLSN, sizeof(currentLSN)); + + fformat(fileStream, "position = %s\n", currentLSN); + if (found) { fformat(fileStream, "basebackup = %s\n", basebackupLocation); diff --git a/src/bin/pg_autoctl/watch.c b/src/bin/pg_autoctl/watch.c index 9426d107d..65bde11a3 100644 --- a/src/bin/pg_autoctl/watch.c +++ b/src/bin/pg_autoctl/watch.c @@ -40,6 +40,7 @@ #include "pidfile.h" #include "state.h" #include "string_utils.h" +#include "system_utils.h" #include "watch.h" #include "watch_colspecs.h" @@ -51,6 +52,7 @@ static bool cli_watch_process_keys(WatchContext *context); static int print_watch_header(WatchContext *context, int r); static int print_watch_footer(WatchContext *context); static int print_nodes_array(WatchContext *context, int r, int c); +static int print_archivers_array(WatchContext *context, int r, int c); static int print_events_array(WatchContext *context, int r, int c); static void print_current_time(WatchContext *context, int r); @@ -249,6 +251,7 @@ cli_watch_update_from_monitor(WatchContext *context) { Monitor *monitor = &(context->monitor); CurrentNodeStateArray *nodesArray = &(context->nodesArray); + ArchiverInfoArray *archiversArray = &(context->archiversArray); MonitorEventsArray *eventsArray = &(context->eventsArray); /* @@ -268,6 +271,12 @@ cli_watch_update_from_monitor(WatchContext *context) return false; } + if (!monitor_get_archivers(monitor, context->formation, archiversArray)) + { + /* errors have already been logged */ + return false; + } + if (!monitor_get_formation_number_sync_standbys( monitor, context->formation, @@ -493,7 +502,20 @@ cli_watch_render(WatchContext *context, WatchContext *previous) int firstNodeRow = nodeHeaderRow + 1; int lastNodeRow = firstNodeRow + context->nodesArray.count - 1; - int eventHeaderRow = lastNodeRow + 2; /* blank line, evenzt headers */ + /* + * The archivers area only takes up screen space when there's at least + * one archiver attached to the formation -- an ordinary cluster with no + * archivers should look exactly like it did before this section existed + * (firstArchiverRow > lastArchiverRow makes it an empty range, which the + * area-selection cascade below naturally skips over). + */ + int archiverHeaderRow = lastNodeRow + 2; /* blank line, archiver headers */ + int firstArchiverRow = archiverHeaderRow + 1; + int lastArchiverRow = firstArchiverRow + context->archiversArray.count - 1; + + int eventHeaderRow = (context->archiversArray.count > 0) + ? lastArchiverRow + 2 + : lastNodeRow + 2; /* blank line, event headers */ int firstEventRow = eventHeaderRow + 1; int lastEventRow = firstEventRow + context->eventsArray.count - 1; @@ -513,9 +535,11 @@ cli_watch_render(WatchContext *context, WatchContext *previous) * that's part of the data: avoid empty separation lines, avoid header * lines. * - * We conceptually divide the screen in two areas: first, the nodes array - * area, and then the events area. When scrolling away from an area we may - * jump to the other area directly. + * We conceptually divide the screen in three areas: the nodes array + * area, the archivers area, and the events area. When scrolling away + * from an area we may jump to the other area directly -- area 2 + * (archivers) is skipped over entirely when there are no archivers to + * show (firstArchiverRow > lastArchiverRow). */ if (context->selectedArea == 1) { @@ -525,17 +549,46 @@ cli_watch_render(WatchContext *context, WatchContext *previous) } else if (context->selectedRow > lastNodeRow) { - context->selectedArea = 2; - context->selectedRow = firstEventRow; + if (context->archiversArray.count > 0) + { + context->selectedArea = 2; + context->selectedRow = firstArchiverRow; + } + else + { + context->selectedArea = 3; + context->selectedRow = firstEventRow; + } } } else if (context->selectedArea == 2) { - if (context->selectedRow < firstEventRow) + if (context->selectedRow < firstArchiverRow) { context->selectedArea = 1; context->selectedRow = lastNodeRow; } + else if (context->selectedRow > lastArchiverRow) + { + context->selectedArea = 3; + context->selectedRow = firstEventRow; + } + } + else if (context->selectedArea == 3) + { + if (context->selectedRow < firstEventRow) + { + if (context->archiversArray.count > 0) + { + context->selectedArea = 2; + context->selectedRow = lastArchiverRow; + } + else + { + context->selectedArea = 1; + context->selectedRow = lastNodeRow; + } + } else if (context->selectedRow > lastEventRow) { context->selectedRow = lastEventRow; @@ -556,6 +609,16 @@ cli_watch_render(WatchContext *context, WatchContext *previous) (void) clear_line_at(printedRows); + if (context->archiversArray.count > 0) + { + (void) clear_line_at(++printedRows); + + int archiverRows = print_archivers_array(context, archiverHeaderRow, 0); + printedRows += archiverRows; + + (void) clear_line_at(printedRows); + } + /* * Now print the events array. Because that operation is more expensive, * and because most of the times there is no event happening, we compare @@ -751,6 +814,93 @@ print_nodes_array(WatchContext *context, int r, int c) } +/* + * print_archivers_array prints one row per archiver attached to the current + * formation: name, host, its ARCHIVING membership's FSM state, and its most + * recently reported storage usage/free space. Unlike print_nodes_array, + * this doesn't go through the ColPolicy width-matching machinery + * (watch_colspecs.h) -- five fixed-width columns is simple enough not to + * need it, and this section only ever appears at all when there's at least + * one archiver to show. + */ +#define ARCHIVER_NAME_COL_LEN 20 +#define ARCHIVER_HOST_COL_LEN 20 +#define ARCHIVER_STATE_COL_LEN 12 +#define ARCHIVER_SIZE_COL_LEN 10 + +static int +print_archivers_array(WatchContext *context, int r, int c) +{ + ArchiverInfoArray *archiversArray = &(context->archiversArray); + + int lines = 0; + int currentRow = r; + + clear_line_at(currentRow); + + attron(A_STANDOUT); + mvprintw(currentRow, c, "%-*s %-*s %-*s %*s %*s ", + ARCHIVER_NAME_COL_LEN, "Archiver Name", + ARCHIVER_HOST_COL_LEN, "Host", + ARCHIVER_STATE_COL_LEN, "State", + ARCHIVER_SIZE_COL_LEN, "Used", + ARCHIVER_SIZE_COL_LEN, "Free"); + attroff(A_STANDOUT); + + ++currentRow; + ++lines; + + for (int index = 0; index < archiversArray->count; index++) + { + ArchiverInfo *archiver = &(archiversArray->archivers[index]); + bool selected = currentRow == context->selectedRow; + + char usedStr[NAMEDATALEN] = "?"; + char freeStr[NAMEDATALEN] = "?"; + + if (archiver->hasStorageStats) + { + pretty_print_bytes(usedStr, sizeof(usedStr), archiver->usedBytes); + pretty_print_bytes(freeStr, sizeof(freeStr), archiver->freeBytes); + } + + const char *stateStr = + archiver->hasNode + ? NodeStateToString(archiver->reportedState) + : "?"; + + clear_line_at(currentRow); + + if (selected) + { + attron(A_REVERSE); + } + + mvprintw(currentRow, c, "%-*s %-*s %-*s %*s %*s ", + ARCHIVER_NAME_COL_LEN, archiver->archiverName, + ARCHIVER_HOST_COL_LEN, archiver->hostname, + ARCHIVER_STATE_COL_LEN, stateStr, + ARCHIVER_SIZE_COL_LEN, usedStr, + ARCHIVER_SIZE_COL_LEN, freeStr); + + if (selected) + { + attroff(A_REVERSE); + } + + ++currentRow; + ++lines; + + if (context->rows <= currentRow) + { + break; + } + } + + return lines; +} + + /* * pick_column_spec chooses which column spec should be used depending on the * current size (rows, cols) of the display, and given update column specs with diff --git a/src/bin/pg_autoctl/watch.h b/src/bin/pg_autoctl/watch.h index 0a2e1e748..f41bc1472 100644 --- a/src/bin/pg_autoctl/watch.h +++ b/src/bin/pg_autoctl/watch.h @@ -51,7 +51,8 @@ typedef struct WatchContext int rows; int cols; int selectedRow; - int selectedArea; /* area 1: node states, area 2: node events */ + int selectedArea; /* area 1: node states, area 2: archivers, + * area 3: node events */ int startCol; WatchMoveFocus move; @@ -69,6 +70,7 @@ typedef struct WatchContext /* data to display */ CurrentNodeStateArray nodesArray; + ArchiverInfoArray archiversArray; MonitorEventsArray eventsArray; MonitorEventsHeaders eventsHeaders; } WatchContext; diff --git a/src/bin/pg_walsender/cmd_base_backup.c b/src/bin/pg_walsender/cmd_base_backup.c index a2851ea86..eb8805c81 100644 --- a/src/bin/pg_walsender/cmd_base_backup.c +++ b/src/bin/pg_walsender/cmd_base_backup.c @@ -31,6 +31,7 @@ */ #include +#include #include #include "postgres_fe.h" @@ -301,6 +302,202 @@ send_position_row(int sock, const char *lsn, const char *tli) } +/* + * find_reachable_end_position and its helpers below compute a base + * backup's "end of backup" position -- see this file's own header comment + * for where that fits in the wire sequence, and cmd_base_backup()'s own + * call site for why it must be a real, currently-reachable target rather + * than a stale re-send of the start position. + * + * Deliberately not pg_walsender/wal_dir_scan.c's own wal_dir_find_latest() + * (this project doesn't share code across its own binaries, see this + * file's own precedent of small, self-contained helpers): that function + * only ever considers a *complete* (non-".partial") segment, which is the + * right, conservative choice for IDENTIFY_SYSTEM/CREATE_REPLICATION_SLOT's + * own "confirmed durable" needs, but wrong here -- an archiver whose only + * WAL activity so far is still sitting in the current ".partial" segment + * (a real, common case: nothing has forced a segment switch yet) would + * make wal_dir_find_latest() report "nothing captured", sending BASE_ + * BACKUP straight back to the same stale start-of-backup fallback this + * whole mechanism exists to avoid. The archiver's walcache always has + * *something* real captured by the time a base backup exists at all + * (pg_receivewal streams from the moment archiving starts); the position + * within the current in-progress segment is exactly as reachable via + * START_REPLICATION as a completed one, once its zero-padded unwritten + * tail (pg_receivewal's own pre-allocation, matching real Postgres's + * XLogFileInitInternal) is trimmed off -- the same trim_trailing_zeros() + * logic cmd_start_replication.c already applies when actually serving it, + * applied here once, up front, to find where its real content ends. + */ +#define CBB_WAL_SEGMENT_SIZE UINT64CONST(0x1000000) +#define CBB_XLOG_SEGMENTS_PER_XLOGID (UINT64CONST(0x100000000) / CBB_WAL_SEGMENT_SIZE) +#define CBB_WAL_FNAME_LEN 24 + + +static bool +is_wal_segment_filename(const char *name) +{ + size_t len = strlen(name); + + if (len != CBB_WAL_FNAME_LEN) + { + return false; + } + + for (size_t i = 0; i < len; i++) + { + if (!isxdigit((unsigned char) name[i])) + { + return false; + } + } + + return true; +} + + +static bool +partial_segment_real_length(const char *path, uint64_t *length) +{ + FILE *file = fopen(path, "rb"); + + if (file == NULL) + { + return false; + } + + char *buffer = malloc(CBB_WAL_SEGMENT_SIZE); + + if (buffer == NULL) + { + fclose(file); + return false; + } + + size_t got = fread(buffer, 1, CBB_WAL_SEGMENT_SIZE, file); + + fclose(file); + + while (got > 0 && buffer[got - 1] == 0) + { + got--; + } + + free(buffer); + + *length = (uint64_t) got; + + return true; +} + + +static bool +find_reachable_end_position(const char *walcacheDir, uint32_t *timeline, + char *endLsn, size_t endLsnSize) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + char bestComplete[CBB_WAL_FNAME_LEN + 1] = { 0 }; + char bestPartial[CBB_WAL_FNAME_LEN + 1] = { 0 }; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + if (is_wal_segment_filename(entry->d_name)) + { + if (bestComplete[0] == '\0' || strcmp(entry->d_name, bestComplete) > 0) + { + strlcpy(bestComplete, entry->d_name, sizeof(bestComplete)); + } + + continue; + } + + const char *partialSuffix = ".partial"; + size_t nameLen = strlen(entry->d_name); + size_t suffixLen = strlen(partialSuffix); + + if (nameLen == CBB_WAL_FNAME_LEN + suffixLen && + strcmp(entry->d_name + CBB_WAL_FNAME_LEN, partialSuffix) == 0) + { + char segPart[CBB_WAL_FNAME_LEN + 1] = { 0 }; + + memcpy(segPart, entry->d_name, CBB_WAL_FNAME_LEN); + + if (is_wal_segment_filename(segPart) && + (bestPartial[0] == '\0' || strcmp(segPart, bestPartial) > 0)) + { + strlcpy(bestPartial, segPart, sizeof(bestPartial)); + } + } + } + + closedir(dir); + + /* + * The current frontier is whichever of the two is numerically later -- + * a ".partial" file only ever exists for the segment actively being + * written, always the same as or newer than the newest complete one. + */ + bool usePartial = bestPartial[0] != '\0' && + (bestComplete[0] == '\0' || + strcmp(bestPartial, bestComplete) >= 0); + + const char *chosen = usePartial ? bestPartial : bestComplete; + + if (chosen[0] == '\0') + { + return false; + } + + char tliHex[9] = { 0 }; + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(tliHex, chosen, 8); + memcpy(logIdHex, chosen + 8, 8); + memcpy(segHex, chosen + 16, 8); + + uint32_t tli = (uint32_t) strtoul(tliHex, NULL, 16); + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + + uint64_t segno = (uint64_t) logId * CBB_XLOG_SEGMENTS_PER_XLOGID + seg; + uint64_t segStart = segno * CBB_WAL_SEGMENT_SIZE; + uint64_t position; + + if (usePartial) + { + char path[MAXPGPATH]; + uint64_t realLength = 0; + + snprintf(path, sizeof(path), "%s/%s.partial", walcacheDir, bestPartial); + + if (!partial_segment_real_length(path, &realLength)) + { + return false; + } + + position = segStart + realLength; + } + else + { + position = segStart + CBB_WAL_SEGMENT_SIZE; + } + + *timeline = tli; + snprintf(endLsn, endLsnSize, "%X/%08X", + (uint32_t) (position >> 32), (uint32_t) (position & 0xFFFFFFFF)); + + return true; +} + + void cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions) { @@ -418,7 +615,49 @@ cmd_base_backup(int sock, const WsRoute *route, const char *rawOptions) return; } - if (!send_position_row(sock, lsn, tliStr)) + /* + * The end-of-backup position must be a real, currently-reachable target + * -- re-sending the same (potentially long-stale) start position here + * would tell a real pg_basebackup's own background WAL streamer + * (--wal-method=stream) to wait for a target it may have already + * passed hours ago, or, worse, one from a since-pruned segment it can + * never reach; either way its background thread hangs the whole + * command forever waiting on a position that will never legitimately + * arrive as "new" data. + * + * route->position is the canonical, out-of-band-maintained value -- + * see service_archiver_update_current_lsn()'s own comment (pg_autoctl's + * service_archiver.c) for why the archiver-serve supervisor computes + * this once, itself, and writes it into the routes file, rather than + * every reader (this one included) independently re-deriving it by + * scanning WAL file content on its own. find_reachable_end_position() + * (this file's own comment) is the fallback for a route that doesn't + * carry one yet (an older archiver-serve binary against a newer pg_ + * walsender, during a rolling upgrade) -- still a real, reachable + * position, just independently re-derived. Falls back further still to + * the start position only if the walcache is completely empty (no base + * backup should exist at all in that case). + */ + char endLsn[32]; + uint32_t endTimeline; + const char *endLsnPtr = lsn; + const char *endTliStr = tliStr; + char endTliBuf[16]; + + if (route->position[0] != '\0') + { + endLsnPtr = route->position; + endTliStr = tliStr; + } + else if (find_reachable_end_position(route->walcacheDir, &endTimeline, endLsn, + sizeof(endLsn))) + { + snprintf(endTliBuf, sizeof(endTliBuf), "%u", endTimeline); + endLsnPtr = endLsn; + endTliStr = endTliBuf; + } + + if (!send_position_row(sock, endLsnPtr, endTliStr)) { return; } diff --git a/src/bin/pg_walsender/cmd_start_replication.c b/src/bin/pg_walsender/cmd_start_replication.c index d6c44f828..347635839 100644 --- a/src/bin/pg_walsender/cmd_start_replication.c +++ b/src/bin/pg_walsender/cmd_start_replication.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -218,6 +219,111 @@ skip_ws(const char *p) } +/* + * find_oldest_segno scans walcacheDir for the lowest-numbered WAL segment + * present on the given timeline (complete or still ".partial" -- either + * counts as "this archiver has it"). Returns false (*oldestSegno untouched) + * if nothing has been captured on that timeline at all yet. + * + * This is what lets the main streaming loop below tell "the requested + * segment hasn't been captured *yet*" (segno >= oldest present -- normal, + * just wait) apart from "the requested segment predates everything this + * archiver has ever captured" (segno < oldest present -- a real, permanent + * gap, not a timing issue): pg_receivewal has no replication slot before + * this project's own recent fix (service_archiver_start_pgreceivewal(), + * pg_autoctl's service_archiver.c), so a pg_receivewal whose very first + * connection attempt loses the startup HBA-propagation race restarts + * streaming from the server's then-current position instead of resuming, + * silently skipping every segment in between -- observed in practice + * during this milestone's own end-to-end testing. Without this check, a + * client asking to stream from inside that permanent gap (e.g. a real pg_ + * basebackup's own --wal-method=stream background receiver, replaying from + * the position a BASE_BACKUP response advertised) would sit in this file's + * own wait_for_more_data_or_client() loop forever, waiting for a segment + * that can never arrive. + */ +static bool +find_oldest_segno(const char *walcacheDir, uint32_t timeline, uint64_t *oldestSegno) +{ + DIR *dir = opendir(walcacheDir); + + if (dir == NULL) + { + return false; + } + + bool found = false; + uint64_t best = 0; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) + { + size_t len = strlen(entry->d_name); + char segPart[25] = { 0 }; + + if (len == 24) + { + memcpy(segPart, entry->d_name, 24); + } + else if (len == 24 + 8 && strcmp(entry->d_name + 24, ".partial") == 0) + { + memcpy(segPart, entry->d_name, 24); + } + else + { + continue; + } + + bool isHex = true; + + for (size_t i = 0; i < 24 && isHex; i++) + { + isHex = isxdigit((unsigned char) segPart[i]); + } + + if (!isHex) + { + continue; + } + + char tliHex[9] = { 0 }; + + memcpy(tliHex, segPart, 8); + + if ((uint32_t) strtoul(tliHex, NULL, 16) != timeline) + { + continue; + } + + char logIdHex[9] = { 0 }; + char segHex[9] = { 0 }; + + memcpy(logIdHex, segPart + 8, 8); + memcpy(segHex, segPart + 16, 8); + + uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); + uint32_t seg = (uint32_t) strtoul(segHex, NULL, 16); + uint64_t segno = (uint64_t) logId * + (UINT64CONST(0x100000000) / WS_WAL_SEGMENT_SIZE) + seg; + + if (!found || segno < best) + { + best = segno; + found = true; + } + } + + closedir(dir); + + if (found) + { + *oldestSegno = best; + } + + return found; +} + + void cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) { @@ -326,6 +432,30 @@ cmd_start_replication(int sock, const WsRoute *route, const char *rawArgs) if (!isComplete && !file_exists(partialPath)) { + uint64_t oldestSegno; + + if (find_oldest_segno(route->walcacheDir, timeline, &oldestSegno) && + segno < oldestSegno) + { + char oldestName[32]; + + wal_segment_filename(timeline, oldestSegno, + oldestName, sizeof(oldestName)); + + log_error("START_REPLICATION: requested segment \"%s\" " + "predates the oldest segment this archiver has " + "captured (\"%s\") -- it was never captured and " + "can never become available, refusing to wait " + "forever for it", + filename, oldestName); + + ws_send_error_response(sock, "58P01", + "requested WAL segment predates this " + "archiver's captured history and will " + "never become available"); + return; + } + /* nothing captured for this segment yet -- wait for it */ if (!wait_for_more_data_or_client(sock, currentLsn, &lastKeepalive)) { diff --git a/src/bin/pg_walsender/routes.c b/src/bin/pg_walsender/routes.c index 539802624..1a3c8469b 100644 --- a/src/bin/pg_walsender/routes.c +++ b/src/bin/pg_walsender/routes.c @@ -132,6 +132,10 @@ routes_load(const char *path, WsRoute **routesOut, int *countOut) { route->timeline = atoi(propValue); } + else if (strcmp(propName, "position") == 0) + { + strlcpy(route->position, propValue, sizeof(route->position)); + } else { log_warn("Ignoring unknown routes file key \"%s\" in section [%s]", diff --git a/src/bin/pg_walsender/routes.h b/src/bin/pg_walsender/routes.h index 50ec272ae..e54941545 100644 --- a/src/bin/pg_walsender/routes.h +++ b/src/bin/pg_walsender/routes.h @@ -30,6 +30,12 @@ typedef struct WsRoute char allowedHosts[1024]; /* comma-separated, empty = unrestricted */ char systemId[32]; /* decimal uint64, as text; "" = unknown */ int timeline; /* 0 = unknown */ + char position[32]; /* "%X/%08X" pg_lsn text; "" = unknown -- + * see service_archiver_update_current_lsn()'s + * own comment (service_archiver.c) for what + * this is and why it lives here rather than + * being re-derived from WAL file content by + * each reader */ } WsRoute; /* diff --git a/src/monitor/pgautofailover--2.2--2.3.sql b/src/monitor/pgautofailover--2.2--2.3.sql index b9b6e6b14..7fdb83314 100644 --- a/src/monitor/pgautofailover--2.2--2.3.sql +++ b/src/monitor/pgautofailover--2.2--2.3.sql @@ -819,10 +819,24 @@ CREATE TABLE pgautofailover.archiver -- this host is allowed to keep running at once maxresidentreplay int NOT NULL DEFAULT 1, + -- storage stats for the archiver's own PGDATA (walcache + basebackups, + -- same root -- see service_archiver_serve.c's own header comment on + -- why an archiver has no other pgdata to speak of), reported + -- periodically by service_archiver_loop(); NULL until the first report. + -- usedbytes is this archiver's own footprint (directory_size() over its + -- whole pgdata); freebytes is the containing filesystem's available + -- space (statvfs's f_bavail, "available to a non-privileged process" -- + -- the number that actually predicts whether the next base backup or + -- WAL segment fits, not f_bfree's superuser-reserved total). + usedbytes bigint, + freebytes bigint, + lastreporttime timestamptz, UNIQUE (archivername), - CHECK (maxresidentreplay >= 0) + CHECK (maxresidentreplay >= 0), + CHECK (usedbytes IS NULL OR usedbytes >= 0), + CHECK (freebytes IS NULL OR freebytes >= 0) ); -- a named, shareable rclone remote configuration -- the literal contents @@ -1176,7 +1190,7 @@ grant execute on function to autoctl_node; CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) - RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup_policy WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; @@ -1242,6 +1256,69 @@ grant execute on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) to autoctl_node; +-- periodic storage heartbeat: usedbytes/freebytes/lastreporttime all move +-- together, from the same service_archiver_loop() tick (service_archiver.c) +-- that already reports this archiver's captured-WAL LSN. +CREATE FUNCTION pgautofailover.report_archiver_storage + (archiverid bigint, usedbytes bigint, freebytes bigint) + RETURNS void LANGUAGE sql SECURITY DEFINER +AS $$ + UPDATE pgautofailover.archiver + SET usedbytes = report_archiver_storage.usedbytes, + freebytes = report_archiver_storage.freebytes, + lastreporttime = now() + WHERE archiver.archiverid = report_archiver_storage.archiverid; +$$; + +comment on function pgautofailover.report_archiver_storage(bigint,bigint,bigint) + is 'record an archiver''s own reported disk usage and free space'; + +grant execute on function + pgautofailover.report_archiver_storage(bigint,bigint,bigint) + to autoctl_node; + +-- one row per archiver attached to formationid, with its FSM state (the +-- 'wal-receiver' archiver_node row created by archiver_add_formation, one +-- per group -- this milestone's own single-membership scope means a +-- single-group formation gets exactly one row per archiver here; a +-- multi-group formation would get one row per (archiver, group), a +-- follow-up concern once an archiver can serve more than one group at +-- once). Used by `pg_autoctl watch`'s own archivers section. +CREATE FUNCTION pgautofailover.get_archivers + ( + IN formationid text default 'default', + OUT archiver_id bigint, + OUT archiver_name text, + OUT hostname text, + OUT used_bytes bigint, + OUT free_bytes bigint, + OUT last_report_time timestamptz, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT a.archiverid, a.archivername, a.hostname, + a.usedbytes, a.freebytes, a.lastreporttime, + n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver a + JOIN pgautofailover.archiver_formation af + ON af.archiverid = a.archiverid + AND af.formationid = get_archivers.formationid + LEFT JOIN pgautofailover.archiver_node an + ON an.archiverid = a.archiverid AND an.kind = 'wal-receiver' + LEFT JOIN pgautofailover.node n + ON n.nodeid = an.nodeid AND n.formationid = get_archivers.formationid + ORDER BY a.archiverid; +$$; + +comment on function pgautofailover.get_archivers(text) + is 'list the archivers attached to a formation, with storage stats and FSM state'; + +grant execute on function pgautofailover.get_archivers(text) + to autoctl_node; + -- named, shareable rclone config objects -- see rclone_config above for -- what belongs in `config` (architecture only, never credentials) CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) @@ -1533,6 +1610,58 @@ comment on function pgautofailover.get_archiver_policy(text,int) grant execute on function pgautofailover.get_archiver_policy(text,int) to autoctl_node; +-- one round trip from the archiver-basebackup side: resolves the +-- basebackup_policy row that applies to (formation, group) via get_ +-- archiver_policy() above, then flattens its interval columns to plain +-- integer seconds -- easy time_t arithmetic on the C side, no interval- +-- text parsing needed. SECURITY DEFINER: reads archiver_policy/ +-- basebackup_policy directly, both created (like every table in this +-- milestone's own schema) after the blanket "GRANT SELECT ON ALL TABLES" +-- near the top of this file, so autoctl_node has no direct grant on +-- either -- same class of gap already hit (and fixed) twice for wal_ +-- archived()/get_latest_basebackup(). +CREATE FUNCTION pgautofailover.get_basebackup_policy_for_group + ( + formationid text, + groupid int, + OUT policyname text, + OUT source pgautofailover.basebackup_source, + OUT replaymode pgautofailover.basebackup_replay_mode, + OUT cache pgautofailover.basebackup_cache, + OUT frequency_seconds int, + OUT maxcount int, + OUT maxage_seconds int, + OUT onpromotion bool, + OUT concurrency int + ) + RETURNS record LANGUAGE plpgsql STABLE SECURITY DEFINER +AS $$ +DECLARE + ap record; +BEGIN + SELECT * INTO ap + FROM pgautofailover.get_archiver_policy( + get_basebackup_policy_for_group.formationid, + get_basebackup_policy_for_group.groupid); + + SELECT p.policyname, p.source, p.replaymode, p.cache, + extract(epoch FROM p.frequency)::int, + p.maxcount, + extract(epoch FROM p.maxage)::int, + p.onpromotion, p.concurrency + INTO policyname, source, replaymode, cache, frequency_seconds, + maxcount, maxage_seconds, onpromotion, concurrency + FROM pgautofailover.basebackup_policy p + WHERE p.basebackuppolicyid = ap.basebackuppolicyid; +END; +$$; + +comment on function pgautofailover.get_basebackup_policy_for_group(text,int) + is 'resolve the full base-backup production/retention policy for (formation, group), intervals flattened to seconds'; + +grant execute on function pgautofailover.get_basebackup_policy_for_group(text,int) + to autoctl_node; + -- the archive_command confirmation check: true iff at least -- archiver_quorum distinct archivers have durably reported %f CREATE FUNCTION pgautofailover.wal_archived @@ -1801,6 +1930,41 @@ grant execute on function pgautofailover.get_latest_basebackup (text,int,pgautofailover.basebackup_source) to autoctl_node; +-- every 'complete' base backup for (formation, group), newest first -- +-- what service_archiver_basebackup.c's own retention pass (maxcount/ +-- maxage) walks to decide what to keep vs. prune, and what a future `pg_ +-- autoctl show basebackup` would list. basebackupid/storagelocation are +-- what report_basebackup_deleted()/an actual directory removal need; +-- startedat_epoch (extract(epoch from lower(period))) is plain integer +-- seconds for the same reason get_basebackup_policy_for_group() flattens +-- its own interval columns -- easy time_t arithmetic, no timestamptz-text +-- parsing on the C side. +CREATE FUNCTION pgautofailover.list_basebackups + ( + formationid text, + groupid int, + OUT basebackupid bigint, + OUT label text, + OUT storagelocation text, + OUT startedat_epoch bigint + ) + RETURNS SETOF record LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT b.basebackupid, b.label, b.storagelocation, + extract(epoch FROM lower(b.period))::bigint + FROM pgautofailover.basebackup b + WHERE b.formationid = list_basebackups.formationid + AND b.groupid = list_basebackups.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC; +$$; + +comment on function pgautofailover.list_basebackups(text,int) + is 'list complete base backups for (formation, group), newest first -- retention/inventory'; + +grant execute on function pgautofailover.list_basebackups(text,int) + to autoctl_node; + -- an archiving node has no sysidentifier of its own (haspgdata = false, -- see that column's own comment): it never runs a real Postgres instance -- to report one. Every other node in the group shares the same physical diff --git a/src/monitor/pgautofailover.sql b/src/monitor/pgautofailover.sql index ce76ffef0..82de96b19 100644 --- a/src/monitor/pgautofailover.sql +++ b/src/monitor/pgautofailover.sql @@ -1401,10 +1401,24 @@ CREATE TABLE pgautofailover.archiver -- this host is allowed to keep running at once maxresidentreplay int NOT NULL DEFAULT 1, + -- storage stats for the archiver's own PGDATA (walcache + basebackups, + -- same root -- see service_archiver_serve.c's own header comment on + -- why an archiver has no other pgdata to speak of), reported + -- periodically by service_archiver_loop(); NULL until the first report. + -- usedbytes is this archiver's own footprint (directory_size() over its + -- whole pgdata); freebytes is the containing filesystem's available + -- space (statvfs's f_bavail, "available to a non-privileged process" -- + -- the number that actually predicts whether the next base backup or + -- WAL segment fits, not f_bfree's superuser-reserved total). + usedbytes bigint, + freebytes bigint, + lastreporttime timestamptz, UNIQUE (archivername), - CHECK (maxresidentreplay >= 0) + CHECK (maxresidentreplay >= 0), + CHECK (usedbytes IS NULL OR usedbytes >= 0), + CHECK (freebytes IS NULL OR freebytes >= 0) ); -- a named, shareable rclone remote configuration -- the literal contents @@ -1758,7 +1772,7 @@ grant execute on function to autoctl_node; CREATE FUNCTION pgautofailover.get_basebackup_policy(policyname text) - RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT + RETURNS pgautofailover.basebackup_policy LANGUAGE sql STRICT SECURITY DEFINER AS $$ SELECT * FROM pgautofailover.basebackup_policy WHERE basebackup_policy.policyname = get_basebackup_policy.policyname; @@ -1824,6 +1838,69 @@ grant execute on function pgautofailover.register_archiver(text,text,text,bigint,bool,int,text) to autoctl_node; +-- periodic storage heartbeat: usedbytes/freebytes/lastreporttime all move +-- together, from the same service_archiver_loop() tick (service_archiver.c) +-- that already reports this archiver's captured-WAL LSN. +CREATE FUNCTION pgautofailover.report_archiver_storage + (archiverid bigint, usedbytes bigint, freebytes bigint) + RETURNS void LANGUAGE sql SECURITY DEFINER +AS $$ + UPDATE pgautofailover.archiver + SET usedbytes = report_archiver_storage.usedbytes, + freebytes = report_archiver_storage.freebytes, + lastreporttime = now() + WHERE archiver.archiverid = report_archiver_storage.archiverid; +$$; + +comment on function pgautofailover.report_archiver_storage(bigint,bigint,bigint) + is 'record an archiver''s own reported disk usage and free space'; + +grant execute on function + pgautofailover.report_archiver_storage(bigint,bigint,bigint) + to autoctl_node; + +-- one row per archiver attached to formationid, with its FSM state (the +-- 'wal-receiver' archiver_node row created by archiver_add_formation, one +-- per group -- this milestone's own single-membership scope means a +-- single-group formation gets exactly one row per archiver here; a +-- multi-group formation would get one row per (archiver, group), a +-- follow-up concern once an archiver can serve more than one group at +-- once). Used by `pg_autoctl watch`'s own archivers section. +CREATE FUNCTION pgautofailover.get_archivers + ( + IN formationid text default 'default', + OUT archiver_id bigint, + OUT archiver_name text, + OUT hostname text, + OUT used_bytes bigint, + OUT free_bytes bigint, + OUT last_report_time timestamptz, + OUT node_id bigint, + OUT reported_state pgautofailover.replication_state, + OUT goal_state pgautofailover.replication_state + ) +RETURNS SETOF record LANGUAGE SQL STRICT SECURITY DEFINER +AS $$ + SELECT a.archiverid, a.archivername, a.hostname, + a.usedbytes, a.freebytes, a.lastreporttime, + n.nodeid, n.reportedstate, n.goalstate + FROM pgautofailover.archiver a + JOIN pgautofailover.archiver_formation af + ON af.archiverid = a.archiverid + AND af.formationid = get_archivers.formationid + LEFT JOIN pgautofailover.archiver_node an + ON an.archiverid = a.archiverid AND an.kind = 'wal-receiver' + LEFT JOIN pgautofailover.node n + ON n.nodeid = an.nodeid AND n.formationid = get_archivers.formationid + ORDER BY a.archiverid; +$$; + +comment on function pgautofailover.get_archivers(text) + is 'list the archivers attached to a formation, with storage stats and FSM state'; + +grant execute on function pgautofailover.get_archivers(text) + to autoctl_node; + -- named, shareable rclone config objects -- see rclone_config above for -- what belongs in `config` (architecture only, never credentials) CREATE FUNCTION pgautofailover.create_rclone_config(name text, config text) @@ -2115,6 +2192,58 @@ comment on function pgautofailover.get_archiver_policy(text,int) grant execute on function pgautofailover.get_archiver_policy(text,int) to autoctl_node; +-- one round trip from the archiver-basebackup side: resolves the +-- basebackup_policy row that applies to (formation, group) via get_ +-- archiver_policy() above, then flattens its interval columns to plain +-- integer seconds -- easy time_t arithmetic on the C side, no interval- +-- text parsing needed. SECURITY DEFINER: reads archiver_policy/ +-- basebackup_policy directly, both created (like every table in this +-- milestone's own schema) after the blanket "GRANT SELECT ON ALL TABLES" +-- near the top of this file, so autoctl_node has no direct grant on +-- either -- same class of gap already hit (and fixed) twice for wal_ +-- archived()/get_latest_basebackup(). +CREATE FUNCTION pgautofailover.get_basebackup_policy_for_group + ( + formationid text, + groupid int, + OUT policyname text, + OUT source pgautofailover.basebackup_source, + OUT replaymode pgautofailover.basebackup_replay_mode, + OUT cache pgautofailover.basebackup_cache, + OUT frequency_seconds int, + OUT maxcount int, + OUT maxage_seconds int, + OUT onpromotion bool, + OUT concurrency int + ) + RETURNS record LANGUAGE plpgsql STABLE SECURITY DEFINER +AS $$ +DECLARE + ap record; +BEGIN + SELECT * INTO ap + FROM pgautofailover.get_archiver_policy( + get_basebackup_policy_for_group.formationid, + get_basebackup_policy_for_group.groupid); + + SELECT p.policyname, p.source, p.replaymode, p.cache, + extract(epoch FROM p.frequency)::int, + p.maxcount, + extract(epoch FROM p.maxage)::int, + p.onpromotion, p.concurrency + INTO policyname, source, replaymode, cache, frequency_seconds, + maxcount, maxage_seconds, onpromotion, concurrency + FROM pgautofailover.basebackup_policy p + WHERE p.basebackuppolicyid = ap.basebackuppolicyid; +END; +$$; + +comment on function pgautofailover.get_basebackup_policy_for_group(text,int) + is 'resolve the full base-backup production/retention policy for (formation, group), intervals flattened to seconds'; + +grant execute on function pgautofailover.get_basebackup_policy_for_group(text,int) + to autoctl_node; + -- the archive_command confirmation check: true iff at least -- archiver_quorum distinct archivers have durably reported %f CREATE FUNCTION pgautofailover.wal_archived @@ -2383,6 +2512,41 @@ grant execute on function pgautofailover.get_latest_basebackup (text,int,pgautofailover.basebackup_source) to autoctl_node; +-- every 'complete' base backup for (formation, group), newest first -- +-- what service_archiver_basebackup.c's own retention pass (maxcount/ +-- maxage) walks to decide what to keep vs. prune, and what a future `pg_ +-- autoctl show basebackup` would list. basebackupid/storagelocation are +-- what report_basebackup_deleted()/an actual directory removal need; +-- startedat_epoch (extract(epoch from lower(period))) is plain integer +-- seconds for the same reason get_basebackup_policy_for_group() flattens +-- its own interval columns -- easy time_t arithmetic, no timestamptz-text +-- parsing on the C side. +CREATE FUNCTION pgautofailover.list_basebackups + ( + formationid text, + groupid int, + OUT basebackupid bigint, + OUT label text, + OUT storagelocation text, + OUT startedat_epoch bigint + ) + RETURNS SETOF record LANGUAGE sql STABLE SECURITY DEFINER +AS $$ + SELECT b.basebackupid, b.label, b.storagelocation, + extract(epoch FROM lower(b.period))::bigint + FROM pgautofailover.basebackup b + WHERE b.formationid = list_basebackups.formationid + AND b.groupid = list_basebackups.groupid + AND b.status = 'complete' + ORDER BY lower(b.period) DESC; +$$; + +comment on function pgautofailover.list_basebackups(text,int) + is 'list complete base backups for (formation, group), newest first -- retention/inventory'; + +grant execute on function pgautofailover.list_basebackups(text,int) + to autoctl_node; + -- an archiving node has no sysidentifier of its own (haspgdata = false, -- see that column's own comment): it never runs a real Postgres instance -- to report one. Every other node in the group shares the same physical From 86b9c6770c03ec414903ed62b2a315ca101b552b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:07:27 +0200 Subject: [PATCH 24/55] tests: fix archiver pgaftest specs for real scheduling/replay behavior archiver_wal_capture.pgaf: fixed a wrong segment-1 assumption (the archiver's replication slot only protects WAL from its own creation time onward -- by the time it's created, node1+node2's own bootstrap has typically already consumed segments up to the empirically observed floor, segment 3) and switched two `wait until ... state is primary` assertions to the real terminal state after a permanent primary loss (`wait_primary`: WAIT_PRIMARY -> PRIMARY requires another node to reach reported SECONDARY, which an archiver never will). archiver_basebackup_generation.pgaf: the schema's own 'default' policy (frequency 24h) no longer produces a second, replay-sourced backup within any sane test window now that scheduling is policy-driven instead of hardcoded "bootstrap live, then exactly one replay". Attach a short-frequency, source=replay policy during setup so the spec's own remaining job -- proving the replay/volatile generation pipeline itself still works -- is still genuinely exercised. archiver_bootstrap_and_fast_forward.pgaf: same wait_primary fix as above, applied where this spec also stops the original primary for good partway through. New: archiver_basebackup_policy.pgaf, covering the base-backup policy feature end to end -- a fast-cycling, maxcount=3 policy created via the real CLI, attached via set_archiver_policy(), reaching and holding a stable retained count after several times its frequency has elapsed. Registered both archiver_basebackup_policy and (previously missing) archiver_bootstrap_and_fast_forward in tests/tap/schedules/node.sch. All four specs verified passing against a from-scratch --no-cache Docker rebuild. --- tests/tap/schedule | 1 + tests/tap/schedules/node.sch | 2 + .../specs/archiver_basebackup_generation.pgaf | 52 +++++++-- .../tap/specs/archiver_basebackup_policy.pgaf | 103 ++++++++++++++++++ .../archiver_bootstrap_and_fast_forward.pgaf | 64 ++++++++--- tests/tap/specs/archiver_wal_capture.pgaf | 86 ++++++++++++--- 6 files changed, 262 insertions(+), 46 deletions(-) create mode 100644 tests/tap/specs/archiver_basebackup_policy.pgaf diff --git a/tests/tap/schedule b/tests/tap/schedule index aa6d8b3dc..dcf765a74 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -33,6 +33,7 @@ wait_primary_draining_deadlock timeline_fork_report_lsn_deadlock archiver_wal_capture archiver_basebackup_generation +archiver_basebackup_policy archiver_bootstrap_and_fast_forward keeper_fsm_gap_209_wait_maintenance keeper_fsm_gap_211_wait_maintenance diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index ac8ee1cc4..d093fd8d6 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -20,3 +20,5 @@ timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect archiver_wal_capture archiver_basebackup_generation +archiver_basebackup_policy +archiver_bootstrap_and_fast_forward diff --git a/tests/tap/specs/archiver_basebackup_generation.pgaf b/tests/tap/specs/archiver_basebackup_generation.pgaf index c5497776e..7090f4495 100644 --- a/tests/tap/specs/archiver_basebackup_generation.pgaf +++ b/tests/tap/specs/archiver_basebackup_generation.pgaf @@ -10,14 +10,25 @@ # loopback, then discard the staging instance. Both are real, # monitor-tracked pgautofailover.basebackup rows by the end. # -# No explicit trigger step is needed here (unlike archiver_wal_capture.pgaf's -# pg_switch_wal() calls): both backups fire on their own, a couple of -# service_archiver_loop() ticks apart, as soon as the archiver starts. -# get_latest_basebackup() only ever reports the single newest row, and this -# pass's own trigger logic produces at most two rows total (live, then -# replay) before going quiet for the group -- so the *final* state is -# deterministic (source = 'replay') even though the intermediate 'live'-only -# state is not something this spec can reliably catch mid-flight. +# The bootstrap backup (a group's very first one) is always sourced live, +# regardless of policy -- a replay needs an existing backup to replay from +# (service_archiver_maybe_generate_basebackup()'s own "bootstrap is always +# live" rule, service_archiver_basebackup.c). Everything *after* bootstrap +# is scheduled and sourced by whichever base-backup policy applies to the +# group -- the schema's own built-in 'default' policy (frequency 24h, +# source 'replay') would make the second backup real, but not within any +# sane test window, so this spec attaches its own short-frequency, +# source=replay policy as soon as archiver1 is up, via the real +# `pg_autoctl create basebackup-policy` CLI + set_archiver_policy() -- the +# same path archiver_basebackup_policy.pgaf (scheduling/retention coverage, +# source=live there) exercises, just with source=replay here so this +# spec's own remaining job -- proving the replay/volatile generation +# pipeline itself actually works (extract the live backup into a staging +# instance, replay this archiver's own captured WAL forward until it +# promotes, pg_basebackup it over loopback, discard the staging instance) +# -- still gets exercised for real. Attached during setup, before test_001's +# own sleep starts, so the fast frequency is already in effect for whichever +# tick first notices the bootstrap backup has landed and a new one is due. # # Predecessor: archiver_wal_capture.pgaf (M4). @@ -30,8 +41,24 @@ cluster { } setup { - wait until primary timeout 60s + # A lone node plus an archiver is a genuine single-node formation -- + # the archiver never counts as a real Postgres secondary (see group_ + # state_machine.c's BuildForPrimaryNodeNodeActiveContext, hasPgData- + # gated), so node1's own correct, stable terminal state here is + # "single", not "primary" (there is no other node to ever promote it + # past that). + wait until node1 state is single timeout 60s wait until archiver1 state is archiving timeout 60s + + exec archiver1 bash -c 'printf "%s" "{\"source\": \"replay\", \"replaymode\": \"volatile\", \"frequency\": \"10 seconds\", \"maxcount\": 3, \"maxage\": \"10 minutes\", \"onpromotion\": false}" > /tmp/replay-policy.json' + exec archiver1 pg_autoctl create basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name replay-fast --config /tmp/replay-policy.json + sql monitor { + SELECT pgautofailover.set_archiver_policy( + 'default', NULL, 1, + (SELECT basebackuppolicyid + FROM pgautofailover.get_basebackup_policy('replay-fast')), + false); + } } teardown { @@ -39,12 +66,13 @@ teardown { } # -# test_001: both the bootstrap live backup and the one-time replay/volatile -# exercise complete on their own; check the final state. +# test_001: the bootstrap live backup and (once the attached fast policy's +# first frequency interval has elapsed) a real replay/volatile +# backup both land on their own; check the final state. # step test_001_replay_backup_lands { - sleep 45s + sleep 60s sql monitor { SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0); } expect { replay } sql monitor { SELECT replaymode::text FROM pgautofailover.get_latest_basebackup('default', 0); } diff --git a/tests/tap/specs/archiver_basebackup_policy.pgaf b/tests/tap/specs/archiver_basebackup_policy.pgaf new file mode 100644 index 000000000..7a6851163 --- /dev/null +++ b/tests/tap/specs/archiver_basebackup_policy.pgaf @@ -0,0 +1,103 @@ +# Archiving & Disaster Recovery, Milestone 5 (appended): base-backup +# production/retention policy -- frequency-driven scheduling and +# maxcount/maxage pruning, appended to M5 rather than left as a follow-up, +# so the archiver's own base-backup production is a real, bounded resource +# before Milestones 6/7/8 (warm standby, PITR, cloud push) start building +# on top of it. +# +# `frequency: 6 seconds` here is illustrative-fast, not the "1 backup a +# minute" example a real deployment might reasonably use (the design doc's +# own "nightly-cloud" example uses 6h; the schema's own default is 24h) -- +# this spec exists to prove the *mechanism* (does a new backup actually +# fire once frequency has elapsed, does retention actually prune once +# maxcount is exceeded), and a multi-minute real interval would make this +# test unnecessarily slow without adding any real coverage. +# +# The policy is created via the real `pg_autoctl create basebackup-policy` +# CLI (exercising its own --config file-reading path end to end), then +# attached to the formation directly via SQL (set_archiver_policy() -- +# the same function `pg_autoctl create archiver --basebackup-policy` +# calls, just without needing archiver1 declared `create and launch +# deferred` only to re-create it by hand the way the disaster-recovery +# spec does for node2). +# +# `source: "live"` rather than "replay": a live pg_basebackup against a +# near-empty test database completes in a couple of seconds, letting +# several full cycles land inside a practical test runtime -- the replay/ +# volatile pipeline itself is already covered by archiver_basebackup_ +# generation.pgaf, this spec's own job is scheduling/retention, not +# re-proving the replay mechanism. +# +# Predecessor: archiver_basebackup_generation.pgaf (M5, base backup +# generation itself). + +cluster { + monitor + formation { + node1 + archiver1 archiver + } +} + +setup { + wait until node1 state is single timeout 60s + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: create a fast-cycling, maxcount=3 policy via the real CLI, +# attach it to the formation, then let enough cycles pass that +# retention has real pruning to do. Reaching *exactly* maxcount +# after several times frequency has elapsed is strong evidence +# both halves work together: if scheduling never fired past the +# bootstrap backup, the count would be stuck at 1, not 3; if +# retention never pruned, the count would keep growing past 3. +# + +step test_001_policy_scheduling_and_retention { + exec archiver1 bash -c 'printf "%s" "{\"source\": \"live\", \"frequency\": \"6 seconds\", \"maxcount\": 3, \"maxage\": \"10 minutes\", \"onpromotion\": false, \"cache\": \"local\"}" > /tmp/basebackup-policy.json' + exec archiver1 pg_autoctl create basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name fast-policy --config /tmp/basebackup-policy.json + + sql monitor { + SELECT pgautofailover.set_archiver_policy( + 'default', NULL, 1, + (SELECT basebackuppolicyid + FROM pgautofailover.get_basebackup_policy('fast-policy')), + false); + } + + # ~70s at 6s/cycle covers roughly 11 possible cycles -- comfortably + # past maxcount=3 even accounting for each live backup itself taking a + # few seconds, so retention has settled into a stable state by the + # time this checks. + sleep 70s + + sql monitor { + SELECT count(*) FROM pgautofailover.list_basebackups('default', 0); + } + expect { 3 } + + # the newest retained backup should be recent -- confirms retention + # kept the *newest* maxcount backups (this file's own ORDER BY ... + # DESC), not an arbitrary set. + sql monitor { + SELECT (max(startedat_epoch) > + extract(epoch FROM now() - interval '20 seconds')::bigint) + FROM pgautofailover.list_basebackups('default', 0); + } + expect { t } +} + +# +# test_002: `show basebackup-policy` reads back exactly what test_001's +# own `create basebackup-policy` wrote, through the real CLI on +# both ends. +# + +step test_002_show_basebackup_policy { + exec archiver1 pg_autoctl show basebackup-policy --monitor postgresql://autoctl_node@monitor/pg_auto_failover --name fast-policy --json +} diff --git a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf index db7013a45..e128dc115 100644 --- a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf +++ b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf @@ -36,6 +36,7 @@ cluster { monitor + ssl off formation { node1 archiver1 archiver @@ -63,15 +64,18 @@ teardown { step test_001_bootstrap_secondary_from_archiver { sleep 30s sql monitor { - SELECT source::text, status::text + SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live'); } - expect { live|complete } - exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --ssl-self-signed --name node2 --hostname node2 --from-archiver + expect { live } + sql monitor { + SELECT status::text + FROM pgautofailover.get_latest_basebackup('default', 0, 'live'); + } + expect { complete } + exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --no-ssl --name node2 --hostname node2 --from-archiver exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" - wait until node2 state is secondary - passing through catchingup - timeout 90s + wait until node2 state is secondary timeout 90s } # @@ -96,29 +100,53 @@ step test_002_stop_secondary_and_advance_primary { # # test_003: kill the primary. node2 is still stopped, so at this instant the # archiver is the only node in the group with any of test_002's -# WAL -- node1 is gone, node2 never received it. +# WAL -- node1 is gone, node2 never received it. node1 has +# already self-demoted from "primary" back to "wait_primary" by +# this point (test_002's own compose stop node2 took away its +# only sync-quorum-satisfying standby), so there's no single +# fixed intermediate assigned-state to assert on here -- just +# kill it and let the monitor's own health check notice. # step test_003_kill_primary_leaving_archiver_only { compose kill node1 - wait until node1 assigned-state = draining timeout 120s + sleep 45s } # -# test_004: bring node2 back. It reports in behind the archiver, the monitor -# assigns fast_forward with the archiver as WAL source, node2 -# fetches the missing WAL from it (standby_fetch_missing_wal, -# already proven against a real archiver during this milestone's -# own development), and promotes. The row count on the other side -# confirms the fetched WAL was real and got applied, not just that -# the FSM label passed through fast_forward. +# test_004: bring node2 back. compose stop/start recreates its container +# from scratch, so test_001's own manually-backgrounded +# `pg_autoctl run` (started via exec, not through the container's +# normal deferred-polling entrypoint) doesn't survive it and +# needs restarting by hand again, the same way test_001 started +# it the first time. Once it does, node2 reports in behind the +# archiver, the monitor assigns fast_forward with the archiver as +# WAL source, node2 fetches the missing WAL from it (standby_ +# fetch_missing_wal, already proven against a real archiver +# during this milestone's own development), and promotes. The +# row count on the other side confirms the fetched WAL was real +# and got applied, not just that the FSM label passed through +# fast_forward. +# +# The terminal assigned state is wait_primary, not primary: this +# is real, correct pg_auto_failover semantics (group_state_ +# machine.c's pos 401-421 PRIMARY_NODE section), not an archiver- +# specific gap -- WAIT_PRIMARY -> PRIMARY requires some other node +# to reach *reported* SECONDARY state first, and neither ever will +# here: node1 is dead (stuck reporting "demoted") and archiver1 is +# a different node kind entirely (reports "archiving", never +# "secondary"). A plain two-node cluster whose original primary +# never rejoins behaves identically -- the surviving node stays in +# wait_primary indefinitely. wait_primary is still a fully active, +# write-serving primary (it's only the synchronous-replication +# guarantee that's unmet), which is exactly what the row-count +# query right below exercises. # step test_004_bring_back_secondary_and_fast_forward { compose start node2 - wait until node2 state is primary - passing through report_lsn, fast_forward, prepare_promotion, wait_primary - timeout 180s + exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" + wait until node2 state is wait_primary timeout 180s sql node2 { SELECT count(*) FROM archiver_ff_probe; } expect { 1000 } } diff --git a/tests/tap/specs/archiver_wal_capture.pgaf b/tests/tap/specs/archiver_wal_capture.pgaf index 520cf82af..6a40a6736 100644 --- a/tests/tap/specs/archiver_wal_capture.pgaf +++ b/tests/tap/specs/archiver_wal_capture.pgaf @@ -10,10 +10,38 @@ # codebase ever called that SQL function, so archiver_wal stayed permanently # empty no matter how much WAL an archiver captured. # -# Segment filenames are deterministic: a freshly initialized primary starts -# WAL at timeline 1, segment 000000010000000000000001, and each -# pg_switch_wal() call on an otherwise idle test database advances exactly -# one segment (confirmed against a real cluster while developing this spec). +# Segment filenames are deterministic within a single run of this spec: each +# pg_switch_wal() call that has real content to flush advances exactly one +# segment (a bare pg_switch_wal() with nothing written since the previous +# one is *not* reliably a no-op in practice -- it still writes the SWITCH +# record itself -- but forcing a long run of them back-to-back with no +# other activity in between measurably slows down how fast pg_receivewal +# can stream and flush all of it under host load, which is exactly the +# wrong kind of margin to add: it turns a segment-numbering assumption into +# a throughput race instead, observed firsthand while developing this +# fix). What is NOT segment 000000010000000000000001, despite an earlier +# version of this spec assuming so: by the time archiver1 finishes +# registering and gets its own replication slot (keeper_create_and_drop_ +# replication_slots(), the same eager per-tick mechanism used for an +# ordinary standby -- see pg_autoctl's service_archiver.c and keeper.c), +# node1 + node2's own cluster/extension bootstrap has typically already +# consumed a few WAL segments -- observed at exactly segment 3 (the +# archiver's slot restart_lsn landing on 0/3000000) across repeated runs of +# this exact Docker image, and segment 3 itself *is* fully retained and +# capturable (it's the archiver's own slot floor, not one before it). A +# replication slot only protects WAL *from its own creation time onward*; +# it can't retroactively un-recycle segments the primary already dropped +# before the slot existed -- confirmed by node2's own real-secondary slot +# *also* not reaching back to segment 1 in the same runs. pgaftest's DSL +# has no way to capture a query result for use in a later query, so there's +# no way to compute "whatever segment we're actually on" dynamically; every +# check below instead uses the fixed, repeatedly-observed floor of segment +# 3 directly, with the minimum number of pg_switch_wal() calls needed +# (each with real content immediately before it) rather than any extra +# margin -- if a future Postgres/build change shifts the real floor, this +# spec will fail fast and clearly (wrong segment name -> wal_archived() +# returns false), not hang or silently pass. +# # The autoctl_node role has no direct SELECT on archiver_wal (see # report_wal_received()'s own SECURITY DEFINER indirection in # pgautofailover.sql) -- wal_archived() is the one function it can call to @@ -46,18 +74,24 @@ teardown { # land durably in archiver_wal (archiver_quorum defaults to 1, and # there is exactly one archiver here, so wal_archived() flips to # true as soon as service_archiver_report_captured_wal()'s next -# tick reports the segment). +# tick reports the segment). Segment 3 is the observed floor (see +# this spec's own header comment) and is itself fully capturable, +# so no throwaway switches are needed before it -- each switch +# below has a real INSERT immediately before it, so it reliably +# produces a genuinely new segment rather than racing pg_ +# receivewal's own throughput under host load. # step test_001_capture_wal { sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } sql node1 { SELECT pg_switch_wal(); } # PG_AUTOCTL_KEEPER_SLEEP_TIME is 1s; this margin covers a slow CI runner. sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000001'); } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } expect { t } - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000002'); } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } expect { t } } @@ -77,9 +111,13 @@ step test_002_archiver_restart_liveness { wait until archiver1 stopped timeout 60s compose start archiver1 wait until archiver1 state is archiving timeout 60s + # one more switch on top of test_001's own two -- completes segment + # 000000010000000000000005 (see this spec's own header comment on why + # these are fixed numbers rather than derived from segment 1). + sql node1 { INSERT INTO t1 VALUES (4); } sql node1 { SELECT pg_switch_wal(); } sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } expect { t } } @@ -98,14 +136,30 @@ step test_002_archiver_restart_liveness { step test_003_failover_continuity { compose stop node1 wait until node1 stopped timeout 60s - wait until node2 state is primary - passing through wait_primary - timeout 120s - wait until archiver1 state is archiving - passing through report_lsn - timeout 120s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000001'); } - expect { t } + # wait_primary, not primary, is the correct terminal state here: node1 + # is only stopped (still a registered group member, never rejoining + # within this spec) and archiver1 is a different node kind entirely + # (reports "archiving", never "secondary") -- group_state_machine.c's + # PRIMARY_NODE section only ever promotes WAIT_PRIMARY -> PRIMARY once + # some other node reaches *reported* SECONDARY state, which neither + # ever will. Same real, correct pg_auto_failover semantics as a plain + # two-node cluster whose original primary never rejoins (see this + # project's own archiver_bootstrap_and_fast_forward.pgaf, test_004, + # for the identical reasoning). + wait until node2 state is wait_primary timeout 120s + # plain terminal-state check, no "passing through report_lsn": that + # clause tracks ASSIGNED-state transitions via LISTEN/NOTIFY, and can + # miss one that already happened before this wait started listening -- + # now that node2 reaches wait_primary much faster (no longer stuck + # waiting on a secondary-quorum condition that could never be + # satisfied), archiver1's own report_lsn -> archiving cycle can + # complete before this wait even starts observing it. + wait until archiver1 state is archiving timeout 120s + # re-check the earliest and latest segments already confirmed by test_001 + # and test_002 -- proves they survived the failover, not that they were + # ever really "segment 1" (see this spec's own header comment). sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } + expect { t } } From b46c41b9ded86326795f6191aa98c98543eab54c Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:07:46 +0200 Subject: [PATCH 25/55] docs: rewrite intro, document ARCHIVING state, add archiving coverage Intro - Rewrote the opening paragraph: pg_auto_failover is a complete system (pg_autoctl runs as its own pid 1 supervising postmaster), not just an extension -- dynamic topology, automated or operator- driven, two modes of operation (command-driven CLI and node.ini + `pg_autoctl node run`). - New "High Availability, Disaster Recovery, and Backups: One System" section with a new two-panel diagram (arch-ha-dr-unified.tex/.svg) contrasting the typical separate-HA-tool/separate-backup-tool split against pg_auto_failover's single control plane for both. Failover State Machine - New "Archiving" subsection in the State reference, covering the ARCHIVING state's real transitions (verified against live `pg_autoctl inspect fsm list --json` output), its exclusion from candidacy/quorum, and its role as a Fast_forward-eligible WAL source. - Added the 3 real archiving edges to the "Node init / join" and "Failover / promotion" mermaid diagrams, with a new archiverState color class and cross-reference notes. Updated the "20 states and 77 transitions" summary line to 21/80. Fault Tolerance - New "Archiving Nodes and Disaster Recovery" section: WAL capture independent of any standby, base backups on a policy, rebuilding a node (or a whole formation) from an archiver's cache, and how archiving nodes participate in (and are excluded from) failover. Operations - New docs/archiving.rst page: registering an archiver, creating and attaching base-backup policies, watching an archiver, and rebuilding a node with `pg_autoctl create postgres --from-archiver` -- including the disaster-recovery case of rebuilding a whole formation from a single surviving archiver. Reference - New CLI reference pages for `pg_autoctl create/show/set basebackup-policy`, registered in their respective toctrees. Verified with a clean `sphinx-build -W --keep-going` (no warnings, no broken references). --- docs/archiving.rst | 157 ++++++ docs/failover-state-machine.rst | 47 +- docs/fault-tolerance.rst | 81 +++ docs/index.rst | 1 + docs/intro.rst | 52 +- docs/ref/pg_autoctl_create.rst | 1 + .../pg_autoctl_create_basebackup_policy.rst | 85 ++++ docs/ref/pg_autoctl_set.rst | 1 + docs/ref/pg_autoctl_set_basebackup_policy.rst | 51 ++ docs/ref/pg_autoctl_show.rst | 1 + .../ref/pg_autoctl_show_basebackup_policy.rst | 47 ++ docs/tikz/arch-ha-dr-unified.svg | 474 ++++++++++++++++++ docs/tikz/arch-ha-dr-unified.tex | 66 +++ 13 files changed, 1059 insertions(+), 5 deletions(-) create mode 100644 docs/archiving.rst create mode 100644 docs/ref/pg_autoctl_create_basebackup_policy.rst create mode 100644 docs/ref/pg_autoctl_set_basebackup_policy.rst create mode 100644 docs/ref/pg_autoctl_show_basebackup_policy.rst create mode 100644 docs/tikz/arch-ha-dr-unified.svg create mode 100644 docs/tikz/arch-ha-dr-unified.tex diff --git a/docs/archiving.rst b/docs/archiving.rst new file mode 100644 index 000000000..40ec1eab1 --- /dev/null +++ b/docs/archiving.rst @@ -0,0 +1,157 @@ +.. _archiving_operations: + +Archiving +========= + +This page covers the operational side of running an **archiver** node: +registering one, attaching a base-backup policy to control how it produces +and prunes base backups, watching what it's captured, and rebuilding a +node from its cache when disaster recovery is what's needed. For the +architecture and the reasoning behind archiving nodes, see +:ref:`archiving_architecture` and :ref:`archiving_fault_tolerance`; for the +``archiving`` state's exact transitions in the keeper's state machine, see +:ref:`failover_state_machine`. + +Registering an archiver +------------------------ + +An archiver is created the same way as any other node kind, with its own +dedicated verb:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver1 \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver1.example.com \ + --formation default \ + --run + +Unlike ``pg_autoctl create postgres``, this does not initialize a +PostgreSQL data directory: ``--pgdata`` here names the archiver's local +cache directory for captured WAL segments and base backups. Once +registered, the archiver starts its own ``pg_receivewal`` against the +formation's current primary, following it across any later promotion, and +reports its progress to the monitor the same way a standby reports +replication state. + +The full set of options:: + + --pgdata path to the archiver's local data/cache directory + --pgctl path to pg_ctl (used to locate pg_receivewal) + --monitor pg_auto_failover Monitor Postgres URL + --hostname hostname to advertise for this archiver + --formation formation this archiver captures WAL and backups for + --basebackup-policy base-backup production/retention policy to attach + (default: "default") + --run create node then run pg_autoctl service + +Base-backup policies +---------------------- + +Every archiver produces full base backups on a schedule, and prunes older +ones, according to a **base-backup policy** attached to its formation (or +overridden per group). A formation that never attaches one of its own +falls back to the schema's built-in ``default`` policy: a base backup +every 24 hours, keeping the 3 most recent, none older than 3 days. + +Create a policy from a JSON document:: + + $ cat > /tmp/nightly.json <<'EOF' + { + "source": "replay", + "replaymode": "volatile", + "frequency": "6h", + "maxcount": 3, + "maxage": "7d", + "onpromotion": true + } + EOF + + $ pg_autoctl create basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --config /tmp/nightly.json + +Attach it to an archiver at creation time with +``--basebackup-policy nightly`` (see above), or to an already-running +archiver's formation with :ref:`pg_autoctl_set`:: + + $ pg_autoctl set basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --config /tmp/nightly.json + +Every archiver whose formation resolves to a changed policy picks up the +change on its own next tick, no restart needed. Read a policy back with:: + + $ pg_autoctl show basebackup-policy \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --name nightly --json + +Two fields are worth calling out: + + - ``source`` chooses whether the *next* base backup is taken ``live`` + (a real ``pg_basebackup`` against a running node) or ``replay`` + (replayed locally from already-captured WAL, at no cost to the live + primary or any standby). An archiver's very first base backup is + always taken live, regardless of policy, since a replay needs an + existing backup to start from. + - ``onpromotion``, when true, forces an extra base backup right after a + failover or switchover, independent of ``frequency`` -- useful when a + fresh backup taken on the new primary's timeline is worth more than + waiting out the rest of the schedule. + +See :ref:`pg_autoctl_create_basebackup_policy` for the full field +reference, and :ref:`pg_autoctl_show_basebackup_policy` / +:ref:`pg_autoctl_set_basebackup_policy` for the read and update commands. + +Watching an archiver +---------------------- + +An archiver reports state through the same node-active protocol as every +other node, so it shows up in the usual commands:: + + $ pg_autoctl show state + $ pg_autoctl watch + +alongside its captured WAL position, replication lag, and current disk +usage on its cache volume -- the same signals an operator already checks +for a standby, applied to an archiver's own job of holding onto WAL and +base backups rather than serving traffic. + +Rebuilding a node from an archiver +------------------------------------- + +When a node needs a fresh copy of the data -- provisioning a new standby +without adding load to the live primary, or rebuilding after every other +node in the formation was lost -- point ``pg_autoctl create postgres`` at +the archiver instead of a live node:: + + $ pg_autoctl create postgres \ + --pgdata /var/lib/postgresql/data \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --formation default \ + --from-archiver + +This bootstraps from the archiver's latest base backup and then catches +up using its cached WAL, the same recovery machinery ``pg_rewind``/ +``pg_basebackup`` fallback already uses elsewhere in pg_auto_failover -- +just sourced from the archiver's cache instead of a running node. Once +caught up, the new node joins the formation and is assigned a role by the +monitor the ordinary way. + +This is also the disaster-recovery path described in +:ref:`archiving_fault_tolerance`: if the primary and every standby are +lost at once, a single surviving archiver is enough to rebuild a new +primary from scratch with ``--from-archiver``, and re-grow standbys from +there. + +See also +-------- + +- :ref:`archiving_architecture` -- what an archiver is and where it fits + among the other architectures +- :ref:`archiving_fault_tolerance` -- WAL capture independent of any + standby, and rebuilding after every other node is lost +- :ref:`failover_state_machine` -- the ``archiving`` state's own + transitions +- :ref:`pg_autoctl_create_basebackup_policy`, + :ref:`pg_autoctl_show_basebackup_policy`, + :ref:`pg_autoctl_set_basebackup_policy` diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index da3130bc2..06401ff10 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -326,6 +326,40 @@ Missing WAL bytes are fetched from one of the most advanced standby nodes by using Postgres cascading replication features: it is possible to use any standby node in the ``primary_conninfo``. +Archiving +^^^^^^^^^ + +The archiving state is assigned to an **archiving node** — a physically +distinct kind of cluster member added with ``pg_autoctl create archiver``, +never a candidate for promotion or failover, since it holds no ``PGDATA`` +of its own to promote (see :ref:`archiving_architecture`). An archiving +node's own state machine only ever visits three states, mirroring just +enough of the ordinary standby lifecycle to participate safely in an +election without ever competing to win one: + +- ``wait_standby`` → ``archiving``, once the group's primary has + authorized the archiver's connection — the same bootstrap step an + ordinary standby goes through, up to this point. +- ``archiving`` → ``report_lsn``, when the group's primary becomes + unreachable and a failover starts: the archiver stops + ``pg_receivewal`` against the now-untrustworthy primary, the same way + an ordinary standby's own `Report_LSN`_ transition detaches it from a + dying upstream. +- ``report_lsn`` → ``archiving``, once a new primary is confirmed: the + archiver re-points ``pg_receivewal`` at it and resumes capturing WAL. + +An archiving node reaching ``report_lsn`` is never itself considered as a +promotion candidate — its own ``haspgdata`` flag (there is no real +Postgres instance to promote) excludes it from candidacy, and since it +never competes for the primary role, it also never counts toward +``number_sync_standbys`` or the quorum a failover election needs to +proceed. What it does contribute during an election is exactly what its +name promises: its own already-captured WAL becomes a real, +`Fast_forward`_-eligible source for whichever candidate did win, if that +candidate turns out to be behind the most advanced standby — including +when every ordinary standby has been lost and the archiver is the only +node left with the data. + Dropped ^^^^^^^ @@ -341,12 +375,12 @@ command, and then the node entry is removed from the monitor. pg_auto_failover keeper's State Machine --------------------------------------- -The full keeper FSM is 20 states and 77 transitions -- legible as a reference +The full keeper FSM is 21 states and 80 transitions -- legible as a reference table, but too dense to read at a glance as a single diagram. ``pg_autoctl inspect fsm mermaid`` renders it instead as five smaller diagrams, one per phase of a node's life, generated directly from ``KeeperFSM[]`` (``src/bin/pg_autoctl/fsm.c``) so they can never drift out of sync with the -actual state machine the way a hand-maintained image can. The 68 edges shown +actual state machine the way a hand-maintained image can. The edges shown below exclude ``join_primary``, a deprecated state (see `Join_primary`_ above) no longer assigned to any node -- ``KeeperFSM[]`` still carries its 9 transitions for backward compatibility with on-disk state from old @@ -395,6 +429,7 @@ restarted. init --> wait_standby : Start following a primary dropped --> wait_standby : Start following a primary init --> report_lsn : Creating a new node from a standby node that is not a candidate. + wait_standby --> archiving : An archiving node's primary is now ready to accept it note right of single : also appears in Node removal / drop note right of dropped : also appears in Node removal / drop @@ -402,11 +437,13 @@ restarted. note right of wait_primary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop note right of wait_standby : also appears in Steady-state / config changes note right of catchingup : also appears in Steady-state / config changes, Failover / promotion, Maintenance, Node removal / drop + note right of archiving : also appears in Failover / promotion classDef metaState fill:#e0e0e0,stroke:#888888,color:#333333 classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a + classDef archiverState fill:#d1ecf1,stroke:#17a2b8,color:#1a1a1a class init metaState class single metaState class dropped metaState @@ -414,6 +451,7 @@ restarted. class wait_primary primaryState class wait_standby secondaryState class catchingup secondaryState + class archiving archiverState Steady-state / config changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -492,6 +530,8 @@ be identical in shape to this one. join_secondary --> secondary : Failover is done, we have a new primary to follow draining --> report_lsn : Reporting the last write-ahead log location after draining demoted --> report_lsn : Reporting the last write-ahead log location after being demoted + archiving --> report_lsn : The group's primary is unreachable, stop pg_receivewal against it + report_lsn --> archiving : A new primary is confirmed, re-point pg_receivewal at it note right of primary : also appears in Steady-state / config changes, Maintenance, Node removal / drop note right of draining : also appears in Node removal / drop @@ -504,17 +544,20 @@ be identical in shape to this one. note right of prepare_promotion : also appears in Node removal / drop note right of stop_replication : also appears in Node removal / drop note right of report_lsn : also appears in Node init / join, Maintenance, Node removal / drop + note right of archiving : also appears in Node init / join classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef demotingState fill:#f8d7da,stroke:#c0392b,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a + classDef archiverState fill:#d1ecf1,stroke:#17a2b8,color:#1a1a1a class primary primaryState class draining demotingState class demoted demotingState class demote_timeout demotingState class apply_settings primaryState class wait_primary primaryState + class archiving archiverState class catchingup secondaryState class secondary secondaryState class prepare_promotion electionState diff --git a/docs/fault-tolerance.rst b/docs/fault-tolerance.rst index ac9da4f87..4e5f3eba6 100644 --- a/docs/fault-tolerance.rst +++ b/docs/fault-tolerance.rst @@ -255,6 +255,87 @@ walkthrough. A standby forks out-of-band; once the mismatch is visible to the monitor, it is pushed to catchingup and rewound within about a second +.. _archiving_fault_tolerance: + +Archiving Nodes and Disaster Recovery +-------------------------------------- + +Everything above concerns keeping the PostgreSQL *service* available: a +healthy primary always answering reads and writes, promoted from a healthy +secondary within seconds of a failure. An **archiver**, introduced in +:ref:`archiving_architecture`, addresses a different failure mode entirely +-- not "the primary went away for a moment," but "the data needs to survive +even if every node that was ever a primary or a standby is gone." + +WAL capture independent of any standby +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A standby's replication connection exists to keep a second copy of the data +directory caught up for a possible promotion; it stops mattering to fault +tolerance the moment that standby is unhealthy, dropped, or was never +configured at all. An archiver's `pg_receivewal`__ connection has no such +dependency: it streams continuously from whichever node is currently the +group's primary, following it across promotions, and it does this whether +the formation has zero standbys or five. A single-node formation with one +archiver already has WAL protection a standby-less formation alone never +would. + +__ https://www.postgresql.org/docs/current/app-pgreceivewal.html + +Because archiving nodes hold no `PGDATA`__ of their own, they are outside +the replication quorum entirely: an unhealthy archiver never triggers +DRAINING on the primary, never disables synchronous replication, and never +factors into ``number_sync_standbys``. Losing an archiver is a +disaster-recovery-posture event the monitor reports, not a service-affecting +one. + +__ https://www.postgresql.org/docs/current/app-initdb.html + +Base backups, on a policy +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +On top of continuous WAL capture, an archiver periodically produces full +base backups from its own local WAL cache, on a schedule and retention +policy attached to the formation or overridden per group (see +:ref:`archiving_operations` for the operational details: creating a +policy, attaching it, and how scheduling and pruning behave). Each +completed backup, and each one pruned by retention, is reported to the +monitor the same way WAL segments are, so an operator or a client library +can always ask the monitor where the most recent recoverable base backup +lives without needing to reach the archiver's storage directly. + +Rebuilding after every other node is lost +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is the scenario the rest of this page doesn't cover: not one node +failing while others compensate, but the primary and every standby gone at +once -- the kind of event no amount of failover automation can route +around, because there is nothing healthy left to fail over *to*. As long as +one archiver survived, the formation is not actually gone: its WAL cache +and latest base backup are enough to rebuild a new primary from scratch, +and from there re-grow standbys the ordinary way. This is the specific +failure mode the split HA-tool/backup-tool approach described in +:ref:`ha_dr_backups` tends to leave untested until the day it's needed -- +here it is the same monitor, the same node-registration path, and the same +archiver that was already running throughout normal operation. + +How archiving nodes participate in failover +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Archiving nodes are health-checked and report state through the node-active +protocol exactly like a primary or secondary, and the monitor tracks their +``archiving`` state the same way it tracks ``primary``/``secondary`` -- +but they are never assigned a ``candidate-priority``-driven role and never +considered for promotion, since there is no data directory to promote. +When the group's primary changes -- whether through an ordinary failover or +an operator-driven switchover -- an archiver notices its `pg_receivewal` +connection has gone stale, stops it, and re-points at the new primary +automatically; see the ``Archiving`` state's transitions in +:ref:`failover_state_machine` for the exact FSM edges involved. From the +perspective of the rest of this page's failover sequences, an archiver is +simply along for the ride: it never blocks a promotion, and it never needs +one of its own. + Failure handling and network partition detection ------------------------------------------------ diff --git a/docs/index.rst b/docs/index.rst index 5f7f8440e..5bc699689 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -76,6 +76,7 @@ __ https://github.com/hapostgres/pg_auto_failover :caption: Operations operations + archiving testing reporting-bugs faq diff --git a/docs/intro.rst b/docs/intro.rst index c96067f7e..84ce9243c 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -1,9 +1,55 @@ Introduction to pg_auto_failover ================================ -pg_auto_failover is an extension for PostgreSQL that monitors and manages -failover for postgres clusters. It is optimised for simplicity and -correctness. +pg_auto_failover is a complete system for operating PostgreSQL in +production, not just an extension. Its ``pg_autoctl`` process runs as its +own pid 1, supervising the Postgres ``postmaster`` underneath it the way +an init system supervises everything else running on a machine; a +dedicated monitor node coordinates state across every node in the +cluster. Together they provide full cluster management with a dynamic +topology: nodes can be added, removed, and reconfigured while the cluster +keeps serving production traffic, whether driven by an operator's own +commands or automatically by the monitor's own health checks. Automated +failover and full high availability are one deliberate configuration +choice this system supports, not the only thing it does. Two modes of +operation are available side by side: the traditional command-driven CLI +(``pg_autoctl create ...``, ``pg_autoctl set ...``), and a specification- +file-driven mode, where a single ``node.ini`` file describes a node's own +desired configuration and ``pg_autoctl node run`` continuously reconciles +reality to match it. + +.. _ha_dr_backups: + +High Availability, Disaster Recovery, and Backups: One System +--------------------------------------------------------------- + +.. figure:: ./tikz/arch-ha-dr-unified.svg + :alt: Two disconnected tools for HA and backups, versus one integrated pg_auto_failover system for both + + Most setups solve HA and DR with two separate tools; pg_auto_failover solves both with one + +Most PostgreSQL setups treat these as two separate problems, solved by two +separate tools: an *HA tool* watches the live cluster and promotes a +standby when the primary goes away, and a *backup tool* — usually +entirely disconnected from the first — periodically archives WAL and base +backups somewhere safe, reached for only once disaster strikes and +someone needs to restore to a point in time. Running both means learning +two tools, trusting two different failure domains, and, very often, +discovering only during a real incident that they were never actually +exercised together. + +pg_auto_failover starts from a different question. The goal was never +"have an HA tool" or "have a backup tool" — it was always "don't lose the +business's data, and keep serving it." That's one problem, and it's best +solved by one system designed around it, not by gluing together two tools +that were each designed in isolation. The same monitor that orchestrates +failover also tracks every archiver's captured WAL and base backups; the +same WAL stream and base backups a failover election already depends on +to guarantee no data loss are what disaster recovery, including +point-in-time recovery, is built on. High Availability and Disaster +Recovery come from a single package, with a single control plane, rather +than from two independently-operated systems an incident is the first +time anyone actually tested together. Single Standby Architecture --------------------------- diff --git a/docs/ref/pg_autoctl_create.rst b/docs/ref/pg_autoctl_create.rst index 117b96001..d88d37374 100644 --- a/docs/ref/pg_autoctl_create.rst +++ b/docs/ref/pg_autoctl_create.rst @@ -13,3 +13,4 @@ pg_autoctl create - Create a pg_auto_failover node, or formation pg_autoctl_create_coordinator pg_autoctl_create_worker pg_autoctl_create_formation + pg_autoctl_create_basebackup_policy diff --git a/docs/ref/pg_autoctl_create_basebackup_policy.rst b/docs/ref/pg_autoctl_create_basebackup_policy.rst new file mode 100644 index 000000000..ebbaeb7fd --- /dev/null +++ b/docs/ref/pg_autoctl_create_basebackup_policy.rst @@ -0,0 +1,85 @@ +.. _pg_autoctl_create_basebackup_policy: + +pg_autoctl create basebackup-policy +==================================== + +pg_autoctl create basebackup-policy - Create a named base-backup +production/retention policy + +Synopsis +-------- + +This command registers a new base-backup production/retention policy on +the monitor. An archiver's own scheduling (when to take the next base +backup) and retention (which older ones to prune) are driven entirely by +whichever policy applies to its (formation, group) -- attach a policy to +an archiver's own formation with ``pg_autoctl create archiver +--basebackup-policy ``:: + + usage: pg_autoctl create basebackup-policy --monitor --name --config + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --config path to a JSON document with the policy body + +Description +----------- + +A base-backup policy controls three independent things for whichever +archiver(s) it applies to: + + - **when** to produce the next base backup (``frequency``, and + ``onpromotion`` to force one immediately after a failover regardless + of ``frequency``), + - **how** to produce it (``source``: ``live``, straight from a running + node, or ``replay``, replayed locally from already-captured WAL -- + and ``replaymode`` when ``source`` is ``replay``), + - **how many to keep** (``maxcount``, ``maxage``: whichever fires first + prunes a given backup -- the directory is removed and the base + backup's own history row is marked deleted, which in turn prunes any + WAL segments no remaining backup still needs). + +A policy is a standalone, independently-referenceable row: the same one +can be shared by every archiver in a fleet, or kept private to a single +(formation, group) via :ref:`pg_autoctl_set` ``archiver-policy``-style +group overrides. A formation that never creates or attaches a policy of +its own uses this schema's own ``default`` policy (nightly-equivalent: +``frequency`` 24 hours, ``maxcount`` 3, ``maxage`` 3 days). + +The ``--config`` document is a flat JSON object with any subset of the +following keys -- any key left out keeps its own default (on ``create``) +or its current value (on :ref:`pg_autoctl_set_basebackup_policy`):: + + { + "source": "replay", + "replaymode": "volatile", + "cache": "local", + "frequency": "6h", + "maxcount": 3, + "maxage": "7d", + "onpromotion": true, + "concurrency": 1 + } + +``frequency`` and ``maxage`` accept any text Postgres itself parses as an +``interval`` (``"6h"``, ``"3 days"``, ``"90 minutes"``, ...). + +Options +------- + +The following options are available to ``pg_autoctl create basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to create. + +--config + + Path to a JSON document with the policy body, as described above. diff --git a/docs/ref/pg_autoctl_set.rst b/docs/ref/pg_autoctl_set.rst index 466bb5d9a..d311582ee 100644 --- a/docs/ref/pg_autoctl_set.rst +++ b/docs/ref/pg_autoctl_set.rst @@ -12,3 +12,4 @@ pg_autoctl set - Set a pg_auto_failover node, or formation setting pg_autoctl_set_node_replication_quorum pg_autoctl_set_node_candidate_priority pg_autoctl_set_node_region + pg_autoctl_set_basebackup_policy diff --git a/docs/ref/pg_autoctl_set_basebackup_policy.rst b/docs/ref/pg_autoctl_set_basebackup_policy.rst new file mode 100644 index 000000000..33d14ec92 --- /dev/null +++ b/docs/ref/pg_autoctl_set_basebackup_policy.rst @@ -0,0 +1,51 @@ +.. _pg_autoctl_set_basebackup_policy: + +pg_autoctl set basebackup-policy +================================== + +pg_autoctl set basebackup-policy - Update a named base-backup +production/retention policy + +Synopsis +-------- + +This command updates a base-backup production/retention policy that +already exists on the monitor:: + + usage: pg_autoctl set basebackup-policy --monitor --name --config + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --config path to a JSON document with the fields to change + +Description +----------- + +Only the fields present in the ``--config`` document change; any field +left out keeps its current value. See :ref:`pg_autoctl_create_basebackup_policy` +for the full set of fields and what each one controls -- the document +shape is identical, just with only the fields you want to change. + +Every archiver whose (formation, group) resolves to this policy (directly, +or through its formation's own default) picks up the change on its next +tick -- there is no need to restart anything. + +Options +------- + +The following options are available to ``pg_autoctl set basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to update. + +--config + + Path to a JSON document with the fields to change. diff --git a/docs/ref/pg_autoctl_show.rst b/docs/ref/pg_autoctl_show.rst index 896b94342..a6301236d 100644 --- a/docs/ref/pg_autoctl_show.rst +++ b/docs/ref/pg_autoctl_show.rst @@ -16,3 +16,4 @@ pg_autoctl show - Show pg_auto_failover information pg_autoctl_show_timeline pg_autoctl_show_file pg_autoctl_show_systemd + pg_autoctl_show_basebackup_policy diff --git a/docs/ref/pg_autoctl_show_basebackup_policy.rst b/docs/ref/pg_autoctl_show_basebackup_policy.rst new file mode 100644 index 000000000..1f07f1821 --- /dev/null +++ b/docs/ref/pg_autoctl_show_basebackup_policy.rst @@ -0,0 +1,47 @@ +.. _pg_autoctl_show_basebackup_policy: + +pg_autoctl show basebackup-policy +=================================== + +pg_autoctl show basebackup-policy - Show a named base-backup +production/retention policy + +Synopsis +-------- + +This command fetches a base-backup production/retention policy by name +from the monitor and prints it:: + + usage: pg_autoctl show basebackup-policy --monitor --name [ --json ] + + --monitor pg_auto_failover Monitor Postgres URL + --name policy name + --json output data in the JSON format + +Description +----------- + +Prints every field of the named policy: ``source``, ``replaymode``, +``cache``, ``frequency``, ``maxcount``, ``maxage``, ``onpromotion``, and +``concurrency`` -- see :ref:`pg_autoctl_create_basebackup_policy` for what +each one controls. + +Options +------- + +The following options are available to ``pg_autoctl show basebackup-policy``: + +--monitor + + Postgres URI used to connect to the monitor. Must use the ``autoctl_node`` + username and target the ``pg_auto_failover`` database name. It is possible + to show the Postgres URI from the monitor node using the command + :ref:`pg_autoctl_show_uri`. + +--name + + Name of the policy to show. + +--json + + Output data in the JSON format. diff --git a/docs/tikz/arch-ha-dr-unified.svg b/docs/tikz/arch-ha-dr-unified.svg new file mode 100644 index 000000000..4f48c5607 --- /dev/null +++ b/docs/tikz/arch-ha-dr-unified.svg @@ -0,0 +1,474 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-ha-dr-unified.tex b/docs/tikz/arch-ha-dr-unified.tex new file mode 100644 index 000000000..356344b82 --- /dev/null +++ b/docs/tikz/arch-ha-dr-unified.tex @@ -0,0 +1,66 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% \draw [help lines] (-16,10) grid (16,26); + + %% left panel: the typical split -- two disconnected tools + \node (splitLabel) at (-10,25) {\large Typical setup}; + + \node (haTool) at (-10,21) [rectangle,draw=stxt,thick, + minimum width=5.5cm,minimum height=2.2cm,text=stxt,fill=async] + {\normalsize HA tool}; + \node (bkTool) at (-10,15) [rectangle,draw=stxt,thick, + minimum width=5.5cm,minimum height=2.2cm,text=stxt,fill=async] + {\normalsize Backup tool}; + + \node (haOut) at (-15.3,21) [text width=3cm,align=center,text=stxt] + {\small High\\Availability}; + \node (bkOut) at (-15.3,15) [text width=3cm,align=center,text=stxt] + {\small Disaster\\Recovery\\(PITR)}; + + \path (haTool) edge[sql] (haOut) + (bkTool) edge[wal] (bkOut); + + \node (gap) at (-10,18) [text=stxt] {\Large ?}; + + %% right panel: pg_auto_failover, one integrated system + \node (unifiedLabel) at (6,25) {\large One integrated system}; + + \node (pgaf) at (6,18) [rectangle,draw=mbox,very thick, + minimum width=6.5cm,minimum height=6.5cm,text=mtxt,fill=mbox!12, + align=center] + {\normalsize pg\_auto\_failover \\[0.4cm] + \small Monitor \\ + \small + WAL capture \\ + \small + Base backups}; + + \node (haOut2) at (13,21) [text width=3cm,align=center,text=mtxt] + {\small High\\Availability}; + \node (bkOut2) at (13,15) [text width=3cm,align=center,text=mtxt] + {\small Disaster\\Recovery\\(PITR)}; + + \path (pgaf.east) edge[sql,out=25,in=180] (haOut2) + (pgaf.east) edge[wal,out=-25,in=180] (bkOut2); + +\end{tikzpicture} + +\end{document} From a9023448a040ee803363c25baabd37328177a45e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:37:00 +0200 Subject: [PATCH 26/55] docs: redesign HA/DR/Backups diagram as two pastel service-boundary panels Replaces the single stacked arch-ha-dr-unified diagram with two separate figures, each a "production architecture" style pair of dashed service-boundary boxes with a header + inner service pills: - arch-ha-dr-typical: High Availability (Patroni, repmgr) next to Disaster Recovery + Backups (pgBackRest, pgBarman) -- two entirely separate boundaries, naming the actual products a typical setup reaches for. - arch-ha-dr-pgautofailover: High Availability + Disaster Recovery collapse into a single pg_auto_failover box; Backups (pgBackRest, pgBarman) remains its own separate boundary. Colors are a muted, readable palette local to these two diagrams (dark-tinted text, pale tints for fills) rather than raw saturated brand colors used directly as text -- the previous version's bright green header/body text (mbox, #9BF00B) was a real readability problem. Node heights are compact (1.35cm pills) instead of the previous 2.3cm/6.4cm boxes, since most of these boxes hold a single line of text. intro.rst's "High Availability, Disaster Recovery, and Backups: One System" section is retitled "High Availability and Disaster Recovery: One System" and its body adjusted to match: Backups, in the narrower sense of retention/cataloguing/cloud tiers, is now described as its own remaining concern rather than folded into "one system," matching what the new diagrams actually show. --- docs/intro.rst | 59 +-- docs/tikz/arch-ha-dr-pgautofailover.svg | 253 +++++++++++++ docs/tikz/arch-ha-dr-pgautofailover.tex | 73 ++++ docs/tikz/arch-ha-dr-typical.svg | 243 ++++++++++++ docs/tikz/arch-ha-dr-typical.tex | 80 ++++ docs/tikz/arch-ha-dr-unified.svg | 474 ------------------------ docs/tikz/arch-ha-dr-unified.tex | 66 ---- 7 files changed, 684 insertions(+), 564 deletions(-) create mode 100644 docs/tikz/arch-ha-dr-pgautofailover.svg create mode 100644 docs/tikz/arch-ha-dr-pgautofailover.tex create mode 100644 docs/tikz/arch-ha-dr-typical.svg create mode 100644 docs/tikz/arch-ha-dr-typical.tex delete mode 100644 docs/tikz/arch-ha-dr-unified.svg delete mode 100644 docs/tikz/arch-ha-dr-unified.tex diff --git a/docs/intro.rst b/docs/intro.rst index 84ce9243c..a1d8cf24b 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -20,36 +20,47 @@ reality to match it. .. _ha_dr_backups: -High Availability, Disaster Recovery, and Backups: One System ---------------------------------------------------------------- +High Availability and Disaster Recovery: One System +------------------------------------------------------ -.. figure:: ./tikz/arch-ha-dr-unified.svg - :alt: Two disconnected tools for HA and backups, versus one integrated pg_auto_failover system for both +.. figure:: ./tikz/arch-ha-dr-typical.svg + :alt: Typical setup, High Availability from Patroni or repmgr, Disaster Recovery and Backups from pgBackRest or pgBarman, two entirely separate boxes - Most setups solve HA and DR with two separate tools; pg_auto_failover solves both with one + A typical setup reaches for a product per box: Patroni or repmgr for + High Availability, pgBackRest or pgBarman for Disaster Recovery and + Backups + +.. figure:: ./tikz/arch-ha-dr-pgautofailover.svg + :alt: With pg_auto_failover, High Availability and Disaster Recovery collapse into a single box, with Backups (pgBackRest or pgBarman) as the one remaining separate concern + + With pg_auto_failover, High Availability and Disaster Recovery collapse + into one system; Backups remains its own concern Most PostgreSQL setups treat these as two separate problems, solved by two -separate tools: an *HA tool* watches the live cluster and promotes a -standby when the primary goes away, and a *backup tool* — usually -entirely disconnected from the first — periodically archives WAL and base -backups somewhere safe, reached for only once disaster strikes and -someone needs to restore to a point in time. Running both means learning -two tools, trusting two different failure domains, and, very often, -discovering only during a real incident that they were never actually -exercised together. +separate products: an *HA tool* — Patroni, repmgr — watches the live +cluster and promotes a standby when the primary goes away, and a *backup +tool* — pgBackRest, pgBarman — usually entirely disconnected from the +first, periodically archives WAL and base backups somewhere safe, reached +for only once disaster strikes and someone needs to restore to a point in +time. Running both means learning two tools, trusting two different +failure domains, and, very often, discovering only during a real incident +that they were never actually exercised together. pg_auto_failover starts from a different question. The goal was never -"have an HA tool" or "have a backup tool" — it was always "don't lose the -business's data, and keep serving it." That's one problem, and it's best -solved by one system designed around it, not by gluing together two tools -that were each designed in isolation. The same monitor that orchestrates -failover also tracks every archiver's captured WAL and base backups; the -same WAL stream and base backups a failover election already depends on -to guarantee no data loss are what disaster recovery, including -point-in-time recovery, is built on. High Availability and Disaster -Recovery come from a single package, with a single control plane, rather -than from two independently-operated systems an incident is the first -time anyone actually tested together. +"have an HA tool" — it was always "don't lose the business's data, and +keep serving it," and high availability and disaster recovery are two +sides of that same problem, best solved by one system designed around it +rather than by gluing together two tools each designed in isolation. The +same monitor that orchestrates failover also tracks every archiver's +captured WAL and base backups; the same WAL stream and base backups a +failover election already depends on to guarantee no data loss are what +disaster recovery, including point-in-time recovery, is built on. High +Availability and Disaster Recovery come from a single package, with a +single control plane, rather than from two independently-operated systems +an incident is the first time anyone actually tested together. Backups — +in the narrower sense of long-term retention, cataloguing, and cloud +storage tiers — remain their own concern, typically still handled by a +dedicated tool like pgBackRest or pgBarman. Single Standby Architecture --------------------------- diff --git a/docs/tikz/arch-ha-dr-pgautofailover.svg b/docs/tikz/arch-ha-dr-pgautofailover.svg new file mode 100644 index 000000000..704fca608 --- /dev/null +++ b/docs/tikz/arch-ha-dr-pgautofailover.svg @@ -0,0 +1,253 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-ha-dr-pgautofailover.tex b/docs/tikz/arch-ha-dr-pgautofailover.tex new file mode 100644 index 000000000..fc13af25e --- /dev/null +++ b/docs/tikz/arch-ha-dr-pgautofailover.tex @@ -0,0 +1,73 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% same pastel, readable palette as arch-ha-dr-typical.tex, plus a + %% muted green for the pg_auto_failover box. + \definecolor{pgafTxt}{HTML}{2E6B4F} + \definecolor{pgafBorder}{HTML}{7FBF9C} + \definecolor{pgafFill}{HTML}{E9F6EF} + + \definecolor{bkTxt}{HTML}{8A5A12} + \definecolor{bkBorder}{HTML}{D3A257} + \definecolor{bkFill}{HTML}{FBF3E6} + + %% \draw [help lines] (-13,0) grid (13,8); + + \newcommand{\boundarybox}[5]{ + %% #1 x1 #2 y1 #3 x2 #4 y2 #5 border/fill color + \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] + (#1,#2) rectangle (#3,#4); + } + + \newcommand{\boundaryheader}[4]{ + %% #1 x (left) #2 y (top) #3 text color #4 text + \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] + at (#1,#2) {#4}; + } + + \newcommand{\servicebox}[5]{ + %% #1 x #2 y #3 border color #4 width #5 text + \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, + text=stxt,minimum width=#4,minimum height=1.35cm,align=center] + at (#1,#2) {\normalsize #5}; + } + + %% -- left boundary: High Availability + Disaster Recovery -- + \boundarybox{-12.5}{0.5}{-0.7}{6.7}{pgafBorder} + \boundaryheader{-12}{6.3}{pgafTxt}{High Availability + Disaster Recovery} + \draw[draw=pgafBorder,line width=0.6pt] (-12,5.6) -- (-1.2,5.6); + + \node[rectangle,rounded corners=5pt,draw=pgafBorder,thick,fill=pgafFill, + text=stxt,minimum width=9.6cm,minimum height=2.2cm,align=center] + at (-6.6,3.25) {\normalsize pg\_auto\_failover}; + + %% -- right boundary: Backups, alone -- + \boundarybox{0.7}{0.5}{12.5}{6.7}{bkBorder} + \boundaryheader{1.2}{6.3}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (1.2,5.6) -- (12,5.6); + + \servicebox{6.6}{4.5}{bkBorder}{7.4cm}{pgBackRest} + \servicebox{6.6}{1.6}{bkBorder}{7.4cm}{pgBarman} + +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/arch-ha-dr-typical.svg b/docs/tikz/arch-ha-dr-typical.svg new file mode 100644 index 000000000..65348993e --- /dev/null +++ b/docs/tikz/arch-ha-dr-typical.svg @@ -0,0 +1,243 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/tikz/arch-ha-dr-typical.tex b/docs/tikz/arch-ha-dr-typical.tex new file mode 100644 index 000000000..4a6394307 --- /dev/null +++ b/docs/tikz/arch-ha-dr-typical.tex @@ -0,0 +1,80 @@ +% Fix for: https://tex.stackexchange.com/a/315027/43228 +\RequirePackage{luatex85} +\documentclass[border=10pt,17pt]{standalone} + +\usepackage{cfr-lm} +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} +\usetikzlibrary{shapes.multipart} + +\begin{document} + +%% sans-serif fonts, large by default, and bold too +\sffamily +\sbweight +\bfseries +\large + +\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] + + \input{common.tex} + + %% pastel, readable palette local to the pair of "typical setup" / + %% "with pg_auto_failover" diagrams -- a dark muted tone for header + %% text (never a raw saturated brand color as body text), a mid + %% pastel tone for borders, a pale tint for the boundary fill. + \definecolor{haTxt}{HTML}{2F5C9B} + \definecolor{haBorder}{HTML}{7FA0CC} + \definecolor{haFill}{HTML}{EAF1FA} + + \definecolor{bkTxt}{HTML}{8A5A12} + \definecolor{bkBorder}{HTML}{D3A257} + \definecolor{bkFill}{HTML}{FBF3E6} + + %% \draw [help lines] (-13,0) grid (13,8); + + %% ----------------------------------------------------------------- + %% "service boundary" box: dashed pastel border, pale tint fill, + %% bold header label + rule -- compact height, no more than the + %% content actually needs. + %% ----------------------------------------------------------------- + + \newcommand{\boundarybox}[5]{ + %% #1 x1 #2 y1 #3 x2 #4 y2 #5 border/fill color + \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] + (#1,#2) rectangle (#3,#4); + } + + \newcommand{\boundaryheader}[4]{ + %% #1 x (left) #2 y (top) #3 text color #4 text + \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] + at (#1,#2) {#4}; + } + + \newcommand{\servicebox}[5]{ + %% #1 x #2 y #3 border color #4 width #5 text + \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, + text=stxt,minimum width=#4,minimum height=1.35cm,align=center] + at (#1,#2) {\normalsize #5}; + } + + %% -- left boundary: High Availability -- + \boundarybox{-12.5}{0.5}{-0.7}{6.7}{haBorder} + \boundaryheader{-12}{6.3}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-12,5.6) -- (-1.2,5.6); + + \servicebox{-6.6}{4.5}{haBorder}{7.4cm}{Patroni} + \servicebox{-6.6}{1.6}{haBorder}{7.4cm}{repmgr} + + %% -- right boundary: Disaster Recovery + Backups -- + \boundarybox{0.7}{0.5}{12.5}{6.7}{bkBorder} + \boundaryheader{1.2}{6.3}{bkTxt}{Disaster Recovery + Backups} + \draw[draw=bkBorder,line width=0.6pt] (1.2,5.6) -- (12,5.6); + + \servicebox{6.6}{4.5}{bkBorder}{7.4cm}{pgBackRest} + \servicebox{6.6}{1.6}{bkBorder}{7.4cm}{pgBarman} + +\end{tikzpicture} + +\end{document} diff --git a/docs/tikz/arch-ha-dr-unified.svg b/docs/tikz/arch-ha-dr-unified.svg deleted file mode 100644 index 4f48c5607..000000000 --- a/docs/tikz/arch-ha-dr-unified.svg +++ /dev/null @@ -1,474 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/tikz/arch-ha-dr-unified.tex b/docs/tikz/arch-ha-dr-unified.tex deleted file mode 100644 index 356344b82..000000000 --- a/docs/tikz/arch-ha-dr-unified.tex +++ /dev/null @@ -1,66 +0,0 @@ -% Fix for: https://tex.stackexchange.com/a/315027/43228 -\RequirePackage{luatex85} -\documentclass[border=10pt,17pt]{standalone} - -\usepackage{cfr-lm} -\usepackage{pgf} -\usepackage{tikz} -\usetikzlibrary{arrows,shapes,snakes,automata,backgrounds,petri} -\usetikzlibrary{shapes.multipart} - -\begin{document} - -%% sans-serif fonts, large by default, and bold too -\sffamily -\sbweight -\bfseries -\large - -\begin{tikzpicture}[>=stealth',bend angle=45,auto,rounded corners] - - \input{common.tex} - - %% \draw [help lines] (-16,10) grid (16,26); - - %% left panel: the typical split -- two disconnected tools - \node (splitLabel) at (-10,25) {\large Typical setup}; - - \node (haTool) at (-10,21) [rectangle,draw=stxt,thick, - minimum width=5.5cm,minimum height=2.2cm,text=stxt,fill=async] - {\normalsize HA tool}; - \node (bkTool) at (-10,15) [rectangle,draw=stxt,thick, - minimum width=5.5cm,minimum height=2.2cm,text=stxt,fill=async] - {\normalsize Backup tool}; - - \node (haOut) at (-15.3,21) [text width=3cm,align=center,text=stxt] - {\small High\\Availability}; - \node (bkOut) at (-15.3,15) [text width=3cm,align=center,text=stxt] - {\small Disaster\\Recovery\\(PITR)}; - - \path (haTool) edge[sql] (haOut) - (bkTool) edge[wal] (bkOut); - - \node (gap) at (-10,18) [text=stxt] {\Large ?}; - - %% right panel: pg_auto_failover, one integrated system - \node (unifiedLabel) at (6,25) {\large One integrated system}; - - \node (pgaf) at (6,18) [rectangle,draw=mbox,very thick, - minimum width=6.5cm,minimum height=6.5cm,text=mtxt,fill=mbox!12, - align=center] - {\normalsize pg\_auto\_failover \\[0.4cm] - \small Monitor \\ - \small + WAL capture \\ - \small + Base backups}; - - \node (haOut2) at (13,21) [text width=3cm,align=center,text=mtxt] - {\small High\\Availability}; - \node (bkOut2) at (13,15) [text width=3cm,align=center,text=mtxt] - {\small Disaster\\Recovery\\(PITR)}; - - \path (pgaf.east) edge[sql,out=25,in=180] (haOut2) - (pgaf.east) edge[wal,out=-25,in=180] (bkOut2); - -\end{tikzpicture} - -\end{document} From a39285cd16c803427efa80f271f909afd63d45d2 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:37:23 +0200 Subject: [PATCH 27/55] docs: regenerate FSM mermaid diagrams from pg_autoctl inspect fsm mermaid The five keeper-FSM mermaid diagrams had drifted from real KeeperFSM[] output -- verified by re-running `pg_autoctl inspect fsm mermaid ` for all five phases and diffing byte-for-byte against what was checked into the docs. Real gaps found and fixed: - Failover / promotion was missing the entire "wherever you were, you're being demoted now" fan-out (init/single/catchingup/secondary/ prepare_promotion/stop_replication/maintenance/prepare_maintenance/ wait_maintenance/report_lsn/fast_forward, each with both a -> demoted and -> demote_timeout edge), plus several report_lsn fan-in edges (fast_forward/prepare_promotion/stop_replication/ demote_timeout/join_secondary -> report_lsn) -- 28 missing edges in this diagram alone. - Node removal / drop was missing wait_maintenance -> single and fast_forward -> single. - Maintenance was missing wait_maintenance -> report_lsn. - The archiving state's edges (added in an earlier, hand-written pass) are now the tool's own generated labels/coloring (electionState amber, not a separate hand-added archiverState class) instead of hand-embellished text not backed by any real KeeperFSM[] comment. Node init / join and Steady-state / config changes already matched exactly. Updated the summary line and the "replaces the old Graphviz diagram" note from the stale 80/68 transition counts to the real total: 21 states, 102 transitions (111 raw KeeperFSM[] edges minus the 9 excluded join_primary ones). Fixed the Failover / promotion intro paragraph's "still less than half the size of the full graph" claim -- at 57 of 102 edges it's now over half, which the added fan-out edges explain (most of that diagram's size is exactly that "interrupted from anywhere" fan-out). Added an explicit `archiving_state` label on the State reference's Archiving entry so other pages can :ref: it directly instead of relying on an implicit, same-document-only section-title link. --- docs/failover-state-machine.rst | 84 +++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/docs/failover-state-machine.rst b/docs/failover-state-machine.rst index 06401ff10..cd4d59412 100644 --- a/docs/failover-state-machine.rst +++ b/docs/failover-state-machine.rst @@ -326,6 +326,8 @@ Missing WAL bytes are fetched from one of the most advanced standby nodes by using Postgres cascading replication features: it is possible to use any standby node in the ``primary_conninfo``. +.. _archiving_state: + Archiving ^^^^^^^^^ @@ -375,7 +377,7 @@ command, and then the node entry is removed from the monitor. pg_auto_failover keeper's State Machine --------------------------------------- -The full keeper FSM is 21 states and 80 transitions -- legible as a reference +The full keeper FSM is 21 states and 102 transitions -- legible as a reference table, but too dense to read at a glance as a single diagram. ``pg_autoctl inspect fsm mermaid`` renders it instead as five smaller diagrams, one per phase of a node's life, generated directly from ``KeeperFSM[]`` @@ -426,12 +428,13 @@ restarted. dropped --> report_lsn : This node is being reinitialized after having been dropped single --> wait_primary : A new secondary was added wait_standby --> catchingup : The primary is now ready to accept a standby + wait_standby --> archiving : wait_standby to archiving init --> wait_standby : Start following a primary dropped --> wait_standby : Start following a primary init --> report_lsn : Creating a new node from a standby node that is not a candidate. - wait_standby --> archiving : An archiving node's primary is now ready to accept it - note right of single : also appears in Node removal / drop + note right of init : also appears in Failover / promotion + note right of single : also appears in Failover / promotion, Node removal / drop note right of dropped : also appears in Node removal / drop note right of report_lsn : also appears in Failover / promotion, Maintenance, Node removal / drop note right of wait_primary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop @@ -443,7 +446,6 @@ restarted. classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a - classDef archiverState fill:#d1ecf1,stroke:#17a2b8,color:#1a1a1a class init metaState class single metaState class dropped metaState @@ -451,7 +453,7 @@ restarted. class wait_primary primaryState class wait_standby secondaryState class catchingup secondaryState - class archiving archiverState + class archiving electionState Steady-state / config changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -493,8 +495,11 @@ Failover / promotion The primary going away and a candidate taking over, including the multi-standby candidate-election machinery (``report_lsn``, -``fast_forward``, ``join_secondary``) -- this is the largest of the five, -still less than half the size of the full graph. Citus coordinator/worker +``fast_forward``, ``join_secondary``) -- this is by far the largest of the +five, over half the size of the full graph on its own, since it is also +where every other phase's states end up if a failover interrupts them: +most of its edges are the "wherever you were, you're being demoted now" +fan-out into ``demoted``/``demote_timeout``. Citus coordinator/worker transitions are not shown separately: every Citus-specific transition in ``KeeperFSM[]`` reuses an edge that already exists here, just with a different underlying implementation, so a separate "Citus diagram" would @@ -510,10 +515,35 @@ be identical in shape to this one. apply_settings --> draining : A failover occurred, stopping writes apply_settings --> demoted : A failover occurred, no longer primary apply_settings --> demote_timeout : A failover occurred, no longer primary - draining --> demote_timeout : Secondary confirms it is receiving no more writes + draining --> demote_timeout : Secondary confirms it's receiving no more writes demote_timeout --> demoted : Demote timeout expired wait_primary --> demoted : A failover occurred, no longer primary + init --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + single --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + catchingup --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + secondary --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_promotion --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + stop_replication --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + wait_maintenance --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + report_lsn --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + fast_forward --> demoted : A different node is taking over as primary, stopping Postgres in case it's still running + init --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + single --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + demoted --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + catchingup --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + secondary --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_promotion --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + stop_replication --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + prepare_maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + wait_maintenance --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + report_lsn --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running + fast_forward --> demote_timeout : A different node is taking over as primary, stopping Postgres in case it's still running demote_timeout --> primary : Detected a network partition, but monitor didn't do failover + archiving --> report_lsn : archiving to report_lsn + report_lsn --> archiving : report_lsn to archiving demoted --> catchingup : A new primary is available. First, try to rewind. If that fails, do a pg_basebackup. secondary --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. catchingup --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. @@ -522,6 +552,11 @@ be identical in shape to this one. prepare_promotion --> wait_primary : Promoting a Citus Worker standby after having blocked writes from the coordinator. secondary --> report_lsn : Reporting the last write-ahead log location received catchingup --> report_lsn : Reporting the last write-ahead log location received + fast_forward --> report_lsn : Reporting the last write-ahead log location received + prepare_promotion --> report_lsn : Reporting the last write-ahead log location received + stop_replication --> report_lsn : Reporting the last write-ahead log location received + demote_timeout --> report_lsn : Reporting the last write-ahead log location received + join_secondary --> report_lsn : Reporting the last write-ahead log location received report_lsn --> prepare_promotion : Stop traffic to primary, wait for it to finish draining. report_lsn --> fast_forward : Fetching missing WAL bits from another standby before promotion fast_forward --> prepare_promotion : Got the missing WAL bytes, promoted @@ -530,8 +565,6 @@ be identical in shape to this one. join_secondary --> secondary : Failover is done, we have a new primary to follow draining --> report_lsn : Reporting the last write-ahead log location after draining demoted --> report_lsn : Reporting the last write-ahead log location after being demoted - archiving --> report_lsn : The group's primary is unreachable, stop pg_receivewal against it - report_lsn --> archiving : A new primary is confirmed, re-point pg_receivewal at it note right of primary : also appears in Steady-state / config changes, Maintenance, Node removal / drop note right of draining : also appears in Node removal / drop @@ -539,31 +572,43 @@ be identical in shape to this one. note right of demote_timeout : also appears in Node removal / drop note right of apply_settings : also appears in Steady-state / config changes, Node removal / drop note right of wait_primary : also appears in Node init / join, Steady-state / config changes, Node removal / drop + note right of init : also appears in Node init / join + note right of single : also appears in Node init / join, Node removal / drop note right of catchingup : also appears in Node init / join, Steady-state / config changes, Maintenance, Node removal / drop note right of secondary : also appears in Steady-state / config changes, Maintenance, Node removal / drop note right of prepare_promotion : also appears in Node removal / drop note right of stop_replication : also appears in Node removal / drop + note right of maintenance : also appears in Maintenance + note right of prepare_maintenance : also appears in Maintenance + note right of wait_maintenance : also appears in Maintenance, Node removal / drop note right of report_lsn : also appears in Node init / join, Maintenance, Node removal / drop + note right of fast_forward : also appears in Node removal / drop note right of archiving : also appears in Node init / join + classDef metaState fill:#e0e0e0,stroke:#888888,color:#333333 classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef demotingState fill:#f8d7da,stroke:#c0392b,color:#1a1a1a + classDef maintenanceState fill:#e8dff5,stroke:#8e6bb0,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a - classDef archiverState fill:#d1ecf1,stroke:#17a2b8,color:#1a1a1a class primary primaryState class draining demotingState class demoted demotingState class demote_timeout demotingState class apply_settings primaryState class wait_primary primaryState - class archiving archiverState + class init metaState + class single metaState class catchingup secondaryState class secondary secondaryState class prepare_promotion electionState class stop_replication electionState + class maintenance maintenanceState + class prepare_maintenance maintenanceState + class wait_maintenance maintenanceState class report_lsn electionState class fast_forward electionState + class archiving electionState class join_secondary electionState Maintenance @@ -586,9 +631,13 @@ Planned maintenance on either a secondary or the primary. prepare_maintenance --> catchingup : Restarting standby after manual maintenance is done. maintenance --> report_lsn : Reporting the last write-ahead log location received prepare_maintenance --> report_lsn : Reporting the last write-ahead log location received + wait_maintenance --> report_lsn : Reporting the last write-ahead log location received note right of primary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop + note right of prepare_maintenance : also appears in Failover / promotion + note right of maintenance : also appears in Failover / promotion note right of secondary : also appears in Steady-state / config changes, Failover / promotion, Node removal / drop + note right of wait_maintenance : also appears in Failover / promotion, Node removal / drop note right of catchingup : also appears in Node init / join, Steady-state / config changes, Failover / promotion, Node removal / drop note right of report_lsn : also appears in Node init / join, Failover / promotion, Node removal / drop @@ -623,11 +672,13 @@ node reacting to the other side of that removal. prepare_promotion --> single : Primary was forcibly removed stop_replication --> single : Went down to force the primary to time out, but then it was removed report_lsn --> single : There is no other node anymore, promote this node + wait_maintenance --> single : Was waiting to be sent to maintenance, but the primary vanished, promote this node + fast_forward --> single : Was fetching missing WAL from another standby, but every other node vanished, promote this node with whatever it has apply_settings --> single : Other node was forcibly removed, now single any_state --> dropped : This node is being dropped from the monitor note right of primary : also appears in Steady-state / config changes, Failover / promotion, Maintenance - note right of single : also appears in Node init / join + note right of single : also appears in Node init / join, Failover / promotion note right of wait_primary : also appears in Node init / join, Steady-state / config changes, Failover / promotion note right of demoted : also appears in Failover / promotion note right of demote_timeout : also appears in Failover / promotion @@ -637,6 +688,8 @@ node reacting to the other side of that removal. note right of prepare_promotion : also appears in Failover / promotion note right of stop_replication : also appears in Failover / promotion note right of report_lsn : also appears in Node init / join, Failover / promotion, Maintenance + note right of wait_maintenance : also appears in Failover / promotion, Maintenance + note right of fast_forward : also appears in Failover / promotion note right of apply_settings : also appears in Steady-state / config changes, Failover / promotion note right of dropped : also appears in Node init / join @@ -644,6 +697,7 @@ node reacting to the other side of that removal. classDef primaryState fill:#cfe2ff,stroke:#3b6fb6,color:#1a1a1a classDef secondaryState fill:#d4edda,stroke:#4c9a5b,color:#1a1a1a classDef demotingState fill:#f8d7da,stroke:#c0392b,color:#1a1a1a + classDef maintenanceState fill:#e8dff5,stroke:#8e6bb0,color:#1a1a1a classDef electionState fill:#fff3cd,stroke:#c99a1e,color:#1a1a1a class primary primaryState class single metaState @@ -656,6 +710,8 @@ node reacting to the other side of that removal. class prepare_promotion electionState class stop_replication electionState class report_lsn electionState + class wait_maintenance maintenanceState + class fast_forward electionState class apply_settings primaryState class any_state metaState class dropped metaState @@ -665,7 +721,7 @@ node reacting to the other side of that removal. This replaces the previous single Graphviz diagram (``pg_autoctl inspect fsm gv | dot -Tsvg``, rendered from a checked-in ``fsm.png`` last regenerated by hand in 2021). The five diagrams above cover every - currently-reachable transition the old single diagram did -- 68 edges, + currently-reachable transition the old single diagram did -- 102 edges, split by phase rather than shown at once -- deliberately excluding only the 9 transitions involving the deprecated ``join_primary`` state, so ``fsm.png`` is no longer needed as documentation. The ``pg_autoctl From 879c440fcd73438bd5d8f68ccb7d7a7bc85d2ebb Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:37:42 +0200 Subject: [PATCH 28/55] docs: add Archiving Architecture internals page New docs/archiving-internals.rst, in the Architecture toctree: the technical reference for how archiving is actually built, meant to be the main place to extend for later milestones (warm standby, PITR, cloud push). Covers, grounded directly in the current source (function names, exact invocations, exact file paths): - The two forked processes per archiver (capture, serve) and the two files (archiver-position, archiver-routes.ini) that are their only channel to each other -- new arch-archiver-internals diagram. - WAL capture: how an archiver's replication slot reuses the exact same mechanism a real standby's slot uses, with zero primary-side special-casing; the exact pg_receivewal invocation; how the real captured position is computed (including .partial-segment trailing- zero trimming) and shared across the fork boundary; what happens to pg_receivewal across a failover. - Base backup generation: the basebackup_policy table and its 3-tier resolution chain; exactly when a backup is due (bootstrap, onpromotion, frequency); the live pg_basebackup invocation; the full replay/volatile pipeline (staging instance, recovery config, promote, basebackup over loopback, discard); retention pruning. - pg_walsender: why it's a from-scratch reimplementation (no frontend-linkable server-side replication library exists), its process model, the routes-file-based auth/routing mechanism, and a full table of every wire command it implements. - How pg_autoctl create postgres --from-archiver and FAST_FORWARD reuse the port==0 archiver-serve-port resolution trick to talk to pg_walsender with no archiver-specific code past that one lookup. - Build/process wiring, and an explicit "extension points" section listing what M6/M7/M8 build on top of, and what's schema-complete but not yet enforced (concurrency, allowed_hosts). Verified clean with sphinx-build -W --keep-going (no warnings, no broken references). --- docs/archiving-internals.rst | 547 ++++++++++++++ docs/index.rst | 1 + docs/tikz/arch-archiver-internals.svg | 987 ++++++++++++++++++++++++++ docs/tikz/arch-archiver-internals.tex | 73 ++ 4 files changed, 1608 insertions(+) create mode 100644 docs/archiving-internals.rst create mode 100644 docs/tikz/arch-archiver-internals.svg create mode 100644 docs/tikz/arch-archiver-internals.tex diff --git a/docs/archiving-internals.rst b/docs/archiving-internals.rst new file mode 100644 index 000000000..4e4a44bb9 --- /dev/null +++ b/docs/archiving-internals.rst @@ -0,0 +1,547 @@ +.. _archiving_internals: + +Archiving Architecture +======================= + +This page is the technical reference for how the :ref:`archiving_architecture` +feature is actually built: the two processes behind every archiver, when and +how ``pg_receivewal`` and ``pg_basebackup`` run, and ``pg_walsender``, the +project's own server-side implementation of enough of the PostgreSQL +replication protocol to serve captured WAL and base backups back out to a +real Postgres instance. It is meant to be the single place to read before +touching any of this code, and the place later milestones (warm standby, +point-in-time recovery, cloud storage push -- see `Extension points for +future milestones`_ below) extend rather than replace. + +For the operator-facing view -- registering an archiver, attaching a +base-backup policy, rebuilding a node from one -- see :ref:`archiving_operations`. +For where an archiver fits in the wider fault-tolerance picture, see +:ref:`archiving_fault_tolerance`. For the ``archiving`` state's exact FSM +transitions, see the :ref:`archiving_state` entry in :ref:`failover_state_machine`. + +Two processes, one archiver +---------------------------- + +``pg_autoctl archiver run`` (or ``pg_autoctl node run`` against a +``kind = archiver`` node spec) supervises exactly two long-running child +processes, registered with this project's ordinary ``supervisor.c`` +machinery the same way any other node kind's services are: + +- **capture** (``service_archiver_loop()``, ``service_archiver.c``) -- + keeps ``pg_receivewal`` running against the group's current primary and + reports progress to the monitor, the node-active loop every other node + kind also runs, just with an archiver's own body. +- **serve** (``service_archiver_serve_loop()``, ``service_archiver_serve.c``) + -- keeps ``pg_walsender`` running and refreshes the small routes file it + reads its configuration from. + +.. figure:: ./tikz/arch-archiver-internals.svg + :alt: pg_autoctl archiver run forks two processes, capture and serve; capture execs pg_receivewal and writes archiver-position, serve reads archiver-position and writes archiver-routes.ini, then execs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches + + Two forked processes per archiver, talking to each other only through + two small files on disk + +Each is its own ``fork()`` of the ``pg_autoctl`` process (no further +``exec()`` at that level -- they keep running as the same binary, in the +same fork-without-exec style ``pg_walsender``'s own accept loop uses one +level down), each with its own independent connection to the monitor, +since a libpq connection is not fork-safe to share. That split is also why +they never touch each other's memory: the only two channels between them +are two small files, both written atomically (write to a ``.tmp`` path, +then ``rename()``) so a concurrent reader never observes a partial write: + +``archiver-position`` + A single line holding the archiver's real, currently-captured WAL + position (see `Tracking the real captured position`_ below), written by + **capture** on every tick and read by **serve** on every routes-file + refresh. Lives next to the archiver's own ``pg_autoctl.cfg`` + (``path_in_same_directory(config->pathnames.config, "archiver-position", + ...)``). + +``archiver-routes.ini`` + The file ``pg_walsender`` itself reads its whole configuration from -- + see `Routing and auth: the routes file`_ below. Lives in the same + directory, written by **serve**. + +WAL capture +----------- + +Reusing the standby replication slot mechanism +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An archiver's WAL capture rides on a mechanism that already existed for +ordinary standbys, with no primary-side special-casing at all. +``keeper_create_and_drop_replication_slots()`` (``keeper.c``) runs on every +primary-role node, every tick, and calls +``postgres_replication_slot_create_and_drop()`` for every *other* node in +the group returned by ``AutoFailoverOtherNodesList()`` +(``src/monitor/node_metadata.c``) -- a list with no ``hasPgData`` filter, +so an archiver's row comes back exactly like a real standby's. The primary +ends up creating and maintaining a slot named +``pgautofailover_standby_`` for the archiver the same +way it would for a secondary, using the same +``REPLICATION_SLOT_NAME_DEFAULT`` prefix (``"pgautofailover_standby"``, +``defaults.h``) -- nothing in the primary's own slot-maintenance code +needs to know an archiver exists. + +That slot is what makes capture reliable in the first place: a replication +slot pins ``restart_lsn`` at *creation time* and the server retains WAL +back to it regardless of how many times the consumer disconnects and +reconnects. Without one, a ``pg_receivewal`` that loses the startup +HBA-propagation race restarts from the server's then-current position +instead of resuming, silently and permanently skipping whatever WAL +existed in the gap -- a real, observed failure mode this project's own +early testing hit before the slot was wired in. + +Starting ``pg_receivewal`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``service_archiver_start_pgreceivewal()`` (``service_archiver.c``) locates +the real, unmodified ``pg_receivewal`` binary next to ``pg_ctl`` +(``path_in_same_directory``), then runs ``fork()``/``execv()`` on it +directly -- a real exec, not a fork-without-exec like the two supervised +services above:: + + pg_receivewal -w -d "host= port= user=streaming_pgautofailover application_name=" \ + -D --no-sync -S pgautofailover_standby_ + +An archiver's own "pgdata" is not a real ``PGDATA`` -- there is no +``initdb``'d cluster underneath it, no Postgres instance ever started +against it. It is repurposed as the archiver's local WAL-cache root: +segments land there directly, complete ones named the usual 24 hex-digit +way, the one currently being written suffixed ``.partial`` (matching real +Postgres's own pre-allocation-to-full-segment-size behavior, +``XLogFileInitInternal``). The same directory later holds a +``basebackups/`` subdirectory once base backups start landing (see +`Base backup generation`_ below). + +The child's pid is tracked in a file-scope variable and reaped every tick +via ``waitpid(WNOHANG)``; a dead child is simply restarted on the next +tick capture notices it. + +Tracking the real captured position +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``service_archiver_update_current_lsn()`` scans the WAL cache directory +every tick for the newest complete segment and the newest ``.partial`` +one. When a ``.partial`` segment is the real frontier (newer than or equal +to the newest complete one), its trailing run of zero bytes -- the +unwritten tail of Postgres's own full-size pre-allocation -- is trimmed +off by reading the whole file and walking backwards from the end, so the +reported position reflects real, captured content, not the segment's +nominal full size. This is what makes an archiving node a real, rankable +candidate in ``pgautofailover.get_most_advanced_standby()`` during a +failover election: that query has no kind-based exclusion, it just needs a +node reporting something other than ``"0/0"``. + +That position is computed once, in the **capture** process, and is the +single source of truth for "how far has this archiver actually captured" +-- consumed directly by the node-active report to the monitor, and written +to the ``archiver-position`` file (`Two processes, one archiver`_ above) +for **serve** to pick up, rather than each side independently re-deriving +it by scanning WAL file content on its own. + +Continuity across a failover +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An archiving node's own FSM is deliberately small -- see the :ref:`archiving_state` +state reference entry in :ref:`failover_state_machine` for the exact three +transitions (``wait_standby`` → ``archiving``, ``archiving`` → +``report_lsn``, ``report_lsn`` → ``archiving``). On ``ARCHIVING`` → +``REPORT_LSN`` (the group's primary becomes unreachable and a failover +starts), the current implementation stops ``pg_receivewal`` outright via +``fsm_archiver_report_lsn()`` rather than leaving it running against the +soon-to-be-former primary. Once a new primary is confirmed and the node is +assigned back to ``ARCHIVING``, ``pg_receivewal`` is started again, +pointed at the new primary, with the same slot name (still keyed on this +archiver's own node id, so the primary-side slot survives the handover +untouched). + +Base backup generation +----------------------- + +The ``basebackup_policy`` table +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Every base backup an archiver produces is scheduled and sourced according +to a policy row in ``pgautofailover.basebackup_policy`` +(``src/monitor/pgautofailover.sql``):: + + CREATE TABLE pgautofailover.basebackup_policy ( + basebackuppolicyid bigserial PRIMARY KEY, + policyname text UNIQUE, + source pgautofailover.basebackup_source NOT NULL DEFAULT 'replay', + replaymode pgautofailover.basebackup_replay_mode DEFAULT 'volatile', + cache pgautofailover.basebackup_cache NOT NULL DEFAULT 'local', + frequency interval NOT NULL DEFAULT '24 hours', + maxcount int NOT NULL DEFAULT 3, + maxage interval NOT NULL DEFAULT '3 days', + onpromotion bool NOT NULL DEFAULT true, + concurrency int NOT NULL DEFAULT 1, + CHECK (source <> 'replay' OR replaymode IS NOT NULL), + CHECK (concurrency >= 1) + ); + +A formation (or a specific group, as an override) attaches a policy via +``pgautofailover.set_archiver_policy()``, the same function both +``pg_autoctl create archiver --basebackup-policy`` and +``pg_autoctl set basebackup-policy`` ultimately call. Resolution is a +three-tier fallback, plpgsql rather than one ``UNION ALL`` query since +branch evaluation order there isn't guaranteed: +``get_archiver_policy(formationid, groupid)`` looks for an exact +``(formation, group)`` override first, then a formation-wide row +(``groupid IS NULL``), then falls back to the schema's own built-in +``'default'`` policy row (nightly-equivalent: 24h frequency, 3 kept, 3 +days max age). ``get_basebackup_policy_for_group()`` wraps that with the +join against ``basebackup_policy`` itself, flattening the ``interval`` +columns to plain integer seconds (``extract(epoch FROM ...)::int``) so the +C side does cheap ``time_t`` arithmetic instead of parsing intervals. + +``concurrency`` is schema-complete and read on the C side, but not yet +enforced -- a single archiver only ever runs one backup job at a time +regardless of its value, a correct simplification for as long as an +archiver belongs to a single ``(formation, group)`` membership (see +`Extension points for future milestones`_). + +Scheduling: when ``pg_basebackup`` runs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``service_archiver_maybe_generate_basebackup()`` +(``service_archiver_basebackup.c``) runs every tick of the capture loop, +alongside the WAL-report cycle. It resolves the group's policy, lists +existing backups (newest first), and decides: + +- **Bootstrap**: a group with zero backups gets one immediately, and it is + **always** sourced ``live`` regardless of the policy's configured + ``source`` -- there is nothing to replay from yet. +- **Forced by promotion**: if ``onpromotion`` is set and the group's + primary has changed since the last tick (tracked via a file-scope + "last known primary" value, seeded lazily so a process's very first + tick never misfires), a backup is forced regardless of ``frequency``. +- **Due by frequency**: otherwise, a backup is due once + ``now - >= policy.frequency``. + +Once a backup is due, generation forks a one-shot child (tracked via its +own pid, reaped the same way ``pg_receivewal``'s is) that runs to +completion and exits -- deliberately not exec'd, so it never blocks the +capture loop's own per-tick node-active/WAL-report cycle for however long +the backup takes. + +Live source +^^^^^^^^^^^^ + +``select_basebackup_source()`` picks the first healthy non-primary node in +the group, falling back to the primary if there is none (rows with +``port == 0`` -- the ARCHIVING-row sentinel -- are skipped, an archiver +never backs up from another archiver). ``run_pg_basebackup()`` then runs +the real, unmodified binary:: + + pg_basebackup -h -p -U streaming_pgautofailover -D \ + --format=plain --wal-method=none --checkpoint=fast --label

' + + '' + + "
" + + '
scroll to zoom · drag to pan · ' + + "double-click to reset · Esc to close
"; + document.body.appendChild(overlay); + + var viewport = overlay.querySelector(".pgaf-zoom-viewport"); + var img = overlay.querySelector(".pgaf-zoom-img"); + var closeBtn = overlay.querySelector(".pgaf-zoom-close"); + + var scale = 1; + var panX = 0; + var panY = 0; + var dragging = false; + var startX = 0; + var startY = 0; + var startPanX = 0; + var startPanY = 0; + var lastFocused = null; + + function applyTransform() { + img.style.transform = + "translate(" + panX + "px, " + panY + "px) scale(" + scale + ")"; + } + + function reset() { + scale = 1; + panX = 0; + panY = 0; + applyTransform(); + } + + function isOpen() { + return overlay.classList.contains("pgaf-zoom-open"); + } + + function open(src, alt) { + lastFocused = document.activeElement; + img.src = src; + img.alt = alt || ""; + reset(); + overlay.classList.add("pgaf-zoom-open"); + document.documentElement.classList.add("pgaf-zoom-locked"); + closeBtn.focus(); + } + + function close() { + overlay.classList.remove("pgaf-zoom-open"); + document.documentElement.classList.remove("pgaf-zoom-locked"); + img.removeAttribute("src"); + if (lastFocused && typeof lastFocused.focus === "function") { + lastFocused.focus(); + } + } + + closeBtn.addEventListener("click", close); + + overlay.addEventListener("click", function (event) { + if (event.target === overlay) { + close(); + } + }); + + document.addEventListener("keydown", function (event) { + if (event.key === "Escape" && isOpen()) { + close(); + } + }); + + viewport.addEventListener( + "wheel", + function (event) { + if (!isOpen()) { + return; + } + event.preventDefault(); + var factor = event.deltaY < 0 ? 1.15 : 1 / 1.15; + scale = Math.min(8, Math.max(0.5, scale * factor)); + applyTransform(); + }, + { passive: false } + ); + + viewport.addEventListener("dblclick", function (event) { + event.preventDefault(); + reset(); + }); + + viewport.addEventListener("mousedown", function (event) { + dragging = true; + startX = event.clientX; + startY = event.clientY; + startPanX = panX; + startPanY = panY; + viewport.classList.add("pgaf-zoom-dragging"); + event.preventDefault(); + }); + + window.addEventListener("mousemove", function (event) { + if (!dragging) { + return; + } + panX = startPanX + (event.clientX - startX); + panY = startPanY + (event.clientY - startY); + applyTransform(); + }); + + window.addEventListener("mouseup", function () { + dragging = false; + viewport.classList.remove("pgaf-zoom-dragging"); + }); + + return { open: open, close: close }; + } + + function isZoomable(img) { + if (img.closest(".pgaf-zoom-overlay")) { + return false; + } + if (img.classList.contains("no-zoom")) { + return false; + } + return !!img.closest("figure"); + } + + function wireImages(zoom) { + var images = document.querySelectorAll("figure img"); + images.forEach(function (img) { + if (!isZoomable(img) || img.dataset.pgafZoomWired) { + return; + } + img.dataset.pgafZoomWired = "1"; + img.classList.add("pgaf-zoomable-img"); + img.tabIndex = 0; + img.setAttribute("role", "button"); + img.setAttribute("aria-label", "Click to zoom: " + (img.alt || "image")); + img.addEventListener("click", function () { + zoom.open(img.currentSrc || img.src, img.alt); + }); + img.addEventListener("keydown", function (event) { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + zoom.open(img.currentSrc || img.src, img.alt); + } + }); + }); + } + + document.addEventListener("DOMContentLoaded", function () { + var zoom = buildOverlay(); + wireImages(zoom); + }); +})(); diff --git a/docs/conf.py b/docs/conf.py index 9c678d65e..859a0e831 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,11 +112,18 @@ def __init__(self, **options): # html_theme_options = {} -# Add our custom CSS +# Add our custom CSS and JS def setup(app): if hasattr(app, "add_css_file"): app.add_css_file("css/citus.css") app.add_css_file("css/pygments.css") + app.add_css_file("css/zoom.css") + if hasattr(app, "add_js_file"): + # Click-to-zoom for our own figures (tikz-rendered diagrams), + # generalizing the pan/scroll-to-zoom already available on + # Mermaid diagrams (mermaid_d3_zoom, above) to every other + # image the docs embed via `.. figure::`. + app.add_js_file("js/zoom.js") # Add any paths that contain custom static files (such as style sheets) here, From 8d6e0e2b92a174379e30bd16582f62b12bc4832b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 02:45:04 +0200 Subject: [PATCH 30/55] docs: add a High Availability section to Architecture Basics New top-level section right after the page's own intro, before "The pg_auto_failover Monitor": frames High Availability as two distinct guarantees -- Service Availability (the Postgres service stays reachable, what the rest of this page/failover-state-machine.rst/ fault-tolerance.rst describe) and Disaster Recovery (the data survives even total loss of every node that ever held it, what archiving-internals.rst and the archiver covers) -- cross-referencing into both rather than duplicating either. Adds a page-level `fault_tolerance` label to fault-tolerance.rst (it had no explicit label of its own) so this new section can :ref: it directly. --- docs/architecture.rst | 48 ++++++++++++++++++++++++++++++++++++++++ docs/fault-tolerance.rst | 2 ++ 2 files changed, 50 insertions(+) diff --git a/docs/architecture.rst b/docs/architecture.rst index 0df322890..c411784ba 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -31,6 +31,54 @@ PostgreSQL service to accept writes when there's a single server available, and opens the service for potential data loss if the primary server were also to fail. +High Availability +------------------ + +pg_auto_failover treats "High Availability" as two related but distinct +guarantees, rather than one. Most of what follows on this page -- +the Monitor, the keeper, synchronous replication, node recovery -- is +in service of the first of the two; :ref:`archiving_internals` and the +pages it links to are in service of the second: + +- **Service Availability**: the Postgres *service* itself stays reachable + and able to accept reads and writes, with as little downtime as + possible when a node is lost. +- **Disaster Recovery**: the *data* survives even in scenarios Service + Availability alone can't help with -- an operator mistake, a bad + deployment, or every node that ever held the data being lost at once. + +Most Postgres setups reach for two separate, independently-operated +products for these -- an HA tool for the first, a backup tool for the +second. pg_auto_failover treats them as one system instead; see +:ref:`ha_dr_backups` for the full comparison. + +Service Availability (failover) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is what the rest of this page, and :ref:`failover_state_machine` / +:ref:`fault_tolerance` in detail, describe: a primary and one or more +secondary nodes, a Monitor orchestrating automated failover when the +primary is lost, and synchronous replication (`Synchronous vs. +asynchronous replication`_ below) guaranteeing no committed write is lost +in the process. This is the guarantee that answers "the primary just +died -- who serves the next query?". + +Disaster Recovery +^^^^^^^^^^^^^^^^^^ + +Service Availability alone can't answer "we lost every node that ever had +this data" or "an operator dropped the wrong table an hour ago" -- a +healthy failover just moves the same problem to a different node just as +fast. Disaster Recovery is handled by a physically distinct kind of node, +the **archiver**, added on top of any of the architectures on this page: +it continuously captures WAL from the group's current primary and +periodically produces base backups, independent of whether any standby is +even healthy or present. See :ref:`archiving_architecture` for where an +archiver fits alongside the architectures below, :ref:`archiving_internals` +for exactly how WAL capture and base-backup generation work, and +:ref:`archiving_fault_tolerance` for how this changes what a total loss of +the rest of the formation actually means. + The pg_auto_failover Monitor ---------------------------- diff --git a/docs/fault-tolerance.rst b/docs/fault-tolerance.rst index 4e5f3eba6..e4f7e8493 100644 --- a/docs/fault-tolerance.rst +++ b/docs/fault-tolerance.rst @@ -1,3 +1,5 @@ +.. _fault_tolerance: + Failover and Fault Tolerance ============================ From 8f52a109b0f6d774ece908b1453f9ae3677de64b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 10:27:46 +0200 Subject: [PATCH 31/55] docs: redraw HA/DR/Backups diagrams as three boxes on one line Three boxes -- High Availability, Disaster Recovery, Backups -- on a single horizontal line in both diagrams, with whichever pair shares a provider wrapped in one outer box: - arch-ha-dr-typical: High Availability stands alone; Disaster Recovery and Backups are wrapped together (the same two products, pgBackRest/pgBarman, cover both roles in a typical setup). - arch-ha-dr-pgautofailover: the same three boxes, same colors, same layout, just regrouped -- High Availability and Disaster Recovery are now the wrapped pair, inside a box labeled pg_auto_failover; Backups stands alone in the slot Disaster Recovery and Backups shared on the other diagram. Replaces the previous, more complicated pass at this (2-column, vertically-stacked nested sub-boxes) with the simpler request: 3 boxes, one line, 2 of them wrapped. --- docs/tikz/arch-ha-dr-pgautofailover.svg | 289 ++++++++++++++---------- docs/tikz/arch-ha-dr-pgautofailover.tex | 63 +++--- docs/tikz/arch-ha-dr-typical.svg | 251 ++++++++++---------- docs/tikz/arch-ha-dr-typical.tex | 58 +++-- 4 files changed, 363 insertions(+), 298 deletions(-) diff --git a/docs/tikz/arch-ha-dr-pgautofailover.svg b/docs/tikz/arch-ha-dr-pgautofailover.svg index 704fca608..55604573e 100644 --- a/docs/tikz/arch-ha-dr-pgautofailover.svg +++ b/docs/tikz/arch-ha-dr-pgautofailover.svg @@ -1,75 +1,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -133,121 +136,159 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + - - - - + + + + diff --git a/docs/tikz/arch-ha-dr-pgautofailover.tex b/docs/tikz/arch-ha-dr-pgautofailover.tex index fc13af25e..910f43625 100644 --- a/docs/tikz/arch-ha-dr-pgautofailover.tex +++ b/docs/tikz/arch-ha-dr-pgautofailover.tex @@ -20,53 +20,60 @@ \input{common.tex} - %% same pastel, readable palette as arch-ha-dr-typical.tex, plus a - %% muted green for the pg_auto_failover box. - \definecolor{pgafTxt}{HTML}{2E6B4F} - \definecolor{pgafBorder}{HTML}{7FBF9C} - \definecolor{pgafFill}{HTML}{E9F6EF} + %% High Availability and Disaster Recovery keep the exact same colors + %% they have on arch-ha-dr-typical.tex, so the two diagrams read as + %% the same three boxes, just regrouped. + \definecolor{haTxt}{HTML}{2F5C9B} + \definecolor{haBorder}{HTML}{7FA0CC} \definecolor{bkTxt}{HTML}{8A5A12} \definecolor{bkBorder}{HTML}{D3A257} - \definecolor{bkFill}{HTML}{FBF3E6} - %% \draw [help lines] (-13,0) grid (13,8); + \definecolor{pgafTxt}{HTML}{2E6B4F} + \definecolor{pgafBorder}{HTML}{7FBF9C} + + %% \draw [help lines] (-13,0) grid (13,7); \newcommand{\boundarybox}[5]{ - %% #1 x1 #2 y1 #3 x2 #4 y2 #5 border/fill color \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] (#1,#2) rectangle (#3,#4); } - \newcommand{\boundaryheader}[4]{ - %% #1 x (left) #2 y (top) #3 text color #4 text \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] at (#1,#2) {#4}; } - \newcommand{\servicebox}[5]{ - %% #1 x #2 y #3 border color #4 width #5 text \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, text=stxt,minimum width=#4,minimum height=1.35cm,align=center] at (#1,#2) {\normalsize #5}; } - %% -- left boundary: High Availability + Disaster Recovery -- - \boundarybox{-12.5}{0.5}{-0.7}{6.7}{pgafBorder} - \boundaryheader{-12}{6.3}{pgafTxt}{High Availability + Disaster Recovery} - \draw[draw=pgafBorder,line width=0.6pt] (-12,5.6) -- (-1.2,5.6); - - \node[rectangle,rounded corners=5pt,draw=pgafBorder,thick,fill=pgafFill, - text=stxt,minimum width=9.6cm,minimum height=2.2cm,align=center] - at (-6.6,3.25) {\normalsize pg\_auto\_failover}; - - %% -- right boundary: Backups, alone -- - \boundarybox{0.7}{0.5}{12.5}{6.7}{bkBorder} - \boundaryheader{1.2}{6.3}{bkTxt}{Backups} - \draw[draw=bkBorder,line width=0.6pt] (1.2,5.6) -- (12,5.6); - - \servicebox{6.6}{4.5}{bkBorder}{7.4cm}{pgBackRest} - \servicebox{6.6}{1.6}{bkBorder}{7.4cm}{pgBarman} + %% three boxes on one horizontal line, the exact same layout as + %% arch-ha-dr-typical.tex -- only which pair is wrapped changes: here + %% High Availability and Disaster Recovery are the pair nested in one + %% outer box (pg_auto_failover, one product covering both), while + %% Backups stands alone on the right, in the slot Disaster Recovery + %% and Backups shared together on the other diagram. + + \boundarybox{-4.4}{0.4}{13.0}{6.6}{pgafBorder} + \boundaryheader{-3.9}{6.2}{pgafTxt}{pg\_auto\_failover} + \draw[draw=pgafBorder,line width=0.6pt] (-3.9,5.5) -- (12.5,5.5); + + \boundarybox{-3.9}{0.9}{3.55}{5.1}{haBorder} + \boundaryheader{-3.4}{4.7}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-3.4,4.0) -- (3.05,4.0); + \servicebox{-0.175}{1.95}{haBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{4.05}{0.9}{12.5}{5.1}{bkBorder} + \boundaryheader{4.55}{4.7}{bkTxt}{Disaster Recovery} + \draw[draw=bkBorder,line width=0.6pt] (4.55,4.0) -- (12.0,4.0); + \servicebox{8.275}{1.95}{bkBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{-13.0}{0.4}{-5.3}{6.6}{bkBorder} + \boundaryheader{-12.5}{6.2}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (-12.5,5.5) -- (-5.8,5.5); + \servicebox{-9.15}{4.3}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{-9.15}{1.7}{bkBorder}{6.2cm}{pgBarman} \end{tikzpicture} diff --git a/docs/tikz/arch-ha-dr-typical.svg b/docs/tikz/arch-ha-dr-typical.svg index 65348993e..7705cb1e0 100644 --- a/docs/tikz/arch-ha-dr-typical.svg +++ b/docs/tikz/arch-ha-dr-typical.svg @@ -1,75 +1,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + @@ -121,123 +118,149 @@ - + - + - + - + - - - - + + + + - + - - - - - - - - - - + + + + + + + + + + - + - - + + - + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + - - - - + + + + diff --git a/docs/tikz/arch-ha-dr-typical.tex b/docs/tikz/arch-ha-dr-typical.tex index 4a6394307..16b6fb360 100644 --- a/docs/tikz/arch-ha-dr-typical.tex +++ b/docs/tikz/arch-ha-dr-typical.tex @@ -20,60 +20,54 @@ \input{common.tex} - %% pastel, readable palette local to the pair of "typical setup" / - %% "with pg_auto_failover" diagrams -- a dark muted tone for header - %% text (never a raw saturated brand color as body text), a mid - %% pastel tone for borders, a pale tint for the boundary fill. \definecolor{haTxt}{HTML}{2F5C9B} \definecolor{haBorder}{HTML}{7FA0CC} - \definecolor{haFill}{HTML}{EAF1FA} \definecolor{bkTxt}{HTML}{8A5A12} \definecolor{bkBorder}{HTML}{D3A257} - \definecolor{bkFill}{HTML}{FBF3E6} - %% \draw [help lines] (-13,0) grid (13,8); + \definecolor{neutralBorder}{HTML}{ABABAB} - %% ----------------------------------------------------------------- - %% "service boundary" box: dashed pastel border, pale tint fill, - %% bold header label + rule -- compact height, no more than the - %% content actually needs. - %% ----------------------------------------------------------------- + %% \draw [help lines] (-13,0) grid (13,7); \newcommand{\boundarybox}[5]{ - %% #1 x1 #2 y1 #3 x2 #4 y2 #5 border/fill color \draw[rounded corners=9pt,draw=#5,very thick,dashed,fill=#5!12] (#1,#2) rectangle (#3,#4); } - \newcommand{\boundaryheader}[4]{ - %% #1 x (left) #2 y (top) #3 text color #4 text \node[anchor=north west,text=#3,font=\bfseries\Large,inner sep=0pt] at (#1,#2) {#4}; } - \newcommand{\servicebox}[5]{ - %% #1 x #2 y #3 border color #4 width #5 text \node[rectangle,rounded corners=5pt,draw=#3,thick,fill=white, text=stxt,minimum width=#4,minimum height=1.35cm,align=center] at (#1,#2) {\normalsize #5}; } - %% -- left boundary: High Availability -- - \boundarybox{-12.5}{0.5}{-0.7}{6.7}{haBorder} - \boundaryheader{-12}{6.3}{haTxt}{High Availability} - \draw[draw=haBorder,line width=0.6pt] (-12,5.6) -- (-1.2,5.6); - - \servicebox{-6.6}{4.5}{haBorder}{7.4cm}{Patroni} - \servicebox{-6.6}{1.6}{haBorder}{7.4cm}{repmgr} - - %% -- right boundary: Disaster Recovery + Backups -- - \boundarybox{0.7}{0.5}{12.5}{6.7}{bkBorder} - \boundaryheader{1.2}{6.3}{bkTxt}{Disaster Recovery + Backups} - \draw[draw=bkBorder,line width=0.6pt] (1.2,5.6) -- (12,5.6); - - \servicebox{6.6}{4.5}{bkBorder}{7.4cm}{pgBackRest} - \servicebox{6.6}{1.6}{bkBorder}{7.4cm}{pgBarman} + %% three boxes on one horizontal line: High Availability, Disaster + %% Recovery, Backups -- Disaster Recovery and Backups wrapped in one + %% outer box (same two products cover both, in a typical setup), + %% High Availability standing alone outside it. + + \boundarybox{-13.0}{0.4}{-5.3}{6.6}{haBorder} + \boundaryheader{-12.5}{6.2}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-12.5,5.5) -- (-5.8,5.5); + \servicebox{-9.15}{4.3}{haBorder}{6.2cm}{Patroni} + \servicebox{-9.15}{1.7}{haBorder}{6.2cm}{repmgr} + + \boundarybox{-4.4}{0.4}{13.0}{6.6}{neutralBorder} + + \boundarybox{-3.9}{0.9}{4.05}{6.1}{bkBorder} + \boundaryheader{-3.4}{5.7}{bkTxt}{Disaster Recovery} + \draw[draw=bkBorder,line width=0.6pt] (-3.4,5.0) -- (3.55,5.0); + \servicebox{0.075}{3.7}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{0.075}{1.4}{bkBorder}{6.2cm}{pgBarman} + + \boundarybox{4.55}{0.9}{12.5}{6.1}{bkBorder} + \boundaryheader{5.05}{5.7}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (5.05,5.0) -- (12.0,5.0); + \servicebox{8.525}{3.7}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{8.525}{1.4}{bkBorder}{6.2cm}{pgBarman} \end{tikzpicture} From 97bf828704fa7e97f4eca24e40f7e3c87b4732cd Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 10:30:47 +0200 Subject: [PATCH 32/55] docs: keep HA | DR | Backups left-to-right order in both diagrams arch-ha-dr-pgautofailover.tex had High Availability and Disaster Recovery wrapped in the pg_auto_failover box on the right and Backups standalone on the left -- reading right-to-left relative to arch-ha-dr-typical.tex's High Availability, Disaster Recovery, Backups order. Swapped positions (same widths/gaps, mirrored placement) so both diagrams read in the same order, only the wrapping differs. --- docs/tikz/arch-ha-dr-pgautofailover.svg | 254 ++++++++++++------------ docs/tikz/arch-ha-dr-pgautofailover.tex | 53 ++--- 2 files changed, 154 insertions(+), 153 deletions(-) diff --git a/docs/tikz/arch-ha-dr-pgautofailover.svg b/docs/tikz/arch-ha-dr-pgautofailover.svg index 55604573e..dc4e3cbf9 100644 --- a/docs/tikz/arch-ha-dr-pgautofailover.svg +++ b/docs/tikz/arch-ha-dr-pgautofailover.svg @@ -136,159 +136,159 @@ - + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - + + + + - + - - - - - - - - - - + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + - - - - + + + + diff --git a/docs/tikz/arch-ha-dr-pgautofailover.tex b/docs/tikz/arch-ha-dr-pgautofailover.tex index 910f43625..51adb8ae4 100644 --- a/docs/tikz/arch-ha-dr-pgautofailover.tex +++ b/docs/tikz/arch-ha-dr-pgautofailover.tex @@ -48,32 +48,33 @@ at (#1,#2) {\normalsize #5}; } - %% three boxes on one horizontal line, the exact same layout as - %% arch-ha-dr-typical.tex -- only which pair is wrapped changes: here - %% High Availability and Disaster Recovery are the pair nested in one - %% outer box (pg_auto_failover, one product covering both), while - %% Backups stands alone on the right, in the slot Disaster Recovery - %% and Backups shared together on the other diagram. - - \boundarybox{-4.4}{0.4}{13.0}{6.6}{pgafBorder} - \boundaryheader{-3.9}{6.2}{pgafTxt}{pg\_auto\_failover} - \draw[draw=pgafBorder,line width=0.6pt] (-3.9,5.5) -- (12.5,5.5); - - \boundarybox{-3.9}{0.9}{3.55}{5.1}{haBorder} - \boundaryheader{-3.4}{4.7}{haTxt}{High Availability} - \draw[draw=haBorder,line width=0.6pt] (-3.4,4.0) -- (3.05,4.0); - \servicebox{-0.175}{1.95}{haBorder}{6.2cm}{pg\_auto\_failover} - - \boundarybox{4.05}{0.9}{12.5}{5.1}{bkBorder} - \boundaryheader{4.55}{4.7}{bkTxt}{Disaster Recovery} - \draw[draw=bkBorder,line width=0.6pt] (4.55,4.0) -- (12.0,4.0); - \servicebox{8.275}{1.95}{bkBorder}{6.2cm}{pg\_auto\_failover} - - \boundarybox{-13.0}{0.4}{-5.3}{6.6}{bkBorder} - \boundaryheader{-12.5}{6.2}{bkTxt}{Backups} - \draw[draw=bkBorder,line width=0.6pt] (-12.5,5.5) -- (-5.8,5.5); - \servicebox{-9.15}{4.3}{bkBorder}{6.2cm}{pgBackRest} - \servicebox{-9.15}{1.7}{bkBorder}{6.2cm}{pgBarman} + %% three boxes on one horizontal line, same left-to-right order as + %% arch-ha-dr-typical.tex -- High Availability, Disaster Recovery, + %% Backups -- only which pair is wrapped changes: here High + %% Availability and Disaster Recovery are the pair nested in one + %% outer box (pg_auto_failover, one product covering both), on the + %% left; Backups stands alone on the right, same as it does on the + %% other diagram. + + \boundarybox{-13.0}{0.4}{4.4}{6.6}{pgafBorder} + \boundaryheader{-12.5}{6.2}{pgafTxt}{pg\_auto\_failover} + \draw[draw=pgafBorder,line width=0.6pt] (-12.5,5.5) -- (3.9,5.5); + + \boundarybox{-12.5}{0.9}{-4.55}{5.1}{haBorder} + \boundaryheader{-12.0}{4.7}{haTxt}{High Availability} + \draw[draw=haBorder,line width=0.6pt] (-12.0,4.0) -- (-5.05,4.0); + \servicebox{-8.525}{1.95}{haBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{-4.05}{0.9}{3.9}{5.1}{bkBorder} + \boundaryheader{-3.55}{4.7}{bkTxt}{Disaster Recovery} + \draw[draw=bkBorder,line width=0.6pt] (-3.55,4.0) -- (3.4,4.0); + \servicebox{-0.075}{1.95}{bkBorder}{6.2cm}{pg\_auto\_failover} + + \boundarybox{5.3}{0.4}{13.0}{6.6}{bkBorder} + \boundaryheader{5.8}{6.2}{bkTxt}{Backups} + \draw[draw=bkBorder,line width=0.6pt] (5.8,5.5) -- (12.5,5.5); + \servicebox{9.15}{4.3}{bkBorder}{6.2cm}{pgBackRest} + \servicebox{9.15}{1.7}{bkBorder}{6.2cm}{pgBarman} \end{tikzpicture} From d9bc50e01c931b41dccc62c55e560ed12fac40d6 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Wed, 5 Aug 2026 20:07:38 +0200 Subject: [PATCH 33/55] docs: rename archiving-internals.rst to archiving-details.rst, rewrite for operators Full rewrite from an operator's viewpoint instead of a contributor's: no function names, no file:line citations, no build/Makefile details, no "extension points for future milestones" section aimed at future code contributors. Keeps the process-tree diagram. New structure: - Data flow: what actually moves (WAL streaming, base backup generation live vs. replay, serving it back out), and that none of it routes through the monitor. - Storage: a concrete directory-listing example, what each file/ subdirectory actually is, and how to reason about disk sizing (base backups ~ maxcount x one backup's size, WAL ~ however much has accumulated since the oldest still-retained backup). - Network exposure: what's listening, how it's authenticated, how to think about firewalling it. - Process model: an ASCII process tree for the simple case, then three scenarios by request -- more/fewer standby nodes in the group (doesn't change the archiver's own process tree at all), several independent formations (one archiver process tree per formation, fully independent), and a Citus formation (honest about current scope: only the coordinator's own group is covered today, worker groups are not yet). - What you can point at an archiver: the wire-protocol commands its serving side understands, as a definition list (matching this project's own CLI-option documentation style) instead of a table. Keeps the `archiving_architecture` label the file already carried, so existing :ref: links to it from architecture.rst/fault-tolerance.rst (currently being edited separately) keep resolving without changes there. Verified with sphinx-build -W --keep-going: clean, no broken references. --- docs/archiving-details.rst | 242 ++++++++++++++++ docs/archiving-internals.rst | 547 ----------------------------------- docs/index.rst | 2 +- 3 files changed, 243 insertions(+), 548 deletions(-) create mode 100644 docs/archiving-details.rst delete mode 100644 docs/archiving-internals.rst diff --git a/docs/archiving-details.rst b/docs/archiving-details.rst new file mode 100644 index 000000000..b2f41f24a --- /dev/null +++ b/docs/archiving-details.rst @@ -0,0 +1,242 @@ +.. _archiving_architecture: + +Archiving in Detail +===================== + +:ref:`archiving_and_disaster_recovery` introduces the archiver at a glance, +and :ref:`archiving_operations` walks through the day-to-day commands for +registering one and attaching a base-backup policy. This page goes one +level deeper: what actually moves over the network and onto disk while an +archiver is running, and what processes are involved -- the level of +detail worth having before sizing storage, deciding where an archiver +should sit on your network, or reasoning about how much of a given +topology (a Citus formation, several independent formations) is actually +covered. + +Data flow +--------- + +An archiver does three things, and none of them ever route through the +monitor -- WAL and base backups always flow directly between the archiver +and whichever node it's talking to, with the monitor only ever seeing +small status reports (what's been captured, what's been backed up, how +much disk is left), never the data itself: + +1. **It streams WAL continuously** from whichever node is currently the + group's primary, over an ordinary PostgreSQL physical replication + connection -- the same kind of connection a standby uses, protected by + its own dedicated replication slot so that nothing already captured is + ever lost, even across a connection that drops and stays down for a + while. If the primary changes, the archiver notices and reconnects to + the new one on its own; no operator action is needed. +2. **It produces base backups on a schedule**, either as a real + ``pg_basebackup`` taken directly from a live node, or entirely on its + own: replaying already-captured WAL against a local copy of the last + base backup until that copy reaches a consistent, promotable state, + and backing up that instead. The second mode never touches the + primary or any standby at all -- useful when you want frequent base + backups without adding load to production. +3. **It hands both back out** on request: a real ``pg_basebackup`` + command, a real standby's own ``primary_conninfo``, or this project's + own restore tooling can all connect to an archiver directly and get + what they ask for, with no special client needed -- see `What you can + point at an archiver`_ below. + +Storage +------- + +Everything an archiver holds lives under one local directory -- the path +given as ``--pgdata`` when the archiver was created. Despite the flag's +name, this is never a real Postgres data directory (there is no +``initdb``, nothing ever starts Postgres against it directly); it's a +cache root:: + + /var/lib/pgaf/archiver1/ + ├── 000000010000000000000041 + ├── 000000010000000000000042 + ├── 000000010000000000000043.partial + ├── archiver-position + ├── archiver-routes.ini + └── basebackups/ + ├── basebackup-20260803T020000Z/ + ├── basebackup-20260804T020000Z/ + └── basebackup-20260805T020000Z/ + +- WAL segments sit directly in this directory, named exactly the way + Postgres itself names them. The most recently-started one carries a + ``.partial`` suffix until it's complete -- archiving doesn't wait for a + segment to fill up before it counts: whatever has already been flushed + into that ``.partial`` file is captured too. +- Each retained base backup is its own subdirectory under + ``basebackups/``, in the same layout an ordinary ``pg_basebackup`` run + by hand would produce. You could point ``postgres -D`` straight at one + of them and it would start -- that's exactly what disaster recovery + relies on. +- ``archiver-position`` and ``archiver-routes.ini`` are small internal + bookkeeping files: coordinates and status, never a copy of any actual + data. Safe to ignore day to day, and not something that needs backing + up itself -- both are regenerated automatically on the archiver's own + next tick. + +Sizing disk for an archiver comes down to two mostly-independent numbers: + +- **Base backups**: roughly the policy's ``maxcount`` times the size of + one backup, since retention prunes anything beyond that count (or + older than ``maxage``, whichever comes first) right after each new one + lands. See :ref:`archiving_operations` for how to set these. +- **WAL**: however much WAL has accumulated since your *oldest + still-retained* base backup -- once a base backup is pruned, the WAL + segments only it still needed are pruned right along with it. A longer + retention window keeps more history recoverable, at the cost of more + WAL kept around to cover it. + +Network exposure +----------------- + +An archiver listens on a TCP port (``6543`` by default) speaking a subset +of the PostgreSQL replication protocol, authenticated the same trust-based +way every node's own replication connections already are in a +pg_auto_failover cluster (there is no password or TLS on this connection +in the current release). Treat it the same way you'd treat any other +node's own replication port: reachable from wherever you expect to run +``pg_basebackup``, point a standby's ``primary_conninfo`` at it, or run a +restore from, and firewalled off from everywhere else. + +Process model +-------------- + +Once started (``pg_autoctl archiver run``, or ``pg_autoctl node run`` +against a ``kind = archiver`` node specification), an archiver supervises +exactly two long-running processes, which in turn each run one more of +their own -- four processes total, all on the one host, none of them +sharing memory. They hand off exactly two small files (`Storage`_ above) +and nothing else: + +.. figure:: ./tikz/arch-archiver-internals.svg + :alt: pg_autoctl archiver run supervises two processes, capture and serve; capture runs pg_receivewal and writes archiver-position, serve reads archiver-position and writes archiver-routes.ini, then runs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches + + Two supervised processes per archiver, talking to each other only + through two small files on disk + +:: + + pg_autoctl archiver run + ├── capture -- keeps WAL streaming alive, reports progress to the monitor + │ └── pg_receivewal + └── serve -- keeps the archiver reachable over the network + └── pg_walsender --port 6543 --routes archiver-routes.ini + +If either child stops unexpectedly, the parent notices on its next tick +and restarts it -- an archiver recovering from a crashed +``pg_receivewal`` or ``pg_walsender`` needs no operator action, the same +way a keeper recovers a crashed Postgres. + +More or fewer standby nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This process tree doesn't change shape based on how many standby nodes +are in the group. WAL capture always talks to whichever node is currently +primary, never to a standby directly, so a two-node group and a +five-node group look identical from the archiver's side. The only place +standby count matters at all is when a base backup is sourced live: with +more healthy standbys available, there are more candidates to pick from +before falling back to the primary -- everything else about the archiver +is unaffected. + +Several formations +^^^^^^^^^^^^^^^^^^^ + +An archiver is attached to one formation at a time. Covering several +formations -- each, say, with its own group of two or three standby +nodes -- means registering one archiver per formation, each with its own +``--pgdata`` directory and its own identity, whether that's several +archiver processes on one host or spread across several hosts: + +:: + + host archiver-a host archiver-b + (attached to formation "default") (attached to formation "billing") + + pg_autoctl archiver run pg_autoctl archiver run + ├── capture -> pg_receivewal ├── capture -> pg_receivewal + └── serve -> pg_walsender └── serve -> pg_walsender + +Each of these process trees is entirely independent -- separate storage +directory, separate WAL stream, separate base-backup schedule, no shared +state of any kind. Losing one has no effect on the others. + +A Citus formation +^^^^^^^^^^^^^^^^^^ + +A Citus formation is really several node groups under one name: the +coordinator's own group, plus one group per worker. Today, registering an +archiver against a Citus formation covers the coordinator's group only -- +worker groups don't yet get their own WAL capture or base backups from +that same archiver. If disaster recovery coverage for worker data matters +to you today, plan around this limitation; formation-wide coverage across +every worker group from a single archiver is on the roadmap but not yet +available. + +What you can point at an archiver +------------------------------------ + +An archiver's serving side understands enough of the real PostgreSQL +replication protocol that ordinary, unmodified tools can talk to it +directly -- nothing here needs a custom client. The commands below are +what those tools actually send; useful to know if you're connecting by +hand with ``psql "... replication=database"`` to check on an archiver, or +deciding what else could talk to one. + +``IDENTIFY_SYSTEM`` + + The first thing any of these tools asks: which system and timeline the + archiver is tracking, and how far it's captured so far. + +``BASE_BACKUP`` + + Streams the archiver's most recent base backup, in the same plain tar + format a real ``pg_basebackup --format=plain`` produces. Point a real, + unmodified ``pg_basebackup`` at an archiver and it works exactly as it + would against a live node -- this is what ``pg_autoctl create postgres + --from-archiver`` uses to bootstrap a brand new node straight from an + archiver's cache instead of a live primary or secondary. + +``START_REPLICATION`` + + Streams WAL from a given position onward, the same way a live primary + would. This is what lets a real standby's own ``primary_conninfo`` + point at an archiver instead of a live node, and what a multi-standby + failover election falls back on to fetch WAL a promoted candidate is + still missing, straight from the archiver's own cache, when no live + node has it anymore. + +``TIMELINE_HISTORY`` + + Returns the timeline history for a given timeline -- needed by any + streaming client following a timeline change, such as after a + failover. + +``CREATE_REPLICATION_SLOT`` / ``READ_REPLICATION_SLOT`` + + Basic physical replication slot support, for tools that expect to + manage their own slot against whatever they're streaming from. + +Fetching a single WAL file + + A small side channel used by this project's own ``restore_command`` + tooling: ask for one file by name, get its exact bytes back. This is + what makes an archiver usable as a ``restore_command`` target on its + own, without needing a full streaming connection just to recover one + missing segment. + +See also +-------- + +- :ref:`archiving_and_disaster_recovery` -- what an archiver is and + where it fits among the other architectures +- :ref:`archiving_operations` -- registering an archiver, attaching a + base-backup policy, rebuilding a node from one +- :ref:`archiving_fault_tolerance` -- what changes about fault tolerance + once an archiver is in the picture +- :ref:`failover_state_machine` -- the ``archiving`` state's own + transitions diff --git a/docs/archiving-internals.rst b/docs/archiving-internals.rst deleted file mode 100644 index 4e4a44bb9..000000000 --- a/docs/archiving-internals.rst +++ /dev/null @@ -1,547 +0,0 @@ -.. _archiving_internals: - -Archiving Architecture -======================= - -This page is the technical reference for how the :ref:`archiving_architecture` -feature is actually built: the two processes behind every archiver, when and -how ``pg_receivewal`` and ``pg_basebackup`` run, and ``pg_walsender``, the -project's own server-side implementation of enough of the PostgreSQL -replication protocol to serve captured WAL and base backups back out to a -real Postgres instance. It is meant to be the single place to read before -touching any of this code, and the place later milestones (warm standby, -point-in-time recovery, cloud storage push -- see `Extension points for -future milestones`_ below) extend rather than replace. - -For the operator-facing view -- registering an archiver, attaching a -base-backup policy, rebuilding a node from one -- see :ref:`archiving_operations`. -For where an archiver fits in the wider fault-tolerance picture, see -:ref:`archiving_fault_tolerance`. For the ``archiving`` state's exact FSM -transitions, see the :ref:`archiving_state` entry in :ref:`failover_state_machine`. - -Two processes, one archiver ----------------------------- - -``pg_autoctl archiver run`` (or ``pg_autoctl node run`` against a -``kind = archiver`` node spec) supervises exactly two long-running child -processes, registered with this project's ordinary ``supervisor.c`` -machinery the same way any other node kind's services are: - -- **capture** (``service_archiver_loop()``, ``service_archiver.c``) -- - keeps ``pg_receivewal`` running against the group's current primary and - reports progress to the monitor, the node-active loop every other node - kind also runs, just with an archiver's own body. -- **serve** (``service_archiver_serve_loop()``, ``service_archiver_serve.c``) - -- keeps ``pg_walsender`` running and refreshes the small routes file it - reads its configuration from. - -.. figure:: ./tikz/arch-archiver-internals.svg - :alt: pg_autoctl archiver run forks two processes, capture and serve; capture execs pg_receivewal and writes archiver-position, serve reads archiver-position and writes archiver-routes.ini, then execs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches - - Two forked processes per archiver, talking to each other only through - two small files on disk - -Each is its own ``fork()`` of the ``pg_autoctl`` process (no further -``exec()`` at that level -- they keep running as the same binary, in the -same fork-without-exec style ``pg_walsender``'s own accept loop uses one -level down), each with its own independent connection to the monitor, -since a libpq connection is not fork-safe to share. That split is also why -they never touch each other's memory: the only two channels between them -are two small files, both written atomically (write to a ``.tmp`` path, -then ``rename()``) so a concurrent reader never observes a partial write: - -``archiver-position`` - A single line holding the archiver's real, currently-captured WAL - position (see `Tracking the real captured position`_ below), written by - **capture** on every tick and read by **serve** on every routes-file - refresh. Lives next to the archiver's own ``pg_autoctl.cfg`` - (``path_in_same_directory(config->pathnames.config, "archiver-position", - ...)``). - -``archiver-routes.ini`` - The file ``pg_walsender`` itself reads its whole configuration from -- - see `Routing and auth: the routes file`_ below. Lives in the same - directory, written by **serve**. - -WAL capture ------------ - -Reusing the standby replication slot mechanism -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An archiver's WAL capture rides on a mechanism that already existed for -ordinary standbys, with no primary-side special-casing at all. -``keeper_create_and_drop_replication_slots()`` (``keeper.c``) runs on every -primary-role node, every tick, and calls -``postgres_replication_slot_create_and_drop()`` for every *other* node in -the group returned by ``AutoFailoverOtherNodesList()`` -(``src/monitor/node_metadata.c``) -- a list with no ``hasPgData`` filter, -so an archiver's row comes back exactly like a real standby's. The primary -ends up creating and maintaining a slot named -``pgautofailover_standby_`` for the archiver the same -way it would for a secondary, using the same -``REPLICATION_SLOT_NAME_DEFAULT`` prefix (``"pgautofailover_standby"``, -``defaults.h``) -- nothing in the primary's own slot-maintenance code -needs to know an archiver exists. - -That slot is what makes capture reliable in the first place: a replication -slot pins ``restart_lsn`` at *creation time* and the server retains WAL -back to it regardless of how many times the consumer disconnects and -reconnects. Without one, a ``pg_receivewal`` that loses the startup -HBA-propagation race restarts from the server's then-current position -instead of resuming, silently and permanently skipping whatever WAL -existed in the gap -- a real, observed failure mode this project's own -early testing hit before the slot was wired in. - -Starting ``pg_receivewal`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``service_archiver_start_pgreceivewal()`` (``service_archiver.c``) locates -the real, unmodified ``pg_receivewal`` binary next to ``pg_ctl`` -(``path_in_same_directory``), then runs ``fork()``/``execv()`` on it -directly -- a real exec, not a fork-without-exec like the two supervised -services above:: - - pg_receivewal -w -d "host= port= user=streaming_pgautofailover application_name=" \ - -D --no-sync -S pgautofailover_standby_ - -An archiver's own "pgdata" is not a real ``PGDATA`` -- there is no -``initdb``'d cluster underneath it, no Postgres instance ever started -against it. It is repurposed as the archiver's local WAL-cache root: -segments land there directly, complete ones named the usual 24 hex-digit -way, the one currently being written suffixed ``.partial`` (matching real -Postgres's own pre-allocation-to-full-segment-size behavior, -``XLogFileInitInternal``). The same directory later holds a -``basebackups/`` subdirectory once base backups start landing (see -`Base backup generation`_ below). - -The child's pid is tracked in a file-scope variable and reaped every tick -via ``waitpid(WNOHANG)``; a dead child is simply restarted on the next -tick capture notices it. - -Tracking the real captured position -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``service_archiver_update_current_lsn()`` scans the WAL cache directory -every tick for the newest complete segment and the newest ``.partial`` -one. When a ``.partial`` segment is the real frontier (newer than or equal -to the newest complete one), its trailing run of zero bytes -- the -unwritten tail of Postgres's own full-size pre-allocation -- is trimmed -off by reading the whole file and walking backwards from the end, so the -reported position reflects real, captured content, not the segment's -nominal full size. This is what makes an archiving node a real, rankable -candidate in ``pgautofailover.get_most_advanced_standby()`` during a -failover election: that query has no kind-based exclusion, it just needs a -node reporting something other than ``"0/0"``. - -That position is computed once, in the **capture** process, and is the -single source of truth for "how far has this archiver actually captured" --- consumed directly by the node-active report to the monitor, and written -to the ``archiver-position`` file (`Two processes, one archiver`_ above) -for **serve** to pick up, rather than each side independently re-deriving -it by scanning WAL file content on its own. - -Continuity across a failover -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An archiving node's own FSM is deliberately small -- see the :ref:`archiving_state` -state reference entry in :ref:`failover_state_machine` for the exact three -transitions (``wait_standby`` → ``archiving``, ``archiving`` → -``report_lsn``, ``report_lsn`` → ``archiving``). On ``ARCHIVING`` → -``REPORT_LSN`` (the group's primary becomes unreachable and a failover -starts), the current implementation stops ``pg_receivewal`` outright via -``fsm_archiver_report_lsn()`` rather than leaving it running against the -soon-to-be-former primary. Once a new primary is confirmed and the node is -assigned back to ``ARCHIVING``, ``pg_receivewal`` is started again, -pointed at the new primary, with the same slot name (still keyed on this -archiver's own node id, so the primary-side slot survives the handover -untouched). - -Base backup generation ------------------------ - -The ``basebackup_policy`` table -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Every base backup an archiver produces is scheduled and sourced according -to a policy row in ``pgautofailover.basebackup_policy`` -(``src/monitor/pgautofailover.sql``):: - - CREATE TABLE pgautofailover.basebackup_policy ( - basebackuppolicyid bigserial PRIMARY KEY, - policyname text UNIQUE, - source pgautofailover.basebackup_source NOT NULL DEFAULT 'replay', - replaymode pgautofailover.basebackup_replay_mode DEFAULT 'volatile', - cache pgautofailover.basebackup_cache NOT NULL DEFAULT 'local', - frequency interval NOT NULL DEFAULT '24 hours', - maxcount int NOT NULL DEFAULT 3, - maxage interval NOT NULL DEFAULT '3 days', - onpromotion bool NOT NULL DEFAULT true, - concurrency int NOT NULL DEFAULT 1, - CHECK (source <> 'replay' OR replaymode IS NOT NULL), - CHECK (concurrency >= 1) - ); - -A formation (or a specific group, as an override) attaches a policy via -``pgautofailover.set_archiver_policy()``, the same function both -``pg_autoctl create archiver --basebackup-policy`` and -``pg_autoctl set basebackup-policy`` ultimately call. Resolution is a -three-tier fallback, plpgsql rather than one ``UNION ALL`` query since -branch evaluation order there isn't guaranteed: -``get_archiver_policy(formationid, groupid)`` looks for an exact -``(formation, group)`` override first, then a formation-wide row -(``groupid IS NULL``), then falls back to the schema's own built-in -``'default'`` policy row (nightly-equivalent: 24h frequency, 3 kept, 3 -days max age). ``get_basebackup_policy_for_group()`` wraps that with the -join against ``basebackup_policy`` itself, flattening the ``interval`` -columns to plain integer seconds (``extract(epoch FROM ...)::int``) so the -C side does cheap ``time_t`` arithmetic instead of parsing intervals. - -``concurrency`` is schema-complete and read on the C side, but not yet -enforced -- a single archiver only ever runs one backup job at a time -regardless of its value, a correct simplification for as long as an -archiver belongs to a single ``(formation, group)`` membership (see -`Extension points for future milestones`_). - -Scheduling: when ``pg_basebackup`` runs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``service_archiver_maybe_generate_basebackup()`` -(``service_archiver_basebackup.c``) runs every tick of the capture loop, -alongside the WAL-report cycle. It resolves the group's policy, lists -existing backups (newest first), and decides: - -- **Bootstrap**: a group with zero backups gets one immediately, and it is - **always** sourced ``live`` regardless of the policy's configured - ``source`` -- there is nothing to replay from yet. -- **Forced by promotion**: if ``onpromotion`` is set and the group's - primary has changed since the last tick (tracked via a file-scope - "last known primary" value, seeded lazily so a process's very first - tick never misfires), a backup is forced regardless of ``frequency``. -- **Due by frequency**: otherwise, a backup is due once - ``now - >= policy.frequency``. - -Once a backup is due, generation forks a one-shot child (tracked via its -own pid, reaped the same way ``pg_receivewal``'s is) that runs to -completion and exits -- deliberately not exec'd, so it never blocks the -capture loop's own per-tick node-active/WAL-report cycle for however long -the backup takes. - -Live source -^^^^^^^^^^^^ - -``select_basebackup_source()`` picks the first healthy non-primary node in -the group, falling back to the primary if there is none (rows with -``port == 0`` -- the ARCHIVING-row sentinel -- are skipped, an archiver -never backs up from another archiver). ``run_pg_basebackup()`` then runs -the real, unmodified binary:: - - pg_basebackup -h -p -U streaming_pgautofailover -D \ - --format=plain --wal-method=none --checkpoint=fast --label

--route / " "--filename --output \n\n" @@ -105,7 +106,11 @@ main_fetch_file(int argc, char **argv) case 'p': { - port = atoi(optarg); + if (!stringToInt(optarg, &port)) + { + log_fatal("Invalid --port value \"%s\"", optarg); + return 1; + } break; } @@ -138,8 +143,9 @@ main_fetch_file(int argc, char **argv) if (host[0] == '\0' || route[0] == '\0' || filename[0] == '\0' || output[0] == '\0') { - fprintf(stderr, "fetch-file: --host, --route, --filename, and " - "--output are all required\n"); + fprintf(stderr, /* IGNORE-BANNED */ + "fetch-file: --host, --route, --filename, and " + "--output are all required\n"); usage(argv[0]); return 1; } @@ -181,7 +187,11 @@ main(int argc, char **argv) { case 'p': { - config.port = atoi(optarg); + if (!stringToInt(optarg, &(config.port))) + { + log_fatal("Invalid --port value \"%s\"", optarg); + return 1; + } break; } diff --git a/src/bin/pg_walsender/repl_command.c b/src/bin/pg_walsender/repl_command.c index 4db7fe859..fa31a4bef 100644 --- a/src/bin/pg_walsender/repl_command.c +++ b/src/bin/pg_walsender/repl_command.c @@ -13,6 +13,8 @@ #include "postgres_fe.h" +#include "string_utils.h" + #include "repl_command.h" #include "cmd_base_backup.h" #include "cmd_identify_system.h" @@ -83,7 +85,12 @@ repl_command_parse(const char *query, WsCommand *cmd) if (strncasecmp(p, "TIMELINE_HISTORY", strlen("TIMELINE_HISTORY")) == 0) { p = skip_whitespace(p + strlen("TIMELINE_HISTORY")); - cmd->timeline = atoi(p); + + if (!stringToInt(p, &(cmd->timeline))) + { + return false; + } + cmd->type = WS_CMD_TIMELINE_HISTORY; return true; } diff --git a/src/bin/pg_walsender/routes.c b/src/bin/pg_walsender/routes.c index 1a3c8469b..107cef1b4 100644 --- a/src/bin/pg_walsender/routes.c +++ b/src/bin/pg_walsender/routes.c @@ -26,6 +26,7 @@ #include "routes.h" #include "file_utils.h" #include "log.h" +#include "string_utils.h" bool @@ -130,7 +131,7 @@ routes_load(const char *path, WsRoute **routesOut, int *countOut) } else if (strcmp(propName, "timeline") == 0) { - route->timeline = atoi(propValue); + (void) stringToInt(propValue, &(route->timeline)); } else if (strcmp(propName, "position") == 0) { diff --git a/src/bin/pg_walsender/startup.c b/src/bin/pg_walsender/startup.c index 341a31fdc..0a3859091 100644 --- a/src/bin/pg_walsender/startup.c +++ b/src/bin/pg_walsender/startup.c @@ -46,7 +46,7 @@ ws_startup_negotiate(int sock, WsStartupParams *params) int32_t code; - memcpy(&code, payload, 4); + memcpy(&code, payload, 4); /* IGNORE-BANNED */ code = ntohl(code); if (code == SSL_REQUEST_CODE || code == GSS_REQUEST_CODE) diff --git a/src/bin/pg_walsender/tar_stream.c b/src/bin/pg_walsender/tar_stream.c index 689553395..fc0a48a67 100644 --- a/src/bin/pg_walsender/tar_stream.c +++ b/src/bin/pg_walsender/tar_stream.c @@ -17,6 +17,7 @@ #include "pgtar.h" #include "tar_stream.h" +#include "file_utils.h" #include "log.h" /* matches basebackup.c's own TAR_NUM_TERMINATION_BLOCKS */ @@ -75,7 +76,7 @@ emit_header(TarWalkState *state, const char *memberName, static bool emit_file_contents(TarWalkState *state, const char *path, off_t size) { - FILE *file = fopen(path, "rb"); + FILE *file = fopen(path, "rb"); /* IGNORE-BANNED */ if (file == NULL) { @@ -137,7 +138,7 @@ walk_directory(TarWalkState *state, const char *rootDir, const char *relDir) } else { - snprintf(fullDir, sizeof(fullDir), "%s/%s", rootDir, relDir); + sformat(fullDir, sizeof(fullDir), "%s/%s", rootDir, relDir); } DIR *dir = opendir(fullDir); @@ -160,7 +161,7 @@ walk_directory(TarWalkState *state, const char *rootDir, const char *relDir) char fullPath[MAXPGPATH]; char relPath[MAXPGPATH]; - snprintf(fullPath, sizeof(fullPath), "%s/%s", fullDir, entry->d_name); + sformat(fullPath, sizeof(fullPath), "%s/%s", fullDir, entry->d_name); if (relDir[0] == '\0') { @@ -168,7 +169,7 @@ walk_directory(TarWalkState *state, const char *rootDir, const char *relDir) } else { - snprintf(relPath, sizeof(relPath), "%s/%s", relDir, entry->d_name); + sformat(relPath, sizeof(relPath), "%s/%s", relDir, entry->d_name); } struct stat st; diff --git a/src/bin/pg_walsender/vendor/tar.c b/src/bin/pg_walsender/vendor/tar.c index 8049b4995..626439053 100644 --- a/src/bin/pg_walsender/vendor/tar.c +++ b/src/bin/pg_walsender/vendor/tar.c @@ -250,10 +250,10 @@ tarCreateHeader(char *h, const char *filename, const char *linktarget, } /* Magic 6 */ - strcpy(&h[TAR_OFFSET_MAGIC], "ustar"); + strcpy(&h[TAR_OFFSET_MAGIC], "ustar"); /* IGNORE-BANNED */ /* Version 2 */ - memcpy(&h[TAR_OFFSET_VERSION], "00", 2); + memcpy(&h[TAR_OFFSET_VERSION], "00", 2); /* IGNORE-BANNED */ /* User 32 */ /* XXX: Do we need to care about setting correct username? */ diff --git a/src/bin/pg_walsender/wal_dir_scan.c b/src/bin/pg_walsender/wal_dir_scan.c index 2ebf26bbc..cbd9d43bf 100644 --- a/src/bin/pg_walsender/wal_dir_scan.c +++ b/src/bin/pg_walsender/wal_dir_scan.c @@ -15,6 +15,7 @@ #include "postgres_fe.h" #include "wal_dir_scan.h" +#include "file_utils.h" /* default WAL segment size (16MB), matching cmd_show.c's own * "SHOW wal_segment_size" -> "16MB" answer */ @@ -52,7 +53,7 @@ wal_segment_filename(uint32_t timeline, uint64_t segno, char *dest, size_t destS uint32_t logId = (uint32_t) (segno / WS_XLOG_SEGMENTS_PER_XLOGID); uint32_t seg = (uint32_t) (segno % WS_XLOG_SEGMENTS_PER_XLOGID); - snprintf(dest, destSize, "%08X%08X%08X", timeline, logId, seg); + sformat(dest, destSize, "%08X%08X%08X", timeline, logId, seg); } @@ -94,9 +95,9 @@ wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, char logIdHex[9] = { 0 }; char segHex[9] = { 0 }; - memcpy(tliHex, best, 8); - memcpy(logIdHex, best + 8, 8); - memcpy(segHex, best + 16, 8); + memcpy(tliHex, best, 8); /* IGNORE-BANNED */ + memcpy(logIdHex, best + 8, 8); /* IGNORE-BANNED */ + memcpy(segHex, best + 16, 8); /* IGNORE-BANNED */ uint32_t tli = (uint32_t) strtoul(tliHex, NULL, 16); uint32_t logId = (uint32_t) strtoul(logIdHex, NULL, 16); @@ -106,8 +107,8 @@ wal_dir_find_latest(const char *walcacheDir, uint32_t *timeline, uint64_t endOfSegment = (segno + 1) * WS_WAL_SEGMENT_SIZE; *timeline = tli; - snprintf(endLsn, endLsnSize, "%X/%08X", - (uint32_t) (endOfSegment >> 32), (uint32_t) (endOfSegment & 0xFFFFFFFF)); + sformat(endLsn, endLsnSize, "%X/%08X", + (uint32_t) (endOfSegment >> 32), (uint32_t) (endOfSegment & 0xFFFFFFFF)); return true; } From c67cf3f806bb4e8e7c169c7797471dd843cc6e89 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:04:53 +0200 Subject: [PATCH 38/55] pgaftest: top-level archiver { } syntax + region support An archiver can now be declared directly inside cluster { }, as its own "archiver { }" block -- a sibling of monitor/formation, not nested inside a formation_block the way it had to be before. This matches the real data model (pgautofailover.archiver has no formation column at all; it attaches to one or more formations by name, it isn't a member of any one of them): archiver archiver1 { formation default # required; exactly one at create time region eu-west # optional; default "default" create and launch deferred # optional } A real shift/reduce ambiguity turned up while designing this: a brace-less "formation " archiver option is indistinguishable, at one token of lookahead, from a brand new top-level formation_block starting right after (formation_block's own opening also accepts a bare name). Braces around the archiver's own option list resolve it. Internally, fold_archivers_into_formations() (called once, right after yyparse() returns) turns each declared archiver into an ordinary TestNode appended to its own named formation's node list -- so every existing per-node code path (ini writing, "pg_autoctl node run " as the container command, healthcheck/depends_on ordering, create/launch deferred handling) already used for an archiver nested inside a formation_block just works for these too, completely unmodified. Only one formation is accepted per archiver, since pg_autoctl create archiver's own ini-driven bootstrap has no notion of attaching to more than one at create time (unlike the CLI's own repeatable --formation) -- a clear error directs to attaching the rest dynamically once running instead of silently dropping them. TestNode's own existing "region" support (already wired through to ordinary nodes' --region) now also reaches an archiver, via build_membership_keeper()'s sibling in nodespec.c gaining the matching --region push for kind = archiver. --- src/bin/pgaftest/test_spec.h | 82 ++ src/bin/pgaftest/test_spec_parse.c | 1698 +++++++++++++++------------- src/bin/pgaftest/test_spec_parse.h | 2 +- src/bin/pgaftest/test_spec_parse.y | 179 +++ 4 files changed, 1200 insertions(+), 761 deletions(-) diff --git a/src/bin/pgaftest/test_spec.h b/src/bin/pgaftest/test_spec.h index 3acb7bc9b..0f6dc90dc 100644 --- a/src/bin/pgaftest/test_spec.h +++ b/src/bin/pgaftest/test_spec.h @@ -18,11 +18,16 @@ #define PGAF_MAX_STEPS 256 #define PGAF_MAX_SEQ 256 #define PGAF_TIMEOUT_DEFAULT 90 +#define PGAF_MAX_ARCHIVERS 8 +#define PGAF_MAX_ARCHIVER_FORMATIONS 8 /* ----------------------------------------------------------------------- * Cluster topology (from the cluster { } block) * * Hierarchy: cluster → monitor + formations → nodes + * ↘ archivers (attach to one or more formations by name, + * not nested inside any one of them -- see + * TestArchiverNode's own comment below) * * Syntax: * @@ -45,6 +50,16 @@ * w1 worker group 1 * w2 worker group 1 * } + * + * # Top-level archiver, sibling to monitor/formation -- see + * # TestArchiverNode's own comment for why this isn't nested inside + * # a formation_block the way ordinary nodes are. Braces are + * # mandatory here (unlike monitor's own bare form) -- see + * # archiver_block's own comment in test_spec_parse.y for why. + * archiver archiver1 { + * formation default + * region dc1 + * } * } * * When "formation" has no name it defaults to "default". @@ -98,11 +113,78 @@ typedef struct TestFormation int nodeCount; } TestFormation; +/* ----------------------------------------------------------------------- + * Top-level archiver nodes (from a cluster-level "archiver { ... }" + * declaration, sibling to "monitor" and "formation" -- NOT nested inside a + * formation_block's node_list the way ordinary/coordinator/worker nodes + * are). This matches the real data model: pgautofailover.archiver has no + * formationid column at all, and attaches to one or more formations + * through the separate archiver_formation join table -- an archiver is a + * process identity that formations attach to, not a member of any one of + * them. + * + * Syntax: + * + * archiver archiver1 { + * formation default + * region eu-west # optional; defaults to "default" + * create and launch deferred # optional; see below + * } + * + * Despite being declared at the top level, an archiver ends up represented + * internally as an ordinary TestNode (kind = NODE_KIND_ARCHIVER), appended + * to its own declared formation's own node list -- see parse_test_spec()'s + * own fold_archivers_into_formations() call, run once right after + * yyparse() returns. This means compose_gen.c's existing, fully-featured + * per-node machinery (writing a real pg_autoctl_node.ini, "pg_autoctl node + * run " as the container's own command, healthcheck/depends_on + * ordering, create/launch-deferred handling) already used for an archiver + * nested directly inside a formation_block -- the older, still-supported + * spelling -- just works for these too, completely unmodified. Only the + * *declaration* needs to be top-level, to match the real data model + * (pgautofailover.archiver has no formationid column at all, it attaches + * to formations through the separate archiver_formation join table, so an + * archiver isn't a member of any one formation the way an ordinary node + * genuinely is) -- once parsed, there is no other difference left. + * + * Only ever attaches to the FIRST formation listed: pg_autoctl create + * archiver's own ini-driven bootstrap (nodespec.c) has no notion of more + * than one --formation at create time. An archiver that needs to cover + * more than one formation from the very start should still declare just + * that first one here, then attach the rest once it's running (see + * archiver_multi_formation.pgaf for the pattern: a direct `sql monitor { + * SELECT pgautofailover.archiver_add_formation(...) }` step) -- + * fold_archivers_into_formations() exits with a clear error rather than + * silently dropping any formation past the first. + * + * "create and launch deferred" (or either half alone) behaves exactly as + * it does for an ordinary node: the container still runs "pg_autoctl node + * run ", but the ini's own [launch] section makes that command poll + * and wait rather than actually registering -- `exec pg_autoctl + * node start` un-defers it explicitly, same as any other deferred node + * (see citus_basic_operation.pgaf's own test_011 for why a Citus + * formation's archiver needs this: it must not attempt to register before + * every worker group already exists, and nothing here waits for that on + * its own). + * ----------------------------------------------------------------------- */ +typedef struct TestArchiverNode +{ + char name[128]; + char region[64]; /* --region NAME; "" = omit (defaults to "default") */ + char formations[PGAF_MAX_ARCHIVER_FORMATIONS][128]; + int formationCount; + bool createDeferred; /* node waits before pg_autoctl create */ + bool launchDeferred; /* node waits for pg_autoctl node start */ +} TestArchiverNode; + typedef struct TestCluster { TestFormation formations[PGAF_MAX_FORMATIONS]; int formationCount; + TestArchiverNode archivers[PGAF_MAX_ARCHIVERS]; + int archiverCount; + bool withMonitor; /* true when "monitor" keyword appears in cluster{} */ bool withCitus; bool bindSource; /* bind-source: mount repo root → /usr/src/pg_auto_failover */ diff --git a/src/bin/pgaftest/test_spec_parse.c b/src/bin/pgaftest/test_spec_parse.c index bc99f37b0..1dcbdd61d 100644 --- a/src/bin/pgaftest/test_spec_parse.c +++ b/src/bin/pgaftest/test_spec_parse.c @@ -454,6 +454,7 @@ static TestCmd *current_promote_cmd = NULL; static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ static TestFormation *current_formation = NULL; static TestNode *current_node = NULL; +static TestArchiverNode *current_archiver = NULL; @@ -477,7 +478,7 @@ static TestNode *current_node = NULL; #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED typedef union YYSTYPE -#line 145 "test_spec_parse.y" +#line 146 "test_spec_parse.y" { int ival; char *str; @@ -485,7 +486,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 193 of yacc.c. */ -#line 489 "test_spec_parse.c" +#line 490 "test_spec_parse.c" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 @@ -498,7 +499,7 @@ typedef union YYSTYPE /* Line 216 of yacc.c. */ -#line 502 "test_spec_parse.c" +#line 503 "test_spec_parse.c" #ifdef short # undef short @@ -713,16 +714,16 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 21 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 634 +#define YYLAST 609 /* YYNTOKENS -- Number of terminals. */ #define YYNTOKENS 122 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 65 +#define YYNNTS 69 /* YYNRULES -- Number of rules. */ -#define YYNRULES 215 +#define YYNRULES 226 /* YYNRULES -- Number of states. */ -#define YYNSTATES 356 +#define YYNSTATES 376 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 @@ -781,126 +782,132 @@ static const yytype_uint16 yyprhs[] = { 0, 0, 3, 5, 8, 10, 12, 14, 16, 18, 19, 25, 26, 29, 31, 33, 35, 37, 39, 41, - 43, 45, 47, 51, 55, 59, 63, 68, 73, 80, - 83, 86, 89, 92, 95, 98, 101, 102, 109, 110, - 113, 115, 117, 119, 121, 123, 125, 128, 131, 132, - 135, 137, 139, 140, 141, 146, 147, 155, 156, 159, - 161, 163, 165, 167, 169, 171, 173, 176, 179, 184, - 187, 189, 191, 193, 196, 199, 202, 205, 208, 211, - 214, 217, 220, 223, 226, 229, 232, 235, 239, 243, - 246, 249, 253, 257, 258, 261, 263, 265, 267, 269, - 271, 273, 275, 277, 279, 281, 283, 285, 287, 289, - 291, 293, 297, 300, 304, 307, 311, 314, 318, 321, - 323, 325, 327, 332, 337, 339, 343, 344, 347, 349, - 351, 355, 359, 360, 370, 371, 381, 389, 397, 403, - 410, 416, 423, 425, 427, 431, 435, 436, 439, 442, - 447, 448, 451, 455, 462, 469, 476, 483, 487, 490, - 493, 497, 501, 504, 506, 510, 513, 518, 524, 532, - 536, 540, 546, 552, 555, 558, 562, 566, 570, 575, - 579, 583, 587, 588, 594, 600, 604, 609, 615, 620, - 626, 629, 630, 633, 635, 637, 639, 641, 643, 645, - 647, 649, 651, 653, 655, 657, 659, 661, 663, 665, - 667, 669, 671, 673, 675, 677 + 43, 45, 47, 48, 55, 56, 59, 62, 65, 68, + 73, 76, 79, 81, 85, 89, 93, 97, 102, 107, + 114, 117, 120, 123, 126, 129, 132, 135, 136, 143, + 144, 147, 149, 151, 153, 155, 157, 159, 162, 165, + 166, 169, 171, 173, 174, 175, 180, 181, 189, 190, + 193, 195, 197, 199, 201, 203, 205, 207, 210, 213, + 218, 221, 223, 225, 227, 230, 233, 236, 239, 242, + 245, 248, 251, 254, 257, 260, 263, 266, 269, 273, + 277, 280, 283, 287, 291, 292, 295, 297, 299, 301, + 303, 305, 307, 309, 311, 313, 315, 317, 319, 321, + 323, 325, 327, 331, 334, 338, 341, 345, 348, 352, + 355, 357, 359, 361, 366, 371, 373, 377, 378, 381, + 383, 385, 389, 393, 394, 404, 405, 415, 423, 431, + 437, 444, 450, 457, 459, 461, 465, 469, 470, 473, + 476, 481, 482, 485, 489, 496, 503, 510, 517, 521, + 524, 527, 531, 535, 538, 540, 544, 547, 552, 558, + 566, 570, 574, 580, 586, 589, 592, 596, 600, 604, + 609, 613, 617, 621, 622, 628, 634, 638, 643, 649, + 654, 660, 663, 664, 667, 669, 671, 673, 675, 677, + 679, 681, 683, 685, 687, 689, 691, 693, 695, 697, + 699, 701, 703, 705, 707, 709, 711 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { 123, 0, -1, 124, -1, 123, 124, -1, 125, -1, - 147, -1, 148, -1, 149, -1, 183, -1, -1, 3, - 103, 126, 127, 104, -1, -1, 127, 128, -1, 129, - -1, 130, -1, 132, -1, 133, -1, 131, -1, 134, - -1, 45, -1, 46, -1, 4, -1, 4, 41, 118, - -1, 4, 14, 118, -1, 4, 37, 117, -1, 4, - 38, 119, -1, 4, 118, 26, 28, -1, 4, 118, - 32, 96, -1, 4, 118, 26, 28, 38, 119, -1, - 13, 119, -1, 13, 118, -1, 44, 118, -1, 44, - 119, -1, 15, 118, -1, 16, 118, -1, 17, 118, - -1, -1, 18, 135, 136, 103, 139, 104, -1, -1, - 136, 138, -1, 118, -1, 119, -1, 16, -1, 4, - -1, 5, -1, 137, -1, 19, 117, -1, 57, 30, - -1, -1, 139, 142, -1, 118, -1, 4, -1, -1, - -1, 140, 141, 143, 145, -1, -1, 5, 118, 141, - 144, 103, 145, 104, -1, -1, 145, 146, -1, 20, - -1, 21, -1, 22, -1, 23, -1, 24, -1, 25, - -1, 28, -1, 26, 28, -1, 27, 28, -1, 27, - 77, 26, 28, -1, 26, 29, -1, 29, -1, 34, - -1, 35, -1, 36, 117, -1, 47, 118, -1, 47, - 119, -1, 102, 117, -1, 37, 117, -1, 40, 118, - -1, 41, 118, -1, 15, 118, -1, 16, 118, -1, - 17, 118, -1, 42, 31, -1, 42, 30, -1, 43, - 119, -1, 39, 119, -1, 33, 118, 118, -1, 33, - 118, 119, -1, 8, 150, -1, 9, 150, -1, 10, - 186, 150, -1, 103, 151, 104, -1, -1, 151, 152, - -1, 153, -1, 159, -1, 166, -1, 167, -1, 168, - -1, 169, -1, 171, -1, 172, -1, 174, -1, 175, - -1, 176, -1, 177, -1, 180, -1, 181, -1, 182, - -1, 173, -1, 70, 118, 121, -1, 70, 118, -1, - 71, 118, 121, -1, 71, 118, -1, 72, 118, 121, - -1, 72, 118, -1, 73, 118, 121, -1, 73, 118, - -1, 73, -1, 12, -1, 78, -1, 118, 99, 154, - 185, -1, 118, 99, 154, 118, -1, 155, -1, 156, - 77, 155, -1, -1, 109, 158, -1, 185, -1, 118, - -1, 158, 105, 185, -1, 158, 105, 118, -1, -1, - 74, 75, 118, 99, 154, 185, 160, 157, 165, -1, - -1, 74, 75, 118, 99, 154, 118, 161, 157, 165, - -1, 74, 75, 118, 100, 154, 185, 165, -1, 74, - 75, 118, 100, 154, 118, 165, -1, 74, 75, 118, - 96, 165, -1, 74, 75, 118, 80, 118, 165, -1, - 74, 75, 162, 163, 165, -1, 74, 75, 155, 77, - 156, 165, -1, 185, -1, 118, -1, 162, 105, 185, - -1, 162, 105, 118, -1, -1, 101, 164, -1, 102, - 117, -1, 164, 105, 102, 117, -1, -1, 76, 117, - -1, 79, 76, 117, -1, 81, 118, 99, 154, 185, - 165, -1, 81, 118, 99, 154, 118, 165, -1, 81, - 118, 100, 154, 185, 165, -1, 81, 118, 100, 154, - 118, 165, -1, 82, 118, 120, -1, 83, 120, -1, - 83, 84, -1, 83, 84, 118, -1, 83, 84, 117, - -1, 85, 170, -1, 118, -1, 170, 105, 118, -1, - 86, 87, -1, 86, 87, 102, 117, -1, 86, 87, - 101, 18, 118, -1, 86, 87, 101, 18, 118, 102, - 117, -1, 88, 89, 118, -1, 88, 90, 118, -1, - 48, 110, 118, 118, 118, -1, 48, 111, 118, 118, - 118, -1, 91, 117, -1, 92, 93, -1, 92, 94, - 118, -1, 92, 95, 118, -1, 92, 97, 118, -1, - 92, 98, 118, 121, -1, 95, 106, 140, -1, 94, - 106, 140, -1, 112, 10, 140, -1, -1, 108, 179, - 103, 151, 104, -1, 81, 140, 107, 185, 178, -1, - 110, 118, 118, -1, 113, 118, 115, 119, -1, 113, - 118, 114, 115, 119, -1, 113, 118, 116, 119, -1, - 113, 118, 114, 116, 119, -1, 11, 184, -1, -1, - 184, 186, -1, 49, -1, 50, -1, 51, -1, 52, - -1, 53, -1, 54, -1, 55, -1, 56, -1, 57, - -1, 58, -1, 59, -1, 60, -1, 61, -1, 62, - -1, 63, -1, 64, -1, 65, -1, 66, -1, 67, - -1, 68, -1, 69, -1, 118, -1, 119, -1 + 151, -1, 152, -1, 153, -1, 187, -1, -1, 3, + 103, 126, 127, 104, -1, -1, 127, 128, -1, 133, + -1, 134, -1, 136, -1, 137, -1, 135, -1, 138, + -1, 129, -1, 45, -1, 46, -1, -1, 22, 118, + 130, 103, 131, 104, -1, -1, 131, 132, -1, 18, + 118, -1, 47, 118, -1, 47, 119, -1, 27, 77, + 26, 28, -1, 26, 28, -1, 27, 28, -1, 4, + -1, 4, 41, 118, -1, 4, 14, 118, -1, 4, + 37, 117, -1, 4, 38, 119, -1, 4, 118, 26, + 28, -1, 4, 118, 32, 96, -1, 4, 118, 26, + 28, 38, 119, -1, 13, 119, -1, 13, 118, -1, + 44, 118, -1, 44, 119, -1, 15, 118, -1, 16, + 118, -1, 17, 118, -1, -1, 18, 139, 140, 103, + 143, 104, -1, -1, 140, 142, -1, 118, -1, 119, + -1, 16, -1, 4, -1, 5, -1, 141, -1, 19, + 117, -1, 57, 30, -1, -1, 143, 146, -1, 118, + -1, 4, -1, -1, -1, 144, 145, 147, 149, -1, + -1, 5, 118, 145, 148, 103, 149, 104, -1, -1, + 149, 150, -1, 20, -1, 21, -1, 22, -1, 23, + -1, 24, -1, 25, -1, 28, -1, 26, 28, -1, + 27, 28, -1, 27, 77, 26, 28, -1, 26, 29, + -1, 29, -1, 34, -1, 35, -1, 36, 117, -1, + 47, 118, -1, 47, 119, -1, 102, 117, -1, 37, + 117, -1, 40, 118, -1, 41, 118, -1, 15, 118, + -1, 16, 118, -1, 17, 118, -1, 42, 31, -1, + 42, 30, -1, 43, 119, -1, 39, 119, -1, 33, + 118, 118, -1, 33, 118, 119, -1, 8, 154, -1, + 9, 154, -1, 10, 190, 154, -1, 103, 155, 104, + -1, -1, 155, 156, -1, 157, -1, 163, -1, 170, + -1, 171, -1, 172, -1, 173, -1, 175, -1, 176, + -1, 178, -1, 179, -1, 180, -1, 181, -1, 184, + -1, 185, -1, 186, -1, 177, -1, 70, 118, 121, + -1, 70, 118, -1, 71, 118, 121, -1, 71, 118, + -1, 72, 118, 121, -1, 72, 118, -1, 73, 118, + 121, -1, 73, 118, -1, 73, -1, 12, -1, 78, + -1, 118, 99, 158, 189, -1, 118, 99, 158, 118, + -1, 159, -1, 160, 77, 159, -1, -1, 109, 162, + -1, 189, -1, 118, -1, 162, 105, 189, -1, 162, + 105, 118, -1, -1, 74, 75, 118, 99, 158, 189, + 164, 161, 169, -1, -1, 74, 75, 118, 99, 158, + 118, 165, 161, 169, -1, 74, 75, 118, 100, 158, + 189, 169, -1, 74, 75, 118, 100, 158, 118, 169, + -1, 74, 75, 118, 96, 169, -1, 74, 75, 118, + 80, 118, 169, -1, 74, 75, 166, 167, 169, -1, + 74, 75, 159, 77, 160, 169, -1, 189, -1, 118, + -1, 166, 105, 189, -1, 166, 105, 118, -1, -1, + 101, 168, -1, 102, 117, -1, 168, 105, 102, 117, + -1, -1, 76, 117, -1, 79, 76, 117, -1, 81, + 118, 99, 158, 189, 169, -1, 81, 118, 99, 158, + 118, 169, -1, 81, 118, 100, 158, 189, 169, -1, + 81, 118, 100, 158, 118, 169, -1, 82, 118, 120, + -1, 83, 120, -1, 83, 84, -1, 83, 84, 118, + -1, 83, 84, 117, -1, 85, 174, -1, 118, -1, + 174, 105, 118, -1, 86, 87, -1, 86, 87, 102, + 117, -1, 86, 87, 101, 18, 118, -1, 86, 87, + 101, 18, 118, 102, 117, -1, 88, 89, 118, -1, + 88, 90, 118, -1, 48, 110, 118, 118, 118, -1, + 48, 111, 118, 118, 118, -1, 91, 117, -1, 92, + 93, -1, 92, 94, 118, -1, 92, 95, 118, -1, + 92, 97, 118, -1, 92, 98, 118, 121, -1, 95, + 106, 144, -1, 94, 106, 144, -1, 112, 10, 144, + -1, -1, 108, 183, 103, 155, 104, -1, 81, 144, + 107, 189, 182, -1, 110, 118, 118, -1, 113, 118, + 115, 119, -1, 113, 118, 114, 115, 119, -1, 113, + 118, 116, 119, -1, 113, 118, 114, 116, 119, -1, + 11, 188, -1, -1, 188, 190, -1, 49, -1, 50, + -1, 51, -1, 52, -1, 53, -1, 54, -1, 55, + -1, 56, -1, 57, -1, 58, -1, 59, -1, 60, + -1, 61, -1, 62, -1, 63, -1, 64, -1, 65, + -1, 66, -1, 67, -1, 68, -1, 69, -1, 118, + -1, 119, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 216, 216, 217, 221, 222, 223, 224, 225, 238, - 237, 247, 249, 253, 254, 255, 256, 257, 258, 259, - 260, 273, 277, 284, 291, 297, 304, 311, 318, 331, - 337, 347, 353, 363, 373, 379, 390, 389, 406, 408, - 417, 418, 419, 420, 421, 425, 430, 434, 440, 442, - 461, 462, 471, 488, 487, 495, 494, 502, 504, 508, - 513, 518, 522, 526, 530, 534, 540, 545, 549, 554, - 558, 562, 566, 570, 574, 579, 584, 588, 592, 598, - 604, 609, 614, 619, 623, 627, 633, 639, 653, 674, - 681, 692, 710, 725, 728, 736, 737, 738, 739, 740, - 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, - 751, 765, 772, 778, 785, 791, 798, 804, 812, 818, - 845, 845, 856, 871, 889, 890, 905, 907, 911, 919, - 927, 934, 946, 945, 957, 956, 967, 976, 985, 999, - 1007, 1021, 1036, 1042, 1049, 1055, 1068, 1070, 1074, 1079, - 1087, 1088, 1089, 1100, 1108, 1116, 1124, 1142, 1157, 1164, - 1168, 1174, 1187, 1195, 1203, 1224, 1231, 1238, 1246, 1262, - 1268, 1289, 1297, 1312, 1326, 1330, 1336, 1342, 1368, 1402, - 1408, 1429, 1446, 1446, 1451, 1470, 1495, 1504, 1513, 1522, - 1538, 1541, 1543, 1565, 1566, 1567, 1568, 1569, 1570, 1571, - 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, - 1582, 1583, 1584, 1585, 1593, 1594 + 0, 217, 217, 218, 222, 223, 224, 225, 226, 239, + 238, 248, 250, 254, 255, 256, 257, 258, 259, 260, + 261, 262, 285, 284, 302, 304, 308, 322, 327, 332, + 339, 343, 359, 363, 370, 377, 383, 390, 397, 404, + 417, 423, 433, 439, 449, 459, 465, 476, 475, 492, + 494, 503, 504, 505, 506, 507, 511, 516, 520, 526, + 528, 547, 548, 557, 574, 573, 581, 580, 588, 590, + 594, 599, 604, 608, 612, 616, 620, 626, 631, 635, + 640, 644, 648, 652, 656, 660, 665, 670, 674, 678, + 684, 690, 695, 700, 705, 709, 713, 719, 725, 739, + 760, 767, 778, 796, 811, 814, 822, 823, 824, 825, + 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, + 836, 837, 851, 858, 864, 871, 877, 884, 890, 898, + 904, 931, 931, 942, 957, 975, 976, 991, 993, 997, + 1005, 1013, 1020, 1032, 1031, 1043, 1042, 1053, 1062, 1071, + 1085, 1093, 1107, 1122, 1128, 1135, 1141, 1154, 1156, 1160, + 1165, 1173, 1174, 1175, 1186, 1194, 1202, 1210, 1228, 1243, + 1250, 1254, 1260, 1273, 1281, 1289, 1310, 1317, 1324, 1332, + 1348, 1354, 1375, 1383, 1398, 1412, 1416, 1422, 1428, 1454, + 1488, 1494, 1515, 1532, 1532, 1537, 1556, 1581, 1590, 1599, + 1608, 1624, 1627, 1629, 1651, 1652, 1653, 1654, 1655, 1656, + 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, + 1667, 1668, 1669, 1670, 1671, 1679, 1680 }; #endif @@ -936,17 +943,18 @@ static const char *const yytname[] = "T_GET", "T_FSM", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", "T_INTEGER", "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", "spec", "spec_item", "cluster_block", "@1", "cluster_item_list", - "cluster_item", "monitor_line", "image_line", "extension_version_line", - "ssl_line", "auth_line", "formation_block", "@2", "formation_opt_list", + "cluster_item", "archiver_block", "@2", "archiver_opt_list", + "archiver_opt", "monitor_line", "image_line", "extension_version_line", + "ssl_line", "auth_line", "formation_block", "@3", "formation_opt_list", "bare_name", "formation_opt", "node_list", "node_name", "init_node_slot", - "node_line", "@3", "@4", "node_opt_list", "node_opt", "setup_block", + "node_line", "@4", "@5", "node_opt_list", "node_opt", "setup_block", "teardown_block", "named_step", "cmd_block", "cmd_list", "step_cmd", "exec_cmd", "state_op", "wait_multi_condition", "wait_multi_condition_list", "opt_passing_through", "pass_state_list", - "wait_cmd", "@5", "@6", "state_name_list", "opt_in_group", "group_items", + "wait_cmd", "@6", "@7", "state_name_list", "opt_in_group", "group_items", "opt_timeout", "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", "promote_list", "perform_cmd", "network_cmd", "nodeini_cmd", "sleep_cmd", - "compose_cmd", "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@7", + "compose_cmd", "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@8", "stays_while_cmd", "set_monitor_cmd", "logs_cmd", "sequence_block", "sequence_names", "fsm_state", "ident_or_string", 0 }; @@ -978,26 +986,27 @@ static const yytype_uint8 yyr1[] = { 0, 122, 123, 123, 124, 124, 124, 124, 124, 126, 125, 127, 127, 128, 128, 128, 128, 128, 128, 128, - 128, 129, 129, 129, 129, 129, 129, 129, 129, 130, - 130, 131, 131, 132, 133, 133, 135, 134, 136, 136, - 137, 137, 137, 137, 137, 138, 138, 138, 139, 139, - 140, 140, 141, 143, 142, 144, 142, 145, 145, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 147, - 148, 149, 150, 151, 151, 152, 152, 152, 152, 152, - 152, 152, 152, 152, 152, 152, 152, 152, 152, 152, - 152, 153, 153, 153, 153, 153, 153, 153, 153, 153, - 154, 154, 155, 155, 156, 156, 157, 157, 158, 158, - 158, 158, 160, 159, 161, 159, 159, 159, 159, 159, - 159, 159, 162, 162, 162, 162, 163, 163, 164, 164, - 165, 165, 165, 166, 166, 166, 166, 167, 168, 168, - 168, 168, 169, 170, 170, 171, 171, 171, 171, 172, - 172, 173, 173, 174, 175, 175, 175, 175, 175, 176, - 176, 177, 179, 178, 180, 181, 182, 182, 182, 182, - 183, 184, 184, 185, 185, 185, 185, 185, 185, 185, - 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, - 185, 185, 185, 185, 186, 186 + 128, 128, 130, 129, 131, 131, 132, 132, 132, 132, + 132, 132, 133, 133, 133, 133, 133, 133, 133, 133, + 134, 134, 135, 135, 136, 137, 137, 139, 138, 140, + 140, 141, 141, 141, 141, 141, 142, 142, 142, 143, + 143, 144, 144, 145, 147, 146, 148, 146, 149, 149, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 151, 152, 153, 154, 155, 155, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 157, 157, 157, 157, 157, 157, 157, 157, + 157, 158, 158, 159, 159, 160, 160, 161, 161, 162, + 162, 162, 162, 164, 163, 165, 163, 163, 163, 163, + 163, 163, 163, 166, 166, 166, 166, 167, 167, 168, + 168, 169, 169, 169, 170, 170, 170, 170, 171, 172, + 172, 172, 172, 173, 174, 174, 175, 175, 175, 175, + 176, 176, 177, 177, 178, 179, 179, 179, 179, 179, + 180, 180, 181, 183, 182, 184, 185, 186, 186, 186, + 186, 187, 188, 188, 189, 189, 189, 189, 189, 189, + 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, + 189, 189, 189, 189, 189, 190, 190 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1005,26 +1014,27 @@ static const yytype_uint8 yyr2[] = { 0, 2, 1, 2, 1, 1, 1, 1, 1, 0, 5, 0, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 3, 3, 3, 3, 4, 4, 6, 2, - 2, 2, 2, 2, 2, 2, 0, 6, 0, 2, - 1, 1, 1, 1, 1, 1, 2, 2, 0, 2, - 1, 1, 0, 0, 4, 0, 7, 0, 2, 1, - 1, 1, 1, 1, 1, 1, 2, 2, 4, 2, - 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 3, 3, 2, - 2, 3, 3, 0, 2, 1, 1, 1, 1, 1, + 1, 1, 0, 6, 0, 2, 2, 2, 2, 4, + 2, 2, 1, 3, 3, 3, 3, 4, 4, 6, + 2, 2, 2, 2, 2, 2, 2, 0, 6, 0, + 2, 1, 1, 1, 1, 1, 1, 2, 2, 0, + 2, 1, 1, 0, 0, 4, 0, 7, 0, 2, + 1, 1, 1, 1, 1, 1, 1, 2, 2, 4, + 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, + 2, 2, 3, 3, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 2, 3, 2, 3, 2, 3, 2, 1, - 1, 1, 4, 4, 1, 3, 0, 2, 1, 1, - 3, 3, 0, 9, 0, 9, 7, 7, 5, 6, - 5, 6, 1, 1, 3, 3, 0, 2, 2, 4, - 0, 2, 3, 6, 6, 6, 6, 3, 2, 2, - 3, 3, 2, 1, 3, 2, 4, 5, 7, 3, - 3, 5, 5, 2, 2, 3, 3, 3, 4, 3, - 3, 3, 0, 5, 5, 3, 4, 5, 4, 5, - 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 3, 2, 3, 2, 3, 2, 3, 2, + 1, 1, 1, 4, 4, 1, 3, 0, 2, 1, + 1, 3, 3, 0, 9, 0, 9, 7, 7, 5, + 6, 5, 6, 1, 1, 3, 3, 0, 2, 2, + 4, 0, 2, 3, 6, 6, 6, 6, 3, 2, + 2, 3, 3, 2, 1, 3, 2, 4, 5, 7, + 3, 3, 5, 5, 2, 2, 3, 3, 3, 4, + 3, 3, 3, 0, 5, 5, 3, 4, 5, 4, + 5, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -1032,290 +1042,290 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint8 yydefact[] = { - 0, 0, 0, 0, 0, 191, 0, 2, 4, 5, - 6, 7, 8, 9, 93, 89, 90, 214, 215, 0, - 190, 1, 3, 11, 0, 91, 192, 0, 0, 0, - 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 92, 0, 0, 0, 94, 95, - 96, 97, 98, 99, 100, 101, 102, 110, 103, 104, - 105, 106, 107, 108, 109, 21, 0, 0, 0, 0, - 36, 0, 19, 20, 10, 12, 13, 14, 17, 15, - 16, 18, 0, 0, 112, 114, 116, 118, 0, 51, - 50, 0, 0, 159, 158, 163, 162, 165, 0, 0, - 173, 174, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 30, 29, 33, 34, - 35, 38, 31, 32, 0, 0, 111, 113, 115, 117, - 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, - 213, 143, 0, 146, 142, 0, 0, 0, 157, 161, - 160, 0, 0, 0, 169, 170, 175, 176, 177, 0, - 50, 180, 179, 185, 181, 0, 0, 0, 23, 24, - 25, 22, 0, 0, 0, 0, 0, 0, 150, 0, - 0, 0, 0, 0, 150, 120, 121, 0, 0, 0, - 164, 0, 166, 178, 0, 0, 186, 188, 26, 27, - 43, 44, 42, 0, 0, 48, 40, 41, 45, 39, - 171, 172, 150, 0, 0, 138, 0, 0, 0, 124, - 150, 0, 147, 145, 144, 140, 150, 150, 150, 150, - 182, 184, 167, 187, 189, 0, 46, 47, 0, 139, - 151, 0, 134, 132, 150, 150, 0, 0, 141, 148, - 0, 154, 153, 156, 155, 0, 0, 28, 0, 37, - 52, 49, 152, 126, 126, 137, 136, 0, 125, 0, - 93, 168, 52, 53, 0, 150, 150, 123, 122, 149, - 0, 55, 57, 129, 127, 128, 135, 133, 183, 0, - 54, 0, 57, 0, 0, 0, 59, 60, 61, 62, - 63, 64, 0, 0, 65, 70, 0, 71, 72, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 58, 131, - 130, 0, 80, 81, 82, 66, 69, 67, 0, 0, - 73, 77, 86, 78, 79, 84, 83, 85, 74, 75, - 76, 56, 0, 87, 88, 68 + 0, 0, 0, 0, 0, 202, 0, 2, 4, 5, + 6, 7, 8, 9, 104, 100, 101, 225, 226, 0, + 201, 1, 3, 11, 0, 102, 203, 0, 0, 0, + 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 103, 0, 0, 0, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 121, 114, 115, + 116, 117, 118, 119, 120, 32, 0, 0, 0, 0, + 47, 0, 0, 20, 21, 10, 12, 19, 13, 14, + 17, 15, 16, 18, 0, 0, 123, 125, 127, 129, + 0, 62, 61, 0, 0, 170, 169, 174, 173, 176, + 0, 0, 184, 185, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 41, 40, + 44, 45, 46, 49, 22, 42, 43, 0, 0, 122, + 124, 126, 128, 204, 205, 206, 207, 208, 209, 210, + 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 154, 0, 157, 153, 0, 0, + 0, 168, 172, 171, 0, 0, 0, 180, 181, 186, + 187, 188, 0, 61, 191, 190, 196, 192, 0, 0, + 0, 34, 35, 36, 33, 0, 0, 0, 0, 0, + 0, 0, 161, 0, 0, 0, 0, 0, 161, 131, + 132, 0, 0, 0, 175, 0, 177, 189, 0, 0, + 197, 199, 37, 38, 54, 55, 53, 0, 0, 59, + 51, 52, 56, 50, 24, 182, 183, 161, 0, 0, + 149, 0, 0, 0, 135, 161, 0, 158, 156, 155, + 151, 161, 161, 161, 161, 193, 195, 178, 198, 200, + 0, 57, 58, 0, 0, 150, 162, 0, 145, 143, + 161, 161, 0, 0, 152, 159, 0, 165, 164, 167, + 166, 0, 0, 39, 0, 48, 63, 60, 0, 0, + 0, 0, 23, 25, 163, 137, 137, 148, 147, 0, + 136, 0, 104, 179, 63, 64, 26, 30, 31, 0, + 27, 28, 0, 161, 161, 134, 133, 160, 0, 66, + 68, 0, 140, 138, 139, 146, 144, 194, 0, 65, + 29, 0, 68, 0, 0, 0, 70, 71, 72, 73, + 74, 75, 0, 0, 76, 81, 0, 82, 83, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 69, 142, + 141, 0, 91, 92, 93, 77, 80, 78, 0, 0, + 84, 88, 97, 89, 90, 95, 94, 96, 85, 86, + 87, 67, 0, 98, 99, 79 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 6, 7, 8, 23, 27, 75, 76, 77, 78, - 79, 80, 81, 121, 184, 218, 219, 248, 91, 283, - 271, 292, 299, 300, 328, 9, 10, 11, 15, 24, - 48, 49, 197, 152, 230, 285, 294, 50, 274, 273, - 153, 194, 232, 225, 51, 52, 53, 54, 96, 55, - 56, 57, 58, 59, 60, 61, 241, 265, 62, 63, - 64, 12, 20, 154, 19 + -1, 6, 7, 8, 23, 27, 76, 77, 188, 254, + 283, 78, 79, 80, 81, 82, 83, 123, 187, 222, + 223, 253, 93, 295, 277, 310, 318, 319, 348, 9, + 10, 11, 15, 24, 48, 49, 201, 155, 235, 303, + 313, 50, 286, 285, 156, 198, 237, 230, 51, 52, + 53, 54, 98, 55, 56, 57, 58, 59, 60, 61, + 246, 271, 62, 63, 64, 12, 20, 157, 19 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -179 +#define YYPACT_NINF -180 static const yytype_int16 yypact[] = { - 47, -64, -54, -54, -51, -179, 85, -179, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -54, - -51, -179, -179, -179, 455, -179, -179, 8, -33, -21, - -18, -5, 0, -16, -1, 10, -69, 19, 39, -3, - -52, -22, 25, 26, -179, 20, 129, 37, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -179, -179, -179, -179, -4, -29, 38, 45, 55, - -179, -27, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -179, 66, 67, 36, 65, 71, 77, 153, -179, - 2, 92, 80, -10, -179, -179, 118, 14, 106, 107, - -179, -179, 108, 110, 133, 134, 5, 5, 135, 5, - -86, 136, 138, 139, 141, 16, -179, -179, -179, -179, - -179, -179, -179, -179, 142, 143, -179, -179, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -53, 180, -70, -179, 6, 6, 565, -179, -179, - -179, 144, 245, 147, -179, -179, -179, -179, -179, 145, - -179, -179, -179, -179, -179, -12, 146, 148, -179, -179, - -179, -179, 240, 173, 3, 152, 175, 176, -59, 6, - 6, 177, 194, 181, -59, -179, -179, 223, 251, 189, - -179, 203, -179, -179, 179, 204, -179, -179, 284, -179, - -179, -179, -179, 207, 295, -179, -179, -179, -179, -179, - -179, -179, -59, 209, 252, -179, 293, 321, 228, -179, - -15, 212, 225, -179, -179, -179, -59, -59, -59, -59, - -179, -179, 229, -179, -179, 213, -179, -179, 1, -179, - -179, 216, 257, 258, -59, -59, 6, 177, -179, -179, - 234, -179, -179, -179, -179, 235, 220, -179, 221, -179, - -179, -179, -179, 231, 231, -179, -179, 363, -179, 246, - -179, -179, -179, -179, 391, -59, -59, -179, -179, -179, - 500, -179, -179, -179, 259, -179, -179, -179, -179, 262, - 154, 433, -179, 248, 249, 250, -179, -179, -179, -179, - -179, -179, 81, -14, -179, -179, 273, -179, -179, 275, - 276, 277, 279, 280, 94, 281, 15, 278, -179, -179, - -179, 125, -179, -179, -179, -179, -179, -179, 368, 17, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -179, 371, -179, -179, -179 + 65, -89, -87, -87, -49, -180, 130, -180, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -87, + -49, -180, -180, -180, 430, -180, -180, 8, -32, -96, + -86, -82, -68, -18, -1, -52, -71, -4, 31, 21, + 43, 50, 10, 16, -180, 56, 116, 62, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, -180, -3, 9, 72, 85, 91, + -180, 99, 17, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, 127, 154, 153, 155, 156, 157, + 34, -180, 54, 168, 159, 38, -180, -180, 175, 71, + 163, 164, -180, -180, 165, 166, 167, 169, 5, 5, + 192, 5, 35, 193, 195, 194, 196, 29, -180, -180, + -180, -180, -180, -180, -180, -180, -180, 197, 220, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, -53, 209, -72, -180, 27, 27, + 540, -180, -180, -180, 221, 322, 224, -180, -180, -180, + -180, -180, 222, -180, -180, -180, -180, -180, 86, 223, + 225, -180, -180, -180, -180, 317, 250, 1, 244, 230, + 231, 232, 30, 27, 27, 233, 251, 170, 30, -180, + -180, 198, 240, 246, -180, 234, -180, -180, 236, 237, + -180, -180, 319, -180, -180, -180, -180, 263, 351, -180, + -180, -180, -180, -180, -180, -180, -180, 30, 265, 307, + -180, 268, 310, 285, -180, 55, 291, 280, -180, -180, + -180, 30, 30, 30, 30, -180, -180, 308, -180, -180, + 290, -180, -180, 3, 33, -180, -180, 294, 335, 336, + 30, 30, 27, 233, -180, -180, 312, -180, -180, -180, + -180, 313, 298, -180, 299, -180, -180, -180, 300, 391, + -10, 97, -180, -180, -180, 311, 311, -180, -180, 338, + -180, 304, -180, -180, -180, -180, -180, -180, -180, 396, + -180, -180, 380, 30, 30, -180, -180, -180, 475, -180, + -180, 395, -180, 320, -180, -180, -180, -180, 321, 171, + -180, 408, -180, 309, 332, 333, -180, -180, -180, -180, + -180, -180, 212, 0, -180, -180, 334, -180, -180, 337, + 362, 361, 363, 364, 238, 365, 124, 366, -180, -180, + -180, 142, -180, -180, -180, -180, -180, -180, 400, 152, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, 425, -180, -180, -180 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -179, -179, 395, -179, -179, -179, -179, -179, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -105, 120, - -179, -179, -179, 101, -179, -179, -179, -179, 13, 124, - -179, -179, -145, -178, -179, 131, -179, -179, -179, -179, - -179, -179, -179, -156, -179, -179, -179, -179, -179, -179, - -179, -179, -179, -179, -179, -179, -179, -179, -179, -179, - -179, -179, -179, -157, 386 + -180, -180, 449, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -107, 191, -180, -180, -180, 172, -180, -180, + -180, -180, 12, 199, -180, -180, -149, -155, -180, 200, + -180, -180, -180, -180, -180, -180, -180, -179, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, + -180, -180, -180, -180, -180, -180, -180, -160, 467 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -124 +#define YYTABLE_NINF -135 static const yytype_int16 yytable[] = { - 199, 171, 172, 89, 174, 89, 268, 210, 211, 89, - 111, 198, 65, 229, 337, 93, 16, 223, 195, 212, - 224, 66, 213, 67, 68, 69, 70, 187, 175, 176, - 177, 192, 25, 112, 113, 193, 234, 114, 235, 13, - 237, 239, 182, 188, 226, 227, 189, 190, 183, 14, - 1, 94, 71, 72, 73, 2, 3, 4, 5, 88, - 214, 223, 257, 338, 224, 100, 249, 17, 18, 253, - 255, 101, 102, 103, 258, 104, 105, 82, 83, 278, - 261, 262, 263, 264, 196, 21, 98, 99, 1, 116, - 117, 122, 123, 2, 3, 4, 5, 84, 275, 276, - 85, 155, 156, 204, 205, 269, 215, 159, 160, 335, - 336, 277, 74, 86, 115, 162, 163, 90, 87, 170, - 288, 216, 217, 170, 345, 346, 97, 295, 92, 296, - 297, 106, 107, 348, 349, 353, 354, 95, 108, 109, - 303, 304, 305, 270, 330, 306, 307, 308, 309, 310, - 311, 312, 313, 314, 315, 110, 118, 126, 316, 317, - 318, 319, 320, 119, 321, 322, 323, 324, 325, 303, - 304, 305, 326, 120, 306, 307, 308, 309, 310, 311, - 312, 313, 314, 315, 124, 125, 127, 316, 317, 318, - 319, 320, 128, 321, 322, 323, 324, 325, 129, 157, - 158, 326, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 161, 164, 165, 166, 327, 167, 351, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 203, 174, 175, 91, 177, 214, 215, 91, 274, 91, + 202, 113, 65, 95, 13, 16, 14, 216, 298, 240, + 217, 66, 86, 67, 68, 69, 70, 191, 357, 196, + 71, 25, 87, 197, 114, 115, 88, 239, 116, 199, + 234, 242, 244, 192, 231, 232, 193, 194, 255, 96, + 89, 278, 72, 73, 74, 185, 264, 90, 218, 279, + 280, 186, 267, 268, 269, 270, 94, 299, 1, 17, + 18, 259, 261, 2, 3, 4, 5, 358, 84, 85, + 281, 287, 288, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 168, 169, 173, 178, 179, 327, 191, 180, 181, - 185, 186, 200, 201, 202, 206, 203, 207, 208, 209, - 220, 151, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 221, 222, 228, 231, 240, 243, 233, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 242, 245, 244, 246, 247, 250, 256, 251, 259, - 260, 266, 267, 272, -123, -122, 279, 281, 280, 282, - 284, 236, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 289, 301, 302, 332, 333, 334, 238, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 339, 340, 341, 352, 350, 342, 343, 344, 355, - 347, 22, 291, 331, 290, 286, 26, 0, 0, 0, - 0, 252, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 0, 0, 0, 0, 0, 0, 254, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 287, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, - 148, 149, 150, 28, 0, 0, 0, 0, 0, 293, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 29, 30, 31, 32, 33, - 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, - 37, 38, 0, 39, 0, 0, 40, 41, 28, 42, - 43, 329, 0, 0, 0, 0, 0, 0, 0, 44, - 0, 0, 0, 0, 0, 45, 0, 46, 47, 0, + 150, 151, 152, 153, 219, 200, 228, 275, 290, 229, + 100, 101, 75, 289, 97, 117, 108, 92, 99, 220, + 221, 173, 109, 173, 315, 316, 111, 118, 119, 306, + 21, 228, 263, 1, 229, 125, 126, 282, 2, 3, + 4, 5, 314, 103, 104, 105, 276, 106, 107, 178, + 179, 180, 154, 158, 159, 162, 163, 323, 324, 325, + 102, 350, 326, 327, 328, 329, 330, 331, 332, 333, + 334, 335, 165, 166, 110, 336, 337, 338, 339, 340, + 112, 341, 342, 343, 344, 345, 323, 324, 325, 346, + 120, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 335, 208, 209, 121, 336, 337, 338, 339, 340, 122, + 341, 342, 343, 344, 345, 300, 301, 124, 346, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 355, 356, 368, 369, 347, 127, 371, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 365, 366, + 373, 374, 128, 347, 129, 160, 130, 131, 132, 161, + 164, 167, 168, 169, 170, 171, 195, 172, 238, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 176, 181, 182, 183, 184, 189, 241, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 190, 204, + 205, 206, 210, 207, 211, 212, 213, 224, 225, 226, + 227, 233, 247, 236, 245, 248, 249, 250, 243, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 251, 252, 256, 257, 262, 266, 258, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 265, 273, + 272, 284, -134, -133, 291, 293, 292, 294, 296, 297, + 302, 307, 311, 320, 322, 321, 372, 352, 260, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 353, 354, 359, 375, 360, 22, 305, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, + 146, 147, 148, 149, 150, 151, 152, 153, 28, 361, + 362, 363, 364, 370, 367, 309, 304, 26, 0, 0, + 0, 308, 0, 0, 351, 0, 0, 0, 312, 0, 29, 30, 31, 32, 33, 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, 37, 38, 0, 39, 0, - 0, 40, 41, 0, 42, 43, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, - 45, 0, 46, 47, 130, 131, 132, 133, 134, 135, - 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150 + 0, 40, 41, 28, 42, 43, 349, 0, 0, 0, + 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, + 45, 0, 46, 47, 0, 29, 30, 31, 32, 33, + 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, + 37, 38, 0, 39, 0, 0, 40, 41, 0, 42, + 43, 0, 0, 0, 0, 0, 0, 0, 0, 317, + 0, 0, 0, 0, 0, 45, 0, 46, 47, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153 }; static const yytype_int16 yycheck[] = { - 157, 106, 107, 4, 109, 4, 5, 4, 5, 4, - 14, 156, 4, 191, 28, 84, 3, 76, 12, 16, - 79, 13, 19, 15, 16, 17, 18, 80, 114, 115, - 116, 101, 19, 37, 38, 105, 193, 41, 194, 103, - 197, 198, 26, 96, 189, 190, 99, 100, 32, 103, - 3, 120, 44, 45, 46, 8, 9, 10, 11, 75, - 57, 76, 77, 77, 79, 117, 222, 118, 119, 226, - 227, 93, 94, 95, 230, 97, 98, 110, 111, 257, - 236, 237, 238, 239, 78, 0, 89, 90, 3, 118, - 119, 118, 119, 8, 9, 10, 11, 118, 254, 255, - 118, 99, 100, 115, 116, 104, 103, 117, 118, 28, - 29, 256, 104, 118, 118, 101, 102, 118, 118, 118, - 277, 118, 119, 118, 30, 31, 87, 284, 118, 285, - 286, 106, 106, 118, 119, 118, 119, 118, 118, 10, - 15, 16, 17, 248, 301, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 118, 118, 121, 33, 34, - 35, 36, 37, 118, 39, 40, 41, 42, 43, 15, - 16, 17, 47, 118, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 118, 118, 121, 33, 34, 35, - 36, 37, 121, 39, 40, 41, 42, 43, 121, 107, - 120, 47, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 105, 118, 118, 118, 102, 118, 104, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 118, 118, 118, 118, 117, 102, 77, 119, 118, - 118, 118, 118, 18, 117, 119, 121, 119, 28, 96, - 118, 118, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 118, 118, 118, 102, 108, 119, 118, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 118, 38, 119, 117, 30, 117, 99, 76, 117, - 105, 102, 119, 117, 77, 77, 102, 117, 103, 118, - 109, 118, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 117, 105, 103, 118, 118, 118, 118, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 118, 117, 117, 26, 117, 119, 118, 118, 28, - 119, 6, 282, 302, 280, 274, 20, -1, -1, -1, - -1, 118, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, -1, -1, -1, -1, -1, -1, 118, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 118, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 48, -1, -1, -1, -1, -1, 118, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 70, 71, 72, 73, 74, - -1, -1, -1, -1, -1, -1, 81, 82, 83, -1, - 85, 86, -1, 88, -1, -1, 91, 92, 48, 94, - 95, 118, -1, -1, -1, -1, -1, -1, -1, 104, - -1, -1, -1, -1, -1, 110, -1, 112, 113, -1, + 160, 108, 109, 4, 111, 4, 5, 4, 5, 4, + 159, 14, 4, 84, 103, 3, 103, 16, 28, 198, + 19, 13, 118, 15, 16, 17, 18, 80, 28, 101, + 22, 19, 118, 105, 37, 38, 118, 197, 41, 12, + 195, 201, 202, 96, 193, 194, 99, 100, 227, 120, + 118, 18, 44, 45, 46, 26, 235, 75, 57, 26, + 27, 32, 241, 242, 243, 244, 118, 77, 3, 118, + 119, 231, 232, 8, 9, 10, 11, 77, 110, 111, + 47, 260, 261, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 103, 78, 76, 104, 263, 79, + 89, 90, 104, 262, 118, 118, 106, 118, 87, 118, + 119, 118, 106, 118, 303, 304, 10, 118, 119, 289, + 0, 76, 77, 3, 79, 118, 119, 104, 8, 9, + 10, 11, 302, 93, 94, 95, 253, 97, 98, 114, + 115, 116, 118, 99, 100, 117, 118, 15, 16, 17, + 117, 321, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 101, 102, 118, 33, 34, 35, 36, 37, + 118, 39, 40, 41, 42, 43, 15, 16, 17, 47, + 118, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 115, 116, 118, 33, 34, 35, 36, 37, 118, + 39, 40, 41, 42, 43, 118, 119, 118, 47, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 28, 29, 118, 119, 102, 118, 104, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 30, 31, + 118, 119, 118, 102, 121, 107, 121, 121, 121, 120, + 105, 118, 118, 118, 118, 118, 77, 118, 118, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 118, 118, 117, 119, 118, 118, 118, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 118, 118, + 18, 117, 119, 121, 119, 28, 96, 103, 118, 118, + 118, 118, 118, 102, 108, 119, 119, 38, 118, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 117, 30, 117, 76, 99, 105, 118, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 117, 119, + 102, 117, 77, 77, 102, 117, 103, 118, 118, 28, + 109, 117, 26, 28, 103, 105, 26, 118, 118, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 118, 118, 118, 28, 117, 6, 118, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 48, 117, + 119, 118, 118, 117, 119, 294, 286, 20, -1, -1, + -1, 292, -1, -1, 322, -1, -1, -1, 118, -1, 70, 71, 72, 73, 74, -1, -1, -1, -1, -1, -1, 81, 82, 83, -1, 85, 86, -1, 88, -1, - -1, 91, 92, -1, 94, 95, -1, -1, -1, -1, + -1, 91, 92, 48, 94, 95, 118, -1, -1, -1, -1, -1, -1, -1, 104, -1, -1, -1, -1, -1, - 110, -1, 112, 113, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, - 65, 66, 67, 68, 69 + 110, -1, 112, 113, -1, 70, 71, 72, 73, 74, + -1, -1, -1, -1, -1, -1, 81, 82, 83, -1, + 85, 86, -1, 88, -1, -1, 91, 92, -1, 94, + 95, -1, -1, -1, -1, -1, -1, -1, -1, 104, + -1, -1, -1, -1, -1, 110, -1, 112, 113, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 3, 8, 9, 10, 11, 123, 124, 125, 147, - 148, 149, 183, 103, 103, 150, 150, 118, 119, 186, - 184, 0, 124, 126, 151, 150, 186, 127, 48, 70, + 0, 3, 8, 9, 10, 11, 123, 124, 125, 151, + 152, 153, 187, 103, 103, 154, 154, 118, 119, 190, + 188, 0, 124, 126, 155, 154, 190, 127, 48, 70, 71, 72, 73, 74, 81, 82, 83, 85, 86, 88, - 91, 92, 94, 95, 104, 110, 112, 113, 152, 153, - 159, 166, 167, 168, 169, 171, 172, 173, 174, 175, - 176, 177, 180, 181, 182, 4, 13, 15, 16, 17, - 18, 44, 45, 46, 104, 128, 129, 130, 131, 132, - 133, 134, 110, 111, 118, 118, 118, 118, 75, 4, - 118, 140, 118, 84, 120, 118, 170, 87, 89, 90, - 117, 93, 94, 95, 97, 98, 106, 106, 118, 10, - 118, 14, 37, 38, 41, 118, 118, 119, 118, 118, - 118, 135, 118, 119, 118, 118, 121, 121, 121, 121, - 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 118, 155, 162, 185, 99, 100, 107, 120, 117, - 118, 105, 101, 102, 118, 118, 118, 118, 118, 118, - 118, 140, 140, 118, 140, 114, 115, 116, 118, 117, - 119, 118, 26, 32, 136, 118, 118, 80, 96, 99, - 100, 77, 101, 105, 163, 12, 78, 154, 154, 185, - 118, 18, 117, 121, 115, 116, 119, 119, 28, 96, - 4, 5, 16, 19, 57, 103, 118, 119, 137, 138, - 118, 118, 118, 76, 79, 165, 154, 154, 118, 155, - 156, 102, 164, 118, 185, 165, 118, 185, 118, 185, - 108, 178, 118, 119, 119, 38, 117, 30, 139, 165, - 117, 76, 118, 185, 118, 185, 99, 77, 165, 117, - 105, 165, 165, 165, 165, 179, 102, 119, 5, 104, - 140, 142, 117, 161, 160, 165, 165, 154, 155, 102, - 103, 117, 118, 141, 109, 157, 157, 118, 185, 117, - 151, 141, 143, 118, 158, 185, 165, 165, 104, 144, - 145, 105, 103, 15, 16, 17, 20, 21, 22, 23, + 91, 92, 94, 95, 104, 110, 112, 113, 156, 157, + 163, 170, 171, 172, 173, 175, 176, 177, 178, 179, + 180, 181, 184, 185, 186, 4, 13, 15, 16, 17, + 18, 22, 44, 45, 46, 104, 128, 129, 133, 134, + 135, 136, 137, 138, 110, 111, 118, 118, 118, 118, + 75, 4, 118, 144, 118, 84, 120, 118, 174, 87, + 89, 90, 117, 93, 94, 95, 97, 98, 106, 106, + 118, 10, 118, 14, 37, 38, 41, 118, 118, 119, + 118, 118, 118, 139, 118, 118, 119, 118, 118, 121, + 121, 121, 121, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 118, 159, 166, 189, 99, 100, + 107, 120, 117, 118, 105, 101, 102, 118, 118, 118, + 118, 118, 118, 118, 144, 144, 118, 144, 114, 115, + 116, 118, 117, 119, 118, 26, 32, 140, 130, 118, + 118, 80, 96, 99, 100, 77, 101, 105, 167, 12, + 78, 158, 158, 189, 118, 18, 117, 121, 115, 116, + 119, 119, 28, 96, 4, 5, 16, 19, 57, 103, + 118, 119, 141, 142, 103, 118, 118, 118, 76, 79, + 169, 158, 158, 118, 159, 160, 102, 168, 118, 189, + 169, 118, 189, 118, 189, 108, 182, 118, 119, 119, + 38, 117, 30, 143, 131, 169, 117, 76, 118, 189, + 118, 189, 99, 77, 169, 117, 105, 169, 169, 169, + 169, 183, 102, 119, 5, 104, 144, 146, 18, 26, + 27, 47, 104, 132, 117, 165, 164, 169, 169, 158, + 159, 102, 103, 117, 118, 145, 118, 28, 28, 77, + 118, 119, 109, 161, 161, 118, 189, 117, 155, 145, + 147, 26, 118, 162, 189, 169, 169, 104, 148, 149, + 28, 105, 103, 15, 16, 17, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 33, 34, 35, 36, - 37, 39, 40, 41, 42, 43, 47, 102, 146, 118, - 185, 145, 118, 118, 118, 28, 29, 28, 77, 118, + 37, 39, 40, 41, 42, 43, 47, 102, 150, 118, + 189, 149, 118, 118, 118, 28, 29, 28, 77, 118, 117, 117, 119, 118, 118, 30, 31, 119, 118, 119, 117, 104, 26, 118, 119, 28 }; @@ -2132,7 +2142,7 @@ yyparse () switch (yyn) { case 9: -#line 238 "test_spec_parse.y" +#line 239 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, "self-signed", sizeof(current_spec->cluster.ssl)); @@ -2141,25 +2151,100 @@ yyparse () ;} break; - case 19: -#line 259 "test_spec_parse.y" + case 20: +#line 261 "test_spec_parse.y" { current_spec->cluster.bindSource = true; ;} break; - case 20: -#line 260 "test_spec_parse.y" + case 21: +#line 262 "test_spec_parse.y" { current_spec->cluster.legacyStartup = true; ;} break; - case 21: -#line 274 "test_spec_parse.y" + case 22: +#line 285 "test_spec_parse.y" + { + TestCluster *cl = ¤t_spec->cluster; + + if (cl->archiverCount >= PGAF_MAX_ARCHIVERS) + { + fprintf(stderr, "pgaftest: too many archivers (max %d)\n", + PGAF_MAX_ARCHIVERS); + exit(1); + } + + current_archiver = &cl->archivers[cl->archiverCount++]; + strlcpy(current_archiver->name, (yyvsp[(2) - (2)].str), sizeof(current_archiver->name)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 26: +#line 309 "test_spec_parse.y" + { + if (current_archiver->formationCount >= PGAF_MAX_ARCHIVER_FORMATIONS) + { + fprintf(stderr, + "pgaftest: too many --formation entries for archiver " + "\"%s\" (max %d)\n", + current_archiver->name, PGAF_MAX_ARCHIVER_FORMATIONS); + exit(1); + } + strlcpy(current_archiver->formations[current_archiver->formationCount++], + (yyvsp[(2) - (2)].str), sizeof(current_archiver->formations[0])); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 27: +#line 323 "test_spec_parse.y" + { + strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 28: +#line 328 "test_spec_parse.y" + { + strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); + free((yyvsp[(2) - (2)].str)); + ;} + break; + + case 29: +#line 333 "test_spec_parse.y" + { + /* bare "create and launch deferred" = both gates, matching + * node_opt's own identical form */ + current_archiver->createDeferred = true; + current_archiver->launchDeferred = true; + ;} + break; + + case 30: +#line 340 "test_spec_parse.y" + { + current_archiver->launchDeferred = true; + ;} + break; + + case 31: +#line 344 "test_spec_parse.y" + { + current_archiver->createDeferred = true; + ;} + break; + + case 32: +#line 360 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; ;} break; - case 22: -#line 278 "test_spec_parse.y" + case 33: +#line 364 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorDebianCluster, (yyvsp[(3) - (3)].str), @@ -2168,8 +2253,8 @@ yyparse () ;} break; - case 23: -#line 285 "test_spec_parse.y" + case 34: +#line 371 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorImageTarget, (yyvsp[(3) - (3)].str), @@ -2178,8 +2263,8 @@ yyparse () ;} break; - case 24: -#line 292 "test_spec_parse.y" + case 35: +#line 378 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; /* monitor port not stored in TestCluster yet; ignore */ @@ -2187,8 +2272,8 @@ yyparse () ;} break; - case 25: -#line 298 "test_spec_parse.y" + case 36: +#line 384 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorPassword, (yyvsp[(3) - (3)].str), @@ -2197,8 +2282,8 @@ yyparse () ;} break; - case 26: -#line 305 "test_spec_parse.y" + case 37: +#line 391 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2207,8 +2292,8 @@ yyparse () ;} break; - case 27: -#line 312 "test_spec_parse.y" + case 38: +#line 398 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2217,8 +2302,8 @@ yyparse () ;} break; - case 28: -#line 319 "test_spec_parse.y" + case 39: +#line 405 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (6)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2229,8 +2314,8 @@ yyparse () ;} break; - case 29: -#line 332 "test_spec_parse.y" + case 40: +#line 418 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2238,8 +2323,8 @@ yyparse () ;} break; - case 30: -#line 338 "test_spec_parse.y" + case 41: +#line 424 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2247,8 +2332,8 @@ yyparse () ;} break; - case 31: -#line 348 "test_spec_parse.y" + case 42: +#line 434 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2256,8 +2341,8 @@ yyparse () ;} break; - case 32: -#line 354 "test_spec_parse.y" + case 43: +#line 440 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2265,8 +2350,8 @@ yyparse () ;} break; - case 33: -#line 364 "test_spec_parse.y" + case 44: +#line 450 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.ssl)); @@ -2274,8 +2359,8 @@ yyparse () ;} break; - case 34: -#line 374 "test_spec_parse.y" + case 45: +#line 460 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2283,8 +2368,8 @@ yyparse () ;} break; - case 35: -#line 380 "test_spec_parse.y" + case 46: +#line 466 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2292,8 +2377,8 @@ yyparse () ;} break; - case 36: -#line 390 "test_spec_parse.y" + case 47: +#line 476 "test_spec_parse.y" { TestCluster *cl = ¤t_spec->cluster; if (cl->formationCount >= PGAF_MAX_FORMATIONS) @@ -2309,65 +2394,65 @@ yyparse () ;} break; - case 40: -#line 417 "test_spec_parse.y" + case 51: +#line 503 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 41: -#line 418 "test_spec_parse.y" + case 52: +#line 504 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 42: -#line 419 "test_spec_parse.y" + case 53: +#line 505 "test_spec_parse.y" { (yyval.str) = strdup("auth"); ;} break; - case 43: -#line 420 "test_spec_parse.y" + case 54: +#line 506 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; - case 44: -#line 421 "test_spec_parse.y" + case 55: +#line 507 "test_spec_parse.y" { (yyval.str) = strdup("node"); ;} break; - case 45: -#line 426 "test_spec_parse.y" + case 56: +#line 512 "test_spec_parse.y" { strlcpy(current_formation->name, (yyvsp[(1) - (1)].str), sizeof(current_formation->name)); free((yyvsp[(1) - (1)].str)); ;} break; - case 46: -#line 431 "test_spec_parse.y" + case 57: +#line 517 "test_spec_parse.y" { current_formation->numSync = (yyvsp[(2) - (2)].ival); ;} break; - case 47: -#line 435 "test_spec_parse.y" + case 58: +#line 521 "test_spec_parse.y" { current_formation->disableSecondary = true; ;} break; - case 50: -#line 461 "test_spec_parse.y" + case 61: +#line 547 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 51: -#line 462 "test_spec_parse.y" + case 62: +#line 548 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; - case 52: -#line 471 "test_spec_parse.y" + case 63: +#line 557 "test_spec_parse.y" { if (current_formation->nodeCount >= PGAF_MAX_NODES) { @@ -2382,68 +2467,68 @@ yyparse () ;} break; - case 53: -#line 488 "test_spec_parse.y" + case 64: +#line 574 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(1) - (2)].str), sizeof(current_node->name)); free((yyvsp[(1) - (2)].str)); ;} break; - case 55: -#line 495 "test_spec_parse.y" + case 66: +#line 581 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(2) - (3)].str), sizeof(current_node->name)); free((yyvsp[(2) - (3)].str)); ;} break; - case 59: -#line 509 "test_spec_parse.y" + case 70: +#line 595 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_COORDINATOR; current_spec->cluster.withCitus = true; ;} break; - case 60: -#line 514 "test_spec_parse.y" + case 71: +#line 600 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_WORKER; current_spec->cluster.withCitus = true; ;} break; - case 61: -#line 519 "test_spec_parse.y" + case 72: +#line 605 "test_spec_parse.y" { current_node->kind = NODE_KIND_ARCHIVER; ;} break; - case 62: -#line 523 "test_spec_parse.y" + case 73: +#line 609 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; - case 63: -#line 527 "test_spec_parse.y" + case 74: +#line 613 "test_spec_parse.y" { current_node->noMonitor = true; ;} break; - case 64: -#line 531 "test_spec_parse.y" + case 75: +#line 617 "test_spec_parse.y" { current_node->suspended = true; ;} break; - case 65: -#line 535 "test_spec_parse.y" + case 76: +#line 621 "test_spec_parse.y" { /* bare "deferred" = create and launch deferred (both gates) */ current_node->createDeferred = true; @@ -2451,96 +2536,96 @@ yyparse () ;} break; - case 66: -#line 541 "test_spec_parse.y" + case 77: +#line 627 "test_spec_parse.y" { /* "launch deferred" alone = run-deferred only, create immediate */ current_node->launchDeferred = true; ;} break; - case 67: -#line 546 "test_spec_parse.y" + case 78: +#line 632 "test_spec_parse.y" { current_node->createDeferred = true; ;} break; - case 68: -#line 550 "test_spec_parse.y" + case 79: +#line 636 "test_spec_parse.y" { current_node->createDeferred = true; current_node->launchDeferred = true; ;} break; - case 69: -#line 555 "test_spec_parse.y" + case 80: +#line 641 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; - case 70: -#line 559 "test_spec_parse.y" + case 81: +#line 645 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; - case 71: -#line 563 "test_spec_parse.y" + case 82: +#line 649 "test_spec_parse.y" { current_node->listen = true; ;} break; - case 72: -#line 567 "test_spec_parse.y" + case 83: +#line 653 "test_spec_parse.y" { current_node->citusSecondary = true; ;} break; - case 73: -#line 571 "test_spec_parse.y" + case 84: +#line 657 "test_spec_parse.y" { current_node->candidatePriority = (yyvsp[(2) - (2)].ival); ;} break; - case 74: -#line 575 "test_spec_parse.y" + case 85: +#line 661 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); ;} break; - case 75: -#line 580 "test_spec_parse.y" + case 86: +#line 666 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); ;} break; - case 76: -#line 585 "test_spec_parse.y" + case 87: +#line 671 "test_spec_parse.y" { current_node->group = (yyvsp[(2) - (2)].ival); ;} break; - case 77: -#line 589 "test_spec_parse.y" + case 88: +#line 675 "test_spec_parse.y" { current_node->pgPort = (yyvsp[(2) - (2)].ival); ;} break; - case 78: -#line 593 "test_spec_parse.y" + case 89: +#line 679 "test_spec_parse.y" { strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), sizeof(current_node->citusClusterName)); @@ -2548,8 +2633,8 @@ yyparse () ;} break; - case 79: -#line 599 "test_spec_parse.y" + case 90: +#line 685 "test_spec_parse.y" { strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), sizeof(current_node->debianCluster)); @@ -2557,46 +2642,46 @@ yyparse () ;} break; - case 80: -#line 605 "test_spec_parse.y" + case 91: +#line 691 "test_spec_parse.y" { strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); free((yyvsp[(2) - (2)].str)); ;} break; - case 81: -#line 610 "test_spec_parse.y" + case 92: +#line 696 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); ;} break; - case 82: -#line 615 "test_spec_parse.y" + case 93: +#line 701 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); ;} break; - case 83: -#line 620 "test_spec_parse.y" + case 94: +#line 706 "test_spec_parse.y" { current_node->replicationQuorum = true; ;} break; - case 84: -#line 624 "test_spec_parse.y" + case 95: +#line 710 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; - case 85: -#line 628 "test_spec_parse.y" + case 96: +#line 714 "test_spec_parse.y" { strlcpy(current_node->replicationPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->replicationPassword)); @@ -2604,8 +2689,8 @@ yyparse () ;} break; - case 86: -#line 634 "test_spec_parse.y" + case 97: +#line 720 "test_spec_parse.y" { strlcpy(current_node->monitorPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->monitorPassword)); @@ -2613,8 +2698,8 @@ yyparse () ;} break; - case 87: -#line 640 "test_spec_parse.y" + case 98: +#line 726 "test_spec_parse.y" { /* volume — adds a named Docker volume */ int vi = current_node->volumeCount; @@ -2630,8 +2715,8 @@ yyparse () ;} break; - case 88: -#line 654 "test_spec_parse.y" + case 99: +#line 740 "test_spec_parse.y" { /* volume "/path/with spaces" */ int vi = current_node->volumeCount; @@ -2647,22 +2732,22 @@ yyparse () ;} break; - case 89: -#line 675 "test_spec_parse.y" + case 100: +#line 761 "test_spec_parse.y" { current_spec->setup = (yyvsp[(2) - (2)].step); ;} break; - case 90: -#line 682 "test_spec_parse.y" + case 101: +#line 768 "test_spec_parse.y" { current_spec->teardown = (yyvsp[(2) - (2)].step); ;} break; - case 91: -#line 693 "test_spec_parse.y" + case 102: +#line 779 "test_spec_parse.y" { TestStep *s = (yyvsp[(3) - (3)].step); strncpy(s->name, (yyvsp[(2) - (3)].str), sizeof(s->name) - 1); @@ -2671,8 +2756,8 @@ yyparse () ;} break; - case 92: -#line 711 "test_spec_parse.y" + case 103: +#line 797 "test_spec_parse.y" { /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ for (TestCmd *c = (yyvsp[(2) - (3)].step)->commands; c; c = c->next) @@ -2685,103 +2770,103 @@ yyparse () ;} break; - case 93: -#line 725 "test_spec_parse.y" + case 104: +#line 811 "test_spec_parse.y" { (yyval.step) = make_step(""); ;} break; - case 94: -#line 729 "test_spec_parse.y" + case 105: +#line 815 "test_spec_parse.y" { if ((yyvsp[(2) - (2)].cmd)) append_cmd((yyvsp[(1) - (2)].step), (yyvsp[(2) - (2)].cmd)); (yyval.step) = (yyvsp[(1) - (2)].step); ;} break; - case 95: -#line 736 "test_spec_parse.y" + case 106: +#line 822 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 96: -#line 737 "test_spec_parse.y" + case 107: +#line 823 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 97: -#line 738 "test_spec_parse.y" + case 108: +#line 824 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 98: -#line 739 "test_spec_parse.y" + case 109: +#line 825 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 99: -#line 740 "test_spec_parse.y" + case 110: +#line 826 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 100: -#line 741 "test_spec_parse.y" + case 111: +#line 827 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 101: -#line 742 "test_spec_parse.y" + case 112: +#line 828 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 102: -#line 743 "test_spec_parse.y" + case 113: +#line 829 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 103: -#line 744 "test_spec_parse.y" + case 114: +#line 830 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 104: -#line 745 "test_spec_parse.y" + case 115: +#line 831 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 105: -#line 746 "test_spec_parse.y" + case 116: +#line 832 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 106: -#line 747 "test_spec_parse.y" + case 117: +#line 833 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 107: -#line 748 "test_spec_parse.y" + case 118: +#line 834 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 108: -#line 749 "test_spec_parse.y" + case 119: +#line 835 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 109: -#line 750 "test_spec_parse.y" + case 120: +#line 836 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 110: -#line 751 "test_spec_parse.y" + case 121: +#line 837 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; - case 111: -#line 766 "test_spec_parse.y" + case 122: +#line 852 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2790,8 +2875,8 @@ yyparse () ;} break; - case 112: -#line 773 "test_spec_parse.y" + case 123: +#line 859 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2799,8 +2884,8 @@ yyparse () ;} break; - case 113: -#line 779 "test_spec_parse.y" + case 124: +#line 865 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2809,8 +2894,8 @@ yyparse () ;} break; - case 114: -#line 786 "test_spec_parse.y" + case 125: +#line 872 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2818,8 +2903,8 @@ yyparse () ;} break; - case 115: -#line 792 "test_spec_parse.y" + case 126: +#line 878 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2828,8 +2913,8 @@ yyparse () ;} break; - case 116: -#line 799 "test_spec_parse.y" + case 127: +#line 885 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2837,8 +2922,8 @@ yyparse () ;} break; - case 117: -#line 805 "test_spec_parse.y" + case 128: +#line 891 "test_spec_parse.y" { /* "pg_autoctl perform failover --formation auth" * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ @@ -2848,8 +2933,8 @@ yyparse () ;} break; - case 118: -#line 813 "test_spec_parse.y" + case 129: +#line 899 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); strlcpy((yyval.cmd)->args, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->args)); @@ -2857,15 +2942,15 @@ yyparse () ;} break; - case 119: -#line 819 "test_spec_parse.y" + case 130: +#line 905 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); ;} break; - case 122: -#line 857 "test_spec_parse.y" + case 133: +#line 943 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2882,8 +2967,8 @@ yyparse () ;} break; - case 123: -#line 872 "test_spec_parse.y" + case 134: +#line 958 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2900,8 +2985,8 @@ yyparse () ;} break; - case 128: -#line 912 "test_spec_parse.y" + case 139: +#line 998 "test_spec_parse.y" { /* current_pass_cmd set by the enclosing wait_cmd rule */ if (current_pass_cmd && @@ -2911,8 +2996,8 @@ yyparse () ;} break; - case 129: -#line 920 "test_spec_parse.y" + case 140: +#line 1006 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2922,8 +3007,8 @@ yyparse () ;} break; - case 130: -#line 928 "test_spec_parse.y" + case 141: +#line 1014 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2932,8 +3017,8 @@ yyparse () ;} break; - case 131: -#line 935 "test_spec_parse.y" + case 142: +#line 1021 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -2943,16 +3028,16 @@ yyparse () ;} break; - case 132: -#line 946 "test_spec_parse.y" + case 143: +#line 1032 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); ;} break; - case 133: -#line 951 "test_spec_parse.y" + case 144: +#line 1037 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2960,16 +3045,16 @@ yyparse () ;} break; - case 134: -#line 957 "test_spec_parse.y" + case 145: +#line 1043 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); free((yyvsp[(3) - (6)].str)); free((yyvsp[(6) - (6)].str)); ;} break; - case 135: -#line 962 "test_spec_parse.y" + case 146: +#line 1048 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -2977,8 +3062,8 @@ yyparse () ;} break; - case 136: -#line 968 "test_spec_parse.y" + case 147: +#line 1054 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -2989,8 +3074,8 @@ yyparse () ;} break; - case 137: -#line 977 "test_spec_parse.y" + case 148: +#line 1063 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -3001,8 +3086,8 @@ yyparse () ;} break; - case 138: -#line 986 "test_spec_parse.y" + case 149: +#line 1072 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STOPPED); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3011,8 +3096,8 @@ yyparse () ;} break; - case 139: -#line 1000 "test_spec_parse.y" + case 150: +#line 1086 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_LSN); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3022,8 +3107,8 @@ yyparse () ;} break; - case 140: -#line 1008 "test_spec_parse.y" + case 151: +#line 1094 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); @@ -3031,8 +3116,8 @@ yyparse () ;} break; - case 141: -#line 1022 "test_spec_parse.y" + case 152: +#line 1108 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); @@ -3040,8 +3125,8 @@ yyparse () ;} break; - case 142: -#line 1037 "test_spec_parse.y" + case 153: +#line 1123 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3049,8 +3134,8 @@ yyparse () ;} break; - case 143: -#line 1043 "test_spec_parse.y" + case 154: +#line 1129 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3059,8 +3144,8 @@ yyparse () ;} break; - case 144: -#line 1050 "test_spec_parse.y" + case 155: +#line 1136 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3068,8 +3153,8 @@ yyparse () ;} break; - case 145: -#line 1056 "test_spec_parse.y" + case 156: +#line 1142 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3078,39 +3163,39 @@ yyparse () ;} break; - case 148: -#line 1075 "test_spec_parse.y" + case 159: +#line 1161 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(2) - (2)].ival); ;} break; - case 149: -#line 1080 "test_spec_parse.y" + case 160: +#line 1166 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(4) - (4)].ival); ;} break; - case 150: -#line 1087 "test_spec_parse.y" + case 161: +#line 1173 "test_spec_parse.y" { (yyval.ival) = PGAF_TIMEOUT_DEFAULT; ;} break; - case 151: -#line 1088 "test_spec_parse.y" + case 162: +#line 1174 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} break; - case 152: -#line 1089 "test_spec_parse.y" + case 163: +#line 1175 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 153: -#line 1101 "test_spec_parse.y" + case 164: +#line 1187 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3120,8 +3205,8 @@ yyparse () ;} break; - case 154: -#line 1109 "test_spec_parse.y" + case 165: +#line 1195 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3131,8 +3216,8 @@ yyparse () ;} break; - case 155: -#line 1117 "test_spec_parse.y" + case 166: +#line 1203 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3142,8 +3227,8 @@ yyparse () ;} break; - case 156: -#line 1125 "test_spec_parse.y" + case 167: +#line 1211 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3153,8 +3238,8 @@ yyparse () ;} break; - case 157: -#line 1143 "test_spec_parse.y" + case 168: +#line 1229 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SQL); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3163,8 +3248,8 @@ yyparse () ;} break; - case 158: -#line 1158 "test_spec_parse.y" + case 169: +#line 1244 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT); strlcpy((yyval.cmd)->expected, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->expected)); @@ -3173,15 +3258,15 @@ yyparse () ;} break; - case 159: -#line 1165 "test_spec_parse.y" + case 170: +#line 1251 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); ;} break; - case 160: -#line 1169 "test_spec_parse.y" + case 171: +#line 1255 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); strlcpy((yyval.cmd)->state, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->state)); @@ -3189,8 +3274,8 @@ yyparse () ;} break; - case 161: -#line 1175 "test_spec_parse.y" + case 172: +#line 1261 "test_spec_parse.y" { /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); @@ -3198,16 +3283,16 @@ yyparse () ;} break; - case 162: -#line 1188 "test_spec_parse.y" + case 173: +#line 1274 "test_spec_parse.y" { (yyval.cmd) = current_promote_cmd; current_promote_cmd = NULL; ;} break; - case 163: -#line 1196 "test_spec_parse.y" + case 174: +#line 1282 "test_spec_parse.y" { current_promote_cmd = make_cmd(CMD_PROMOTE); current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; @@ -3217,8 +3302,8 @@ yyparse () ;} break; - case 164: -#line 1204 "test_spec_parse.y" + case 175: +#line 1290 "test_spec_parse.y" { if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], @@ -3227,8 +3312,8 @@ yyparse () ;} break; - case 165: -#line 1225 "test_spec_parse.y" + case 176: +#line 1311 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3237,8 +3322,8 @@ yyparse () ;} break; - case 166: -#line 1232 "test_spec_parse.y" + case 177: +#line 1318 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3247,8 +3332,8 @@ yyparse () ;} break; - case 167: -#line 1239 "test_spec_parse.y" + case 178: +#line 1325 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3258,8 +3343,8 @@ yyparse () ;} break; - case 168: -#line 1247 "test_spec_parse.y" + case 179: +#line 1333 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (7)].str), sizeof((yyval.cmd)->service)); @@ -3269,8 +3354,8 @@ yyparse () ;} break; - case 169: -#line 1263 "test_spec_parse.y" + case 180: +#line 1349 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_OFF); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3278,8 +3363,8 @@ yyparse () ;} break; - case 170: -#line 1269 "test_spec_parse.y" + case 181: +#line 1355 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_ON); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3287,8 +3372,8 @@ yyparse () ;} break; - case 171: -#line 1290 "test_spec_parse.y" + case 182: +#line 1376 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_SET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3298,8 +3383,8 @@ yyparse () ;} break; - case 172: -#line 1298 "test_spec_parse.y" + case 183: +#line 1384 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_GET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3309,23 +3394,23 @@ yyparse () ;} break; - case 173: -#line 1313 "test_spec_parse.y" + case 184: +#line 1399 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SLEEP); (yyval.cmd)->timeoutSeconds = (yyvsp[(2) - (2)].ival); ;} break; - case 174: -#line 1327 "test_spec_parse.y" + case 185: +#line 1413 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_DOWN); ;} break; - case 175: -#line 1331 "test_spec_parse.y" + case 186: +#line 1417 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_START); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3333,8 +3418,8 @@ yyparse () ;} break; - case 176: -#line 1337 "test_spec_parse.y" + case 187: +#line 1423 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_STOP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3342,8 +3427,8 @@ yyparse () ;} break; - case 177: -#line 1343 "test_spec_parse.y" + case 188: +#line 1429 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_KILL); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3351,8 +3436,8 @@ yyparse () ;} break; - case 178: -#line 1369 "test_spec_parse.y" + case 189: +#line 1455 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_INJECT); strlcpy((yyval.cmd)->expected, (yyvsp[(3) - (4)].str), sizeof((yyval.cmd)->expected)); /* image */ @@ -3377,8 +3462,8 @@ yyparse () ;} break; - case 179: -#line 1403 "test_spec_parse.y" + case 190: +#line 1489 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STOP_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3386,8 +3471,8 @@ yyparse () ;} break; - case 180: -#line 1409 "test_spec_parse.y" + case 191: +#line 1495 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_START_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3395,8 +3480,8 @@ yyparse () ;} break; - case 181: -#line 1430 "test_spec_parse.y" + case 192: +#line 1516 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FSM_STEP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3404,18 +3489,18 @@ yyparse () ;} break; - case 182: -#line 1446 "test_spec_parse.y" + case 193: +#line 1532 "test_spec_parse.y" { pgaf_next_brace_is_while = 1; ;} break; - case 183: -#line 1447 "test_spec_parse.y" + case 194: +#line 1533 "test_spec_parse.y" { (yyval.step) = (yyvsp[(4) - (5)].step); ;} break; - case 184: -#line 1452 "test_spec_parse.y" + case 195: +#line 1538 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STAYS_WHILE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3425,8 +3510,8 @@ yyparse () ;} break; - case 185: -#line 1471 "test_spec_parse.y" + case 196: +#line 1557 "test_spec_parse.y" { /* only "set monitor " is supported; $2 must be "monitor" */ if (strcmp((yyvsp[(2) - (3)].str), "monitor") != 0) @@ -3441,8 +3526,8 @@ yyparse () ;} break; - case 186: -#line 1496 "test_spec_parse.y" + case 197: +#line 1582 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3453,8 +3538,8 @@ yyparse () ;} break; - case 187: -#line 1505 "test_spec_parse.y" + case 198: +#line 1591 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3465,8 +3550,8 @@ yyparse () ;} break; - case 188: -#line 1514 "test_spec_parse.y" + case 199: +#line 1600 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3477,8 +3562,8 @@ yyparse () ;} break; - case 189: -#line 1523 "test_spec_parse.y" + case 200: +#line 1609 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3489,8 +3574,8 @@ yyparse () ;} break; - case 192: -#line 1544 "test_spec_parse.y" + case 203: +#line 1630 "test_spec_parse.y" { int i = current_spec->sequenceLength; if (i < PGAF_MAX_SEQ) @@ -3504,124 +3589,124 @@ yyparse () ;} break; - case 193: -#line 1565 "test_spec_parse.y" + case 204: +#line 1651 "test_spec_parse.y" { (yyval.str) = "init"; ;} break; - case 194: -#line 1566 "test_spec_parse.y" + case 205: +#line 1652 "test_spec_parse.y" { (yyval.str) = "single"; ;} break; - case 195: -#line 1567 "test_spec_parse.y" + case 206: +#line 1653 "test_spec_parse.y" { (yyval.str) = "primary"; ;} break; - case 196: -#line 1568 "test_spec_parse.y" + case 207: +#line 1654 "test_spec_parse.y" { (yyval.str) = "wait_primary"; ;} break; - case 197: -#line 1569 "test_spec_parse.y" + case 208: +#line 1655 "test_spec_parse.y" { (yyval.str) = "wait_standby"; ;} break; - case 198: -#line 1570 "test_spec_parse.y" + case 209: +#line 1656 "test_spec_parse.y" { (yyval.str) = "demoted"; ;} break; - case 199: -#line 1571 "test_spec_parse.y" + case 210: +#line 1657 "test_spec_parse.y" { (yyval.str) = "demote_timeout"; ;} break; - case 200: -#line 1572 "test_spec_parse.y" + case 211: +#line 1658 "test_spec_parse.y" { (yyval.str) = "draining"; ;} break; - case 201: -#line 1573 "test_spec_parse.y" + case 212: +#line 1659 "test_spec_parse.y" { (yyval.str) = "secondary"; ;} break; - case 202: -#line 1574 "test_spec_parse.y" + case 213: +#line 1660 "test_spec_parse.y" { (yyval.str) = "catchingup"; ;} break; - case 203: -#line 1575 "test_spec_parse.y" + case 214: +#line 1661 "test_spec_parse.y" { (yyval.str) = "prepare_promotion"; ;} break; - case 204: -#line 1576 "test_spec_parse.y" + case 215: +#line 1662 "test_spec_parse.y" { (yyval.str) = "stop_replication"; ;} break; - case 205: -#line 1577 "test_spec_parse.y" + case 216: +#line 1663 "test_spec_parse.y" { (yyval.str) = "maintenance"; ;} break; - case 206: -#line 1578 "test_spec_parse.y" + case 217: +#line 1664 "test_spec_parse.y" { (yyval.str) = "join_primary"; ;} break; - case 207: -#line 1579 "test_spec_parse.y" + case 218: +#line 1665 "test_spec_parse.y" { (yyval.str) = "apply_settings"; ;} break; - case 208: -#line 1580 "test_spec_parse.y" + case 219: +#line 1666 "test_spec_parse.y" { (yyval.str) = "prepare_maintenance"; ;} break; - case 209: -#line 1581 "test_spec_parse.y" + case 220: +#line 1667 "test_spec_parse.y" { (yyval.str) = "wait_maintenance"; ;} break; - case 210: -#line 1582 "test_spec_parse.y" + case 221: +#line 1668 "test_spec_parse.y" { (yyval.str) = "report_lsn"; ;} break; - case 211: -#line 1583 "test_spec_parse.y" + case 222: +#line 1669 "test_spec_parse.y" { (yyval.str) = "fast_forward"; ;} break; - case 212: -#line 1584 "test_spec_parse.y" + case 223: +#line 1670 "test_spec_parse.y" { (yyval.str) = "join_secondary"; ;} break; - case 213: -#line 1585 "test_spec_parse.y" + case 224: +#line 1671 "test_spec_parse.y" { (yyval.str) = "dropped"; ;} break; - case 214: -#line 1593 "test_spec_parse.y" + case 225: +#line 1679 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 215: -#line 1594 "test_spec_parse.y" + case 226: +#line 1680 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; /* Line 1267 of yacc.c. */ -#line 3625 "test_spec_parse.c" +#line 3710 "test_spec_parse.c" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); @@ -3835,7 +3920,98 @@ yyparse () } -#line 1597 "test_spec_parse.y" +#line 1683 "test_spec_parse.y" + + +/* + * fold_archivers_into_formations turns each top-level "archiver { }" + * declaration (TestArchiverNode, cluster->archivers[]) into an ordinary + * TestNode of kind NODE_KIND_ARCHIVER, appended to its own declared + * formation's own node list -- see TestArchiverNode's own comment + * (test_spec.h) for why the *declaration* still needs to be top-level even + * though it ends up represented identically to the older, still-supported + * "archiver nested inside a formation_block" spelling once parsed. Called + * once, right after yyparse() returns, so every caller downstream of + * parse_test_spec() (compose_gen.c included) only ever sees ordinary + * TestNode entries and needs no awareness of TestArchiverNode at all. + * + * cluster->archiverCount is reset to 0 once every entry has been folded, + * so cluster->archivers[] is never a second, stale source of truth for + * the very same nodes now living in cluster->formations[].nodes[]. + */ +static void +fold_archivers_into_formations(TestCluster *cluster) +{ + for (int ai = 0; ai < cluster->archiverCount; ai++) + { + TestArchiverNode *a = &cluster->archivers[ai]; + + if (a->formationCount == 0) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" needs at least one " + "\"formation \" entry\n", a->name); + exit(1); + } + + if (a->formationCount > 1) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" lists %d formations, but " + "pg_autoctl create archiver's own ini-driven bring-up " + "only attaches to one at create time -- declare just " + "\"formation %s\" here and attach the rest (e.g. " + "\"%s\") dynamically once it's running instead, via a " + "direct \"sql monitor { SELECT pgautofailover." + "archiver_add_formation(...) }\" step -- see " + "archiver_multi_formation.pgaf for the pattern\n", + a->name, a->formationCount, a->formations[0], + a->formations[1]); + exit(1); + } + + TestFormation *form = NULL; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + if (strcmp(cluster->formations[fi].name, a->formations[0]) == 0) + { + form = &cluster->formations[fi]; + break; + } + } + + if (form == NULL) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" attaches to formation " + "\"%s\", which is not declared in this cluster{} " + "block\n", a->name, a->formations[0]); + exit(1); + } + + if (form->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, + "pgaftest: too many nodes in formation \"%s\" (max %d)\n", + form->name, PGAF_MAX_NODES); + exit(1); + } + + TestNode *node = &form->nodes[form->nodeCount++]; + + memset(node, 0, sizeof(*node)); + strlcpy(node->name, a->name, sizeof(node->name)); + node->kind = NODE_KIND_ARCHIVER; + node->candidatePriority = 50; + node->replicationQuorum = true; + strlcpy(node->region, a->region, sizeof(node->region)); + node->createDeferred = a->createDeferred; + node->launchDeferred = a->launchDeferred; + } + + cluster->archiverCount = 0; +} /* ----------------------------------------------------------------------- @@ -3864,6 +4040,8 @@ parse_test_spec(const char *filename) yyparse(); fclose(f); + fold_archivers_into_formations(&spec->cluster); + /* * If the file has no explicit sequence{} block, default to running * steps in declaration order. Populated here (not just in the CI diff --git a/src/bin/pgaftest/test_spec_parse.h b/src/bin/pgaftest/test_spec_parse.h index d0ad03e5d..85a36a2e1 100644 --- a/src/bin/pgaftest/test_spec_parse.h +++ b/src/bin/pgaftest/test_spec_parse.h @@ -286,7 +286,7 @@ #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED typedef union YYSTYPE -#line 145 "test_spec_parse.y" +#line 146 "test_spec_parse.y" { int ival; char *str; diff --git a/src/bin/pgaftest/test_spec_parse.y b/src/bin/pgaftest/test_spec_parse.y index 2670bcb5a..a785c86a6 100644 --- a/src/bin/pgaftest/test_spec_parse.y +++ b/src/bin/pgaftest/test_spec_parse.y @@ -139,6 +139,7 @@ static TestCmd *current_promote_cmd = NULL; static TestCmd *current_pass_cmd = NULL; /* for opt_passing_through */ static TestFormation *current_formation = NULL; static TestNode *current_node = NULL; +static TestArchiverNode *current_archiver = NULL; %} @@ -256,10 +257,95 @@ cluster_item: | auth_line | extension_version_line | formation_block + | archiver_block | T_BIND_SOURCE { current_spec->cluster.bindSource = true; } | T_LEGACY_STARTUP { current_spec->cluster.legacyStartup = true; } ; +/* + * archiver { formation [formation ...] [region ] } + * + * Top-level, sibling to "monitor" and "formation" -- NOT nested inside a + * formation_block's node_list the way ordinary/coordinator/worker nodes + * are (see TestArchiverNode's own comment in test_spec.h for why: an + * archiver attaches to one or more formations by name, it isn't a member + * of any one of them). May appear more than once, for a cluster with + * several archivers. + * + * Braces are mandatory here (unlike monitor_line's own bare/flat form): + * archiver_opt's own "T_FORMATION T_IDENT" would otherwise be + * indistinguishable, at one token of lookahead, from a brand new + * top-level formation_block starting right after this one (formation_ + * block's own opening is also "T_FORMATION bare_name ...", bare_name + * itself accepting a plain T_IDENT) -- a real shift/reduce ambiguity + * caught while writing this grammar, not a stylistic choice. + */ +archiver_block: + T_ARCHIVER T_IDENT + { + TestCluster *cl = ¤t_spec->cluster; + + if (cl->archiverCount >= PGAF_MAX_ARCHIVERS) + { + fprintf(stderr, "pgaftest: too many archivers (max %d)\n", + PGAF_MAX_ARCHIVERS); + exit(1); + } + + current_archiver = &cl->archivers[cl->archiverCount++]; + strlcpy(current_archiver->name, $2, sizeof(current_archiver->name)); + free($2); + } + T_LBRACE archiver_opt_list T_RBRACE + ; + +archiver_opt_list: + /* empty */ + | archiver_opt_list archiver_opt + ; + +archiver_opt: + T_FORMATION T_IDENT + { + if (current_archiver->formationCount >= PGAF_MAX_ARCHIVER_FORMATIONS) + { + fprintf(stderr, + "pgaftest: too many --formation entries for archiver " + "\"%s\" (max %d)\n", + current_archiver->name, PGAF_MAX_ARCHIVER_FORMATIONS); + exit(1); + } + strlcpy(current_archiver->formations[current_archiver->formationCount++], + $2, sizeof(current_archiver->formations[0])); + free($2); + } + | T_REGION T_IDENT + { + strlcpy(current_archiver->region, $2, sizeof(current_archiver->region)); + free($2); + } + | T_REGION T_STRING + { + strlcpy(current_archiver->region, $2, sizeof(current_archiver->region)); + free($2); + } + | T_CREATE T_AND T_LAUNCH T_DEFERRED + { + /* bare "create and launch deferred" = both gates, matching + * node_opt's own identical form */ + current_archiver->createDeferred = true; + current_archiver->launchDeferred = true; + } + | T_LAUNCH T_DEFERRED + { + current_archiver->launchDeferred = true; + } + | T_CREATE T_DEFERRED + { + current_archiver->createDeferred = true; + } + ; + /* * monitor [port N] * @@ -1596,6 +1682,97 @@ ident_or_string: %% +/* + * fold_archivers_into_formations turns each top-level "archiver { }" + * declaration (TestArchiverNode, cluster->archivers[]) into an ordinary + * TestNode of kind NODE_KIND_ARCHIVER, appended to its own declared + * formation's own node list -- see TestArchiverNode's own comment + * (test_spec.h) for why the *declaration* still needs to be top-level even + * though it ends up represented identically to the older, still-supported + * "archiver nested inside a formation_block" spelling once parsed. Called + * once, right after yyparse() returns, so every caller downstream of + * parse_test_spec() (compose_gen.c included) only ever sees ordinary + * TestNode entries and needs no awareness of TestArchiverNode at all. + * + * cluster->archiverCount is reset to 0 once every entry has been folded, + * so cluster->archivers[] is never a second, stale source of truth for + * the very same nodes now living in cluster->formations[].nodes[]. + */ +static void +fold_archivers_into_formations(TestCluster *cluster) +{ + for (int ai = 0; ai < cluster->archiverCount; ai++) + { + TestArchiverNode *a = &cluster->archivers[ai]; + + if (a->formationCount == 0) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" needs at least one " + "\"formation \" entry\n", a->name); + exit(1); + } + + if (a->formationCount > 1) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" lists %d formations, but " + "pg_autoctl create archiver's own ini-driven bring-up " + "only attaches to one at create time -- declare just " + "\"formation %s\" here and attach the rest (e.g. " + "\"%s\") dynamically once it's running instead, via a " + "direct \"sql monitor { SELECT pgautofailover." + "archiver_add_formation(...) }\" step -- see " + "archiver_multi_formation.pgaf for the pattern\n", + a->name, a->formationCount, a->formations[0], + a->formations[1]); + exit(1); + } + + TestFormation *form = NULL; + + for (int fi = 0; fi < cluster->formationCount; fi++) + { + if (strcmp(cluster->formations[fi].name, a->formations[0]) == 0) + { + form = &cluster->formations[fi]; + break; + } + } + + if (form == NULL) + { + fprintf(stderr, + "pgaftest: archiver \"%s\" attaches to formation " + "\"%s\", which is not declared in this cluster{} " + "block\n", a->name, a->formations[0]); + exit(1); + } + + if (form->nodeCount >= PGAF_MAX_NODES) + { + fprintf(stderr, + "pgaftest: too many nodes in formation \"%s\" (max %d)\n", + form->name, PGAF_MAX_NODES); + exit(1); + } + + TestNode *node = &form->nodes[form->nodeCount++]; + + memset(node, 0, sizeof(*node)); + strlcpy(node->name, a->name, sizeof(node->name)); + node->kind = NODE_KIND_ARCHIVER; + node->candidatePriority = 50; + node->replicationQuorum = true; + strlcpy(node->region, a->region, sizeof(node->region)); + node->createDeferred = a->createDeferred; + node->launchDeferred = a->launchDeferred; + } + + cluster->archiverCount = 0; +} + + /* ----------------------------------------------------------------------- * Public entry point * ----------------------------------------------------------------------- */ @@ -1622,6 +1799,8 @@ parse_test_spec(const char *filename) yyparse(); fclose(f); + fold_archivers_into_formations(&spec->cluster); + /* * If the file has no explicit sequence{} block, default to running * steps in declaration order. Populated here (not just in the CI From 76ff170d597ce8156d06d25b138ba938601a667f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:05:13 +0200 Subject: [PATCH 39/55] tests: new/updated pgaftest specs for multi-membership + region archiver_multi_formation.pgaf: one archiver, brought up ARCHIVING for a "default" formation, then attached to a second, independent formation purely through a monitor-side RPC while already running -- proving the reconciler's own dynamic-attach discovery, not just static coverage at creation time. citus_basic_operation.pgaf: test_011/test_012 cover a single archiver attached to a whole Citus formation, ending up with one membership per group (coordinator's group 0, plus each worker group) -- proving Citus coverage isn't limited to the coordinator. Uses the new top-level "archiver { }" syntax with "create and launch deferred" + `pg_autoctl node start`, triggered only once every worker group is already registered. archiver_budget_architecture_regions.pgaf: the "budget architecture" (see docs/architecture.rst) -- node1/dc1, node2/dc2, archiver1/dc3 -- proving --region round-trips correctly for both node kinds and that the archiver still does real work regardless of its own label. archiver_two_regions.pgaf: two archivers (eu-west/us-east) attached to the same formation, proving both independently capture WAL (via an archiverQuorum raise after the fact, since wal_archived() itself aggregates across every attached archiver and doesn't expose a per-archiver breakdown). --- .../archiver_budget_architecture_regions.pgaf | 89 ++++++++ tests/tap/specs/archiver_multi_formation.pgaf | 197 ++++++++++++++++++ tests/tap/specs/archiver_two_regions.pgaf | 113 ++++++++++ tests/tap/specs/citus_basic_operation.pgaf | 77 +++++++ 4 files changed, 476 insertions(+) create mode 100644 tests/tap/specs/archiver_budget_architecture_regions.pgaf create mode 100644 tests/tap/specs/archiver_multi_formation.pgaf create mode 100644 tests/tap/specs/archiver_two_regions.pgaf diff --git a/tests/tap/specs/archiver_budget_architecture_regions.pgaf b/tests/tap/specs/archiver_budget_architecture_regions.pgaf new file mode 100644 index 000000000..dd84ed9eb --- /dev/null +++ b/tests/tap/specs/archiver_budget_architecture_regions.pgaf @@ -0,0 +1,89 @@ +# Archiving & Disaster Recovery: the "budget architecture" (see +# docs/architecture.rst's own "Service Availability" section) -- an +# ordinary two-node primary/secondary group plus an archiver added on top +# for Disaster Recovery, rather than a third live standby. This spec's own +# focus is the --region label pg_autoctl create postgres/create archiver +# both accept: node1 in "dc1", node2 in "dc2", archiver1 (which the +# monitor logically sits alongside, for this topology) in "dc3" -- a +# realistic geo-distributed budget deployment, and the first pgaftest spec +# to actually exercise region end to end for either node kind. +# +# region is purely informational (pg_autoctl watch's own display, get_ +# archivers()'s own output column for an archiver) -- it never affects +# placement, quorum, or failover decisions on its own. This spec proves +# the label round-trips correctly (set at create time, readable back from +# the monitor afterwards) and that the archiver itself still does real +# work regardless of which region it's labelled with. +# +# archiver1 uses pgaftest's top-level "archiver { }" syntax (see +# TestArchiverNode's own comment in test_spec.h), folded into "default"'s +# own node list and launched immediately (the normal default for this +# syntax) via the ordinary ini-driven "pg_autoctl node run" path -- no +# deferred launch needed: group 0 of "default" already exists as soon as +# node1 (which archiver1 depends on being healthy) has registered, +# regardless of whether promotion has completed yet. +# +# Predecessor: basic_operation.pgaf (the same two-node group, without the +# archiver or region labels); archiver_wal_capture.pgaf (the WAL-capture +# proof this borrows its pattern from). + +cluster { + monitor + formation { + node1 region dc1 + node2 region dc2 + } + archiver archiver1 { + formation default + region dc3 + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: region labels round-trip correctly for both ordinary nodes and +# the archiver -- set via --region at create time, readable back +# from the monitor afterwards. pgautofailover.node is covered by +# the blanket "GRANT SELECT ON ALL TABLES" near the top of +# pgautofailover.sql, so a direct SELECT works for node1/node2; +# pgautofailover.archiver itself is not (granted much later in +# the same file), so archiver1's region goes through get_ +# archivers('default'), the same function pg_autoctl watch uses. +# + +step test_001_region_labels_round_trip { + sql monitor { SELECT region FROM pgautofailover.node WHERE nodename = 'node1'; } + expect { dc1 } + sql monitor { SELECT region FROM pgautofailover.node WHERE nodename = 'node2'; } + expect { dc2 } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver1'; } + expect { dc3 } +} + +# +# test_002: the archiver itself still does real work regardless of its own +# region label -- same idiom as archiver_wal_capture.pgaf's own +# test_001 (segment 3 is this image's own observed slot-creation +# floor; see that spec's header comment for the full reasoning). +# + +step test_002_archiver_captures_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } + expect { t } +} diff --git a/tests/tap/specs/archiver_multi_formation.pgaf b/tests/tap/specs/archiver_multi_formation.pgaf new file mode 100644 index 000000000..ea1d6edce --- /dev/null +++ b/tests/tap/specs/archiver_multi_formation.pgaf @@ -0,0 +1,197 @@ +# Archiving & Disaster Recovery, Milestone 5: dynamic multi-formation +# membership for a single archiver identity. +# +# Covers service_archiver_reconciler.c: one archiver process (archiver1) +# is brought up ARCHIVING only "default" (a plain two-node formation), then +# -- while it is already running and capturing that formation's WAL -- is +# attached to a *second*, independent formation ("formation2") purely +# through a monitor-side RPC (pgautofailover.archiver_add_formation()), +# with no restart of the archiver process and no CLI/config change on its +# side at all. archiver_reconciler_tick() only re-lists this archiver's +# memberships (pgautofailover.list_archiver_memberships()) once every +# ARCHIVER_RECONCILER_INTERVAL_SECONDS (30s, service_archiver_reconciler.c) +# -- so the second membership's WAL capture can only start once that +# periodic tick actually notices it, not immediately. That discovery path +# is the thing this spec exists to prove; everything else here (bringing +# up two independent 2-node formations, forcing WAL switches, checking +# wal_archived()) is the same idiom as archiver_wal_capture.pgaf. +# +# Two independent formations coming up concurrently is why this spec uses +# the explicit per-node "wait until state is " forms throughout +# rather than the aggregate "wait until primary, secondary" form: the +# aggregate form is not formation-scoped (test_runner.c's +# wait_for_states()/monitor pg_autoctl inspect monitor formation-states +# path just checks "does *any* node report primary and *any* node report +# secondary" cluster-wide), which is ambiguous once two formations are +# each independently electing their own primary/secondary at once. +# +# archiver1 ends up with two rows in pgautofailover.node once attached to +# both formations (one per (formation, group) membership, both nodename = +# 'archiver1', both groupid = 0 since each formation here has a single, +# plain-Postgres group) -- so the generic "wait until archiver1 state is +# archiving" form (SELECT reportedstate, goalstate FROM pgautofailover.node +# WHERE nodename = $1 LIMIT 1, no ORDER BY -- see test_runner.c's +# monitor_get_node_state()) becomes ambiguous the moment the second +# membership exists: it may observe either row. Every check on archiver1's +# per-membership state after test_003 attaches the second membership goes +# through an explicit `sql monitor` query naming both nodename *and* +# formationid instead. +# +# The autoctl_node role has no direct SELECT on pgautofailover.archiver +# (granted much later in pgautofailover.sql than the blanket "GRANT SELECT +# ON ALL TABLES IN SCHEMA pgautofailover" near its own top) -- archiver1's +# archiverid is resolved via pgautofailover.get_archivers('default'), which +# *is* granted and returns archiver_id as its first output column. +# +# archiver1 is declared with pgaftest's top-level "archiver { }" syntax +# (sibling to monitor/formation, not nested inside either formation block) +# -- an archiver attaches to formations by name, it isn't a member of any +# one of them (pgautofailover.archiver has no formationid column at all; +# see TestArchiverNode's own comment in test_spec.h). Internally it's +# folded into "default"'s own node list (the one formation it declares), +# so it launches immediately -- the normal default for this syntax, same +# as any ordinary node -- via the ordinary ini-driven "pg_autoctl node +# run" path: no exec, no deferred launch needed here, since group 0 of +# "default" already exists as soon as node1 (which archiver1 depends on +# being healthy, same as node2 does) has registered, regardless of +# whether promotion has completed yet. +# +# Predecessor: none -- first pgaftest spec covering dynamic multi-formation +# archiver membership; see archiver_wal_capture.pgaf for the base single- +# formation WAL-capture mechanism this builds on. + +cluster { + monitor + formation { + node1 + node2 + } + formation formation2 { + node3 + node4 + } + archiver archiver1 { + formation default + } +} + +setup { + wait until node1 state is primary timeout 90s + wait until node2 state is secondary timeout 90s + promote node1 + wait until archiver1 state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: baseline -- confirm formation "default"'s WAL is captured +# before formation2 even exists, exactly as in +# archiver_wal_capture.pgaf. Segment numbering here follows that +# spec's own observed floor (segment 3 is the archiver's slot +# restart_lsn floor on this image -- see archiver_wal_capture. +# pgaf's header comment for the full reasoning); each switch +# below has a real INSERT immediately before it. +# + +step test_001_capture_formation1_wal { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } + expect { t } +} + +# +# test_002: bring up formation2 independently -- registration against a +# non-default formation can lag behind the default formation's +# own (compose_gen.c: data nodes retry registration until their +# formation exists), so this is generous on timeout. +# + +step test_002_bring_up_formation2 { + wait until node3 state is primary timeout 120s + wait until node4 state is secondary timeout 120s + promote node3 +} + +# +# test_003: the actual behavior under test. Attach the already-running +# archiver1 to formation2 purely via the monitor RPC (not the +# CLI's --formation flag, which only matters at `create archiver` +# time) and wait for the reconciler's own periodic tick (30s, +# ARCHIVER_RECONCILER_INTERVAL_SECONDS) to notice the new +# membership and start a second WAL-capture child for it. +# archiver_add_formation() is idempotent and, since formation2 +# has exactly one group (group 0, plain nodes, no Citus), attaches +# exactly one new ARCHIVING node row. +# + +step test_003_dynamic_attach_to_formation2 { + sql monitor { + SELECT pgautofailover.archiver_add_formation( + (SELECT archiver_id FROM pgautofailover.get_archivers('default') LIMIT 1), + 'formation2'); + } + # one reconciler tick (30s) plus margin for it to notice, fork the new + # capture child, and for that child to register/report far enough to + # reach ARCHIVING_STATE. + sleep 50s + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'formation2'; } + expect { archiving } +} + +# +# test_004: confirm formation2's WAL is actually being captured by the +# newly-started capture child, not just that the FSM state looks +# right. formation2 is a fresh formation/group -- its own +# archiver slot restart_lsn floor is expected to be an early, +# low-numbered segment (no unrelated bootstrap traffic beyond its +# own two nodes joining), so this switches enough times to be +# independent of the exact floor rather than hardcoding a segment +# number the way test_001 does for the already-characterized +# "default" formation. +# + +step test_004_capture_formation2_wal { + sql node3 { CREATE TABLE t2(a int); INSERT INTO t2 VALUES (1), (2); } + sql node3 { SELECT pg_switch_wal(); } + sql node3 { INSERT INTO t2 VALUES (3); } + sql node3 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000001'); } + expect { t } + sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000002'); } + expect { t } +} + +# +# test_005: formation "default"'s own capture must still be uninterrupted +# -- proves the reconciler's dynamic add of formation2 did not +# restart the archiver process or disrupt the pre-existing +# membership's already-running pg_receivewal. One more switch on +# top of test_001's own two. +# + +step test_005_formation1_still_healthy { + sql node1 { INSERT INTO t1 VALUES (4); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } + expect { t } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default'; } + expect { archiving } +} + +sequence + test_001_capture_formation1_wal + test_002_bring_up_formation2 + test_003_dynamic_attach_to_formation2 + test_004_capture_formation2_wal + test_005_formation1_still_healthy diff --git a/tests/tap/specs/archiver_two_regions.pgaf b/tests/tap/specs/archiver_two_regions.pgaf new file mode 100644 index 000000000..6f988ab5d --- /dev/null +++ b/tests/tap/specs/archiver_two_regions.pgaf @@ -0,0 +1,113 @@ +# Archiving & Disaster Recovery: two independent archivers, in two +# different regions, both attached to the very same formation -- the +# geographically-redundant DR coverage pattern archiver.region exists for +# (see that column's own comment, pgautofailover.sql, and +# pg_autoctl_create_archiver.rst's own --region section). +# +# archiver_add_formation() names each ARCHIVING node row +# 'archiver--', so two different archivers attaching +# to the same (formation, group) get distinct rows and fully independent +# WAL streams/replication slots against the same primary -- confirmed +# already at the SQL-regression level (src/monitor/sql/archiving_schema. +# sql's own "a second archiver serving the same formation/group" case); +# this spec is the first to prove it end to end, with two real archiver +# processes. +# +# Rather than trying to peek at which specific archiver reported a given +# WAL segment (pgautofailover.wal_archived() aggregates across every +# archiver attached to the group, it doesn't expose a per-archiver +# breakdown to autoctl_node), test_002 proves both are independently +# capturing by raising archiverQuorum from its default of 1 to 2 *after* +# a segment is already confirmed archived at quorum 1, then re-checking +# the very same segment: if only one of the two archivers had actually +# captured and reported it, raising the quorum would make wal_archived() +# flip back to false for that segment. It doesn't -- proving both +# archivers, not just one, independently streamed and reported it. +# +# Both archivers use pgaftest's top-level "archiver { }" syntax (see +# TestArchiverNode's own comment in test_spec.h), each folded into +# "default"'s own node list and launched immediately (the normal default +# for this syntax) via the ordinary ini-driven "pg_autoctl node run" path +# -- no deferred launch needed: group 0 of "default" already exists as +# soon as node1 has registered, regardless of promotion. +# +# Predecessor: archiver_wal_capture.pgaf (single-archiver WAL-capture +# proof this borrows its segment-numbering reasoning from); +# archiver_budget_architecture_regions.pgaf (the first spec to exercise +# --region at all, for a single archiver). + +cluster { + monitor + formation { + node1 + node2 + } + archiver archiver-eu { + formation default + region eu-west + } + archiver archiver-us { + formation default + region us-east + } +} + +setup { + wait until primary, secondary timeout 120s + promote node1 + wait until archiver-eu state is archiving timeout 60s + wait until archiver-us state is archiving timeout 60s +} + +teardown { + compose down +} + +# +# test_001: both archivers attached to the same formation, each correctly +# reporting its own region -- get_archivers('default') returns +# one row per archiver here (both groupid 0, the only group in +# this plain-Postgres formation), distinguished by archiver_name. +# + +step test_001_both_archivers_attached_and_labelled { + sql monitor { SELECT reported_state::text FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-eu'; } + expect { archiving } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-eu'; } + expect { eu-west } + sql monitor { SELECT reported_state::text FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-us'; } + expect { archiving } + sql monitor { SELECT region FROM pgautofailover.get_archivers('default') WHERE archiver_name = 'archiver-us'; } + expect { us-east } +} + +# +# test_002: both archivers are independently, actually capturing WAL -- +# not just one of them with the other silently idle -- proven by +# raising archiverQuorum after the fact, see this file's own +# header comment for the full reasoning. Segment 3 is archiver_ +# wal_capture.pgaf's own observed floor for a single archiver on +# this image (see that spec's header comment for the general +# reasoning: a few segments get consumed by node1+node2's own +# bootstrap before any archiver's replication slot exists, +# observed at exactly segment 3, repeatably). Two archivers here +# instead of one shouldn't move that floor -- both slots get +# created around the same bootstrap point and segment 3 hasn't +# been recycled yet by either -- but this hasn't been separately +# confirmed against a real two-archiver run the way the single- +# archiver floor has; if this segment number turns out wrong +# against the real image, this is the line to adjust. +# + +step test_002_both_archivers_capture_independently { + sql node1 { CREATE TABLE t1(a int); INSERT INTO t1 VALUES (1), (2); } + sql node1 { SELECT pg_switch_wal(); } + sql node1 { INSERT INTO t1 VALUES (3); } + sql node1 { SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } + sql monitor { SELECT pgautofailover.set_archiver_policy('default', NULL, 2, NULL, NULL); } + sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } + expect { t } +} diff --git a/tests/tap/specs/citus_basic_operation.pgaf b/tests/tap/specs/citus_basic_operation.pgaf index 47a0cd571..62b9e57c8 100644 --- a/tests/tap/specs/citus_basic_operation.pgaf +++ b/tests/tap/specs/citus_basic_operation.pgaf @@ -1,5 +1,10 @@ # Test basic Citus cluster operations: coordinator HA, worker HA with two # worker groups, distributed table writes/reads, and failover at each level. +# test_011/test_012 additionally cover Archiving & Disaster Recovery for a +# Citus formation: a single archiver attached once ends up with one +# membership per group -- the coordinator's own group 0, plus worker1's +# group 1 and worker2's group 2 -- each independently WAL-capturing, proving +# a Citus formation's archiver coverage isn't limited to the coordinator. # # Ported from tests/test_basic_citus_operation.py # Predecessor: tests/test_basic_citus_operation.py @@ -14,6 +19,10 @@ cluster { worker2a worker group 2 worker2b worker group 2 } + archiver archiver1 { + formation default + create and launch deferred + } } setup { @@ -132,3 +141,71 @@ step test_010_perform_failover_coordinator { and coordinator1b state is primary timeout 90s } + +# +# test_011: bring up an archiver attached to this Citus formation. +# Declared with pgaftest's top-level "archiver { }" syntax (see +# the cluster{} block above) with "create and launch deferred": +# its container still runs the ordinary "pg_autoctl node run +# " command, but the ini's own [launch] section makes that +# poll and wait rather than actually registering, until this +# step's `pg_autoctl node start` un-defers it -- exactly the +# same idiom as any other deferred node in this DSL (see +# basic_operation.pgaf's own node3/test_017). Triggered only +# here, well after test_002_init_workers confirms every group +# (0, 1, 2) already exists on the monitor: +# pgautofailover.archiver_add_formation() attaches one ARCHIVING +# row per group already present in the formation at the moment +# it's called, so creating the archiver this late (rather than +# at cluster boot, concurrently with the worker groups still +# registering) guarantees a single `pg_autoctl create archiver +# --formation default` call here covers all three groups, not +# just whichever happened to exist first. +# +# archiver1 ends up with three rows in pgautofailover.node (one +# per group), all sharing nodename = 'archiver1' -- the generic +# "wait until archiver1 state is archiving" form is ambiguous +# once more than one such row exists (test_runner.c's +# monitor_get_node_state() does "... WHERE nodename = $1 LIMIT 1", +# no ORDER BY), so every check below names formationid and +# groupid explicitly instead. +# + +step test_011_bring_up_archiver { + exec archiver1 pg_autoctl node start + sleep 40s + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 0; } + expect { archiving } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 1; } + expect { archiving } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 2; } + expect { archiving } +} + +# +# test_012: prove the archiver is actually capturing real WAL for every +# group, not just reporting a state label. Writes go to each +# group's *current* primary -- coordinator1b (failed over in +# test_010), worker1a (never failed over in this spec), worker2b +# (failed over in test_009) -- and each membership's own +# reportedlsn is checked against '0/0': service_archiver_update_ +# current_lsn() (service_archiver.c) never reports anything until +# a real segment has actually been captured, so a value beyond +# '0/0' can only mean WAL genuinely landed for that group. This +# avoids hardcoding an exact segment name/number, which this +# spec's own substantial prior WAL traffic (distributed table +# creation, several failovers) would make fragile to predict. +# + +step test_012_archiver_captures_every_group { + sql coordinator1b { CREATE TABLE archiver_probe_coord(a int); INSERT INTO archiver_probe_coord SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + sql worker1a { CREATE TABLE archiver_probe_w1(a int); INSERT INTO archiver_probe_w1 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + sql worker2b { CREATE TABLE archiver_probe_w2(a int); INSERT INTO archiver_probe_w2 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } + sleep 15s + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 0; } + expect { t } + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 1; } + expect { t } + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 2; } + expect { t } +} From 1330956a18c883fdb060e3f5194ef323d1c9fbbb Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:05:32 +0200 Subject: [PATCH 40/55] docs: integrate archiving into the top-level HA/DR architecture narrative intro.rst, architecture.rst, fault-tolerance.rst: fold Archiving & Disaster Recovery into the existing Service Availability / Business Continuity framing rather than treating it as a bolt-on -- the "budget architecture" (two Postgres nodes plus an archiver, in place of a third live standby) as a named trade-off, WARM standby via cascading replication-protocol serving, and PITR via "transient" nodes that can be reified into new groups. --- docs/architecture.rst | 62 +++++++++++++---- docs/fault-tolerance.rst | 113 +++++++++++++++--------------- docs/intro.rst | 145 ++++++++++++++++++++------------------- 3 files changed, 177 insertions(+), 143 deletions(-) diff --git a/docs/architecture.rst b/docs/architecture.rst index c411784ba..1c30248d7 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -35,10 +35,10 @@ High Availability ------------------ pg_auto_failover treats "High Availability" as two related but distinct -guarantees, rather than one. Most of what follows on this page -- -the Monitor, the keeper, synchronous replication, node recovery -- is -in service of the first of the two; :ref:`archiving_internals` and the -pages it links to are in service of the second: +guarantees, rather than one. Most of what follows on this page -- the +Monitor, the keeper, synchronous replication, node recovery -- is in service +of the first of the two; :ref:`archiving_architecture` and the pages it +links to are in service of the second: - **Service Availability**: the Postgres *service* itself stays reachable and able to accept reads and writes, with as little downtime as @@ -63,19 +63,34 @@ asynchronous replication`_ below) guaranteeing no committed write is lost in the process. This is the guarantee that answers "the primary just died -- who serves the next query?". +Service Availability can be setup to obtain Business Continuity in the face +of production incidents with a basic setup of two Postgres nodes, and +Postgres High Availability starting with a setup of three Postgres nodes. + +Given integrated archiving support, a trade-off or *budget* architecture can +be easily deployed with two Postgres nodes and an archiver to obtain an HA +setup that complies with many production needs. + Disaster Recovery ^^^^^^^^^^^^^^^^^^ -Service Availability alone can't answer "we lost every node that ever had -this data" or "an operator dropped the wrong table an hour ago" -- a -healthy failover just moves the same problem to a different node just as -fast. Disaster Recovery is handled by a physically distinct kind of node, -the **archiver**, added on top of any of the architectures on this page: -it continuously captures WAL from the group's current primary and -periodically produces base backups, independent of whether any standby is -even healthy or present. See :ref:`archiving_architecture` for where an -archiver fits alongside the architectures below, :ref:`archiving_internals` -for exactly how WAL capture and base-backup generation work, and +Service Availability only makes sense for a database system when there is a +compliant setup for durability, or data safety. When using PostgreSQL, that +means a proper archiving implementation that allows *Point in Time Recovery* +operations. + +With PITR it's possible to mitigate data loss operations such as a missing +WHERE clause in a DELETE or a DROP TABLE done in production instead of the +development environment, also known as human errors. + +Disaster Recovery is handled by a physically distinct kind of node, the +**archiver**, added on top of any of the architectures on this page: it +continuously captures WAL from the group's current primary and periodically +produces base backups. + +See :ref:`archiving_and_disaster_recovery` for where an archiver fits +alongside the architectures below, :ref:`archiving_architecture` for exactly +how WAL capture and base-backup generation work, and :ref:`archiving_fault_tolerance` for how this changes what a total loss of the rest of the formation actually means. @@ -200,6 +215,25 @@ As a result, refrain from naming your nodes with the role you intend for them. Their roles can change. If they didn't, your system wouldn't need pg_auto_failover! +Archiver +^^^^^^^^ + +An archiver is a server (virtual or physical) that runs PostgreSQL archiving +storage for one or many formations. The archiver hosts any number of +*archiving nodes* and schedules *base backups* in order to be able to +implement Postgres `Point in Time Recovery`__ which is the foundations for +implementing Disaster Recovery. + +__ https://www.postgresql.org/docs/current/continuous-archiving.html + +Archiving Node +^^^^^^^^^^^^^^ + +A process managed in an archiver instance that reports to the monitor as a +node in a group and that runs ``pg_receivewal``. An archiving node as no +PGDATA, it can participate in the replication quorum but can not be a +failover candidate: its ``candidate_priority`` is always zero. + State ^^^^^ diff --git a/docs/fault-tolerance.rst b/docs/fault-tolerance.rst index e4f7e8493..c981e8465 100644 --- a/docs/fault-tolerance.rst +++ b/docs/fault-tolerance.rst @@ -262,64 +262,61 @@ walkthrough. Archiving Nodes and Disaster Recovery -------------------------------------- -Everything above concerns keeping the PostgreSQL *service* available: a -healthy primary always answering reads and writes, promoted from a healthy -secondary within seconds of a failure. An **archiver**, introduced in -:ref:`archiving_architecture`, addresses a different failure mode entirely --- not "the primary went away for a moment," but "the data needs to survive -even if every node that was ever a primary or a standby is gone." - -WAL capture independent of any standby -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A standby's replication connection exists to keep a second copy of the data -directory caught up for a possible promotion; it stops mattering to fault -tolerance the moment that standby is unhealthy, dropped, or was never -configured at all. An archiver's `pg_receivewal`__ connection has no such -dependency: it streams continuously from whichever node is currently the -group's primary, following it across promotions, and it does this whether -the formation has zero standbys or five. A single-node formation with one -archiver already has WAL protection a standby-less formation alone never -would. - -__ https://www.postgresql.org/docs/current/app-pgreceivewal.html - -Because archiving nodes hold no `PGDATA`__ of their own, they are outside -the replication quorum entirely: an unhealthy archiver never triggers -DRAINING on the primary, never disables synchronous replication, and never -factors into ``number_sync_standbys``. Losing an archiver is a -disaster-recovery-posture event the monitor reports, not a service-affecting -one. - -__ https://www.postgresql.org/docs/current/app-initdb.html - -Base backups, on a policy -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On top of continuous WAL capture, an archiver periodically produces full -base backups from its own local WAL cache, on a schedule and retention -policy attached to the formation or overridden per group (see -:ref:`archiving_operations` for the operational details: creating a -policy, attaching it, and how scheduling and pruning behave). Each -completed backup, and each one pruned by retention, is reported to the -monitor the same way WAL segments are, so an operator or a client library -can always ask the monitor where the most recent recoverable base backup -lives without needing to reach the archiver's storage directly. - -Rebuilding after every other node is lost -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This is the scenario the rest of this page doesn't cover: not one node -failing while others compensate, but the primary and every standby gone at -once -- the kind of event no amount of failover automation can route -around, because there is nothing healthy left to fail over *to*. As long as -one archiver survived, the formation is not actually gone: its WAL cache -and latest base backup are enough to rebuild a new primary from scratch, -and from there re-grow standbys the ordinary way. This is the specific -failure mode the split HA-tool/backup-tool approach described in -:ref:`ha_dr_backups` tends to leave untested until the day it's needed -- -here it is the same monitor, the same node-registration path, and the same -archiver that was already running throughout normal operation. +On-top of the Service Availability a database system needs Data +Availability, and it is expected to survive some data loss scenarios that +are not covered with the previous sections about fault tolerance. + +Typically, an erroneous ``DELETE`` without a ``WHERE`` clause, or a ``DROP +TABLE`` that happened on the wrong server, by mistake or because of a +security exploit of some sorts. + +.. note:: + + Always make sure to have a separate role for the normal application + activities that is separate from the database owner, and use yet another + specific role for database schema upgrade, or migrations. + + This alone avoids most of the security risk surface. + +While the previous sections concerns keeping the PostgreSQL *service* +available thanks to being able to failover from a primary node to its +secondary within seconds of a failure, an **archiver** addresses a different +failure mode entirely: either the loss of multiple (all) nodes at the same +time, or a data loss that happens while the service is running fine. + +See also :ref:`archiving_architecture` for more details about the archiving +support in pg_auto_failover. + +When an archiver is enabled on a pg_auto_failover architecture in +production, the following operations are covered: + + - Point in Time Recovery can be driven on transient nodes created from the + archives. + + - Disaster Recovery can be implemented by copying the data recovered in a + transient PITR node up to the current primary, a manual operation, or by + reifying the transient PITR node into its own new group in the + formation, allowing to redeploy a new cluster from a selected position + in the WAL history. + + - Archiving nodes may paritipate in the replication quorum, and as they + only implement ``pg_receivewal`` without maintaining a full PGDATA + directory, there is no crash recovery happening on the WAL stream -- it + is often the case that an archiving node would be the first to report + LSN progress. + + - Taking base backup happens on the primary node by default (a live source + setting) and can also be setup as a replay source, meaning that a new + node is created from the latest base backup and instructed to replay all + the WAL that have been archived since this base backup, up to the + current moment in time. The replay source can in turn be setup as a + volatile or a persistent node. + + - It is possible to maintain standby servers that only connect to the + archive, because we have added a way to serve the archives using the + Postgres protocol replication. Such a standby would be named a WARM + standby, even though it can be using WAL streaming, with a cascading hop + in the archives. How archiving nodes participate in failover ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/intro.rst b/docs/intro.rst index a1d8cf24b..84c13676c 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -2,21 +2,27 @@ Introduction to pg_auto_failover ================================ pg_auto_failover is a complete system for operating PostgreSQL in -production, not just an extension. Its ``pg_autoctl`` process runs as its -own pid 1, supervising the Postgres ``postmaster`` underneath it the way -an init system supervises everything else running on a machine; a -dedicated monitor node coordinates state across every node in the -cluster. Together they provide full cluster management with a dynamic -topology: nodes can be added, removed, and reconfigured while the cluster -keeps serving production traffic, whether driven by an operator's own -commands or automatically by the monitor's own health checks. Automated -failover and full high availability are one deliberate configuration -choice this system supports, not the only thing it does. Two modes of -operation are available side by side: the traditional command-driven CLI -(``pg_autoctl create ...``, ``pg_autoctl set ...``), and a specification- -file-driven mode, where a single ``node.ini`` file describes a node's own -desired configuration and ``pg_autoctl node run`` continuously reconciles -reality to match it. +production. Its ``pg_autoctl`` process may runs `pid 1` or init in a +container based environment and supervises the Postgres ``postmaster`` +underneath it. A dedicated monitor node coordinates state across every node +in the cluster: the monitor is a Postgres instance with the +``pgautofailover`` extension installed to implement our inter-node +communication protocol. + +Together they provide full cluster management with a dynamic topology: nodes +can be added, removed, and reconfigured while the cluster keeps serving +production traffic, whether driven by an operator's own commands or +automatically by the monitor's own health checks. + +Automated failover and full high availability can both be implemented and a +production cluster can evolve from simple failover capabilities to enhanced +data protection settings. + +Two modes of operation are available side by side: the traditional +command-driven CLI (``pg_autoctl create ...``, ``pg_autoctl set ...``), and +a specification- file-driven mode, where a single ``node.ini`` file +describes a node's own desired configuration and :ref:`pg_autoctl_node_run` +continuously reconciles reality to match it. .. _ha_dr_backups: @@ -36,30 +42,40 @@ High Availability and Disaster Recovery: One System With pg_auto_failover, High Availability and Disaster Recovery collapse into one system; Backups remains its own concern -Most PostgreSQL setups treat these as two separate problems, solved by two -separate products: an *HA tool* — Patroni, repmgr — watches the live -cluster and promotes a standby when the primary goes away, and a *backup -tool* — pgBackRest, pgBarman — usually entirely disconnected from the -first, periodically archives WAL and base backups somewhere safe, reached -for only once disaster strikes and someone needs to restore to a point in -time. Running both means learning two tools, trusting two different -failure domains, and, very often, discovering only during a real incident -that they were never actually exercised together. - -pg_auto_failover starts from a different question. The goal was never -"have an HA tool" — it was always "don't lose the business's data, and -keep serving it," and high availability and disaster recovery are two -sides of that same problem, best solved by one system designed around it -rather than by gluing together two tools each designed in isolation. The -same monitor that orchestrates failover also tracks every archiver's +With RDBMS such as PostgreSQL the concept of High Availability applies to +the service and also the data. Where most PostgreSQL setups treat these as +two separate problems, solved by two separate products, pg_auto_failover +addresses both HA aspects into a single deployment. + +Postgres backup systems need to be able Point in Time Recovery, which +requires an archiving implemnentation when using Postgres. Also, Disaster +Recovery is built on-top of PITR. As a consequence, most systems are +implementing Disaster Recovery with their backup software solution, not +their High Availability solution. + +Running both solutions together means trusting two different failure +domains, and, very often, discovering only during a real incident that they +were never actually exercised together. + +pg_auto_failover starts from a different question: how to make things so +simple to setup and test that they just work once shipped in production? + +High Availability of the Postgres service and Disaster Recovery of its data +set are two sides of the same problem, best solved by one system designed +around it rather than by gluing together two tools each designed in +isolation. + +The same monitor that orchestrates failover also tracks every archiver's captured WAL and base backups; the same WAL stream and base backups a failover election already depends on to guarantee no data loss are what -disaster recovery, including point-in-time recovery, is built on. High -Availability and Disaster Recovery come from a single package, with a +disaster recovery, including point-in-time recovery, is built on. + +High Availability and Disaster Recovery come from a single package, with a single control plane, rather than from two independently-operated systems -an incident is the first time anyone actually tested together. Backups — -in the narrower sense of long-term retention, cataloguing, and cloud -storage tiers — remain their own concern, typically still handled by a +that are only put to the test when a production incident happens. + +Backups — in the narrower sense of long-term retention, cataloguing, and +cloud storage tiers — remain their own concern, typically still handled by a dedicated tool like pgBackRest or pgBarman. Single Standby Architecture @@ -93,7 +109,7 @@ setting on the *primary* node. Until the *secondary* is back to being monitored healthy, failover and switchover operations are not allowed, preventing data loss. -.. _archiving_architecture: +.. _archiving_and_disaster_recovery: Archiving & Disaster Recovery Architecture ------------------------------------------- @@ -103,40 +119,27 @@ Archiving & Disaster Recovery Architecture pg_auto_failover architecture with a primary, a standby, and an archiver -An **archiver** is a separate physical entity, added on top of any of the -architectures on this page — it applies just as well to the single-standby -setup above as it does to a multi-standby fleet, since it addresses a -different concern: disaster recovery, independent of how many nodes -currently participate in the failover quorum. - -Unlike a standby, an archiver holds no copy of the primary's data directory -and never takes writes or reads for the application. It runs its own -`pg_receivewal`__ continuously against the group's current primary, -capturing every WAL segment into a local cache the moment it's generated, -and periodically produces full base backups from that cache. Both are -reported back to the pg_auto_failover Monitor, the same way a standby -reports its own replication state — so the Monitor can tell an operator, -or a client library, when a given segment has landed durably on enough -archivers to be considered safe (``archiver_quorum``), and where the most -recent base backup lives. - -__ https://www.postgresql.org/docs/current/app-pgreceivewal.html - -The pg_auto_failover Monitor tracks an archiver's participation in a group -as its own node, in the ``archiving`` **archiving node** state — reported -and monitored the same way ``primary``/``secondary`` are, but never a -candidate for promotion or failover: an archiving node holds no -`PGDATA`__ of its own, so there is nothing to promote it *to*. - -__ https://www.postgresql.org/docs/current/app-initdb.html - -Because the archiver keeps a complete, continuously updated copy of the -group's WAL stream and periodic base backups independent of any single -standby, it serves two purposes beyond ordinary high availability: a new -node can be provisioned straight from an archiver's cache instead of -placing extra load on a live primary or secondary, and a group that has -lost every other node still has everything needed to rebuild from scratch, -as long as the archiver itself survived. +An **archiver** is a separate node, added on top of any of the architectures +on this page — it applies just as well to the single-standby setup above as +it does to a multi-standby fleet, since it addresses a different concern: +disaster recovery, independent of how many nodes currently participate in +the failover quorum. + +An archiver then register archiving nodes to groups on formations managed by +the monitor it reports to. An archiving node is running ``pg_receivewal`` to +maintain the Postgres PITR archive storage, and schedules regular base +backup activity using ``pg_basebackup``. The *archiving node* reports to the +pg_auto_failover Monitor and participates in a group Finite State Machine: +it reports its WAL position and can be used in the replication quorum, and +other nodes in the same group can fetch WAL from an *archiving node* (see +REPORT_LSN and FORWARD_LSN states in the :ref:`failover_state_machine`:. + +For that, pg_auto_failover implements its own server-side implementation of +the PostgreSQL replication protocol, a ``pg_walsender`` process that knows +how to serve the data from the archive local on-disk location (or remote +Cloud Object Storage) to the PostgreSQL client replication tools already +listed: ``pg_basebackup`` and `pg_receivewal``, as described in more details +in :ref:`archiving_architecture`. Multiple Standby Architecture ----------------------------- From afa4a9366241c1c5983367b9495755f4a19a6e8b Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:05:59 +0200 Subject: [PATCH 41/55] docs: multi-membership archiver, --region, and node.ini/node run reference fixes archiving-details.rst: rewrite storage layout for per-/ subdirectories, process model for the reconciler + N capture children (diagram regenerated to match), and the "Several formations"/"A Citus formation" sections that previously and incorrectly described a one-archiver-per-formation limitation. New docs/ref/pg_autoctl_create_archiver.rst (this command had no reference page or man page at all before), plus create/show/set basebackup-policy man page entries that were missing from conf.py's own man_pages list despite already having rst pages. archiving.rst: document repeatable --formation and --region on `create archiver`, including the geographically-redundant-DR use case for multiple archivers on one formation. pg_autoctl_node.rst / pg_autoctl_node_run.rst / pg_autoctl_node_start.rst / operations.rst: fix a real, pre-existing inaccuracy found while verifying this session's own reliance on the ini-driven archiver bring-up path -- these pages documented a single "[launch] mode = deferred" key, but the actual implementation (nodespec.c) has always used two independent keys, "create" and "run", each gating a different step (node creation vs. actually starting Postgres/the supervisor). Also: "[node] kind" was missing "archiver" as a valid value entirely. pgaftest.rst: document the new top-level "archiver { }" syntax and its single-formation-at-create-time constraint, and fix its own stale "launch deferred = sleep infinity" claim (containers deferred this way still run the ordinary `pg_autoctl node run ` command; the ini's own [launch] section is what makes it wait). --- docs/archiving-details.rst | 214 ++++++---- docs/archiving.rst | 47 ++- docs/conf.py | 28 ++ docs/operations.rst | 18 +- docs/ref/pg_autoctl_create.rst | 1 + docs/ref/pg_autoctl_create_archiver.rst | 143 +++++++ docs/ref/pg_autoctl_node.rst | 86 ++-- docs/ref/pg_autoctl_node_run.rst | 53 ++- docs/ref/pg_autoctl_node_start.rst | 40 +- docs/ref/pgaftest.rst | 73 +++- docs/tikz/arch-archiver-internals.svg | 527 +++++++++++++----------- docs/tikz/arch-archiver-internals.tex | 13 +- 12 files changed, 848 insertions(+), 395 deletions(-) create mode 100644 docs/ref/pg_autoctl_create_archiver.rst diff --git a/docs/archiving-details.rst b/docs/archiving-details.rst index 60c1d21e6..676861118 100644 --- a/docs/archiving-details.rst +++ b/docs/archiving-details.rst @@ -9,9 +9,9 @@ registering one and attaching a base-backup policy. This page goes one level deeper: what actually moves over the network and onto disk while an archiver is running, and what processes are involved -- the level of detail worth having before sizing storage, deciding where an archiver -should sit on your network, or reasoning about how much of a given -topology (a Citus formation, several independent formations) is actually -covered. +should sit on your network, or reasoning about how a single archiver +covers a whole topology (every group of a Citus formation, or several +independent formations at once). Data flow --------- @@ -23,12 +23,15 @@ small status reports (what's been captured, what's been backed up, how much disk is left), never the data itself: 1. **It streams WAL continuously** from whichever node is currently the - group's primary, over an ordinary PostgreSQL physical replication - connection -- the same kind of connection a standby uses, protected by - its own dedicated replication slot so that nothing already captured is - ever lost, even across a connection that drops and stays down for a - while. If the primary changes, the archiver notices and reconnects to - the new one on its own; no operator action is needed. + primary, over an ordinary PostgreSQL physical replication connection -- + the same kind of connection a standby uses, protected by its own + dedicated replication slot so that nothing already captured is ever + lost, even across a connection that drops and stays down for a while. + If the primary changes, the archiver notices and reconnects to the new + one on its own; no operator action is needed. An archiver attached to + several groups (see `Process model`_ below) runs one of these streams + per group, entirely independently -- one group's primary changing, or + its stream stalling, has no effect on any other group's. 2. **It produces base backups on a schedule**, either as a real ``pg_basebackup`` taken directly from a live node, or entirely on its own: replaying already-captured WAL against a local copy of the last @@ -51,42 +54,58 @@ name, this is never a real Postgres data directory (there is no ``initdb``, nothing ever starts Postgres against it directly); it's a cache root. -One archiver directory belongs to exactly one source -- one primary's -WAL, one base-backup lineage -- so nothing inside it is namespaced by -formation, group, or node: there's nothing else that could ever write -there to collide with. An archiver covering more than one source (several -formations, each its own archiver -- see `Several formations`_ below) -does so as several entirely separate archivers, each with its own -directory, not as one archiver internally partitioning a shared one:: +A single archiver can be attached to more than one (formation, group) at +once -- every group of a Citus formation, or several independent +formations altogether (see `Process model`_ below). Each such membership +gets its own subdirectory, one level under the archiver's own root, named +after the formation and group it belongs to, so that two memberships' +WAL and base backups never collide even though they share one archiver +identity and one root directory:: /var/lib/pgaf/archiver1/ - ├── 000000010000000000000041 - ├── 000000010000000000000042 - ├── 000000010000000000000043.partial - ├── archiver-position ├── archiver-routes.ini - └── basebackups/ - ├── basebackup-20260803T020000Z/ - ├── basebackup-20260804T020000Z/ - └── basebackup-20260805T020000Z/ - -- WAL segments sit directly in this directory, named exactly the way - Postgres itself names them. The most recently-started one carries a - ``.partial`` suffix until it's complete -- archiving doesn't wait for a - segment to fill up before it counts: whatever has already been flushed - into that ``.partial`` file is captured too. -- Each retained base backup is its own subdirectory under - ``basebackups/``, in the same layout an ordinary ``pg_basebackup`` run - by hand would produce. You could point ``postgres -D`` straight at one - of them and it would start -- that's exactly what disaster recovery - relies on. -- ``archiver-position`` and ``archiver-routes.ini`` are small internal - bookkeeping files: coordinates and status, never a copy of any actual - data. Safe to ignore day to day, and not something that needs backing - up itself -- both are regenerated automatically on the archiver's own - next tick. - -Sizing disk for an archiver comes down to two mostly-independent numbers: + ├── default/ + │ └── 0/ + │ ├── 000000010000000000000041 + │ ├── 000000010000000000000042 + │ ├── 000000010000000000000043.partial + │ ├── archiver-position + │ └── basebackups/ + │ ├── basebackup-20260803T020000Z/ + │ ├── basebackup-20260804T020000Z/ + │ └── basebackup-20260805T020000Z/ + └── billing/ + └── 0/ + ├── 000000010000000000000012 + ├── archiver-position + └── basebackups/ + └── basebackup-20260805T030000Z/ + +- WAL segments sit directly under their own ``//`` + subdirectory, named exactly the way Postgres itself names them. The + most recently-started one carries a ``.partial`` suffix until it's + complete -- archiving doesn't wait for a segment to fill up before it + counts: whatever has already been flushed into that ``.partial`` file + is captured too. +- Each retained base backup is its own subdirectory under that + membership's own ``basebackups/``, in the same layout an ordinary + ``pg_basebackup`` run by hand would produce. You could point + ``postgres -D`` straight at one of them and it would start -- that's + exactly what disaster recovery relies on. +- Each membership has its own ``archiver-position`` file, tracking that + group's own captured LSN. ``archiver-routes.ini`` sits at the archiver's + own root instead, one section per membership. All of these are small + internal bookkeeping files -- coordinates and status, never a copy of + any actual data. Safe to ignore day to day, and not something that + needs backing up itself -- all of them are regenerated automatically on + the archiver's own next tick. + +A single-membership archiver (the common case: one formation, one group) +looks the same, just with only one ``//`` subdirectory +under its root. + +Sizing disk for one membership comes down to two mostly-independent +numbers: - **Base backups**: roughly the policy's ``maxcount`` times the size of one backup, since retention prunes anything beyond that count (or @@ -98,6 +117,10 @@ Sizing disk for an archiver comes down to two mostly-independent numbers: retention window keeps more history recoverable, at the cost of more WAL kept around to cover it. +An archiver attached to several groups needs the sum of this across every +membership -- each has its own base-backup policy and its own WAL +retention, sized independently. + Network exposure ----------------- @@ -115,75 +138,98 @@ Process model Once started (``pg_autoctl archiver run``, or ``pg_autoctl node run`` against a ``kind = archiver`` node specification), an archiver supervises -exactly two long-running processes, which in turn each run one more of -their own -- four processes total, all on the one host, none of them -sharing memory. They hand off exactly two small files (`Storage`_ above) -and nothing else: +exactly two long-running processes: ``serve``, and a ``reconciler`` that +in turn keeps one WAL-capture child running per (formation, group) +membership this archiver currently holds -- added and removed on its own +as the archiver is attached to or detached from a formation, no restart +of the archiver itself required. They hand off small files (`Storage`_ +above) and nothing else: .. figure:: ./tikz/arch-archiver-internals.svg - :alt: pg_autoctl archiver run supervises two processes, capture and serve; capture runs pg_receivewal and writes archiver-position, serve reads archiver-position and writes archiver-routes.ini, then runs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches + :alt: pg_autoctl archiver run supervises two processes, reconciler and serve; reconciler forks one capture child per membership, each running pg_receivewal and writing its own archiver-position; serve writes archiver-routes.ini (one section per membership) and runs pg_walsender, which reads the WAL cache and routes file and serves pg_basebackup, streaming standbys, and restore_command fetches - Two supervised processes per archiver, talking to each other only - through two small files on disk + Two supervised top-level processes per archiver; the reconciler forks + one WAL-capture child per membership underneath it :: pg_autoctl archiver run - ├── capture -- keeps WAL streaming alive, reports progress to the monitor - │ └── pg_receivewal - └── serve -- keeps the archiver reachable over the network + ├── reconciler -- keeps the set of running captures in sync with the + │ │ monitor's own membership list for this archiver + │ ├── capture (default/0) -- one per membership, reports its own + │ │ └── pg_receivewal progress to the monitor independently + │ └── capture (billing/0) + │ └── pg_receivewal + └── serve -- keeps the archiver reachable over the network, └── pg_walsender --port 6543 --routes archiver-routes.ini + (serves every membership through the one process) -If either child stops unexpectedly, the parent notices on its next tick +If any child stops unexpectedly, its supervisor notices on its next tick and restarts it -- an archiver recovering from a crashed ``pg_receivewal`` or ``pg_walsender`` needs no operator action, the same -way a keeper recovers a crashed Postgres. +way a keeper recovers a crashed Postgres. A crash of the reconciler +itself is likewise just restarted by the top-level supervisor; on +restart it re-discovers its current memberships from the monitor and +resumes capturing all of them -- a replication slot keeps the WAL a +capture needs regardless of how many times its own consumer reconnects, +so this costs nothing. More or fewer standby nodes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This process tree doesn't change shape based on how many standby nodes -are in the group. WAL capture always talks to whichever node is currently -primary, never to a standby directly, so a two-node group and a -five-node group look identical from the archiver's side. The only place -standby count matters at all is when a base backup is sourced live: with -more healthy standbys available, there are more candidates to pick from -before falling back to the primary -- everything else about the archiver -is unaffected. +A membership's own capture process doesn't change shape based on how many +standby nodes are in its group. WAL capture always talks to whichever +node is currently primary, never to a standby directly, so a two-node +group and a five-node group look identical from the archiver's side. The +only place standby count matters at all is when a base backup is sourced +live: with more healthy standbys available, there are more candidates to +pick from before falling back to the primary -- everything else about +the archiver is unaffected. Several formations ^^^^^^^^^^^^^^^^^^^ -An archiver is attached to one formation at a time. Covering several -formations -- each, say, with its own group of two or three standby -nodes -- means registering one archiver per formation, each with its own -``--pgdata`` directory and its own identity, whether that's several -archiver processes on one host or spread across several hosts: +One archiver can be attached to several formations at once -- each with +its own group of two or three standby nodes, say -- with no need to run a +separate archiver process per formation (see :ref:`archiving_operations` +for the repeated ``--formation`` this takes at creation time). Each +formation attached this way is one more membership, which shows up as one +more ``capture`` child under the reconciler and one more section in +``archiver-routes.ini``; nothing about the archiver's own identity, port, +or ``--pgdata`` root changes: :: - host archiver-a host archiver-b - (attached to formation "default") (attached to formation "billing") - - pg_autoctl archiver run pg_autoctl archiver run - ├── capture -> pg_receivewal ├── capture -> pg_receivewal - └── serve -> pg_walsender └── serve -> pg_walsender - -Each of these process trees is entirely independent -- separate storage -directory, separate WAL stream, separate base-backup schedule, no shared -state of any kind. Losing one has no effect on the others. + pg_autoctl archiver run + ├── reconciler + │ ├── capture (default/0) -> pg_receivewal + │ └── capture (billing/0) -> pg_receivewal + └── serve -> pg_walsender (serves both memberships) + +Each membership's own capture is entirely independent -- separate storage +subdirectory, separate WAL stream, separate base-backup schedule, no +shared state with any other membership. Losing one (its capture process +crashing, say) has no effect on the others; the reconciler restarts just +that one. Running one archiver per formation instead, on separate hosts, +is still a perfectly reasonable choice -- for isolating blast radius, or +spreading load across machines -- just no longer a requirement. A Citus formation ^^^^^^^^^^^^^^^^^^ A Citus formation is really several node groups under one name: the -coordinator's own group, plus one group per worker. Today, registering an -archiver against a Citus formation covers the coordinator's group only -- -worker groups don't yet get their own WAL capture or base backups from -that same archiver. If disaster recovery coverage for worker data matters -to you today, plan around this limitation; formation-wide coverage across -every worker group from a single archiver is on the roadmap but not yet -available. +coordinator's own group, plus one group per worker. Attaching an archiver +to a Citus formation attaches it to every group that already exists in +that formation at the time -- the coordinator's and every worker's -- +each becoming its own membership with its own capture process, exactly +like several independent formations would. A worker group added to the +formation *afterwards* is not picked up on its own: the reconciler only +ever starts capture for memberships the monitor already knows about, and +nothing today re-attaches an archiver to a formation automatically when +that formation grows a new group. Re-running the attach for that +formation covers the new group too (existing memberships are left alone), +and the reconciler picks it up on its own next periodic check, no +archiver restart required. What you can point at an archiver ------------------------------------ diff --git a/docs/archiving.rst b/docs/archiving.rst index 40ec1eab1..4b3de6535 100644 --- a/docs/archiving.rst +++ b/docs/archiving.rst @@ -28,10 +28,44 @@ dedicated verb:: Unlike ``pg_autoctl create postgres``, this does not initialize a PostgreSQL data directory: ``--pgdata`` here names the archiver's local cache directory for captured WAL segments and base backups. Once -registered, the archiver starts its own ``pg_receivewal`` against the -formation's current primary, following it across any later promotion, and -reports its progress to the monitor the same way a standby reports -replication state. +registered, the archiver starts one ``pg_receivewal`` per group of the +formation it's attached to (every worker of a Citus formation included, +not just the coordinator) against each group's current primary, following +it across any later promotion, and reports its progress to the monitor +the same way a standby reports replication state -- see +:ref:`archiving_architecture` for the full process model. + +``--formation`` may be given more than once, to attach the same archiver +to several formations right from the start:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver1 \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver1.example.com \ + --formation default \ + --formation billing \ + --run + +There is currently no command to attach an already-running archiver to a +further formation later on -- covering an additional formation, or a +worker group added to an already-attached Citus formation, requires +specifying every formation up front with a repeated ``--formation``. + +``--region`` labels which data-centre or availability zone this archiver +runs in -- purely informational, shown by ``pg_autoctl watch``. More than +one archiver can be attached to the very same formation at once (each +gets its own independent capture and its own replication slot against +that formation's primary), so registering a second archiver in a +different region against the same formation is how geographically +redundant disaster-recovery coverage is set up:: + + $ pg_autoctl create archiver \ + --pgdata /var/lib/pgaf/archiver-eu \ + --monitor postgresql://autoctl_node@monitor/pg_auto_failover \ + --hostname archiver-eu.example.com \ + --formation default \ + --region eu-west \ + --run The full set of options:: @@ -39,7 +73,10 @@ The full set of options:: --pgctl path to pg_ctl (used to locate pg_receivewal) --monitor pg_auto_failover Monitor Postgres URL --hostname hostname to advertise for this archiver - --formation formation this archiver captures WAL and backups for + --formation formation to attach to, may be repeated + (default: "default") + --region data-centre or availability-zone label for this + archiver (default: "default") --basebackup-policy base-backup production/retention policy to attach (default: "default") --run create node then run pg_autoctl service diff --git a/docs/conf.py b/docs/conf.py index 859a0e831..a2454decd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -471,6 +471,34 @@ def setup(app): [author], 1, ), + ( + "ref/pg_autoctl_create_archiver", + "pg_autoctl create archiver", + "pg_autoctl create archiver", + [author], + 1, + ), + ( + "ref/pg_autoctl_create_basebackup_policy", + "pg_autoctl create basebackup-policy", + "pg_autoctl create basebackup-policy", + [author], + 1, + ), + ( + "ref/pg_autoctl_show_basebackup_policy", + "pg_autoctl show basebackup-policy", + "pg_autoctl show basebackup-policy", + [author], + 1, + ), + ( + "ref/pg_autoctl_set_basebackup_policy", + "pg_autoctl set basebackup-policy", + "pg_autoctl set basebackup-policy", + [author], + 1, + ), ( "ref/pg_autoctl_activate", "pg_autoctl activate", diff --git a/docs/operations.rst b/docs/operations.rst index 928f3ef62..309efb47f 100644 --- a/docs/operations.rst +++ b/docs/operations.rst @@ -583,19 +583,23 @@ A single command then creates the node if absent and starts the supervisor:: This makes ``pg_autoctl node run`` a natural ``CMD`` (Docker) or ``command:`` (Kubernetes) entry-point for every node type. The same image -works for monitor, primary, standby, coordinator, and worker nodes — per-node -differences live entirely in the bind-mounted ini file. +works for monitor, primary, standby, coordinator, worker, and +:ref:`archiving_architecture` archiver nodes — per-node differences live +entirely in the bind-mounted ini file. **Live reconfiguration** — the supervisor watches the ini file. Editing ``candidate_priority``, ``replication_quorum``, ``ssl`` settings, or ``monitor.pguri`` and saving the file is sufficient to converge the running node; no restart is required. -**Ordered startup** — add ``[launch] mode = deferred`` to any node that -should wait for an external signal before initialising. Call -``pg_autoctl node start `` from a sidecar or init container to release -it. This replaces external orchestration for the common case where data -nodes must wait until the monitor is ready. +**Ordered startup** — add ``[launch] create = deferred`` and/or ``run = +deferred`` to any node that should wait for an external signal before +initialising. Call ``pg_autoctl node start `` from a sidecar or +init container to release it (clears both flags). This replaces external +orchestration for the common case where data nodes must wait until the +monitor is ready — or, for an archiver, until its target formation (every +group of it, for a Citus formation) is already registered, since +``pg_autoctl create archiver`` has no retry-until-ready loop of its own. For the full property reference and mutability table see :ref:`pg_autoctl_node`. diff --git a/docs/ref/pg_autoctl_create.rst b/docs/ref/pg_autoctl_create.rst index d88d37374..0e0c10157 100644 --- a/docs/ref/pg_autoctl_create.rst +++ b/docs/ref/pg_autoctl_create.rst @@ -12,5 +12,6 @@ pg_autoctl create - Create a pg_auto_failover node, or formation pg_autoctl_create_postgres pg_autoctl_create_coordinator pg_autoctl_create_worker + pg_autoctl_create_archiver pg_autoctl_create_formation pg_autoctl_create_basebackup_policy diff --git a/docs/ref/pg_autoctl_create_archiver.rst b/docs/ref/pg_autoctl_create_archiver.rst new file mode 100644 index 000000000..4633a034d --- /dev/null +++ b/docs/ref/pg_autoctl_create_archiver.rst @@ -0,0 +1,143 @@ +.. _pg_autoctl_create_archiver: + +pg_autoctl create archiver +=========================== + +pg_autoctl create archiver - Initialize a pg_auto_failover archiver node + +Synopsis +-------- + +The command ``pg_autoctl create archiver`` registers a new **Archiver** +identity on the monitor and attaches it to one or more formations for +Archiving & Disaster Recovery. See :ref:`archiving_architecture` for what +an archiver actually does once running, and :ref:`archiving_operations` +for the operational side of this command. + +:: + + usage: pg_autoctl create archiver + + --pgdata path to the archiver's local data/cache directory + --pgctl path to pg_ctl (used to locate pg_receivewal) + --monitor pg_auto_failover Monitor Postgres URL + --hostname hostname to advertise for this archiver + --name archiver name (default: derived from hostname) + --formation formation to attach to, may be repeated + (default: "default") + --region data-centre or availability-zone label for this + archiver (default: "default") + --basebackup-policy base-backup production/retention policy to attach + (default: "default") + --run create node then run pg_autoctl service + +Description +----------- + +Unlike ``pg_autoctl create postgres`` and the other node kinds, this +command does not initialize a PostgreSQL data directory: ``--pgdata`` +here names the archiver's local cache directory for captured WAL segments +and base backups, and no ``initdb`` ever runs against it. Once +registered, the archiver starts one WAL-capture process per group of +every formation it is attached to (every worker of a Citus formation +included, not just the coordinator), each following its own group's +current primary and reconnecting on its own across any later promotion, +and reports its progress to the monitor the same way an ordinary standby +reports replication state. + +``--formation`` may be given more than once, to attach the same archiver +to several formations from the start -- there is currently no separate +command to attach an already-running archiver to a further formation +later on, so every formation (and, for a Citus formation gaining a new +worker group afterwards, that new group too) needs to be covered by a +repeated ``--formation`` up front. See :ref:`archiving_architecture`'s own +"Several formations" and "A Citus formation" sections for the process +model this produces. + +``--basebackup-policy`` attaches a named base-backup production/retention +policy (see :ref:`pg_autoctl_create_basebackup_policy`) to every formation +given, formation-wide. A formation that never gets a policy of its own +this way, or via :ref:`pg_autoctl_set_basebackup_policy`, falls back to +the schema's own built-in ``default`` policy. + +Options +------- + +The following options are available to ``pg_autoctl create archiver``: + +--pgdata + + Path to the archiver's local cache directory for captured WAL segments + and base backups. Despite the flag's name shared with every other node + kind, this is never a real Postgres data directory. Defaults to the + environment variable ``PGDATA``. + +--pgctl + + Path to the ``pg_ctl`` tool, used only to locate the ``pg_receivewal`` + binary the archiver runs alongside it. Same discovery rules as + :ref:`pg_autoctl_create_postgres`'s own ``--pgctl``. + +--monitor + + Postgres URI used to connect to the monitor. Must use the + ``autoctl_node`` username and target the ``pg_auto_failover`` database + name. It is possible to show the Postgres URI from the monitor node + using the command :ref:`pg_autoctl_show_uri`. + +--hostname + + Hostname or IP address other nodes and clients use to reach this + archiver -- in particular, what a standby's ``primary_conninfo`` or a + ``restore_command`` would point at when using this archiver as a + disaster-recovery source. Same discovery rules as + :ref:`pg_autoctl_create_postgres`'s own ``--hostname`` when not + provided. + +--name + + Archiver name used on the monitor. Defaults to ``--hostname`` when not + provided. + +--formation + + Formation to attach this archiver to. May be repeated to attach the + same archiver to several formations at once; defaults to the + ``default`` formation when not given at all. + +--region + + Free-form label identifying the data-centre or availability zone this + archiver runs in. Purely informational, same convention as + :ref:`pg_autoctl_create_postgres`'s own ``--region``: displayed by + ``pg_autoctl watch``'s archivers section, does not affect any placement + or quorum decision on its own. More than one archiver can be attached + to the very same formation at once -- distinct regions across them is + the intended shape for geographically-redundant disaster-recovery + coverage of one formation. + +--basebackup-policy + + Name of an existing base-backup policy (see + :ref:`pg_autoctl_create_basebackup_policy`) to attach to every + ``--formation`` given, formation-wide. + +--run + + Immediately run the ``pg_autoctl`` archiver service after having + created this node, instead of requiring a separate ``pg_autoctl run`` + invocation afterwards. + +See Also +-------- + +:ref:`pg_autoctl_node_run` provides a declarative alternative to this +command: describe the node once in a ``pg_autoctl_node.ini`` file and run +``pg_autoctl node run`` — it creates the archiver if absent and starts +the supervisor in one step. See :ref:`pg_autoctl_node` for the full +reference. + +:ref:`archiving_architecture` covers what runs once an archiver is +started, and :ref:`archiving_operations` covers the rest of the +day-to-day commands (attaching a policy after the fact, watching what's +captured, rebuilding a node from an archiver's cache). diff --git a/docs/ref/pg_autoctl_node.rst b/docs/ref/pg_autoctl_node.rst index ad4b05840..51e177489 100644 --- a/docs/ref/pg_autoctl_node.rst +++ b/docs/ref/pg_autoctl_node.rst @@ -35,8 +35,8 @@ Description and Kubernetes deployments. The complete node description lives in one ini file that can be version-controlled, templated, and bind-mounted into a container. The same image and the same entry-point work for every node type -(monitor, primary, standby, Citus coordinator, Citus worker); per-node -differences live entirely in the mounted ini file. +(monitor, primary, standby, Citus coordinator, Citus worker, archiver); +per-node differences live entirely in the mounted ini file. The ``pg_autoctl_node.ini`` File -------------------------------- @@ -94,8 +94,9 @@ node is created or started from scratch. ``kind`` - Node role. One of ``postgres``, ``monitor``, ``coordinator``, or - ``worker``. Required; immutable. + Node role. One of ``postgres``, ``monitor``, ``coordinator``, + ``worker``, or ``archiver`` (see :ref:`archiving_architecture`). + Required; immutable. ``name`` @@ -147,7 +148,19 @@ node is created or started from scratch. ``group`` Citus group identifier. ``0`` means coordinator. Defaults to ``0``. - Immutable. + Immutable. Not meaningful for ``kind = archiver`` -- see below. + +For ``kind = archiver``, this section works the same way but with one +real difference in behavior worth knowing: an ordinary node's own +registration retries until its target formation exists (and, once it +does, applies immediately), while ``pg_autoctl create archiver`` (which +this section's ``name`` ultimately drives, once through ``[launch]`` +below) has no such retry -- it attaches to whichever groups already exist +in that formation at the exact moment it runs, and never re-attaches to +groups added afterwards on its own. If the target formation (or, for a +Citus formation, all of its groups) might not exist yet when this node's +own container would otherwise start, use ``[launch]`` below to hold it +back until an operator or orchestrator confirms the formation is ready. ``[settings]`` ^^^^^^^^^^^^^^ @@ -215,11 +228,30 @@ SSL live via ``pg_autoctl enable ssl``. ``[launch]`` ^^^^^^^^^^^^ -``mode`` +Two independent gates, both defaulting to ``immediate``. ``pg_autoctl node +run`` checks ``create`` first (holding back node creation entirely), then +-- once the node exists, whether this run just created it or it already +existed -- checks ``run`` (holding back actually starting Postgres and the +supervisor). Setting only one of the two is meaningful: ``create = +immediate`` with ``run = deferred`` creates the node right away but leaves +it stopped; ``create = deferred`` with ``run = immediate`` (the common +case, usually set together as ``create = deferred`` / ``run = deferred``) +waits before doing anything at all. + +``create`` + + When set to ``deferred``, ``pg_autoctl node run`` polls this file every + 0.5s and waits instead of running ``pg_autoctl create --run`` + immediately. Call ``pg_autoctl node start`` to release it (clears both + ``create`` and ``run`` together). Defaults to ``immediate``. See + :ref:`pg_autoctl_node_start`. + +``run`` - When set to ``deferred``, the node starts a polling loop and waits instead - of creating or starting Postgres immediately. Call ``pg_autoctl node - start`` to release it. Defaults to ``immediate``. See + When set to ``deferred``, ``pg_autoctl node run`` polls this file every + 0.5s and waits (after any pending ``create`` has already resolved) + instead of exec'ing into ``pg_autoctl run`` immediately. Call + ``pg_autoctl node start`` to release it. Defaults to ``immediate``. See :ref:`pg_autoctl_node_start`. ``[formation ]`` @@ -275,25 +307,31 @@ Changing an **immutable** field (``kind``, ``pgdata``, ``hostname``, ``port``, ``auth``, ``pg_hba_lan``) while the node is running is logged as a warning; the value takes effect the next time the node is started. -The ``launch = deferred`` Pattern ----------------------------------- +The Deferred-Launch Pattern +---------------------------- :: [launch] - mode = deferred - -A node configured with ``mode = deferred`` starts a polling loop and waits. -A sidecar container or init script then calls:: - - pg_autoctl node start /etc/pgaf/node.ini - -which rewrites the ini file with ``mode = immediate``. The waiting node -detects the change within the poll interval and proceeds to create or run. -This enables ordered startup without an external orchestrator: the monitor -container can be given ``mode = immediate`` while all data nodes start with -``mode = deferred``, and each data node is released with ``node start`` only -after the monitor is confirmed ready. + create = deferred + run = deferred + +A node configured this way still runs the ordinary ``pg_autoctl node run`` +command, but it starts a polling loop and waits rather than creating or +starting Postgres. A sidecar container or init script then calls:: + + pg_autoctl node start + +which clears both flags (rewriting the ini file with ``create = immediate`` +/ ``run = immediate``). The waiting node detects the change within the +poll interval and proceeds. This enables ordered startup without an +external orchestrator: the monitor container can be given the defaults +(``immediate``) while all data nodes start deferred, and each data node is +released with ``node start`` only after the monitor is confirmed ready -- +or, for an :ref:`archiving_architecture` archiver that needs every group of +its target formation to already exist (a Citus formation's several worker +groups, in particular), only after every one of those groups is confirmed +registered. See Also -------- diff --git a/docs/ref/pg_autoctl_node_run.rst b/docs/ref/pg_autoctl_node_run.rst index c57ab76d9..90f8a6a08 100644 --- a/docs/ref/pg_autoctl_node_run.rst +++ b/docs/ref/pg_autoctl_node_run.rst @@ -22,48 +22,63 @@ Description deployments. Given a ``pg_autoctl_node.ini`` file it: 1. Reads and validates the ini file. -2. If ``[launch] mode = deferred``, polls the file until the section is - removed or changed to ``mode = immediate`` (see ``pg_autoctl node start``). +2. If ``[launch] create = deferred``, polls the file every 0.5s until it's + changed to ``create = immediate`` (see ``pg_autoctl node start``). 3. Checks whether the node already exists (looks for ``pg_autoctl.cfg`` inside ``pgdata``). - - **First start** — builds the ``pg_autoctl create [flags] --run`` - argument list from the ini file and exec's into it, which creates Postgres - and starts the supervisor in one step. - - **Subsequent starts** — applies any mutable setting changes found in the - ini file, then exec's into ``pg_autoctl run --pgdata

``. + - **First start** — runs ``pg_autoctl create [flags]`` (built + from the ini file, *without* ``--run``) to create the node. + - **Subsequent starts** — applies any mutable setting changes found in + the ini file to the already-existing node. -4. Sets the ``PG_AUTOCTL_NODESPEC`` environment variable to the ini file +4. If ``[launch] run = deferred``, polls the file every 0.5s until it's + changed to ``run = immediate``. +5. Exec's into ``pg_autoctl run --pgdata ``, which starts Postgres (if + applicable for this node kind) and the supervisor. +6. Sets the ``PG_AUTOCTL_NODESPEC`` environment variable to the ini file path before exec'ing, so the supervisor can watch the file for live changes to ``[settings]``. +``create`` and ``run`` are independent gates: creating the node (step 3) +and starting it (step 5) can each be deferred on their own. Setting only +``create = deferred`` (leaving ``run`` at its default) creates the node +immediately once released and starts it right away in the same +invocation; setting only ``run = deferred`` creates the node immediately +but leaves it stopped until separately released. + Because the command uses ``execv()``, the pg_autoctl supervisor becomes the direct child process (PID 1 in a container), preserving the standard Unix signal contract — ``SIGTERM`` stops the supervisor cleanly, ``SIGHUP`` reloads configuration. See :ref:`pg_autoctl_stop` for what a graceful ``SIGTERM`` actually does before the node stops. -The ``launch = deferred`` pattern ----------------------------------- +The deferred-launch pattern +---------------------------- The ``[launch]`` section enables ordered startup without an external orchestrator:: [launch] - mode = deferred + create = deferred + run = deferred -A node with ``mode = deferred`` starts the polling loop and waits. A -second container, sidecar, or init script calls:: +A node configured this way starts the polling loop and waits. A second +container, sidecar, or init script calls:: - pg_autoctl node start /etc/pgaf/node.ini + pg_autoctl node start -which rewrites the file with ``mode = immediate``. The waiting node detects -the change and proceeds. This is useful when you need to ensure the monitor -is fully up before any data node attempts registration, or when bringing up -Citus workers in a specific order. +which clears both flags. The waiting node detects the change and +proceeds. This is useful when you need to ensure the monitor is fully up +before any data node attempts registration, when bringing up Citus +workers in a specific order, or -- for an :ref:`archiving_architecture` +archiver -- when its target formation (or, for a Citus formation, every +one of its groups) might not exist yet: ``pg_autoctl create archiver`` +has no retry-until-ready loop of its own the way an ordinary node's +registration does, so it must not run before the formation is ready. See Also -------- :ref:`pg_autoctl_node`, :ref:`pg_autoctl_create_postgres`, -:ref:`pg_autoctl_run` +:ref:`pg_autoctl_create_archiver`, :ref:`pg_autoctl_run` diff --git a/docs/ref/pg_autoctl_node_start.rst b/docs/ref/pg_autoctl_node_start.rst index c70718658..30af6d705 100644 --- a/docs/ref/pg_autoctl_node_start.rst +++ b/docs/ref/pg_autoctl_node_start.rst @@ -3,7 +3,7 @@ pg_autoctl node start ===================== -pg_autoctl node start - Release a node waiting in launch=deferred mode +pg_autoctl node start - Release a node waiting in a deferred launch Synopsis -------- @@ -18,30 +18,38 @@ Synopsis Description ----------- -``pg_autoctl node start`` releases a node that is waiting in -``[launch] mode = deferred``. It rewrites the ini file with -``mode = immediate``; the waiting node detects the change within the poll -interval and proceeds to create or run. +``pg_autoctl node start`` releases a node that is waiting on either or +both of ``[launch] create = deferred`` / ``run = deferred``. It clears +both flags in the ini file (rewriting them to ``immediate``); the waiting +node detects the change within the poll interval and proceeds. This command is idempotent: calling it on a node that is already running -(or has ``mode = immediate``) is a no-op. +(both flags already ``immediate``) is a no-op. -The ``launch = deferred`` Pattern ----------------------------------- +The Deferred-Launch Pattern +---------------------------- -A node configured with ``[launch] mode = deferred`` starts a polling loop -and waits instead of immediately creating or starting Postgres. This -enables ordered startup without an external orchestrator:: +A node configured with ``[launch] create = deferred`` and/or ``run = +deferred`` starts a polling loop and waits instead of immediately +creating or starting Postgres. This enables ordered startup without an +external orchestrator:: # In the ini file for each data node: [launch] - mode = deferred + create = deferred + run = deferred -The monitor can be given ``mode = immediate`` (the default), while data nodes -start with ``mode = deferred``. Once the monitor is confirmed ready, release -each data node:: +The monitor can be left at the defaults (``immediate``), while data nodes +start deferred. Once the monitor is confirmed ready, release each data +node:: - pg_autoctl node start /etc/pgaf/node.ini + pg_autoctl node start + +An :ref:`archiving_architecture` archiver whose target formation (or, for +a Citus formation, one or more of its groups) might not exist yet at +container-start time follows the same pattern -- see +:ref:`pg_autoctl_node_run`'s own note on why an archiver specifically +needs this, unlike an ordinary node. See Also -------- diff --git a/docs/ref/pgaftest.rst b/docs/ref/pgaftest.rst index a4889d511..df64f6e1c 100644 --- a/docs/ref/pgaftest.rst +++ b/docs/ref/pgaftest.rst @@ -438,13 +438,28 @@ Node modifiers: ``candidate-priority `` Failover priority 0–100 (default: 50) ``region `` Data-centre / availability-zone label (``--region``; default: ``default``) -``launch deferred`` Container starts with ``sleep infinity``; - use ``exec node pg_autoctl node start`` +``create deferred`` Container still runs the ordinary + ``pg_autoctl node run `` command, but + the ini's own ``[launch] create = deferred`` + makes it poll and wait rather than actually + registering; release with + ``exec node pg_autoctl node start`` +``launch deferred`` Same mechanism, gating only the final + "start Postgres and the supervisor" step + (``[launch] run = deferred``) -- the node is + still created, just not started yet +``create and launch deferred`` Both gates at once -- the common case, + matching how ``pg_autoctl create + --run`` bundles create+run for an + immediate node ``suspended`` The node-active service never transitions on its own; drive it explicitly with the ``fsm step `` DSL command (see `Suspended nodes`_ below) ``coordinator`` / ``worker group `` Citus role +``archiver`` Archiving & Disaster Recovery node + (see `Top-level archiver nodes`_ below for + the more common declaration form) ``no-monitor`` Standalone node (no monitor) ``listen`` Bind all interfaces (``--listen 0.0.0.0``) ``auth `` Per-node auth override @@ -452,6 +467,60 @@ Node modifiers: ``volume `` Mount a named Docker volume at ```` ============================================ ============================================= +Top-level archiver nodes +~~~~~~~~~~~~~~~~~~~~~~~~~ + +An archiver may also be declared directly inside ``cluster { }``, as its own +``archiver { }`` block -- a sibling of ``monitor``/``formation``, not +nested inside either. This matches the real data model +(``pgautofailover.archiver`` has no formation column at all; it attaches to +one or more formations by name, it isn't a member of any one of them), and +is the recommended form over declaring an ``archiver`` node inline inside a +``formation { }`` block: + +.. code-block:: text + + cluster { + monitor + formation { + node1 + node2 + } + archiver archiver1 { + formation default # required; exactly one + region eu-west # optional; default "default" + create and launch deferred # optional -- see below + } + } + +Internally this is folded into an ordinary node entry in the named +formation's own node list right after parsing, so it launches immediately +by default and supports every modifier above (``region``, the deferred +forms, ...) exactly the same way an ordinary node does -- there is nothing +archiver-specific about the mechanism, only about where it's declared. + +Only one ``formation `` is accepted: ``pg_autoctl create archiver``'s +own ini-driven bootstrap has no notion of attaching to more than one +formation at create time (unlike the CLI's own repeatable ``--formation`` +flag). To cover a second formation, attach it dynamically once the +archiver is already running -- a direct ``sql monitor { SELECT +pgautofailover.archiver_add_formation(...) }`` step is the idiom used by +the ``archiver_multi_formation.pgaf`` spec in this test suite. + +Immediate (the default) launch is only safe when the target formation's +group already exists by the time the archiver's own container starts -- +guaranteed for a formation whose other nodes it already ``depends_on`` +(the ordinary node-ordering rules apply the same way here), but *not* +guaranteed across independent formations or a Citus formation's several +groups, since ``pg_autoctl create archiver`` has no retry-until-ready loop +the way ordinary nodes' registration does. Use ``create and launch +deferred`` plus an explicit ``exec pg_autoctl node start`` step +once every target group is confirmed to exist whenever that ordering +isn't otherwise guaranteed -- see the ``citus_basic_operation.pgaf`` +spec's own archiver step for a worked example (a Citus formation's worker +groups must all be registered before the archiver attaches, so it can +cover every one of them in a single call). + Node registration order ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/tikz/arch-archiver-internals.svg b/docs/tikz/arch-archiver-internals.svg index 41913881f..1c6209957 100644 --- a/docs/tikz/arch-archiver-internals.svg +++ b/docs/tikz/arch-archiver-internals.svg @@ -90,58 +90,61 @@ - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + @@ -153,12 +156,9 @@ - + - - - @@ -189,55 +189,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -315,22 +315,22 @@ - + - + - + - + - + - + @@ -400,7 +400,7 @@ - + @@ -475,52 +475,66 @@ - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - + + - + @@ -552,14 +566,14 @@ - - + + - - - - + + + + @@ -581,140 +595,187 @@ - + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - - - - + + + + - - + + - - - - - - - - + + + + + + + + - + - + - - + + - - - + + + - - + + - + - - - - - - - + + + + + + + - - - - + + + + - - + + - - - - - - - - + + + + + + + + - - - + + + - - - + + + - - - - + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - + + + + + + @@ -730,16 +791,16 @@ - + - - - + + + - + @@ -752,28 +813,28 @@ - - + + - - - - - + + + + + - - + + - + - + - + @@ -789,11 +850,11 @@ - + - + @@ -816,7 +877,7 @@ - + @@ -826,59 +887,59 @@ - - + + - - - - + + + + - - + + - + - + - - - + + + - - + + - - + + - - + + - + - + - + - + @@ -887,32 +948,32 @@ - - - + + + - + - + - + - - + + - + - + @@ -920,18 +981,18 @@ - + - + - + diff --git a/docs/tikz/arch-archiver-internals.tex b/docs/tikz/arch-archiver-internals.tex index ef053380c..d0264dba8 100644 --- a/docs/tikz/arch-archiver-internals.tex +++ b/docs/tikz/arch-archiver-internals.tex @@ -34,19 +34,22 @@ \node (super) at (0,18.6) [proc] {\normalsize \texttt{pg\_autoctl archiver run}\\[2pt]\small two supervised services}; - \node (cap) at (-6.4,14.6) [proc] {\normalsize capture\\[2pt]\small \texttt{service\_archiver\_loop()}}; + \node (recon) at (-6.4,14.6) [proc] {\normalsize reconciler\\[2pt]\small \texttt{service\_archiver\_reconciler\_loop()}}; \node (serve) at (6.4,14.6) [proc] {\normalsize serve\\[2pt]\small \texttt{service\_archiver\_serve\_loop()}}; - \path (super.west) edge[->,thick,out=200,in=90] node[left,pos=0.5] {\small fork} (cap.north) + \path (super.west) edge[->,thick,out=200,in=90] node[left,pos=0.5] {\small fork} (recon.north) (super.east) edge[->,thick,out=-20,in=90] node[right,pos=0.5] {\small fork} (serve.north); - \node (recv) at (-9.4,10.4) [child] {\ttfamily\small pg\_receivewal}; + \node (cap) at (-6.4,11.0) [child] {\ttfamily\small capture}; + \path (recon.south) edge[->,thick] node[left,pos=0.5,align=left] {\small fork,\\[-2pt]\small one per\\[-2pt]\small membership} (cap.north); + + \node (recv) at (-9.4,7.4) [child] {\ttfamily\small pg\_receivewal}; \path (cap.south) edge[->,thick,out=230,in=90] node[left,pos=0.55,align=left] {\small fork + exec\\[-2pt]\scriptsize \texttt{-S }} (recv.north); - \node (walcache) at (-9.4,6.9) [file] {WAL cache / basebackups}; + \node (walcache) at (-9.4,3.9) [file] {WAL cache / basebackups}; \path (recv.south) edge[->,thick] node[right] {\small writes} (walcache.north); - \node (pos) at (-3.1,10.4) [file] {archiver-position}; + \node (pos) at (-3.1,7.4) [file] {archiver-position}; \path (cap.south) edge[->,dashed,color=abox,out=-70,in=110] node[below,sloped] {\small writes} (pos.north); \node (routes) at (3.1,10.4) [file] {archiver-routes.ini}; From c64ff70015b2a4dda5571f57978d5d1082eb855a Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:28:35 +0200 Subject: [PATCH 42/55] tests: add PG19-specific expected output for archiving_schema PostgreSQL 19 (beta) changed pg_lsn's own text output to always zero-pad the lower 32 bits to 8 hex digits (0/500000 -> 0/00500000); 14-18 all still use the shorter, non-padded form. This is an upstream Postgres change, unrelated to this branch's own work -- the affected lines are archiving_schema.sql's pre-existing basebackup/pitr_node_status content. Matches the project's own established per-version override convention (src/monitor/expected/pg19/expected/, already used by several other tests) -- archiving_schema just never had one yet, since nobody had run it against PG19 before. Fixes the "Build run image (PG19)" and "Build test image (PG19)" CI job failures from run 84226284129. --- .../pg19/expected/archiving_schema.out | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 src/monitor/expected/pg19/expected/archiving_schema.out diff --git a/src/monitor/expected/pg19/expected/archiving_schema.out b/src/monitor/expected/pg19/expected/archiving_schema.out new file mode 100644 index 000000000..c6acb9bba --- /dev/null +++ b/src/monitor/expected/pg19/expected/archiving_schema.out @@ -0,0 +1,360 @@ +-- Copyright (c) Microsoft Corporation. All rights reserved. +-- Licensed under the PostgreSQL License. +-- +-- Regression tests for the Archiving & Disaster Recovery schema and its +-- monitor API (milestone 1: schema + monitor API only -- no +-- service_archiver process involved, everything here is exercised via +-- direct SQL calls against the schema alone). See +-- ~/dev/temp/archiving-disaster-recovery.md for the full design. +\x on +-- A dedicated formation, like every other test in this schedule: 'default' +-- is the seed formation CREATE EXTENSION itself creates, and by this point +-- in regress_schedule it may already have real nodes registered into it by +-- earlier tests, so it's the one name this file must NOT reuse. The +-- 'default' basebackup_policy row (also a CREATE EXTENSION seed) is shared +-- on purpose: this file's own focus is exercising it, not creating another. +-- Two ordinary nodes stand in for a group's primary+secondary, inserted +-- directly rather than through register_node()/node_active(): the ordinary +-- node FSM has its own dedicated coverage elsewhere, this file's own focus +-- is the archiver schema layered on top of it. +SELECT pgautofailover.create_formation('archiving_test', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+------------------------------------ +create_formation | (archiving_test,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test', 0, 'node1', 'node1.local', 5432, 111, + 'primary', 'primary'), + ('archiving_test', 0, 'node2', 'node2.local', 5432, 111, + 'secondary', 'secondary'); +-- ── register_archiver ──────────────────────────────────────────────────── +SELECT pgautofailover.register_archiver('archiver1', 'archiver1.local') + AS archiverid \gset +SELECT archiverid, archivername, hostname, region, basebackuppolicyid, + autoregister, maxresidentreplay + FROM pgautofailover.archiver; +-[ RECORD 1 ]------+---------------- +archiverid | 1 +archivername | archiver1 +hostname | archiver1.local +region | default +basebackuppolicyid | 1 +autoregister | t +maxresidentreplay | 1 + +-- the mandatory 'local' storage target is created in the same call +SELECT archiverstorageid, archiverid, storagemethod, storagepath, rcloneconfigid + FROM pgautofailover.archiver_storage; +-[ RECORD 1 ]-----+------ +archiverstorageid | 1 +archiverid | 1 +storagemethod | local +storagepath | +rcloneconfigid | + +-- ── archiver_add_formation: the budget setup's own fan-out ───────────────── +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 40 + +SELECT nodeid, formationid, groupid, nodename, nodehost, nodeport, + goalstate, reportedstate, haspgdata + FROM pgautofailover.node + WHERE haspgdata = false; +-[ RECORD 1 ]-+---------------- +nodeid | 40 +formationid | archiving_test +groupid | 0 +nodename | archiver-1-0 +nodehost | archiver1.local +nodeport | 0 +goalstate | wait_standby +reportedstate | wait_standby +haspgdata | f + +SELECT archivernodeid, archiverid, kind, nodeid + FROM pgautofailover.archiver_node + WHERE kind = 'wal-receiver'; +-[ RECORD 1 ]--+------------- +archivernodeid | 1 +archiverid | 1 +kind | wal-receiver +nodeid | 40 + +SELECT nodeid FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false \gset +-- calling archiver_add_formation() again for the same (archiver, formation) +-- must be a safe no-op -- no error, no duplicate node/archiver_node rows -- +-- since a real archiver's own reconciler calls this periodically to pick up +-- newly-added groups (e.g. a Citus formation growing a worker), not just +-- once at creation time +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test'); +(0 rows) + +SELECT count(*) AS should_still_be_one FROM pgautofailover.node + WHERE formationid = 'archiving_test' AND groupid = 0 AND haspgdata = false; +-[ RECORD 1 ]-------+-- +should_still_be_one | 1 + +-- ── list_archiver_memberships: what an archiver process discovers ────────── +SELECT * FROM pgautofailover.list_archiver_memberships(:archiverid); +-[ RECORD 1 ]--+--------------- +formation_id | archiving_test +group_id | 0 +node_id | 40 +reported_state | wait_standby +goal_state | wait_standby + +-- a second formation attached to the same archiver shows up alongside the +-- first -- this is the multi-membership case: one archiver, several +-- (formation, group) rows, each its own WAL stream and base-backup schedule +SELECT pgautofailover.create_formation('archiving_test_2', 'pgsql', 'postgres', + true, 1); +-[ RECORD 1 ]----+-------------------------------------- +create_formation | (archiving_test_2,pgsql,postgres,t,1) + +INSERT INTO pgautofailover.node + (formationid, groupid, nodename, nodehost, nodeport, sysidentifier, + goalstate, reportedstate) +VALUES ('archiving_test_2', 0, 'node3', 'node3.local', 5432, 222, + 'primary', 'primary'); +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 43 + +SELECT formation_id, group_id + FROM pgautofailover.list_archiver_memberships(:archiverid) + ORDER BY formation_id; +-[ RECORD 1 ]+----------------- +formation_id | archiving_test +group_id | 0 +-[ RECORD 2 ]+----------------- +formation_id | archiving_test_2 +group_id | 0 + +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test_2'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +-- a second archiver serving the same formation/group shares the same +-- (nodehost, nodeport) = (its own hostname, 0) with the first -- the +-- node_nodehost_nodeport_haspgdata_idx partial unique index (scoped to +-- haspgdata rows only) must not reject this. Registered with an explicit, +-- distinct region from archiver1's own default -- this is the intended +-- shape for geographically-redundant DR coverage of the same formation +-- (see archiver.region's own comment); get_archivers() below must surface +-- both regions distinctly. +SELECT pgautofailover.register_archiver('archiver2', 'archiver1.local', + region => 'eu-west') + AS archiverid2 \gset +SELECT * FROM pgautofailover.archiver_add_formation(:archiverid2, 'archiving_test'); +-[ RECORD 1 ]----------+--- +archiver_add_formation | 44 + +SELECT archiver_id, archiver_name, region + FROM pgautofailover.get_archivers('archiving_test') + ORDER BY archiver_id; +-[ RECORD 1 ]-+---------- +archiver_id | 1 +archiver_name | archiver1 +region | default +-[ RECORD 2 ]-+---------- +archiver_id | 2 +archiver_name | archiver2 +region | eu-west + +-- ── WAL capture confirmation: wal_archived() / report_wal_received() ─────── +SELECT pgautofailover.report_wal_received( + :nodeid, '000000010000000000000001', '0/1000000'); +-[ RECORD 1 ]-------+- +report_wal_received | + +-- default archiver_quorum is 1: a single archiver's report already satisfies it +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | t + +-- bump the formation-wide default to 2: the same segment, reported by only +-- one archiver, no longer satisfies quorum +SELECT pgautofailover.set_archiver_policy('archiving_test', NULL, 2, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT pgautofailover.wal_archived('archiving_test', 0, '000000010000000000000001'); +-[ RECORD 1 ]+-- +wal_archived | f + +-- a group-specific override takes precedence over the formation-wide default +SELECT pgautofailover.set_archiver_policy('archiving_test', 0, 1, NULL, NULL); +-[ RECORD 1 ]-------+- +set_archiver_policy | + +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 0); +-[ RECORD 1 ]-------------+-- +archiverquorum | 1 +basebackuppolicyid | +replicationquorumeligible | f + +-- group 1 has no override of its own: falls back to the formation default (2) +SELECT * FROM pgautofailover.get_archiver_policy('archiving_test', 1); +-[ RECORD 1 ]-------------+-- +archiverquorum | 2 +basebackuppolicyid | +replicationquorumeligible | f + +-- ── base backup lifecycle ─────────────────────────────────────────────────── +SELECT pgautofailover.report_basebackup_started( + :archiverid, 'archiving_test', 0, 'base_20260804', 1, '0/500000', 'live') + AS basebackupid \gset +SELECT pgautofailover.report_basebackup_completed( + :basebackupid, '0/1000000', 123456789, + '/var/lib/pgaf-archiver/backups/base_20260804'); +-[ RECORD 1 ]---------------+- +report_basebackup_completed | + +SELECT basebackupid, status, startlsn, endlsn, sizebytes + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+----------- +basebackupid | 1 +status | complete +startlsn | 0/00500000 +endlsn | 0/01000000 +sizebytes | 123456789 + +SELECT basebackupid, formationid, groupid, status + FROM pgautofailover.get_latest_basebackup('archiving_test', 0); +-[ RECORD 1 ]+--------------- +basebackupid | 1 +formationid | archiving_test +groupid | 0 +status | complete + +-- nothing to prune yet: the captured segment's LSN isn't older than this +-- backup's own startlsn +SELECT pgautofailover.prune_archiver_wal('archiving_test', 0); +-[ RECORD 1 ]------+-- +prune_archiver_wal | 0 + +-- report_basebackup_deleted() marks status='deleted' (never a real DELETE) +-- and prunes -- with no 'complete' backup left for this group, there's no +-- anchor point to replay forward from, so nothing prunes either +SELECT pgautofailover.report_basebackup_deleted(:basebackupid); +-[ RECORD 1 ]-------------+- +report_basebackup_deleted | + +SELECT basebackupid, status, deletedat IS NOT NULL AS was_deleted + FROM pgautofailover.basebackup; +-[ RECORD 1 ]+-------- +basebackupid | 1 +status | deleted +was_deleted | t + +-- ── rclone_config + archiver_storage ───────────────────────────────────── +SELECT pgautofailover.create_rclone_config( + 'minio-test', '[minio]' || chr(10) || 'type = s3') + AS rcloneconfigid \gset +SELECT pgautofailover.archiver_add_storage(:archiverid, 'minio-test') + AS archiverstorageid \gset +SELECT archiverstorageid, storagemethod, rcloneconfigid + FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid + ORDER BY archiverstorageid; +-[ RECORD 1 ]-----+------- +archiverstorageid | 1 +storagemethod | local +rcloneconfigid | +-[ RECORD 2 ]-----+------- +archiverstorageid | 3 +storagemethod | rclone +rcloneconfigid | 1 + +-- the mandatory local target cannot be removed +SELECT archiverstorageid AS local_storageid FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid AND storagemethod = 'local' \gset +SELECT pgautofailover.archiver_remove_storage(:local_storageid); +ERROR: archiver_storage 1 does not exist, or is the mandatory local target +CONTEXT: PL/pgSQL function pgautofailover.archiver_remove_storage(bigint) line 8 at RAISE +-- the non-local target can be +SELECT pgautofailover.archiver_remove_storage(:archiverstorageid); +-[ RECORD 1 ]-----------+- +archiver_remove_storage | + +SELECT count(*) AS remaining_storage_targets FROM pgautofailover.archiver_storage + WHERE archiverid = :archiverid; +-[ RECORD 1 ]-------------+-- +remaining_storage_targets | 1 + +-- ── warm-standby archiver_node + maxresidentreplay cap ────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby', + NULL, NULL, 'archiving_test', 0, 'continuous') + AS archivernodeid1 \gset +-- default maxresidentreplay is 1: a second resident warm-standby on the +-- same archiver must be refused +SELECT pgautofailover.create_archiver_node( + :archiverid, 'warm-standby', '/var/lib/pgaf-archiver/standby2', + NULL, NULL, 'archiving_test', 0, 'continuous'); +ERROR: archiver 1 is already at its maxresidentreplay cap (1) +CONTEXT: PL/pgSQL function pgautofailover.create_archiver_node(bigint,pgautofailover.archiver_node_kind,text,text,bigint,text,integer,pgautofailover.archiver_node_cadence,text,pgautofailover.pitr_status) line 18 at RAISE +-- ── PITR lifecycle ─────────────────────────────────────────────────────── +SELECT pgautofailover.create_archiver_node( + :archiverid, 'pitr', '/var/lib/pgaf-archiver/pitr-recovery', + NULL, NULL, NULL, NULL, NULL, NULL, 'restoring') + AS pitrnodeid \gset +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'create', + '{"restore_target_time": "2026-08-04 00:00:00+00"}'::jsonb, + NULL, NULL, 'not paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT pgautofailover.set_archiver_node_pitr_status(:pitrnodeid, 'paused'); +-[ RECORD 1 ]-----------------+- +set_archiver_node_pitr_status | + +SELECT pgautofailover.report_pitr_status( + :pitrnodeid, 'status', NULL, '0/900000'::pg_lsn, '2026-08-04 00:00:05+00', 'paused'); +-[ RECORD 1 ]------+- +report_pitr_status | + +SELECT archivernodeid, archiverid, pitrstatus, lastoperation, + observedlsn, observedpausestate + FROM pgautofailover.pitr_node_status; +-[ RECORD 1 ]------+----------- +archivernodeid | 5 +archiverid | 1 +pitrstatus | paused +lastoperation | status +observedlsn | 0/00900000 +observedpausestate | paused + +-- ── PITR command queue: pops and clears exactly once ──────────────────────── +SELECT pgautofailover.pitr_queue_command(:pitrnodeid, 'promote', NULL); +-[ RECORD 1 ]------+- +pitr_queue_command | + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+-------- +pitr_next_command | promote + +SELECT pgautofailover.pitr_next_command(:pitrnodeid); +-[ RECORD 1 ]-----+----- +pitr_next_command | none + +-- ── archiver_remove_formation cleans up the ARCHIVING node row ────────────── +SELECT pgautofailover.archiver_remove_formation(:archiverid, 'archiving_test'); +-[ RECORD 1 ]-------------+- +archiver_remove_formation | + +SELECT count(*) AS should_be_zero FROM pgautofailover.node + WHERE haspgdata = false AND nodeid = :nodeid; +-[ RECORD 1 ]--+-- +should_be_zero | 0 + +SELECT count(*) AS should_also_be_zero FROM pgautofailover.archiver_node + WHERE archiverid = :archiverid AND kind = 'wal-receiver'; +-[ RECORD 1 ]-------+-- +should_also_be_zero | 0 + From ac78b2b4be2231eb058c1d3328300d0a79ec8e69 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:34:02 +0200 Subject: [PATCH 43/55] Dockerfile: make pg_walsender COPY optional via bracket-glob (stopgap) tests/upgrade builds the current Dockerfile against old-release source trees that predate pg_walsender, so the build stage never produces that binary and the literal COPY fails. The [r] bracket-expression is treated as a glob by BuildKit; an empty match is not an error for COPY (unlike a literal missing path), so this makes the copy optional without touching the old-release source or the upgrade-test tooling. Temporary workaround -- revisit after the release. --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 75293188b..223c1c605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,7 +120,12 @@ COPY --from=build /usr/lib/postgresql/${PGVERSION}/lib/pgautofailover.so \ COPY --from=build /usr/share/postgresql/${PGVERSION}/extension/pgautofailover* \ /usr/share/postgresql/${PGVERSION}/extension/ COPY --from=build /usr/local/bin/pg_autoctl /usr/local/bin/ -COPY --from=build /usr/local/bin/pg_walsender /usr/local/bin/ +# Bracket-glob makes this an optional copy: BuildKit treats [r] as a glob, +# and an empty glob match is not an error for COPY (unlike a literal missing +# path). This lets tests/upgrade build the "current" Dockerfile against an +# old release's source tree, which predates pg_walsender and has no binary +# to copy. Stopgap only -- revisit after the release with a cleaner fix. +COPY --from=build /usr/local/bin/pg_walsende[r] /usr/local/bin/ RUN mkdir -p /var/lib/postgres \ && chown -R docker /var/lib/postgres From 5d976a190ca41fe8c7f1316d67a1fe620c285a4f Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 00:47:57 +0200 Subject: [PATCH 44/55] Fix deferred-archiver ini corruption and multi-membership state-file collision Two real bugs found by Docker verification of the archiver/region redesign: - nodespec_write() had no case for NODE_KIND_ARCHIVER, falling through to "postgres". cli_node_start() (pg_autoctl node start) reads the spec, clears the deferred flags, and rewrites the ini via this same function -- silently downgrading a deferred archiver's own ini to a plain postgres node kind, losing --formation/--region in the process. Root cause of citus_basic_operation.pgaf's test_011 failure. While in there, also stopped nodespec_write() from dropping [settings] region entirely on every round-trip (it was never emitted at all), which would silently erase a deferred node's region on 'node start'. - build_membership_keeper() shallow-copies the archiver-level template Keeper, inheriting its already-computed config.pathnames. The subsequent keeper_config_set_pathnames_from_pgdata() call is then a no-op, since SetConfigFilePath/SetStateFilePath/SetNodesFilePath all skip already-nonempty fields -- so every membership beyond the first silently pointed at the template's (or an earlier membership's) state file instead of its own. Root cause of archiver_multi_formation.pgaf's test_003 failure. Fixed by memset-ing pathnames to zero before recomputing them. Both confirmed via a fresh build + citus_indent + banned-API check; Docker/pgaftest re-verification of the two previously-failing specs to follow. --- src/bin/pg_autoctl/nodespec.c | 19 ++++++++++++++++--- .../pg_autoctl/service_archiver_reconciler.c | 13 +++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/bin/pg_autoctl/nodespec.c b/src/bin/pg_autoctl/nodespec.c index 14903e20f..fb9ddc74a 100644 --- a/src/bin/pg_autoctl/nodespec.c +++ b/src/bin/pg_autoctl/nodespec.c @@ -391,6 +391,12 @@ nodespec_write(const NodeSpec *spec, FILE *out) break; } + case NODE_KIND_ARCHIVER: + { + kindStr = "archiver"; + break; + } + default: { kindStr = "postgres"; @@ -449,14 +455,21 @@ nodespec_write(const NodeSpec *spec, FILE *out) fformat(out, "[settings]\n" "candidate_priority = %d\n" - "replication_quorum = %s\n" + "replication_quorum = %s\n", + spec->candidate_priority, + spec->replication_quorum ? "true" : "false"); + + if (!IS_EMPTY_STRING_BUFFER(spec->region)) + { + fformat(out, "region = %s\n", spec->region); + } + + fformat(out, "\n" "[options]\n" "ssl = %s\n" "auth = %s\n" "pg_hba_lan = %s\n", - spec->candidate_priority, - spec->replication_quorum ? "true" : "false", spec->ssl, spec->auth, spec->pg_hba_lan ? "true" : "false"); diff --git a/src/bin/pg_autoctl/service_archiver_reconciler.c b/src/bin/pg_autoctl/service_archiver_reconciler.c index b52b3370d..e83f3ed5d 100644 --- a/src/bin/pg_autoctl/service_archiver_reconciler.c +++ b/src/bin/pg_autoctl/service_archiver_reconciler.c @@ -326,6 +326,19 @@ build_membership_keeper(Keeper *templateKeeper, ArchiverMembership *membership, return false; } + /* + * The shallow copy above inherited the template keeper's own + * already-computed pathnames (config/state/nodes/pid, derived from + * the archiver-level pgdata). keeper_config_set_pathnames_from_pgdata()'s + * setters each skip an already-nonempty field, so without this reset + * every membership beyond the first would silently keep pointing at + * the template's (or an earlier membership's) files instead of its + * own -- clear them so they're recomputed from this membership's own + * pgdata below. + */ + memset(&(membershipKeeper->config.pathnames), 0, + sizeof(membershipKeeper->config.pathnames)); + if (!keeper_config_set_pathnames_from_pgdata( &(membershipKeeper->config.pathnames), membershipKeeper->config.pgSetup.pgdata)) From bc8610fe54397387500a3cf93f87b00b2a14887e Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 01:57:49 +0200 Subject: [PATCH 45/55] pgaf specs: fix nodename literal mismatch in archiver monitor SQL checks pgautofailover.archiver_add_formation() always synthesizes an ARCHIVING membership row's nodename as 'archiver--', never the plain --name given at create-archiver time. citus_basic_operation. pgaf and archiver_multi_formation.pgaf both asserted WHERE nodename = 'archiver1' in their monitor SQL checks (test_011/ test_012 and test_003/final teardown check respectively), which never matched -- these checks always returned zero rows. Both specs already constrain formationid (and groupid where relevant) and have exactly one archiver each, so WHERE nodename LIKE 'archiver-%' is a safe, unambiguous fix. Also corrects both header comments' stale assumption that these rows share the plain --name. Confirmed via live Docker/pgaftest run: citus_basic_operation.pgaf now passes all 16 steps (previously failed at test_011). archiver_multi_ formation.pgaf's test_003 (the dynamic-attach step this nodename fix covers) now passes too; a separate, unrelated bug in that spec's test_004 (goalstate incorrectly reset to wait_standby for a dynamically-attached second membership) is still under investigation. --- tests/tap/specs/archiver_multi_formation.pgaf | 26 ++++++++++--------- tests/tap/specs/citus_basic_operation.pgaf | 23 +++++++++------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/tests/tap/specs/archiver_multi_formation.pgaf b/tests/tap/specs/archiver_multi_formation.pgaf index ea1d6edce..090683544 100644 --- a/tests/tap/specs/archiver_multi_formation.pgaf +++ b/tests/tap/specs/archiver_multi_formation.pgaf @@ -26,16 +26,18 @@ # each independently electing their own primary/secondary at once. # # archiver1 ends up with two rows in pgautofailover.node once attached to -# both formations (one per (formation, group) membership, both nodename = -# 'archiver1', both groupid = 0 since each formation here has a single, -# plain-Postgres group) -- so the generic "wait until archiver1 state is -# archiving" form (SELECT reportedstate, goalstate FROM pgautofailover.node -# WHERE nodename = $1 LIMIT 1, no ORDER BY -- see test_runner.c's -# monitor_get_node_state()) becomes ambiguous the moment the second -# membership exists: it may observe either row. Every check on archiver1's -# per-membership state after test_003 attaches the second membership goes -# through an explicit `sql monitor` query naming both nodename *and* -# formationid instead. +# both formations (one per (formation, group) membership, both groupid = 0 +# since each formation here has a single, plain-Postgres group). Each row +# is named by archiver_add_formation() itself as 'archiver-- +# ' -- never the plain --name -- so the generic "wait until +# archiver1 state is archiving" form (SELECT reportedstate, goalstate FROM +# pgautofailover.node WHERE nodename = $1 LIMIT 1, no ORDER BY -- see +# test_runner.c's monitor_get_node_state()) wouldn't match either row to +# begin with, and would be ambiguous between the two even if it did. Every +# check on archiver1's per-membership state after test_003 attaches the +# second membership goes through an explicit `sql monitor` query matching +# nodename LIKE 'archiver-%' (safe: this spec has exactly one archiver) +# *and* formationid instead. # # The autoctl_node role has no direct SELECT on pgautofailover.archiver # (granted much later in pgautofailover.sql than the blanket "GRANT SELECT @@ -143,7 +145,7 @@ step test_003_dynamic_attach_to_formation2 { # capture child, and for that child to register/report far enough to # reach ARCHIVING_STATE. sleep 50s - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'formation2'; } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'formation2'; } expect { archiving } } @@ -185,7 +187,7 @@ step test_005_formation1_still_healthy { sleep 15s sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } expect { t } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default'; } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default'; } expect { archiving } } diff --git a/tests/tap/specs/citus_basic_operation.pgaf b/tests/tap/specs/citus_basic_operation.pgaf index 62b9e57c8..3afb916d6 100644 --- a/tests/tap/specs/citus_basic_operation.pgaf +++ b/tests/tap/specs/citus_basic_operation.pgaf @@ -163,22 +163,25 @@ step test_010_perform_failover_coordinator { # just whichever happened to exist first. # # archiver1 ends up with three rows in pgautofailover.node (one -# per group), all sharing nodename = 'archiver1' -- the generic -# "wait until archiver1 state is archiving" form is ambiguous -# once more than one such row exists (test_runner.c's +# per group), each named by archiver_add_formation() itself as +# 'archiver--' (never the plain --name) -- +# the generic "wait until archiver1 state is archiving" form is +# ambiguous once more than one such row exists (test_runner.c's # monitor_get_node_state() does "... WHERE nodename = $1 LIMIT 1", -# no ORDER BY), so every check below names formationid and +# no ORDER BY) and wouldn't match this synthesized name anyway, +# so every check below matches nodename LIKE 'archiver-%' (safe: +# this spec has exactly one archiver) and names formationid and # groupid explicitly instead. # step test_011_bring_up_archiver { exec archiver1 pg_autoctl node start sleep 40s - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 0; } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0; } expect { archiving } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 1; } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1; } expect { archiving } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 2; } + sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2; } expect { archiving } } @@ -202,10 +205,10 @@ step test_012_archiver_captures_every_group { sql worker1a { CREATE TABLE archiver_probe_w1(a int); INSERT INTO archiver_probe_w1 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } sql worker2b { CREATE TABLE archiver_probe_w2(a int); INSERT INTO archiver_probe_w2 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } sleep 15s - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 0; } + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0; } expect { t } - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 1; } + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1; } expect { t } - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename = 'archiver1' AND formationid = 'default' AND groupid = 2; } + sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2; } expect { t } } From 2c8c0cbfb3b78d125838c9822ee7f85176784d25 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 02:08:22 +0200 Subject: [PATCH 46/55] pg_walsender: report the real build's PG_VERSION, not a hardcoded 16.4 WS_SERVER_VERSION was a fixed "16.4" MVP placeholder (already flagged as a known follow-up in its own comment). Real libpq clients (pg_basebackup, pg_receivewal) reject a server reporting a version newer than themselves, so every pg_walsender build for a PG version other than 16 made "create postgres --from-archiver" fail with "pg_basebackup: error: incompatible server version 16.4" -- caught by CI run 84233594160's node schedule on PG14/PG15/PG19. pg_walsender is built once per PGVERSION, against that version's own server headers (Makefile.common's pg_config --includedir-server), so PG_VERSION/PG_VERSION_NUM (from pg_config.h, via postgres_fe.h) are already this build's real target version -- no new plumbing needed, just stop shadowing them with a fixed string. --- src/bin/pg_walsender/defaults.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/bin/pg_walsender/defaults.h b/src/bin/pg_walsender/defaults.h index a2e48bc32..95a90392d 100644 --- a/src/bin/pg_walsender/defaults.h +++ b/src/bin/pg_walsender/defaults.h @@ -15,6 +15,8 @@ #ifndef WS_DEFAULTS_H #define WS_DEFAULTS_H +#include "postgres_fe.h" + #define PG_AUTOCTL_REPLICA_USERNAME "pgautofailover_replicator" #define WS_DEFAULT_PORT 6543 @@ -22,12 +24,15 @@ /* * Reported as the "server_version" startup parameter so that real libpq * clients (pg_basebackup, pg_receivewal) compute a sane PQserverVersion(). - * MVP: a fixed, reasonably-current value; wiring this to the archived - * group's actual tracked pg_version (see the Postgres/Citus version - * tracking prerequisite, milestone 0) is a follow-up, not required for the - * protocol to function. + * PG_VERSION/PG_VERSION_NUM (from pg_config.h, pulled in via postgres_fe.h) + * are this build's own real target version -- pg_walsender is built once + * per PGVERSION, against that version's own server headers (Makefile.common's + * pg_config --includedir-server), so this is already the archived group's + * actual pg_version, not a stand-in for it. A previous fixed "16.4" value + * here made every non-PG16 build report a version mismatch to real + * pg_basebackup/pg_receivewal clients ("incompatible server version"). */ -#define WS_SERVER_VERSION "16.4" -#define WS_SERVER_VERSION_NUM 160004 +#define WS_SERVER_VERSION PG_VERSION +#define WS_SERVER_VERSION_NUM PG_VERSION_NUM #endif /* WS_DEFAULTS_H */ From e1cd4435df62dda824e7c36fe7e57a3a7fa5d1ff Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 02:08:29 +0200 Subject: [PATCH 47/55] ci: split archiver specs out of node.sch into their own schedule Adding archiver_wal_capture/archiver_basebackup_generation/archiver_ basebackup_policy/archiver_bootstrap_and_fast_forward to node.sch pushed every PG version over the pgaftest job's 20-minute step timeout: CI run 84233594160 shows PG16/PG18 timing out outright, and PG14/PG15/PG19 hitting real failures before they'd have gotten there either. node.sch's own header already documents this exact pattern from when the FSM edge-gap specs were split out to node-fsm-gaps.sch. New tests/tap/schedules/archiver.sch runs on every PG version, not PG17 only like node-fsm-gaps.sch: pg_walsender speaks the real Postgres wire protocol, so its correctness is genuinely version- sensitive (see the previous commit's server_version fix, caught by exactly this multi-version coverage). --- .github/workflows/ci.yml | 10 ++++++++++ tests/tap/schedules/archiver.sch | 18 ++++++++++++++++++ tests/tap/schedules/node.sch | 7 +++---- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/tap/schedules/archiver.sch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81027169d..4a9a7b74c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,6 +323,16 @@ jobs: - { PGVERSION: 17, schedule: node } - { PGVERSION: 18, schedule: node } - { PGVERSION: 19, schedule: node } + # archiver: WAL capture, base backups, rebuild-from-archiver — all + # PG versions, since pg_walsender's wire-protocol + # correctness is version-sensitive (see + # tests/tap/schedules/archiver.sch's own header comment) + - { PGVERSION: 14, schedule: archiver } + - { PGVERSION: 15, schedule: archiver } + - { PGVERSION: 16, schedule: archiver } + - { PGVERSION: 17, schedule: archiver } + - { PGVERSION: 18, schedule: archiver } + - { PGVERSION: 19, schedule: archiver } # ssl: enable_ssl, ssl_self_signed, ssl_cert - { PGVERSION: 14, schedule: ssl } - { PGVERSION: 15, schedule: ssl } diff --git a/tests/tap/schedules/archiver.sch b/tests/tap/schedules/archiver.sch new file mode 100644 index 000000000..26d9d229a --- /dev/null +++ b/tests/tap/schedules/archiver.sch @@ -0,0 +1,18 @@ +# Archiving & Disaster Recovery: WAL capture, base backups, and rebuild- +# from-archiver. Split out of node.sch: adding these 4 specs pushed every +# PG version of that already-tight schedule over the CI step's 20-minute +# timeout (CI run 84233594160: PG16/PG18 timed out at 20 minutes, PG14/ +# PG15/PG19 hit real failures before even getting there -- a cumulative +# time-budget overrun on top of real bugs, same pattern node-fsm-gaps.sch +# was split out for). +# +# Unlike node-fsm-gaps.sch, this schedule runs on every PG version rather +# than PG17 only: pg_walsender speaks the real Postgres replication wire +# protocol to real pg_basebackup/pg_receivewal clients, so its correctness +# is genuinely version-sensitive (this exact split was prompted by a +# version-specific bug: a hardcoded server_version made every non-PG16 +# build fail "incompatible server version" against real pg_basebackup). +archiver_wal_capture +archiver_basebackup_generation +archiver_basebackup_policy +archiver_bootstrap_and_fast_forward diff --git a/tests/tap/schedules/node.sch b/tests/tap/schedules/node.sch index d093fd8d6..8149d11cd 100644 --- a/tests/tap/schedules/node.sch +++ b/tests/tap/schedules/node.sch @@ -4,6 +4,9 @@ # edge-gap specs that used to live here were split out to node-fsm-gaps.sch # (PG17-only) once this schedule's own combined runtime started timing out # the CI step on every PG version -- see that file's own header comment. +# The archiver specs that briefly lived here too were split out to +# archiver.sch (all PG versions) for the same reason -- see that file's +# own header comment. create_standby_with_pgdata launch_deferred_set_metadata fsm_step_report_advance @@ -18,7 +21,3 @@ replication_stall_3dc demote_timeout_wait_primary_deadlock timeline_fork_report_lsn_deadlock timeline_fork_3node_auto_detect -archiver_wal_capture -archiver_basebackup_generation -archiver_basebackup_policy -archiver_bootstrap_and_fast_forward From b506b464ba67f457cc752648d9e1fe2c4f41b111 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 02:38:00 +0200 Subject: [PATCH 48/55] monitor: scope ReportAutoFailoverNodeState by nodeid, not (nodehost, nodeport) Every node's periodic node_active() report writes through this function. For ordinary Postgres nodes, (nodehost, nodeport) is a unique key. For an ARCHIVING membership row it isn't: archiver_add_ formation() gives every membership nodeport = 0 (a permanent sentinel -- an archiver has no postmaster to be reachable on) and nodehost = the owning archiver's own hostname, both identical across every (formation, group) membership belonging to the same archiver identity. Once an archiver has 2+ memberships, any one membership's routine report silently overwrote reportedstate on every other membership sharing the same archiver, without touching their goalstate -- leaving reportedstate/goalstate inconsistent and putting the affected membership's local FSM into an unrecoverable crash loop ("does not know how to reach state wait_standby from archiving"). Root-caused live (reproduced twice) via SQL statement logging while debugging archiver_multi_formation.pgaf's test_004 failure. Fixed by scoping the UPDATE on nodeid, the column that's actually unique per row, using the nodeId the caller (node_active_protocol.c) already has on hand from its own node lookup -- no new plumbing needed. Verified: full src/monitor SQL regression suite passes (20/20, including node_active_protocol, archiving_schema, and all 6 concurrent-report tests -- no regression for ordinary-node reporting). Live Docker/pgaftest: the target bug is confirmed fixed end-to-end (both memberships now hold independent, consistent reportedstate/ goalstate, no crash loop); citus_basic_operation.pgaf still 16/16, archiver_wal_capture.pgaf and archiver_budget_architecture_regions.pgaf unaffected. --- src/monitor/node_active_protocol.c | 3 +-- src/monitor/node_metadata.c | 19 +++++++++++++------ src/monitor/node_metadata.h | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/monitor/node_active_protocol.c b/src/monitor/node_active_protocol.c index a1e22db01..d96f66cf1 100644 --- a/src/monitor/node_active_protocol.c +++ b/src/monitor/node_active_protocol.c @@ -524,8 +524,7 @@ NodeActive(char *formationId, AutoFailoverNodeState *currentNodeState) * Report the current state. The state might not have changed, but in * that case we still update the last report time. */ - ReportAutoFailoverNodeState(pgAutoFailoverNode->nodeHost, - pgAutoFailoverNode->nodePort, + ReportAutoFailoverNodeState(pgAutoFailoverNode->nodeId, currentNodeState->replicationState, currentNodeState->pgIsRunning, currentNodeState->pgsrSyncState, diff --git a/src/monitor/node_metadata.c b/src/monitor/node_metadata.c index e3f0988b4..143ca57e8 100644 --- a/src/monitor/node_metadata.c +++ b/src/monitor/node_metadata.c @@ -1592,9 +1592,18 @@ SetNodeGoalState(AutoFailoverNode *pgAutoFailoverNode, * a node. * * We use SPI to automatically handle triggers, function calls, etc. + * + * Scoped by nodeid, not (nodehost, nodeport): an ARCHIVING row's nodeport + * is a permanent 0 sentinel and its nodehost is the owning archiver's own + * hostname, both identical across every (formation, group) membership of + * the same archiver identity (see archiver_add_formation()'s own comment + * on this, pgautofailover.sql). Scoping on that pair used to make any one + * membership's routine report blindly overwrite reportedstate on every + * other membership sharing the same archiver -- nodeid is the one column + * that's actually unique per row. */ void -ReportAutoFailoverNodeState(char *nodeHost, int nodePort, +ReportAutoFailoverNodeState(int64 nodeId, ReplicationState reportedState, bool pgIsRunning, SyncState pgSyncState, int reportedTLI, @@ -1609,8 +1618,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, TEXTOID, /* pg_stat_replication.sync_state */ INT4OID, /* reportedtli */ LSNOID, /* reportedlsn */ - TEXTOID, /* nodehost */ - INT4OID /* nodeport */ + INT8OID /* nodeid */ }; Datum argValues[] = { @@ -1619,8 +1627,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, CStringGetTextDatum(SyncStateToString(pgSyncState)), /* sync_state */ Int32GetDatum(reportedTLI), /* reportedtli */ LSNGetDatum(reportedLSN), /* reportedlsn */ - CStringGetTextDatum(nodeHost), /* nodehost */ - Int32GetDatum(nodePort) /* nodeport */ + Int64GetDatum(nodeId) /* nodeid */ }; const int argCount = sizeof(argValues) / sizeof(argValues[0]); @@ -1645,7 +1652,7 @@ ReportAutoFailoverNodeState(char *nodeHost, int nodePort, " THEN COALESCE(replication_stall_since, now()) " " ELSE NULL " "END " - "WHERE nodehost = $6 AND nodeport = $7"; + "WHERE nodeid = $6"; SPI_connect(); diff --git a/src/monitor/node_metadata.h b/src/monitor/node_metadata.h index db5f5a69f..7c4ca50b9 100644 --- a/src/monitor/node_metadata.h +++ b/src/monitor/node_metadata.h @@ -229,7 +229,7 @@ extern int AddAutoFailoverNode(char *formationId, extern void SetNodeGoalState(AutoFailoverNode *pgAutoFailoverNode, ReplicationState goalState, const char *message); -extern void ReportAutoFailoverNodeState(char *nodeHost, int nodePort, +extern void ReportAutoFailoverNodeState(int64 nodeId, ReplicationState reportedState, bool pgIsRunning, SyncState pgSyncState, From 34ed48d41ac84b85a1d248a81734820b7979ada8 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 02:49:09 +0200 Subject: [PATCH 49/55] archiver_multi_formation.pgaf: fix wrong WAL floor assumption; wire multi-region specs into CI test_004_capture_formation2_wal assumed formation2's archiver slot restart_lsn floor would start at an early, low-numbered segment since it's a fresh formation/group. Live testing (during the ReportAutoFailoverNodeState fix's own verification) showed this is wrong: the floor is segment 3, the same bootstrap-consumption behavior archiver_wal_capture.pgaf's own header comment already documents for the "default" formation -- a freshly-created replication slot's restart_lsn reflects whatever WAL bootstrap/registration itself generated before the slot existed, independent of which formation it belongs to. Segments 1/2 never actually get archived; only 3 onward do. Fixed the two hardcoded segment numbers and corrected the header comment's wrong assumption. Also adds a new tests/tap/schedules/archiver-multi.sch (PG17-only, matching node-fsm-gaps.sch's own rationale -- this is reconciler/SQL logic coverage, not pg_walsender wire-protocol coverage) wiring archiver_multi_formation.pgaf, archiver_budget_architecture_regions. pgaf, and archiver_two_regions.pgaf into CI for the first time -- all three existed as pgaftest specs but were never reachable by any CI schedule until now. Verified: all 3 specs pass in full (5/5, 2/2, 2/2); the combined schedule runs in 3m25s wall-clock, comfortably under the 20-minute CI step timeout that a related schedule in this same PR already blew once. --- .github/workflows/ci.yml | 1 + tests/tap/schedules/archiver-multi.sch | 16 ++++++++++++++++ tests/tap/specs/archiver_multi_formation.pgaf | 19 ++++++++++--------- 3 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 tests/tap/schedules/archiver-multi.sch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a9a7b74c..413d08f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -345,6 +345,7 @@ jobs: - { PGVERSION: 17, schedule: multi-misc } - { PGVERSION: 17, schedule: multi-async } - { PGVERSION: 17, schedule: node-fsm-gaps } + - { PGVERSION: 17, schedule: archiver-multi } - { PGVERSION: 17, schedule: citus-1 } - { PGVERSION: 17, schedule: citus-2 } # citus on PG18 (supported); allow failure until officially validated diff --git a/tests/tap/schedules/archiver-multi.sch b/tests/tap/schedules/archiver-multi.sch new file mode 100644 index 000000000..64342d1d6 --- /dev/null +++ b/tests/tap/schedules/archiver-multi.sch @@ -0,0 +1,16 @@ +# Archiving & Disaster Recovery: dynamic multi-formation attach and +# geo-redundant region coverage. Kept out of archiver.sch (WAL capture, +# base backups, rebuild-from-archiver): these specs exercise the +# reconciler's own membership-diffing and the region column's SQL/CLI +# round-trip -- monitor-side and CLI logic, not pg_walsender's wire +# protocol -- so PG17-only matches node-fsm-gaps.sch's own rationale +# ("this is FSM/logic coverage, not version-specific code paths") rather +# than archiver.sch's all-versions one. Also keeps this schedule light: +# archiver_multi_formation.pgaf alone has a mandatory 50s sleep (one +# reconciler tick, ARCHIVER_RECONCILER_INTERVAL_SECONDS) plus several +# more, and archiver.sch already learned the hard way (this same PR, +# CI run 84233594160) what happens when a schedule's own runtime creeps +# past the 20-minute step timeout. +archiver_multi_formation +archiver_budget_architecture_regions +archiver_two_regions diff --git a/tests/tap/specs/archiver_multi_formation.pgaf b/tests/tap/specs/archiver_multi_formation.pgaf index 090683544..9085c83e1 100644 --- a/tests/tap/specs/archiver_multi_formation.pgaf +++ b/tests/tap/specs/archiver_multi_formation.pgaf @@ -152,13 +152,14 @@ step test_003_dynamic_attach_to_formation2 { # # test_004: confirm formation2's WAL is actually being captured by the # newly-started capture child, not just that the FSM state looks -# right. formation2 is a fresh formation/group -- its own -# archiver slot restart_lsn floor is expected to be an early, -# low-numbered segment (no unrelated bootstrap traffic beyond its -# own two nodes joining), so this switches enough times to be -# independent of the exact floor rather than hardcoding a segment -# number the way test_001 does for the already-characterized -# "default" formation. +# right. formation2 is a fresh formation/group, but its archiver +# slot's restart_lsn floor still lands on segment 3 -- the same +# bootstrap-consumption behavior archiver_wal_capture.pgaf's own +# header comment documents for "default": a freshly-created +# replication slot's restart_lsn already reflects whatever WAL +# bootstrap/registration itself generated before the slot existed, +# independent of which formation it belongs to. Confirmed live +# (segment 1/2 never actually get archived -- only 3 onward do). # step test_004_capture_formation2_wal { @@ -167,9 +168,9 @@ step test_004_capture_formation2_wal { sql node3 { INSERT INTO t2 VALUES (3); } sql node3 { SELECT pg_switch_wal(); } sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000001'); } + sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000003'); } expect { t } - sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000002'); } + sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000004'); } expect { t } } From 7c61fa6c5d8b98950b0624bd4ffba2ad6874dfdb Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 02:57:07 +0200 Subject: [PATCH 50/55] archiver_basebackup_generation.pgaf: fix flaky fixed-sleep before replay-backup assertion test_001_replay_backup_lands slept a fixed 60s then asserted the replay/volatile backup had landed. CI run 84233594160 (PG14 node schedule) failed this with "expected replay, got live": the bootstrap live backup had landed but the subsequent replay backup hadn't yet, under CI resource contention. pgaftest's DSL has no generic SQL-condition polling primitive to switch to (test_spec_parse.y's "wait until" forms are all node-state- specific -- state/assigned-state/stopped/replays-lsn), so this follows archiver_basebackup_policy.pgaf's own established precedent for backup-timing checks: a generous fixed sleep, not a tight one. Bumped 60s to 120s -- scheduling itself is checked every 1s (PG_AUTOCTL_KEEPER_SLEEP_TIME) so the 10s policy interval is noticed promptly, but generating a replay/volatile backup is several real Postgres-instance lifecycles (extract, replay, promote, backup, discard), not a single fast pg_basebackup call, so its own wall-clock cost is the real variable here. Verified: 3/3 consecutive local runs pass, ~120.4s each (no C files touched; docker-check/banned.h.sh clean regardless). --- .../specs/archiver_basebackup_generation.pgaf | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/tap/specs/archiver_basebackup_generation.pgaf b/tests/tap/specs/archiver_basebackup_generation.pgaf index 7090f4495..8d86fa1c6 100644 --- a/tests/tap/specs/archiver_basebackup_generation.pgaf +++ b/tests/tap/specs/archiver_basebackup_generation.pgaf @@ -70,9 +70,27 @@ teardown { # first frequency interval has elapsed) a real replay/volatile # backup both land on their own; check the final state. # +# service_archiver_maybe_generate_basebackup() is checked once +# per service_archiver_loop() tick (PG_AUTOCTL_KEEPER_SLEEP_TIME, +# 1s), so scheduling itself notices the 10s frequency promptly -- +# the real variable cost is generating the replay/volatile backup +# itself once it's due: extract the live backup into a staging +# instance, replay this archiver's own captured WAL forward, +# poll for promotion (wait_for_replay_promotion(), 1s steps), +# pg_basebackup it over loopback, then discard the staging +# instance -- several real Postgres-instance lifecycles, not a +# single fast pg_basebackup call like archiver_basebackup_ +# policy.pgaf's own live-backup cycles. CI run 84233594160 hit +# this exact margin: 60s wasn't enough under load (bootstrap + +# 10s policy interval + a slow replay cycle), so this follows +# that spec's own precedent of a generous fixed sleep (there is +# no SQL-condition polling primitive in pgaftest's DSL -- +# test_spec_parse.y's `wait until` forms are all node-state- +# specific) rather than a tight one. +# step test_001_replay_backup_lands { - sleep 60s + sleep 120s sql monitor { SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0); } expect { replay } sql monitor { SELECT replaymode::text FROM pgautofailover.get_latest_basebackup('default', 0); } From 344a4cc0b950db3ed7346235782f675ef1111193 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 04:43:09 +0200 Subject: [PATCH 51/55] pgaftest: add wait-until-SQL polling primitive + 3 sugar verbs Several archiver specs used a "sleep N seconds, then run one SQL query, then assert" pattern to wait for an async condition (a WAL segment archived, an archiver reaching a state, a base backup landing) instead of actually polling. This caused real CI flakiness -- a fixed sleep either wastes time past a condition that was already true, or isn't long enough under load and produces a flaky failure. Adds a new CMD_WAIT_SQL command: wait until sql { SQL } is { value } [timeout Ns] which polls exec_sql_on_service() every second until its (substring- matched, same semantics as `expect { }`) output contains , or times out. This is the building block; three sugar forms cover the repeated shapes found across the archiver specs, all lowering to CMD_WAIT_SQL at parse time with no new runtime machinery: wait until wal segment "" archived in / wait until archiver state is in [/] wait until basebackup is in / The archiver-state form exists because an ARCHIVING membership row's nodename is always synthesized by archiver_add_formation() as 'archiver--', never the plain --name given at create-archiver time -- the ordinary "wait until state is " form (which matches on nodename = $1) can't see these rows at all, let alone disambiguate more than one membership sharing the same archiver. Grammar changes regenerated via `make generate` (src/bin/pgaftest), zero bison conflicts. docs/ref/pgaftest.rst documents all four forms. Verified: full grammar round-trip via `pgaftest indent` on every new form, a hand-written timeout-path spec confirms clean 5s failure (no hang, no false pass), and end-to-end Docker/pgaftest runs across all 8 specs that use or sit next to this feature (see next commit for the migration itself). --- docs/ref/pgaftest.rst | 43 + src/bin/pgaftest/cli_indent.c | 20 + src/bin/pgaftest/test_runner.c | 61 ++ src/bin/pgaftest/test_spec.h | 11 + src/bin/pgaftest/test_spec_parse.c | 1456 ++++++++++++++----------- src/bin/pgaftest/test_spec_parse.h | 32 +- src/bin/pgaftest/test_spec_parse.y | 124 +++ src/bin/pgaftest/test_spec_scan.c | 1614 ++++++++++++++-------------- src/bin/pgaftest/test_spec_scan.l | 6 + 9 files changed, 1917 insertions(+), 1450 deletions(-) diff --git a/docs/ref/pgaftest.rst b/docs/ref/pgaftest.rst index df64f6e1c..88c6f6f82 100644 --- a/docs/ref/pgaftest.rst +++ b/docs/ref/pgaftest.rst @@ -630,6 +630,49 @@ propagated. expect { } expect error [] +**SQL-condition waits** + +.. code-block:: text + + wait until sql { SELECT ... } is { } [timeout s] + + wait until wal segment "" archived in / [timeout s] + wait until archiver state is in [/] [timeout s] + wait until basebackup is in / [timeout s] + +The generic form polls an arbitrary scalar SQL expression every second +until its (substring-matched, same semantics as ``expect``) result contains +````, or the timeout elapses — the primitive to reach for when a +condition can't be expressed as a node-state wait and none of the sugar +forms below fit. It exists specifically to replace ``sleep s`` followed +by a single ``sql``/``expect`` pair: a fixed sleep either wastes time +waiting past a condition that was already true, or — under CI load — isn't +long enough and produces a flaky failure; polling adapts to how long the +condition actually takes. + +The three sugar forms below are just this primitive with a pre-built SQL +query, covering the checks archiver specs need most: + +- ``wait until wal segment "" archived in /`` + polls ``pgautofailover.wal_archived()``. The segment name must be quoted + (it's all digits, which would otherwise be lexed as an integer and + overflow). +- ``wait until archiver state is in [/]`` polls + an archiver's own ``reportedstate``, matching on + ``nodename LIKE 'archiver-%'`` and ``formationid`` (and ``groupid`` when + given) rather than a plain node name: an ``ARCHIVING`` row's ``nodename`` + is always synthesized by ``archiver_add_formation()`` as + ``archiver--``, never the plain ``--name`` given at + ``create archiver`` time, so the ordinary ``wait until state is + `` form can't see these rows at all, let alone disambiguate more + than one membership sharing the same archiver. Omit the group when the + formation has exactly one archiver membership; give it to disambiguate a + multi-group Citus formation. +- ``wait until basebackup is in + /`` polls ``pgautofailover.get_latest_basebackup()``'s + 2-argument form. For the 3-argument ``preferred_source`` overload, or any + other ``pgautofailover.*`` function, use the generic form directly. + **Network** .. code-block:: text diff --git a/src/bin/pgaftest/cli_indent.c b/src/bin/pgaftest/cli_indent.c index 5f69fd44e..c2048fd78 100644 --- a/src/bin/pgaftest/cli_indent.c +++ b/src/bin/pgaftest/cli_indent.c @@ -900,6 +900,26 @@ print_cmd(FILE *out, const TestCmd *cmd, int indent) break; } + case CMD_WAIT_SQL: + { + /* + * Always the canonical generic form: "wal segment ... archived", + * "archiver state is ...", and "basebackup ... is ..." are all + * sugar folded into a plain SQL/expected pair at parse time, so + * there's no surface syntax left to distinguish and round-trip + * -- this always re-renders as the generic form, same as + * CMD_SQL normalises embedded newlines rather than preserving + * original formatting. + */ + char norm[8192]; + normalize_sql(cmd->args, norm, sizeof(norm)); + + fformat(out, "%*swait until sql %s { %s } is { %s } timeout %ds\n", + indent, "", cmd->service, norm, cmd->expected, + cmd->timeoutSeconds); + break; + } + case CMD_PROMOTE: { fformat(out, "%*spromote", indent, ""); diff --git a/src/bin/pgaftest/test_runner.c b/src/bin/pgaftest/test_runner.c index 40cec5b2e..8bf4d1288 100644 --- a/src/bin/pgaftest/test_runner.c +++ b/src/bin/pgaftest/test_runner.c @@ -116,6 +116,15 @@ test_cmd_print(FILE *f, const TestCmd *cmd, int indent) break; } + case CMD_WAIT_SQL: + { + fprintf(f, /* IGNORE-BANNED */ + "%swait until sql %s { %s } is { %s } timeout %ds\n", + pad, cmd->service, cmd->args, cmd->expected, + cmd->timeoutSeconds); + break; + } + case CMD_EXPECT: { fprintf(f, "%sexpect { %s }\n", pad, cmd->expected); /* IGNORE-BANNED */ @@ -3329,6 +3338,50 @@ runner_exec_cmd(TestRunner *r, TestCmd *cmd, char *errBuf, int errLen) return true; } + case CMD_WAIT_SQL: + { + /* + * Generic SQL-condition poll: re-run cmd->args on cmd->service + * every second until its output contains cmd->expected (same + * substring semantics as CMD_EXPECT) or the timeout elapses. + * Reuses exec_sql_on_service() rather than the LISTEN/NOTIFY + * machinery wait_for_state()/wait_for_states() use: those key + * off specific goalstate/reportedstate convergence events, + * which an arbitrary scalar SQL expression has none of. + */ + time_t deadline = time(NULL) + cmd->timeoutSeconds; + char output[4096] = ""; + bool matched = false; + + for (;;) + { + if (exec_sql_on_service(r, cmd->service, cmd->args, + output, sizeof(output)) && + strstr(output, cmd->expected) != NULL) + { + matched = true; + break; + } + + if (time(NULL) >= deadline) + { + break; + } + + sleep(1); + } + + if (!matched) + { + sformat(errBuf, errLen, + "timeout: sql on %s never matched \"%s\" " + "(last output: \"%s\")", + cmd->service, cmd->expected, output); + return false; + } + return true; + } + case CMD_EXPECT_ERROR: { if (!r->lastSqlFailed) @@ -4338,6 +4391,14 @@ cmd_label(const TestCmd *cmd, char *buf, int len) break; } + case CMD_WAIT_SQL: + { + inline_text(cmd->args, tmp, sizeof(tmp)); + sformat(buf, len, "wait until sql %s { %s } is { %s } timeout %ds", + cmd->service, tmp, cmd->expected, cmd->timeoutSeconds); + break; + } + case CMD_EXPECT: { if (strchr(cmd->expected, '\n')) diff --git a/src/bin/pgaftest/test_spec.h b/src/bin/pgaftest/test_spec.h index 0f6dc90dc..18939dfe4 100644 --- a/src/bin/pgaftest/test_spec.h +++ b/src/bin/pgaftest/test_spec.h @@ -278,6 +278,17 @@ typedef enum TestCmdKind * last-replayed LSN has caught up to that captured * value. service = node to poll, state = source * node to capture the LSN from. */ + CMD_WAIT_SQL, /* wait until sql { SQL } is { value } [timeout Ns] + * — polls an arbitrary scalar SQL expression until + * its (substring-matched, same semantics as + * CMD_EXPECT) result contains , or times + * out. The building block "wait until wal segment + * ... archived", "wait until archiver state is + * ...", and "wait until basebackup ... is ..." are + * all sugar for at parse time -- reach for this + * directly only when none of those fit. + * service = target service (e.g. "monitor"), + * args = SQL text, expected = value to match. */ } TestCmdKind; typedef struct TestCmd diff --git a/src/bin/pgaftest/test_spec_parse.c b/src/bin/pgaftest/test_spec_parse.c index 1dcbdd61d..cf7e61f06 100644 --- a/src/bin/pgaftest/test_spec_parse.c +++ b/src/bin/pgaftest/test_spec_parse.c @@ -180,11 +180,16 @@ T_NOT = 369, T_CONTAINS = 370, T_MATCHES = 371, - T_INTEGER = 372, - T_IDENT = 373, - T_STRING = 374, - T_BLOCK = 375, - T_SHELL_ARGS = 376 + T_WAL = 372, + T_SEGMENT = 373, + T_ARCHIVED = 374, + T_BASEBACKUP = 375, + T_SLASH = 376, + T_INTEGER = 377, + T_IDENT = 378, + T_STRING = 379, + T_BLOCK = 380, + T_SHELL_ARGS = 381 }; #endif /* Tokens. */ @@ -302,11 +307,16 @@ #define T_NOT 369 #define T_CONTAINS 370 #define T_MATCHES 371 -#define T_INTEGER 372 -#define T_IDENT 373 -#define T_STRING 374 -#define T_BLOCK 375 -#define T_SHELL_ARGS 376 +#define T_WAL 372 +#define T_SEGMENT 373 +#define T_ARCHIVED 374 +#define T_BASEBACKUP 375 +#define T_SLASH 376 +#define T_INTEGER 377 +#define T_IDENT 378 +#define T_STRING 379 +#define T_BLOCK 380 +#define T_SHELL_ARGS 381 @@ -486,7 +496,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 193 of yacc.c. */ -#line 490 "test_spec_parse.c" +#line 500 "test_spec_parse.c" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 @@ -499,7 +509,7 @@ typedef union YYSTYPE /* Line 216 of yacc.c. */ -#line 503 "test_spec_parse.c" +#line 513 "test_spec_parse.c" #ifdef short # undef short @@ -714,20 +724,20 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 21 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 609 +#define YYLAST 683 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 122 +#define YYNTOKENS 127 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 69 +#define YYNNTS 71 /* YYNRULES -- Number of rules. */ -#define YYNRULES 226 +#define YYNRULES 234 /* YYNRULES -- Number of states. */ -#define YYNSTATES 376 +#define YYNSTATES 412 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 376 +#define YYMAXUTOK 381 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -772,7 +782,8 @@ static const yytype_uint8 yytranslate[] = 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120, 121 + 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, + 125, 126 }; #if YYDEBUG @@ -795,119 +806,126 @@ static const yytype_uint16 yyprhs[] = 323, 325, 327, 331, 334, 338, 341, 345, 348, 352, 355, 357, 359, 361, 366, 371, 373, 377, 378, 381, 383, 385, 389, 393, 394, 404, 405, 415, 423, 431, - 437, 444, 450, 457, 459, 461, 465, 469, 470, 473, - 476, 481, 482, 485, 489, 496, 503, 510, 517, 521, - 524, 527, 531, 535, 538, 540, 544, 547, 552, 558, - 566, 570, 574, 580, 586, 589, 592, 596, 600, 604, - 609, 613, 617, 621, 622, 628, 634, 638, 643, 649, - 654, 660, 663, 664, 667, 669, 671, 673, 675, 677, - 679, 681, 683, 685, 687, 689, 691, 693, 695, 697, - 699, 701, 703, 705, 707, 709, 711 + 437, 444, 450, 457, 466, 478, 489, 501, 503, 505, + 509, 513, 514, 517, 520, 525, 526, 529, 533, 540, + 547, 554, 561, 565, 568, 571, 575, 579, 582, 584, + 588, 591, 596, 602, 610, 614, 618, 624, 630, 633, + 636, 640, 644, 648, 653, 657, 661, 665, 666, 672, + 678, 682, 687, 693, 698, 704, 707, 708, 711, 713, + 715, 717, 719, 721, 723, 725, 727, 729, 731, 733, + 735, 737, 739, 741, 743, 745, 747, 749, 751, 753, + 755, 757, 759, 761, 762 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { - 123, 0, -1, 124, -1, 123, 124, -1, 125, -1, - 151, -1, 152, -1, 153, -1, 187, -1, -1, 3, - 103, 126, 127, 104, -1, -1, 127, 128, -1, 133, - -1, 134, -1, 136, -1, 137, -1, 135, -1, 138, - -1, 129, -1, 45, -1, 46, -1, -1, 22, 118, - 130, 103, 131, 104, -1, -1, 131, 132, -1, 18, - 118, -1, 47, 118, -1, 47, 119, -1, 27, 77, + 128, 0, -1, 129, -1, 128, 129, -1, 130, -1, + 156, -1, 157, -1, 158, -1, 192, -1, -1, 3, + 103, 131, 132, 104, -1, -1, 132, 133, -1, 138, + -1, 139, -1, 141, -1, 142, -1, 140, -1, 143, + -1, 134, -1, 45, -1, 46, -1, -1, 22, 123, + 135, 103, 136, 104, -1, -1, 136, 137, -1, 18, + 123, -1, 47, 123, -1, 47, 124, -1, 27, 77, 26, 28, -1, 26, 28, -1, 27, 28, -1, 4, - -1, 4, 41, 118, -1, 4, 14, 118, -1, 4, - 37, 117, -1, 4, 38, 119, -1, 4, 118, 26, - 28, -1, 4, 118, 32, 96, -1, 4, 118, 26, - 28, 38, 119, -1, 13, 119, -1, 13, 118, -1, - 44, 118, -1, 44, 119, -1, 15, 118, -1, 16, - 118, -1, 17, 118, -1, -1, 18, 139, 140, 103, - 143, 104, -1, -1, 140, 142, -1, 118, -1, 119, - -1, 16, -1, 4, -1, 5, -1, 141, -1, 19, - 117, -1, 57, 30, -1, -1, 143, 146, -1, 118, - -1, 4, -1, -1, -1, 144, 145, 147, 149, -1, - -1, 5, 118, 145, 148, 103, 149, 104, -1, -1, - 149, 150, -1, 20, -1, 21, -1, 22, -1, 23, + -1, 4, 41, 123, -1, 4, 14, 123, -1, 4, + 37, 122, -1, 4, 38, 124, -1, 4, 123, 26, + 28, -1, 4, 123, 32, 96, -1, 4, 123, 26, + 28, 38, 124, -1, 13, 124, -1, 13, 123, -1, + 44, 123, -1, 44, 124, -1, 15, 123, -1, 16, + 123, -1, 17, 123, -1, -1, 18, 144, 145, 103, + 148, 104, -1, -1, 145, 147, -1, 123, -1, 124, + -1, 16, -1, 4, -1, 5, -1, 146, -1, 19, + 122, -1, 57, 30, -1, -1, 148, 151, -1, 123, + -1, 4, -1, -1, -1, 149, 150, 152, 154, -1, + -1, 5, 123, 150, 153, 103, 154, 104, -1, -1, + 154, 155, -1, 20, -1, 21, -1, 22, -1, 23, -1, 24, -1, 25, -1, 28, -1, 26, 28, -1, 27, 28, -1, 27, 77, 26, 28, -1, 26, 29, - -1, 29, -1, 34, -1, 35, -1, 36, 117, -1, - 47, 118, -1, 47, 119, -1, 102, 117, -1, 37, - 117, -1, 40, 118, -1, 41, 118, -1, 15, 118, - -1, 16, 118, -1, 17, 118, -1, 42, 31, -1, - 42, 30, -1, 43, 119, -1, 39, 119, -1, 33, - 118, 118, -1, 33, 118, 119, -1, 8, 154, -1, - 9, 154, -1, 10, 190, 154, -1, 103, 155, 104, - -1, -1, 155, 156, -1, 157, -1, 163, -1, 170, - -1, 171, -1, 172, -1, 173, -1, 175, -1, 176, - -1, 178, -1, 179, -1, 180, -1, 181, -1, 184, - -1, 185, -1, 186, -1, 177, -1, 70, 118, 121, - -1, 70, 118, -1, 71, 118, 121, -1, 71, 118, - -1, 72, 118, 121, -1, 72, 118, -1, 73, 118, - 121, -1, 73, 118, -1, 73, -1, 12, -1, 78, - -1, 118, 99, 158, 189, -1, 118, 99, 158, 118, - -1, 159, -1, 160, 77, 159, -1, -1, 109, 162, - -1, 189, -1, 118, -1, 162, 105, 189, -1, 162, - 105, 118, -1, -1, 74, 75, 118, 99, 158, 189, - 164, 161, 169, -1, -1, 74, 75, 118, 99, 158, - 118, 165, 161, 169, -1, 74, 75, 118, 100, 158, - 189, 169, -1, 74, 75, 118, 100, 158, 118, 169, - -1, 74, 75, 118, 96, 169, -1, 74, 75, 118, - 80, 118, 169, -1, 74, 75, 166, 167, 169, -1, - 74, 75, 159, 77, 160, 169, -1, 189, -1, 118, - -1, 166, 105, 189, -1, 166, 105, 118, -1, -1, - 101, 168, -1, 102, 117, -1, 168, 105, 102, 117, - -1, -1, 76, 117, -1, 79, 76, 117, -1, 81, - 118, 99, 158, 189, 169, -1, 81, 118, 99, 158, - 118, 169, -1, 81, 118, 100, 158, 189, 169, -1, - 81, 118, 100, 158, 118, 169, -1, 82, 118, 120, - -1, 83, 120, -1, 83, 84, -1, 83, 84, 118, - -1, 83, 84, 117, -1, 85, 174, -1, 118, -1, - 174, 105, 118, -1, 86, 87, -1, 86, 87, 102, - 117, -1, 86, 87, 101, 18, 118, -1, 86, 87, - 101, 18, 118, 102, 117, -1, 88, 89, 118, -1, - 88, 90, 118, -1, 48, 110, 118, 118, 118, -1, - 48, 111, 118, 118, 118, -1, 91, 117, -1, 92, - 93, -1, 92, 94, 118, -1, 92, 95, 118, -1, - 92, 97, 118, -1, 92, 98, 118, 121, -1, 95, - 106, 144, -1, 94, 106, 144, -1, 112, 10, 144, - -1, -1, 108, 183, 103, 155, 104, -1, 81, 144, - 107, 189, 182, -1, 110, 118, 118, -1, 113, 118, - 115, 119, -1, 113, 118, 114, 115, 119, -1, 113, - 118, 116, 119, -1, 113, 118, 114, 116, 119, -1, - 11, 188, -1, -1, 188, 190, -1, 49, -1, 50, - -1, 51, -1, 52, -1, 53, -1, 54, -1, 55, - -1, 56, -1, 57, -1, 58, -1, 59, -1, 60, - -1, 61, -1, 62, -1, 63, -1, 64, -1, 65, - -1, 66, -1, 67, -1, 68, -1, 69, -1, 118, - -1, 119, -1 + -1, 29, -1, 34, -1, 35, -1, 36, 122, -1, + 47, 123, -1, 47, 124, -1, 102, 122, -1, 37, + 122, -1, 40, 123, -1, 41, 123, -1, 15, 123, + -1, 16, 123, -1, 17, 123, -1, 42, 31, -1, + 42, 30, -1, 43, 124, -1, 39, 124, -1, 33, + 123, 123, -1, 33, 123, 124, -1, 8, 159, -1, + 9, 159, -1, 10, 195, 159, -1, 103, 160, 104, + -1, -1, 160, 161, -1, 162, -1, 168, -1, 175, + -1, 176, -1, 177, -1, 178, -1, 180, -1, 181, + -1, 183, -1, 184, -1, 185, -1, 186, -1, 189, + -1, 190, -1, 191, -1, 182, -1, 70, 123, 126, + -1, 70, 123, -1, 71, 123, 126, -1, 71, 123, + -1, 72, 123, 126, -1, 72, 123, -1, 73, 123, + 126, -1, 73, 123, -1, 73, -1, 12, -1, 78, + -1, 123, 99, 163, 194, -1, 123, 99, 163, 123, + -1, 164, -1, 165, 77, 164, -1, -1, 109, 167, + -1, 194, -1, 123, -1, 167, 105, 194, -1, 167, + 105, 123, -1, -1, 74, 75, 123, 99, 163, 194, + 169, 166, 174, -1, -1, 74, 75, 123, 99, 163, + 123, 170, 166, 174, -1, 74, 75, 123, 100, 163, + 194, 174, -1, 74, 75, 123, 100, 163, 123, 174, + -1, 74, 75, 123, 96, 174, -1, 74, 75, 123, + 80, 123, 174, -1, 74, 75, 171, 172, 174, -1, + 74, 75, 164, 77, 165, 174, -1, 74, 75, 82, + 123, 125, 78, 125, 174, -1, 74, 75, 117, 118, + 124, 119, 101, 123, 121, 122, 174, -1, 74, 75, + 22, 99, 163, 196, 101, 123, 197, 174, -1, 74, + 75, 120, 123, 78, 123, 101, 123, 121, 122, 174, + -1, 194, -1, 123, -1, 171, 105, 194, -1, 171, + 105, 123, -1, -1, 101, 173, -1, 102, 122, -1, + 173, 105, 102, 122, -1, -1, 76, 122, -1, 79, + 76, 122, -1, 81, 123, 99, 163, 194, 174, -1, + 81, 123, 99, 163, 123, 174, -1, 81, 123, 100, + 163, 194, 174, -1, 81, 123, 100, 163, 123, 174, + -1, 82, 123, 125, -1, 83, 125, -1, 83, 84, + -1, 83, 84, 123, -1, 83, 84, 122, -1, 85, + 179, -1, 123, -1, 179, 105, 123, -1, 86, 87, + -1, 86, 87, 102, 122, -1, 86, 87, 101, 18, + 123, -1, 86, 87, 101, 18, 123, 102, 122, -1, + 88, 89, 123, -1, 88, 90, 123, -1, 48, 110, + 123, 123, 123, -1, 48, 111, 123, 123, 123, -1, + 91, 122, -1, 92, 93, -1, 92, 94, 123, -1, + 92, 95, 123, -1, 92, 97, 123, -1, 92, 98, + 123, 126, -1, 95, 106, 149, -1, 94, 106, 149, + -1, 112, 10, 149, -1, -1, 108, 188, 103, 160, + 104, -1, 81, 149, 107, 194, 187, -1, 110, 123, + 123, -1, 113, 123, 115, 124, -1, 113, 123, 114, + 115, 124, -1, 113, 123, 116, 124, -1, 113, 123, + 114, 116, 124, -1, 11, 193, -1, -1, 193, 195, + -1, 49, -1, 50, -1, 51, -1, 52, -1, 53, + -1, 54, -1, 55, -1, 56, -1, 57, -1, 58, + -1, 59, -1, 60, -1, 61, -1, 62, -1, 63, + -1, 64, -1, 65, -1, 66, -1, 67, -1, 68, + -1, 69, -1, 123, -1, 124, -1, 194, -1, 123, + -1, -1, 121, 122, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 217, 217, 218, 222, 223, 224, 225, 226, 239, - 238, 248, 250, 254, 255, 256, 257, 258, 259, 260, - 261, 262, 285, 284, 302, 304, 308, 322, 327, 332, - 339, 343, 359, 363, 370, 377, 383, 390, 397, 404, - 417, 423, 433, 439, 449, 459, 465, 476, 475, 492, - 494, 503, 504, 505, 506, 507, 511, 516, 520, 526, - 528, 547, 548, 557, 574, 573, 581, 580, 588, 590, - 594, 599, 604, 608, 612, 616, 620, 626, 631, 635, - 640, 644, 648, 652, 656, 660, 665, 670, 674, 678, - 684, 690, 695, 700, 705, 709, 713, 719, 725, 739, - 760, 767, 778, 796, 811, 814, 822, 823, 824, 825, - 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, - 836, 837, 851, 858, 864, 871, 877, 884, 890, 898, - 904, 931, 931, 942, 957, 975, 976, 991, 993, 997, - 1005, 1013, 1020, 1032, 1031, 1043, 1042, 1053, 1062, 1071, - 1085, 1093, 1107, 1122, 1128, 1135, 1141, 1154, 1156, 1160, - 1165, 1173, 1174, 1175, 1186, 1194, 1202, 1210, 1228, 1243, - 1250, 1254, 1260, 1273, 1281, 1289, 1310, 1317, 1324, 1332, - 1348, 1354, 1375, 1383, 1398, 1412, 1416, 1422, 1428, 1454, - 1488, 1494, 1515, 1532, 1532, 1537, 1556, 1581, 1590, 1599, - 1608, 1624, 1627, 1629, 1651, 1652, 1653, 1654, 1655, 1656, - 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, - 1667, 1668, 1669, 1670, 1671, 1679, 1680 + 0, 220, 220, 221, 225, 226, 227, 228, 229, 242, + 241, 251, 253, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 288, 287, 305, 307, 311, 325, 330, 335, + 342, 346, 362, 366, 373, 380, 386, 393, 400, 407, + 420, 426, 436, 442, 452, 462, 468, 479, 478, 495, + 497, 506, 507, 508, 509, 510, 514, 519, 523, 529, + 531, 550, 551, 560, 577, 576, 584, 583, 591, 593, + 597, 602, 607, 611, 615, 619, 623, 629, 634, 638, + 643, 647, 651, 655, 659, 663, 668, 673, 677, 681, + 687, 693, 698, 703, 708, 712, 716, 722, 728, 742, + 763, 770, 781, 799, 814, 817, 825, 826, 827, 828, + 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, + 839, 840, 854, 861, 867, 874, 880, 887, 893, 901, + 907, 934, 934, 945, 960, 978, 979, 994, 996, 1000, + 1008, 1016, 1023, 1035, 1034, 1046, 1045, 1056, 1065, 1074, + 1088, 1096, 1110, 1125, 1142, 1166, 1195, 1225, 1231, 1238, + 1244, 1257, 1259, 1263, 1268, 1276, 1277, 1278, 1289, 1297, + 1305, 1313, 1331, 1346, 1353, 1357, 1363, 1376, 1384, 1392, + 1413, 1420, 1427, 1435, 1451, 1457, 1478, 1486, 1501, 1515, + 1519, 1525, 1531, 1557, 1591, 1597, 1618, 1635, 1635, 1640, + 1659, 1684, 1693, 1702, 1711, 1727, 1730, 1732, 1754, 1755, + 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, + 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1782, + 1783, 1794, 1795, 1803, 1804 }; #endif @@ -940,23 +958,25 @@ static const char *const yytname[] = "T_DOWN", "T_START", "T_STOP", "T_STOPPED", "T_KILL", "T_INJECT", "T_STATE", "T_ASSIGNED_STATE", "T_IN", "T_GROUP", "T_LBRACE", "T_RBRACE", "T_COMMA", "T_POSTGRES", "T_STAYS", "T_WHILE", "T_THROUGH", "T_SET", - "T_GET", "T_FSM", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", - "T_INTEGER", "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", - "spec", "spec_item", "cluster_block", "@1", "cluster_item_list", - "cluster_item", "archiver_block", "@2", "archiver_opt_list", - "archiver_opt", "monitor_line", "image_line", "extension_version_line", - "ssl_line", "auth_line", "formation_block", "@3", "formation_opt_list", - "bare_name", "formation_opt", "node_list", "node_name", "init_node_slot", - "node_line", "@4", "@5", "node_opt_list", "node_opt", "setup_block", - "teardown_block", "named_step", "cmd_block", "cmd_list", "step_cmd", - "exec_cmd", "state_op", "wait_multi_condition", - "wait_multi_condition_list", "opt_passing_through", "pass_state_list", - "wait_cmd", "@6", "@7", "state_name_list", "opt_in_group", "group_items", - "opt_timeout", "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", - "promote_list", "perform_cmd", "network_cmd", "nodeini_cmd", "sleep_cmd", - "compose_cmd", "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@8", + "T_GET", "T_FSM", "T_LOGS", "T_NOT", "T_CONTAINS", "T_MATCHES", "T_WAL", + "T_SEGMENT", "T_ARCHIVED", "T_BASEBACKUP", "T_SLASH", "T_INTEGER", + "T_IDENT", "T_STRING", "T_BLOCK", "T_SHELL_ARGS", "$accept", "spec", + "spec_item", "cluster_block", "@1", "cluster_item_list", "cluster_item", + "archiver_block", "@2", "archiver_opt_list", "archiver_opt", + "monitor_line", "image_line", "extension_version_line", "ssl_line", + "auth_line", "formation_block", "@3", "formation_opt_list", "bare_name", + "formation_opt", "node_list", "node_name", "init_node_slot", "node_line", + "@4", "@5", "node_opt_list", "node_opt", "setup_block", "teardown_block", + "named_step", "cmd_block", "cmd_list", "step_cmd", "exec_cmd", + "state_op", "wait_multi_condition", "wait_multi_condition_list", + "opt_passing_through", "pass_state_list", "wait_cmd", "@6", "@7", + "state_name_list", "opt_in_group", "group_items", "opt_timeout", + "assert_cmd", "sql_cmd", "expect_cmd", "promote_cmd", "promote_list", + "perform_cmd", "network_cmd", "nodeini_cmd", "sleep_cmd", "compose_cmd", + "postgres_ctl_cmd", "fsm_step_cmd", "while_body", "@8", "stays_while_cmd", "set_monitor_cmd", "logs_cmd", "sequence_block", - "sequence_names", "fsm_state", "ident_or_string", 0 + "sequence_names", "fsm_state", "ident_or_string", "wait_state_name", + "opt_wait_group", 0 }; #endif @@ -977,36 +997,37 @@ static const yytype_uint16 yytoknum[] = 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, - 375, 376 + 375, 376, 377, 378, 379, 380, 381 }; # endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint8 yyr1[] = { - 0, 122, 123, 123, 124, 124, 124, 124, 124, 126, - 125, 127, 127, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 130, 129, 131, 131, 132, 132, 132, 132, - 132, 132, 133, 133, 133, 133, 133, 133, 133, 133, - 134, 134, 135, 135, 136, 137, 137, 139, 138, 140, - 140, 141, 141, 141, 141, 141, 142, 142, 142, 143, - 143, 144, 144, 145, 147, 146, 148, 146, 149, 149, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 151, 152, 153, 154, 155, 155, 156, 156, 156, 156, - 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, - 156, 156, 157, 157, 157, 157, 157, 157, 157, 157, - 157, 158, 158, 159, 159, 160, 160, 161, 161, 162, - 162, 162, 162, 164, 163, 165, 163, 163, 163, 163, - 163, 163, 163, 166, 166, 166, 166, 167, 167, 168, - 168, 169, 169, 169, 170, 170, 170, 170, 171, 172, - 172, 172, 172, 173, 174, 174, 175, 175, 175, 175, - 176, 176, 177, 177, 178, 179, 179, 179, 179, 179, - 180, 180, 181, 183, 182, 184, 185, 186, 186, 186, - 186, 187, 188, 188, 189, 189, 189, 189, 189, 189, - 189, 189, 189, 189, 189, 189, 189, 189, 189, 189, - 189, 189, 189, 189, 189, 190, 190 + 0, 127, 128, 128, 129, 129, 129, 129, 129, 131, + 130, 132, 132, 133, 133, 133, 133, 133, 133, 133, + 133, 133, 135, 134, 136, 136, 137, 137, 137, 137, + 137, 137, 138, 138, 138, 138, 138, 138, 138, 138, + 139, 139, 140, 140, 141, 142, 142, 144, 143, 145, + 145, 146, 146, 146, 146, 146, 147, 147, 147, 148, + 148, 149, 149, 150, 152, 151, 153, 151, 154, 154, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 156, 157, 158, 159, 160, 160, 161, 161, 161, 161, + 161, 161, 161, 161, 161, 161, 161, 161, 161, 161, + 161, 161, 162, 162, 162, 162, 162, 162, 162, 162, + 162, 163, 163, 164, 164, 165, 165, 166, 166, 167, + 167, 167, 167, 169, 168, 170, 168, 168, 168, 168, + 168, 168, 168, 168, 168, 168, 168, 171, 171, 171, + 171, 172, 172, 173, 173, 174, 174, 174, 175, 175, + 175, 175, 176, 177, 177, 177, 177, 178, 179, 179, + 180, 180, 180, 180, 181, 181, 182, 182, 183, 184, + 184, 184, 184, 184, 185, 185, 186, 188, 187, 189, + 190, 191, 191, 191, 191, 192, 193, 193, 194, 194, + 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, + 194, 194, 194, 194, 194, 194, 194, 194, 194, 195, + 195, 196, 196, 197, 197 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1027,14 +1048,15 @@ static const yytype_uint8 yyr2[] = 1, 1, 3, 2, 3, 2, 3, 2, 3, 2, 1, 1, 1, 4, 4, 1, 3, 0, 2, 1, 1, 3, 3, 0, 9, 0, 9, 7, 7, 5, - 6, 5, 6, 1, 1, 3, 3, 0, 2, 2, - 4, 0, 2, 3, 6, 6, 6, 6, 3, 2, - 2, 3, 3, 2, 1, 3, 2, 4, 5, 7, - 3, 3, 5, 5, 2, 2, 3, 3, 3, 4, - 3, 3, 3, 0, 5, 5, 3, 4, 5, 4, - 5, 2, 0, 2, 1, 1, 1, 1, 1, 1, + 6, 5, 6, 8, 11, 10, 11, 1, 1, 3, + 3, 0, 2, 2, 4, 0, 2, 3, 6, 6, + 6, 6, 3, 2, 2, 3, 3, 2, 1, 3, + 2, 4, 5, 7, 3, 3, 5, 5, 2, 2, + 3, 3, 3, 4, 3, 3, 3, 0, 5, 5, + 3, 4, 5, 4, 5, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 2 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -1042,113 +1064,123 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint8 yydefact[] = { - 0, 0, 0, 0, 0, 202, 0, 2, 4, 5, - 6, 7, 8, 9, 104, 100, 101, 225, 226, 0, - 201, 1, 3, 11, 0, 102, 203, 0, 0, 0, + 0, 0, 0, 0, 0, 206, 0, 2, 4, 5, + 6, 7, 8, 9, 104, 100, 101, 229, 230, 0, + 205, 1, 3, 11, 0, 102, 207, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 105, 106, 107, 108, 109, 110, 111, 112, 113, 121, 114, 115, 116, 117, 118, 119, 120, 32, 0, 0, 0, 0, 47, 0, 0, 20, 21, 10, 12, 19, 13, 14, 17, 15, 16, 18, 0, 0, 123, 125, 127, 129, - 0, 62, 61, 0, 0, 170, 169, 174, 173, 176, - 0, 0, 184, 185, 0, 0, 0, 0, 0, 0, + 0, 62, 61, 0, 0, 174, 173, 178, 177, 180, + 0, 0, 188, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 40, 44, 45, 46, 49, 22, 42, 43, 0, 0, 122, - 124, 126, 128, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - 221, 222, 223, 224, 154, 0, 157, 153, 0, 0, - 0, 168, 172, 171, 0, 0, 0, 180, 181, 186, - 187, 188, 0, 61, 191, 190, 196, 192, 0, 0, - 0, 34, 35, 36, 33, 0, 0, 0, 0, 0, - 0, 0, 161, 0, 0, 0, 0, 0, 161, 131, - 132, 0, 0, 0, 175, 0, 177, 189, 0, 0, - 197, 199, 37, 38, 54, 55, 53, 0, 0, 59, - 51, 52, 56, 50, 24, 182, 183, 161, 0, 0, - 149, 0, 0, 0, 135, 161, 0, 158, 156, 155, - 151, 161, 161, 161, 161, 193, 195, 178, 198, 200, - 0, 57, 58, 0, 0, 150, 162, 0, 145, 143, - 161, 161, 0, 0, 152, 159, 0, 165, 164, 167, - 166, 0, 0, 39, 0, 48, 63, 60, 0, 0, - 0, 0, 23, 25, 163, 137, 137, 148, 147, 0, - 136, 0, 104, 179, 63, 64, 26, 30, 31, 0, - 27, 28, 0, 161, 161, 134, 133, 160, 0, 66, - 68, 0, 140, 138, 139, 146, 144, 194, 0, 65, - 29, 0, 68, 0, 0, 0, 70, 71, 72, 73, - 74, 75, 0, 0, 76, 81, 0, 82, 83, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 69, 142, - 141, 0, 91, 92, 93, 77, 80, 78, 0, 0, - 84, 88, 97, 89, 90, 95, 94, 96, 85, 86, - 87, 67, 0, 98, 99, 79 + 124, 126, 128, 0, 208, 209, 210, 211, 212, 213, + 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 0, 0, 0, 158, 0, + 161, 157, 0, 0, 0, 172, 176, 175, 0, 0, + 0, 184, 185, 190, 191, 192, 0, 61, 195, 194, + 200, 196, 0, 0, 0, 34, 35, 36, 33, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 165, 0, 0, 0, 0, 0, 165, 131, 132, 0, + 0, 0, 179, 0, 181, 193, 0, 0, 201, 203, + 37, 38, 54, 55, 53, 0, 0, 59, 51, 52, + 56, 50, 24, 186, 187, 0, 0, 0, 0, 165, + 0, 0, 149, 0, 0, 0, 135, 165, 0, 162, + 160, 159, 151, 165, 165, 165, 165, 197, 199, 182, + 202, 204, 0, 57, 58, 0, 0, 232, 231, 0, + 0, 0, 0, 150, 166, 0, 145, 143, 165, 165, + 0, 0, 152, 163, 0, 169, 168, 171, 170, 0, + 0, 39, 0, 48, 63, 60, 0, 0, 0, 0, + 23, 25, 0, 165, 0, 0, 167, 137, 137, 148, + 147, 0, 136, 0, 104, 183, 63, 64, 26, 30, + 31, 0, 27, 28, 233, 153, 0, 0, 0, 165, + 165, 134, 133, 164, 0, 66, 68, 0, 0, 165, + 0, 0, 140, 138, 139, 146, 144, 198, 0, 65, + 29, 234, 155, 165, 165, 0, 68, 0, 0, 0, + 70, 71, 72, 73, 74, 75, 0, 0, 76, 81, + 0, 82, 83, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 69, 154, 156, 142, 141, 0, 91, 92, + 93, 77, 80, 78, 0, 0, 84, 88, 97, 89, + 90, 95, 94, 96, 85, 86, 87, 67, 0, 98, + 99, 79 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 6, 7, 8, 23, 27, 76, 77, 188, 254, - 283, 78, 79, 80, 81, 82, 83, 123, 187, 222, - 223, 253, 93, 295, 277, 310, 318, 319, 348, 9, - 10, 11, 15, 24, 48, 49, 201, 155, 235, 303, - 313, 50, 286, 285, 156, 198, 237, 230, 51, 52, + -1, 6, 7, 8, 23, 27, 76, 77, 192, 266, + 301, 78, 79, 80, 81, 82, 83, 123, 191, 230, + 231, 265, 93, 317, 295, 336, 348, 349, 382, 9, + 10, 11, 15, 24, 48, 49, 209, 159, 247, 329, + 343, 50, 308, 307, 160, 206, 249, 242, 51, 52, 53, 54, 98, 55, 56, 57, 58, 59, 60, 61, - 246, 271, 62, 63, 64, 12, 20, 157, 19 + 258, 289, 62, 63, 64, 12, 20, 161, 19, 269, + 339 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -180 +#define YYPACT_NINF -204 static const yytype_int16 yypact[] = { - 65, -89, -87, -87, -49, -180, 130, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -87, - -49, -180, -180, -180, 430, -180, -180, 8, -32, -96, - -86, -82, -68, -18, -1, -52, -71, -4, 31, 21, - 43, 50, 10, 16, -180, 56, 116, 62, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -3, 9, 72, 85, 91, - -180, 99, 17, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, 127, 154, 153, 155, 156, 157, - 34, -180, 54, 168, 159, 38, -180, -180, 175, 71, - 163, 164, -180, -180, 165, 166, 167, 169, 5, 5, - 192, 5, 35, 193, 195, 194, 196, 29, -180, -180, - -180, -180, -180, -180, -180, -180, -180, 197, 220, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -53, 209, -72, -180, 27, 27, - 540, -180, -180, -180, 221, 322, 224, -180, -180, -180, - -180, -180, 222, -180, -180, -180, -180, -180, 86, 223, - 225, -180, -180, -180, -180, 317, 250, 1, 244, 230, - 231, 232, 30, 27, 27, 233, 251, 170, 30, -180, - -180, 198, 240, 246, -180, 234, -180, -180, 236, 237, - -180, -180, 319, -180, -180, -180, -180, 263, 351, -180, - -180, -180, -180, -180, -180, -180, -180, 30, 265, 307, - -180, 268, 310, 285, -180, 55, 291, 280, -180, -180, - -180, 30, 30, 30, 30, -180, -180, 308, -180, -180, - 290, -180, -180, 3, 33, -180, -180, 294, 335, 336, - 30, 30, 27, 233, -180, -180, 312, -180, -180, -180, - -180, 313, 298, -180, 299, -180, -180, -180, 300, 391, - -10, 97, -180, -180, -180, 311, 311, -180, -180, 338, - -180, 304, -180, -180, -180, -180, -180, -180, -180, 396, - -180, -180, 380, 30, 30, -180, -180, -180, 475, -180, - -180, 395, -180, 320, -180, -180, -180, -180, 321, 171, - -180, 408, -180, 309, 332, 333, -180, -180, -180, -180, - -180, -180, 212, 0, -180, -180, 334, -180, -180, 337, - 362, 361, 363, 364, 238, 365, 124, 366, -180, -180, - -180, 142, -180, -180, -180, -180, -180, -180, 400, 152, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, 425, -180, -180, -180 + 102, -70, -56, -56, -98, -204, 82, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -56, + -98, -204, -204, -204, 504, -204, -204, 52, -33, -74, + -63, -57, -51, -18, 7, -20, -66, -2, 21, -6, + 10, 25, 11, 23, -204, 14, 144, 32, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -7, -29, 34, 36, 37, + -204, 38, -22, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, 39, 40, 60, 61, 62, 63, + 116, -204, 15, 83, 67, -16, -204, -204, 88, 33, + 74, 86, -204, -204, 87, 94, 100, 101, 8, 8, + 104, 8, -27, 105, 89, 106, 108, -3, -204, -204, + -204, -204, -204, -204, -204, -204, -204, 109, 111, -204, + -204, -204, -204, 126, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, 112, 119, 120, -61, 152, + -73, -204, 3, 3, 614, -204, -204, -204, 121, 220, + 133, -204, -204, -204, -204, -204, 130, -204, -204, -204, + -204, -204, 26, 139, 145, -204, -204, -204, -204, 229, + 174, 1, 168, 150, 151, 3, 153, 155, 197, 154, + -39, 3, 3, 157, 180, 235, -39, -204, -204, 256, + 279, 218, -204, 226, -204, -204, 227, 228, -204, -204, + 238, -204, -204, -204, -204, 231, 320, -204, -204, -204, + -204, -204, -204, -204, -204, 331, 276, 236, 233, -39, + 237, 281, -204, 354, 375, 261, -204, -15, 239, 257, + -204, -204, -204, -39, -39, -39, -39, -204, -204, 262, + -204, -204, 241, -204, -204, 5, -5, -204, -204, 265, + 242, 267, 268, -204, -204, 248, 286, 294, -39, -39, + 3, 157, -204, -204, 270, -204, -204, -204, -204, 271, + 251, -204, 252, -204, -204, -204, 253, 349, -14, 16, + -204, -204, 255, -39, 278, 322, -204, 337, 337, -204, + -204, 406, -204, 325, -204, -204, -204, -204, -204, -204, + -204, 422, -204, -204, 328, -204, 329, 330, 450, -39, + -39, -204, -204, -204, 549, -204, -204, 424, 356, -39, + 357, 358, -204, 348, -204, -204, -204, -204, 373, 225, + -204, -204, -204, -39, -39, 481, -204, 359, 360, 361, + -204, -204, -204, -204, -204, -204, 115, -4, -204, -204, + 362, -204, -204, 364, 365, 366, 368, 369, 118, 370, + 22, 367, -204, -204, -204, -204, -204, 179, -204, -204, + -204, -204, -204, -204, 455, 29, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, 460, -204, + -204, -204 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -180, -180, 449, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -107, 191, -180, -180, -180, 172, -180, -180, - -180, -180, 12, 199, -180, -180, -149, -155, -180, 200, - -180, -180, -180, -180, -180, -180, -180, -179, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -180, -180, -180, - -180, -180, -180, -180, -180, -180, -180, -160, 467 + -204, -204, 487, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -107, 181, -204, -204, -204, 140, -204, -204, + -204, -204, 24, 206, -204, -204, -147, -195, -204, 187, + -204, -204, -204, -204, -204, -204, -204, -203, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -204, -204, -204, + -204, -204, -204, -204, -204, -204, -204, -164, 501, -204, + -204 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If @@ -1158,176 +1190,196 @@ static const yytype_int16 yypgoto[] = #define YYTABLE_NINF -135 static const yytype_int16 yytable[] = { - 203, 174, 175, 91, 177, 214, 215, 91, 274, 91, - 202, 113, 65, 95, 13, 16, 14, 216, 298, 240, - 217, 66, 86, 67, 68, 69, 70, 191, 357, 196, - 71, 25, 87, 197, 114, 115, 88, 239, 116, 199, - 234, 242, 244, 192, 231, 232, 193, 194, 255, 96, - 89, 278, 72, 73, 74, 185, 264, 90, 218, 279, - 280, 186, 267, 268, 269, 270, 94, 299, 1, 17, - 18, 259, 261, 2, 3, 4, 5, 358, 84, 85, - 281, 287, 288, 133, 134, 135, 136, 137, 138, 139, + 211, 178, 179, 252, 181, 222, 223, 113, 246, 91, + 292, 91, 91, 296, 320, 207, 210, 224, 95, 199, + 225, 297, 298, 189, 393, 17, 18, 16, 204, 190, + 114, 115, 205, 13, 116, 200, 273, 240, 201, 202, + 241, 251, 299, 25, 282, 254, 256, 14, 235, 86, + 285, 286, 287, 288, 243, 244, 65, 90, 226, 96, + 87, 240, 281, 321, 241, 66, 88, 67, 68, 69, + 70, 268, 89, 394, 71, 309, 310, 84, 85, 277, + 279, 208, 21, 100, 101, 1, 312, 182, 183, 184, + 2, 3, 4, 5, 118, 119, 72, 73, 74, 300, + 325, 125, 126, 94, 227, 1, 166, 167, 99, 293, + 2, 3, 4, 5, 162, 163, 117, 108, 103, 104, + 105, 97, 106, 107, 228, 229, 345, 346, 177, 109, + 92, 177, 102, 311, 169, 170, 352, 110, 133, 322, + 323, 216, 217, 391, 392, 404, 405, 332, 401, 402, + 383, 384, 409, 410, 111, 112, 75, 120, 294, 121, + 122, 124, 127, 128, 344, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 129, 130, 131, 132, + 164, 386, 165, 168, 357, 358, 359, 171, 155, 360, + 361, 362, 363, 364, 365, 366, 367, 368, 369, 172, + 173, 186, 370, 371, 372, 373, 374, 174, 375, 376, + 377, 378, 379, 175, 176, 195, 380, 180, 185, 203, + 187, 188, 193, 156, 194, 196, 157, 197, 213, 158, + 357, 358, 359, 198, 212, 360, 361, 362, 363, 364, + 365, 366, 367, 368, 369, 214, 215, 220, 370, 371, + 372, 373, 374, 218, 375, 376, 377, 378, 379, 219, + 221, 232, 380, 233, 234, 238, 262, 239, 236, 237, + 245, 381, 248, 407, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 219, 200, 228, 275, 290, 229, - 100, 101, 75, 289, 97, 117, 108, 92, 99, 220, - 221, 173, 109, 173, 315, 316, 111, 118, 119, 306, - 21, 228, 263, 1, 229, 125, 126, 282, 2, 3, - 4, 5, 314, 103, 104, 105, 276, 106, 107, 178, - 179, 180, 154, 158, 159, 162, 163, 323, 324, 325, - 102, 350, 326, 327, 328, 329, 330, 331, 332, 333, - 334, 335, 165, 166, 110, 336, 337, 338, 339, 340, - 112, 341, 342, 343, 344, 345, 323, 324, 325, 346, - 120, 326, 327, 328, 329, 330, 331, 332, 333, 334, - 335, 208, 209, 121, 336, 337, 338, 339, 340, 122, - 341, 342, 343, 344, 345, 300, 301, 124, 346, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, - 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - 355, 356, 368, 369, 347, 127, 371, 133, 134, 135, - 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 365, 366, - 373, 374, 128, 347, 129, 160, 130, 131, 132, 161, - 164, 167, 168, 169, 170, 171, 195, 172, 238, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, - 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - 176, 181, 182, 183, 184, 189, 241, 133, 134, 135, + 150, 151, 152, 153, 154, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 257, 381, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 190, 204, - 205, 206, 210, 207, 211, 212, 213, 224, 225, 226, - 227, 233, 247, 236, 245, 248, 249, 250, 243, 133, + 146, 147, 148, 149, 150, 151, 152, 153, 154, 259, + 264, 260, 261, 263, 270, 271, 272, 275, 250, 274, + 280, 283, 284, -134, 290, 291, 302, 303, 304, 305, + 306, -133, 313, 315, 314, 316, 318, 319, 324, 253, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - 251, 252, 256, 257, 262, 266, 258, 133, 134, 135, - 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 265, 273, - 272, 284, -134, -133, 291, 293, 292, 294, 296, 297, - 302, 307, 311, 320, 322, 321, 372, 352, 260, 133, + 154, 326, 255, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 327, 328, 333, 337, 338, + 340, 341, 350, 355, 267, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, + 149, 150, 151, 152, 153, 154, 356, 276, 351, 353, + 354, 408, 388, 389, 390, 395, 396, 397, 411, 406, + 398, 399, 400, 22, 403, 330, 387, 335, 278, 134, + 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + 334, 26, 0, 0, 0, 0, 0, 0, 0, 331, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - 353, 354, 359, 375, 360, 22, 305, 133, 134, 135, - 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 28, 361, - 362, 363, 364, 370, 367, 309, 304, 26, 0, 0, - 0, 308, 0, 0, 351, 0, 0, 0, 312, 0, - 29, 30, 31, 32, 33, 0, 0, 0, 0, 0, - 0, 34, 35, 36, 0, 37, 38, 0, 39, 0, - 0, 40, 41, 28, 42, 43, 349, 0, 0, 0, - 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, - 45, 0, 46, 47, 0, 29, 30, 31, 32, 33, - 0, 0, 0, 0, 0, 0, 34, 35, 36, 0, - 37, 38, 0, 39, 0, 0, 40, 41, 0, 42, - 43, 0, 0, 0, 0, 0, 0, 0, 0, 317, - 0, 0, 0, 0, 0, 45, 0, 46, 47, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, - 144, 145, 146, 147, 148, 149, 150, 151, 152, 153 + 154, 0, 28, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 342, 29, 30, 31, 32, 33, 0, + 0, 0, 0, 0, 0, 34, 35, 36, 0, 37, + 38, 0, 39, 0, 0, 40, 41, 28, 42, 43, + 0, 0, 0, 0, 385, 0, 0, 0, 44, 0, + 0, 0, 0, 0, 45, 0, 46, 47, 0, 29, + 30, 31, 32, 33, 0, 0, 0, 0, 0, 0, + 34, 35, 36, 0, 37, 38, 0, 39, 0, 0, + 40, 41, 0, 42, 43, 0, 0, 0, 0, 0, + 0, 0, 0, 347, 0, 0, 0, 0, 0, 45, + 0, 46, 47, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154 }; static const yytype_int16 yycheck[] = { - 160, 108, 109, 4, 111, 4, 5, 4, 5, 4, - 159, 14, 4, 84, 103, 3, 103, 16, 28, 198, - 19, 13, 118, 15, 16, 17, 18, 80, 28, 101, - 22, 19, 118, 105, 37, 38, 118, 197, 41, 12, - 195, 201, 202, 96, 193, 194, 99, 100, 227, 120, - 118, 18, 44, 45, 46, 26, 235, 75, 57, 26, - 27, 32, 241, 242, 243, 244, 118, 77, 3, 118, - 119, 231, 232, 8, 9, 10, 11, 77, 110, 111, - 47, 260, 261, 49, 50, 51, 52, 53, 54, 55, + 164, 108, 109, 206, 111, 4, 5, 14, 203, 4, + 5, 4, 4, 18, 28, 12, 163, 16, 84, 80, + 19, 26, 27, 26, 28, 123, 124, 3, 101, 32, + 37, 38, 105, 103, 41, 96, 239, 76, 99, 100, + 79, 205, 47, 19, 247, 209, 210, 103, 195, 123, + 253, 254, 255, 256, 201, 202, 4, 75, 57, 125, + 123, 76, 77, 77, 79, 13, 123, 15, 16, 17, + 18, 235, 123, 77, 22, 278, 279, 110, 111, 243, + 244, 78, 0, 89, 90, 3, 281, 114, 115, 116, + 8, 9, 10, 11, 123, 124, 44, 45, 46, 104, + 303, 123, 124, 123, 103, 3, 122, 123, 87, 104, + 8, 9, 10, 11, 99, 100, 123, 106, 93, 94, + 95, 123, 97, 98, 123, 124, 329, 330, 123, 106, + 123, 123, 122, 280, 101, 102, 339, 123, 22, 123, + 124, 115, 116, 28, 29, 123, 124, 311, 30, 31, + 353, 354, 123, 124, 10, 123, 104, 123, 265, 123, + 123, 123, 123, 123, 328, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 126, 126, 126, 126, + 107, 355, 125, 105, 15, 16, 17, 123, 82, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 123, + 123, 122, 33, 34, 35, 36, 37, 123, 39, 40, + 41, 42, 43, 123, 123, 99, 47, 123, 123, 77, + 124, 123, 123, 117, 123, 123, 120, 118, 18, 123, + 15, 16, 17, 123, 123, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 122, 126, 28, 33, 34, + 35, 36, 37, 124, 39, 40, 41, 42, 43, 124, + 96, 103, 47, 123, 123, 78, 38, 123, 125, 124, + 123, 102, 102, 104, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 108, 102, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 69, 123, + 30, 124, 124, 122, 78, 119, 123, 76, 123, 122, + 99, 122, 105, 77, 102, 124, 101, 125, 101, 101, + 122, 77, 102, 122, 103, 123, 123, 28, 123, 123, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 123, 123, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 103, 78, 76, 104, 263, 79, - 89, 90, 104, 262, 118, 118, 106, 118, 87, 118, - 119, 118, 106, 118, 303, 304, 10, 118, 119, 289, - 0, 76, 77, 3, 79, 118, 119, 104, 8, 9, - 10, 11, 302, 93, 94, 95, 253, 97, 98, 114, - 115, 116, 118, 99, 100, 117, 118, 15, 16, 17, - 117, 321, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 101, 102, 118, 33, 34, 35, 36, 37, - 118, 39, 40, 41, 42, 43, 15, 16, 17, 47, - 118, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 29, 115, 116, 118, 33, 34, 35, 36, 37, 118, - 39, 40, 41, 42, 43, 118, 119, 118, 47, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 28, 29, 118, 119, 102, 118, 104, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 30, 31, - 118, 119, 118, 102, 121, 107, 121, 121, 121, 120, - 105, 118, 118, 118, 118, 118, 77, 118, 118, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 118, 118, 117, 119, 118, 118, 118, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 118, 118, - 18, 117, 119, 121, 119, 28, 96, 103, 118, 118, - 118, 118, 118, 102, 108, 119, 119, 38, 118, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 117, 30, 117, 76, 99, 105, 118, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 117, 119, - 102, 117, 77, 77, 102, 117, 103, 118, 118, 28, - 109, 117, 26, 28, 103, 105, 26, 118, 118, 49, + 66, 67, 68, 69, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 123, 109, 122, 26, 121, + 121, 121, 28, 105, 123, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 103, 123, 122, 122, + 122, 26, 123, 123, 123, 123, 122, 122, 28, 122, + 124, 123, 123, 6, 124, 308, 356, 316, 123, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 118, 118, 118, 28, 117, 6, 118, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, - 62, 63, 64, 65, 66, 67, 68, 69, 48, 117, - 119, 118, 118, 117, 119, 294, 286, 20, -1, -1, - -1, 292, -1, -1, 322, -1, -1, -1, 118, -1, - 70, 71, 72, 73, 74, -1, -1, -1, -1, -1, - -1, 81, 82, 83, -1, 85, 86, -1, 88, -1, - -1, 91, 92, 48, 94, 95, 118, -1, -1, -1, - -1, -1, -1, -1, 104, -1, -1, -1, -1, -1, - 110, -1, 112, 113, -1, 70, 71, 72, 73, 74, - -1, -1, -1, -1, -1, -1, 81, 82, 83, -1, - 85, 86, -1, 88, -1, -1, 91, 92, -1, 94, - 95, -1, -1, -1, -1, -1, -1, -1, -1, 104, - -1, -1, -1, -1, -1, 110, -1, 112, 113, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 + 314, 20, -1, -1, -1, -1, -1, -1, -1, 123, + 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, -1, 48, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 123, 70, 71, 72, 73, 74, -1, + -1, -1, -1, -1, -1, 81, 82, 83, -1, 85, + 86, -1, 88, -1, -1, 91, 92, 48, 94, 95, + -1, -1, -1, -1, 123, -1, -1, -1, 104, -1, + -1, -1, -1, -1, 110, -1, 112, 113, -1, 70, + 71, 72, 73, 74, -1, -1, -1, -1, -1, -1, + 81, 82, 83, -1, 85, 86, -1, 88, -1, -1, + 91, 92, -1, 94, 95, -1, -1, -1, -1, -1, + -1, -1, -1, 104, -1, -1, -1, -1, -1, 110, + -1, 112, 113, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, + 66, 67, 68, 69 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 3, 8, 9, 10, 11, 123, 124, 125, 151, - 152, 153, 187, 103, 103, 154, 154, 118, 119, 190, - 188, 0, 124, 126, 155, 154, 190, 127, 48, 70, + 0, 3, 8, 9, 10, 11, 128, 129, 130, 156, + 157, 158, 192, 103, 103, 159, 159, 123, 124, 195, + 193, 0, 129, 131, 160, 159, 195, 132, 48, 70, 71, 72, 73, 74, 81, 82, 83, 85, 86, 88, - 91, 92, 94, 95, 104, 110, 112, 113, 156, 157, - 163, 170, 171, 172, 173, 175, 176, 177, 178, 179, - 180, 181, 184, 185, 186, 4, 13, 15, 16, 17, - 18, 22, 44, 45, 46, 104, 128, 129, 133, 134, - 135, 136, 137, 138, 110, 111, 118, 118, 118, 118, - 75, 4, 118, 144, 118, 84, 120, 118, 174, 87, - 89, 90, 117, 93, 94, 95, 97, 98, 106, 106, - 118, 10, 118, 14, 37, 38, 41, 118, 118, 119, - 118, 118, 118, 139, 118, 118, 119, 118, 118, 121, - 121, 121, 121, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, - 66, 67, 68, 69, 118, 159, 166, 189, 99, 100, - 107, 120, 117, 118, 105, 101, 102, 118, 118, 118, - 118, 118, 118, 118, 144, 144, 118, 144, 114, 115, - 116, 118, 117, 119, 118, 26, 32, 140, 130, 118, - 118, 80, 96, 99, 100, 77, 101, 105, 167, 12, - 78, 158, 158, 189, 118, 18, 117, 121, 115, 116, - 119, 119, 28, 96, 4, 5, 16, 19, 57, 103, - 118, 119, 141, 142, 103, 118, 118, 118, 76, 79, - 169, 158, 158, 118, 159, 160, 102, 168, 118, 189, - 169, 118, 189, 118, 189, 108, 182, 118, 119, 119, - 38, 117, 30, 143, 131, 169, 117, 76, 118, 189, - 118, 189, 99, 77, 169, 117, 105, 169, 169, 169, - 169, 183, 102, 119, 5, 104, 144, 146, 18, 26, - 27, 47, 104, 132, 117, 165, 164, 169, 169, 158, - 159, 102, 103, 117, 118, 145, 118, 28, 28, 77, - 118, 119, 109, 161, 161, 118, 189, 117, 155, 145, - 147, 26, 118, 162, 189, 169, 169, 104, 148, 149, - 28, 105, 103, 15, 16, 17, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 33, 34, 35, 36, - 37, 39, 40, 41, 42, 43, 47, 102, 150, 118, - 189, 149, 118, 118, 118, 28, 29, 28, 77, 118, - 117, 117, 119, 118, 118, 30, 31, 119, 118, 119, - 117, 104, 26, 118, 119, 28 + 91, 92, 94, 95, 104, 110, 112, 113, 161, 162, + 168, 175, 176, 177, 178, 180, 181, 182, 183, 184, + 185, 186, 189, 190, 191, 4, 13, 15, 16, 17, + 18, 22, 44, 45, 46, 104, 133, 134, 138, 139, + 140, 141, 142, 143, 110, 111, 123, 123, 123, 123, + 75, 4, 123, 149, 123, 84, 125, 123, 179, 87, + 89, 90, 122, 93, 94, 95, 97, 98, 106, 106, + 123, 10, 123, 14, 37, 38, 41, 123, 123, 124, + 123, 123, 123, 144, 123, 123, 124, 123, 123, 126, + 126, 126, 126, 22, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 82, 117, 120, 123, 164, + 171, 194, 99, 100, 107, 125, 122, 123, 105, 101, + 102, 123, 123, 123, 123, 123, 123, 123, 149, 149, + 123, 149, 114, 115, 116, 123, 122, 124, 123, 26, + 32, 145, 135, 123, 123, 99, 123, 118, 123, 80, + 96, 99, 100, 77, 101, 105, 172, 12, 78, 163, + 163, 194, 123, 18, 122, 126, 115, 116, 124, 124, + 28, 96, 4, 5, 16, 19, 57, 103, 123, 124, + 146, 147, 103, 123, 123, 163, 125, 124, 78, 123, + 76, 79, 174, 163, 163, 123, 164, 165, 102, 173, + 123, 194, 174, 123, 194, 123, 194, 108, 187, 123, + 124, 124, 38, 122, 30, 148, 136, 123, 194, 196, + 78, 119, 123, 174, 122, 76, 123, 194, 123, 194, + 99, 77, 174, 122, 105, 174, 174, 174, 174, 188, + 102, 124, 5, 104, 149, 151, 18, 26, 27, 47, + 104, 137, 101, 125, 101, 101, 122, 170, 169, 174, + 174, 163, 164, 102, 103, 122, 123, 150, 123, 28, + 28, 77, 123, 124, 123, 174, 123, 123, 109, 166, + 166, 123, 194, 122, 160, 150, 152, 26, 121, 197, + 121, 121, 123, 167, 194, 174, 174, 104, 153, 154, + 28, 122, 174, 122, 122, 105, 103, 15, 16, 17, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, + 47, 102, 155, 174, 174, 123, 194, 154, 123, 123, + 123, 28, 29, 28, 77, 123, 122, 122, 124, 123, + 123, 30, 31, 124, 123, 124, 122, 104, 26, 123, + 124, 28 }; #define yyerrok (yyerrstatus = 0) @@ -2142,7 +2194,7 @@ yyparse () switch (yyn) { case 9: -#line 239 "test_spec_parse.y" +#line 242 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, "self-signed", sizeof(current_spec->cluster.ssl)); @@ -2152,17 +2204,17 @@ yyparse () break; case 20: -#line 261 "test_spec_parse.y" +#line 264 "test_spec_parse.y" { current_spec->cluster.bindSource = true; ;} break; case 21: -#line 262 "test_spec_parse.y" +#line 265 "test_spec_parse.y" { current_spec->cluster.legacyStartup = true; ;} break; case 22: -#line 285 "test_spec_parse.y" +#line 288 "test_spec_parse.y" { TestCluster *cl = ¤t_spec->cluster; @@ -2180,7 +2232,7 @@ yyparse () break; case 26: -#line 309 "test_spec_parse.y" +#line 312 "test_spec_parse.y" { if (current_archiver->formationCount >= PGAF_MAX_ARCHIVER_FORMATIONS) { @@ -2197,7 +2249,7 @@ yyparse () break; case 27: -#line 323 "test_spec_parse.y" +#line 326 "test_spec_parse.y" { strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); free((yyvsp[(2) - (2)].str)); @@ -2205,7 +2257,7 @@ yyparse () break; case 28: -#line 328 "test_spec_parse.y" +#line 331 "test_spec_parse.y" { strlcpy(current_archiver->region, (yyvsp[(2) - (2)].str), sizeof(current_archiver->region)); free((yyvsp[(2) - (2)].str)); @@ -2213,7 +2265,7 @@ yyparse () break; case 29: -#line 333 "test_spec_parse.y" +#line 336 "test_spec_parse.y" { /* bare "create and launch deferred" = both gates, matching * node_opt's own identical form */ @@ -2223,28 +2275,28 @@ yyparse () break; case 30: -#line 340 "test_spec_parse.y" +#line 343 "test_spec_parse.y" { current_archiver->launchDeferred = true; ;} break; case 31: -#line 344 "test_spec_parse.y" +#line 347 "test_spec_parse.y" { current_archiver->createDeferred = true; ;} break; case 32: -#line 360 "test_spec_parse.y" +#line 363 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; ;} break; case 33: -#line 364 "test_spec_parse.y" +#line 367 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorDebianCluster, (yyvsp[(3) - (3)].str), @@ -2254,7 +2306,7 @@ yyparse () break; case 34: -#line 371 "test_spec_parse.y" +#line 374 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorImageTarget, (yyvsp[(3) - (3)].str), @@ -2264,7 +2316,7 @@ yyparse () break; case 35: -#line 378 "test_spec_parse.y" +#line 381 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; /* monitor port not stored in TestCluster yet; ignore */ @@ -2273,7 +2325,7 @@ yyparse () break; case 36: -#line 384 "test_spec_parse.y" +#line 387 "test_spec_parse.y" { current_spec->cluster.withMonitor = true; strlcpy(current_spec->cluster.monitorPassword, (yyvsp[(3) - (3)].str), @@ -2283,7 +2335,7 @@ yyparse () break; case 37: -#line 391 "test_spec_parse.y" +#line 394 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2293,7 +2345,7 @@ yyparse () break; case 38: -#line 398 "test_spec_parse.y" +#line 401 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (4)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2303,7 +2355,7 @@ yyparse () break; case 39: -#line 405 "test_spec_parse.y" +#line 408 "test_spec_parse.y" { strlcpy(current_spec->cluster.secondMonitorName, (yyvsp[(2) - (6)].str), sizeof(current_spec->cluster.secondMonitorName)); @@ -2315,7 +2367,7 @@ yyparse () break; case 40: -#line 418 "test_spec_parse.y" +#line 421 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2324,7 +2376,7 @@ yyparse () break; case 41: -#line 424 "test_spec_parse.y" +#line 427 "test_spec_parse.y" { strlcpy(current_spec->cluster.image, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.image)); @@ -2333,7 +2385,7 @@ yyparse () break; case 42: -#line 434 "test_spec_parse.y" +#line 437 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2342,7 +2394,7 @@ yyparse () break; case 43: -#line 440 "test_spec_parse.y" +#line 443 "test_spec_parse.y" { strlcpy(current_spec->cluster.extensionVersion, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.extensionVersion)); @@ -2351,7 +2403,7 @@ yyparse () break; case 44: -#line 450 "test_spec_parse.y" +#line 453 "test_spec_parse.y" { strlcpy(current_spec->cluster.ssl, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.ssl)); @@ -2360,7 +2412,7 @@ yyparse () break; case 45: -#line 460 "test_spec_parse.y" +#line 463 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2369,7 +2421,7 @@ yyparse () break; case 46: -#line 466 "test_spec_parse.y" +#line 469 "test_spec_parse.y" { strlcpy(current_spec->cluster.auth, (yyvsp[(2) - (2)].str), sizeof(current_spec->cluster.auth)); @@ -2378,7 +2430,7 @@ yyparse () break; case 47: -#line 476 "test_spec_parse.y" +#line 479 "test_spec_parse.y" { TestCluster *cl = ¤t_spec->cluster; if (cl->formationCount >= PGAF_MAX_FORMATIONS) @@ -2395,32 +2447,32 @@ yyparse () break; case 51: -#line 503 "test_spec_parse.y" +#line 506 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; case 52: -#line 504 "test_spec_parse.y" +#line 507 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; case 53: -#line 505 "test_spec_parse.y" +#line 508 "test_spec_parse.y" { (yyval.str) = strdup("auth"); ;} break; case 54: -#line 506 "test_spec_parse.y" +#line 509 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; case 55: -#line 507 "test_spec_parse.y" +#line 510 "test_spec_parse.y" { (yyval.str) = strdup("node"); ;} break; case 56: -#line 512 "test_spec_parse.y" +#line 515 "test_spec_parse.y" { strlcpy(current_formation->name, (yyvsp[(1) - (1)].str), sizeof(current_formation->name)); free((yyvsp[(1) - (1)].str)); @@ -2428,31 +2480,31 @@ yyparse () break; case 57: -#line 517 "test_spec_parse.y" +#line 520 "test_spec_parse.y" { current_formation->numSync = (yyvsp[(2) - (2)].ival); ;} break; case 58: -#line 521 "test_spec_parse.y" +#line 524 "test_spec_parse.y" { current_formation->disableSecondary = true; ;} break; case 61: -#line 547 "test_spec_parse.y" +#line 550 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; case 62: -#line 548 "test_spec_parse.y" +#line 551 "test_spec_parse.y" { (yyval.str) = strdup("monitor"); ;} break; case 63: -#line 557 "test_spec_parse.y" +#line 560 "test_spec_parse.y" { if (current_formation->nodeCount >= PGAF_MAX_NODES) { @@ -2468,7 +2520,7 @@ yyparse () break; case 64: -#line 574 "test_spec_parse.y" +#line 577 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(1) - (2)].str), sizeof(current_node->name)); free((yyvsp[(1) - (2)].str)); @@ -2476,7 +2528,7 @@ yyparse () break; case 66: -#line 581 "test_spec_parse.y" +#line 584 "test_spec_parse.y" { strlcpy(current_node->name, (yyvsp[(2) - (3)].str), sizeof(current_node->name)); free((yyvsp[(2) - (3)].str)); @@ -2484,7 +2536,7 @@ yyparse () break; case 70: -#line 595 "test_spec_parse.y" +#line 598 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_COORDINATOR; current_spec->cluster.withCitus = true; @@ -2492,7 +2544,7 @@ yyparse () break; case 71: -#line 600 "test_spec_parse.y" +#line 603 "test_spec_parse.y" { current_node->kind = NODE_KIND_CITUS_WORKER; current_spec->cluster.withCitus = true; @@ -2500,35 +2552,35 @@ yyparse () break; case 72: -#line 605 "test_spec_parse.y" +#line 608 "test_spec_parse.y" { current_node->kind = NODE_KIND_ARCHIVER; ;} break; case 73: -#line 609 "test_spec_parse.y" +#line 612 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; case 74: -#line 613 "test_spec_parse.y" +#line 616 "test_spec_parse.y" { current_node->noMonitor = true; ;} break; case 75: -#line 617 "test_spec_parse.y" +#line 620 "test_spec_parse.y" { current_node->suspended = true; ;} break; case 76: -#line 621 "test_spec_parse.y" +#line 624 "test_spec_parse.y" { /* bare "deferred" = create and launch deferred (both gates) */ current_node->createDeferred = true; @@ -2537,7 +2589,7 @@ yyparse () break; case 77: -#line 627 "test_spec_parse.y" +#line 630 "test_spec_parse.y" { /* "launch deferred" alone = run-deferred only, create immediate */ current_node->launchDeferred = true; @@ -2545,14 +2597,14 @@ yyparse () break; case 78: -#line 632 "test_spec_parse.y" +#line 635 "test_spec_parse.y" { current_node->createDeferred = true; ;} break; case 79: -#line 636 "test_spec_parse.y" +#line 639 "test_spec_parse.y" { current_node->createDeferred = true; current_node->launchDeferred = true; @@ -2560,42 +2612,42 @@ yyparse () break; case 80: -#line 641 "test_spec_parse.y" +#line 644 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; case 81: -#line 645 "test_spec_parse.y" +#line 648 "test_spec_parse.y" { current_node->launchDeferred = false; ;} break; case 82: -#line 649 "test_spec_parse.y" +#line 652 "test_spec_parse.y" { current_node->listen = true; ;} break; case 83: -#line 653 "test_spec_parse.y" +#line 656 "test_spec_parse.y" { current_node->citusSecondary = true; ;} break; case 84: -#line 657 "test_spec_parse.y" +#line 660 "test_spec_parse.y" { current_node->candidatePriority = (yyvsp[(2) - (2)].ival); ;} break; case 85: -#line 661 "test_spec_parse.y" +#line 664 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); @@ -2603,7 +2655,7 @@ yyparse () break; case 86: -#line 666 "test_spec_parse.y" +#line 669 "test_spec_parse.y" { strlcpy(current_node->region, (yyvsp[(2) - (2)].str), sizeof(current_node->region)); free((yyvsp[(2) - (2)].str)); @@ -2611,21 +2663,21 @@ yyparse () break; case 87: -#line 671 "test_spec_parse.y" +#line 674 "test_spec_parse.y" { current_node->group = (yyvsp[(2) - (2)].ival); ;} break; case 88: -#line 675 "test_spec_parse.y" +#line 678 "test_spec_parse.y" { current_node->pgPort = (yyvsp[(2) - (2)].ival); ;} break; case 89: -#line 679 "test_spec_parse.y" +#line 682 "test_spec_parse.y" { strlcpy(current_node->citusClusterName, (yyvsp[(2) - (2)].str), sizeof(current_node->citusClusterName)); @@ -2634,7 +2686,7 @@ yyparse () break; case 90: -#line 685 "test_spec_parse.y" +#line 688 "test_spec_parse.y" { strlcpy(current_node->debianCluster, (yyvsp[(2) - (2)].str), sizeof(current_node->debianCluster)); @@ -2643,7 +2695,7 @@ yyparse () break; case 91: -#line 691 "test_spec_parse.y" +#line 694 "test_spec_parse.y" { strlcpy(current_node->ssl, (yyvsp[(2) - (2)].str), sizeof(current_node->ssl)); free((yyvsp[(2) - (2)].str)); @@ -2651,7 +2703,7 @@ yyparse () break; case 92: -#line 696 "test_spec_parse.y" +#line 699 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); @@ -2659,7 +2711,7 @@ yyparse () break; case 93: -#line 701 "test_spec_parse.y" +#line 704 "test_spec_parse.y" { strlcpy(current_node->auth, (yyvsp[(2) - (2)].str), sizeof(current_node->auth)); free((yyvsp[(2) - (2)].str)); @@ -2667,21 +2719,21 @@ yyparse () break; case 94: -#line 706 "test_spec_parse.y" +#line 709 "test_spec_parse.y" { current_node->replicationQuorum = true; ;} break; case 95: -#line 710 "test_spec_parse.y" +#line 713 "test_spec_parse.y" { current_node->replicationQuorum = false; ;} break; case 96: -#line 714 "test_spec_parse.y" +#line 717 "test_spec_parse.y" { strlcpy(current_node->replicationPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->replicationPassword)); @@ -2690,7 +2742,7 @@ yyparse () break; case 97: -#line 720 "test_spec_parse.y" +#line 723 "test_spec_parse.y" { strlcpy(current_node->monitorPassword, (yyvsp[(2) - (2)].str), sizeof(current_node->monitorPassword)); @@ -2699,7 +2751,7 @@ yyparse () break; case 98: -#line 726 "test_spec_parse.y" +#line 729 "test_spec_parse.y" { /* volume — adds a named Docker volume */ int vi = current_node->volumeCount; @@ -2716,7 +2768,7 @@ yyparse () break; case 99: -#line 740 "test_spec_parse.y" +#line 743 "test_spec_parse.y" { /* volume "/path/with spaces" */ int vi = current_node->volumeCount; @@ -2733,21 +2785,21 @@ yyparse () break; case 100: -#line 761 "test_spec_parse.y" +#line 764 "test_spec_parse.y" { current_spec->setup = (yyvsp[(2) - (2)].step); ;} break; case 101: -#line 768 "test_spec_parse.y" +#line 771 "test_spec_parse.y" { current_spec->teardown = (yyvsp[(2) - (2)].step); ;} break; case 102: -#line 779 "test_spec_parse.y" +#line 782 "test_spec_parse.y" { TestStep *s = (yyvsp[(3) - (3)].step); strncpy(s->name, (yyvsp[(2) - (3)].str), sizeof(s->name) - 1); @@ -2757,7 +2809,7 @@ yyparse () break; case 103: -#line 797 "test_spec_parse.y" +#line 800 "test_spec_parse.y" { /* post-process: CMD_SQL immediately before CMD_EXPECT_ERROR */ for (TestCmd *c = (yyvsp[(2) - (3)].step)->commands; c; c = c->next) @@ -2771,14 +2823,14 @@ yyparse () break; case 104: -#line 811 "test_spec_parse.y" +#line 814 "test_spec_parse.y" { (yyval.step) = make_step(""); ;} break; case 105: -#line 815 "test_spec_parse.y" +#line 818 "test_spec_parse.y" { if ((yyvsp[(2) - (2)].cmd)) append_cmd((yyvsp[(1) - (2)].step), (yyvsp[(2) - (2)].cmd)); (yyval.step) = (yyvsp[(1) - (2)].step); @@ -2786,87 +2838,87 @@ yyparse () break; case 106: -#line 822 "test_spec_parse.y" +#line 825 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 107: -#line 823 "test_spec_parse.y" +#line 826 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 108: -#line 824 "test_spec_parse.y" +#line 827 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 109: -#line 825 "test_spec_parse.y" +#line 828 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 110: -#line 826 "test_spec_parse.y" +#line 829 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 111: -#line 827 "test_spec_parse.y" +#line 830 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 112: -#line 828 "test_spec_parse.y" +#line 831 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 113: -#line 829 "test_spec_parse.y" +#line 832 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 114: -#line 830 "test_spec_parse.y" +#line 833 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 115: -#line 831 "test_spec_parse.y" +#line 834 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 116: -#line 832 "test_spec_parse.y" +#line 835 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 117: -#line 833 "test_spec_parse.y" +#line 836 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 118: -#line 834 "test_spec_parse.y" +#line 837 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 119: -#line 835 "test_spec_parse.y" +#line 838 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 120: -#line 836 "test_spec_parse.y" +#line 839 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 121: -#line 837 "test_spec_parse.y" +#line 840 "test_spec_parse.y" { (yyval.cmd) = (yyvsp[(1) - (1)].cmd); ;} break; case 122: -#line 852 "test_spec_parse.y" +#line 855 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2876,7 +2928,7 @@ yyparse () break; case 123: -#line 859 "test_spec_parse.y" +#line 862 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2885,7 +2937,7 @@ yyparse () break; case 124: -#line 865 "test_spec_parse.y" +#line 868 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2895,7 +2947,7 @@ yyparse () break; case 125: -#line 872 "test_spec_parse.y" +#line 875 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXEC_FAILS); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2904,7 +2956,7 @@ yyparse () break; case 126: -#line 878 "test_spec_parse.y" +#line 881 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -2914,7 +2966,7 @@ yyparse () break; case 127: -#line 885 "test_spec_parse.y" +#line 888 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_RUN); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->service)); @@ -2923,7 +2975,7 @@ yyparse () break; case 128: -#line 891 "test_spec_parse.y" +#line 894 "test_spec_parse.y" { /* "pg_autoctl perform failover --formation auth" * EXEC_ARGS returns T_IDENT for first word, T_SHELL_ARGS for rest */ @@ -2934,7 +2986,7 @@ yyparse () break; case 129: -#line 899 "test_spec_parse.y" +#line 902 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); strlcpy((yyval.cmd)->args, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->args)); @@ -2943,14 +2995,14 @@ yyparse () break; case 130: -#line 905 "test_spec_parse.y" +#line 908 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_PG_AUTOCTL); ;} break; case 133: -#line 943 "test_spec_parse.y" +#line 946 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2968,7 +3020,7 @@ yyparse () break; case 134: -#line 958 "test_spec_parse.y" +#line 961 "test_spec_parse.y" { if (!current_wait_cmd) current_wait_cmd = make_cmd(CMD_WAIT_MULTI); @@ -2986,7 +3038,7 @@ yyparse () break; case 139: -#line 998 "test_spec_parse.y" +#line 1001 "test_spec_parse.y" { /* current_pass_cmd set by the enclosing wait_cmd rule */ if (current_pass_cmd && @@ -2997,7 +3049,7 @@ yyparse () break; case 140: -#line 1006 "test_spec_parse.y" +#line 1009 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -3008,7 +3060,7 @@ yyparse () break; case 141: -#line 1014 "test_spec_parse.y" +#line 1017 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -3018,7 +3070,7 @@ yyparse () break; case 142: -#line 1021 "test_spec_parse.y" +#line 1024 "test_spec_parse.y" { if (current_pass_cmd && current_pass_cmd->passThroughCount < PGAF_MAX_WAIT_STATES) @@ -3029,7 +3081,7 @@ yyparse () break; case 143: -#line 1032 "test_spec_parse.y" +#line 1035 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); @@ -3037,7 +3089,7 @@ yyparse () break; case 144: -#line 1037 "test_spec_parse.y" +#line 1040 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -3046,7 +3098,7 @@ yyparse () break; case 145: -#line 1043 "test_spec_parse.y" +#line 1046 "test_spec_parse.y" { current_pass_cmd = make_cmd(CMD_WAIT_STATE); strlcpy(current_pass_cmd->service, (yyvsp[(3) - (6)].str), sizeof(current_pass_cmd->service)); strlcpy(current_pass_cmd->state, (yyvsp[(6) - (6)].str), sizeof(current_pass_cmd->state)); @@ -3054,7 +3106,7 @@ yyparse () break; case 146: -#line 1048 "test_spec_parse.y" +#line 1051 "test_spec_parse.y" { current_pass_cmd->timeoutSeconds = (yyvsp[(9) - (9)].ival); (yyval.cmd) = current_pass_cmd; @@ -3063,7 +3115,7 @@ yyparse () break; case 147: -#line 1054 "test_spec_parse.y" +#line 1057 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -3075,7 +3127,7 @@ yyparse () break; case 148: -#line 1063 "test_spec_parse.y" +#line 1066 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STATE); (yyval.cmd)->kind = CMD_ASSERT_ASSIGNED; @@ -3087,7 +3139,7 @@ yyparse () break; case 149: -#line 1072 "test_spec_parse.y" +#line 1075 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_STOPPED); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3097,7 +3149,7 @@ yyparse () break; case 150: -#line 1086 "test_spec_parse.y" +#line 1089 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_WAIT_LSN); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3108,7 +3160,7 @@ yyparse () break; case 151: -#line 1094 "test_spec_parse.y" +#line 1097 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(5) - (5)].ival); @@ -3117,7 +3169,7 @@ yyparse () break; case 152: -#line 1108 "test_spec_parse.y" +#line 1111 "test_spec_parse.y" { (yyval.cmd) = current_wait_cmd; (yyval.cmd)->timeoutSeconds = (yyvsp[(6) - (6)].ival); @@ -3126,7 +3178,81 @@ yyparse () break; case 153: -#line 1123 "test_spec_parse.y" +#line 1126 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, (yyvsp[(4) - (8)].str), sizeof((yyval.cmd)->service)); + strlcpy((yyval.cmd)->args, (yyvsp[(5) - (8)].str), sizeof((yyval.cmd)->args)); + strlcpy((yyval.cmd)->expected, (yyvsp[(7) - (8)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(8) - (8)].ival); + free((yyvsp[(4) - (8)].str)); free((yyvsp[(5) - (8)].str)); free((yyvsp[(7) - (8)].str)); + ;} + break; + + case 154: +#line 1143 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT pgautofailover.wal_archived('%s', %d, '%s')", + (yyvsp[(8) - (11)].str), (yyvsp[(10) - (11)].ival), (yyvsp[(5) - (11)].str)); + strlcpy((yyval.cmd)->expected, "t", sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(11) - (11)].ival); + free((yyvsp[(5) - (11)].str)); free((yyvsp[(8) - (11)].str)); + ;} + break; + + case 155: +#line 1167 "test_spec_parse.y" + { + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + if ((yyvsp[(9) - (10)].ival) >= 0) + { + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'" + " AND groupid = %d", (yyvsp[(8) - (10)].str), (yyvsp[(9) - (10)].ival)); + } + else + { + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'", (yyvsp[(8) - (10)].str)); + } + strlcpy((yyval.cmd)->expected, (yyvsp[(6) - (10)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(10) - (10)].ival); + free((yyvsp[(6) - (10)].str)); free((yyvsp[(8) - (10)].str)); + ;} + break; + + case 156: +#line 1196 "test_spec_parse.y" + { + if (strcmp((yyvsp[(4) - (11)].str), "source") != 0 && + strcmp((yyvsp[(4) - (11)].str), "status") != 0 && + strcmp((yyvsp[(4) - (11)].str), "replaymode") != 0) + { + fprintf(stderr, + "pgaftest: line %d: \"wait until basebackup %s ...\" -- " + "unknown property (expected source, status, or replaymode)\n", + pgaf_line_number, (yyvsp[(4) - (11)].str)); + exit(1); + } + (yyval.cmd) = make_cmd(CMD_WAIT_SQL); + strlcpy((yyval.cmd)->service, "monitor", sizeof((yyval.cmd)->service)); + sformat((yyval.cmd)->args, sizeof((yyval.cmd)->args), + "SELECT %s::text FROM pgautofailover.get_latest_basebackup('%s', %d)", + (yyvsp[(4) - (11)].str), (yyvsp[(8) - (11)].str), (yyvsp[(10) - (11)].ival)); + strlcpy((yyval.cmd)->expected, (yyvsp[(6) - (11)].str), sizeof((yyval.cmd)->expected)); + (yyval.cmd)->timeoutSeconds = (yyvsp[(11) - (11)].ival); + free((yyvsp[(4) - (11)].str)); free((yyvsp[(6) - (11)].str)); free((yyvsp[(8) - (11)].str)); + ;} + break; + + case 157: +#line 1226 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3134,8 +3260,8 @@ yyparse () ;} break; - case 154: -#line 1129 "test_spec_parse.y" + case 158: +#line 1232 "test_spec_parse.y" { current_wait_cmd = make_cmd(CMD_WAIT_STATES); strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3144,8 +3270,8 @@ yyparse () ;} break; - case 155: -#line 1136 "test_spec_parse.y" + case 159: +#line 1239 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3153,8 +3279,8 @@ yyparse () ;} break; - case 156: -#line 1142 "test_spec_parse.y" + case 160: +#line 1245 "test_spec_parse.y" { if (current_wait_cmd->waitStateCount < PGAF_MAX_WAIT_STATES) strlcpy(current_wait_cmd->waitStates[current_wait_cmd->waitStateCount++], @@ -3163,39 +3289,39 @@ yyparse () ;} break; - case 159: -#line 1161 "test_spec_parse.y" + case 163: +#line 1264 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(2) - (2)].ival); ;} break; - case 160: -#line 1166 "test_spec_parse.y" + case 164: +#line 1269 "test_spec_parse.y" { if (current_wait_cmd->waitGroupCount < PGAF_MAX_WAIT_GROUPS) current_wait_cmd->waitGroups[current_wait_cmd->waitGroupCount++] = (yyvsp[(4) - (4)].ival); ;} break; - case 161: -#line 1173 "test_spec_parse.y" + case 165: +#line 1276 "test_spec_parse.y" { (yyval.ival) = PGAF_TIMEOUT_DEFAULT; ;} break; - case 162: -#line 1174 "test_spec_parse.y" + case 166: +#line 1277 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} break; - case 163: -#line 1175 "test_spec_parse.y" + case 167: +#line 1278 "test_spec_parse.y" { (yyval.ival) = (yyvsp[(3) - (3)].ival); ;} break; - case 164: -#line 1187 "test_spec_parse.y" + case 168: +#line 1290 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3205,8 +3331,8 @@ yyparse () ;} break; - case 165: -#line 1195 "test_spec_parse.y" + case 169: +#line 1298 "test_spec_parse.y" { (yyval.cmd) = make_cmd((yyvsp[(6) - (6)].ival) > 0 ? CMD_WAIT_STATE : CMD_ASSERT_STATE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3216,8 +3342,8 @@ yyparse () ;} break; - case 166: -#line 1203 "test_spec_parse.y" + case 170: +#line 1306 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3227,8 +3353,8 @@ yyparse () ;} break; - case 167: -#line 1211 "test_spec_parse.y" + case 171: +#line 1314 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_ASSERT_ASSIGNED); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (6)].str), sizeof((yyval.cmd)->service)); @@ -3238,8 +3364,8 @@ yyparse () ;} break; - case 168: -#line 1229 "test_spec_parse.y" + case 172: +#line 1332 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SQL); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3248,8 +3374,8 @@ yyparse () ;} break; - case 169: -#line 1244 "test_spec_parse.y" + case 173: +#line 1347 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT); strlcpy((yyval.cmd)->expected, (yyvsp[(2) - (2)].str), sizeof((yyval.cmd)->expected)); @@ -3258,15 +3384,15 @@ yyparse () ;} break; - case 170: -#line 1251 "test_spec_parse.y" + case 174: +#line 1354 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); ;} break; - case 171: -#line 1255 "test_spec_parse.y" + case 175: +#line 1358 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); strlcpy((yyval.cmd)->state, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->state)); @@ -3274,8 +3400,8 @@ yyparse () ;} break; - case 172: -#line 1261 "test_spec_parse.y" + case 176: +#line 1364 "test_spec_parse.y" { /* SQLSTATE codes like 25006 are all digits, lexed as T_INTEGER */ (yyval.cmd) = make_cmd(CMD_EXPECT_ERROR); @@ -3283,16 +3409,16 @@ yyparse () ;} break; - case 173: -#line 1274 "test_spec_parse.y" + case 177: +#line 1377 "test_spec_parse.y" { (yyval.cmd) = current_promote_cmd; current_promote_cmd = NULL; ;} break; - case 174: -#line 1282 "test_spec_parse.y" + case 178: +#line 1385 "test_spec_parse.y" { current_promote_cmd = make_cmd(CMD_PROMOTE); current_promote_cmd->timeoutSeconds = PGAF_TIMEOUT_DEFAULT; @@ -3302,8 +3428,8 @@ yyparse () ;} break; - case 175: -#line 1290 "test_spec_parse.y" + case 179: +#line 1393 "test_spec_parse.y" { if (current_promote_cmd->promoteCount < PGAF_MAX_PROMOTE_NODES) strlcpy(current_promote_cmd->promoteNodes[current_promote_cmd->promoteCount++], @@ -3312,8 +3438,8 @@ yyparse () ;} break; - case 176: -#line 1311 "test_spec_parse.y" + case 180: +#line 1414 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3322,8 +3448,8 @@ yyparse () ;} break; - case 177: -#line 1318 "test_spec_parse.y" + case 181: +#line 1421 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, "default", sizeof((yyval.cmd)->service)); @@ -3332,8 +3458,8 @@ yyparse () ;} break; - case 178: -#line 1325 "test_spec_parse.y" + case 182: +#line 1428 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3343,8 +3469,8 @@ yyparse () ;} break; - case 179: -#line 1333 "test_spec_parse.y" + case 183: +#line 1436 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FAILOVER); strlcpy((yyval.cmd)->service, (yyvsp[(5) - (7)].str), sizeof((yyval.cmd)->service)); @@ -3354,8 +3480,8 @@ yyparse () ;} break; - case 180: -#line 1349 "test_spec_parse.y" + case 184: +#line 1452 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_OFF); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3363,8 +3489,8 @@ yyparse () ;} break; - case 181: -#line 1355 "test_spec_parse.y" + case 185: +#line 1458 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NETWORK_ON); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3372,8 +3498,8 @@ yyparse () ;} break; - case 182: -#line 1376 "test_spec_parse.y" + case 186: +#line 1479 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_SET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3383,8 +3509,8 @@ yyparse () ;} break; - case 183: -#line 1384 "test_spec_parse.y" + case 187: +#line 1487 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_NODEINI_GET); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3394,23 +3520,23 @@ yyparse () ;} break; - case 184: -#line 1399 "test_spec_parse.y" + case 188: +#line 1502 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_SLEEP); (yyval.cmd)->timeoutSeconds = (yyvsp[(2) - (2)].ival); ;} break; - case 185: -#line 1413 "test_spec_parse.y" + case 189: +#line 1516 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_DOWN); ;} break; - case 186: -#line 1417 "test_spec_parse.y" + case 190: +#line 1520 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_START); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3418,8 +3544,8 @@ yyparse () ;} break; - case 187: -#line 1423 "test_spec_parse.y" + case 191: +#line 1526 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_STOP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3427,8 +3553,8 @@ yyparse () ;} break; - case 188: -#line 1429 "test_spec_parse.y" + case 192: +#line 1532 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_KILL); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3436,8 +3562,8 @@ yyparse () ;} break; - case 189: -#line 1455 "test_spec_parse.y" + case 193: +#line 1558 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_COMPOSE_INJECT); strlcpy((yyval.cmd)->expected, (yyvsp[(3) - (4)].str), sizeof((yyval.cmd)->expected)); /* image */ @@ -3462,8 +3588,8 @@ yyparse () ;} break; - case 190: -#line 1489 "test_spec_parse.y" + case 194: +#line 1592 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STOP_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3471,8 +3597,8 @@ yyparse () ;} break; - case 191: -#line 1495 "test_spec_parse.y" + case 195: +#line 1598 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_START_POSTGRES); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3480,8 +3606,8 @@ yyparse () ;} break; - case 192: -#line 1516 "test_spec_parse.y" + case 196: +#line 1619 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_FSM_STEP); strlcpy((yyval.cmd)->service, (yyvsp[(3) - (3)].str), sizeof((yyval.cmd)->service)); @@ -3489,18 +3615,18 @@ yyparse () ;} break; - case 193: -#line 1532 "test_spec_parse.y" + case 197: +#line 1635 "test_spec_parse.y" { pgaf_next_brace_is_while = 1; ;} break; - case 194: -#line 1533 "test_spec_parse.y" + case 198: +#line 1636 "test_spec_parse.y" { (yyval.step) = (yyvsp[(4) - (5)].step); ;} break; - case 195: -#line 1538 "test_spec_parse.y" + case 199: +#line 1641 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_STAYS_WHILE); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3510,8 +3636,8 @@ yyparse () ;} break; - case 196: -#line 1557 "test_spec_parse.y" + case 200: +#line 1660 "test_spec_parse.y" { /* only "set monitor " is supported; $2 must be "monitor" */ if (strcmp((yyvsp[(2) - (3)].str), "monitor") != 0) @@ -3526,8 +3652,8 @@ yyparse () ;} break; - case 197: -#line 1582 "test_spec_parse.y" + case 201: +#line 1685 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3538,8 +3664,8 @@ yyparse () ;} break; - case 198: -#line 1591 "test_spec_parse.y" + case 202: +#line 1694 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3550,8 +3676,8 @@ yyparse () ;} break; - case 199: -#line 1600 "test_spec_parse.y" + case 203: +#line 1703 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (4)].str), sizeof((yyval.cmd)->service)); @@ -3562,8 +3688,8 @@ yyparse () ;} break; - case 200: -#line 1609 "test_spec_parse.y" + case 204: +#line 1712 "test_spec_parse.y" { (yyval.cmd) = make_cmd(CMD_LOGS_CHECK); strlcpy((yyval.cmd)->service, (yyvsp[(2) - (5)].str), sizeof((yyval.cmd)->service)); @@ -3574,8 +3700,8 @@ yyparse () ;} break; - case 203: -#line 1630 "test_spec_parse.y" + case 207: +#line 1733 "test_spec_parse.y" { int i = current_spec->sequenceLength; if (i < PGAF_MAX_SEQ) @@ -3589,124 +3715,144 @@ yyparse () ;} break; - case 204: -#line 1651 "test_spec_parse.y" + case 208: +#line 1754 "test_spec_parse.y" { (yyval.str) = "init"; ;} break; - case 205: -#line 1652 "test_spec_parse.y" + case 209: +#line 1755 "test_spec_parse.y" { (yyval.str) = "single"; ;} break; - case 206: -#line 1653 "test_spec_parse.y" + case 210: +#line 1756 "test_spec_parse.y" { (yyval.str) = "primary"; ;} break; - case 207: -#line 1654 "test_spec_parse.y" + case 211: +#line 1757 "test_spec_parse.y" { (yyval.str) = "wait_primary"; ;} break; - case 208: -#line 1655 "test_spec_parse.y" + case 212: +#line 1758 "test_spec_parse.y" { (yyval.str) = "wait_standby"; ;} break; - case 209: -#line 1656 "test_spec_parse.y" + case 213: +#line 1759 "test_spec_parse.y" { (yyval.str) = "demoted"; ;} break; - case 210: -#line 1657 "test_spec_parse.y" + case 214: +#line 1760 "test_spec_parse.y" { (yyval.str) = "demote_timeout"; ;} break; - case 211: -#line 1658 "test_spec_parse.y" + case 215: +#line 1761 "test_spec_parse.y" { (yyval.str) = "draining"; ;} break; - case 212: -#line 1659 "test_spec_parse.y" + case 216: +#line 1762 "test_spec_parse.y" { (yyval.str) = "secondary"; ;} break; - case 213: -#line 1660 "test_spec_parse.y" + case 217: +#line 1763 "test_spec_parse.y" { (yyval.str) = "catchingup"; ;} break; - case 214: -#line 1661 "test_spec_parse.y" + case 218: +#line 1764 "test_spec_parse.y" { (yyval.str) = "prepare_promotion"; ;} break; - case 215: -#line 1662 "test_spec_parse.y" + case 219: +#line 1765 "test_spec_parse.y" { (yyval.str) = "stop_replication"; ;} break; - case 216: -#line 1663 "test_spec_parse.y" + case 220: +#line 1766 "test_spec_parse.y" { (yyval.str) = "maintenance"; ;} break; - case 217: -#line 1664 "test_spec_parse.y" + case 221: +#line 1767 "test_spec_parse.y" { (yyval.str) = "join_primary"; ;} break; - case 218: -#line 1665 "test_spec_parse.y" + case 222: +#line 1768 "test_spec_parse.y" { (yyval.str) = "apply_settings"; ;} break; - case 219: -#line 1666 "test_spec_parse.y" + case 223: +#line 1769 "test_spec_parse.y" { (yyval.str) = "prepare_maintenance"; ;} break; - case 220: -#line 1667 "test_spec_parse.y" + case 224: +#line 1770 "test_spec_parse.y" { (yyval.str) = "wait_maintenance"; ;} break; - case 221: -#line 1668 "test_spec_parse.y" + case 225: +#line 1771 "test_spec_parse.y" { (yyval.str) = "report_lsn"; ;} break; - case 222: -#line 1669 "test_spec_parse.y" + case 226: +#line 1772 "test_spec_parse.y" { (yyval.str) = "fast_forward"; ;} break; - case 223: -#line 1670 "test_spec_parse.y" + case 227: +#line 1773 "test_spec_parse.y" { (yyval.str) = "join_secondary"; ;} break; - case 224: -#line 1671 "test_spec_parse.y" + case 228: +#line 1774 "test_spec_parse.y" { (yyval.str) = "dropped"; ;} break; - case 225: -#line 1679 "test_spec_parse.y" + case 229: +#line 1782 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; - case 226: -#line 1680 "test_spec_parse.y" + case 230: +#line 1783 "test_spec_parse.y" + { (yyval.str) = (yyvsp[(1) - (1)].str); ;} + break; + + case 231: +#line 1794 "test_spec_parse.y" + { (yyval.str) = strdup((yyvsp[(1) - (1)].str)); ;} + break; + + case 232: +#line 1795 "test_spec_parse.y" { (yyval.str) = (yyvsp[(1) - (1)].str); ;} break; + case 233: +#line 1803 "test_spec_parse.y" + { (yyval.ival) = -1; ;} + break; + + case 234: +#line 1804 "test_spec_parse.y" + { (yyval.ival) = (yyvsp[(2) - (2)].ival); ;} + break; + /* Line 1267 of yacc.c. */ -#line 3710 "test_spec_parse.c" +#line 3856 "test_spec_parse.c" default: break; } YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); @@ -3920,7 +4066,7 @@ yyparse () } -#line 1683 "test_spec_parse.y" +#line 1807 "test_spec_parse.y" /* diff --git a/src/bin/pgaftest/test_spec_parse.h b/src/bin/pgaftest/test_spec_parse.h index 85a36a2e1..a840f1a49 100644 --- a/src/bin/pgaftest/test_spec_parse.h +++ b/src/bin/pgaftest/test_spec_parse.h @@ -153,11 +153,16 @@ T_NOT = 369, T_CONTAINS = 370, T_MATCHES = 371, - T_INTEGER = 372, - T_IDENT = 373, - T_STRING = 374, - T_BLOCK = 375, - T_SHELL_ARGS = 376 + T_WAL = 372, + T_SEGMENT = 373, + T_ARCHIVED = 374, + T_BASEBACKUP = 375, + T_SLASH = 376, + T_INTEGER = 377, + T_IDENT = 378, + T_STRING = 379, + T_BLOCK = 380, + T_SHELL_ARGS = 381 }; #endif /* Tokens. */ @@ -275,11 +280,16 @@ #define T_NOT 369 #define T_CONTAINS 370 #define T_MATCHES 371 -#define T_INTEGER 372 -#define T_IDENT 373 -#define T_STRING 374 -#define T_BLOCK 375 -#define T_SHELL_ARGS 376 +#define T_WAL 372 +#define T_SEGMENT 373 +#define T_ARCHIVED 374 +#define T_BASEBACKUP 375 +#define T_SLASH 376 +#define T_INTEGER 377 +#define T_IDENT 378 +#define T_STRING 379 +#define T_BLOCK 380 +#define T_SHELL_ARGS 381 @@ -294,7 +304,7 @@ typedef union YYSTYPE TestCmd *cmd; } /* Line 1529 of yacc.c. */ -#line 298 "test_spec_parse.h" +#line 308 "test_spec_parse.h" YYSTYPE; # define yystype YYSTYPE /* obsolescent; will be withdrawn */ # define YYSTYPE_IS_DECLARED 1 diff --git a/src/bin/pgaftest/test_spec_parse.y b/src/bin/pgaftest/test_spec_parse.y index a785c86a6..f4140aadb 100644 --- a/src/bin/pgaftest/test_spec_parse.y +++ b/src/bin/pgaftest/test_spec_parse.y @@ -191,6 +191,7 @@ static TestArchiverNode *current_archiver = NULL; %token T_POSTGRES T_STAYS T_WHILE T_THROUGH T_SET T_GET %token T_FSM %token T_LOGS T_NOT T_CONTAINS T_MATCHES +%token T_WAL T_SEGMENT T_ARCHIVED T_BASEBACKUP T_SLASH /* ---- Tokens with values ---- */ %token T_INTEGER @@ -200,6 +201,7 @@ static TestArchiverNode *current_archiver = NULL; %type ident_or_string %type bare_name %type fsm_state +%type wait_state_name %type node_name %type cmd_block cmd_list %type step_cmd @@ -209,6 +211,7 @@ static TestArchiverNode *current_archiver = NULL; %type fsm_step_cmd %type nodeini_cmd %type opt_timeout +%type opt_wait_group %type while_body %% @@ -1110,6 +1113,106 @@ wait_cmd: $$->timeoutSeconds = $6; current_wait_cmd = NULL; } + /* + * Generic form: wait until sql { SQL } is { value } [timeout Ns] + * + * Polls an arbitrary scalar SQL expression until its (substring- + * matched, same semantics as `expect { }`) result contains . + * The "wal segment ... archived", "archiver state is ...", and + * "basebackup ... is ..." forms below are all sugar for this at parse + * time -- reach for this directly only when none of those fit. + */ + | T_WAIT T_UNTIL T_SQL T_IDENT T_BLOCK T_IS T_BLOCK opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, $4, sizeof($$->service)); + strlcpy($$->args, $5, sizeof($$->args)); + strlcpy($$->expected, $7, sizeof($$->expected)); + $$->timeoutSeconds = $8; + free($4); free($5); free($7); + } + /* + * wait until wal segment "" archived in / [timeout Ns] + * + * Sugar for polling pgautofailover.wal_archived(). The segment name is + * quoted (T_STRING) rather than bare: a real segment name is all + * digits, which the lexer's own T_INTEGER rule would otherwise + * swallow (and overflow -- a segment name is 24 digits, an int isn't). + */ + | T_WAIT T_UNTIL T_WAL T_SEGMENT T_STRING T_ARCHIVED T_IN T_IDENT T_SLASH T_INTEGER opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + sformat($$->args, sizeof($$->args), + "SELECT pgautofailover.wal_archived('%s', %d, '%s')", + $8, $10, $5); + strlcpy($$->expected, "t", sizeof($$->expected)); + $$->timeoutSeconds = $11; + free($5); free($8); + } + /* + * wait until archiver state is in [/] [timeout Ns] + * + * Sugar for the nodename LIKE 'archiver-%' idiom every multi- + * membership archiver spec needs: archiver_add_formation() (pgautofailover.sql) + * never uses the plain --name given at create-archiver time as an + * ARCHIVING row's own nodename, so the ordinary "wait until + * state is " form (which matches on nodename = $1) can't see these + * rows at all, let alone disambiguate more than one. Group is + * optional: omit it when the formation has exactly one archiver + * membership (the common case, and formationid alone is unambiguous), + * give it to disambiguate a multi-group Citus formation. + */ + | T_WAIT T_UNTIL T_ARCHIVER T_STATE state_op wait_state_name T_IN T_IDENT opt_wait_group opt_timeout + { + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + if ($9 >= 0) + { + sformat($$->args, sizeof($$->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'" + " AND groupid = %d", $8, $9); + } + else + { + sformat($$->args, sizeof($$->args), + "SELECT reportedstate::text FROM pgautofailover.node" + " WHERE nodename LIKE 'archiver-%%' AND formationid = '%s'", $8); + } + strlcpy($$->expected, $6, sizeof($$->expected)); + $$->timeoutSeconds = $10; + free($6); free($8); + } + /* + * wait until basebackup is in / [timeout Ns] + * + * Sugar for polling pgautofailover.get_latest_basebackup(). + * is validated here rather than tokenized: it's the one piece of this + * command that's genuinely open content (a column name), not fixed + * syntax, so a clear parse-time error beats a cryptic runtime SQL one. + */ + | T_WAIT T_UNTIL T_BASEBACKUP T_IDENT T_IS T_IDENT T_IN T_IDENT T_SLASH T_INTEGER opt_timeout + { + if (strcmp($4, "source") != 0 && + strcmp($4, "status") != 0 && + strcmp($4, "replaymode") != 0) + { + fprintf(stderr, + "pgaftest: line %d: \"wait until basebackup %s ...\" -- " + "unknown property (expected source, status, or replaymode)\n", + pgaf_line_number, $4); + exit(1); + } + $$ = make_cmd(CMD_WAIT_SQL); + strlcpy($$->service, "monitor", sizeof($$->service)); + sformat($$->args, sizeof($$->args), + "SELECT %s::text FROM pgautofailover.get_latest_basebackup('%s', %d)", + $4, $8, $10); + strlcpy($$->expected, $6, sizeof($$->expected)); + $$->timeoutSeconds = $11; + free($4); free($6); free($8); + } ; /* @@ -1680,6 +1783,27 @@ ident_or_string: | T_STRING { $$ = $1; } ; +/* + * wait_state_name — a state name for "wait until archiver state is X", + * accepting both known FSM state tokens and bare idents (e.g. "archiving", + * which has no T_FS_* token of its own -- see fsm_state's own list). Always + * returns a heap-owned string so the caller can unconditionally free() it, + * unlike fsm_state itself (whose branches return static literals). + */ +wait_state_name: + fsm_state { $$ = strdup($1); } + | T_IDENT { $$ = $1; } + ; + +/* + * opt_wait_group — optional "/" suffix for "wait until archiver + * state is X in [/]". -1 means "no group filter". + */ +opt_wait_group: + /* empty */ { $$ = -1; } + | T_SLASH T_INTEGER { $$ = $2; } + ; + %% /* diff --git a/src/bin/pgaftest/test_spec_scan.c b/src/bin/pgaftest/test_spec_scan.c index 8e5d33bb0..2d16af03b 100644 --- a/src/bin/pgaftest/test_spec_scan.c +++ b/src/bin/pgaftest/test_spec_scan.c @@ -356,8 +356,8 @@ static void yynoreturn yy_fatal_error ( const char* msg ); (yy_hold_char) = *yy_cp; \ *yy_cp = '\0'; \ (yy_c_buf_p) = yy_cp; -#define YY_NUM_RULES 164 -#define YY_END_OF_BUFFER 165 +#define YY_NUM_RULES 170 +#define YY_END_OF_BUFFER 171 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -365,146 +365,149 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[1259] = +static const flex_int16_t yy_accept[1284] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 18, 17, 16, 18, 1, 12, 11, 13, 13, - 13, 13, 13, 13, 15, 164, 21, 20, 164, 19, + 171, 18, 17, 16, 18, 1, 12, 11, 13, 13, + 13, 13, 13, 13, 15, 170, 21, 20, 170, 19, 63, 62, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 65, 66, 103, 102, 164, 101, 139, 154, 138, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 156, - 157, 160, 159, 161, 162, 163, 17, 0, 14, 1, - 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, + 100, 65, 66, 103, 102, 170, 101, 145, 119, 160, + 144, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 162, 163, 166, 165, 167, 168, 169, 17, 0, + 14, 1, 12, 12, 13, 13, 13, 13, 13, 13, - 21, 0, 64, 19, 63, 63, 100, 100, 100, 100, + 13, 13, 21, 0, 64, 19, 63, 63, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 103, 0, 155, 101, - 154, 154, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 130, - 136, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 160, 159, 162, 13, 13, - - 13, 13, 13, 13, 13, 13, 45, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 103, 0, + 161, 101, 160, 160, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 136, 142, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 166, + + 165, 168, 13, 13, 13, 13, 13, 13, 13, 13, + 45, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 25, 100, 100, - 100, 100, 100, 100, 135, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 141, 148, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 151, 158, 158, 158, 158, 158, - 158, 158, 158, 106, 158, 147, 158, 158, 113, 158, - - 158, 158, 158, 158, 158, 158, 158, 158, 13, 13, - 13, 4, 13, 13, 9, 13, 100, 100, 100, 27, + 100, 25, 100, 100, 100, 100, 100, 100, 141, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 147, 154, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 157, 164, 164, 164, 164, 164, 164, 164, 164, 106, + + 164, 164, 153, 164, 164, 113, 164, 164, 164, 164, + 164, 164, 164, 114, 164, 164, 13, 13, 13, 4, + 13, 13, 9, 13, 100, 100, 100, 27, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 100, 67, 100, - 100, 100, 100, 100, 100, 100, 30, 100, 100, 54, - 100, 100, 100, 100, 100, 100, 100, 100, 100, 44, - 100, 100, 100, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 125, 158, 158, 158, 105, 158, 158, - 158, 158, 158, 67, 158, 158, 129, 150, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - - 158, 158, 158, 158, 158, 158, 158, 142, 127, 158, - 158, 158, 108, 158, 137, 13, 13, 13, 13, 7, - 13, 100, 100, 34, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 43, 100, 100, - 100, 53, 24, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 67, 100, 100, 100, + 100, 100, 100, 100, 30, 100, 100, 54, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 44, 100, 100, + 100, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 131, 164, 164, 164, 105, 164, 164, + 164, 164, 164, 67, 164, 164, 135, 156, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 148, 133, + 164, 164, 164, 108, 164, 143, 13, 13, 13, 13, + 7, 13, 100, 100, 34, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 43, 100, + 100, 100, 53, 24, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 115, 158, 158, 158, 158, 158, 158, 134, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - - 158, 158, 158, 158, 158, 158, 158, 158, 122, 126, - 131, 143, 158, 158, 158, 158, 158, 109, 158, 158, - 144, 13, 13, 13, 13, 13, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 40, 100, 100, 100, 100, + 100, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 121, 164, 164, 164, 164, + + 164, 164, 140, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 128, 132, 137, 149, 164, 164, 164, 164, + 164, 109, 164, 164, 150, 13, 13, 13, 13, 13, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 40, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 39, 100, 49, 100, 100, 100, 100, 100, - 100, 100, 52, 100, 100, 100, 68, 100, 100, 100, - 100, 48, 100, 100, 100, 100, 100, 100, 32, 158, - 158, 112, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 114, 158, 158, 158, 158, 149, 158, - - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 68, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 13, - 13, 2, 3, 13, 13, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 100, 100, 74, - 100, 99, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 22, 100, 100, 100, 100, - 69, 100, 100, 100, 100, 100, 100, 47, 100, 100, - 100, 100, 100, 100, 100, 158, 158, 158, 158, 158, - 123, 121, 158, 158, 158, 74, 158, 158, 99, 158, - - 158, 158, 158, 158, 158, 158, 158, 158, 158, 153, - 119, 124, 158, 117, 158, 158, 158, 69, 116, 110, - 158, 158, 158, 158, 158, 128, 146, 111, 158, 158, - 158, 158, 158, 158, 13, 13, 10, 8, 100, 100, + 100, 100, 100, 100, 100, 100, 39, 100, 49, 100, + 100, 100, 100, 100, 100, 100, 52, 100, 100, 100, + 68, 100, 100, 100, 100, 48, 100, 100, 100, 100, + 100, 100, 32, 164, 164, 164, 112, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 120, + 164, 164, 164, 164, 155, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 68, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 13, 13, 2, 3, + 13, 13, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 74, 100, 99, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 22, 100, 100, 100, 100, 69, 100, 100, + 100, 100, 100, 100, 47, 100, 100, 100, 100, 100, + + 100, 100, 164, 164, 164, 164, 164, 164, 164, 129, + 127, 164, 164, 164, 74, 164, 164, 99, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 159, 125, + 130, 164, 123, 164, 164, 164, 69, 122, 110, 164, + 164, 164, 115, 164, 164, 134, 152, 111, 164, 164, + 164, 164, 164, 164, 13, 13, 10, 8, 100, 100, 33, 100, 100, 100, 100, 100, 100, 100, 100, 41, 100, 100, 77, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 29, 37, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 158, 158, 158, 158, 158, - 152, 158, 158, 158, 77, 158, 118, 158, 158, 158, - - 158, 158, 158, 158, 158, 0, 158, 140, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 13, 13, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 100, 28, 100, - 42, 46, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 78, 100, 100, - 36, 100, 100, 100, 100, 100, 100, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 158, 28, 158, - 158, 158, 158, 158, 0, 158, 158, 158, 158, 158, - 158, 158, 78, 158, 158, 158, 158, 158, 158, 158, - - 158, 13, 13, 100, 100, 100, 100, 100, 79, 100, + + 100, 100, 100, 100, 100, 164, 164, 116, 118, 164, + 164, 164, 164, 158, 164, 164, 164, 77, 164, 124, + 164, 164, 164, 164, 164, 164, 164, 164, 0, 164, + 146, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 13, 13, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 28, 100, 42, 46, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 35, 100, 100, 100, - 100, 100, 94, 93, 100, 100, 100, 100, 100, 100, - 100, 100, 158, 158, 158, 158, 79, 158, 158, 120, - 104, 158, 158, 158, 158, 158, 158, 158, 0, 107, - 158, 158, 158, 158, 94, 93, 158, 158, 158, 158, - 158, 158, 158, 158, 13, 13, 100, 100, 26, 60, - 100, 100, 100, 31, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 84, 100, 100, 100, + 78, 100, 100, 36, 100, 100, 100, 100, 100, 100, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 28, 164, 164, 164, 164, 164, 0, 164, + 164, 164, 164, 164, 164, 164, 78, 164, 164, 164, + 164, 164, 164, 164, 164, 13, 13, 100, 100, 100, + 100, 100, 79, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 158, 158, 158, 158, 158, 158, 158, 158, 158, - 158, 158, 158, 84, 0, 158, 158, 158, 158, 158, - 158, 158, 158, 158, 158, 158, 158, 13, 6, 100, - 100, 100, 100, 100, 100, 100, 100, 100, 96, 95, - 23, 86, 100, 85, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 71, 73, 100, 70, 72, - 158, 158, 158, 158, 158, 158, 96, 95, 86, 158, - 85, 158, 0, 158, 158, 158, 158, 158, 158, 158, - 71, 73, 158, 70, 72, 13, 100, 100, 100, 100, + 35, 100, 100, 100, 100, 100, 94, 93, 100, 100, + 100, 100, 100, 100, 100, 100, 164, 164, 164, 117, + 164, 79, 164, 164, 126, 104, 164, 164, 164, 164, + 164, 164, 164, 0, 107, 164, 164, 164, 164, 94, + 93, 164, 164, 164, 164, 164, 164, 164, 164, 13, + 13, 100, 100, 26, 60, 100, 100, 100, 31, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 100, 100, 100, 100, 100, 100, 158, - 158, 158, 158, 158, 158, 158, 158, 0, 158, 158, - 158, 158, 158, 158, 158, 158, 13, 88, 87, 100, - 100, 100, 56, 76, 75, 100, 98, 97, 61, 100, + 100, 84, 100, 100, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 84, 0, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 13, 6, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 96, 95, 23, 86, 100, 85, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, + 71, 73, 100, 70, 72, 164, 164, 164, 164, 164, + + 164, 96, 95, 86, 164, 85, 164, 0, 164, 164, + 164, 164, 164, 164, 164, 71, 73, 164, 70, 72, + 13, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 88, 87, 132, 158, 76, 75, 98, 97, 0, 158, - 158, 158, 158, 158, 158, 158, 158, 13, 100, 100, - 50, 100, 100, 100, 100, 100, 100, 100, 100, 100, - 100, 100, 100, 158, 145, 158, 158, 158, 158, 158, - - 158, 158, 158, 13, 100, 100, 100, 38, 100, 100, - 100, 100, 100, 100, 83, 82, 92, 91, 158, 158, - 158, 158, 158, 83, 82, 92, 91, 5, 100, 100, - 59, 100, 81, 100, 80, 100, 100, 158, 158, 81, - 158, 80, 51, 55, 100, 100, 100, 57, 133, 158, - 158, 90, 89, 100, 90, 89, 58, 0 + 100, 100, 100, 100, 164, 164, 164, 164, 164, 164, + 164, 164, 0, 164, 164, 164, 164, 164, 164, 164, + 164, 13, 88, 87, 100, 100, 100, 56, 76, 75, + 100, 98, 97, 61, 100, 100, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 88, 87, 138, 164, 76, + 75, 98, 97, 0, 164, 164, 164, 164, 164, 164, + + 164, 164, 13, 100, 100, 50, 100, 100, 100, 100, + 100, 100, 100, 100, 100, 100, 100, 100, 164, 151, + 164, 164, 164, 164, 164, 164, 164, 164, 13, 100, + 100, 100, 38, 100, 100, 100, 100, 100, 100, 83, + 82, 92, 91, 164, 164, 164, 164, 164, 83, 82, + 92, 91, 5, 100, 100, 59, 100, 81, 100, 80, + 100, 100, 164, 164, 81, 164, 80, 51, 55, 100, + 100, 100, 57, 139, 164, 164, 90, 89, 100, 90, + 89, 58, 0 } ; static const YY_CHAR yy_ec[256] = @@ -513,16 +516,16 @@ static const YY_CHAR yy_ec[256] = 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 5, 6, 1, 1, 1, 1, 1, - 1, 1, 1, 7, 8, 1, 1, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, - 10, 1, 1, 1, 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 11, 11, 12, 11, 11, 11, 11, 11, 11, 11, - 1, 1, 1, 1, 13, 1, 14, 15, 16, 17, - - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 11, 39, 1, 40, 1, 1, 1, 1, 1, + 1, 1, 1, 7, 8, 1, 9, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 1, 1, 1, + 11, 1, 1, 1, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 13, 12, 12, 12, 12, 12, 12, 12, + 1, 1, 1, 1, 14, 1, 15, 16, 17, 18, + + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 12, 40, 1, 41, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -539,616 +542,629 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[41] = +static const YY_CHAR yy_meta[42] = { 0, - 1, 2, 3, 1, 1, 1, 1, 4, 4, 1, + 1, 2, 3, 1, 1, 1, 1, 4, 1, 4, + 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 1, 1 + 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, + 1 } ; -static const flex_int16_t yy_base[1272] = +static const flex_int16_t yy_base[1297] = { 0, - 0, 0, 40, 0, 80, 0, 119, 121, 1351, 1350, - 1352, 1355, 123, 1355, 1346, 0, 117, 1355, 0, 106, - 1322, 1321, 112, 1330, 1355, 1355, 130, 1355, 1342, 0, - 124, 1355, 0, 108, 1324, 124, 123, 1308, 129, 1313, - 121, 1315, 144, 136, 125, 137, 1324, 145, 1310, 1312, - 146, 1355, 1355, 167, 1355, 1334, 0, 1355, 161, 1355, - 0, 153, 147, 166, 150, 158, 158, 164, 1310, 1315, - 1308, 1321, 174, 181, 180, 185, 183, 1307, 199, 1355, - 1355, 0, 1331, 1355, 0, 1355, 204, 1327, 1355, 0, - 207, 1355, 0, 1298, 1296, 1302, 1311, 192, 1309, 1312, - - 222, 1320, 1355, 0, 218, 1355, 0, 1307, 1294, 1306, - 1283, 1287, 1292, 184, 1285, 1289, 1298, 214, 209, 1282, - 203, 1283, 1285, 217, 1290, 1289, 1276, 1289, 1276, 1285, - 1279, 224, 1279, 1272, 1272, 216, 216, 1286, 1274, 1275, - 1271, 1266, 1263, 1271, 1273, 1263, 240, 1288, 1355, 0, - 237, 1355, 0, 1275, 1262, 1258, 220, 225, 1263, 1256, - 1251, 234, 1255, 236, 234, 1254, 1258, 1250, 1254, 235, - 0, 1259, 1255, 1259, 237, 1245, 238, 1245, 1245, 1262, - 1242, 245, 1244, 1245, 167, 1244, 1252, 1244, 246, 1237, - 1241, 1233, 1243, 1242, 1230, 0, 1260, 0, 1227, 1228, - - 1237, 1240, 1223, 1222, 1226, 1223, 0, 1228, 1231, 1224, - 1229, 1232, 1231, 1231, 1212, 1214, 1230, 1221, 1224, 1213, - 1218, 1210, 1220, 1205, 1203, 1209, 1200, 1213, 1214, 1198, - 1203, 1202, 1214, 1194, 1199, 1203, 1198, 1205, 1214, 1189, - 1187, 1190, 1192, 1195, 247, 1188, 1195, 0, 1185, 1184, - 1194, 1177, 1177, 1185, 0, 1183, 258, 1190, 1190, 1176, - 250, 1176, 1187, 1175, 1179, 1171, 1171, 1182, 1179, 1171, - 1162, 1168, 0, 0, 1159, 1159, 1173, 1163, 1164, 1156, - 1160, 1170, 1149, 1166, 0, 1151, 1163, 1167, 1147, 1150, - 1152, 1151, 253, 0, 1148, 0, 1155, 1156, 0, 251, - - 1144, 1143, 1143, 1152, 1147, 1135, 1142, 1145, 1133, 1131, - 1130, 0, 1144, 1132, 0, 1143, 1121, 1136, 1141, 1148, - 1147, 1132, 1132, 1120, 1134, 1117, 1135, 1117, 1114, 1119, - 1116, 1117, 1125, 277, 1128, 1112, 1122, 1122, 1116, 278, - 1121, 1120, 1117, 1101, 1100, 1104, 0, 1099, 1094, 0, - 1115, 1114, 1099, 1104, 1094, 1097, 1098, 279, 1104, 0, - 1095, 280, 1102, 1081, 1087, 1097, 1094, 1094, 1086, 1095, - 1098, 1078, 1082, 0, 1082, 1079, 1076, 1098, 1089, 1076, - 286, 1089, 1073, 0, 1085, 287, 0, 0, 1067, 1078, - 1070, 1075, 1074, 1067, 1060, 1073, 1078, 1077, 1062, 1075, - - 1057, 1060, 1061, 1056, 1051, 1065, 1050, 0, 288, 1047, - 1052, 1054, 289, 1060, 0, 1069, 1058, 1047, 1047, 0, - 1045, 290, 1037, 0, 1045, 1038, 1052, 1046, 1059, 1044, - 1047, 1037, 1032, 1044, 1039, 1042, 1027, 0, 1039, 1038, - 1023, 0, 1047, 1032, 1039, 275, 277, 1031, 1013, 1023, - 1031, 1020, 1020, 1008, 1017, 1013, 1012, 1015, 1025, 1007, - 1022, 1020, 1006, 1005, 1017, 1007, 1015, 284, 286, 1001, - 306, 998, 1003, 1012, 1006, 995, 1010, 1003, 1006, 996, - 1000, 1003, 0, 1001, 986, 983, 998, 997, 982, 0, - 981, 291, 292, 995, 994, 980, 983, 982, 977, 974, - - 975, 974, 973, 970, 964, 968, 983, 981, 0, 0, - 0, 0, 967, 966, 978, 975, 960, 0, 296, 300, - 0, 295, 962, 961, 975, 954, 957, 956, 969, 968, - 957, 970, 956, 311, 955, 0, 973, 962, 322, 952, - 961, 955, 948, 947, 952, 940, 958, 946, 939, 951, - 937, 949, 0, 958, 0, 938, 933, 941, 935, 930, - 942, 921, 0, 944, 325, 943, 0, 938, 937, 937, - 936, 0, 938, 920, 917, 935, 917, 914, 0, 914, - 913, 0, 926, 929, 915, 923, 907, 912, 328, 911, - 910, 919, 921, 0, 916, 905, 904, 909, 0, 899, - - 911, 897, 909, 899, 893, 900, 901, 902, 895, 892, - 901, 900, 879, 898, 883, 329, 900, 0, 895, 894, - 894, 889, 876, 894, 876, 873, 891, 873, 870, 874, - 873, 0, 0, 882, 872, 880, 879, 865, 862, 860, - 860, 872, 866, 872, 875, 872, 870, 853, 852, 0, - 864, 0, 855, 851, 850, 852, 865, 845, 852, 854, - 859, 852, 857, 840, 857, 862, 836, 852, 850, 336, - 0, 833, 840, 839, 832, 833, 832, 0, 842, 837, - 835, 839, 830, 122, 247, 250, 261, 288, 301, 320, - 0, 0, 314, 314, 315, 0, 332, 331, 0, 330, - - 322, 323, 324, 328, 335, 342, 337, 344, 347, 0, - 0, 0, 360, 0, 349, 334, 359, 0, 0, 0, - 343, 344, 339, 342, 344, 0, 0, 0, 352, 353, - 362, 355, 356, 365, 352, 350, 0, 0, 349, 350, - 0, 363, 354, 368, 353, 354, 373, 357, 366, 0, - 370, 371, 0, 367, 359, 360, 370, 367, 381, 362, - 375, 374, 377, 376, 372, 379, 378, 380, 0, 0, - 383, 384, 389, 382, 383, 378, 392, 393, 402, 393, - 395, 395, 396, 398, 398, 393, 394, 420, 411, 396, - 0, 409, 410, 417, 0, 409, 0, 399, 400, 410, - - 412, 411, 414, 413, 415, 441, 413, 0, 421, 422, - 417, 420, 415, 429, 430, 429, 431, 431, 432, 434, - 434, 431, 439, 431, 432, 438, 451, 460, 440, 438, - 443, 444, 439, 449, 450, 469, 464, 465, 0, 460, - 0, 0, 467, 455, 469, 457, 471, 470, 473, 457, - 475, 459, 477, 461, 465, 467, 468, 0, 474, 475, - 0, 465, 485, 483, 468, 488, 486, 471, 472, 474, - 499, 479, 483, 484, 478, 480, 499, 500, 0, 501, - 489, 503, 491, 503, 499, 496, 508, 492, 510, 494, - 499, 500, 0, 506, 507, 497, 517, 515, 500, 520, - - 518, 519, 519, 516, 517, 523, 523, 513, 0, 510, - 517, 514, 514, 529, 530, 514, 519, 520, 534, 522, - 537, 524, 539, 526, 540, 527, 0, 538, 533, 540, - 535, 537, 0, 0, 549, 550, 549, 537, 554, 552, - 540, 557, 551, 552, 542, 547, 0, 559, 560, 0, - 0, 548, 549, 550, 565, 552, 567, 567, 555, 0, - 565, 560, 567, 562, 0, 0, 575, 576, 575, 563, - 580, 578, 566, 583, 577, 569, 574, 575, 0, 0, - 572, 586, 588, 0, 573, 579, 580, 591, 593, 594, - 579, 575, 600, 577, 602, 584, 0, 586, 592, 594, - - 594, 596, 615, 610, 611, 599, 589, 590, 602, 592, - 593, 605, 606, 620, 604, 608, 609, 621, 622, 602, - 627, 604, 629, 0, 616, 618, 620, 620, 622, 635, - 636, 624, 614, 615, 627, 617, 618, 630, 0, 638, - 639, 638, 630, 648, 645, 630, 631, 635, 0, 0, - 0, 0, 636, 0, 637, 635, 634, 638, 644, 640, - 646, 646, 644, 645, 665, 0, 0, 666, 0, 0, - 661, 662, 650, 662, 651, 652, 0, 0, 0, 656, - 0, 657, 655, 657, 663, 659, 665, 661, 662, 682, - 0, 0, 683, 0, 0, 684, 667, 668, 673, 694, - - 672, 673, 672, 673, 675, 670, 671, 681, 683, 694, - 680, 696, 682, 702, 683, 696, 697, 693, 694, 690, - 691, 706, 697, 693, 694, 690, 691, 710, 713, 699, - 715, 701, 713, 714, 710, 711, 706, 0, 0, 709, - 714, 704, 0, 0, 0, 721, 0, 0, 0, 713, - 718, 724, 720, 726, 717, 722, 723, 724, 737, 738, - 0, 0, 0, 724, 0, 0, 0, 0, 735, 730, - 736, 732, 738, 733, 734, 747, 748, 737, 744, 753, - 0, 740, 752, 756, 743, 758, 745, 742, 744, 749, - 750, 760, 761, 758, 1355, 767, 754, 769, 756, 758, - - 759, 769, 770, 758, 757, 765, 765, 0, 766, 767, - 768, 769, 761, 764, 0, 0, 0, 0, 766, 773, - 774, 775, 776, 0, 0, 0, 0, 0, 766, 787, - 0, 790, 0, 791, 0, 780, 783, 772, 795, 0, - 796, 0, 0, 0, 795, 796, 784, 0, 0, 798, - 799, 0, 0, 801, 0, 0, 0, 1355, 818, 822, - 826, 830, 829, 834, 838, 837, 842, 846, 845, 850, - 854 + 0, 0, 41, 0, 82, 0, 122, 124, 1377, 1376, + 1378, 1381, 126, 1381, 1372, 0, 119, 1381, 0, 108, + 1347, 1346, 114, 1355, 1381, 1381, 133, 1381, 1368, 0, + 126, 1381, 0, 110, 1349, 126, 125, 1333, 131, 1338, + 123, 1340, 146, 138, 127, 139, 1349, 147, 1335, 1337, + 148, 1381, 1381, 170, 1381, 1360, 0, 1381, 1381, 163, + 1381, 0, 155, 1349, 149, 170, 152, 171, 160, 170, + 1334, 1339, 1332, 1345, 172, 190, 175, 189, 184, 1331, + 202, 1381, 1381, 0, 1356, 1381, 0, 1381, 193, 1352, + 1381, 0, 203, 1381, 0, 1322, 1320, 1326, 1335, 187, + + 1333, 1336, 224, 1345, 1381, 0, 217, 1381, 0, 1331, + 1318, 1330, 1307, 1311, 1316, 201, 1309, 1313, 1322, 217, + 216, 1306, 206, 1307, 1309, 219, 1314, 1313, 1300, 1313, + 1300, 1309, 1303, 230, 1303, 1296, 1296, 224, 219, 1310, + 1298, 1299, 1295, 1290, 1287, 1295, 1297, 1287, 249, 1313, + 1381, 0, 242, 1381, 0, 1299, 1286, 1298, 1281, 1280, + 226, 214, 1285, 1278, 1273, 241, 1277, 238, 236, 1276, + 1280, 1272, 1276, 238, 0, 1281, 1277, 1281, 240, 1267, + 246, 1267, 1267, 1284, 1264, 248, 1266, 1267, 255, 1266, + 1274, 1266, 263, 1259, 1263, 1255, 258, 1265, 1253, 0, + + 1284, 0, 1250, 1251, 1260, 1263, 1246, 1245, 1249, 1246, + 0, 1251, 1254, 1247, 1252, 1255, 1254, 1254, 1235, 1237, + 1253, 1244, 1247, 1236, 1241, 1233, 1243, 1228, 1226, 1232, + 1223, 1236, 1237, 1221, 1226, 1225, 1237, 1217, 1222, 1226, + 1221, 1228, 1238, 1212, 1210, 1213, 1215, 1218, 257, 1211, + 1218, 0, 1208, 1207, 1217, 1200, 1200, 1208, 0, 1206, + 1209, 268, 1211, 1211, 1211, 1197, 245, 1197, 1208, 1196, + 1200, 1192, 1192, 1203, 1200, 1192, 1183, 1189, 0, 0, + 1180, 1180, 1194, 1184, 1185, 1177, 1181, 1191, 1170, 1187, + 0, 1172, 1184, 1188, 1168, 1171, 1173, 1172, 259, 0, + + 1169, 1170, 0, 1175, 1176, 0, 261, 1164, 1163, 1163, + 1172, 1167, 1155, 0, 1162, 1165, 1153, 1151, 1150, 0, + 1164, 1152, 0, 1163, 1141, 1156, 1161, 1169, 1168, 1152, + 1152, 1140, 1154, 1137, 1155, 1137, 1134, 1139, 1136, 1137, + 1145, 282, 1148, 1132, 1142, 1142, 1136, 289, 1141, 1140, + 1137, 1121, 1120, 1124, 0, 1119, 1114, 0, 1135, 1134, + 1119, 1124, 1114, 1117, 1118, 290, 1124, 0, 1115, 291, + 1122, 1101, 1116, 1106, 1116, 1120, 1112, 1112, 1104, 1113, + 1116, 1096, 1100, 0, 1100, 1097, 1094, 1117, 1107, 1094, + 293, 1107, 1091, 0, 1103, 294, 0, 0, 1085, 1096, + + 1088, 1093, 1092, 1085, 1078, 1091, 1096, 1095, 1080, 1093, + 1075, 1078, 1086, 1078, 1073, 1068, 1082, 1067, 0, 298, + 1064, 1069, 1071, 301, 1077, 0, 1087, 1075, 1064, 1064, + 0, 1062, 302, 1054, 0, 1062, 1055, 1069, 1063, 1077, + 1061, 1064, 1054, 1049, 1061, 1056, 1059, 1044, 0, 1056, + 1055, 1040, 0, 1065, 1049, 1056, 281, 288, 1048, 1030, + 1040, 1048, 1037, 1037, 1025, 1034, 1030, 1029, 1032, 1042, + 1024, 1039, 1037, 1023, 1022, 1034, 1024, 1032, 292, 296, + 1018, 316, 1013, 1014, 1019, 1031, 1027, 1021, 1010, 1025, + 1018, 1021, 1011, 1015, 1018, 0, 1016, 1001, 998, 1013, + + 1012, 997, 0, 996, 301, 302, 1010, 1009, 995, 998, + 997, 992, 989, 990, 989, 988, 985, 979, 983, 998, + 987, 995, 0, 0, 0, 0, 981, 980, 992, 989, + 974, 0, 306, 310, 0, 310, 976, 975, 989, 968, + 971, 970, 983, 982, 971, 984, 970, 321, 969, 0, + 988, 976, 334, 966, 975, 969, 962, 961, 966, 954, + 972, 960, 953, 965, 951, 963, 0, 973, 0, 952, + 947, 955, 949, 944, 956, 935, 0, 958, 336, 957, + 0, 952, 951, 951, 950, 0, 952, 934, 931, 949, + 931, 928, 0, 928, 927, 940, 0, 939, 940, 941, + + 927, 935, 919, 924, 337, 923, 922, 931, 933, 0, + 928, 917, 916, 921, 0, 911, 923, 909, 921, 911, + 905, 912, 913, 914, 907, 904, 913, 912, 891, 910, + 895, 345, 912, 892, 0, 906, 905, 905, 900, 887, + 905, 887, 884, 902, 884, 881, 885, 884, 0, 0, + 893, 883, 891, 890, 876, 873, 871, 871, 883, 877, + 883, 886, 883, 881, 864, 863, 0, 875, 0, 866, + 862, 861, 863, 876, 856, 863, 865, 870, 863, 868, + 850, 864, 870, 113, 158, 196, 348, 0, 224, 239, + 240, 262, 283, 287, 0, 322, 323, 326, 342, 335, + + 337, 346, 344, 345, 347, 348, 342, 334, 348, 0, + 0, 337, 337, 338, 0, 354, 353, 0, 352, 344, + 345, 346, 351, 358, 365, 360, 367, 370, 0, 0, + 0, 384, 0, 372, 357, 383, 0, 0, 0, 366, + 367, 362, 0, 365, 366, 0, 0, 0, 375, 376, + 385, 378, 379, 388, 375, 373, 0, 0, 372, 373, + 0, 386, 377, 391, 376, 377, 396, 380, 389, 0, + 393, 394, 0, 390, 382, 383, 393, 390, 404, 385, + 398, 397, 400, 399, 395, 402, 401, 403, 0, 0, + 406, 407, 412, 405, 406, 401, 415, 416, 425, 416, + + 418, 418, 419, 421, 421, 416, 417, 0, 0, 444, + 418, 435, 420, 0, 433, 434, 441, 0, 433, 0, + 423, 424, 434, 436, 435, 438, 437, 439, 466, 437, + 0, 445, 446, 441, 444, 439, 453, 454, 453, 455, + 455, 456, 458, 458, 455, 463, 455, 456, 462, 475, + 485, 464, 462, 467, 468, 463, 472, 474, 494, 488, + 489, 0, 484, 0, 0, 491, 479, 493, 481, 495, + 494, 497, 481, 499, 483, 501, 485, 489, 491, 492, + 0, 498, 499, 0, 489, 509, 507, 492, 512, 510, + 495, 496, 498, 502, 525, 504, 508, 509, 503, 505, + + 524, 525, 0, 526, 514, 528, 516, 528, 524, 521, + 533, 517, 535, 519, 524, 525, 0, 531, 532, 522, + 542, 540, 525, 545, 543, 544, 544, 541, 542, 548, + 548, 538, 0, 535, 542, 539, 539, 554, 555, 539, + 544, 545, 559, 547, 562, 549, 564, 551, 565, 552, + 0, 563, 558, 565, 560, 562, 0, 0, 574, 575, + 574, 562, 579, 577, 565, 582, 576, 577, 567, 0, + 572, 0, 584, 585, 0, 0, 573, 574, 575, 590, + 577, 592, 592, 580, 0, 590, 585, 592, 587, 0, + 0, 600, 601, 600, 588, 605, 603, 591, 608, 602, + + 594, 599, 600, 0, 0, 597, 611, 613, 0, 598, + 604, 605, 616, 618, 619, 604, 600, 625, 602, 627, + 609, 0, 611, 617, 619, 619, 621, 641, 635, 636, + 624, 614, 615, 627, 617, 618, 630, 631, 645, 629, + 633, 634, 646, 647, 627, 652, 629, 654, 0, 641, + 643, 645, 645, 647, 660, 661, 649, 639, 640, 652, + 642, 643, 655, 0, 663, 664, 663, 655, 673, 670, + 655, 656, 660, 0, 0, 0, 0, 661, 0, 662, + 660, 659, 663, 669, 665, 671, 671, 669, 670, 690, + 0, 0, 691, 0, 0, 686, 687, 675, 687, 676, + + 677, 0, 0, 0, 681, 0, 682, 680, 682, 688, + 684, 690, 686, 687, 707, 0, 0, 708, 0, 0, + 709, 692, 693, 698, 720, 697, 698, 697, 698, 700, + 695, 696, 706, 708, 719, 705, 721, 707, 727, 708, + 721, 722, 718, 719, 715, 716, 731, 722, 718, 719, + 715, 716, 735, 738, 724, 740, 726, 738, 739, 735, + 736, 731, 0, 0, 734, 739, 729, 0, 0, 0, + 746, 0, 0, 0, 738, 743, 749, 745, 751, 742, + 747, 748, 749, 762, 763, 0, 0, 0, 749, 0, + 0, 0, 0, 760, 755, 761, 757, 763, 758, 759, + + 772, 773, 762, 769, 778, 0, 765, 777, 781, 768, + 783, 770, 767, 769, 774, 775, 785, 786, 783, 1381, + 792, 779, 794, 781, 783, 784, 794, 795, 783, 782, + 790, 790, 0, 791, 792, 793, 794, 786, 789, 0, + 0, 0, 0, 791, 798, 799, 800, 801, 0, 0, + 0, 0, 0, 791, 812, 0, 815, 0, 816, 0, + 805, 808, 797, 820, 0, 821, 0, 0, 0, 820, + 821, 809, 0, 0, 823, 824, 0, 0, 826, 0, + 0, 0, 1381, 844, 848, 852, 856, 855, 860, 864, + 863, 868, 872, 871, 876, 880 + } ; -static const flex_int16_t yy_def[1272] = +static const flex_int16_t yy_def[1297] = { 0, - 1258, 1, 1258, 3, 1258, 5, 1259, 1259, 1260, 1260, - 1258, 1258, 1258, 1258, 1261, 1262, 1258, 1258, 1263, 1263, - 1263, 1263, 1263, 1263, 1258, 1258, 1258, 1258, 1264, 1265, - 1258, 1258, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1258, 1258, 1258, 1258, 1267, 1268, 1258, 1258, 1258, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, - 1258, 1270, 1258, 1258, 1271, 1258, 1258, 1261, 1258, 1262, - 1258, 1258, 1263, 1263, 1263, 1263, 1263, 1263, 1263, 1263, - - 1258, 1264, 1258, 1265, 1258, 1258, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1258, 1267, 1258, 1268, - 1258, 1258, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1270, 1258, 1271, 1263, 1263, - - 1263, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1263, - 1263, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1263, 1263, 1263, 1263, 1263, - 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, - 1263, 1263, 1263, 1263, 1263, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1263, 1263, 1263, 1263, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1263, 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - - 1269, 1263, 1263, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1263, 1263, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1263, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1258, 1269, 1269, 1269, 1269, 1269, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, 1266, 1266, - - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1258, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1269, 1258, 1269, 1269, 1269, 1269, 1269, - - 1269, 1269, 1269, 1263, 1266, 1266, 1266, 1266, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, - 1269, 1269, 1269, 1269, 1269, 1269, 1269, 1263, 1266, 1266, - 1266, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, 1269, - 1269, 1269, 1266, 1266, 1266, 1266, 1266, 1266, 1269, 1269, - 1269, 1266, 1266, 1266, 1269, 1269, 1266, 0, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258 + 1283, 1, 1283, 3, 1283, 5, 1284, 1284, 1285, 1285, + 1283, 1283, 1283, 1283, 1286, 1287, 1283, 1283, 1288, 1288, + 1288, 1288, 1288, 1288, 1283, 1283, 1283, 1283, 1289, 1290, + 1283, 1283, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1283, 1283, 1283, 1283, 1292, 1293, 1283, 1283, 1283, + 1283, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1283, 1283, 1295, 1283, 1283, 1296, 1283, 1283, 1286, + 1283, 1287, 1283, 1283, 1288, 1288, 1288, 1288, 1288, 1288, + + 1288, 1288, 1283, 1289, 1283, 1290, 1283, 1283, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1283, 1292, + 1283, 1293, 1283, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1295, + + 1283, 1296, 1288, 1288, 1288, 1288, 1288, 1288, 1288, 1288, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, 1288, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, + 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + + 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1288, 1288, 1288, 1288, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1288, 1288, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1288, 1288, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1288, + + 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1288, 1288, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1283, 1294, 1294, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1294, 1283, 1294, 1294, 1294, 1294, 1294, 1294, + + 1294, 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1294, 1283, + 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1294, 1288, 1291, + 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1294, 1294, + 1294, 1294, 1288, 1291, 1291, 1291, 1291, 1291, 1291, 1291, + 1291, 1291, 1294, 1294, 1294, 1294, 1294, 1291, 1291, 1291, + 1291, 1291, 1291, 1294, 1294, 1294, 1291, 1291, 1291, 1294, + 1294, 1291, 0, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283 + } ; -static const flex_int16_t yy_nxt[1396] = +static const flex_int16_t yy_nxt[1423] = { 0, - 12, 13, 14, 13, 15, 16, 12, 12, 17, 18, - 19, 19, 19, 19, 19, 20, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 21, 22, 19, 19, 19, - 19, 23, 24, 19, 19, 19, 19, 19, 25, 12, - 26, 27, 28, 27, 29, 30, 26, 26, 31, 32, - 33, 33, 33, 34, 35, 36, 37, 38, 39, 40, - 33, 41, 42, 33, 43, 44, 45, 33, 46, 33, - 47, 48, 49, 33, 50, 51, 33, 33, 52, 53, - 26, 54, 55, 54, 56, 57, 58, 26, 59, 60, - 61, 61, 61, 62, 61, 63, 64, 65, 66, 67, - - 61, 68, 69, 70, 71, 72, 73, 61, 74, 61, - 75, 76, 77, 78, 61, 79, 61, 61, 80, 81, - 83, 84, 83, 84, 87, 91, 87, 94, 92, 98, - 95, 101, 105, 101, 108, 106, 109, 114, 110, 111, - 118, 112, 121, 784, 99, 115, 124, 125, 92, 130, - 134, 116, 132, 119, 117, 106, 122, 127, 133, 145, - 157, 128, 138, 131, 135, 129, 139, 136, 147, 151, - 147, 165, 152, 146, 158, 168, 140, 141, 142, 154, - 163, 155, 295, 159, 156, 166, 164, 160, 169, 167, - 170, 176, 152, 161, 178, 171, 162, 183, 179, 296, - - 180, 177, 185, 190, 191, 87, 186, 87, 181, 187, - 213, 182, 193, 184, 188, 91, 214, 189, 92, 194, - 195, 203, 221, 101, 204, 101, 105, 224, 218, 106, - 228, 237, 219, 242, 225, 244, 222, 243, 92, 220, - 238, 147, 229, 147, 245, 151, 258, 265, 152, 106, - 260, 261, 259, 268, 284, 270, 276, 277, 281, 300, - 785, 266, 290, 301, 269, 271, 291, 786, 152, 282, - 285, 354, 292, 302, 355, 365, 370, 400, 787, 366, - 401, 405, 371, 406, 439, 446, 463, 468, 407, 440, - 447, 464, 469, 487, 492, 513, 519, 527, 488, 493, - - 514, 520, 528, 549, 788, 551, 550, 465, 552, 573, - 630, 576, 574, 580, 577, 575, 515, 578, 581, 600, - 602, 624, 601, 603, 625, 627, 643, 626, 628, 648, - 631, 629, 673, 789, 649, 694, 721, 674, 650, 790, - 695, 722, 644, 771, 696, 791, 792, 793, 772, 794, - 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, - 805, 806, 806, 806, 807, 808, 809, 811, 812, 813, - 814, 810, 815, 816, 817, 818, 819, 820, 821, 822, - 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, - 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, - - 843, 844, 845, 846, 847, 848, 849, 850, 851, 853, - 855, 852, 854, 856, 857, 858, 859, 860, 861, 862, - 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, - 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, - 883, 884, 806, 806, 806, 886, 887, 889, 891, 888, - 890, 892, 893, 894, 895, 896, 897, 898, 899, 900, - 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, - 911, 912, 913, 885, 914, 915, 916, 917, 918, 919, + 12, 13, 14, 13, 15, 16, 12, 12, 12, 17, + 18, 19, 19, 19, 19, 19, 20, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 21, 22, 19, 19, + 19, 19, 23, 24, 19, 19, 19, 19, 19, 25, + 12, 26, 27, 28, 27, 29, 30, 26, 26, 26, + 31, 32, 33, 33, 33, 34, 35, 36, 37, 38, + 39, 40, 33, 41, 42, 33, 43, 44, 45, 33, + 46, 33, 47, 48, 49, 33, 50, 51, 33, 33, + 52, 53, 26, 54, 55, 54, 56, 57, 58, 26, + 59, 60, 61, 62, 62, 62, 63, 64, 65, 66, + + 67, 68, 69, 62, 70, 71, 72, 73, 74, 75, + 62, 76, 62, 77, 78, 79, 80, 62, 81, 62, + 62, 82, 83, 85, 86, 85, 86, 89, 93, 89, + 96, 94, 100, 97, 103, 107, 103, 110, 108, 111, + 116, 112, 113, 120, 114, 123, 788, 101, 117, 126, + 127, 94, 132, 136, 118, 134, 121, 119, 108, 124, + 129, 135, 147, 161, 130, 140, 133, 137, 131, 141, + 138, 149, 153, 149, 789, 154, 148, 162, 172, 142, + 143, 144, 156, 167, 157, 169, 158, 159, 163, 168, + 180, 173, 164, 187, 89, 154, 89, 174, 165, 170, + + 181, 166, 175, 171, 182, 194, 195, 189, 183, 188, + 184, 190, 93, 790, 191, 94, 197, 207, 185, 192, + 208, 186, 193, 198, 199, 103, 107, 103, 217, 108, + 225, 228, 222, 232, 218, 94, 223, 241, 229, 248, + 266, 267, 246, 224, 226, 233, 247, 242, 249, 108, + 149, 153, 149, 264, 154, 271, 274, 793, 276, 265, + 282, 283, 287, 290, 794, 795, 296, 275, 277, 272, + 297, 301, 380, 288, 154, 302, 298, 307, 381, 291, + 313, 308, 362, 314, 410, 363, 374, 411, 303, 450, + 375, 309, 416, 796, 417, 451, 457, 474, 479, 418, + + 500, 505, 458, 475, 480, 527, 501, 506, 533, 541, + 563, 528, 797, 564, 534, 542, 798, 565, 587, 476, + 566, 588, 590, 594, 589, 591, 647, 529, 592, 595, + 616, 618, 641, 617, 619, 642, 644, 660, 643, 645, + 799, 665, 646, 690, 713, 800, 648, 666, 801, 691, + 714, 667, 740, 661, 715, 791, 802, 803, 741, 804, + 805, 792, 806, 807, 808, 810, 811, 812, 813, 814, + 815, 816, 817, 818, 819, 820, 821, 822, 809, 823, + 824, 825, 826, 827, 828, 829, 829, 829, 830, 831, + 832, 834, 835, 836, 837, 838, 833, 839, 840, 841, + + 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, + 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, + 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, + 872, 873, 874, 876, 878, 875, 877, 879, 880, 881, + 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, + 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, + 902, 903, 904, 905, 906, 907, 908, 829, 829, 829, + 910, 911, 913, 915, 912, 914, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, - 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, - - 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, - 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, - 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, - 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, - 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, - 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, - 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, - 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, - 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, - 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, - - 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, - 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, - 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, - 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, - 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, - 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, - 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, - 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, - 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, - 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, - - 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, - 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, - 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, - 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, - 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, - 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, - 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, - 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, - 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, - 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, - - 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, - 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 82, 82, - 82, 82, 85, 85, 85, 85, 88, 88, 88, 88, - 90, 90, 93, 90, 102, 102, 102, 102, 104, 104, - 107, 104, 148, 148, 148, 148, 150, 150, 153, 150, - 196, 783, 782, 196, 198, 198, 781, 198, 780, 779, - 778, 777, 776, 775, 774, 773, 770, 769, 768, 767, - 766, 765, 764, 763, 762, 761, 760, 759, 758, 757, - 756, 755, 754, 753, 752, 751, 750, 749, 748, 747, - 746, 745, 744, 743, 742, 741, 740, 739, 738, 737, - + 930, 931, 932, 933, 934, 935, 936, 937, 938, 909, + + 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, + 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, + 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, + 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, + 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, + 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, + 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, + 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, + 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, + 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + + 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, + 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, + 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, + 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, + 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, + 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, + 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, + 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, + 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, + 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, + + 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, + 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, + 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, + 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, + 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, + 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, + 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, + 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, + 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, + 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, + + 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, + 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, + 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, + 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, + 1279, 1280, 1281, 1282, 84, 84, 84, 84, 87, 87, + 87, 87, 90, 90, 90, 90, 92, 92, 95, 92, + 104, 104, 104, 104, 106, 106, 109, 106, 150, 150, + 150, 150, 152, 152, 155, 152, 200, 787, 786, 200, + 202, 202, 785, 202, 784, 783, 782, 781, 780, 779, + 778, 777, 776, 775, 774, 773, 772, 771, 770, 769, + + 768, 767, 766, 765, 764, 763, 762, 761, 760, 759, + 758, 757, 756, 755, 754, 753, 752, 751, 750, 749, + 748, 747, 746, 745, 744, 743, 742, 739, 738, 737, 736, 735, 734, 733, 732, 731, 730, 729, 728, 727, - 726, 725, 724, 723, 720, 719, 718, 717, 716, 715, - 714, 713, 712, 711, 710, 709, 708, 707, 706, 705, - 704, 703, 702, 701, 700, 699, 698, 697, 693, 692, - 691, 690, 689, 688, 687, 686, 685, 684, 683, 682, - 681, 680, 679, 678, 677, 676, 675, 672, 671, 670, - 669, 668, 667, 666, 665, 664, 663, 662, 661, 660, - 659, 658, 657, 656, 655, 654, 653, 652, 651, 647, - 646, 645, 642, 641, 640, 639, 638, 637, 636, 635, - 634, 633, 632, 623, 622, 621, 620, 619, 618, 617, - - 616, 615, 614, 613, 612, 611, 610, 609, 608, 607, - 606, 605, 604, 599, 598, 597, 596, 595, 594, 593, - 592, 591, 590, 589, 588, 587, 586, 585, 584, 583, - 582, 579, 572, 571, 570, 569, 568, 567, 566, 565, - 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, - 554, 553, 548, 547, 546, 545, 544, 543, 542, 541, - 540, 539, 538, 537, 536, 535, 534, 533, 532, 531, - 530, 529, 526, 525, 524, 523, 522, 521, 518, 517, - 516, 512, 511, 510, 509, 508, 507, 506, 505, 504, - 503, 502, 501, 500, 499, 498, 497, 496, 495, 494, - - 491, 490, 489, 486, 485, 484, 483, 482, 481, 480, - 479, 478, 477, 476, 475, 474, 473, 472, 471, 470, - 467, 466, 462, 461, 460, 459, 458, 457, 456, 455, - 454, 453, 452, 451, 450, 449, 448, 445, 444, 443, - 442, 441, 438, 437, 436, 435, 434, 433, 432, 431, - 430, 429, 428, 427, 426, 425, 424, 423, 422, 421, - 420, 419, 418, 417, 416, 415, 414, 413, 412, 411, - 410, 409, 408, 404, 403, 402, 399, 398, 397, 396, - 395, 394, 393, 392, 391, 390, 389, 388, 387, 386, - 385, 384, 383, 382, 381, 380, 379, 378, 377, 376, - - 375, 374, 373, 372, 369, 368, 367, 364, 363, 362, - 361, 360, 359, 358, 357, 356, 353, 352, 351, 350, - 349, 348, 347, 346, 345, 344, 343, 342, 341, 340, - 339, 338, 337, 336, 335, 334, 333, 332, 331, 330, - 329, 328, 327, 326, 325, 324, 323, 322, 321, 320, - 319, 318, 317, 316, 315, 314, 313, 312, 311, 310, - 309, 197, 308, 307, 306, 305, 304, 303, 299, 298, - 297, 294, 293, 289, 288, 287, 286, 283, 280, 279, - 278, 275, 274, 273, 272, 267, 264, 263, 262, 257, - 256, 255, 149, 254, 253, 252, 251, 250, 249, 248, - - 247, 246, 241, 240, 239, 236, 235, 234, 233, 232, - 231, 230, 227, 226, 223, 217, 216, 215, 212, 211, - 210, 209, 208, 207, 103, 206, 205, 202, 201, 200, - 199, 89, 197, 192, 175, 174, 173, 172, 149, 144, - 143, 137, 126, 123, 120, 113, 103, 100, 97, 96, - 89, 1258, 86, 86, 11, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258 - + 726, 725, 724, 723, 722, 721, 720, 719, 718, 717, + 716, 712, 711, 710, 709, 708, 707, 706, 705, 704, + 703, 702, 701, 700, 699, 698, 697, 696, 695, 694, + 693, 692, 689, 688, 687, 686, 685, 684, 683, 682, + 681, 680, 679, 678, 677, 676, 675, 674, 673, 672, + 671, 670, 669, 668, 664, 663, 662, 659, 658, 657, + + 656, 655, 654, 653, 652, 651, 650, 649, 640, 639, + 638, 637, 636, 635, 634, 633, 632, 631, 630, 629, + 628, 627, 626, 625, 624, 623, 622, 621, 620, 615, + 614, 613, 612, 611, 610, 609, 608, 607, 606, 605, + 604, 603, 602, 601, 600, 599, 598, 597, 596, 593, + 586, 585, 584, 583, 582, 581, 580, 579, 578, 577, + 576, 575, 574, 573, 572, 571, 570, 569, 568, 567, + 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, + 552, 551, 550, 549, 548, 547, 546, 545, 544, 543, + 540, 539, 538, 537, 536, 535, 532, 531, 530, 526, + + 525, 524, 523, 522, 521, 520, 519, 518, 517, 516, + 515, 514, 513, 512, 511, 510, 509, 508, 507, 504, + 503, 502, 499, 498, 497, 496, 495, 494, 493, 492, + 491, 490, 489, 488, 487, 486, 485, 484, 483, 482, + 481, 478, 477, 473, 472, 471, 470, 469, 468, 467, + 466, 465, 464, 463, 462, 461, 460, 459, 456, 455, + 454, 453, 452, 449, 448, 447, 446, 445, 444, 443, + 442, 441, 440, 439, 438, 437, 436, 435, 434, 433, + 432, 431, 430, 429, 428, 427, 426, 425, 424, 423, + 422, 421, 420, 419, 415, 414, 413, 412, 409, 408, + + 407, 406, 405, 404, 403, 402, 401, 400, 399, 398, + 397, 396, 395, 394, 393, 392, 391, 390, 389, 388, + 387, 386, 385, 384, 383, 382, 379, 378, 377, 376, + 373, 372, 371, 370, 369, 368, 367, 366, 365, 364, + 361, 360, 359, 358, 357, 356, 355, 354, 353, 352, + 351, 350, 349, 348, 347, 346, 345, 344, 343, 342, + 341, 340, 339, 338, 337, 336, 335, 334, 333, 332, + 331, 330, 329, 328, 327, 326, 325, 324, 323, 322, + 321, 320, 319, 318, 317, 201, 316, 315, 312, 311, + 310, 306, 305, 304, 300, 299, 295, 294, 293, 292, + + 289, 286, 285, 284, 281, 280, 279, 278, 273, 270, + 269, 268, 263, 262, 261, 260, 259, 151, 258, 257, + 256, 255, 254, 253, 252, 251, 250, 245, 244, 243, + 240, 239, 238, 237, 236, 235, 234, 231, 230, 227, + 221, 220, 219, 216, 215, 214, 213, 212, 211, 105, + 210, 209, 206, 205, 204, 203, 91, 201, 196, 179, + 178, 177, 176, 160, 151, 146, 145, 139, 128, 125, + 122, 115, 105, 102, 99, 98, 91, 1283, 88, 88, + 11, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283 } ; -static const flex_int16_t yy_chk[1396] = +static const flex_int16_t yy_chk[1423] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 7, 7, 8, 8, 13, 17, 13, 20, 17, 23, - 20, 27, 31, 27, 34, 31, 34, 36, 34, 34, - 37, 34, 39, 684, 23, 36, 41, 41, 17, 44, - 46, 36, 45, 37, 36, 31, 39, 43, 45, 51, - 63, 43, 48, 44, 46, 43, 48, 46, 54, 59, - 54, 66, 59, 51, 63, 67, 48, 48, 48, 62, - 65, 62, 185, 64, 62, 66, 65, 64, 67, 66, - 68, 73, 59, 64, 74, 68, 64, 75, 74, 185, - - 74, 73, 76, 77, 77, 87, 76, 87, 74, 76, - 114, 74, 79, 75, 76, 91, 114, 76, 91, 79, - 79, 98, 119, 101, 98, 101, 105, 121, 118, 105, - 124, 132, 118, 136, 121, 137, 119, 136, 91, 118, - 132, 147, 124, 147, 137, 151, 157, 162, 151, 105, - 158, 158, 157, 164, 177, 165, 170, 170, 175, 189, - 685, 162, 182, 189, 164, 165, 182, 686, 151, 175, - 177, 245, 182, 189, 245, 257, 261, 293, 687, 257, - 293, 300, 261, 300, 334, 340, 358, 362, 300, 334, - 340, 358, 362, 381, 386, 409, 413, 422, 381, 386, - - 409, 413, 422, 446, 688, 447, 446, 358, 447, 468, - 522, 469, 468, 471, 469, 468, 409, 469, 471, 492, - 493, 519, 492, 493, 519, 520, 534, 519, 520, 539, - 522, 520, 565, 689, 539, 589, 616, 565, 539, 690, - 589, 616, 534, 670, 589, 693, 694, 695, 670, 697, - 698, 700, 701, 702, 703, 704, 705, 706, 707, 708, - 709, 713, 713, 713, 715, 716, 717, 721, 722, 723, - 724, 717, 725, 729, 730, 731, 732, 733, 734, 735, - 736, 739, 740, 742, 743, 744, 745, 746, 747, 748, - 749, 751, 752, 754, 755, 756, 757, 758, 759, 760, - - 761, 762, 763, 764, 765, 766, 767, 768, 771, 772, - 773, 771, 772, 774, 775, 776, 777, 778, 779, 780, - 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, - 792, 793, 794, 796, 798, 799, 800, 801, 802, 803, - 804, 805, 806, 806, 806, 807, 809, 810, 811, 809, - 810, 812, 813, 814, 815, 816, 817, 818, 819, 820, - 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, - 831, 832, 833, 806, 834, 835, 836, 837, 838, 840, - 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, - 853, 854, 855, 856, 857, 859, 860, 862, 863, 864, - - 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, - 875, 876, 877, 878, 880, 881, 882, 883, 884, 885, - 886, 887, 888, 889, 890, 891, 892, 894, 895, 896, - 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, - 907, 908, 910, 911, 912, 913, 914, 915, 916, 917, - 918, 919, 920, 921, 922, 923, 924, 925, 926, 928, - 929, 930, 931, 932, 935, 936, 937, 938, 939, 940, - 941, 942, 943, 944, 945, 946, 948, 949, 952, 953, - 954, 955, 956, 957, 958, 959, 961, 962, 963, 964, - 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, - - 977, 978, 981, 982, 983, 985, 986, 987, 988, 989, - 990, 991, 992, 993, 994, 995, 996, 998, 999, 1000, - 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, - 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, - 1021, 1022, 1023, 1025, 1026, 1027, 1028, 1029, 1030, 1031, - 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1040, 1041, 1042, - 1043, 1044, 1045, 1046, 1047, 1048, 1053, 1055, 1056, 1057, - 1058, 1059, 1060, 1061, 1062, 1062, 1063, 1064, 1065, 1068, - 1071, 1072, 1073, 1074, 1075, 1076, 1080, 1082, 1083, 1084, - 1085, 1086, 1087, 1088, 1089, 1090, 1093, 1096, 1097, 1098, - - 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, - 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, - 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, - 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1140, - 1141, 1142, 1146, 1150, 1151, 1152, 1153, 1154, 1155, 1156, - 1157, 1158, 1159, 1160, 1164, 1169, 1170, 1171, 1172, 1173, - 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1182, 1183, 1184, - 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, - 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, - 1206, 1207, 1209, 1210, 1211, 1212, 1213, 1214, 1219, 1220, - - 1221, 1222, 1223, 1229, 1230, 1232, 1234, 1236, 1237, 1238, - 1239, 1241, 1245, 1246, 1247, 1250, 1251, 1254, 1259, 1259, - 1259, 1259, 1260, 1260, 1260, 1260, 1261, 1261, 1261, 1261, - 1262, 1262, 1263, 1262, 1264, 1264, 1264, 1264, 1265, 1265, - 1266, 1265, 1267, 1267, 1267, 1267, 1268, 1268, 1269, 1268, - 1270, 683, 682, 1270, 1271, 1271, 681, 1271, 680, 679, - 677, 676, 675, 674, 673, 672, 669, 668, 667, 666, - 665, 664, 663, 662, 661, 660, 659, 658, 657, 656, - 655, 654, 653, 651, 649, 648, 647, 646, 645, 644, - 643, 642, 641, 640, 639, 638, 637, 636, 635, 634, - - 631, 630, 629, 628, 627, 626, 625, 624, 623, 622, - 621, 620, 619, 617, 615, 614, 613, 612, 611, 610, - 609, 608, 607, 606, 605, 604, 603, 602, 601, 600, - 598, 597, 596, 595, 593, 592, 591, 590, 588, 587, - 586, 585, 584, 583, 581, 580, 578, 577, 576, 575, - 574, 573, 571, 570, 569, 568, 566, 564, 562, 561, - 560, 559, 558, 557, 556, 554, 552, 551, 550, 549, - 548, 547, 546, 545, 544, 543, 542, 541, 540, 538, - 537, 535, 533, 532, 531, 530, 529, 528, 527, 526, - 525, 524, 523, 517, 516, 515, 514, 513, 508, 507, - - 506, 505, 504, 503, 502, 501, 500, 499, 498, 497, - 496, 495, 494, 491, 489, 488, 487, 486, 485, 484, - 482, 481, 480, 479, 478, 477, 476, 475, 474, 473, - 472, 470, 467, 466, 465, 464, 463, 462, 461, 460, - 459, 458, 457, 456, 455, 454, 453, 452, 451, 450, - 449, 448, 445, 444, 443, 441, 440, 439, 437, 436, - 435, 434, 433, 432, 431, 430, 429, 428, 427, 426, - 425, 423, 421, 419, 418, 417, 416, 414, 412, 411, - 410, 407, 406, 405, 404, 403, 402, 401, 400, 399, - 398, 397, 396, 395, 394, 393, 392, 391, 390, 389, - - 385, 383, 382, 380, 379, 378, 377, 376, 375, 373, - 372, 371, 370, 369, 368, 367, 366, 365, 364, 363, - 361, 359, 357, 356, 355, 354, 353, 352, 351, 349, - 348, 346, 345, 344, 343, 342, 341, 339, 338, 337, - 336, 335, 333, 332, 331, 330, 329, 328, 327, 326, - 325, 324, 323, 322, 321, 320, 319, 318, 317, 316, - 314, 313, 311, 310, 309, 308, 307, 306, 305, 304, - 303, 302, 301, 298, 297, 295, 292, 291, 290, 289, - 288, 287, 286, 284, 283, 282, 281, 280, 279, 278, - 277, 276, 275, 272, 271, 270, 269, 268, 267, 266, - - 265, 264, 263, 262, 260, 259, 258, 256, 254, 253, - 252, 251, 250, 249, 247, 246, 244, 243, 242, 241, - 240, 239, 238, 237, 236, 235, 234, 233, 232, 231, - 230, 229, 228, 227, 226, 225, 224, 223, 222, 221, - 220, 219, 218, 217, 216, 215, 214, 213, 212, 211, - 210, 209, 208, 206, 205, 204, 203, 202, 201, 200, - 199, 197, 195, 194, 193, 192, 191, 190, 188, 187, - 186, 184, 183, 181, 180, 179, 178, 176, 174, 173, - 172, 169, 168, 167, 166, 163, 161, 160, 159, 156, - 155, 154, 148, 146, 145, 144, 143, 142, 141, 140, - - 139, 138, 135, 134, 133, 131, 130, 129, 128, 127, - 126, 125, 123, 122, 120, 117, 116, 115, 113, 112, - 111, 110, 109, 108, 102, 100, 99, 97, 96, 95, - 94, 88, 83, 78, 72, 71, 70, 69, 56, 50, - 49, 47, 42, 40, 38, 35, 29, 24, 22, 21, - 15, 11, 10, 9, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, 1258, - 1258, 1258, 1258, 1258, 1258 - + 5, 5, 5, 7, 7, 8, 8, 13, 17, 13, + 20, 17, 23, 20, 27, 31, 27, 34, 31, 34, + 36, 34, 34, 37, 34, 39, 684, 23, 36, 41, + 41, 17, 44, 46, 36, 45, 37, 36, 31, 39, + 43, 45, 51, 65, 43, 48, 44, 46, 43, 48, + 46, 54, 60, 54, 685, 60, 51, 65, 69, 48, + 48, 48, 63, 67, 63, 68, 63, 63, 66, 67, + 75, 69, 66, 77, 89, 60, 89, 70, 66, 68, + + 75, 66, 70, 68, 76, 79, 79, 78, 76, 77, + 76, 78, 93, 686, 78, 93, 81, 100, 76, 78, + 100, 76, 78, 81, 81, 103, 107, 103, 116, 107, + 121, 123, 120, 126, 116, 93, 120, 134, 123, 139, + 162, 162, 138, 120, 121, 126, 138, 134, 139, 107, + 149, 153, 149, 161, 153, 166, 168, 689, 169, 161, + 174, 174, 179, 181, 690, 691, 186, 168, 169, 166, + 186, 189, 267, 179, 153, 189, 186, 193, 267, 181, + 197, 193, 249, 197, 299, 249, 262, 299, 189, 342, + 262, 193, 307, 692, 307, 342, 348, 366, 370, 307, + + 391, 396, 348, 366, 370, 420, 391, 396, 424, 433, + 457, 420, 693, 457, 424, 433, 694, 458, 479, 366, + 458, 479, 480, 482, 479, 480, 536, 420, 480, 482, + 505, 506, 533, 505, 506, 533, 534, 548, 533, 534, + 696, 553, 534, 579, 605, 697, 536, 553, 698, 579, + 605, 553, 632, 548, 605, 687, 699, 700, 632, 701, + 702, 687, 703, 704, 705, 706, 707, 708, 709, 712, + 713, 714, 716, 717, 719, 720, 721, 722, 705, 723, + 724, 725, 726, 727, 728, 732, 732, 732, 734, 735, + 736, 740, 741, 742, 744, 745, 736, 749, 750, 751, + + 752, 753, 754, 755, 756, 759, 760, 762, 763, 764, + 765, 766, 767, 768, 769, 771, 772, 774, 775, 776, + 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, + 787, 788, 791, 792, 793, 791, 792, 794, 795, 796, + 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, + 807, 810, 811, 812, 813, 815, 816, 817, 819, 821, + 822, 823, 824, 825, 826, 827, 828, 829, 829, 829, + 830, 832, 833, 834, 832, 833, 835, 836, 837, 838, + 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, + 849, 850, 851, 852, 853, 854, 855, 856, 857, 829, + + 858, 859, 860, 861, 863, 866, 867, 868, 869, 870, + 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, + 882, 883, 885, 886, 887, 888, 889, 890, 891, 892, + 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, + 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, + 914, 915, 916, 918, 919, 920, 921, 922, 923, 924, + 925, 926, 927, 928, 929, 930, 931, 932, 934, 935, + 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, + 946, 947, 948, 949, 950, 952, 953, 954, 955, 956, + 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, + + 969, 971, 973, 974, 977, 978, 979, 980, 981, 982, + 983, 984, 986, 987, 988, 989, 992, 993, 994, 995, + 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1006, 1007, + 1008, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, + 1019, 1020, 1021, 1023, 1024, 1025, 1026, 1027, 1028, 1029, + 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, + 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1050, + 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, + 1061, 1062, 1063, 1065, 1066, 1067, 1068, 1069, 1070, 1071, + 1072, 1073, 1078, 1080, 1081, 1082, 1083, 1084, 1085, 1086, + + 1087, 1087, 1088, 1089, 1090, 1093, 1096, 1097, 1098, 1099, + 1100, 1101, 1105, 1107, 1108, 1109, 1110, 1111, 1112, 1113, + 1114, 1115, 1118, 1121, 1122, 1123, 1124, 1125, 1126, 1127, + 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, + 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, + 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, + 1158, 1159, 1160, 1161, 1162, 1165, 1166, 1167, 1171, 1175, + 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, + 1189, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, + 1203, 1204, 1205, 1207, 1208, 1209, 1210, 1211, 1212, 1213, + + 1214, 1215, 1216, 1217, 1218, 1219, 1221, 1222, 1223, 1224, + 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1234, 1235, + 1236, 1237, 1238, 1239, 1244, 1245, 1246, 1247, 1248, 1254, + 1255, 1257, 1259, 1261, 1262, 1263, 1264, 1266, 1270, 1271, + 1272, 1275, 1276, 1279, 1284, 1284, 1284, 1284, 1285, 1285, + 1285, 1285, 1286, 1286, 1286, 1286, 1287, 1287, 1288, 1287, + 1289, 1289, 1289, 1289, 1290, 1290, 1291, 1290, 1292, 1292, + 1292, 1292, 1293, 1293, 1294, 1293, 1295, 683, 682, 1295, + 1296, 1296, 681, 1296, 680, 679, 678, 677, 676, 675, + 674, 673, 672, 671, 670, 668, 666, 665, 664, 663, + + 662, 661, 660, 659, 658, 657, 656, 655, 654, 653, + 652, 651, 648, 647, 646, 645, 644, 643, 642, 641, + 640, 639, 638, 637, 636, 634, 633, 631, 630, 629, + 628, 627, 626, 625, 624, 623, 622, 621, 620, 619, + 618, 617, 616, 614, 613, 612, 611, 609, 608, 607, + 606, 604, 603, 602, 601, 600, 599, 598, 596, 595, + 594, 592, 591, 590, 589, 588, 587, 585, 584, 583, + 582, 580, 578, 576, 575, 574, 573, 572, 571, 570, + 568, 566, 565, 564, 563, 562, 561, 560, 559, 558, + 557, 556, 555, 554, 552, 551, 549, 547, 546, 545, + + 544, 543, 542, 541, 540, 539, 538, 537, 531, 530, + 529, 528, 527, 522, 521, 520, 519, 518, 517, 516, + 515, 514, 513, 512, 511, 510, 509, 508, 507, 504, + 502, 501, 500, 499, 498, 497, 495, 494, 493, 492, + 491, 490, 489, 488, 487, 486, 485, 484, 483, 481, + 478, 477, 476, 475, 474, 473, 472, 471, 470, 469, + 468, 467, 466, 465, 464, 463, 462, 461, 460, 459, + 456, 455, 454, 452, 451, 450, 448, 447, 446, 445, + 444, 443, 442, 441, 440, 439, 438, 437, 436, 434, + 432, 430, 429, 428, 427, 425, 423, 422, 421, 418, + + 417, 416, 415, 414, 413, 412, 411, 410, 409, 408, + 407, 406, 405, 404, 403, 402, 401, 400, 399, 395, + 393, 392, 390, 389, 388, 387, 386, 385, 383, 382, + 381, 380, 379, 378, 377, 376, 375, 374, 373, 372, + 371, 369, 367, 365, 364, 363, 362, 361, 360, 359, + 357, 356, 354, 353, 352, 351, 350, 349, 347, 346, + 345, 344, 343, 341, 340, 339, 338, 337, 336, 335, + 334, 333, 332, 331, 330, 329, 328, 327, 326, 325, + 324, 322, 321, 319, 318, 317, 316, 315, 313, 312, + 311, 310, 309, 308, 305, 304, 302, 301, 298, 297, + + 296, 295, 294, 293, 292, 290, 289, 288, 287, 286, + 285, 284, 283, 282, 281, 278, 277, 276, 275, 274, + 273, 272, 271, 270, 269, 268, 266, 265, 264, 263, + 261, 260, 258, 257, 256, 255, 254, 253, 251, 250, + 248, 247, 246, 245, 244, 243, 242, 241, 240, 239, + 238, 237, 236, 235, 234, 233, 232, 231, 230, 229, + 228, 227, 226, 225, 224, 223, 222, 221, 220, 219, + 218, 217, 216, 215, 214, 213, 212, 210, 209, 208, + 207, 206, 205, 204, 203, 201, 199, 198, 196, 195, + 194, 192, 191, 190, 188, 187, 185, 184, 183, 182, + + 180, 178, 177, 176, 173, 172, 171, 170, 167, 165, + 164, 163, 160, 159, 158, 157, 156, 150, 148, 147, + 146, 145, 144, 143, 142, 141, 140, 137, 136, 135, + 133, 132, 131, 130, 129, 128, 127, 125, 124, 122, + 119, 118, 117, 115, 114, 113, 112, 111, 110, 104, + 102, 101, 99, 98, 97, 96, 90, 85, 80, 74, + 73, 72, 71, 64, 56, 50, 49, 47, 42, 40, + 38, 35, 29, 24, 22, 21, 15, 11, 10, 9, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, 1283, + 1283, 1283 } ; static yy_state_type yy_last_accepting_state; @@ -1243,9 +1259,9 @@ static char *pgaf_strdup(const char *s) * call flex's static input() function. */ static void pgaf_read_raw_block(void); -#line 1246 "test_spec_scan.c" +#line 1262 "test_spec_scan.c" -#line 1248 "test_spec_scan.c" +#line 1264 "test_spec_scan.c" #define INITIAL 0 #define CLUSTER_BODY 1 @@ -1467,7 +1483,7 @@ YY_DECL #line 89 "test_spec_scan.l" -#line 1470 "test_spec_scan.c" +#line 1486 "test_spec_scan.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1494,13 +1510,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1259 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_current_state != 1258 ); + while ( yy_current_state != 1283 ); yy_cp = (yy_last_accepting_cpos); yy_current_state = (yy_last_accepting_state); @@ -2140,225 +2156,255 @@ YY_RULE_SETUP case 114: YY_RULE_SETUP #line 273 "test_spec_scan.l" -{ return T_EXPECT; } +{ return T_WAL; } YY_BREAK case 115: YY_RULE_SETUP #line 274 "test_spec_scan.l" -{ return T_ERROR; } +{ return T_SEGMENT; } YY_BREAK case 116: YY_RULE_SETUP #line 275 "test_spec_scan.l" -{ return T_PROMOTE; } +{ return T_ARCHIVED; } YY_BREAK case 117: YY_RULE_SETUP #line 276 "test_spec_scan.l" -{ return T_PERFORM; } +{ return T_BASEBACKUP; } YY_BREAK case 118: YY_RULE_SETUP #line 277 "test_spec_scan.l" -{ return T_FAILOVER; } +{ return T_ARCHIVER; } YY_BREAK case 119: YY_RULE_SETUP #line 278 "test_spec_scan.l" -{ return T_NETWORK; } +{ return T_SLASH; } YY_BREAK case 120: YY_RULE_SETUP #line 279 "test_spec_scan.l" -{ return T_DISCONNECT; } +{ return T_EXPECT; } YY_BREAK case 121: YY_RULE_SETUP #line 280 "test_spec_scan.l" -{ return T_CONNECT; } +{ return T_ERROR; } YY_BREAK case 122: YY_RULE_SETUP #line 281 "test_spec_scan.l" -{ return T_SLEEP; } +{ return T_PROMOTE; } YY_BREAK case 123: YY_RULE_SETUP #line 282 "test_spec_scan.l" -{ return T_COMPOSE; } +{ return T_PERFORM; } YY_BREAK case 124: YY_RULE_SETUP #line 283 "test_spec_scan.l" -{ return T_NODEINI; } +{ return T_FAILOVER; } YY_BREAK case 125: YY_RULE_SETUP #line 284 "test_spec_scan.l" -{ return T_DOWN; } +{ return T_NETWORK; } YY_BREAK case 126: YY_RULE_SETUP #line 285 "test_spec_scan.l" -{ return T_START; } +{ return T_DISCONNECT; } YY_BREAK case 127: YY_RULE_SETUP #line 286 "test_spec_scan.l" -{ return T_STOP; } +{ return T_CONNECT; } YY_BREAK case 128: YY_RULE_SETUP #line 287 "test_spec_scan.l" -{ return T_STOPPED; } +{ return T_SLEEP; } YY_BREAK case 129: YY_RULE_SETUP #line 288 "test_spec_scan.l" -{ return T_KILL; } +{ return T_COMPOSE; } YY_BREAK case 130: YY_RULE_SETUP #line 289 "test_spec_scan.l" -{ return T_IN; } +{ return T_NODEINI; } YY_BREAK case 131: YY_RULE_SETUP #line 290 "test_spec_scan.l" -{ return T_STATE; } +{ return T_DOWN; } YY_BREAK case 132: YY_RULE_SETUP #line 291 "test_spec_scan.l" -{ return T_ASSIGNED_STATE; } +{ return T_START; } YY_BREAK case 133: YY_RULE_SETUP #line 292 "test_spec_scan.l" -{ return T_CANDIDATE_PRIORITY; } +{ return T_STOP; } YY_BREAK case 134: YY_RULE_SETUP #line 293 "test_spec_scan.l" -{ return T_GROUP; } +{ return T_STOPPED; } YY_BREAK case 135: YY_RULE_SETUP #line 294 "test_spec_scan.l" -{ return T_AND; } +{ return T_KILL; } YY_BREAK case 136: YY_RULE_SETUP #line 295 "test_spec_scan.l" -{ return T_IS; } +{ return T_IN; } YY_BREAK case 137: YY_RULE_SETUP #line 296 "test_spec_scan.l" -{ return T_WITH; } +{ return T_STATE; } YY_BREAK case 138: YY_RULE_SETUP #line 297 "test_spec_scan.l" -{ return T_EQUALS; } +{ return T_ASSIGNED_STATE; } YY_BREAK case 139: YY_RULE_SETUP #line 298 "test_spec_scan.l" -{ return T_COMMA; } +{ return T_CANDIDATE_PRIORITY; } YY_BREAK case 140: YY_RULE_SETUP #line 299 "test_spec_scan.l" -{ return T_POSTGRES; } +{ return T_GROUP; } YY_BREAK case 141: YY_RULE_SETUP #line 300 "test_spec_scan.l" -{ return T_FSM; } +{ return T_AND; } YY_BREAK case 142: YY_RULE_SETUP #line 301 "test_spec_scan.l" -{ return T_STEP; } +{ return T_IS; } YY_BREAK case 143: YY_RULE_SETUP #line 302 "test_spec_scan.l" -{ return T_STAYS; } +{ return T_WITH; } YY_BREAK case 144: YY_RULE_SETUP #line 303 "test_spec_scan.l" -{ return T_WHILE; } +{ return T_EQUALS; } YY_BREAK case 145: -/* rule 145 can match eol */ YY_RULE_SETUP #line 304 "test_spec_scan.l" -{ return T_THROUGH; } +{ return T_COMMA; } YY_BREAK case 146: YY_RULE_SETUP #line 305 "test_spec_scan.l" -{ return T_THROUGH; } +{ return T_POSTGRES; } YY_BREAK case 147: YY_RULE_SETUP #line 306 "test_spec_scan.l" -{ return T_SET; } +{ return T_FSM; } YY_BREAK case 148: YY_RULE_SETUP #line 307 "test_spec_scan.l" -{ return T_GET; } +{ return T_STEP; } YY_BREAK case 149: YY_RULE_SETUP #line 308 "test_spec_scan.l" -{ BEGIN(EXEC_ARGS); return T_INJECT; } +{ return T_STAYS; } YY_BREAK case 150: YY_RULE_SETUP #line 309 "test_spec_scan.l" -{ return T_LOGS; } +{ return T_WHILE; } YY_BREAK case 151: +/* rule 151 can match eol */ YY_RULE_SETUP #line 310 "test_spec_scan.l" -{ return T_NOT; } +{ return T_THROUGH; } YY_BREAK case 152: YY_RULE_SETUP #line 311 "test_spec_scan.l" -{ return T_CONTAINS; } +{ return T_THROUGH; } YY_BREAK case 153: YY_RULE_SETUP #line 312 "test_spec_scan.l" -{ return T_MATCHES; } +{ return T_SET; } YY_BREAK case 154: YY_RULE_SETUP +#line 313 "test_spec_scan.l" +{ return T_GET; } + YY_BREAK +case 155: +YY_RULE_SETUP #line 314 "test_spec_scan.l" +{ BEGIN(EXEC_ARGS); return T_INJECT; } + YY_BREAK +case 156: +YY_RULE_SETUP +#line 315 "test_spec_scan.l" +{ return T_LOGS; } + YY_BREAK +case 157: +YY_RULE_SETUP +#line 316 "test_spec_scan.l" +{ return T_NOT; } + YY_BREAK +case 158: +YY_RULE_SETUP +#line 317 "test_spec_scan.l" +{ return T_CONTAINS; } + YY_BREAK +case 159: +YY_RULE_SETUP +#line 318 "test_spec_scan.l" +{ return T_MATCHES; } + YY_BREAK +case 160: +YY_RULE_SETUP +#line 320 "test_spec_scan.l" { yylval.ival = atoi(yytext); return T_INTEGER; } YY_BREAK -case 155: -/* rule 155 can match eol */ +case 161: +/* rule 161 can match eol */ YY_RULE_SETUP -#line 319 "test_spec_scan.l" +#line 325 "test_spec_scan.l" { yytext[yyleng - 1] = '\0'; yylval.str = pgaf_strdup(yytext + 1); return T_STRING; } YY_BREAK -case 156: +case 162: YY_RULE_SETUP -#line 325 "test_spec_scan.l" +#line 331 "test_spec_scan.l" { if (pgaf_next_brace_is_while) { pgaf_next_brace_is_while = 0; @@ -2369,9 +2415,9 @@ YY_RULE_SETUP return T_BLOCK; } YY_BREAK -case 157: +case 163: YY_RULE_SETUP -#line 335 "test_spec_scan.l" +#line 341 "test_spec_scan.l" { if (pgaf_step_brace_depth > 0) { pgaf_step_brace_depth--; @@ -2382,40 +2428,40 @@ YY_RULE_SETUP return T_RBRACE; } YY_BREAK -case 158: +case 164: YY_RULE_SETUP -#line 345 "test_spec_scan.l" +#line 351 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); return T_IDENT; } YY_BREAK -case 159: +case 165: YY_RULE_SETUP -#line 350 "test_spec_scan.l" +#line 356 "test_spec_scan.l" { /* skip whitespace before service name */ } YY_BREAK -case 160: +case 166: YY_RULE_SETUP -#line 352 "test_spec_scan.l" +#line 358 "test_spec_scan.l" { yylval.str = pgaf_strdup(yytext); BEGIN(EXEC_ARGS_REST); return T_IDENT; } YY_BREAK -case 161: -/* rule 161 can match eol */ +case 167: +/* rule 167 can match eol */ YY_RULE_SETUP -#line 358 "test_spec_scan.l" +#line 364 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 162: +case 168: YY_RULE_SETUP -#line 363 "test_spec_scan.l" +#line 369 "test_spec_scan.l" { char *p = yytext; while (*p == ' ' || *p == '\t') p++; @@ -2424,21 +2470,21 @@ YY_RULE_SETUP return T_SHELL_ARGS; } YY_BREAK -case 163: -/* rule 163 can match eol */ +case 169: +/* rule 169 can match eol */ YY_RULE_SETUP -#line 371 "test_spec_scan.l" +#line 377 "test_spec_scan.l" { pgaf_line_number++; BEGIN(STEP_BODY); } YY_BREAK -case 164: +case 170: YY_RULE_SETUP -#line 376 "test_spec_scan.l" +#line 382 "test_spec_scan.l" ECHO; YY_BREAK -#line 2441 "test_spec_scan.c" +#line 2487 "test_spec_scan.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(CLUSTER_BODY): case YY_STATE_EOF(STEP_BODY): @@ -2740,7 +2786,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1259 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2768,11 +2814,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1259 ) + if ( yy_current_state >= 1284 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 1258); + yy_is_jam = (yy_current_state == 1283); return yy_is_jam ? 0 : yy_current_state; } @@ -3411,7 +3457,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 376 "test_spec_scan.l" +#line 382 "test_spec_scan.l" static void diff --git a/src/bin/pgaftest/test_spec_scan.l b/src/bin/pgaftest/test_spec_scan.l index 1ad9d10e7..51b06120f 100644 --- a/src/bin/pgaftest/test_spec_scan.l +++ b/src/bin/pgaftest/test_spec_scan.l @@ -270,6 +270,12 @@ static void pgaf_read_raw_block(void); "timeout" { return T_TIMEOUT; } "assert" { return T_ASSERT; } "sql" { return T_SQL; } +"wal" { return T_WAL; } +"segment" { return T_SEGMENT; } +"archived" { return T_ARCHIVED; } +"basebackup" { return T_BASEBACKUP; } +"archiver" { return T_ARCHIVER; } +"/" { return T_SLASH; } "expect" { return T_EXPECT; } "error" { return T_ERROR; } "promote" { return T_PROMOTE; } From 42dfd6110eae9a081dbdb2431495c28863a2d69c Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 04:43:31 +0200 Subject: [PATCH 52/55] tests: migrate 7 archiver specs to the new wait-until-SQL syntax Replaces every "sleep N + sql + expect" call site that was polling for an async condition with the new wait-until-SQL forms (previous commit), across: archiver_wal_capture.pgaf archiver_multi_formation.pgaf citus_basic_operation.pgaf archiver_budget_architecture_regions.pgaf archiver_two_regions.pgaf archiver_basebackup_generation.pgaf archiver_bootstrap_and_fast_forward.pgaf archiver_basebackup_generation.pgaf's own fixed 120s sleep (added in an earlier commit as a stopgap for CI flakiness) is replaced outright by the new "wait until basebackup ... is ..." polling form, which is the real fix that stopgap was standing in for. archiver_basebackup_policy.pgaf is deliberately NOT migrated: its `count(*) = 3` check needs a stable, settled value after enough retention cycles have elapsed, not a first-reach-true poll -- a naive poll-until-true would risk a false pass on a transient count. Its existing fixed sleep is correct by design, not a flakiness bug. Also fixes a second, real bug this migration surfaced in archiver_bootstrap_and_fast_forward.pgaf's test_001: the monitor's own "basebackup complete" status and pg_walsender's actual ability to serve that backup are two different things. cmd_base_backup.c checks route->basebackupDir, which service_archiver_serve.c only refreshes every ARCHIVER_SERVE_ROUTES_REFRESH_TICKS (30) ticks -- so there's a real window where the monitor says "complete" before the archiver's own route is servable. The instant wait-until-SQL poll exposed this race (the old spec's blind sleep 30s happened to also absorb it by accident). There's no SQL-observable signal for "the archiver's route is ready", so this bridges the known 30s refresh window with a documented sleep rather than guessing at, or inventing new machinery for, something that isn't visible from the monitor side. Verified end-to-end via Docker/pgaftest: all 7 migrated specs pass in full, plus archiver_basebackup_policy.pgaf as an unmigrated regression check (2/2, unaffected). citus_basic_operation.pgaf runs its full 16-step Citus HA suite clean (~3.5 min). No C files touched by this commit; make docker-check / banned.h.sh are clean regardless. --- .../specs/archiver_basebackup_generation.pgaf | 36 ++++++------------ .../archiver_bootstrap_and_fast_forward.pgaf | 32 +++++++++++----- .../archiver_budget_architecture_regions.pgaf | 7 +--- tests/tap/specs/archiver_multi_formation.pgaf | 38 +++++++------------ tests/tap/specs/archiver_two_regions.pgaf | 7 +--- tests/tap/specs/archiver_wal_capture.pgaf | 12 ++---- tests/tap/specs/citus_basic_operation.pgaf | 31 ++++++--------- 7 files changed, 66 insertions(+), 97 deletions(-) diff --git a/tests/tap/specs/archiver_basebackup_generation.pgaf b/tests/tap/specs/archiver_basebackup_generation.pgaf index 8d86fa1c6..c4d8fdc51 100644 --- a/tests/tap/specs/archiver_basebackup_generation.pgaf +++ b/tests/tap/specs/archiver_basebackup_generation.pgaf @@ -70,31 +70,19 @@ teardown { # first frequency interval has elapsed) a real replay/volatile # backup both land on their own; check the final state. # -# service_archiver_maybe_generate_basebackup() is checked once -# per service_archiver_loop() tick (PG_AUTOCTL_KEEPER_SLEEP_TIME, -# 1s), so scheduling itself notices the 10s frequency promptly -- -# the real variable cost is generating the replay/volatile backup -# itself once it's due: extract the live backup into a staging -# instance, replay this archiver's own captured WAL forward, -# poll for promotion (wait_for_replay_promotion(), 1s steps), -# pg_basebackup it over loopback, then discard the staging -# instance -- several real Postgres-instance lifecycles, not a -# single fast pg_basebackup call like archiver_basebackup_ -# policy.pgaf's own live-backup cycles. CI run 84233594160 hit -# this exact margin: 60s wasn't enough under load (bootstrap + -# 10s policy interval + a slow replay cycle), so this follows -# that spec's own precedent of a generous fixed sleep (there is -# no SQL-condition polling primitive in pgaftest's DSL -- -# test_spec_parse.y's `wait until` forms are all node-state- -# specific) rather than a tight one. +# Polls rather than sleeping a fixed guess: generating the +# replay/volatile backup is several real Postgres-instance +# lifecycles (extract the live backup into a staging instance, +# replay this archiver's own captured WAL forward, poll for +# promotion, pg_basebackup it over loopback, discard the staging +# instance), not a single fast pg_basebackup call like archiver_ +# basebackup_policy.pgaf's own live-backup cycles, so its wall- +# clock cost varies with host load (CI run 84233594160 hit this +# directly: a fixed 60s sleep wasn't always enough). # step test_001_replay_backup_lands { - sleep 120s - sql monitor { SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0); } - expect { replay } - sql monitor { SELECT replaymode::text FROM pgautofailover.get_latest_basebackup('default', 0); } - expect { volatile } - sql monitor { SELECT status::text FROM pgautofailover.get_latest_basebackup('default', 0); } - expect { complete } + wait until basebackup source is replay in default/0 timeout 150s + wait until basebackup replaymode is volatile in default/0 timeout 5s + wait until basebackup status is complete in default/0 timeout 30s } diff --git a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf index e128dc115..edbd86ef8 100644 --- a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf +++ b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf @@ -60,19 +60,31 @@ teardown { # a plain 'live' check here is the same thing this spec can # observe from the monitor side), then bootstrap node2 from it. # +# The monitor's own "complete" status and pg_walsender's actual +# ability to serve that backup are two different things: the +# archiver only re-reads its routes file (which is what +# cmd_base_backup.c actually checks -- route->basebackupDir) +# once every ARCHIVER_SERVE_ROUTES_REFRESH_TICKS ticks +# (service_archiver_serve.c, currently 30 x the 1s tick), so a +# real gap exists between "the monitor says complete" and "the +# archiver's own pg_walsender will actually serve it" -- there's +# no SQL-observable signal for the second half, so this bridges +# the known refresh window with a plain sleep rather than +# guessing at (or inventing new machinery for) something that +# isn't visible from the monitor side. +# step test_001_bootstrap_secondary_from_archiver { + # get_latest_basebackup's 3-arg preferred_source overload -- not the + # plain 2-arg form the "wait until basebackup ..." verb generates -- + # so this stays the generic sql-polling form. + wait until sql monitor { + SELECT source::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live') + } is { live } timeout 30s + wait until sql monitor { + SELECT status::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live') + } is { complete } timeout 10s sleep 30s - sql monitor { - SELECT source::text - FROM pgautofailover.get_latest_basebackup('default', 0, 'live'); - } - expect { live } - sql monitor { - SELECT status::text - FROM pgautofailover.get_latest_basebackup('default', 0, 'live'); - } - expect { complete } exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --no-ssl --name node2 --hostname node2 --from-archiver exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" wait until node2 state is secondary timeout 90s diff --git a/tests/tap/specs/archiver_budget_architecture_regions.pgaf b/tests/tap/specs/archiver_budget_architecture_regions.pgaf index dd84ed9eb..0440f277d 100644 --- a/tests/tap/specs/archiver_budget_architecture_regions.pgaf +++ b/tests/tap/specs/archiver_budget_architecture_regions.pgaf @@ -81,9 +81,6 @@ step test_002_archiver_captures_wal { sql node1 { SELECT pg_switch_wal(); } sql node1 { INSERT INTO t1 VALUES (3); } sql node1 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } - expect { t } - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s } diff --git a/tests/tap/specs/archiver_multi_formation.pgaf b/tests/tap/specs/archiver_multi_formation.pgaf index 9085c83e1..a27c7aeb8 100644 --- a/tests/tap/specs/archiver_multi_formation.pgaf +++ b/tests/tap/specs/archiver_multi_formation.pgaf @@ -35,9 +35,9 @@ # test_runner.c's monitor_get_node_state()) wouldn't match either row to # begin with, and would be ambiguous between the two even if it did. Every # check on archiver1's per-membership state after test_003 attaches the -# second membership goes through an explicit `sql monitor` query matching -# nodename LIKE 'archiver-%' (safe: this spec has exactly one archiver) -# *and* formationid instead. +# second membership uses "wait until archiver state is ... in " +# instead, which matches nodename LIKE 'archiver-%' AND formationid under +# the hood (safe: this spec has exactly one archiver). # # The autoctl_node role has no direct SELECT on pgautofailover.archiver # (granted much later in pgautofailover.sql than the blanket "GRANT SELECT @@ -103,11 +103,8 @@ step test_001_capture_formation1_wal { sql node1 { SELECT pg_switch_wal(); } sql node1 { INSERT INTO t1 VALUES (3); } sql node1 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } - expect { t } - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s } # @@ -141,12 +138,11 @@ step test_003_dynamic_attach_to_formation2 { (SELECT archiver_id FROM pgautofailover.get_archivers('default') LIMIT 1), 'formation2'); } - # one reconciler tick (30s) plus margin for it to notice, fork the new - # capture child, and for that child to register/report far enough to - # reach ARCHIVING_STATE. - sleep 50s - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'formation2'; } - expect { archiving } + # polls through the reconciler's own periodic tick (30s, + # ARCHIVER_RECONCILER_INTERVAL_SECONDS) noticing the new membership, + # forking the new capture child, and that child registering/reporting + # far enough to reach ARCHIVING_STATE. + wait until archiver state is archiving in formation2 timeout 90s } # @@ -167,11 +163,8 @@ step test_004_capture_formation2_wal { sql node3 { SELECT pg_switch_wal(); } sql node3 { INSERT INTO t2 VALUES (3); } sql node3 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000003'); } - expect { t } - sql monitor { SELECT pgautofailover.wal_archived('formation2', 0, '000000010000000000000004'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in formation2/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in formation2/0 timeout 30s } # @@ -185,11 +178,8 @@ step test_004_capture_formation2_wal { step test_005_formation1_still_healthy { sql node1 { INSERT INTO t1 VALUES (4); } sql node1 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } - expect { t } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default'; } - expect { archiving } + wait until wal segment "000000010000000000000005" archived in default/0 timeout 30s + wait until archiver state is archiving in default timeout 30s } sequence diff --git a/tests/tap/specs/archiver_two_regions.pgaf b/tests/tap/specs/archiver_two_regions.pgaf index 6f988ab5d..3b65e983b 100644 --- a/tests/tap/specs/archiver_two_regions.pgaf +++ b/tests/tap/specs/archiver_two_regions.pgaf @@ -104,10 +104,7 @@ step test_002_both_archivers_capture_independently { sql node1 { SELECT pg_switch_wal(); } sql node1 { INSERT INTO t1 VALUES (3); } sql node1 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s sql monitor { SELECT pgautofailover.set_archiver_policy('default', NULL, 2, NULL, NULL); } - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s } diff --git a/tests/tap/specs/archiver_wal_capture.pgaf b/tests/tap/specs/archiver_wal_capture.pgaf index 6a40a6736..a0302f24c 100644 --- a/tests/tap/specs/archiver_wal_capture.pgaf +++ b/tests/tap/specs/archiver_wal_capture.pgaf @@ -87,12 +87,8 @@ step test_001_capture_wal { sql node1 { SELECT pg_switch_wal(); } sql node1 { INSERT INTO t1 VALUES (3); } sql node1 { SELECT pg_switch_wal(); } - # PG_AUTOCTL_KEEPER_SLEEP_TIME is 1s; this margin covers a slow CI runner. - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000003'); } - expect { t } - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000004'); } - expect { t } + wait until wal segment "000000010000000000000003" archived in default/0 timeout 30s + wait until wal segment "000000010000000000000004" archived in default/0 timeout 30s } # @@ -116,9 +112,7 @@ step test_002_archiver_restart_liveness { # these are fixed numbers rather than derived from segment 1). sql node1 { INSERT INTO t1 VALUES (4); } sql node1 { SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT pgautofailover.wal_archived('default', 0, '000000010000000000000005'); } - expect { t } + wait until wal segment "000000010000000000000005" archived in default/0 timeout 30s } # diff --git a/tests/tap/specs/citus_basic_operation.pgaf b/tests/tap/specs/citus_basic_operation.pgaf index 3afb916d6..16e26ddac 100644 --- a/tests/tap/specs/citus_basic_operation.pgaf +++ b/tests/tap/specs/citus_basic_operation.pgaf @@ -166,23 +166,18 @@ step test_010_perform_failover_coordinator { # per group), each named by archiver_add_formation() itself as # 'archiver--' (never the plain --name) -- # the generic "wait until archiver1 state is archiving" form is -# ambiguous once more than one such row exists (test_runner.c's -# monitor_get_node_state() does "... WHERE nodename = $1 LIMIT 1", -# no ORDER BY) and wouldn't match this synthesized name anyway, -# so every check below matches nodename LIKE 'archiver-%' (safe: -# this spec has exactly one archiver) and names formationid and -# groupid explicitly instead. +# ambiguous once more than one such row exists (it matches on +# nodename = $1, no ORDER BY) and wouldn't match this synthesized +# name anyway, so every check below uses "wait until archiver +# state is ... in default/" instead (matches nodename +# LIKE 'archiver-%' AND formationid/groupid under the hood). # step test_011_bring_up_archiver { exec archiver1 pg_autoctl node start - sleep 40s - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0; } - expect { archiving } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1; } - expect { archiving } - sql monitor { SELECT reportedstate FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2; } - expect { archiving } + wait until archiver state is archiving in default/0 timeout 60s + wait until archiver state is archiving in default/1 timeout 60s + wait until archiver state is archiving in default/2 timeout 60s } # @@ -204,11 +199,7 @@ step test_012_archiver_captures_every_group { sql coordinator1b { CREATE TABLE archiver_probe_coord(a int); INSERT INTO archiver_probe_coord SELECT generate_series(1, 100); SELECT pg_switch_wal(); } sql worker1a { CREATE TABLE archiver_probe_w1(a int); INSERT INTO archiver_probe_w1 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } sql worker2b { CREATE TABLE archiver_probe_w2(a int); INSERT INTO archiver_probe_w2 SELECT generate_series(1, 100); SELECT pg_switch_wal(); } - sleep 15s - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0; } - expect { t } - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1; } - expect { t } - sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2; } - expect { t } + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 0 } is { t } timeout 30s + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 1 } is { t } timeout 30s + wait until sql monitor { SELECT reportedlsn > '0/0' FROM pgautofailover.node WHERE nodename LIKE 'archiver-%' AND formationid = 'default' AND groupid = 2 } is { t } timeout 30s } From b1d8f6fcf0e666d28c9eacc1161de2e5e7fe4b86 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 04:43:53 +0200 Subject: [PATCH 53/55] pg_walsender: respond to real libpq's protocol-version GREASE probe Real PG19 libpq performs a "GREASE" self-test on every new connection (borrowed from TLS): it deliberately requests a bogus minor protocol version (major=3, minor=9999) plus a "_pq_.test_protocol_negotiation" startup option, to verify the server negotiates down properly rather than silently accepting whatever was asked. A server that accepts it without negotiating is treated as broken and the connection is refused: "server incorrectly accepted \"grease\" protocol version 3.9999 without negotiation" -- this broke every PG19 archiver connection in CI (pgaftest / archiver (PG19), the exact "create postgres --from-archiver" bootstrap path). ws_startup_negotiate() only ever checked the major version ((code >> 16) != 3) and ignored the minor version entirely, so it just proceeded with whatever was requested, including the grease probe's own nonsense value. Fixed with a real NegotiateProtocolVersion ('v') response, matching Postgres's own backend behaviour: - new ws_send_negotiate_protocol_version() (framing.c/.h) sends the full encoded version (major<<16 | newest supported minor) followed by a count and list of unrecognized "_pq_.*" startup options -- real libpq's own pqGetNegotiateProtocolVersion3() rejects a response that isn't properly encoded as "downgrade to pre-3.0", and separately requires any _pq_.* option the client sent to be echoed back as unsupported (we don't parse any, so every one seen is unsupported by definition). - ws_startup_negotiate() now parses the startup packet's key/value pairs before responding (needed to collect the _pq_.* option names), and sends the negotiate message whenever the requested minor version isn't 0 (all pg_walsender actually implements), continuing the connection at that version rather than closing it. Verified against real PG19 beta2 psql (which performs the same GREASE probe as libpq) connecting directly to a standalone pg_walsender: IDENTIFY_SYSTEM succeeds, no negotiation error. End to end: archiver_bootstrap_and_fast_forward.pgaf passes all 4 steps on PG19, including the exact bootstrap step that failed in CI. Full regression pass (archiver_wal_capture, archiver_basebackup_generation, archiver_basebackup_policy) on PG19 unaffected. --- src/bin/pg_walsender/framing.c | 45 ++++++++++++++++++++++++++++++++++ src/bin/pg_walsender/framing.h | 3 +++ src/bin/pg_walsender/startup.c | 44 ++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_walsender/framing.c b/src/bin/pg_walsender/framing.c index 728fbda8a..4965dc839 100644 --- a/src/bin/pg_walsender/framing.c +++ b/src/bin/pg_walsender/framing.c @@ -286,6 +286,51 @@ ws_send_backend_key_data(int sock, int32_t pid, int32_t secret) } +/* + * ws_send_negotiate_protocol_version sends the 'v' NegotiateProtocolVersion + * message. Per the wire protocol, the first Int32 is *not* a bare minor + * version -- it's the full negotiated protocol version (major<<16|minor), + * exactly like the version code in a StartupMessage; real libpq's + * pqGetNegotiateProtocolVersion3() compares it against PG_PROTOCOL(3, 0) + * and rejects anything smaller as "downgrade to pre-3.0 protocol version". + * newestMinor is the highest minor protocol version we actually support + * (always 0 -- only protocol 3.0 is implemented), combined here with major + * version 3. unsupportedOptions/nUnsupportedOptions lists any "_pq_.*" + * startup options the client asked for that we don't recognize (we don't + * parse any, so this is every "_pq_.*" key seen) -- real libpq's own + * protocol-GREASE self-test requires the server to echo back + * "_pq_.test_protocol_negotiation" here, or it fails the connection with + * "server did not report the unsupported ... parameter". See startup.c's + * own caller for why this exists. + */ +bool +ws_send_negotiate_protocol_version(int sock, int32_t newestMinor, + const char **unsupportedOptions, + int nUnsupportedOptions) +{ + PQExpBuffer buf = createPQExpBuffer(); + + int32_t netVersion = htonl((3 << 16) | (newestMinor & 0xFFFF)); + int32_t netOptionCount = htonl(nUnsupportedOptions); + + appendBinaryPQExpBuffer(buf, (const char *) &netVersion, 4); + appendBinaryPQExpBuffer(buf, (const char *) &netOptionCount, 4); + + for (int i = 0; i < nUnsupportedOptions; i++) + { + appendBinaryPQExpBuffer(buf, unsupportedOptions[i], + strlen(unsupportedOptions[i]) + 1); + } + + bool ok = !PQExpBufferBroken(buf) && + ws_send_message(sock, 'v', buf->data, buf->len); + + destroyPQExpBuffer(buf); + + return ok; +} + + bool ws_send_ready_for_query(int sock) { diff --git a/src/bin/pg_walsender/framing.h b/src/bin/pg_walsender/framing.h index 8a1cd38a5..c7184de0b 100644 --- a/src/bin/pg_walsender/framing.h +++ b/src/bin/pg_walsender/framing.h @@ -64,6 +64,9 @@ bool ws_send_message(int sock, char type, const char *data, int32_t dataLen); bool ws_send_authentication_ok(int sock); bool ws_send_parameter_status(int sock, const char *name, const char *value); bool ws_send_backend_key_data(int sock, int32_t pid, int32_t secret); +bool ws_send_negotiate_protocol_version(int sock, int32_t newestMinor, + const char **unsupportedOptions, + int nUnsupportedOptions); bool ws_send_ready_for_query(int sock); bool ws_send_error_response(int sock, const char *sqlstate, const char *message); bool ws_send_command_complete(int sock, const char *tag); diff --git a/src/bin/pg_walsender/startup.c b/src/bin/pg_walsender/startup.c index 0a3859091..e5d61b624 100644 --- a/src/bin/pg_walsender/startup.c +++ b/src/bin/pg_walsender/startup.c @@ -80,10 +80,25 @@ ws_startup_negotiate(int sock, WsStartupParams *params) return false; } - /* parse the NUL-separated key/value pairs following the version code */ + /* + * Parse the NUL-separated key/value pairs following the version + * code first -- we need to know which "_pq_.*" options (if any) the + * client sent *before* we can answer NegotiateProtocolVersion below: + * real libpq's protocol-GREASE self-test sends + * "_pq_.test_protocol_negotiation" and requires the server to echo + * it back as unsupported (we don't parse any "_pq_.*" options, so + * every one seen here is unsupported by definition). + */ const char *ptr = payload + 4; const char *end = payload + payloadLen; + enum + { + WS_MAX_UNSUPPORTED_OPTIONS = 16 + }; + const char *unsupportedOptions[WS_MAX_UNSUPPORTED_OPTIONS]; + int nUnsupportedOptions = 0; + while (ptr < end && *ptr != '\0') { const char *key = ptr; @@ -118,6 +133,33 @@ ws_startup_negotiate(int sock, WsStartupParams *params) strcasecmp(value, "true") == 0 || params->replicationDatabase); } + else if (strncmp(key, "_pq_.", 5) == 0 && + nUnsupportedOptions < WS_MAX_UNSUPPORTED_OPTIONS) + { + unsupportedOptions[nUnsupportedOptions++] = key; + } + } + + /* + * Only protocol 3.0 is implemented. A client is free to ask for a + * newer minor version than we understand -- real libpq deliberately + * probes with a bogus one (protocol "GREASE", e.g. 3.9999) to + * verify a server properly negotiates rather than silently + * accepting whatever was asked for, and refuses to proceed against + * a server that gets this wrong. Tell it the newest minor version + * we actually speak (0) via NegotiateProtocolVersion, matching real + * Postgres's own backend behaviour, then continue the connection at + * that version rather than closing it. + */ + if ((code & 0xFFFF) != 0) + { + if (!ws_send_negotiate_protocol_version(sock, 0, + unsupportedOptions, + nUnsupportedOptions)) + { + free(payload); + return false; + } } free(payload); From b8f61cf769dedd8c3933af4399a4161fe56e5dac Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 17:04:26 +0200 Subject: [PATCH 54/55] docs: document the routes.ini refresh mechanism archiving-details.rst mentioned archiver-routes.ini in passing but never explained when or why it gets rewritten -- "regenerated automatically on the archiver's own next tick" was the full extent of it. Adds a dedicated "Keeping the routes file current" section to the Process model, covering: - why pg_walsender never queries the monitor directly (staying serve-capable through a monitor outage, staying a small standalone/testable binary) - the atomic write pattern (temp file + rename) and why it makes concurrent reads inherently safe, no locking needed - all four refresh triggers: startup, the 30s periodic tick, immediate refresh on SIGUSR1 after a base backup completes (see the next commit), and SIGHUP - that multiple memberships' base backups can genuinely run concurrently, with no archiver-wide serialization Also includes a real, verbatim sample of the file's contents, and tightens the Storage section's own brief mention to cross-reference the new section instead of repeating a vaguer version of the same explanation. Verified: `make -C docs html` builds clean, no warnings, no unresolved cross-references. --- docs/archiving-details.rst | 55 ++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/archiving-details.rst b/docs/archiving-details.rst index 676861118..94cedbd0e 100644 --- a/docs/archiving-details.rst +++ b/docs/archiving-details.rst @@ -94,11 +94,12 @@ identity and one root directory:: exactly what disaster recovery relies on. - Each membership has its own ``archiver-position`` file, tracking that group's own captured LSN. ``archiver-routes.ini`` sits at the archiver's - own root instead, one section per membership. All of these are small - internal bookkeeping files -- coordinates and status, never a copy of - any actual data. Safe to ignore day to day, and not something that - needs backing up itself -- all of them are regenerated automatically on - the archiver's own next tick. + own root instead, one section per membership -- see `Keeping the + routes file current`_ below for exactly when and why it gets rewritten. + All of these are small internal bookkeeping files -- coordinates and + status, never a copy of any actual data. Safe to ignore day to day, and + not something that needs backing up itself -- all of them are + regenerated automatically. A single-membership archiver (the common case: one formation, one group) looks the same, just with only one ``//`` subdirectory @@ -174,6 +175,50 @@ resumes capturing all of them -- a replication slot keeps the WAL a capture needs regardless of how many times its own consumer reconnects, so this costs nothing. +Keeping the routes file current +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pg_walsender`` never queries the monitor itself, on purpose: an +archiver exists to keep serving already-captured data even when the +monitor it would otherwise depend on is unreachable, and staying free of +that dependency also keeps ``pg_walsender`` a small, standalone binary +with nothing to mock or stand up just to test it. ``archiver-routes.ini`` +is the decoupling point -- ``serve`` is the one process that actually +talks to the monitor, resolving each membership's current WAL-cache +directory and latest complete base backup and writing them here; every +``pg_walsender`` connection just reads this one local file straight off +disk, fresh, with no monitor round trip on its own hot path. One section +per membership:: + + [default/0] + walcache = /var/lib/pgaf/archiver1/default/0 + position = 0/0 + basebackup = /var/lib/pgaf/archiver1/default/0/basebackups/basebackup-20260806T132954Z + timeline = 1 + systemid = 7670908901798703128 + +The file is always rewritten as a whole -- one full pass over every +membership this archiver currently holds, written to a temporary file +and atomically renamed into place -- never patched in place. A +connection arriving mid-refresh always sees either the complete previous +version or the complete new one, never a torn write; nothing here needs +a lock. ``serve`` triggers a rewrite: + +- once at startup, before ``pg_walsender`` is even started; +- every 30 seconds, as a periodic catch-all -- covers anything not + otherwise signaled, such as a membership having just been attached; +- immediately, the moment a base backup finishes and is reported + complete -- the process that just produced it signals ``serve`` + directly, rather than leaving a freshly-completed backup unservable + for up to that 30-second window; and +- on ``SIGHUP``, the same reload signal every other pg_autoctl process + already understands. + +Each membership generates its own base backups independently (its own +schedule, its own retention), so more than one can genuinely be in +progress at once on a multi-membership archiver -- there's no archiver- +wide lock serializing them. + More or fewer standby nodes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From c2c264c3c237936d0918abd0b32668eaaf99ea45 Mon Sep 17 00:00:00 2001 From: Dimitri Fontaine Date: Thu, 6 Aug 2026 17:04:53 +0200 Subject: [PATCH 55/55] archiver: refresh routes.ini immediately after a base backup completes service_archiver_serve.c's routes file only refreshed on a blind 30s timer (ARCHIVER_SERVE_ROUTES_REFRESH_TICKS), so there was a real window where the monitor already reported a base backup "complete" before pg_walsender's own route (route->basebackupDir, cmd_base_ backup.c) reflected it -- previously worked around in archiver_bootstrap_and_fast_forward.pgaf with a blind 30s sleep. Adds a dedicated SIGUSR1 signal so the capture process that just finished generating and reporting a backup can prompt an immediate refresh instead of waiting for the next tick: - src/bin/common/signals.c/.h: new asked_to_refresh_routes flag, wired to SIGUSR1 via catch_refresh_routes(), registered in the shared set_signal_handlers() (installed everywhere, same as asked_to_reload/SIGHUP, harmless where nothing checks it) and added to block_signals()'s masked set. - service_archiver_serve.c: service_archiver_serve_loop() checks the flag every iteration, same pattern as the existing SIGHUP check, and refreshes immediately when set. - keeper_config.h: new archiverPidFilePath field, carrying the archiver-level supervisor's own shared pidfile path (one " " line per supervised service) across into a per-membership KeeperConfig, whose own pathnames.pid gets overwritten with a different value moments later. - service_archiver_reconciler.c: build_membership_keeper() stashes this path from the template keeper right after the shallow copy, before the per-membership pathname recompute overwrites it. - service_archiver_basebackup.c: new notify_archiver_serve_of_new_ basebackup(), called from the forked generation child right after a successful backup. Looks up archiver-serve's own pid via supervisor_find_service_pid() (the project's existing helper for resolving one named service inside a shared multi-service pidfile) and sends it SIGUSR1. Best-effort: any failure here (pidfile missing, process already gone) is logged and otherwise ignored -- archiver-serve's own periodic refresh is still the fallback, and this must never turn an already-successful base backup into a failure. Caught mid-implementation: an earlier version of this used a plain read_pidfile() on archiverPidFilePath, which only reads the first line -- the supervisor's own pid, not archiver-serve's, since that pidfile has one line per supervised service. Confirmed via live Docker testing: the signal reached the supervisor (which has the handler installed everywhere, per the above) but nothing ever ran service_archiver_serve_loop() in that process, so the flag was set in the wrong process's memory and routes.ini stayed on the 30s tick the whole time (observed once at 117s -- 4x the tick, not immediate). Fixed by switching to supervisor_find_service_pid(), which is what this project already uses elsewhere to look a specific service up by name in exactly this pidfile format. Also removes the archiver_bootstrap_and_fast_forward.pgaf bridging sleep this was meant to replace, and updates that step's own header comment accordingly. Verification status: local build (zero warnings), citus_indent, and banned.h.sh are all clean. The corrected pid-lookup logic was traced carefully against supervisor_find_service_pid()'s own implementation and the pidfile format it expects, and a live Docker pass confirmed the mechanism doesn't crash or hang archiver-serve and that concurrent backups on different memberships both land correctly in routes.ini with no corruption. A full, clean, uncontaminated end-to-end timing proof (routes.ini updating within ~1s of a backup completing, not up to 30s later) was attempted but not obtained in this session -- repeated verification passes hit infrastructure issues (a Docker Compose project-name collision between two concurrent local investigations, then agent/watchdog stalls on image rebuilds) rather than any observed test failure after the pid-lookup fix landed. Worth a clean, solo re-verification pass before relying on this further. --- src/bin/common/signals.c | 17 +++++- src/bin/common/signals.h | 12 ++++ src/bin/pg_autoctl/keeper_config.h | 21 +++++++ .../pg_autoctl/service_archiver_basebackup.c | 56 +++++++++++++++++++ .../pg_autoctl/service_archiver_reconciler.c | 12 ++++ src/bin/pg_autoctl/service_archiver_serve.c | 13 +++++ .../archiver_bootstrap_and_fast_forward.pgaf | 21 +++---- 7 files changed, 139 insertions(+), 13 deletions(-) diff --git a/src/bin/common/signals.c b/src/bin/common/signals.c index 58687da49..90b1041da 100644 --- a/src/bin/common/signals.c +++ b/src/bin/common/signals.c @@ -25,6 +25,7 @@ volatile sig_atomic_t asked_to_stop = 0; /* SIGTERM */ volatile sig_atomic_t asked_to_stop_fast = 0; /* SIGINT */ volatile sig_atomic_t asked_to_reload = 0; /* SIGHUP */ volatile sig_atomic_t asked_to_quit = 0; /* SIGQUIT */ +volatile sig_atomic_t asked_to_refresh_routes = 0; /* SIGUSR1 */ /* * set_signal_handlers sets our signal handlers for the 4 signals that we @@ -39,6 +40,7 @@ set_signal_handlers(bool exitOnQuit) pqsignal(SIGHUP, catch_reload); pqsignal(SIGINT, catch_int); pqsignal(SIGTERM, catch_term); + pqsignal(SIGUSR1, catch_refresh_routes); if (exitOnQuit) { @@ -59,7 +61,7 @@ set_signal_handlers(bool exitOnQuit) bool block_signals(sigset_t *mask, sigset_t *orig_mask) { - int signals[] = { SIGHUP, SIGINT, SIGTERM, SIGQUIT, -1 }; + int signals[] = { SIGHUP, SIGINT, SIGTERM, SIGQUIT, SIGUSR1, -1 }; if (sigemptyset(mask) == -1) { @@ -128,6 +130,19 @@ catch_reload(SIGNAL_ARGS) } +/* + * catch_refresh_routes receives the SIGUSR1 signal. + */ +void +catch_refresh_routes(SIGNAL_ARGS) +{ + int sig = postgres_signal_arg; + + asked_to_refresh_routes = 1; + pqsignal(sig, catch_refresh_routes); +} + + /* * catch_int receives the SIGINT signal. */ diff --git a/src/bin/common/signals.h b/src/bin/common/signals.h index f82b7a9ce..b4542c0a8 100644 --- a/src/bin/common/signals.h +++ b/src/bin/common/signals.h @@ -20,6 +20,17 @@ extern volatile sig_atomic_t asked_to_stop_fast; /* SIGINT */ extern volatile sig_atomic_t asked_to_reload; /* SIGHUP */ extern volatile sig_atomic_t asked_to_quit; /* SIGQUIT */ +/* + * Prompts service_archiver_serve_loop() to refresh its routes file on its + * next iteration instead of waiting for the next periodic tick -- see + * service_archiver_maybe_generate_basebackup()'s own comment (service_ + * archiver_basebackup.c) on why a freshly-completed base backup needs + * this. Harmless in every other process: nothing else checks it, same as + * asked_to_reload is already installed everywhere regardless of whether a + * given service body reacts to it. + */ +extern volatile sig_atomic_t asked_to_refresh_routes; /* SIGUSR1 */ + #define CHECK_FOR_FAST_SHUTDOWN { if (asked_to_stop_fast) { break; } \ } @@ -31,6 +42,7 @@ void catch_int(SIGNAL_ARGS); void catch_term(SIGNAL_ARGS); void catch_quit(SIGNAL_ARGS); void catch_quit_and_exit(SIGNAL_ARGS); +void catch_refresh_routes(SIGNAL_ARGS); int get_current_signal(int defaultSignal); int pick_stronger_signal(int sig1, int sig2); diff --git a/src/bin/pg_autoctl/keeper_config.h b/src/bin/pg_autoctl/keeper_config.h index 1e17fed1c..131277aa0 100644 --- a/src/bin/pg_autoctl/keeper_config.h +++ b/src/bin/pg_autoctl/keeper_config.h @@ -61,6 +61,27 @@ typedef struct KeeperConfig char archiverIdStr[INTSTRING_MAX_DIGITS]; int64_t archiverId; + /* + * The archiver-level (not per-membership) supervisor's own pidfile + * path, stashed by service_archiver_reconciler.c's build_membership_ + * keeper() from the template keeper's pathnames.pid before they get + * overwritten with this membership's own per-(formation, group) + * paths. This is the *shared* pidfile every one of this archiver's + * supervised services (archiver-serve, archiver-reconciler, each + * archiver-capture--) has one line in -- not a + * dedicated pidfile of its own -- so a reader must look up a specific + * service's own pid by name (supervisor_find_service_pid(), + * SERVICE_NAME_ARCHIVER_SERVE), not just read the first line. + * + * A capture child that just finished generating a base backup + * (service_archiver_basebackup.c) uses this to find archiver-serve's + * pid and signal it (SIGUSR1) to prompt an immediate routes refresh, + * rather than leaving pg_walsender to serve a stale route for up to + * ARCHIVER_SERVE_ROUTES_REFRESH_TICKS more ticks. Only meaningful for + * a per-membership keeper built that way; empty otherwise. + */ + char archiverPidFilePath[MAXPGPATH]; + /* PostgreSQL setup */ PostgresSetup pgSetup; diff --git a/src/bin/pg_autoctl/service_archiver_basebackup.c b/src/bin/pg_autoctl/service_archiver_basebackup.c index 1a8f5d128..9235e9276 100644 --- a/src/bin/pg_autoctl/service_archiver_basebackup.c +++ b/src/bin/pg_autoctl/service_archiver_basebackup.c @@ -82,6 +82,7 @@ #include "runprogram.h" #include "signals.h" #include "string_utils.h" +#include "supervisor.h" /* * One base backup generation child at a time, mirroring @@ -1015,6 +1016,56 @@ get_current_primary_node_id(Keeper *keeper, int64_t *primaryNodeId) } +/* + * notify_archiver_serve_of_new_basebackup signals the archiver-serve + * process (SIGUSR1) to refresh its routes file immediately, rather than + * leaving pg_walsender to serve a stale route for up to ARCHIVER_SERVE_ + * ROUTES_REFRESH_TICKS more ticks after the monitor already knows this + * backup is complete. Best-effort: archiver-serve's own periodic refresh + * is still there as a fallback, so any failure here (pidfile missing or + * stale, process already gone) is logged and otherwise ignored -- it must + * never turn an already-successful base backup into a failure. + * + * config->archiverPidFilePath is the archiver-level *supervisor's* own + * shared pidfile, with one " " line per supervised + * service (archiver-serve, archiver-reconciler, each archiver-capture-*) + * -- not a dedicated pidfile of archiver-serve's own. Reading its first + * line (as a plain read_pidfile() would) gives the supervisor's own pid, + * not archiver-serve's; supervisor_find_service_pid() is what actually + * looks a specific service up by name. + */ +static void +notify_archiver_serve_of_new_basebackup(KeeperConfig *config) +{ + if (IS_EMPTY_STRING_BUFFER(config->archiverPidFilePath)) + { + return; + } + + pid_t archiverServePid = 0; + + if (!supervisor_find_service_pid(config->archiverPidFilePath, + SERVICE_NAME_ARCHIVER_SERVE, + &archiverServePid) || + archiverServePid <= 0) + { + log_debug("Could not find archiver-serve's pid in \"%s\" to " + "prompt an immediate routes refresh; it will pick up " + "this base backup on its own next periodic tick", + config->archiverPidFilePath); + return; + } + + if (kill(archiverServePid, SIGUSR1) != 0) + { + log_debug("Could not signal archiver-serve (pid %d) to prompt an " + "immediate routes refresh: %m; it will pick up this " + "base backup on its own next periodic tick", + archiverServePid); + } +} + + /* * service_archiver_maybe_generate_basebackup checks, once per * service_archiver_loop() tick, whether a base backup generation is due @@ -1193,6 +1244,11 @@ service_archiver_maybe_generate_basebackup(Keeper *keeper) : generate_replay_basebackup(keeper, sourceBackupDir, backupDir, label, &policy); + if (ok) + { + notify_archiver_serve_of_new_basebackup(config); + } + exit(ok ? EXIT_CODE_QUIT : EXIT_CODE_INTERNAL_ERROR); } diff --git a/src/bin/pg_autoctl/service_archiver_reconciler.c b/src/bin/pg_autoctl/service_archiver_reconciler.c index e83f3ed5d..7132e167f 100644 --- a/src/bin/pg_autoctl/service_archiver_reconciler.c +++ b/src/bin/pg_autoctl/service_archiver_reconciler.c @@ -307,6 +307,18 @@ build_membership_keeper(Keeper *templateKeeper, ArchiverMembership *membership, * process doesn't already own, so a shallow copy is a real copy */ *membershipKeeper = *templateKeeper; + /* + * Stash the archiver-level supervisor's own shared pidfile path (still + * correct at this exact point, inherited from templateKeeper) into its + * own dedicated field before the pathnames recompute below overwrites + * config.pathnames.pid with this membership's own value -- see + * KeeperConfig's own comment on archiverPidFilePath for why a capture + * child needs this. + */ + strlcpy(membershipKeeper->config.archiverPidFilePath, + templateKeeper->config.pathnames.pid, + sizeof(membershipKeeper->config.archiverPidFilePath)); + strlcpy(membershipKeeper->config.formation, membership->formation, sizeof(membershipKeeper->config.formation)); membershipKeeper->config.groupId = membership->groupId; diff --git a/src/bin/pg_autoctl/service_archiver_serve.c b/src/bin/pg_autoctl/service_archiver_serve.c index 827ff0bb8..556db7881 100644 --- a/src/bin/pg_autoctl/service_archiver_serve.c +++ b/src/bin/pg_autoctl/service_archiver_serve.c @@ -567,6 +567,19 @@ service_archiver_serve_loop(Keeper *keeper) (void) service_archiver_serve_refresh_routes(keeper); } + /* + * SIGUSR1: a capture child just finished generating and reporting + * a base backup (service_archiver_maybe_generate_basebackup(), + * service_archiver_basebackup.c) and is prompting an immediate + * refresh rather than leaving pg_walsender to serve a stale route + * for up to ARCHIVER_SERVE_ROUTES_REFRESH_TICKS more ticks. + */ + if (asked_to_refresh_routes) + { + asked_to_refresh_routes = 0; + (void) service_archiver_serve_refresh_routes(keeper); + } + if (!service_archiver_serve_walsender_is_running()) { log_warn("pg_walsender is not running anymore, restarting it"); diff --git a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf index edbd86ef8..4e8d75c7b 100644 --- a/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf +++ b/tests/tap/specs/archiver_bootstrap_and_fast_forward.pgaf @@ -61,17 +61,15 @@ teardown { # observe from the monitor side), then bootstrap node2 from it. # # The monitor's own "complete" status and pg_walsender's actual -# ability to serve that backup are two different things: the -# archiver only re-reads its routes file (which is what -# cmd_base_backup.c actually checks -- route->basebackupDir) -# once every ARCHIVER_SERVE_ROUTES_REFRESH_TICKS ticks -# (service_archiver_serve.c, currently 30 x the 1s tick), so a -# real gap exists between "the monitor says complete" and "the -# archiver's own pg_walsender will actually serve it" -- there's -# no SQL-observable signal for the second half, so this bridges -# the known refresh window with a plain sleep rather than -# guessing at (or inventing new machinery for) something that -# isn't visible from the monitor side. +# ability to serve that backup used to be two different things: +# the archiver only re-read its routes file (what cmd_base_ +# backup.c actually checks -- route->basebackupDir) once every +# ARCHIVER_SERVE_ROUTES_REFRESH_TICKS ticks (service_archiver_ +# serve.c, 30 x the 1s tick). service_archiver_basebackup.c now +# signals archiver-serve (SIGUSR1) the moment a backup finishes +# generating and gets reported complete, prompting an immediate +# refresh instead of waiting for the next tick -- no bridging +# sleep needed here anymore. # step test_001_bootstrap_secondary_from_archiver { @@ -84,7 +82,6 @@ step test_001_bootstrap_secondary_from_archiver { wait until sql monitor { SELECT status::text FROM pgautofailover.get_latest_basebackup('default', 0, 'live') } is { complete } timeout 10s - sleep 30s exec node2 pg_autoctl create postgres --pgdata /var/lib/postgres/pgaf --monitor postgresql://autoctl_node@monitor/pg_auto_failover --auth trust --no-ssl --name node2 --hostname node2 --from-archiver exec node2 bash -c "nohup pg_autoctl run --pgdata /var/lib/postgres/pgaf > /tmp/node2-run.log 2>&1 & echo backgrounded pid $!" wait until node2 state is secondary timeout 90s