From 63ed0f87d6e0d018681100e0134d6a577f35cde9 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Wed, 26 Aug 2026 14:54:30 +0100 Subject: [PATCH] Seed standbys from backups --- CHANGELOG.md | 2 + README.md | 4 + config/postgres.example.yml | 1 + lib/active_postgres/components/repmgr.rb | 93 +++++++++++-------- lib/active_postgres/configuration.rb | 5 + .../configuration/standby_policy.rb | 24 +++++ lib/active_postgres/deployment_flow.rb | 6 +- lib/active_postgres/installer.rb | 4 +- .../standby_deployment_flow.rb | 18 +++- test/deployment_flow_test.rb | 38 ++++++++ test/jump_host_configuration_test.rb | 37 ++++++++ test/repmgr_test.rb | 12 +++ test/test_helper.rb | 4 + 13 files changed, 203 insertions(+), 45 deletions(-) create mode 100644 lib/active_postgres/configuration/standby_policy.rb create mode 100644 test/deployment_flow_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae4b40..b52abaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project follows [Semantic Versioning](https://semver.org/). - Support strict operator SSH through declared jump hosts. - Allow a standby to override repmgr failover and priority policy. +- Allow a standby to be seeded from the latest valid pgBackRest backup. +- Make deployment-flow `--dry-run` stop before confirmation or mutation. ## 0.9.6 - 2026-08-20 diff --git a/README.md b/README.md index 294865b..befc26b 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ production: - host: 100.63.4.227 private_ip: 10.8.0.12 jump_host: app-oci + seed_method: pgbackrest repmgr: auto_failover: false priority: 0 @@ -71,6 +72,9 @@ production: connection through a declared host. A standby-level `repmgr` block overrides the cluster defaults, so a cross-provider disaster-recovery replica can remain manual-promotion while the ordinary standby retains automatic failover. +`seed_method: pgbackrest` restores the latest valid backup with standby recovery +settings before repmgr registration, avoiding a full base-backup stream from +the primary. The `pgbackrest` component must be enabled. Add credentials (`rails credentials:edit`): diff --git a/config/postgres.example.yml b/config/postgres.example.yml index 1d06603..519d43c 100644 --- a/config/postgres.example.yml +++ b/config/postgres.example.yml @@ -281,6 +281,7 @@ production: private_ip: 10.8.0.102 # Optional: Private IP for replication label: eu-west-1 # jump_host: app-oci + # seed_method: pgbackrest # Restore latest backup instead of cloning from primary # Keep a cross-provider DR standby out of automatic promotion elections. # repmgr: # auto_failover: false diff --git a/lib/active_postgres/components/repmgr.rb b/lib/active_postgres/components/repmgr.rb index 52a644f..0313c25 100644 --- a/lib/active_postgres/components/repmgr.rb +++ b/lib/active_postgres/components/repmgr.rb @@ -244,6 +244,7 @@ def setup_standby(standby_host) repmgr_db = config.repmgr_database postgres_user = config.postgres_user replication_user = config.replication_user + seed_method = config.standby_seed_method_for(standby_host) # Variables used in ERB templates via binding _ = host @@ -300,58 +301,63 @@ def setup_standby(standby_host) nil end - info 'Cloning from primary over private network...' - # Create a temporary repmgr config for cloning that points to the primary - # The regular config points to localhost which doesn't work during initial clone - temp_repmgr_conf = <<~CONF - node_id=#{node_id} - node_name='#{standby_host}' - conninfo='host=#{primary_replication_host} user=#{repmgr_user} dbname=#{repmgr_db} connect_timeout=10' - data_directory='/var/lib/postgresql/#{version}/main' - CONF - - info 'Creating temporary repmgr config for cloning...' - upload! StringIO.new(temp_repmgr_conf), '/tmp/repmgr_clone.conf' - execute :sudo, 'mv', '/tmp/repmgr_clone.conf', '/etc/repmgr_clone.conf' - execute :sudo, 'chown', 'postgres:postgres', '/etc/repmgr_clone.conf' - execute :sudo, 'chmod', '600', '/etc/repmgr_clone.conf' - - info 'Running: repmgr standby clone (this may take a few minutes to copy the database)...' + if seed_method == :repmgr + info 'Cloning from primary over private network...' + # Create a temporary repmgr config for cloning that points to the primary. + # The regular config points to localhost, which does not work during the initial clone. + temp_repmgr_conf = <<~CONF + node_id=#{node_id} + node_name='#{standby_host}' + conninfo='host=#{primary_replication_host} user=#{repmgr_user} dbname=#{repmgr_db} connect_timeout=10' + data_directory='/var/lib/postgresql/#{version}/main' + CONF + + info 'Creating temporary repmgr config for cloning...' + upload! StringIO.new(temp_repmgr_conf), '/tmp/repmgr_clone.conf' + execute :sudo, 'mv', '/tmp/repmgr_clone.conf', '/etc/repmgr_clone.conf' + execute :sudo, 'chown', 'postgres:postgres', '/etc/repmgr_clone.conf' + execute :sudo, 'chmod', '600', '/etc/repmgr_clone.conf' + info 'Running: repmgr standby clone (this may take a few minutes to copy the database)...' + else + info 'Restoring the latest valid pgBackRest backup as a standby...' + end clone_success = false begin - execute :sudo, '-u', postgres_user, 'env', - 'HOME=/var/lib/postgresql', - 'repmgr', 'standby', 'clone', - '-h', primary_replication_host, - '-U', repmgr_user, - '-d', repmgr_db, - '-f', '/etc/repmgr_clone.conf', - '--force', - '--verbose' - - info 'Clone command completed' + if seed_method == :pgbackrest + standby_label = config.node_label_for(standby_host) || "standby-#{standby_host.split('.').first}" + primary_conninfo = build_primary_conninfo(standby_label) + execute(*pgbackrest_restore_arguments(primary_conninfo)) + info 'pgBackRest standby restore completed' + else + execute :sudo, '-u', postgres_user, 'env', + 'HOME=/var/lib/postgresql', + 'repmgr', 'standby', 'clone', + '-h', primary_replication_host, + '-U', repmgr_user, + '-d', repmgr_db, + '-f', '/etc/repmgr_clone.conf', + '--force', + '--verbose' + info 'Clone command completed' + end - # Check if clone actually succeeded if test(:sudo, 'test', '-d', "/var/lib/postgresql/#{version}/main") clone_success = true - info 'Data directory successfully cloned!' - - # Clean up temporary clone config - execute :sudo, 'rm', '-f', '/etc/repmgr_clone.conf' + info 'Standby data directory successfully seeded!' + execute :sudo, 'rm', '-f', '/etc/repmgr_clone.conf' if seed_method == :repmgr else - error "Data directory /var/lib/postgresql/#{version}/main was not created even though command succeeded!" + error "Data directory /var/lib/postgresql/#{version}/main was not created even though seeding succeeded!" end rescue SSHKit::Command::Failed => e - error "Clone command failed with exit code #{e.message}" + error "Standby seed command failed with exit code #{e.message}" rescue StandardError => e - error "Clone command raised exception: #{e.class}: #{e.message}" + error "Standby seed command raised exception: #{e.class}: #{e.message}" end - # If clone failed, raise error unless clone_success - error '✗ repmgr standby clone failed' - raise 'repmgr standby clone failed to create data directory' + error '✗ Standby data seeding failed' + raise 'standby data seeding failed to create data directory' end end @@ -907,6 +913,15 @@ def build_primary_conninfo(standby_label) "host=#{primary_host} user=#{user} dbname=#{dbname} application_name=#{standby_label}" end + def pgbackrest_restore_arguments(primary_conninfo) + [ + :sudo, '-u', config.postgres_user, + 'pgbackrest', '--stanza=main', '--type=standby', + "--recovery-option=primary_conninfo=#{primary_conninfo}", + 'restore' + ] + end + def normalize_repmgr_password(raw_password) password = raw_password.to_s.rstrip diff --git a/lib/active_postgres/configuration.rb b/lib/active_postgres/configuration.rb index 11cb181..87e5331 100644 --- a/lib/active_postgres/configuration.rb +++ b/lib/active_postgres/configuration.rb @@ -1,7 +1,10 @@ require 'yaml' +require_relative 'configuration/standby_policy' module ActivePostgres class Configuration + include StandbyPolicy + attr_reader :environment, :version, :user, :ssh_key, :ssh_known_hosts_file, :ssh_host_key_verification, :primary, :standbys, :jump_hosts, :components, :secrets_config, :database_config @@ -178,6 +181,8 @@ def validate! raise Error, "Jump host #{jump_host_name} must include host" if jump_host['host'].to_s.strip.empty? end + validate_standby_seed_methods! + if component_enabled?(:repmgr) dns_failover = component_config(:repmgr)[:dns_failover] if dns_failover && dns_failover[:enabled] diff --git a/lib/active_postgres/configuration/standby_policy.rb b/lib/active_postgres/configuration/standby_policy.rb new file mode 100644 index 0000000..d9a832e --- /dev/null +++ b/lib/active_postgres/configuration/standby_policy.rb @@ -0,0 +1,24 @@ +module ActivePostgres + class Configuration + module StandbyPolicy + def standby_seed_method_for(host) + (standby_config_for(host)&.fetch('seed_method', nil) || 'repmgr').to_s.to_sym + end + + private + + def validate_standby_seed_methods! + @standbys.each do |standby| + host = standby.fetch('host') + seed_method = standby_seed_method_for(host) + unless %i[repmgr pgbackrest].include?(seed_method) + raise Error, "Invalid seed_method '#{seed_method}' for standby #{host}. Use 'repmgr' or 'pgbackrest'." + end + if seed_method == :pgbackrest && !component_enabled?(:pgbackrest) + raise Error, "Standby #{host} requires the pgbackrest component for seed_method 'pgbackrest'" + end + end + end + end + end +end diff --git a/lib/active_postgres/deployment_flow.rb b/lib/active_postgres/deployment_flow.rb index a1471a7..7ecccef 100644 --- a/lib/active_postgres/deployment_flow.rb +++ b/lib/active_postgres/deployment_flow.rb @@ -13,12 +13,16 @@ def initialize(config, ssh_executor:, secrets:, logger:, rollback_manager:, skip @skip_validation = skip_validation end - def execute + def execute(dry_run: false) ErrorHandler.with_handling(context: { operation: operation_name }) do print_header validate_prerequisites run_preflight_checks unless skip_validation print_deployment_plan + if dry_run + logger.info "\n[DRY RUN] No changes made." + return + end return unless confirm_deployment rollback_manager.with_rollback(description: operation_name) do diff --git a/lib/active_postgres/installer.rb b/lib/active_postgres/installer.rb index bf5cf2e..05d4da6 100644 --- a/lib/active_postgres/installer.rb +++ b/lib/active_postgres/installer.rb @@ -28,7 +28,7 @@ def setup rollback_manager: rollback_manager, skip_validation: skip_validation ) - flow.execute + flow.execute(dry_run: dry_run) end def setup_component(component_name) @@ -117,7 +117,7 @@ def setup_standby_only(standby_host) rollback_manager: rollback_manager, skip_validation: skip_validation ) - flow.execute + flow.execute(dry_run: dry_run) end end end diff --git a/lib/active_postgres/standby_deployment_flow.rb b/lib/active_postgres/standby_deployment_flow.rb index 1387e3b..c05c4df 100644 --- a/lib/active_postgres/standby_deployment_flow.rb +++ b/lib/active_postgres/standby_deployment_flow.rb @@ -49,17 +49,22 @@ def check_if_standby_already_deployed def list_deployment_steps logger.info " • Install PostgreSQL #{config.version} packages on #{standby_host}" - logger.info " • Clone data from primary: #{config.primary_host}" + if config.standby_seed_method_for(standby_host) == :pgbackrest + logger.info ' • Restore the latest valid pgBackRest backup as a standby' + else + logger.info " • Clone data from primary: #{config.primary_host}" + end logger.info ' • Register standby with repmgr cluster' end def list_warnings - logger.info "\n⚠️ Primary database will NOT be touched" + logger.info "\n⚠️ Primary stays online; setup performs idempotent role and repmgr checks" end def deploy_components deploy_ssl if config.component_enabled?(:ssl) deploy_core + deploy_pgbackrest if config.component_enabled?(:pgbackrest) deploy_repmgr deploy_optional_components end @@ -79,8 +84,15 @@ def deploy_core end end + def deploy_pgbackrest + logger.task('Installing pgBackRest on standby') do + component = Components::PgBackRest.new(config, ssh_executor, secrets) + component.install_on_standby(standby_host) + end + end + def deploy_repmgr - logger.task('Setting up repmgr and cloning from primary') do + logger.task('Setting up repmgr and seeding standby') do component = Components::Repmgr.new(config, ssh_executor, secrets) register_rollback('repmgr', component) component.setup_standby_only(standby_host) diff --git a/test/deployment_flow_test.rb b/test/deployment_flow_test.rb new file mode 100644 index 0000000..068138f --- /dev/null +++ b/test/deployment_flow_test.rb @@ -0,0 +1,38 @@ +require 'test_helper' + +class DeploymentFlowTest < Minitest::Test + class DryRunFlow < ActivePostgres::DeploymentFlow + attr_reader :deployed + + private + + def operation_name = 'Dry run' + def print_targets; end + def run_preflight_checks; end + def list_deployment_steps; end + def list_next_steps; end + + def deploy_components + @deployed = true + end + end + + def test_dry_run_stops_before_confirmation_and_deployment + config = Object.new + config.define_singleton_method(:validate!) { true } + config.define_singleton_method(:environment) { 'test' } + logger = ActivePostgres::Logger.new + + flow = DryRunFlow.new( + config, + ssh_executor: Object.new, + secrets: Object.new, + logger:, + rollback_manager: Object.new + ) + + flow.execute(dry_run: true) + + refute flow.deployed + end +end diff --git a/test/jump_host_configuration_test.rb b/test/jump_host_configuration_test.rb index 7c69e5c..e35c93f 100644 --- a/test/jump_host_configuration_test.rb +++ b/test/jump_host_configuration_test.rb @@ -48,6 +48,43 @@ def test_standby_repmgr_policy_overrides_the_cluster_default assert_equal 0, policy.fetch(:priority) end + def test_standby_can_be_seeded_from_pgbackrest + config = build_config( + 'components' => { 'pgbackrest' => { 'enabled' => true } }, + 'standby' => [ + { 'host' => 'standby.example.com', 'seed_method' => 'pgbackrest' } + ] + ) + + config.validate! + + assert_equal :pgbackrest, config.standby_seed_method_for('standby.example.com') + end + + def test_pgbackrest_seed_requires_the_component + config = build_config( + 'standby' => [ + { 'host' => 'standby.example.com', 'seed_method' => 'pgbackrest' } + ] + ) + + error = assert_raises(ActivePostgres::Error) { config.validate! } + + assert_match(/requires the pgbackrest component/, error.message) + end + + def test_unknown_seed_method_fails_validation + config = build_config( + 'standby' => [ + { 'host' => 'standby.example.com', 'seed_method' => 'snapshot' } + ] + ) + + error = assert_raises(ActivePostgres::Error) { config.validate! } + + assert_match(/Invalid seed_method 'snapshot'/, error.message) + end + private def build_config(overrides) diff --git a/test/repmgr_test.rb b/test/repmgr_test.rb index a0a2978..1c52f72 100644 --- a/test/repmgr_test.rb +++ b/test/repmgr_test.rb @@ -49,6 +49,18 @@ def test_build_primary_conninfo_uses_replication_user_when_password_present assert_equal 'host=10.0.0.10 user=repl_user dbname=replication application_name=standby-node', conninfo end + def test_pgbackrest_restore_uses_standby_mode_and_primary_conninfo + conninfo = 'host=10.0.0.10 user=replication dbname=replication application_name=aws-standby' + arguments = @component.send(:pgbackrest_restore_arguments, conninfo) + + assert_equal [ + :sudo, '-u', 'postgres', + 'pgbackrest', '--stanza=main', '--type=standby', + "--recovery-option=primary_conninfo=#{conninfo}", + 'restore' + ], arguments + end + def test_manual_failover_policy_disables_repmgrd commands = [] backend = Object.new diff --git a/test/test_helper.rb b/test/test_helper.rb index b4b30d4..acb3beb 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -106,6 +106,10 @@ def repmgr_config_for(host) component_config(:repmgr).merge(overrides.transform_keys(&:to_sym)) end + def standby_seed_method_for(host) + (standby_config_for(host)&.fetch('seed_method', nil) || 'repmgr').to_s.to_sym + end + def node_label_for(host) if host == primary_host primary['label']