diff --git a/CHANGELOG.md b/CHANGELOG.md index 406982a..5ae4b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project follows [Semantic Versioning](https://semver.org/). ## Unreleased +### Added + +- Support strict operator SSH through declared jump hosts. +- Allow a standby to override repmgr failover and priority policy. + ## 0.9.6 - 2026-08-20 ### Changed diff --git a/README.md b/README.md index 250f099..294865b 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ production: version: 18 user: ubuntu ssh_key: ~/.ssh/id_rsa + ssh_known_hosts_file: config/ssh_known_hosts + + jump_hosts: + app-oci: + host: 129.213.194.40 primary: host: 34.12.234.81 # Public IP for SSH @@ -43,6 +48,12 @@ production: standby: - host: 52.23.45.67 private_ip: 10.8.0.11 + - host: 100.63.4.227 + private_ip: 10.8.0.12 + jump_host: app-oci + repmgr: + auto_failover: false + priority: 0 components: repmgr: {enabled: true} @@ -56,6 +67,11 @@ production: app_password: $POSTGRES_APP_PASSWORD ``` +`jump_host` keeps the operator SSH key on the workstation and proxies the +connection through a declared host. A standby-level `repmgr` block overrides +the cluster defaults, so a cross-provider disaster-recovery replica can remain +manual-promotion while the ordinary standby retains automatic failover. + Add credentials (`rails credentials:edit`): ```yaml diff --git a/config/postgres.example.yml b/config/postgres.example.yml index 07e74b2..1d06603 100644 --- a/config/postgres.example.yml +++ b/config/postgres.example.yml @@ -13,8 +13,17 @@ shared: &shared version: 18 user: ubuntu ssh_key: ~/.ssh/id_rsa + # Use one reviewed host-key file for direct and proxied SSH connections. + # ssh_known_hosts_file: config/ssh_known_hosts # SSH host key verification: always (strict) or accept_new (first connection) ssh_host_key_verification: always + + # Optional operator jump hosts. Standbys reference these by name; private + # keys remain on the operator workstation. + # jump_hosts: + # app-oci: + # host: 129.213.194.40 + # user: ubuntu # Components (enable as needed) components: @@ -271,6 +280,11 @@ production: - host: 18.156.78.90 # Public IP for SSH private_ip: 10.8.0.102 # Optional: Private IP for replication label: eu-west-1 + # jump_host: app-oci + # Keep a cross-provider DR standby out of automatic promotion elections. + # repmgr: + # auto_failover: false + # priority: 0 # Optional: Add more replicas for read scaling # - host: 35.178.45.23 diff --git a/lib/active_postgres/components/repmgr.rb b/lib/active_postgres/components/repmgr.rb index 05791e2..52a644f 100644 --- a/lib/active_postgres/components/repmgr.rb +++ b/lib/active_postgres/components/repmgr.rb @@ -103,7 +103,7 @@ def setup_primary puts ' Setting up primary with repmgr...' host = config.primary_host - repmgr_config = config.component_config(:repmgr) + repmgr_config = config.repmgr_config_for(host) version = config.version repmgr_password = normalize_repmgr_password(secrets.resolve('repmgr_password')) replication_password = normalize_replication_password(secrets.resolve('replication_password')) @@ -236,7 +236,7 @@ def setup_standby(standby_host) puts " Setting up standby: #{standby_host}..." host = standby_host - repmgr_config = config.component_config(:repmgr) + repmgr_config = config.repmgr_config_for(standby_host) primary_replication_host = config.primary_replication_host version = config.version secrets_obj = secrets @@ -1101,7 +1101,12 @@ def update_postgres_configs_on_standby(host, version) end def enable_repmgrd_if_configured(host, repmgr_config) - return if repmgr_config[:auto_failover] == false + if repmgr_config[:auto_failover] == false + ssh_executor.execute_on_host(host) do + execute :sudo, 'systemctl', 'disable', '--now', 'repmgrd', raise_on_non_zero_exit: false + end + return + end default_config = <<~CONF REPMGRD_ENABLED=yes diff --git a/lib/active_postgres/configuration.rb b/lib/active_postgres/configuration.rb index a81b9b3..11cb181 100644 --- a/lib/active_postgres/configuration.rb +++ b/lib/active_postgres/configuration.rb @@ -2,8 +2,8 @@ module ActivePostgres class Configuration - attr_reader :environment, :version, :user, :ssh_key, :ssh_host_key_verification, :primary, :standbys, :components, :secrets_config, - :database_config + attr_reader :environment, :version, :user, :ssh_key, :ssh_known_hosts_file, :ssh_host_key_verification, + :primary, :standbys, :jump_hosts, :components, :secrets_config, :database_config def initialize(config_hash, environment = 'development') @environment = environment @@ -14,6 +14,7 @@ def initialize(config_hash, environment = 'development') @version = env_config['version'] || 18 @user = env_config['user'] || 'ubuntu' @ssh_key = File.expand_path(env_config['ssh_key'] || '~/.ssh/id_rsa') + @ssh_known_hosts_file = optional_path(env_config['ssh_known_hosts_file']) @ssh_host_key_verification = normalize_ssh_host_key_verification( env_config['ssh_host_key_verification'] || env_config['ssh_verify_host_key'] ) @@ -21,6 +22,7 @@ def initialize(config_hash, environment = 'development') @primary = env_config['primary'] || {} @standbys = env_config['standby'] || [] @standbys = [@standbys] unless @standbys.is_a?(Array) + @jump_hosts = env_config['jump_hosts'] || {} @components = parse_components(env_config['components'] || {}) @secrets_config = env_config['secrets'] || {} @@ -111,6 +113,24 @@ def standby_config_for(host) @standbys.find { |s| s['host'] == host } end + def node_config_for(host) + return @primary if host == primary_host + + standby_config_for(host) + end + + def jump_host_config_for(host) + jump_host_name = node_config_for(host)&.dig('jump_host') + return unless jump_host_name + + @jump_hosts[jump_host_name] || raise(Error, "Host #{host} jump_host not found: #{jump_host_name}") + end + + def repmgr_config_for(host) + overrides = node_config_for(host)&.dig('repmgr') || {} + component_config(:repmgr).merge(symbolize_keys(overrides)) + end + def node_label_for(host) if host == primary_host @primary['label'] @@ -149,6 +169,15 @@ def validate! end end + all_hosts.each do |host| + jump_host_name = node_config_for(host)&.dig('jump_host') + next unless jump_host_name + + jump_host = @jump_hosts[jump_host_name] + raise Error, "Host #{host} jump_host not found: #{jump_host_name}" unless jump_host + raise Error, "Jump host #{jump_host_name} must include host" if jump_host['host'].to_s.strip.empty? + end + if component_enabled?(:repmgr) dns_failover = component_config(:repmgr)[:dns_failover] if dns_failover && dns_failover[:enabled] @@ -246,12 +275,6 @@ def parse_components(components_config) result end - def node_config_for(host) - return @primary if host == primary_host - - standby_config_for(host) - end - def private_ip_for(node_config) return unless node_config @@ -295,5 +318,9 @@ def normalize_ssh_host_key_verification(value) raise Error, "Invalid ssh_host_key_verification '#{value}'. Use 'always' or 'accept_new'." end + + def optional_path(path) + File.expand_path(path) if path + end end end diff --git a/lib/active_postgres/ssh_executor.rb b/lib/active_postgres/ssh_executor.rb index b5a13b5..eb38233 100644 --- a/lib/active_postgres/ssh_executor.rb +++ b/lib/active_postgres/ssh_executor.rb @@ -1,6 +1,8 @@ require 'sshkit' require 'sshkit/dsl' +require 'net/ssh/proxy/command' require 'securerandom' +require 'shellwords' require 'stringio' module ActivePostgres @@ -20,11 +22,11 @@ def quiet? end def execute_on_host(host, &) - on("#{config.user}@#{host}", &) + on(ssh_host_for(host), &) end def execute_on_host_as(host, user, &) - on("#{user}@#{host}", &) + on(ssh_host_for(host, user:), &) end def execute_on_primary(&) @@ -32,12 +34,12 @@ def execute_on_primary(&) end def execute_on_standbys(&) - hosts = config.standby_hosts.map { |h| "#{config.user}@#{h}" } + hosts = config.standby_hosts.map { |host| ssh_host_for(host) } on(hosts, in: :parallel, &) end def execute_on_all_hosts(&) - hosts = config.all_hosts.map { |h| "#{config.user}@#{h}" } + hosts = config.all_hosts.map { |host| ssh_host_for(host) } on(hosts, in: :parallel, &) end @@ -307,6 +309,8 @@ def setup_sshkit number_of_password_prompts: 0 } + options[:user_known_hosts_file] = [config.ssh_known_hosts_file] if config.ssh_known_hosts_file + if config.ssh_key && File.exist?(config.ssh_key) options[:keys] = [config.ssh_key] options[:keys_only] = true @@ -315,5 +319,58 @@ def setup_sshkit ssh.ssh_options = options end end + + def ssh_host_for(host, user: config.user) + ssh_host = SSHKit::Host.new(host) + ssh_host.user = user + + node_config = config.node_config_for(host) || {} + ssh_host.port = node_config['ssh_port'] || node_config['port'] if node_config['ssh_port'] || node_config['port'] + + options = { + verify_host_key: config.ssh_host_key_verification || :always + } + options[:user_known_hosts_file] = [config.ssh_known_hosts_file] if config.ssh_known_hosts_file + + if node_config['ssh_key'] + options[:keys] = [File.expand_path(node_config['ssh_key'])] + options[:keys_only] = true + end + + if (jump_host_config = config.jump_host_config_for(host)) + options[:proxy] = jump_proxy(jump_host_config) + end + + ssh_host.ssh_options = options + ssh_host + end + + def jump_proxy(jump_host_config) + jump_user = jump_host_config['ssh_user'] || jump_host_config['user'] || config.user + jump_port = jump_host_config['ssh_port'] || jump_host_config['port'] + jump_key = jump_host_config['ssh_key'] || config.ssh_key + command = [ + 'ssh', + '-F', '/dev/null', + '-o', 'BatchMode=yes', + '-o', 'ForwardAgent=no', + '-o', "StrictHostKeyChecking=#{open_ssh_host_key_mode}" + ] + + command.push('-o', "UserKnownHostsFile=#{config.ssh_known_hosts_file}") if config.ssh_known_hosts_file + + command.push('-i', File.expand_path(jump_key), '-o', 'IdentitiesOnly=yes') if + jump_key && File.exist?(File.expand_path(jump_key)) + + command.push('-p', jump_port.to_s) if jump_port + command.push('-W', '%h:%p', "#{jump_user}@#{jump_host_config.fetch('host')}") + + command_line = Shellwords.join(command).gsub('\\%h', '%h').gsub('\\%p', '%p') + Net::SSH::Proxy::Command.new(command_line) + end + + def open_ssh_host_key_mode + config.ssh_host_key_verification == :accept_new ? 'accept-new' : 'yes' + end end end diff --git a/test/jump_host_configuration_test.rb b/test/jump_host_configuration_test.rb new file mode 100644 index 0000000..7c69e5c --- /dev/null +++ b/test/jump_host_configuration_test.rb @@ -0,0 +1,65 @@ +require 'test_helper' + +class JumpHostConfigurationTest < Minitest::Test + def test_standby_can_reference_a_declared_jump_host + config = build_config( + 'jump_hosts' => { + 'app-oci' => { 'host' => '1.2.3.4', 'user' => 'ubuntu' } + }, + 'standby' => [ + { 'host' => '5.6.7.8', 'private_ip' => '10.0.0.11', 'jump_host' => 'app-oci' } + ] + ) + + assert_equal '1.2.3.4', config.jump_host_config_for('5.6.7.8').fetch('host') + end + + def test_unknown_jump_host_fails_validation + config = build_config( + 'components' => { 'repmgr' => { 'enabled' => true } }, + 'standby' => [ + { 'host' => '5.6.7.8', 'private_ip' => '10.0.0.11', 'jump_host' => 'missing' } + ], + 'secrets' => { 'replication_password' => 'secret' } + ) + + error = assert_raises(ActivePostgres::Error) { config.validate! } + + assert_match(/jump_host not found: missing/, error.message) + end + + def test_standby_repmgr_policy_overrides_the_cluster_default + config = build_config( + 'components' => { + 'repmgr' => { 'enabled' => true, 'auto_failover' => true, 'priority' => 100 } + }, + 'standby' => [ + { + 'host' => 'standby.example.com', + 'private_ip' => '10.0.0.11', + 'repmgr' => { 'auto_failover' => false, 'priority' => 0 } + } + ] + ) + + policy = config.repmgr_config_for('standby.example.com') + + assert_equal false, policy.fetch(:auto_failover) + assert_equal 0, policy.fetch(:priority) + end + + private + + def build_config(overrides) + config = { + 'test' => { + 'primary' => { 'host' => 'primary.example.com' }, + 'standby' => [{ 'host' => 'standby.example.com' }], + 'components' => {}, + 'secrets' => {} + }.merge(overrides) + } + + ActivePostgres::Configuration.new(config, 'test') + end +end diff --git a/test/repmgr_test.rb b/test/repmgr_test.rb index b085147..a0a2978 100644 --- a/test/repmgr_test.rb +++ b/test/repmgr_test.rb @@ -49,6 +49,26 @@ def test_build_primary_conninfo_uses_replication_user_when_password_present assert_equal 'host=10.0.0.10 user=repl_user dbname=replication application_name=standby-node', conninfo end + def test_manual_failover_policy_disables_repmgrd + commands = [] + backend = Object.new + backend.define_singleton_method(:execute) do |*args, **options| + commands << [args, options] + end + + executor = Object.new + executor.define_singleton_method(:execute_on_host) do |_host, &block| + backend.instance_exec(&block) + end + + component = ActivePostgres::Components::Repmgr.new(@config, executor, @secrets) + component.send(:enable_repmgrd_if_configured, 'standby.example.com', auto_failover: false) + + assert_equal [ + [[:sudo, 'systemctl', 'disable', '--now', 'repmgrd'], { raise_on_non_zero_exit: false }] + ], commands + end + def test_normalize_dns_servers_handles_hash_and_string raw = [ { 'host' => 'public.example.com', 'private_ip' => '10.0.0.10' }, diff --git a/test/ssh_executor_test.rb b/test/ssh_executor_test.rb new file mode 100644 index 0000000..8a4efe9 --- /dev/null +++ b/test/ssh_executor_test.rb @@ -0,0 +1,58 @@ +require 'test_helper' +require 'tempfile' + +class SSHExecutorTest < Minitest::Test + def test_standby_can_connect_through_a_declared_jump_host + Tempfile.create('known-hosts') do |known_hosts| + config = build_config( + 'ssh_key' => File::NULL, + 'ssh_known_hosts_file' => known_hosts.path, + 'ssh_host_key_verification' => 'accept_new', + 'jump_hosts' => { + 'app-oci' => { 'host' => '1.2.3.4', 'user' => 'ubuntu' } + }, + 'standby' => [ + { 'host' => '5.6.7.8', 'private_ip' => '10.0.0.11', 'jump_host' => 'app-oci' } + ] + ) + executor = ActivePostgres::SSHExecutor.new(config, quiet: true) + + host = executor.send(:ssh_host_for, '5.6.7.8') + proxy = host.ssh_options.fetch(:proxy) + + assert_instance_of Net::SSH::Proxy::Command, proxy + assert_equal [known_hosts.path], host.ssh_options.fetch(:user_known_hosts_file) + assert_equal :accept_new, host.ssh_options.fetch(:verify_host_key) + assert_includes proxy.command_line_template, 'ssh -F /dev/null' + assert_includes proxy.command_line_template, '-o BatchMode\\=yes' + assert_includes proxy.command_line_template, '-o ForwardAgent\\=no' + assert_includes proxy.command_line_template, '-o StrictHostKeyChecking\\=accept-new' + assert_includes proxy.command_line_template, "UserKnownHostsFile\\=#{known_hosts.path}" + assert_includes proxy.command_line_template, '-W %h:%p ubuntu@1.2.3.4' + end + end + + def test_direct_host_does_not_receive_a_proxy + config = build_config('standby' => [{ 'host' => '5.6.7.8', 'private_ip' => '10.0.0.11' }]) + executor = ActivePostgres::SSHExecutor.new(config, quiet: true) + + host = executor.send(:ssh_host_for, '5.6.7.8') + + refute host.ssh_options.key?(:proxy) + end + + private + + def build_config(overrides) + config = { + 'test' => { + 'primary' => { 'host' => 'primary.example.com' }, + 'standby' => [{ 'host' => 'standby.example.com' }], + 'components' => {}, + 'secrets' => {} + }.merge(overrides) + } + + ActivePostgres::Configuration.new(config, 'test') + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 00cbff1..b4b30d4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -20,7 +20,7 @@ module TestHelpers class ConfigStub attr_accessor :primary_host, :standby_hosts, :version, :primary, :standbys, :environment, :secrets_config, :ssh_key, - :ssh_host_key_verification, :user, :postgres_user, :pgbouncer_user, :app_user, :app_database, + :ssh_known_hosts_file, :ssh_host_key_verification, :user, :postgres_user, :pgbouncer_user, :app_user, :app_database, :repmgr_user, :repmgr_database, :replication_user def initialize(attrs = {}) @@ -32,6 +32,7 @@ def initialize(attrs = {}) @environment = attrs[:environment] || 'test' @secrets_config = attrs[:secrets_config] || {} @ssh_key = attrs[:ssh_key] || '~/.ssh/id_rsa' + @ssh_known_hosts_file = attrs[:ssh_known_hosts_file] @ssh_host_key_verification = attrs[:ssh_host_key_verification] || :always @user = attrs[:user] || 'ubuntu' @postgres_user = attrs[:postgres_user] || 'postgres' @@ -96,6 +97,15 @@ def standby_config_for(host) standbys.find { |s| s['host'] == host } end + def repmgr_config_for(host) + overrides = if host == primary_host + primary&.fetch('repmgr', {}) + else + standby_config_for(host)&.fetch('repmgr', {}) + end + component_config(:repmgr).merge(overrides.transform_keys(&:to_sym)) + end + def node_label_for(host) if host == primary_host primary['label']