diff --git a/README.md b/README.md index befc26b..4a86939 100644 --- a/README.md +++ b/README.md @@ -199,8 +199,13 @@ components: pgbackrest: retention_full: 7 retention_archive: 14 - # You can schedule full + incremental separately: - # schedule_full: "0 2 * * *" + # Reduce incremental backup churn for large, frequently changed relations. + # Block backups require repository bundling. + repo_bundle: true + repo_block: true + # You can schedule full, differential, and incremental backups separately: + # schedule_full: "0 2 * * 0" + # schedule_differential: "35 2 * * 1-6" # schedule_incremental: "0 * * * *" ``` diff --git a/lib/active_postgres/components/pgbackrest.rb b/lib/active_postgres/components/pgbackrest.rb index da79d0c..250493c 100644 --- a/lib/active_postgres/components/pgbackrest.rb +++ b/lib/active_postgres/components/pgbackrest.rb @@ -26,6 +26,7 @@ def uninstall ssh_executor.execute_on_host(host) do execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest' execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup' + execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-differential' execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental' execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE @@ -162,10 +163,12 @@ def setup_backup_schedules(host, pgbackrest_config) def backup_schedules(pgbackrest_config) schedule_full = pgbackrest_config[:schedule_full] || pgbackrest_config[:schedule] + schedule_differential = pgbackrest_config[:schedule_differential] schedule_incremental = pgbackrest_config[:schedule_incremental] schedules = [] schedules << { type: 'full', schedule: schedule_full, file: '/etc/cron.d/pgbackrest-backup' } if schedule_full + schedules << { type: 'diff', schedule: schedule_differential, file: '/etc/cron.d/pgbackrest-backup-differential' } if schedule_differential schedules << { type: 'incr', schedule: schedule_incremental, file: '/etc/cron.d/pgbackrest-backup-incremental' } if schedule_incremental schedules @@ -192,6 +195,7 @@ def setup_backup_schedule(host, schedule, backup_type = 'full', cron_file = '/et def remove_backup_schedule(host) ssh_executor.execute_on_host(host) do execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup' + execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-differential' execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental' end end diff --git a/lib/active_postgres/configuration.rb b/lib/active_postgres/configuration.rb index 87e5331..d3ffcf8 100644 --- a/lib/active_postgres/configuration.rb +++ b/lib/active_postgres/configuration.rb @@ -1,8 +1,10 @@ require 'yaml' +require_relative 'configuration/pgbackrest_policy' require_relative 'configuration/standby_policy' module ActivePostgres class Configuration + include PgBackRestPolicy include StandbyPolicy attr_reader :environment, :version, :user, :ssh_key, :ssh_known_hosts_file, :ssh_host_key_verification, @@ -156,14 +158,7 @@ def validate! raise Error, 'monitoring.grafana.host is required when grafana is enabled' end end - if component_enabled?(:pgbackrest) - pg_config = component_config(:pgbackrest) - retention_full = pg_config[:retention_full] - retention_archive = pg_config[:retention_archive] - if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i - raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety' - end - end + validate_pgbackrest! if component_enabled?(:pgbouncer) pgbouncer_app_hosts.each do |app_host| diff --git a/lib/active_postgres/configuration/pgbackrest_policy.rb b/lib/active_postgres/configuration/pgbackrest_policy.rb new file mode 100644 index 0000000..908bc49 --- /dev/null +++ b/lib/active_postgres/configuration/pgbackrest_policy.rb @@ -0,0 +1,19 @@ +module ActivePostgres + class Configuration + module PgBackRestPolicy + private + + def validate_pgbackrest! + return unless component_enabled?(:pgbackrest) + + pg_config = component_config(:pgbackrest) + retention_full = pg_config[:retention_full] + retention_archive = pg_config[:retention_archive] + if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i + raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety' + end + raise Error, 'pgbackrest.repo_block requires repo_bundle' if pg_config[:repo_block] && !pg_config[:repo_bundle] + end + end + end +end diff --git a/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb b/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb index e1c5863..1135aa2 100644 --- a/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb +++ b/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb @@ -57,6 +57,13 @@ shared: &shared pgbackrest: enabled: false + # For large, frequently changed relation files, bundle repository objects + # and back up changed blocks instead of copying each whole file. + # repo_bundle: true + # repo_block: true + # schedule_full: "0 2 * * 0" + # schedule_differential: "35 2 * * 1-6" + # schedule_incremental: "5 */4 * * *" monitoring: enabled: false diff --git a/templates/pgbackrest.conf.erb b/templates/pgbackrest.conf.erb index def15d8..0cdabae 100644 --- a/templates/pgbackrest.conf.erb +++ b/templates/pgbackrest.conf.erb @@ -40,6 +40,15 @@ repo1-path=<%= pgbackrest_config[:repo_path] || '/var/lib/pgbackrest' %> repo1-retention-full=<%= pgbackrest_config[:retention_full] || 7 %> repo1-retention-archive=<%= pgbackrest_config[:retention_archive] || 14 %> +<% if pgbackrest_config[:repo_bundle] %> +# Bundle small files and metadata into fewer repository objects. +repo1-bundle=y +<% end %> +<% if pgbackrest_config[:repo_block] %> +# Back up changed blocks instead of entire relation files. +repo1-block=y +<% end %> + # Compression compress-type=lz4 compress-level=3 @@ -69,4 +78,3 @@ pg1-path=/var/lib/postgresql/<%= config.version %>/main pg1-port=5432 pg1-socket-path=/var/run/postgresql pg1-user=<%= config.postgres_user %> - diff --git a/test/components/pgbackrest_templates_test.rb b/test/components/pgbackrest_templates_test.rb index 6557347..f919fd0 100644 --- a/test/components/pgbackrest_templates_test.rb +++ b/test/components/pgbackrest_templates_test.rb @@ -77,6 +77,20 @@ def test_conf_default_retention assert_includes content, 'repo1-retention-archive=14' end + def test_conf_enables_bundled_block_incremental_backups + content = render_conf(repo_type: 'local', repo_bundle: true, repo_block: true) + + assert_includes content, 'repo1-bundle=y' + assert_includes content, 'repo1-block=y' + end + + def test_conf_leaves_bundled_block_incremental_backups_disabled_by_default + content = render_conf(repo_type: 'local') + + refute_includes content, 'repo1-bundle=' + refute_includes content, 'repo1-block=' + end + def test_conf_includes_compression content = render_conf(repo_type: 'local') diff --git a/test/components/pgbackrest_test.rb b/test/components/pgbackrest_test.rb index eba401d..8c2a330 100644 --- a/test/components/pgbackrest_test.rb +++ b/test/components/pgbackrest_test.rb @@ -35,13 +35,16 @@ def test_backup_schedules_prefers_schedule_full schedules = component.send(:backup_schedules, { schedule: '0 2 * * *', schedule_full: '0 3 * * *', + schedule_differential: '30 3 * * 1-6', schedule_incremental: '0 * * * *' }) full = schedules.find { |entry| entry[:type] == 'full' } + differential = schedules.find { |entry| entry[:type] == 'diff' } incremental = schedules.find { |entry| entry[:type] == 'incr' } assert_equal '0 3 * * *', full[:schedule] + assert_equal '30 3 * * 1-6', differential[:schedule] assert_equal '0 * * * *', incremental[:schedule] end @@ -61,12 +64,14 @@ def test_setup_backup_schedules_rewrites_existing_cron_files_before_installing_n component.send(:setup_backup_schedules, 'primary.example.com', { schedule_full: '0 2 * * *', + schedule_differential: '30 2 * * 1-6', schedule_incremental: '0 * * * *' }) assert_equal [:remove, 'primary.example.com'], calls.first assert_equal [:schedule, 'primary.example.com', '0 2 * * *', 'full', '/etc/cron.d/pgbackrest-backup'], calls[1] - assert_equal [:schedule, 'primary.example.com', '0 * * * *', 'incr', '/etc/cron.d/pgbackrest-backup-incremental'], calls[2] + assert_equal [:schedule, 'primary.example.com', '30 2 * * 1-6', 'diff', '/etc/cron.d/pgbackrest-backup-differential'], calls[2] + assert_equal [:schedule, 'primary.example.com', '0 * * * *', 'incr', '/etc/cron.d/pgbackrest-backup-incremental'], calls[3] end def test_setup_backup_schedule_uses_backup_type diff --git a/test/configuration_test.rb b/test/configuration_test.rb index aa7da38..61960fa 100644 --- a/test/configuration_test.rb +++ b/test/configuration_test.rb @@ -170,6 +170,34 @@ def test_pgbackrest_retention_validation_passes_when_archive_gte_full assert config.validate! end + def test_pgbackrest_block_backups_require_bundling + config = build_config( + 'components' => { + 'pgbackrest' => { + 'enabled' => true, + 'repo_block' => true + } + } + ) + + error = assert_raises(ActivePostgres::Error) { config.validate! } + assert_match(/repo_block requires repo_bundle/i, error.message) + end + + def test_pgbackrest_block_backups_accept_bundling + config = build_config( + 'components' => { + 'pgbackrest' => { + 'enabled' => true, + 'repo_bundle' => true, + 'repo_block' => true + } + } + ) + + assert config.validate! + end + def test_standby_config_for_returns_standby_hash config = build_config( 'standby' => [