Add bundled block backups and differential scheduling

README.md

@@ -199,8 +199,13 @@ components:
   pgbackrest:
     retention_full: 7
     retention_archive: 14
-    # You can schedule full + incremental separately:
-    # schedule_full: "0 2 * * *"
+    # Reduce incremental backup churn for large, frequently changed relations.
+    # Block backups require repository bundling.
+    repo_bundle: true
+    repo_block: true
+    # You can schedule full, differential, and incremental backups separately:
+    # schedule_full: "0 2 * * 0"
+    # schedule_differential: "35 2 * * 1-6"
     # schedule_incremental: "0 * * * *"
 ```
 

lib/active_postgres/components/pgbackrest.rb

@@ -26,6 +26,7 @@ def uninstall
           ssh_executor.execute_on_host(host) do
             execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest'
             execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
+            execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-differential'
             execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
             execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE
             execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE
@@ -162,10 +163,12 @@ def setup_backup_schedules(host, pgbackrest_config)
 
       def backup_schedules(pgbackrest_config)
         schedule_full = pgbackrest_config[:schedule_full] || pgbackrest_config[:schedule]
+        schedule_differential = pgbackrest_config[:schedule_differential]
         schedule_incremental = pgbackrest_config[:schedule_incremental]
 
         schedules = []
         schedules << { type: 'full', schedule: schedule_full, file: '/etc/cron.d/pgbackrest-backup' } if schedule_full
+        schedules << { type: 'diff', schedule: schedule_differential, file: '/etc/cron.d/pgbackrest-backup-differential' } if schedule_differential
         schedules << { type: 'incr', schedule: schedule_incremental, file: '/etc/cron.d/pgbackrest-backup-incremental' } if schedule_incremental
 
         schedules
@@ -192,6 +195,7 @@ def setup_backup_schedule(host, schedule, backup_type = 'full', cron_file = '/et
       def remove_backup_schedule(host)
         ssh_executor.execute_on_host(host) do
           execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
+          execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-differential'
           execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
         end
       end

lib/active_postgres/configuration.rb

@@ -1,8 +1,10 @@
 require 'yaml'
+require_relative 'configuration/pgbackrest_policy'
 require_relative 'configuration/standby_policy'
 
 module ActivePostgres
   class Configuration
+    include PgBackRestPolicy
     include StandbyPolicy
 
     attr_reader :environment, :version, :user, :ssh_key, :ssh_known_hosts_file, :ssh_host_key_verification,
@@ -156,14 +158,7 @@ def validate!
           raise Error, 'monitoring.grafana.host is required when grafana is enabled'
         end
       end
-      if component_enabled?(:pgbackrest)
-        pg_config = component_config(:pgbackrest)
-        retention_full = pg_config[:retention_full]
-        retention_archive = pg_config[:retention_archive]
-        if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
-          raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
-        end
-      end
+      validate_pgbackrest!
 
       if component_enabled?(:pgbouncer)
         pgbouncer_app_hosts.each do |app_host|

lib/active_postgres/configuration/pgbackrest_policy.rb

@@ -0,0 +1,19 @@
+module ActivePostgres
+  class Configuration
+    module PgBackRestPolicy
+      private
+
+      def validate_pgbackrest!
+        return unless component_enabled?(:pgbackrest)
+
+        pg_config = component_config(:pgbackrest)
+        retention_full = pg_config[:retention_full]
+        retention_archive = pg_config[:retention_archive]
+        if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
+          raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
+        end
+        raise Error, 'pgbackrest.repo_block requires repo_bundle' if pg_config[:repo_block] && !pg_config[:repo_bundle]
+      end
+    end
+  end
+end

lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb

@@ -57,6 +57,13 @@ shared: &shared
 
     pgbackrest:
       enabled: false
+      # For large, frequently changed relation files, bundle repository objects
+      # and back up changed blocks instead of copying each whole file.
+      # repo_bundle: true
+      # repo_block: true
+      # schedule_full: "0 2 * * 0"
+      # schedule_differential: "35 2 * * 1-6"
+      # schedule_incremental: "5 */4 * * *"
 
     monitoring:
       enabled: false

templates/pgbackrest.conf.erb

@@ -40,6 +40,15 @@ repo1-path=<%= pgbackrest_config[:repo_path] || '/var/lib/pgbackrest' %>
 repo1-retention-full=<%= pgbackrest_config[:retention_full] || 7 %>
 repo1-retention-archive=<%= pgbackrest_config[:retention_archive] || 14 %>
 
+<% if pgbackrest_config[:repo_bundle] %>
+# Bundle small files and metadata into fewer repository objects.
+repo1-bundle=y
+<% end %>
+<% if pgbackrest_config[:repo_block] %>
+# Back up changed blocks instead of entire relation files.
+repo1-block=y
+<% end %>
+
 # Compression
 compress-type=lz4
 compress-level=3
@@ -69,4 +78,3 @@ pg1-path=/var/lib/postgresql/<%= config.version %>/main
 pg1-port=5432
 pg1-socket-path=/var/run/postgresql
 pg1-user=<%= config.postgres_user %>
-

test/components/pgbackrest_templates_test.rb

@@ -77,6 +77,20 @@ def test_conf_default_retention
     assert_includes content, 'repo1-retention-archive=14'
   end
 
+  def test_conf_enables_bundled_block_incremental_backups
+    content = render_conf(repo_type: 'local', repo_bundle: true, repo_block: true)
+
+    assert_includes content, 'repo1-bundle=y'
+    assert_includes content, 'repo1-block=y'
+  end
+
+  def test_conf_leaves_bundled_block_incremental_backups_disabled_by_default
+    content = render_conf(repo_type: 'local')
+
+    refute_includes content, 'repo1-bundle='
+    refute_includes content, 'repo1-block='
+  end
+
   def test_conf_includes_compression
     content = render_conf(repo_type: 'local')
 

test/components/pgbackrest_test.rb

@@ -35,13 +35,16 @@ def test_backup_schedules_prefers_schedule_full
     schedules = component.send(:backup_schedules, {
                                  schedule: '0 2 * * *',
                                  schedule_full: '0 3 * * *',
+                                 schedule_differential: '30 3 * * 1-6',
                                  schedule_incremental: '0 * * * *'
                                })
 
     full = schedules.find { |entry| entry[:type] == 'full' }
+    differential = schedules.find { |entry| entry[:type] == 'diff' }
     incremental = schedules.find { |entry| entry[:type] == 'incr' }
 
     assert_equal '0 3 * * *', full[:schedule]
+    assert_equal '30 3 * * 1-6', differential[:schedule]
     assert_equal '0 * * * *', incremental[:schedule]
   end
 
@@ -61,12 +64,14 @@ def test_setup_backup_schedules_rewrites_existing_cron_files_before_installing_n
 
     component.send(:setup_backup_schedules, 'primary.example.com', {
                      schedule_full: '0 2 * * *',
+                     schedule_differential: '30 2 * * 1-6',
                      schedule_incremental: '0 * * * *'
                    })
 
     assert_equal [:remove, 'primary.example.com'], calls.first
     assert_equal [:schedule, 'primary.example.com', '0 2 * * *', 'full', '/etc/cron.d/pgbackrest-backup'], calls[1]
-    assert_equal [:schedule, 'primary.example.com', '0 * * * *', 'incr', '/etc/cron.d/pgbackrest-backup-incremental'], calls[2]
+    assert_equal [:schedule, 'primary.example.com', '30 2 * * 1-6', 'diff', '/etc/cron.d/pgbackrest-backup-differential'], calls[2]
+    assert_equal [:schedule, 'primary.example.com', '0 * * * *', 'incr', '/etc/cron.d/pgbackrest-backup-incremental'], calls[3]
   end
 
   def test_setup_backup_schedule_uses_backup_type

test/configuration_test.rb

@@ -170,6 +170,34 @@ def test_pgbackrest_retention_validation_passes_when_archive_gte_full
     assert config.validate!
   end
 
+  def test_pgbackrest_block_backups_require_bundling
+    config = build_config(
+      'components' => {
+        'pgbackrest' => {
+          'enabled' => true,
+          'repo_block' => true
+        }
+      }
+    )
+
+    error = assert_raises(ActivePostgres::Error) { config.validate! }
+    assert_match(/repo_block requires repo_bundle/i, error.message)
+  end
+
+  def test_pgbackrest_block_backups_accept_bundling
+    config = build_config(
+      'components' => {
+        'pgbackrest' => {
+          'enabled' => true,
+          'repo_bundle' => true,
+          'repo_block' => true
+        }
+      }
+    )
+
+    assert config.validate!
+  end
+
   def test_standby_config_for_returns_standby_hash
     config = build_config(
       'standby' => [