diff --git a/src/playbooks/backup/backup.yaml b/src/playbooks/backup/backup.yaml new file mode 100644 index 000000000..a5bec08ee --- /dev/null +++ b/src/playbooks/backup/backup.yaml @@ -0,0 +1,16 @@ +--- +- name: Backup Databases and configuration + hosts: quadlet + become: true + gather_facts: true + vars_files: + - "../../vars/defaults.yml" + - "../../vars/flavors/{{ flavor }}.yml" + - "../../vars/{{ certificates_source }}_certificates.yml" + - "../../vars/foreman.yml" + - "../../vars/database.yml" + - "../../vars/database_iop.yml" + - "../../vars/base.yaml" + - "../../roles/pulp/defaults/main.yaml" + roles: + - backup diff --git a/src/playbooks/backup/metadata.obsah.yaml b/src/playbooks/backup/metadata.obsah.yaml new file mode 100644 index 000000000..a45d035fe --- /dev/null +++ b/src/playbooks/backup/metadata.obsah.yaml @@ -0,0 +1,20 @@ +--- +help: | + Create offline backup of Foreman databases and configuration + +variables: + backup_dir: + parameter: backup_dir + help: Directory where backup files will be stored + type: AbsolutePath + persist: false + + skip_pulp_content: + help: Skip Pulp content directory backup + action: store_true + persist: false + + wait_for_tasks: + help: Wait for running tasks to complete instead of failing immediately + action: store_true + persist: false \ No newline at end of file diff --git a/src/playbooks/checks/checks.yaml b/src/playbooks/checks/checks.yaml index 95863cc8e..3123d2b09 100644 --- a/src/playbooks/checks/checks.yaml +++ b/src/playbooks/checks/checks.yaml @@ -4,6 +4,8 @@ gather_facts: true vars_files: - "../../vars/defaults.yml" + - "../../vars/flavors/{{ flavor }}.yml" - "../../vars/database.yml" + - "../../vars/database_iop.yml" roles: - checks diff --git a/src/playbooks/restore/metadata.obsah.yaml b/src/playbooks/restore/metadata.obsah.yaml new file mode 100644 index 000000000..dc7b2bcaf --- /dev/null +++ b/src/playbooks/restore/metadata.obsah.yaml @@ -0,0 +1,18 @@ +--- +help: | + Restore Foreman from an offline backup + + Validates backup contents, extracts configuration files, restores databases, + restores Pulp content, and redeploys the system. + +variables: + backup_dir: + parameter: backup_dir + help: Directory containing the backup files + type: AbsolutePath + persist: false + + dry_run: + help: Validate backup without making any changes + action: store_true + persist: false diff --git a/src/playbooks/restore/restore.yaml b/src/playbooks/restore/restore.yaml new file mode 100644 index 000000000..653cc1a61 --- /dev/null +++ b/src/playbooks/restore/restore.yaml @@ -0,0 +1,16 @@ +--- +- name: Restore from offline backup + hosts: quadlet + become: true + gather_facts: true + vars_files: + - "../../vars/defaults.yml" + - "../../vars/flavors/{{ flavor }}.yml" + - "../../vars/{{ certificates_source }}_certificates.yml" + - "../../vars/foreman.yml" + - "../../vars/database.yml" + - "../../vars/database_iop.yml" + - "../../vars/base.yaml" + - "../../roles/pulp/defaults/main.yaml" + roles: + - restore diff --git a/src/roles/backup/defaults/main.yaml b/src/roles/backup/defaults/main.yaml new file mode 100644 index 000000000..602514749 --- /dev/null +++ b/src/roles/backup/defaults/main.yaml @@ -0,0 +1,11 @@ +--- +# Timeout and retry settings +backup_task_wait_retries: 60 +backup_task_wait_delay: 10 +backup_postgresql_ready_retries: 10 +backup_postgresql_ready_delay: 2 +backup_postgresql_stop_retries: 30 +backup_postgresql_stop_delay: 1 +# State tracking variables +backup_service_stopped: false +backup_postgresql_started: false diff --git a/src/roles/backup/tasks/database_dumps.yaml b/src/roles/backup/tasks/database_dumps.yaml new file mode 100644 index 000000000..c74efd30f --- /dev/null +++ b/src/roles/backup/tasks/database_dumps.yaml @@ -0,0 +1,34 @@ +--- +- name: Dump databases + ansible.builtin.command: + cmd: > + pg_dump + -h {{ item.host }} + -p {{ item.port }} + -U {{ item.user }} + -Fc + -f {{ backup_dir_full }}/{{ item.name }}.dump + {{ item.database }} + environment: + PGPASSWORD: "{{ item.password }}" + loop: "{{ backup_databases_config }}" + changed_when: true + no_log: true + +- name: Calculate total backup size + ansible.builtin.find: + paths: "{{ backup_dir_full }}" + patterns: "*.dump" + register: backup_files + +- name: Calculate total size + ansible.builtin.set_fact: + backup_total_size_bytes: "{{ backup_files.files | map(attribute='size') | sum }}" + +- name: Display backup summary + ansible.builtin.debug: + msg: | + Database dumps completed: + - Total files: {{ backup_files.matched }} + - Total size: {{ backup_total_size_bytes | int | human_readable }} + - Location: {{ backup_dir_full }} diff --git a/src/roles/backup/tasks/main.yaml b/src/roles/backup/tasks/main.yaml new file mode 100644 index 000000000..ba9a6b251 --- /dev/null +++ b/src/roles/backup/tasks/main.yaml @@ -0,0 +1,226 @@ +--- +- name: Set backup timestamp + ansible.builtin.set_fact: + backup_timestamp: "{{ ansible_date_time.iso8601_basic_short }}" + +- name: Set full backup directory path + ansible.builtin.set_fact: + backup_dir_full: "{{ backup_dir }}/foreman-backup-{{ backup_timestamp }}" + +- name: Ensure backup directory exists + ansible.builtin.file: + path: "{{ backup_dir }}" + state: directory + mode: '0755' + +- name: Test write permissions + ansible.builtin.file: + path: "{{ backup_dir }}/.write_test" + state: touch + mode: '0644' + register: backup_write_test + changed_when: false + +- name: Remove write test file + ansible.builtin.file: + path: "{{ backup_dir }}/.write_test" + state: absent + when: backup_write_test is succeeded + changed_when: false + +- name: Perform backup operations + block: + - name: Create timestamped backup directory + ansible.builtin.file: + path: "{{ backup_dir_full }}" + state: directory + mode: '0755' + + - name: Run preflight checks + ansible.builtin.include_tasks: + file: preflight.yaml + + - name: Stop Foreman services + ansible.builtin.systemd: + name: foreman.target + state: stopped + + - name: Mark services as stopped + ansible.builtin.set_fact: + backup_service_stopped: true + + - name: Wait for PostgreSQL to fully stop + ansible.builtin.systemd: + name: postgresql.service + register: backup_postgres_status + until: backup_postgres_status.status.ActiveState == 'inactive' + retries: "{{ backup_postgresql_stop_retries }}" + delay: "{{ backup_postgresql_stop_delay }}" + when: database_mode == 'internal' + changed_when: false + + - name: Start PostgreSQL for dumps + ansible.builtin.systemd: + name: postgresql.service + state: started + when: database_mode == 'internal' + + - name: Mark PostgreSQL as started + ansible.builtin.set_fact: + backup_postgresql_started: true + when: database_mode == 'internal' + + - name: Wait for PostgreSQL readiness + ansible.builtin.command: + cmd: pg_isready -h {{ database_host }} -p {{ database_port }} + register: backup_pg_ready + retries: "{{ backup_postgresql_ready_retries }}" + delay: "{{ backup_postgresql_ready_delay }}" + until: backup_pg_ready.rc == 0 + changed_when: false + + - name: Initialize databases_config with Foreman database + ansible.builtin.set_fact: + backup_databases_config: + - name: foreman + database: "{{ foreman_database_name }}" + host: "{{ foreman_database_host }}" + port: "{{ foreman_database_port }}" + user: "{{ foreman_database_user }}" + password: "{{ foreman_database_password }}" + no_log: true + + - name: Add Katello databases (Candlepin and Pulp) + ansible.builtin.set_fact: + backup_databases_config: "{{ backup_databases_config + katello_databases }}" + vars: + katello_databases: + - name: candlepin + database: "{{ candlepin_database_name }}" + host: "{{ candlepin_database_host }}" + port: "{{ candlepin_database_port }}" + user: "{{ candlepin_database_user }}" + password: "{{ candlepin_database_password }}" + - name: pulp + database: "{{ pulp_database_name }}" + host: "{{ pulp_database_host }}" + port: "{{ pulp_database_port }}" + user: "{{ pulp_database_user }}" + password: "{{ pulp_database_password }}" + when: "'katello' in enabled_features" + no_log: true + + - name: Add IOP databases + ansible.builtin.set_fact: + backup_databases_config: "{{ backup_databases_config + iop_databases }}" + vars: + iop_databases: + - name: iop_advisor + database: "{{ iop_advisor_database_name }}" + host: "{{ database_host }}" + port: "{{ database_port }}" + user: "{{ iop_advisor_database_user }}" + password: "{{ iop_advisor_database_password }}" + - name: iop_inventory + database: "{{ iop_inventory_database_name }}" + host: "{{ database_host }}" + port: "{{ database_port }}" + user: "{{ iop_inventory_database_user }}" + password: "{{ iop_inventory_database_password }}" + - name: iop_remediations + database: "{{ iop_remediation_database_name }}" + host: "{{ database_host }}" + port: "{{ database_port }}" + user: "{{ iop_remediation_database_user }}" + password: "{{ iop_remediation_database_password }}" + - name: iop_vmaas + database: "{{ iop_vmaas_database_name }}" + host: "{{ database_host }}" + port: "{{ database_port }}" + user: "{{ iop_vmaas_database_user }}" + password: "{{ iop_vmaas_database_password }}" + - name: iop_vulnerability + database: "{{ iop_vulnerability_database_name }}" + host: "{{ database_host }}" + port: "{{ database_port }}" + user: "{{ iop_vulnerability_database_user }}" + password: "{{ iop_vulnerability_database_password }}" + when: "'iop' in enabled_features" + no_log: true + + - name: Build database names list for display + ansible.builtin.set_fact: + backup_databases_to_backup: "{{ backup_databases_config | map(attribute='database') | list }}" + + - name: Dump databases + ansible.builtin.include_tasks: + file: database_dumps.yaml + + - name: Backup foremanctl state directory + community.general.archive: + path: "{{ obsah_state_path }}" + dest: "{{ backup_dir_full }}/foremanctl-state.tar.gz" + format: gz + mode: '0644' + + - name: Backup pulp content + ansible.builtin.include_tasks: + file: pulp_content.yaml + when: not skip_pulp_content | default(false) + + - name: Generate backup metadata + ansible.builtin.include_tasks: + file: metadata.yaml + + - name: Stop PostgreSQL + ansible.builtin.systemd: + name: postgresql.service + state: stopped + when: + - database_mode == 'internal' + - backup_postgresql_started | default(false) + + - name: Mark PostgreSQL as stopped + ansible.builtin.set_fact: + backup_postgresql_started: false + when: database_mode == 'internal' + + - name: Start Foreman services + ansible.builtin.systemd: + name: foreman.target + state: started + + - name: Mark services as started + ansible.builtin.set_fact: + backup_service_stopped: false + + - name: Display backup completion + ansible.builtin.debug: + msg: | + Backup completed successfully. + Location: {{ backup_dir_full }} + Databases: {{ backup_databases_to_backup | join(', ') }} + + rescue: + - name: Restore PostgreSQL on failure + ansible.builtin.systemd: + name: postgresql.service + state: stopped + when: + - database_mode == 'internal' + - backup_postgresql_started | default(false) + failed_when: false + + - name: Restore Foreman services on failure + ansible.builtin.systemd: + name: foreman.target + state: started + when: backup_service_stopped | default(false) + failed_when: false + + - name: Report failure + ansible.builtin.fail: + msg: | + Backup failed: {{ ansible_failed_result.msg | default('Unknown error') }} + Services have been restarted. + Partial backup may exist at: {{ backup_dir_full }} diff --git a/src/roles/backup/tasks/metadata.yaml b/src/roles/backup/tasks/metadata.yaml new file mode 100644 index 000000000..db0e6baf0 --- /dev/null +++ b/src/roles/backup/tasks/metadata.yaml @@ -0,0 +1,64 @@ +--- +# Generate backup metadata before stopping services +# This metadata helps with restore operations and compatibility checking + +- name: Gather package facts + ansible.builtin.package_facts: + +- name: Query container images + containers.podman.podman_image_info: + register: backup_container_images_result + failed_when: false + +- name: Build detailed container image list + ansible.builtin.set_fact: + backup_container_images_detailed: >- + {{ + backup_container_images_detailed | default([]) + + [{ + 'name': (item.RepoTags | first) if item.RepoTags | default([]) | length > 0 else '', + 'digest': (item.RepoDigests | first) if item.RepoDigests | default([]) | length > 0 else '', + 'id': item.Id, + 'created': item.Created + }] + }} + loop: "{{ backup_container_images_result.images | default([]) }}" + when: backup_container_images_result is succeeded + no_log: true + +- name: Check if pulp content was backed up + ansible.builtin.stat: + path: "{{ backup_dir_full }}/pulp-content.tar.gz" + register: backup_pulp_content_backup_check + failed_when: false + +- name: Write metadata file + ansible.builtin.copy: + content: "{{ backup_metadata | to_nice_yaml }}" + dest: "{{ backup_dir_full }}/metadata.yml" + mode: '0644' + vars: + backup_metadata: + hostname: "{{ ansible_fqdn }}" + os_version: "{{ ansible_distribution }} {{ ansible_distribution_version }}" + foremanctl_version: "{{ ansible_facts.packages['foremanctl'][0].version | default('unknown') if 'foremanctl' in ansible_facts.packages else 'unknown' }}" + online: false + incremental: false + timestamp: "{{ backup_timestamp }}" + databases: "{{ backup_databases_to_backup }}" + iop_enabled: "{{ 'iop' in enabled_features }}" + enabled_features: "{{ enabled_features | default([]) }}" + database_mode: "{{ database_mode }}" + container_images: "{{ backup_container_images_detailed | default([]) }}" + backed_up_components: >- + {{ + [ + 'databases', + 'container_images', + 'foremanctl_state' + ] + (['pulp_content'] if backup_pulp_content_backup_check.stat.exists | default(false) else []) + }} + +- name: Display metadata location + ansible.builtin.debug: + msg: "Backup metadata written to {{ backup_dir_full }}/metadata.yml" diff --git a/src/roles/backup/tasks/preflight.yaml b/src/roles/backup/tasks/preflight.yaml new file mode 100644 index 000000000..5c381c6b8 --- /dev/null +++ b/src/roles/backup/tasks/preflight.yaml @@ -0,0 +1,117 @@ +--- +# Preflight checks for backup operation +# - Check for running Foreman tasks +# - Check for running Pulp tasks +# - Run amcheck on databases (if available and local) + +- name: Check for running Foreman tasks + theforeman.foreman.resource_info: + server_url: "https://{{ ansible_fqdn }}" + oauth1_consumer_key: "{{ backup_foreman_oauth_consumer_key }}" + oauth1_consumer_secret: "{{ backup_foreman_oauth_consumer_secret }}" + ca_path: "{{ backup_foreman_ca_certificate }}" + resource: foreman_tasks + search: "state=running" + register: backup_foreman_tasks_check + failed_when: false + changed_when: false + no_log: true + +- name: Set Foreman running tasks count + ansible.builtin.set_fact: + backup_foreman_running_tasks: "{{ backup_foreman_tasks_check.resources | default([]) | length }}" + +- name: Wait for Foreman tasks to complete (if --wait-for-tasks) + theforeman.foreman.wait_for_task: + server_url: "https://{{ ansible_fqdn }}" + oauth1_consumer_key: "{{ backup_foreman_oauth_consumer_key }}" + oauth1_consumer_secret: "{{ backup_foreman_oauth_consumer_secret }}" + ca_path: "{{ backup_foreman_ca_certificate }}" + task: "{{ item }}" + timeout: "{{ task_wait_timeout | default(3600) }}" + loop: "{{ backup_foreman_tasks_check.resources | map(attribute='id') | list }}" + when: + - wait_for_tasks | default(false) + - backup_foreman_running_tasks | int > 0 + changed_when: false + no_log: true + +- name: Fail if Foreman tasks are running (without --wait-for-tasks) + ansible.builtin.fail: + msg: | + There are {{ backup_foreman_running_tasks }} running Foreman task(s). + Please wait for these to complete or use --wait-for-tasks flag. + when: + - not (wait_for_tasks | default(false)) + - backup_foreman_running_tasks | int > 0 + +- name: Check for running Pulp tasks + community.postgresql.postgresql_query: + db: "{{ pulp_database_name }}" + login_host: "{{ pulp_database_host }}" + login_port: "{{ pulp_database_port }}" + login_user: "{{ pulp_database_user }}" + login_password: "{{ pulp_database_password }}" + query: "SELECT COUNT(*) as count FROM core_task WHERE state IN ('running', 'waiting')" + register: backup_pulp_tasks_check + failed_when: false + changed_when: false + no_log: true + +- name: Set Pulp running tasks count + ansible.builtin.set_fact: + backup_pulp_running_tasks: "{{ backup_pulp_tasks_check.query_result[0].count | default(0) | int }}" + when: backup_pulp_tasks_check is succeeded + +- name: Wait for Pulp tasks to complete (if --wait-for-tasks) + community.postgresql.postgresql_query: + db: "{{ pulp_database_name }}" + login_host: "{{ pulp_database_host }}" + login_port: "{{ pulp_database_port }}" + login_user: "{{ pulp_database_user }}" + login_password: "{{ pulp_database_password }}" + query: "SELECT COUNT(*) as count FROM core_task WHERE state IN ('running', 'waiting')" + register: backup_pulp_tasks_wait + until: backup_pulp_tasks_wait.query_result[0].count | default(0) == 0 + retries: "{{ backup_task_wait_retries }}" + delay: "{{ backup_task_wait_delay }}" + when: + - wait_for_tasks | default(false) + - backup_pulp_running_tasks | default(0) | int > 0 + changed_when: false + no_log: true + +- name: Fail if Pulp tasks are running (without --wait-for-tasks) + ansible.builtin.fail: + msg: | + There are {{ backup_pulp_running_tasks }} running Pulp task(s). + Please wait for these to complete or use --wait-for-tasks flag. + when: + - not wait_for_tasks | default(false) + - backup_pulp_running_tasks | default(0) | int > 0 + +- name: Run database index integrity checks + ansible.builtin.include_role: + name: check_db_index + vars: + check_db_index_database: "{{ item }}" + loop: + - "{{ foreman_database_name }}" + - "{{ candlepin_database_name }}" + - "{{ pulp_database_name }}" + when: database_mode == 'internal' + +- name: Run IOP database index integrity checks + ansible.builtin.include_role: + name: check_db_index + vars: + check_db_index_database: "{{ item }}" + loop: + - "{{ iop_advisor_database_name }}" + - "{{ iop_inventory_database_name }}" + - "{{ iop_remediation_database_name }}" + - "{{ iop_vmaas_database_name }}" + - "{{ iop_vulnerability_database_name }}" + when: + - database_mode == 'internal' + - "'iop' in enabled_features" diff --git a/src/roles/backup/tasks/pulp_content.yaml b/src/roles/backup/tasks/pulp_content.yaml new file mode 100644 index 000000000..089d90a05 --- /dev/null +++ b/src/roles/backup/tasks/pulp_content.yaml @@ -0,0 +1,30 @@ +--- +- name: Backup pulp content + when: not skip_pulp_content | default(false) + block: + - name: Backup pulp content directory with encryption keys # noqa: command-instead-of-module + ansible.builtin.command: + cmd: > + tar -czf {{ backup_dir_full }}/pulp-content.tar.gz + -C {{ pulp_storage_path }} + --exclude=media/exports + --exclude=media/imports + --exclude=media/sync_imports + media + database_fields.symmetric.key + django_secret_key + register: backup_pulp_content_archive + changed_when: true + + - name: Get pulp content archive info + ansible.builtin.stat: + path: "{{ backup_dir_full }}/pulp-content.tar.gz" + register: backup_pulp_content_archive_stat + when: backup_pulp_content_archive is succeeded + + - name: Display pulp content backup completion + ansible.builtin.debug: + msg: >- + Pulp content backup completed: {{ backup_dir_full }}/pulp-content.tar.gz + ({{ (backup_pulp_content_archive_stat.stat.size / 1024 / 1024) | round(2) }} MB) + when: backup_pulp_content_archive_stat.stat.exists diff --git a/src/roles/check_db_index/tasks/main.yml b/src/roles/check_db_index/tasks/main.yml new file mode 100644 index 000000000..5e99c23bf --- /dev/null +++ b/src/roles/check_db_index/tasks/main.yml @@ -0,0 +1,53 @@ +--- +# Check database indexes using PostgreSQL amcheck extension +# This check verifies the logical consistency of B-tree indexes to detect corruption +# Required parameters: +# check_db_index_database: database name to check + +- name: Check if amcheck extension is installed + community.postgresql.postgresql_query: + db: "{{ check_db_index_database }}" + login_host: "{{ database_host }}" + login_port: "{{ database_port }}" + login_user: postgres + login_password: "{{ postgresql_admin_password }}" + query: SELECT COUNT(*) as count FROM pg_extension WHERE extname = 'amcheck' + register: check_db_index_amcheck_installed + failed_when: false + changed_when: false + +- name: "Run amcheck on database: {{ check_db_index_database }}" + when: + - check_db_index_amcheck_installed is succeeded + - check_db_index_amcheck_installed.query_result[0].count | default(0) > 0 + block: + - name: Execute amcheck integrity check + community.postgresql.postgresql_query: + db: "{{ check_db_index_database }}" + login_host: "{{ database_host }}" + login_port: "{{ database_port }}" + login_user: postgres + login_password: "{{ postgresql_admin_password }}" + query: | + SELECT bt_index_check(index => c.oid, heapallindexed => i.indisunique), + c.relname, + c.relpages + FROM pg_index i + JOIN pg_opclass op ON i.indclass[0] = op.oid + JOIN pg_am am ON op.opcmethod = am.oid + JOIN pg_class c ON i.indexrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE am.amname = 'btree' AND n.nspname = 'public' + AND c.relpersistence != 't' + AND c.relkind = 'i' AND i.indisready AND i.indisvalid + ORDER BY c.relpages DESC; + changed_when: false + + - name: "Report database index check PASSED: {{ check_db_index_database }}" + ansible.builtin.debug: + msg: "{{ check_db_index_database }} database index check: PASSED" + + rescue: + - name: "Report database index check FAILED: {{ check_db_index_database }}" + ansible.builtin.debug: + msg: "{{ check_db_index_database }} database index check: FAILED - indexes may be corrupted" diff --git a/src/roles/checks/tasks/main.yml b/src/roles/checks/tasks/main.yml index 90dbf9e1d..f90acc376 100644 --- a/src/roles/checks/tasks/main.yml +++ b/src/roles/checks/tasks/main.yml @@ -7,6 +7,32 @@ - check_database_connection - check_system_requirements +- name: Run database index integrity checks + ansible.builtin.include_role: + name: check_db_index + vars: + check_db_index_database: "{{ item }}" + loop: + - "{{ foreman_database_name }}" + - "{{ candlepin_database_name }}" + - "{{ pulp_database_name }}" + when: database_mode == 'internal' + +- name: Run IOP database index integrity checks + ansible.builtin.include_role: + name: check_db_index + vars: + check_db_index_database: "{{ item }}" + loop: + - "{{ iop_advisor_database_name }}" + - "{{ iop_inventory_database_name }}" + - "{{ iop_remediation_database_name }}" + - "{{ iop_vmaas_database_name }}" + - "{{ iop_vulnerability_database_name }}" + when: + - database_mode == 'internal' + - "'iop' in enabled_features" + - name: Report status of checks ansible.builtin.fail: msg: "{{ checks_results }}" diff --git a/src/roles/pulp/defaults/main.yaml b/src/roles/pulp/defaults/main.yaml index cbffafa59..cd955313a 100644 --- a/src/roles/pulp/defaults/main.yaml +++ b/src/roles/pulp/defaults/main.yaml @@ -9,11 +9,13 @@ pulp_worker_count: "{{ [8, ansible_facts['processor_nproc']] | min }}" pulp_content_service_worker_count: "{{ (2 * ([8, ansible_facts['processor_nproc']] | min)) + 1 }}" pulp_api_service_worker_count: "{{ ([4, ansible_facts['processor_nproc']] | min) + 1 }}" +pulp_storage_path: /var/lib/pulp + pulp_volumes: >- {{ - ['/var/lib/pulp:/var/lib/pulp:rw'] + - (pulp_import_paths | map('regex_replace', '^(.+)$', '\1:\1:rw') | list) + - (pulp_export_paths | map('regex_replace', '^(.+)$', '\1:\1:rw') | list) + [pulp_storage_path ~ ':' ~ pulp_storage_path] + + (pulp_import_paths | map('regex_replace', '^(.+)$', '\1:\1') | list) + + (pulp_export_paths | map('regex_replace', '^(.+)$', '\1:\1') | list) }} pulp_api_container_name: pulp-api diff --git a/src/roles/restore/defaults/main.yaml b/src/roles/restore/defaults/main.yaml new file mode 100644 index 000000000..797ae2db9 --- /dev/null +++ b/src/roles/restore/defaults/main.yaml @@ -0,0 +1,11 @@ +--- +# Timeout and retry settings for restore operations +restore_postgresql_ready_retries: 10 +restore_postgresql_ready_delay: 2 +restore_postgresql_stop_retries: 30 +restore_postgresql_stop_delay: 1 + +# State tracking variables +restore_service_stopped: false +restore_postgresql_started: false +restore_destructive_action_taken: false diff --git a/src/roles/restore/tasks/deploy_and_verify.yaml b/src/roles/restore/tasks/deploy_and_verify.yaml new file mode 100644 index 000000000..0c78481bb --- /dev/null +++ b/src/roles/restore/tasks/deploy_and_verify.yaml @@ -0,0 +1,101 @@ +--- +# Deploy and verify +# Run foremanctl deploy to regenerate podman secrets from restored credentials + +- name: Stop PostgreSQL + ansible.builtin.systemd: + name: postgresql.service + state: stopped + when: + - database_mode == 'internal' + - restore_postgresql_started | default(false) + +- name: Mark PostgreSQL as stopped + ansible.builtin.set_fact: + restore_postgresql_started: false + when: database_mode == 'internal' + +- name: Display deployment status + ansible.builtin.debug: + msg: | + Running foremanctl deploy to regenerate configuration... + All data has been restored: + - Databases restored + - Pulp content and encryption keys restored + - OAuth keys and passwords restored + - Foremanctl state restored + + Deploy will regenerate podman secrets from restored credentials. + +- name: Run foremanctl deploy + ansible.builtin.command: + cmd: foremanctl deploy + register: restore_deploy_result + changed_when: true + +- name: Display deploy completion + ansible.builtin.debug: + msg: "Deploy completed - podman secrets regenerated from restored state" + +- name: Mark services as started + ansible.builtin.set_fact: + restore_service_stopped: false + +- name: Wait for services to stabilize + ansible.builtin.pause: + seconds: 30 + prompt: "Waiting for deployed services to stabilize..." + +- name: Wait for Foreman API to respond + ansible.builtin.uri: + url: "https://{{ ansible_fqdn }}/api/status" + method: GET + validate_certs: false + status_code: [200, 401] + register: restore_api_check + until: restore_api_check.status in [200, 401] + retries: 30 + delay: 10 + changed_when: false + +- name: Set API status message + ansible.builtin.set_fact: + restore_api_status: "{{ 'responding (authenticated)' if restore_api_check.status == 200 else 'responding (requires auth)' }}" + +- name: Check service health + ansible.builtin.systemd: + name: "{{ item }}" + register: restore_service_check + failed_when: restore_service_check.status.ActiveState != 'active' + loop: + - foreman.target + - foreman.service + - postgresql.service + changed_when: false + +- name: Display restore success + ansible.builtin.debug: + msg: | + ═══════════════════════════════════════════════════════════════ + Restore complete + ═══════════════════════════════════════════════════════════════ + + Restored from: {{ backup_dir }} + System: {{ ansible_fqdn }} + + - Backup validated + - System prepared + - Databases restored ({{ restore_databases_to_restore | length }} databases) + - Pulp content and encryption keys restored + - OAuth keys and passwords restored + - System deployed and verified + + All services are running: + - foreman.target - active + - foreman.service - active + - postgresql.service - active + + Foreman API: https://{{ ansible_fqdn }}/api/status - {{ restore_api_status }} + + Your Foreman instance has been successfully restored. + ═══════════════════════════════════════════════════════════════ diff --git a/src/roles/restore/tasks/main.yaml b/src/roles/restore/tasks/main.yaml new file mode 100644 index 000000000..d87b36357 --- /dev/null +++ b/src/roles/restore/tasks/main.yaml @@ -0,0 +1,80 @@ +--- +# Main restore orchestration + +- name: Run validation checks + ansible.builtin.include_tasks: + file: validate.yaml + +- name: Perform restore operations + block: + - name: Prepare system for restore + ansible.builtin.include_tasks: + file: prepare_system.yaml + + - name: System prepared + ansible.builtin.debug: + msg: | + ═══════════════════════════════════════════════════════════════ + System prepared for restore + ═══════════════════════════════════════════════════════════════ + + - name: Restore databases + ansible.builtin.include_tasks: + file: restore_databases.yaml + + - name: Databases restored + ansible.builtin.debug: + msg: | + ═══════════════════════════════════════════════════════════════ + Databases restored + ═══════════════════════════════════════════════════════════════ + + - name: Restore Pulp content + ansible.builtin.include_tasks: + file: restore_pulp_content.yaml + + - name: Pulp content restored + ansible.builtin.debug: + msg: | + ═══════════════════════════════════════════════════════════════ + Pulp content and encryption keys restored + ═══════════════════════════════════════════════════════════════ + + - name: Restore foremanctl state + ansible.builtin.include_tasks: + file: restore_foremanctl_state.yaml + + - name: Foremanctl state restored + ansible.builtin.debug: + msg: | + ═══════════════════════════════════════════════════════════════ + OAuth keys and passwords restored + ═══════════════════════════════════════════════════════════════ + + - name: Start services and verify + ansible.builtin.include_tasks: + file: deploy_and_verify.yaml + + rescue: + - name: Stop PostgreSQL on failure + ansible.builtin.systemd: + name: postgresql.service + state: stopped + when: + - database_mode == 'internal' + - restore_postgresql_started | default(false) + failed_when: false + + - name: Restart Foreman services on failure + ansible.builtin.systemd: + name: foreman.target + state: started + when: restore_service_stopped | default(false) + failed_when: false + + - name: Report failure + ansible.builtin.fail: + msg: | + Restore failed: {{ ansible_failed_result.msg | default('Unknown error') }} + Services have been restarted. + System is back to running state. diff --git a/src/roles/restore/tasks/prepare_system.yaml b/src/roles/restore/tasks/prepare_system.yaml new file mode 100644 index 000000000..33270e8b7 --- /dev/null +++ b/src/roles/restore/tasks/prepare_system.yaml @@ -0,0 +1,49 @@ +--- +# Phase 2: Prepare system for database restore +# Stop services and ensure PostgreSQL is ready + +- name: Stop Foreman services + ansible.builtin.systemd: + name: foreman.target + state: stopped + +- name: Mark services as stopped + ansible.builtin.set_fact: + restore_service_stopped: true + +- name: Wait for PostgreSQL to fully stop + ansible.builtin.systemd: + name: postgresql.service + register: restore_postgres_status + until: restore_postgres_status.status.ActiveState == 'inactive' + retries: "{{ restore_postgresql_stop_retries }}" + delay: "{{ restore_postgresql_stop_delay }}" + when: database_mode == 'internal' + changed_when: false + +- name: Start PostgreSQL for restore + ansible.builtin.systemd: + name: postgresql.service + state: started + when: database_mode == 'internal' + +- name: Mark PostgreSQL as started + ansible.builtin.set_fact: + restore_postgresql_started: true + when: database_mode == 'internal' + +- name: Wait for PostgreSQL readiness + ansible.builtin.command: + cmd: pg_isready -h {{ database_host }} -p {{ database_port }} + register: restore_pg_ready + retries: "{{ restore_postgresql_ready_retries }}" + delay: "{{ restore_postgresql_ready_delay }}" + until: restore_pg_ready.rc == 0 + changed_when: false + +- name: Display preparation complete + ansible.builtin.debug: + msg: | + Foreman services stopped + PostgreSQL started and ready + System prepared for database restore diff --git a/src/roles/restore/tasks/restore_databases.yaml b/src/roles/restore/tasks/restore_databases.yaml new file mode 100644 index 000000000..8c4909dae --- /dev/null +++ b/src/roles/restore/tasks/restore_databases.yaml @@ -0,0 +1,126 @@ +--- +# Phase 3: Restore databases from backup +# SAFETY: Destructive operations have when: false for initial testing + +- name: Read backup metadata + ansible.builtin.slurp: + path: "{{ backup_dir }}/metadata.yml" + register: restore_metadata_content + +- name: Parse backup metadata + ansible.builtin.set_fact: + restore_backup_metadata: "{{ restore_metadata_content['content'] | b64decode | from_yaml }}" + +- name: Display backup database information + ansible.builtin.debug: + msg: "Databases in backup: {{ restore_backup_metadata.databases | join(', ') }}" + +- name: Build database restore configuration + ansible.builtin.set_fact: + restore_databases_config: + - dump_file: foreman.dump + database_name: "{{ foreman_database_name }}" + owner: "{{ foreman_database_user }}" + - dump_file: candlepin.dump + database_name: "{{ candlepin_database_name }}" + owner: "{{ candlepin_database_user }}" + when_db: candlepin + - dump_file: pulp.dump + database_name: "{{ pulp_database_name }}" + owner: "{{ pulp_database_user }}" + when_db: pulp + +- name: Filter databases that exist in backup + ansible.builtin.set_fact: + restore_databases_to_restore: >- + {{ + restore_databases_config | + selectattr('when_db', 'undefined') | + list + + (restore_databases_config | + selectattr('when_db', 'defined') | + selectattr('when_db', 'in', restore_backup_metadata.databases) | + list) + }} + +- name: Display databases that will be restored + ansible.builtin.debug: + msg: "Will restore {{ restore_databases_to_restore | length }} database(s): {{ restore_databases_to_restore | map(attribute='database_name') | join(', ') }}" + +- name: Verify dump files exist + ansible.builtin.stat: + path: "{{ backup_dir }}/{{ item.dump_file }}" + register: restore_dump_files_check + failed_when: not restore_dump_files_check.stat.exists + loop: "{{ restore_databases_to_restore }}" + loop_control: + label: "{{ item.dump_file }}" + +- name: Drop existing databases + community.postgresql.postgresql_db: + name: "{{ item.database_name }}" + state: absent + login_host: "{{ database_host }}" + login_port: "{{ database_port }}" + login_user: postgres + login_password: "{{ postgresql_admin_password }}" + loop: "{{ restore_databases_to_restore }}" + loop_control: + label: "{{ item.database_name }}" + +- name: Create empty databases + community.postgresql.postgresql_db: + name: "{{ item.database_name }}" + state: present + owner: "{{ item.owner }}" + login_host: "{{ database_host }}" + login_port: "{{ database_port }}" + login_user: postgres + login_password: "{{ postgresql_admin_password }}" + loop: "{{ restore_databases_to_restore }}" + loop_control: + label: "{{ item.database_name }}" + +- name: Restore databases from dump files + ansible.builtin.command: + cmd: > + pg_restore + -h {{ database_host }} + -p {{ database_port }} + -U {{ item.owner }} + -d {{ item.database_name }} + --no-owner + --no-acl + {{ backup_dir }}/{{ item.dump_file }} + environment: + PGPASSWORD: "{{ postgresql_admin_password }}" + loop: "{{ restore_databases_to_restore }}" + loop_control: + label: "{{ item.dump_file }} → {{ item.database_name }}" + changed_when: true + +- name: Fix database ownership + community.postgresql.postgresql_owner: + db: "{{ item.database_name }}" + new_owner: "{{ item.owner }}" + obj_name: "{{ item.database_name }}" + obj_type: database + login_host: "{{ database_host }}" + login_port: "{{ database_port }}" + login_user: postgres + login_password: "{{ postgresql_admin_password }}" + loop: "{{ restore_databases_to_restore }}" + loop_control: + label: "{{ item.database_name }}" + +- name: Display restore completion + ansible.builtin.debug: + msg: | + Database restore configuration built + Dump files verified + Existing databases dropped + Empty databases created + Data restored from dump files + Database ownership fixed + + Databases restored: {{ restore_databases_to_restore | map(attribute='database_name') | join(', ') }} diff --git a/src/roles/restore/tasks/restore_foremanctl_state.yaml b/src/roles/restore/tasks/restore_foremanctl_state.yaml new file mode 100644 index 000000000..9965e9795 --- /dev/null +++ b/src/roles/restore/tasks/restore_foremanctl_state.yaml @@ -0,0 +1,68 @@ +--- +# Restore foremanctl state (OAuth keys, passwords, parameters) +# CRITICAL: Must be restored before starting services + +- name: Check if foremanctl-state archive exists + ansible.builtin.stat: + path: "{{ backup_dir }}/foremanctl-state.tar.gz" + register: restore_state_check + +- name: Fail if foremanctl-state is missing + ansible.builtin.fail: + msg: "CRITICAL: foremanctl-state.tar.gz not found in backup! Cannot restore OAuth keys and passwords." + when: not restore_state_check.stat.exists + +- name: Check if current state directory exists + ansible.builtin.stat: + path: "{{ obsah_state_path }}" + register: restore_current_state + +- name: Backup current foremanctl state + ansible.builtin.command: + cmd: mv {{ obsah_state_path }} {{ obsah_state_path }}.backup-{{ ansible_date_time.epoch }} + when: restore_current_state.stat.exists + changed_when: true + failed_when: false + +- name: Ensure state directory parent exists + ansible.builtin.file: + path: "{{ obsah_state_path | dirname }}" + state: directory + mode: '0755' + +- name: Extract foremanctl state archive + ansible.builtin.unarchive: + src: "{{ backup_dir }}/foremanctl-state.tar.gz" + dest: "{{ obsah_state_path | dirname }}" + remote_src: true + +- name: Verify critical files were restored + ansible.builtin.stat: + path: "{{ obsah_state_path }}/{{ item }}" + register: restore_state_files + failed_when: not restore_state_files.stat.exists + loop: + - parameters.yaml + - foreman-oauth-consumer-key + - foreman-oauth-consumer-secret + - postgresql-admin-password + - foreman-db-password + - candlepin-db-password + - pulp-db-password + +- name: Display foremanctl state restore completion + ansible.builtin.debug: + msg: | + Foremanctl state restored to {{ obsah_state_path }} + OAuth keys restored + Database passwords restored + Parameters.yaml restored + + Critical files verified: + - foreman-oauth-consumer-key + - foreman-oauth-consumer-secret + - postgresql-admin-password + - foreman-db-password + - candlepin-db-password + - pulp-db-password + - parameters.yaml diff --git a/src/roles/restore/tasks/restore_pulp_content.yaml b/src/roles/restore/tasks/restore_pulp_content.yaml new file mode 100644 index 000000000..8cb28dc7f --- /dev/null +++ b/src/roles/restore/tasks/restore_pulp_content.yaml @@ -0,0 +1,77 @@ +--- +# Phase 4: Restore Pulp content and encryption keys +# CRITICAL: Includes database_fields.symmetric.key and django_secret_key + +- name: Check if pulp content archive exists + ansible.builtin.stat: + path: "{{ backup_dir }}/pulp-content.tar.gz" + register: restore_pulp_content_check + +- name: Display pulp content availability + ansible.builtin.debug: + msg: "{{ 'Pulp content found in backup, will restore' if restore_pulp_content_check.stat.exists else 'Pulp content not in backup, skipping (backup used --skip-pulp-content)' }}" + +- name: Restore pulp content and encryption keys + when: restore_pulp_content_check.stat.exists + block: + - name: Ensure pulp storage directory exists + ansible.builtin.file: + path: "{{ pulp_storage_path }}" + state: directory + mode: '0755' + + - name: Check if media directory exists + ansible.builtin.stat: + path: "{{ pulp_storage_path }}/media" + register: restore_pulp_media_dir + + - name: Backup existing pulp media directory + ansible.builtin.command: + cmd: mv {{ pulp_storage_path }}/media {{ pulp_storage_path }}/media.backup-{{ ansible_date_time.epoch }} + when: restore_pulp_media_dir.stat.exists + changed_when: true + failed_when: false + + - name: Extract pulp content archive + ansible.builtin.unarchive: + src: "{{ backup_dir }}/pulp-content.tar.gz" + dest: "{{ pulp_storage_path }}" + remote_src: true + + - name: Verify Pulp encryption key was restored + ansible.builtin.stat: + path: "{{ pulp_storage_path }}/database_fields.symmetric.key" + register: restore_pulp_encryption_key + failed_when: not restore_pulp_encryption_key.stat.exists + + - name: Verify Django secret key was restored + ansible.builtin.stat: + path: "{{ pulp_storage_path }}/django_secret_key" + register: restore_django_secret_key + failed_when: not restore_django_secret_key.stat.exists + + - name: Count restored media files + ansible.builtin.find: + paths: "{{ pulp_storage_path }}/media" + file_type: file + recurse: true + register: restore_pulp_media_files + + - name: Get archive size + ansible.builtin.stat: + path: "{{ backup_dir }}/pulp-content.tar.gz" + register: restore_pulp_content_size + + - name: Display pulp content restore completion + ansible.builtin.debug: + msg: | + Pulp content extracted to {{ pulp_storage_path }} + Pulp encryption key restored: {{ pulp_storage_path }}/database_fields.symmetric.key + Django secret key restored: {{ pulp_storage_path }}/django_secret_key + Media files restored: {{ restore_pulp_media_files.matched }} files + Archive size: {{ (restore_pulp_content_size.stat.size / 1024 / 1024) | round(2) }} MB + +- name: Display skip message + ansible.builtin.debug: + msg: "Pulp content restore skipped (not present in backup)" + when: not restore_pulp_content_check.stat.exists diff --git a/src/roles/restore/tasks/validate.yaml b/src/roles/restore/tasks/validate.yaml new file mode 100644 index 000000000..392d5bfb8 --- /dev/null +++ b/src/roles/restore/tasks/validate.yaml @@ -0,0 +1,45 @@ +--- +# Phase 1: Basic validation - check required files exist +# This runs BEFORE any destructive actions + +- name: Check if backup directory exists + ansible.builtin.stat: + path: "{{ backup_dir }}" + register: restore_backup_dir_stat + +- name: Fail if backup directory does not exist + ansible.builtin.fail: + msg: "Backup directory does not exist: {{ backup_dir }}" + when: not restore_backup_dir_stat.stat.exists + +- name: Check for metadata.yml + ansible.builtin.stat: + path: "{{ backup_dir }}/metadata.yml" + register: restore_metadata_stat + +- name: Fail if metadata.yml is missing + ansible.builtin.fail: + msg: "Backup metadata file not found: {{ backup_dir }}/metadata.yml" + when: not restore_metadata_stat.stat.exists + +- name: Verify required backup files exist + ansible.builtin.stat: + path: "{{ backup_dir }}/{{ item }}" + register: restore_required_files + failed_when: not restore_required_files.stat.exists + loop: + - foreman.dump + - candlepin.dump + - pulp.dump + +- name: Display validation success + ansible.builtin.debug: + msg: | + Backup validation passed + Backup directory exists: {{ backup_dir }} + Metadata file found + Required files present (foreman.dump, candlepin.dump, pulp.dump) + +- name: Stop here if dry-run mode + ansible.builtin.meta: end_play + when: dry_run | default(false) diff --git a/src/vars/base.yaml b/src/vars/base.yaml index b85b9d02f..774a86072 100644 --- a/src/vars/base.yaml +++ b/src/vars/base.yaml @@ -48,3 +48,7 @@ foreman_proxy_oauth_consumer_secret: "{{ foreman_oauth_consumer_secret }}" iop_core_foreman_url: "{{ foreman_url }}" iop_core_foreman_oauth_consumer_key: "{{ foreman_oauth_consumer_key }}" iop_core_foreman_oauth_consumer_secret: "{{ foreman_oauth_consumer_secret }}" + +backup_foreman_oauth_consumer_key: "{{ foreman_oauth_consumer_key }}" +backup_foreman_oauth_consumer_secret: "{{ foreman_oauth_consumer_secret }}" +backup_foreman_ca_certificate: "{{ foreman_ca_certificate }}"