Centralize database configuration into a single reusable list

development/ansible.cfg

@@ -3,4 +3,5 @@ host_key_checking = False
 stdout_callback=debug
 stderr_callback=debug
 roles_path = ./roles:../src/roles
+filter_plugins = ../src/filter_plugins
 display_skipped_hosts = no

development/playbooks/deploy-dev/deploy-dev.yaml

@@ -15,37 +15,22 @@
   pre_tasks:
     - name: Set development postgresql databases
       ansible.builtin.set_fact:
-        postgresql_databases:
-          - name: "{{ candlepin_database_name }}"
-            owner: "{{ candlepin_database_user }}"
-          - name: "{{ foreman_development_database_name }}"
-            owner: "{{ foreman_database_user }}"
-          - name: "{{ foreman_development_database_name }}_test"
-            owner: "{{ foreman_database_user }}"
-          - name: "{{ pulp_database_name }}"
-            owner: "{{ pulp_database_user }}"
-        postgresql_users:
-          - name: "{{ candlepin_database_user }}"
-            password: "{{ candlepin_database_password }}"
-          - name: "{{ foreman_database_user }}"
-            password: "{{ foreman_database_password }}"
-            role_attr_flags: SUPERUSER
-          - name: "{{ pulp_database_user }}"
-            password: "{{ pulp_database_password }}"
+        postgresql_databases: >-
+          {{ all_databases
+             | rejectattr('name', 'equalto', 'foreman')
+             | to_postgresql_databases
+             + [{'name': foreman_development_database_name, 'owner': foreman_database_user},
+                {'name': foreman_development_database_name + '_test', 'owner': foreman_database_user}] }}
+        postgresql_users: >-
+          {{ all_databases
+             | rejectattr('name', 'equalto', 'foreman')
+             | to_postgresql_users
+             + [{'name': foreman_database_user, 'password': foreman_database_password, 'role_attr_flags': 'SUPERUSER'}] }}
 
     - name: Setup iop requirements
       when:
         - "'iop' in enabled_features"
       block:
-        - name: Include iop databases
-          ansible.builtin.include_vars:
-            file: "../../../src/vars/database_iop.yml"
-
-        - name: Combine lists
-          ansible.builtin.set_fact:
-            postgresql_databases: "{{ postgresql_databases + iop_postgresql_databases }}"
-            postgresql_users: "{{ postgresql_users + iop_postgresql_users }}"
-
         - name: Enable foreman_rh_cloud plugin for iop
           ansible.builtin.set_fact:
             foreman_development_enabled_plugins: "{{ foreman_development_enabled_plugins + ['foreman_rh_cloud'] }}"

development/playbooks/remote-database/remote-database.yaml

@@ -4,6 +4,8 @@
     - database
   become: true
   vars_files:
+    - "../../../src/vars/defaults.yml"
+    - "../../../src/vars/flavors/{{ flavor }}.yml"
     - "../../../src/vars/database.yml"
   vars:
     certificates_hostnames:

src/filter_plugins/foremanctl.py

@@ -115,6 +115,14 @@ def has_feature(features, feature):
     return feature in features or any(f.startswith(feature + '/') for f in features)
 
 
+def to_postgresql_databases(databases):
+    return [{'name': db['database'], 'owner': db['user']} for db in databases]
+
+
+def to_postgresql_users(databases):
+    return [{'name': db['user'], 'password': db['password']} for db in databases]
+
+
 class FilterModule(object):
     '''foremanctl filters'''
 
@@ -128,4 +136,6 @@ def filters(self):
             'list_all_features': list_all_features,
             'invalid_features': invalid_features,
             'has_feature': has_feature,
+            'to_postgresql_databases': to_postgresql_databases,
+            'to_postgresql_users': to_postgresql_users,
         }

src/playbooks/checks/checks.yaml

@@ -4,6 +4,9 @@
   gather_facts: true
   vars_files:
     - "../../vars/defaults.yml"
+    - "../../vars/flavors/{{ flavor }}.yml"
     - "../../vars/database.yml"
   roles:
-    - checks
+    - role: checks
+      vars:
+        checks_databases: "{{ all_databases }}"

src/playbooks/deploy/deploy.yaml

@@ -12,23 +12,11 @@
     - "../../vars/database.yml"
     - "../../vars/foreman.yml"
     - "../../vars/base.yaml"
-  pre_tasks:
-    - name: Add iop databases
-      when:
-        - "'iop' in enabled_features"
-        - database_mode == 'internal'
-      block:
-        - name: Include iop databases
-          ansible.builtin.include_vars:
-            file: "../../vars/database_iop.yml"
-
-        - name: Combine lists
-          ansible.builtin.set_fact:
-            postgresql_databases: "{{ postgresql_databases + iop_postgresql_databases }}"
-            postgresql_users: "{{ postgresql_users + iop_postgresql_users }}"
   roles:
     - role: pre_install
     - role: checks
+      vars:
+        checks_databases: "{{ all_databases }}"
     - role: certificates
       when: "certificates_source in ['default', 'custom_server']"
     - role: certificate_checks

src/roles/check_database_connection/tasks/check.yaml

@@ -1,7 +1,7 @@
 - name: Store CA cert to a temporary file
   when:
-    - db_item.ca_cert is defined
-    - db_item.ca_cert is truthy
+    - db_item.ssl_ca is defined
+    - db_item.ssl_ca is truthy
   block:
     - name: Create temporary file
       ansible.builtin.tempfile:
@@ -12,35 +12,35 @@
     - name: Write CA cert to temporary file
       ansible.builtin.copy:
         dest: "{{ _check_database_connection_ca_cert.path }}"
-        src: "{{ db_item.ca_cert }}"
+        src: "{{ db_item.ssl_ca }}"
         mode: '0640'
 
 - name: Check database connectivity to {{ db_item.name }}
   community.postgresql.postgresql_ping:
     login_host: "{{ db_item.host }}"
     login_user: "{{ db_item.user }}"
     login_password: "{{ db_item.password }}"
-    login_db: "{{ db_item.dbname }}"
+    login_db: "{{ db_item.database }}"
     ca_cert: "{{ _check_database_connection_ca_cert.path | default(omit) }}"
-    ssl_mode: "{{ db_item.sslmode | default(omit) }}"
+    ssl_mode: "{{ db_item.ssl_mode | default(omit) }}"
   register: check_database_connection_ping_result
   ignore_errors: true
 
 - name: Delete temporary CA cert file
   when:
-    - db_item.ca_cert is defined
-    - db_item.ca_cert is truthy
+    - db_item.ssl_ca is defined
+    - db_item.ssl_ca is truthy
   block:
     - name: Delete temporary file
       ansible.builtin.file:
         state: absent
         path: "{{ _check_database_connection_ca_cert.path }}"
 
-- name: Assert database is reachable for  {{ db_item.name }}
+- name: Assert database is reachable for {{ db_item.name }}
   ansible.builtin.assert:
     that:
       - check_database_connection_ping_result.is_available
     fail_msg: >
-      Cannot connect to {{ db_item.name }} database '{{ db_item.dbname }}' at {{ db_item.host }}.
+      Cannot connect to {{ db_item.name }} database '{{ db_item.database }}' at {{ db_item.host }}.
       Please verify the database host, port, name, user, and password.
       Error: {{ check_database_connection_ping_result.conn_err_msg | default('No error message available.') }}

src/roles/check_database_connection/tasks/main.yaml

@@ -2,30 +2,8 @@
 - name: Check DB
   ansible.builtin.include_tasks: check.yaml
   no_log: true
-  loop:
-    - name: Foreman
-      host: "{{ foreman_database_host }}"
-      user: "{{ foreman_database_user }}"
-      password: "{{ foreman_database_password }}"
-      dbname: "{{ foreman_database_name }}"
-      ca_cert: "{{ foreman_database_ssl_ca | default('') }}"
-      sslmode: "{{ foreman_database_ssl_mode | default(omit) }}"
-
-    - name: Candlepin
-      host: "{{ candlepin_database_host }}"
-      user: "{{ candlepin_database_user }}"
-      password: "{{ candlepin_database_password }}"
-      dbname: "{{ candlepin_database_name }}"
-      ca_cert: "{{ candlepin_database_ssl_ca | default('') }}"
-      sslmode: "{{ candlepin_database_ssl_mode | default(omit) }}"
-
-    - name: Pulp
-      host: "{{ pulp_database_host }}"
-      user: "{{ pulp_database_user }}"
-      password: "{{ pulp_database_password }}"
-      dbname: "{{ pulp_database_name }}"
-      ca_cert: "{{ pulp_database_ssl_ca | default('') }}"
-      sslmode: "{{ pulp_database_ssl_mode | default(omit) }}"
+  loop: "{{ checks_databases }}"
   loop_control:
     loop_var: db_item
+    label: "{{ db_item.name }}"
   when: database_mode == 'external'

src/roles/checks/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+checks_databases: []

src/vars/database.yml

@@ -35,17 +35,112 @@ foreman_database_port: "{{ database_port }}"
 foreman_database_ssl_mode: "{{ database_ssl_mode }}"
 foreman_database_ssl_ca: "{{ database_ssl_ca }}"
 
-postgresql_databases:
-  - name: "{{ candlepin_database_name }}"
-    owner: "{{ candlepin_database_user }}"
-  - name: "{{ foreman_database_name }}"
-    owner: "{{ foreman_database_user }}"
-  - name: "{{ pulp_database_name }}"
-    owner: "{{ pulp_database_user }}"
-postgresql_users:
-  - name: "{{ candlepin_database_user }}"
-    password: "{{ candlepin_database_password }}"
-  - name: "{{ foreman_database_user }}"
+iop_database_host: host.containers.internal
+iop_database_port: 5432
+
+iop_inventory_database_host: "{{ iop_database_host }}"
+iop_inventory_database_port: "{{ iop_database_port }}"
+iop_inventory_database_name: inventory_db
+iop_inventory_database_user: inventory_admin
+iop_inventory_database_password_file: "{{ obsah_state_path }}/iop-inventory-db-password"
+iop_inventory_database_password: "{{ lookup('ansible.builtin.password', iop_inventory_database_password_file, chars=['ascii_letters', 'digits']) }}"
+
+iop_advisor_database_host: "{{ iop_database_host }}"
+iop_advisor_database_port: "{{ iop_database_port }}"
+iop_advisor_database_name: advisor_db
+iop_advisor_database_user: advisor_user
+iop_advisor_database_password_file: "{{ obsah_state_path }}/iop-advisor-db-password"
+iop_advisor_database_password: "{{ lookup('ansible.builtin.password', iop_advisor_database_password_file, chars=['ascii_letters', 'digits']) }}"
+
+iop_remediation_database_host: "{{ iop_database_host }}"
+iop_remediation_database_port: "{{ iop_database_port }}"
+iop_remediation_database_name: remediations_db
+iop_remediation_database_user: remediations_user
+iop_remediation_database_password_file: "{{ obsah_state_path }}/iop-remediation-db-password"
+iop_remediation_database_password: "{{ lookup('ansible.builtin.password', iop_remediation_database_password_file, chars=['ascii_letters', 'digits']) }}"
+
+iop_vmaas_database_host: "{{ iop_database_host }}"
+iop_vmaas_database_port: "{{ iop_database_port }}"
+iop_vmaas_database_name: vmaas_db
+iop_vmaas_database_user: vmaas_admin
+iop_vmaas_database_password_file: "{{ obsah_state_path }}/iop-vmaas-db-password"
+iop_vmaas_database_password: "{{ lookup('ansible.builtin.password', iop_vmaas_database_password_file, chars=['ascii_letters', 'digits']) }}"
+
+iop_vulnerability_database_host: "{{ iop_database_host }}"
+iop_vulnerability_database_port: "{{ iop_database_port }}"
+iop_vulnerability_database_name: vulnerability_db
+iop_vulnerability_database_user: vulnerability_admin
+iop_vulnerability_database_password_file: "{{ obsah_state_path }}/iop-vulnerability-db-password"
+iop_vulnerability_database_password: "{{ lookup('ansible.builtin.password', iop_vulnerability_database_password_file, chars=['ascii_letters', 'digits']) }}"
+
+databases:
+  - name: foreman
+    database: "{{ foreman_database_name }}"
+    host: "{{ foreman_database_host }}"
+    port: "{{ foreman_database_port }}"
+    user: "{{ foreman_database_user }}"
     password: "{{ foreman_database_password }}"
-  - name: "{{ pulp_database_user }}"
+    ssl_mode: "{{ foreman_database_ssl_mode }}"
+    ssl_ca: "{{ foreman_database_ssl_ca }}"
+    feature: foreman
+  - name: candlepin
+    database: "{{ candlepin_database_name }}"
+    host: "{{ candlepin_database_host }}"
+    port: "{{ candlepin_database_port }}"
+    user: "{{ candlepin_database_user }}"
+    password: "{{ candlepin_database_password }}"
+    ssl_mode: "{{ candlepin_database_ssl_mode }}"
+    ssl_ca: "{{ candlepin_database_ssl_ca }}"
+    feature: katello
+  - name: pulp
+    database: "{{ pulp_database_name }}"
+    host: "{{ pulp_database_host }}"
+    port: "{{ pulp_database_port }}"
+    user: "{{ pulp_database_user }}"
     password: "{{ pulp_database_password }}"
+    ssl_mode: "{{ pulp_database_ssl_mode }}"
+    ssl_ca: "{{ pulp_database_ssl_ca }}"
+    feature: katello
+  - name: iop_advisor
+    database: "{{ iop_advisor_database_name }}"
+    host: "{{ iop_advisor_database_host }}"
+    port: "{{ iop_advisor_database_port }}"
+    user: "{{ iop_advisor_database_user }}"
+    password: "{{ iop_advisor_database_password }}"
+    feature: iop
+  - name: iop_inventory
+    database: "{{ iop_inventory_database_name }}"
+    host: "{{ iop_inventory_database_host }}"
+    port: "{{ iop_inventory_database_port }}"
+    user: "{{ iop_inventory_database_user }}"
+    password: "{{ iop_inventory_database_password }}"
+    feature: iop
+  - name: iop_remediation
+    database: "{{ iop_remediation_database_name }}"
+    host: "{{ iop_remediation_database_host }}"
+    port: "{{ iop_remediation_database_port }}"
+    user: "{{ iop_remediation_database_user }}"
+    password: "{{ iop_remediation_database_password }}"
+    feature: iop
+  - name: iop_vmaas
+    database: "{{ iop_vmaas_database_name }}"
+    host: "{{ iop_vmaas_database_host }}"
+    port: "{{ iop_vmaas_database_port }}"
+    user: "{{ iop_vmaas_database_user }}"
+    password: "{{ iop_vmaas_database_password }}"
+    feature: iop
+  - name: iop_vulnerability
+    database: "{{ iop_vulnerability_database_name }}"
+    host: "{{ iop_vulnerability_database_host }}"
+    port: "{{ iop_vulnerability_database_port }}"
+    user: "{{ iop_vulnerability_database_user }}"
+    password: "{{ iop_vulnerability_database_password }}"
+    feature: iop
+
+all_databases: >-
+  {{ databases | selectattr('feature', 'in', enabled_features) | list }}
+
+postgresql_databases: >-
+  {{ all_databases | to_postgresql_databases }}
+postgresql_users: >-
+  {{ all_databases | to_postgresql_users }}