From d00559bdc9688772581b7aca726dc281cbb307dc Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 07:33:19 +1000
Subject: [PATCH 01/24] feat: add ocp4_workload_multi_tenant_loop role

Generic bridge workload that loops tenant workloads for N users
on a shared cluster. Designed as a stopgap until Babylon gets
native cluster-tenant lifecycle support.

Interface:
- num_users: number of tenants to provision
- tenant_workloads: list of workload FQCNs to run per user
- tenant_remove_workloads: list of workload FQCNs for destroy
- tenant_user_prefix: username prefix (default: user)
- tenant_user_offset: starting number (default: 1)
- tenant_password_length: per-user password length (default: 8)

The role overrides ocp4_workload_tenant_keycloak_username and
common_password per user iteration. All other tenant vars
defined in the AgnosticV catalog item cascade via Jinja2 lazy
evaluation from these two roots.
---
 .../ocp4_workload_multi_tenant_loop/README.md | 100 ++++++++++++++++++
 .../defaults/main.yml                         |  34 ++++++
 .../meta/main.yml                             |  17 +++
 .../tasks/destroy_user.yml                    |  32 ++++++
 .../tasks/main.yml                            |  13 +++
 .../tasks/provision_user.yml                  |  51 +++++++++
 .../tasks/remove_workload.yml                 |  55 ++++++++++
 .../tasks/workload.yml                        |  54 ++++++++++
 8 files changed, 356 insertions(+)
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/README.md
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/meta/main.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml

diff --git a/roles/ocp4_workload_multi_tenant_loop/README.md b/roles/ocp4_workload_multi_tenant_loop/README.md
new file mode 100644
index 0000000..49bb494
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/README.md
@@ -0,0 +1,100 @@
+# ocp4_workload_multi_tenant_loop
+
+Generic bridge workload that provisions tenant workloads for N users on a shared OpenShift cluster.
+
+Designed for **combined catalog items** that merge cluster-level and tenant-level provisioning into a single order. This role sits at the end of the `workloads:` list and loops a separate `tenant_workloads:` list once per user.
+
+## How it works
+
+```
+workloads:                              # run once by openshift-workloads config
+├── ocp4_workload_authentication
+├── ocp4_workload_pipelines
+├── ...cluster workloads...
+└── ocp4_workload_multi_tenant_loop     # this role
+        │
+        │  reads: num_users, tenant_workloads
+        │  loops: user1 → userN
+        │
+        ├── tenant_keycloak_user  (user1)
+        ├── tenant_namespace      (user1)
+        ├── tenant_showroom       (user1)
+        ├── tenant_keycloak_user  (user2)
+        ├── ...
+        └── tenant_showroom       (userN)
+```
+
+For each user, the role overrides two variables via `include_role vars:`:
+
+| Variable | Value |
+|---|---|
+| `ocp4_workload_tenant_keycloak_username` | `{{ tenant_user_prefix }}{{ user_number }}` |
+| `common_password` | Auto-generated per user |
+
+All other tenant variables in the catalog item should be Jinja2 expressions that derive from these two roots. Ansible evaluates Jinja2 lazily, so overriding the root causes the entire variable tree to resolve correctly for each user.
+
+## Required variables
+
+Set these in the AgnosticV catalog item:
+
+| Variable | Type | Description |
+|---|---|---|
+| `num_users` | int | Number of tenant users to provision |
+| `tenant_workloads` | list | Workload FQCNs to run per user (provision order) |
+| `tenant_remove_workloads` | list | Workload FQCNs for destroy (explicit order) |
+
+## Optional variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `tenant_user_prefix` | `user` | Username prefix (users: user1, user2, ...) |
+| `tenant_user_offset` | `1` | Starting user number |
+| `tenant_password_length` | `8` | Per-user password length |
+
+## Example: AgnosticV catalog item
+
+```yaml
+config: openshift-workloads
+num_users: 16
+
+workloads:
+- agnosticd.core_workloads.ocp4_workload_authentication
+- agnosticd.core_workloads.ocp4_workload_pipelines
+# ...cluster workloads...
+- agnosticd.core_workloads.ocp4_workload_multi_tenant_loop
+
+tenant_workloads:
+- agnosticd.namespaced_workloads.ocp4_workload_tenant_keycloak_user
+- agnosticd.namespaced_workloads.ocp4_workload_tenant_namespace
+- agnosticd.showroom.ocp4_workload_showroom
+
+tenant_remove_workloads:
+- agnosticd.showroom.ocp4_workload_showroom
+- agnosticd.namespaced_workloads.ocp4_workload_tenant_namespace
+- agnosticd.namespaced_workloads.ocp4_workload_tenant_keycloak_user
+
+remove_workloads:
+- agnosticd.core_workloads.ocp4_workload_multi_tenant_loop
+- agnosticd.core_workloads.ocp4_workload_pipelines
+- agnosticd.core_workloads.ocp4_workload_authentication
+
+# Tenant vars — all derive from ocp4_workload_tenant_keycloak_username
+ocp4_workload_tenant_namespace_username: >-
+  {{ ocp4_workload_tenant_keycloak_username }}
+ocp4_workload_showroom_namespace: >-
+  {{ ocp4_workload_tenant_keycloak_username }}-showroom
+```
+
+## Destroy behavior
+
+On destroy, the config's `remove_workloads` calls this role first. The role reads `tenant_remove_workloads` and loops users in reverse order (last user destroyed first). After all tenants are cleaned up, the config continues destroying cluster-level workloads.
+
+## Passwords
+
+Each user gets a unique password written to `{{ output_dir }}/tenant_user_{{ N }}_password`. The Ansible `password` lookup plugin generates on first call and returns the same value on re-runs — safe for idempotent provisioning.
+
+## Limitations
+
+- Provisioning is sequential (user1 completes all workloads before user2 starts)
+- If a user fails, subsequent users are not provisioned
+- No parallel provisioning in v1 — can be added as a follow-up if needed
diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
new file mode 100644
index 0000000..3999c38
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -0,0 +1,34 @@
+---
+# =================================================================
+# ocp4_workload_multi_tenant_loop — default variables
+# =================================================================
+#
+# This role is a generic bridge between cluster-level and
+# tenant-level workloads in a combined catalog item.
+#
+# Required variables (must be set in AgnosticV, no defaults):
+#   num_users              - int, number of tenant users to provision
+#   tenant_workloads       - list, workload FQCNs to run per user
+#   tenant_remove_workloads - list, workload FQCNs for destroy
+#
+# The role overrides two "root" vars per user iteration:
+#   ocp4_workload_tenant_keycloak_username
+#   common_password
+#
+# Every other tenant variable in the catalog item should be a
+# Jinja2 expression that derives from these two roots. Ansible's
+# lazy evaluation ensures they resolve correctly per user.
+# =================================================================
+
+# Username prefix. Users are named {prefix}{number}.
+# Examples: user1, user2, ... or lab-user1, lab-user2, ...
+tenant_user_prefix: user
+
+# First user number. Useful when user numbering must start
+# at something other than 1 (e.g. user0 or user100).
+tenant_user_offset: 1
+
+# Length of the auto-generated per-user password.
+# Each user gets a unique password stored at:
+#   {{ output_dir }}/tenant_user_{{ N }}_password
+tenant_password_length: 8
diff --git a/roles/ocp4_workload_multi_tenant_loop/meta/main.yml b/roles/ocp4_workload_multi_tenant_loop/meta/main.yml
new file mode 100644
index 0000000..24f021f
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/meta/main.yml
@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  role_name: ocp4_workload_multi_tenant_loop
+  author: Red Hat, Prakhar Srivastava (psrivast@redhat.com)
+  description: |
+    Generic multi-tenant loop workload. Provisions a list of
+    tenant workloads for N users on a shared OpenShift cluster.
+    Used in combined catalog items that merge cluster and tenant
+    provisioning into a single order.
+  license: MIT
+  min_ansible_version: "2.9"
+  platforms: []
+  galaxy_tags:
+  - ocp
+  - openshift
+  - multi-tenant
+dependencies: []
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
new file mode 100644
index 0000000..d39ef2e
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -0,0 +1,32 @@
+---
+# =================================================================
+# Destroy all tenant workloads for a single user.
+#
+# Mirrors provision_user.yml but uses tenant_remove_workloads
+# (the explicit destroy order defined in the catalog item).
+#
+# The password lookup returns the same password that was generated
+# during provisioning — needed by tenant workload destroy tasks
+# that reference common_password (e.g. to delete Keycloak users).
+# =================================================================
+
+- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+  ansible.builtin.set_fact:
+    _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+    _tenant_password: >-
+      {{ lookup('password',
+           output_dir ~ '/tenant_user_' ~ _tenant_user_num ~ '_password',
+           length=tenant_password_length | int,
+           chars=['ascii_letters', 'digits']
+         ) }}
+
+- name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+  ansible.builtin.include_role:
+    name: "{{ _tenant_workload }}"
+  loop: "{{ tenant_remove_workloads }}"
+  loop_control:
+    loop_var: _tenant_workload
+    label: "{{ _tenant_workload | split('.') | last }}"
+  vars:
+    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
+    common_password: "{{ _tenant_password }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
new file mode 100644
index 0000000..aaf2364
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+# --------------------------------------------------
+# Standard AgnosticD v2 dispatcher.
+# Routes to workload.yml (provision) or
+# remove_workload.yml (destroy) based on ACTION.
+# --------------------------------------------------
+- name: Running workload provision tasks
+  when: ACTION == "provision"
+  ansible.builtin.include_tasks: workload.yml
+
+- name: Running workload removal tasks
+  when: ACTION == "destroy"
+  ansible.builtin.include_tasks: remove_workload.yml
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
new file mode 100644
index 0000000..6f63a77
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -0,0 +1,51 @@
+---
+# =================================================================
+# Provision all tenant workloads for a single user.
+#
+# Called by workload.yml for each user number (_tenant_user_num).
+#
+# How variable override works:
+#   The include_role vars: block overrides two "root" variables:
+#     - ocp4_workload_tenant_keycloak_username
+#     - common_password
+#
+#   These have the highest Ansible precedence within the role
+#   execution. All other tenant vars in the catalog item are
+#   Jinja2 expressions that reference these roots, e.g.:
+#
+#     ocp4_workload_tenant_namespace_username:
+#       "{{ ocp4_workload_tenant_keycloak_username }}"
+#     ocp4_workload_showroom_namespace:
+#       "{{ ocp4_workload_tenant_keycloak_username }}-showroom"
+#
+#   Ansible evaluates Jinja2 lazily at use time, so overriding
+#   the root causes the entire variable tree to resolve correctly
+#   for this user. No need to remap every variable.
+# =================================================================
+
+# Generate a deterministic per-user password.
+# The password lookup writes to a file on first call and returns
+# the same value on subsequent calls — safe for re-runs.
+- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+  ansible.builtin.set_fact:
+    _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+    _tenant_password: >-
+      {{ lookup('password',
+           output_dir ~ '/tenant_user_' ~ _tenant_user_num ~ '_password',
+           length=tenant_password_length | int,
+           chars=['ascii_letters', 'digits']
+         ) }}
+
+# Run each tenant workload for this user.
+# The vars: block scopes the overrides to this include_role call.
+# The next user iteration will set new values.
+- name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+  ansible.builtin.include_role:
+    name: "{{ _tenant_workload }}"
+  loop: "{{ tenant_workloads }}"
+  loop_control:
+    loop_var: _tenant_workload
+    label: "{{ _tenant_workload | split('.') | last }}"
+  vars:
+    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
+    common_password: "{{ _tenant_password }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
new file mode 100644
index 0000000..d552ddc
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
@@ -0,0 +1,55 @@
+---
+# =================================================================
+# Destroy: loop tenant remove workloads for each user (reverse)
+#
+# Mirrors workload.yml but:
+#   - Reads tenant_remove_workloads (explicit destroy order)
+#   - Loops users in reverse order (last user destroyed first)
+#
+# The catalog item's remove_workloads list handles cluster-level
+# destroy. This role handles tenant-level destroy only:
+#
+#   remove_workloads:
+#   - agnosticd.core_workloads.ocp4_workload_multi_tenant_loop  <-- us
+#   - agnosticd.core_workloads.ocp4_workload_devspaces          <-- cluster
+#   - agnosticd.core_workloads.ocp4_workload_authentication     <-- cluster
+#
+# Clean separation: tenant cleanup = this role, cluster cleanup
+# = the config's remove_workloads list.
+# =================================================================
+
+- name: Validate required variables for destroy
+  ansible.builtin.assert:
+    that:
+    - num_users is defined
+    - num_users | int > 0
+    - tenant_remove_workloads is defined
+    - tenant_remove_workloads | length > 0
+    fail_msg: >-
+      num_users (int > 0) and tenant_remove_workloads (non-empty list)
+      must be defined for tenant destroy.
+
+- name: Display tenant loop destroy configuration
+  ansible.builtin.debug:
+    msg:
+    - "Tenant loop: destroying {{ num_users }} user(s)"
+    - "Destroy order: {{ tenant_remove_workloads | map('split', '.') | map('last') | list }}"
+
+# Destroy in reverse user order: user16 first, user1 last.
+# This is a convention — tenant workloads with cross-user
+# dependencies (rare) should be cleaned up last-in-first-out.
+- name: Destroy tenant for each user (reverse order)
+  ansible.builtin.include_tasks: destroy_user.yml
+  loop: >-
+    {{ range(
+         (tenant_user_offset | int) + (num_users | int) - 1,
+         (tenant_user_offset | int) - 1,
+         -1
+       ) | list }}
+  loop_control:
+    loop_var: _tenant_user_num
+    label: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+
+- name: Tenant loop destroy complete
+  ansible.builtin.debug:
+    msg: "All {{ num_users }} tenant(s) destroyed successfully."
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
new file mode 100644
index 0000000..549c548
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -0,0 +1,54 @@
+---
+# =================================================================
+# Provision: loop tenant workloads for each user
+#
+# This is the bridge between cluster-level workloads (which run
+# once via the normal workloads: list) and tenant-level workloads
+# (which run N times, once per user).
+#
+# The catalog item defines:
+#   workloads:         [..., ocp4_workload_multi_tenant_loop]
+#   tenant_workloads:  [list of per-user workloads]
+#   num_users:         16
+#
+# This role reads tenant_workloads and loops it num_users times,
+# overriding ocp4_workload_tenant_keycloak_username and
+# common_password per iteration. All other tenant vars cascade
+# via Jinja2 lazy evaluation.
+# =================================================================
+
+- name: Validate required variables
+  ansible.builtin.assert:
+    that:
+    - num_users is defined
+    - num_users | int > 0
+    - tenant_workloads is defined
+    - tenant_workloads | length > 0
+    fail_msg: >-
+      num_users (int > 0) and tenant_workloads (non-empty list)
+      must be defined in the AgnosticV catalog item.
+
+- name: Display tenant loop configuration
+  ansible.builtin.debug:
+    msg:
+    - "Tenant loop: provisioning {{ num_users }} user(s)"
+    - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
+    - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
+
+# Loop over user numbers sequentially.
+# Each iteration calls provision_user.yml which sets the per-user
+# identity and runs all tenant workloads for that user.
+- name: Provision tenant for each user
+  ansible.builtin.include_tasks: provision_user.yml
+  loop: >-
+    {{ range(
+         tenant_user_offset | int,
+         (tenant_user_offset | int) + (num_users | int)
+       ) | list }}
+  loop_control:
+    loop_var: _tenant_user_num
+    label: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+
+- name: Tenant loop provisioning complete
+  ansible.builtin.debug:
+    msg: "All {{ num_users }} tenant(s) provisioned successfully."

From 3dac4d8d1381ff11f5b907bb543d74ac4cd048bb Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 07:36:42 +1000
Subject: [PATCH 02/24] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20con?=
 =?UTF-8?q?vention=20alignment,=20safe=20defaults,=20shorter=20comments?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../defaults/main.yml                         | 40 ++++++++-----------
 .../meta/main.yml                             |  3 +-
 .../tasks/destroy_user.yml                    | 12 ++----
 .../tasks/main.yml                            |  4 +-
 .../tasks/provision_user.yml                  | 36 ++++-------------
 .../tasks/remove_workload.yml                 | 29 +++-----------
 .../tasks/workload.yml                        | 25 ++----------
 7 files changed, 39 insertions(+), 110 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index 3999c38..c9f2172 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -1,34 +1,28 @@
 ---
 # =================================================================
 # ocp4_workload_multi_tenant_loop — default variables
-# =================================================================
-#
-# This role is a generic bridge between cluster-level and
-# tenant-level workloads in a combined catalog item.
-#
-# Required variables (must be set in AgnosticV, no defaults):
-#   num_users              - int, number of tenant users to provision
-#   tenant_workloads       - list, workload FQCNs to run per user
-#   tenant_remove_workloads - list, workload FQCNs for destroy
-#
-# The role overrides two "root" vars per user iteration:
-#   ocp4_workload_tenant_keycloak_username
-#   common_password
 #
-# Every other tenant variable in the catalog item should be a
-# Jinja2 expression that derives from these two roots. Ansible's
-# lazy evaluation ensures they resolve correctly per user.
+# Generic bridge between cluster-level and tenant-level workloads
+# in a combined catalog item. Overrides two root variables per
+# user iteration (ocp4_workload_tenant_keycloak_username and
+# common_password); all other tenant vars cascade via Jinja2.
 # =================================================================
 
-# Username prefix. Users are named {prefix}{number}.
-# Examples: user1, user2, ... or lab-user1, lab-user2, ...
+# Number of tenant users to provision. Must be > 0 when used.
+num_users: 0
+
+# Workload FQCNs to run per user (provision order). Must be non-empty.
+tenant_workloads: []
+
+# Workload FQCNs for destroy (explicit order). Must be non-empty.
+tenant_remove_workloads: []
+
+# Username prefix. Users are named {prefix}{number}: user1, user2, ...
 tenant_user_prefix: user
 
-# First user number. Useful when user numbering must start
-# at something other than 1 (e.g. user0 or user100).
+# First user number.
 tenant_user_offset: 1
 
-# Length of the auto-generated per-user password.
-# Each user gets a unique password stored at:
-#   {{ output_dir }}/tenant_user_{{ N }}_password
+# Length of auto-generated per-user passwords.
+# Stored at: {{ output_dir }}/tenant_user_{{ N }}_password
 tenant_password_length: 8
diff --git a/roles/ocp4_workload_multi_tenant_loop/meta/main.yml b/roles/ocp4_workload_multi_tenant_loop/meta/main.yml
index 24f021f..a565086 100644
--- a/roles/ocp4_workload_multi_tenant_loop/meta/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/meta/main.yml
@@ -13,5 +13,6 @@ galaxy_info:
   galaxy_tags:
   - ocp
   - openshift
-  - multi-tenant
+  - multitenant
+  - loop
 dependencies: []
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index d39ef2e..ded5b47 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -1,14 +1,8 @@
 ---
-# =================================================================
+# -------------------------------------------------------------------------
 # Destroy all tenant workloads for a single user.
-#
-# Mirrors provision_user.yml but uses tenant_remove_workloads
-# (the explicit destroy order defined in the catalog item).
-#
-# The password lookup returns the same password that was generated
-# during provisioning — needed by tenant workload destroy tasks
-# that reference common_password (e.g. to delete Keycloak users).
-# =================================================================
+# Password lookup returns the same value generated during provisioning.
+# -------------------------------------------------------------------------
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
index aaf2364..23308e3 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/main.yml
@@ -1,8 +1,6 @@
 ---
 # --------------------------------------------------
-# Standard AgnosticD v2 dispatcher.
-# Routes to workload.yml (provision) or
-# remove_workload.yml (destroy) based on ACTION.
+# Do not modify this file
 # --------------------------------------------------
 - name: Running workload provision tasks
   when: ACTION == "provision"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 6f63a77..8d2b766 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -1,31 +1,12 @@
 ---
-# =================================================================
-# Provision all tenant workloads for a single user.
-#
-# Called by workload.yml for each user number (_tenant_user_num).
-#
-# How variable override works:
-#   The include_role vars: block overrides two "root" variables:
-#     - ocp4_workload_tenant_keycloak_username
-#     - common_password
-#
-#   These have the highest Ansible precedence within the role
-#   execution. All other tenant vars in the catalog item are
-#   Jinja2 expressions that reference these roots, e.g.:
-#
-#     ocp4_workload_tenant_namespace_username:
-#       "{{ ocp4_workload_tenant_keycloak_username }}"
-#     ocp4_workload_showroom_namespace:
-#       "{{ ocp4_workload_tenant_keycloak_username }}-showroom"
-#
-#   Ansible evaluates Jinja2 lazily at use time, so overriding
-#   the root causes the entire variable tree to resolve correctly
-#   for this user. No need to remap every variable.
-# =================================================================
+# -------------------------------------------------------------------------
+# Run all tenant workloads for a single user.
+# Overrides ocp4_workload_tenant_keycloak_username and common_password
+# via include_role vars: — all other tenant vars cascade via Jinja2.
+# -------------------------------------------------------------------------
 
-# Generate a deterministic per-user password.
-# The password lookup writes to a file on first call and returns
-# the same value on subsequent calls — safe for re-runs.
+# Password lookup writes to a file on first call and returns the
+# same value on re-runs — safe for idempotent re-provisioning.
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -36,9 +17,6 @@
            chars=['ascii_letters', 'digits']
          ) }}
 
-# Run each tenant workload for this user.
-# The vars: block scopes the overrides to this include_role call.
-# The next user iteration will set new values.
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
index d552ddc..5063916 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
@@ -1,29 +1,14 @@
 ---
-# =================================================================
-# Destroy: loop tenant remove workloads for each user (reverse)
-#
-# Mirrors workload.yml but:
-#   - Reads tenant_remove_workloads (explicit destroy order)
-#   - Loops users in reverse order (last user destroyed first)
-#
-# The catalog item's remove_workloads list handles cluster-level
-# destroy. This role handles tenant-level destroy only:
-#
-#   remove_workloads:
-#   - agnosticd.core_workloads.ocp4_workload_multi_tenant_loop  <-- us
-#   - agnosticd.core_workloads.ocp4_workload_devspaces          <-- cluster
-#   - agnosticd.core_workloads.ocp4_workload_authentication     <-- cluster
-#
-# Clean separation: tenant cleanup = this role, cluster cleanup
-# = the config's remove_workloads list.
-# =================================================================
+# -------------------------------------------------------------------------
+# Destroy tenant workloads for each user (reverse user order).
+# The catalog item's remove_workloads handles cluster-level teardown;
+# this role handles tenant-level teardown only.
+# -------------------------------------------------------------------------
 
 - name: Validate required variables for destroy
   ansible.builtin.assert:
     that:
-    - num_users is defined
     - num_users | int > 0
-    - tenant_remove_workloads is defined
     - tenant_remove_workloads | length > 0
     fail_msg: >-
       num_users (int > 0) and tenant_remove_workloads (non-empty list)
@@ -35,9 +20,7 @@
     - "Tenant loop: destroying {{ num_users }} user(s)"
     - "Destroy order: {{ tenant_remove_workloads | map('split', '.') | map('last') | list }}"
 
-# Destroy in reverse user order: user16 first, user1 last.
-# This is a convention — tenant workloads with cross-user
-# dependencies (rare) should be cleaned up last-in-first-out.
+# Destroy last user first (LIFO) — safe default for cross-user dependencies.
 - name: Destroy tenant for each user (reverse order)
   ansible.builtin.include_tasks: destroy_user.yml
   loop: >-
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index 549c548..75501cd 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -1,28 +1,12 @@
 ---
-# =================================================================
-# Provision: loop tenant workloads for each user
-#
-# This is the bridge between cluster-level workloads (which run
-# once via the normal workloads: list) and tenant-level workloads
-# (which run N times, once per user).
-#
-# The catalog item defines:
-#   workloads:         [..., ocp4_workload_multi_tenant_loop]
-#   tenant_workloads:  [list of per-user workloads]
-#   num_users:         16
-#
-# This role reads tenant_workloads and loops it num_users times,
-# overriding ocp4_workload_tenant_keycloak_username and
-# common_password per iteration. All other tenant vars cascade
-# via Jinja2 lazy evaluation.
-# =================================================================
+# -------------------------------------------------------------------------
+# Provision tenant workloads for each user
+# -------------------------------------------------------------------------
 
 - name: Validate required variables
   ansible.builtin.assert:
     that:
-    - num_users is defined
     - num_users | int > 0
-    - tenant_workloads is defined
     - tenant_workloads | length > 0
     fail_msg: >-
       num_users (int > 0) and tenant_workloads (non-empty list)
@@ -35,9 +19,6 @@
     - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
     - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
 
-# Loop over user numbers sequentially.
-# Each iteration calls provision_user.yml which sets the per-user
-# identity and runs all tenant workloads for that user.
 - name: Provision tenant for each user
   ansible.builtin.include_tasks: provision_user.yml
   loop: >-

From 4cab6b3494bf0e4de18304242ba3a5959c2ae4c9 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 08:59:51 +1000
Subject: [PATCH 03/24] =?UTF-8?q?fix:=20GPTEINFRA-16720=20ignore=20errors?=
 =?UTF-8?q?=20on=20destroy=20=E2=80=94=20never=20block=20cluster=20teardow?=
 =?UTF-8?q?n?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../tasks/destroy_user.yml                    | 30 ++++++++++++-------
 .../tasks/remove_workload.yml                 | 21 +++++--------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index ded5b47..9cab33f 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -1,7 +1,7 @@
 ---
 # -------------------------------------------------------------------------
 # Destroy all tenant workloads for a single user.
-# Password lookup returns the same value generated during provisioning.
+# Wrapped in block/rescue — errors are logged but never block teardown.
 # -------------------------------------------------------------------------
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -14,13 +14,21 @@
            chars=['ascii_letters', 'digits']
          ) }}
 
-- name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
-  ansible.builtin.include_role:
-    name: "{{ _tenant_workload }}"
-  loop: "{{ tenant_remove_workloads }}"
-  loop_control:
-    loop_var: _tenant_workload
-    label: "{{ _tenant_workload | split('.') | last }}"
-  vars:
-    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
-    common_password: "{{ _tenant_password }}"
+- name: "Destroy workloads for {{ _tenant_username }}"
+  block:
+  - name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+    ansible.builtin.include_role:
+      name: "{{ _tenant_workload }}"
+    loop: "{{ tenant_remove_workloads }}"
+    loop_control:
+      loop_var: _tenant_workload
+      label: "{{ _tenant_workload | split('.') | last }}"
+    vars:
+      ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
+      common_password: "{{ _tenant_password }}"
+  rescue:
+  - name: "Destroy failed for {{ _tenant_username }} — continuing"
+    ansible.builtin.debug:
+      msg: >-
+        Tenant destroy failed for {{ _tenant_username }}.
+        Error ignored — cluster teardown will clean up remaining resources.
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
index 5063916..7b6dbd6 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
@@ -1,27 +1,20 @@
 ---
 # -------------------------------------------------------------------------
 # Destroy tenant workloads for each user (reverse user order).
-# The catalog item's remove_workloads handles cluster-level teardown;
-# this role handles tenant-level teardown only.
+# All errors are ignored — destroy must never block cluster teardown.
 # -------------------------------------------------------------------------
 
-- name: Validate required variables for destroy
-  ansible.builtin.assert:
-    that:
-    - num_users | int > 0
-    - tenant_remove_workloads | length > 0
-    fail_msg: >-
-      num_users (int > 0) and tenant_remove_workloads (non-empty list)
-      must be defined for tenant destroy.
-
 - name: Display tenant loop destroy configuration
   ansible.builtin.debug:
     msg:
     - "Tenant loop: destroying {{ num_users }} user(s)"
-    - "Destroy order: {{ tenant_remove_workloads | map('split', '.') | map('last') | list }}"
+    - "Destroy order: {{ tenant_remove_workloads | default([]) | map('split', '.') | map('last') | list }}"
 
-# Destroy last user first (LIFO) — safe default for cross-user dependencies.
+# Skip entirely if nothing to clean up
 - name: Destroy tenant for each user (reverse order)
+  when:
+  - num_users | int > 0
+  - tenant_remove_workloads | default([]) | length > 0
   ansible.builtin.include_tasks: destroy_user.yml
   loop: >-
     {{ range(
@@ -35,4 +28,4 @@
 
 - name: Tenant loop destroy complete
   ansible.builtin.debug:
-    msg: "All {{ num_users }} tenant(s) destroyed successfully."
+    msg: "Tenant destroy finished (errors ignored — cluster teardown continues)."

From be014c2c0d1194dbe1e199162a3b24abafad8b39 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 09:27:32 +1000
Subject: [PATCH 04/24] feat: GPTEINFRA-16720 add configurable pre-loop delay
 for async cluster resources

---
 roles/ocp4_workload_multi_tenant_loop/defaults/main.yml  | 5 +++++
 roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml | 9 +++++++++
 2 files changed, 14 insertions(+)

diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index c9f2172..7359b84 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -26,3 +26,8 @@ tenant_user_offset: 1
 # Length of auto-generated per-user passwords.
 # Stored at: {{ output_dir }}/tenant_user_{{ N }}_password
 tenant_password_length: 8
+
+# Seconds to wait before starting the tenant loop.
+# Gives async cluster resources (ArgoCD apps, operator CRDs, etc.)
+# time to settle after cluster workloads complete.
+tenant_pre_loop_delay: 0
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index 75501cd..a735125 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -18,6 +18,15 @@
     - "Tenant loop: provisioning {{ num_users }} user(s)"
     - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
     - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
+    - "Pre-loop delay: {{ tenant_pre_loop_delay }}s"
+
+# Give async cluster resources time to settle (ArgoCD syncs,
+# operator CRDs, Keycloak realm init, etc.)
+- name: Wait for cluster workloads to settle
+  when: tenant_pre_loop_delay | int > 0
+  ansible.builtin.pause:
+    seconds: "{{ tenant_pre_loop_delay | int }}"
+    prompt: "Waiting {{ tenant_pre_loop_delay }}s for async cluster resources to settle..."
 
 - name: Provision tenant for each user
   ansible.builtin.include_tasks: provision_user.yml

From 85ba30c8a06befbb29daf06595449a5d6eb1f8ba Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 15:56:19 +1000
Subject: [PATCH 05/24] feat: GPTEINFRA-16720 inject user context into
 agnosticd_user_info via module_defaults

Use include_role apply: module_defaults: to set the user parameter
on all agnosticd_user_info calls inside tenant workloads. This makes
each user's data and messages separate in provision_data without
modifying the tenant workloads themselves.
---
 .../tasks/destroy_user.yml                                | 4 ++++
 .../tasks/provision_user.yml                              | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index 9cab33f..b2beeb2 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -19,6 +19,10 @@
   - name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
     ansible.builtin.include_role:
       name: "{{ _tenant_workload }}"
+      apply:
+        module_defaults:
+          agnosticd.core.agnosticd_user_info:
+            user: "{{ _tenant_username }}"
     loop: "{{ tenant_remove_workloads }}"
     loop_control:
       loop_var: _tenant_workload
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 8d2b766..17a43f9 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -3,6 +3,10 @@
 # Run all tenant workloads for a single user.
 # Overrides ocp4_workload_tenant_keycloak_username and common_password
 # via include_role vars: — all other tenant vars cascade via Jinja2.
+#
+# module_defaults injects user: into every agnosticd_user_info call
+# inside the tenant workloads, so each user's data and messages are
+# stored separately in provision_data without modifying the workloads.
 # -------------------------------------------------------------------------
 
 # Password lookup writes to a file on first call and returns the
@@ -20,6 +24,10 @@
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
+    apply:
+      module_defaults:
+        agnosticd.core.agnosticd_user_info:
+          user: "{{ _tenant_username }}"
   loop: "{{ tenant_workloads }}"
   loop_control:
     loop_var: _tenant_workload

From bb5ae03f2195ff3eca187a142a9d01b5d9fe8726 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 16:38:37 +1000
Subject: [PATCH 06/24] feat: GPTEINFRA-16720 register each user with Babylon
 after tenant workloads

Call agnosticd_user_info per user with username and password so
workshop_user_mode: multi has the base credentials for each user.
Lab-specific data (showroom URLs etc) comes from the tenant
workloads own agnosticd_user_info calls via module_defaults.
---
 .../tasks/provision_user.yml                          | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 17a43f9..34c4c49 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -35,3 +35,14 @@
   vars:
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
+
+# Register this user with Babylon so workshop_user_mode: multi
+# has the base credentials. Lab-specific data (showroom URLs etc)
+# comes from tenant workloads' own agnosticd_user_info calls.
+- name: "Register {{ _tenant_username }} with Babylon"
+  agnosticd.core.agnosticd_user_info:
+    user: "{{ _tenant_username }}"
+    data:
+      user: "{{ _tenant_username }}"
+      password: "{{ _tenant_password }}"
+    msg: "{{ _tenant_username }} / {{ _tenant_password }}"

From f3e98e46817c86dfc86738d3297961c8b90ba679 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 16:40:11 +1000
Subject: [PATCH 07/24] feat: GPTEINFRA-16720 include showroom URL in per-user
 registration

---
 .../tasks/provision_user.yml                             | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 34c4c49..4ce4a17 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -37,12 +37,15 @@
     common_password: "{{ _tenant_password }}"
 
 # Register this user with Babylon so workshop_user_mode: multi
-# has the base credentials. Lab-specific data (showroom URLs etc)
-# comes from tenant workloads' own agnosticd_user_info calls.
+# has credentials and showroom URL for each user.
 - name: "Register {{ _tenant_username }} with Babylon"
   agnosticd.core.agnosticd_user_info:
     user: "{{ _tenant_username }}"
     data:
       user: "{{ _tenant_username }}"
       password: "{{ _tenant_password }}"
-    msg: "{{ _tenant_username }} / {{ _tenant_password }}"
+      lab_ui_url: >-
+        https://showroom-{{ _tenant_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+    msg: >-
+      Lab: https://showroom-{{ _tenant_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+      User: {{ _tenant_username }} / {{ _tenant_password }}

From 8347b99c2e1028b5bdc9e28f745168af909d6e82 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 22:30:34 +1000
Subject: [PATCH 08/24] fix: GPTEINFRA-16720 pass _showroom_user so showroom
 reports per-user lab_ui_url

The showroom role uses _showroom_user in its agnosticd_user_info call.
When undefined it uses omit which overrides module_defaults, causing
all users to get the last user's showroom URL. Passing _showroom_user
explicitly fixes per-user lab_ui_url in the portal.
---
 .../ocp4_workload_multi_tenant_loop/tasks/provision_user.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 4ce4a17..7ef07a5 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -7,6 +7,10 @@
 # module_defaults injects user: into every agnosticd_user_info call
 # inside the tenant workloads, so each user's data and messages are
 # stored separately in provision_data without modifying the workloads.
+#
+# _showroom_user is passed explicitly because the showroom role uses
+# user: {{ _showroom_user | default(omit) }} — omit overrides
+# module_defaults, so we must set it directly.
 # -------------------------------------------------------------------------
 
 # Password lookup writes to a file on first call and returns the
@@ -35,6 +39,7 @@
   vars:
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
+    _showroom_user: "{{ _tenant_username }}"
 
 # Register this user with Babylon so workshop_user_mode: multi
 # has credentials and showroom URL for each user.

From be1872cf92f46c67e2ecb1a5f3fbad75b03a8f53 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Mon, 1 Jun 2026 23:56:51 +1000
Subject: [PATCH 09/24] =?UTF-8?q?fix:=20GPTEINFRA-16720=20remove=20=5Fshow?=
 =?UTF-8?q?room=5Fuser=20=E2=80=94=20causes=20double=20namespace=20and=20m?=
 =?UTF-8?q?essage=20leaks?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

_showroom_user triggers the showroom role's multi-user path which
appends the username to the namespace (user1-showroom-user1 instead
of user1-showroom). Also causes cross-user message contamination.

The bridge's own agnosticd_user_info call at the end of each user
is the authoritative source for lab_ui_url — showroom role writes
to global via omit, bridge writes correct URL per-user.
---
 .../tasks/provision_user.yml                         | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 7ef07a5..ea48a69 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -6,11 +6,12 @@
 #
 # module_defaults injects user: into every agnosticd_user_info call
 # inside the tenant workloads, so each user's data and messages are
-# stored separately in provision_data without modifying the workloads.
+# stored separately in provision_data.
 #
-# _showroom_user is passed explicitly because the showroom role uses
-# user: {{ _showroom_user | default(omit) }} — omit overrides
-# module_defaults, so we must set it directly.
+# NOTE: Do NOT pass _showroom_user — it triggers the showroom role's
+# multi-user path which appends the username to the namespace.
+# The bridge's own agnosticd_user_info call at the end sets the
+# correct lab_ui_url per user.
 # -------------------------------------------------------------------------
 
 # Password lookup writes to a file on first call and returns the
@@ -39,10 +40,11 @@
   vars:
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
-    _showroom_user: "{{ _tenant_username }}"
 
 # Register this user with Babylon so workshop_user_mode: multi
 # has credentials and showroom URL for each user.
+# This is the authoritative source for lab_ui_url — the showroom
+# role writes to global (omit), this call writes per-user.
 - name: "Register {{ _tenant_username }} with Babylon"
   agnosticd.core.agnosticd_user_info:
     user: "{{ _tenant_username }}"

From 1736feeac2d52f36c1cc65e49f0807dddf69009c Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 00:07:35 +1000
Subject: [PATCH 10/24] =?UTF-8?q?fix:=20GPTEINFRA-16720=20remove=20module?=
 =?UTF-8?q?=5Fdefaults=20=E2=80=94=20bridge=20owns=20per-user=20data?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

module_defaults injects user: into tenant workloads but conflicts
with showroom role's omit pattern, causing lab_ui_url overwrites.

Simpler design: tenant workloads write to global (we don't care),
the bridge's own agnosticd_user_info call is the sole source of
per-user data (user, password, lab_ui_url). Clean separation.
---
 .../tasks/destroy_user.yml                    |  4 ---
 .../tasks/provision_user.yml                  | 29 +++++++------------
 2 files changed, 10 insertions(+), 23 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index b2beeb2..9cab33f 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -19,10 +19,6 @@
   - name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
     ansible.builtin.include_role:
       name: "{{ _tenant_workload }}"
-      apply:
-        module_defaults:
-          agnosticd.core.agnosticd_user_info:
-            user: "{{ _tenant_username }}"
     loop: "{{ tenant_remove_workloads }}"
     loop_control:
       loop_var: _tenant_workload
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index ea48a69..e5f59f5 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -1,21 +1,17 @@
 ---
 # -------------------------------------------------------------------------
 # Run all tenant workloads for a single user.
-# Overrides ocp4_workload_tenant_keycloak_username and common_password
-# via include_role vars: — all other tenant vars cascade via Jinja2.
 #
-# module_defaults injects user: into every agnosticd_user_info call
-# inside the tenant workloads, so each user's data and messages are
-# stored separately in provision_data.
+# The bridge overrides ocp4_workload_tenant_keycloak_username and
+# common_password per user via include_role vars:. All other tenant
+# vars cascade via Jinja2 lazy evaluation.
 #
-# NOTE: Do NOT pass _showroom_user — it triggers the showroom role's
-# multi-user path which appends the username to the namespace.
-# The bridge's own agnosticd_user_info call at the end sets the
-# correct lab_ui_url per user.
+# Per-user data for Babylon (workshop_user_mode: multi) is handled
+# by the bridge's own agnosticd_user_info call at the end — NOT by
+# module_defaults injection. Tenant workloads write to global data
+# which is fine — the bridge call is the authoritative per-user source.
 # -------------------------------------------------------------------------
 
-# Password lookup writes to a file on first call and returns the
-# same value on re-runs — safe for idempotent re-provisioning.
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -29,10 +25,6 @@
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
-    apply:
-      module_defaults:
-        agnosticd.core.agnosticd_user_info:
-          user: "{{ _tenant_username }}"
   loop: "{{ tenant_workloads }}"
   loop_control:
     loop_var: _tenant_workload
@@ -41,10 +33,9 @@
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
 
-# Register this user with Babylon so workshop_user_mode: multi
-# has credentials and showroom URL for each user.
-# This is the authoritative source for lab_ui_url — the showroom
-# role writes to global (omit), this call writes per-user.
+# Authoritative per-user data for Babylon.
+# This is the ONLY agnosticd_user_info call with user: set.
+# Provides everything the workshop attendee needs.
 - name: "Register {{ _tenant_username }} with Babylon"
   agnosticd.core.agnosticd_user_info:
     user: "{{ _tenant_username }}"

From 8a00223700b2e5390f781b2ac6553c0de3571992 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 00:11:13 +1000
Subject: [PATCH 11/24] =?UTF-8?q?feat:=20GPTEINFRA-16720=20configurable=20?=
 =?UTF-8?q?tenant=5Fuser=5Finfo=5Fdata=20=E2=80=94=20bridge=20stays=20gene?=
 =?UTF-8?q?ric?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move lab-specific data (lab_ui_url pattern) out of the bridge role
and into agnosticv via tenant_user_info_data dict. The bridge just
merges it with the base user/password data per user.

Each lab defines its own URL patterns in agnosticv.
The bridge has zero knowledge of showroom or any specific workload.
---
 .../ocp4_workload_multi_tenant_loop/README.md | 44 +++++++++++++------
 .../defaults/main.yml                         | 17 +++++++
 .../tasks/provision_user.yml                  | 30 +++++++------
 3 files changed, 64 insertions(+), 27 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/README.md b/roles/ocp4_workload_multi_tenant_loop/README.md
index 49bb494..8e920d4 100644
--- a/roles/ocp4_workload_multi_tenant_loop/README.md
+++ b/roles/ocp4_workload_multi_tenant_loop/README.md
@@ -19,9 +19,10 @@ workloads:                              # run once by openshift-workloads config
         ├── tenant_keycloak_user  (user1)
         ├── tenant_namespace      (user1)
         ├── tenant_showroom       (user1)
+        ├── >> register user1 with Babylon
         ├── tenant_keycloak_user  (user2)
         ├── ...
-        └── tenant_showroom       (userN)
+        └── >> register userN with Babylon
 ```
 
 For each user, the role overrides two variables via `include_role vars:`:
@@ -33,6 +34,8 @@ For each user, the role overrides two variables via `include_role vars:`:
 
 All other tenant variables in the catalog item should be Jinja2 expressions that derive from these two roots. Ansible evaluates Jinja2 lazily, so overriding the root causes the entire variable tree to resolve correctly for each user.
 
+After all tenant workloads complete for a user, the bridge registers the user with Babylon via `agnosticd_user_info`. This is the **only** per-user data source for `workshop_user_mode: multi`.
+
 ## Required variables
 
 Set these in the AgnosticV catalog item:
@@ -50,6 +53,22 @@ Set these in the AgnosticV catalog item:
 | `tenant_user_prefix` | `user` | Username prefix (users: user1, user2, ...) |
 | `tenant_user_offset` | `1` | Starting user number |
 | `tenant_password_length` | `8` | Per-user password length |
+| `tenant_pre_loop_delay` | `0` | Seconds to wait before starting tenant loop |
+| `tenant_user_info_data` | `{}` | Per-user data dict reported to Babylon (lab-specific) |
+| `tenant_user_info_msg` | `""` | Per-user message (empty = auto-generated) |
+
+## Per-user data for workshops
+
+The bridge reports per-user data to Babylon for `workshop_user_mode: multi`. Base data (`user`, `password`) is always included. Lab-specific data is defined in `tenant_user_info_data`:
+
+```yaml
+# In agnosticv common.yaml:
+tenant_user_info_data:
+  lab_ui_url: >-
+    https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+```
+
+Jinja2 expressions in `tenant_user_info_data` are evaluated per user — `ocp4_workload_tenant_keycloak_username` resolves to the current user's name.
 
 ## Example: AgnosticV catalog item
 
@@ -69,25 +88,24 @@ tenant_workloads:
 - agnosticd.showroom.ocp4_workload_showroom
 
 tenant_remove_workloads:
-- agnosticd.showroom.ocp4_workload_showroom
-- agnosticd.namespaced_workloads.ocp4_workload_tenant_namespace
-- agnosticd.namespaced_workloads.ocp4_workload_tenant_keycloak_user
+- rhpds.litellm_virtual_keys.ocp4_workload_litellm_virtual_keys
 
 remove_workloads:
 - agnosticd.core_workloads.ocp4_workload_multi_tenant_loop
-- agnosticd.core_workloads.ocp4_workload_pipelines
-- agnosticd.core_workloads.ocp4_workload_authentication
 
-# Tenant vars — all derive from ocp4_workload_tenant_keycloak_username
-ocp4_workload_tenant_namespace_username: >-
-  {{ ocp4_workload_tenant_keycloak_username }}
-ocp4_workload_showroom_namespace: >-
-  {{ ocp4_workload_tenant_keycloak_username }}-showroom
+tenant_user_info_data:
+  lab_ui_url: >-
+    https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+
+__meta__:
+  catalog:
+    workshop_user_mode: multi
+    workshopLabUiRedirect: true
 ```
 
 ## Destroy behavior
 
-On destroy, the config's `remove_workloads` calls this role first. The role reads `tenant_remove_workloads` and loops users in reverse order (last user destroyed first). After all tenants are cleaned up, the config continues destroying cluster-level workloads.
+On destroy, the config's `remove_workloads` calls this role first. The role reads `tenant_remove_workloads` and loops users in reverse order (last user destroyed first). All errors are caught and logged — destroy never blocks cluster teardown.
 
 ## Passwords
 
@@ -97,4 +115,4 @@ Each user gets a unique password written to `{{ output_dir }}/tenant_user_{{ N }
 
 - Provisioning is sequential (user1 completes all workloads before user2 starts)
 - If a user fails, subsequent users are not provisioned
-- No parallel provisioning in v1 — can be added as a follow-up if needed
+- No parallel provisioning in v1 — can be added as a follow-up
diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index 7359b84..f88d5b2 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -31,3 +31,20 @@ tenant_password_length: 8
 # Gives async cluster resources (ArgoCD apps, operator CRDs, etc.)
 # time to settle after cluster workloads complete.
 tenant_pre_loop_delay: 0
+
+# Per-user data reported to Babylon via agnosticd_user_info.
+# Define in agnosticv with lab-specific data (showroom URL, etc.).
+# Jinja2 expressions are evaluated per user — use
+# ocp4_workload_tenant_keycloak_username for the current username.
+#
+# Example (in agnosticv common.yaml):
+#   tenant_user_info_data:
+#     lab_ui_url: "https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}"
+#
+# The bridge always adds 'user' and 'password' automatically.
+tenant_user_info_data: {}
+
+# Per-user message reported to Babylon via agnosticd_user_info.
+# Jinja2 expressions evaluated per user.
+# If empty, the bridge generates a default message with user/password.
+tenant_user_info_msg: ""
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index e5f59f5..d139212 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -6,10 +6,10 @@
 # common_password per user via include_role vars:. All other tenant
 # vars cascade via Jinja2 lazy evaluation.
 #
-# Per-user data for Babylon (workshop_user_mode: multi) is handled
-# by the bridge's own agnosticd_user_info call at the end — NOT by
-# module_defaults injection. Tenant workloads write to global data
-# which is fine — the bridge call is the authoritative per-user source.
+# Per-user data for Babylon (workshop_user_mode: multi) comes from
+# the bridge's own agnosticd_user_info call at the end. Lab-specific
+# data (showroom URL, etc.) is defined in agnosticv via
+# tenant_user_info_data. The bridge is fully generic.
 # -------------------------------------------------------------------------
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -33,17 +33,19 @@
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
 
-# Authoritative per-user data for Babylon.
-# This is the ONLY agnosticd_user_info call with user: set.
-# Provides everything the workshop attendee needs.
+# Report per-user data to Babylon.
+# Base data (user + password) is always included.
+# Lab-specific data comes from tenant_user_info_data (defined in agnosticv).
 - name: "Register {{ _tenant_username }} with Babylon"
-  agnosticd.core.agnosticd_user_info:
-    user: "{{ _tenant_username }}"
-    data:
+  vars:
+    _base_data:
       user: "{{ _tenant_username }}"
       password: "{{ _tenant_password }}"
-      lab_ui_url: >-
-        https://showroom-{{ _tenant_username }}-showroom.{{ openshift_cluster_ingress_domain }}
-    msg: >-
-      Lab: https://showroom-{{ _tenant_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+    _user_data: >-
+      {{ _base_data | combine(tenant_user_info_data | default({})) }}
+    _default_msg: >-
       User: {{ _tenant_username }} / {{ _tenant_password }}
+  agnosticd.core.agnosticd_user_info:
+    user: "{{ _tenant_username }}"
+    data: "{{ _user_data }}"
+    msg: "{{ tenant_user_info_msg | default(_default_msg, true) }}"

From e29d48dec478d91a753570c440367038e9e43975 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 13:28:42 +1000
Subject: [PATCH 12/24] fix: GPTEINFRA-16720 register users AFTER all tenant
 workloads complete
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The bridge's agnosticd_user_info call writes per-user data which
the showroom role reads back via agnosticd_user_data lookup. This
creates a 'users' dict that triggers showroom's multi-user path,
creating wrong namespaces (user2-showroom-user1).

Fix: split into two phases:
1. Loop all users — run tenant workloads only
2. Loop all users again — register with Babylon

This way no per-user data exists in user-data.yaml while showroom
is deploying.
---
 .../tasks/provision_user.yml                  | 27 +----------------
 .../tasks/register_user.yml                   | 29 +++++++++++++++++++
 .../tasks/workload.yml                        | 21 ++++++++++++--
 3 files changed, 49 insertions(+), 28 deletions(-)
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index d139212..c84273a 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -1,15 +1,7 @@
 ---
 # -------------------------------------------------------------------------
 # Run all tenant workloads for a single user.
-#
-# The bridge overrides ocp4_workload_tenant_keycloak_username and
-# common_password per user via include_role vars:. All other tenant
-# vars cascade via Jinja2 lazy evaluation.
-#
-# Per-user data for Babylon (workshop_user_mode: multi) comes from
-# the bridge's own agnosticd_user_info call at the end. Lab-specific
-# data (showroom URL, etc.) is defined in agnosticv via
-# tenant_user_info_data. The bridge is fully generic.
+# No agnosticd_user_info calls — registration happens in phase 2.
 # -------------------------------------------------------------------------
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -32,20 +24,3 @@
   vars:
     ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
     common_password: "{{ _tenant_password }}"
-
-# Report per-user data to Babylon.
-# Base data (user + password) is always included.
-# Lab-specific data comes from tenant_user_info_data (defined in agnosticv).
-- name: "Register {{ _tenant_username }} with Babylon"
-  vars:
-    _base_data:
-      user: "{{ _tenant_username }}"
-      password: "{{ _tenant_password }}"
-    _user_data: >-
-      {{ _base_data | combine(tenant_user_info_data | default({})) }}
-    _default_msg: >-
-      User: {{ _tenant_username }} / {{ _tenant_password }}
-  agnosticd.core.agnosticd_user_info:
-    user: "{{ _tenant_username }}"
-    data: "{{ _user_data }}"
-    msg: "{{ tenant_user_info_msg | default(_default_msg, true) }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
new file mode 100644
index 0000000..be9cc43
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -0,0 +1,29 @@
+---
+# -------------------------------------------------------------------------
+# Register a single user with Babylon (phase 2).
+# Called AFTER all tenant workloads complete for ALL users.
+# -------------------------------------------------------------------------
+
+- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+  ansible.builtin.set_fact:
+    _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+    _tenant_password: >-
+      {{ lookup('password',
+           output_dir ~ '/tenant_user_' ~ _tenant_user_num ~ '_password',
+           length=tenant_password_length | int,
+           chars=['ascii_letters', 'digits']
+         ) }}
+
+- name: "Register {{ _tenant_username }} with Babylon"
+  vars:
+    _base_data:
+      user: "{{ _tenant_username }}"
+      password: "{{ _tenant_password }}"
+    _user_data: >-
+      {{ _base_data | combine(tenant_user_info_data | default({})) }}
+    _default_msg: >-
+      User: {{ _tenant_username }} / {{ _tenant_password }}
+  agnosticd.core.agnosticd_user_info:
+    user: "{{ _tenant_username }}"
+    data: "{{ _user_data }}"
+    msg: "{{ tenant_user_info_msg | default(_default_msg, true) }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index a735125..e0dae45 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -28,7 +28,11 @@
     seconds: "{{ tenant_pre_loop_delay | int }}"
     prompt: "Waiting {{ tenant_pre_loop_delay }}s for async cluster resources to settle..."
 
-- name: Provision tenant for each user
+# Phase 1: Run all tenant workloads for each user.
+# No agnosticd_user_info calls here — showroom reads user-data.yaml
+# via agnosticd_user_data lookup. If per-user data exists at that
+# point, showroom triggers multi-user mode and creates wrong namespaces.
+- name: "Phase 1: Provision tenant workloads"
   ansible.builtin.include_tasks: provision_user.yml
   loop: >-
     {{ range(
@@ -39,6 +43,19 @@
     loop_var: _tenant_user_num
     label: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
 
+# Phase 2: Register all users with Babylon AFTER all workloads complete.
+# Now it's safe to write per-user data — showroom has already deployed.
+- name: "Phase 2: Register users with Babylon"
+  ansible.builtin.include_tasks: register_user.yml
+  loop: >-
+    {{ range(
+         tenant_user_offset | int,
+         (tenant_user_offset | int) + (num_users | int)
+       ) | list }}
+  loop_control:
+    loop_var: _tenant_user_num
+    label: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+
 - name: Tenant loop provisioning complete
   ansible.builtin.debug:
-    msg: "All {{ num_users }} tenant(s) provisioned successfully."
+    msg: "All {{ num_users }} tenant(s) provisioned and registered."

From b3acac908b6a6059bd629e207bbfb40663ae2792 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 14:27:14 +1000
Subject: [PATCH 13/24] fix: GPTEINFRA-16720 set
 ocp4_workload_tenant_keycloak_username in register phase

tenant_user_info_data references ocp4_workload_tenant_keycloak_username
which is only set via include_role vars: in phase 1. Phase 2 needs it
as a fact so the Jinja2 expressions in tenant_user_info_data resolve.
---
 roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
index be9cc43..a5abaaa 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -13,6 +13,7 @@
            length=tenant_password_length | int,
            chars=['ascii_letters', 'digits']
          ) }}
+    ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
 
 - name: "Register {{ _tenant_username }} with Babylon"
   vars:

From 8cf3ef192b233f926147ea25bccb3075048021bb Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 17:02:59 +1000
Subject: [PATCH 14/24] docs: GPTEINFRA-16720 add comprehensive comments
 explaining design decisions

Extensive inline comments for PR reviewers covering:
- Two-phase provisioning (why register after all workloads)
- Variable scoping (Jinja2 lazy eval, include_role vars:)
- Showroom interaction gotchas
- Destroy error tolerance
- Password determinism
---
 .../tasks/destroy_user.yml                    | 14 +++--
 .../tasks/provision_user.yml                  | 38 ++++++++++--
 .../tasks/register_user.yml                   | 23 +++++++-
 .../tasks/remove_workload.yml                 | 23 ++++++--
 .../tasks/workload.yml                        | 58 +++++++++++++++----
 5 files changed, 129 insertions(+), 27 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index 9cab33f..7e1c5f2 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -1,8 +1,14 @@
 ---
-# -------------------------------------------------------------------------
-# Destroy all tenant workloads for a single user.
-# Wrapped in block/rescue — errors are logged but never block teardown.
-# -------------------------------------------------------------------------
+# =========================================================================
+# Destroy external resources for a single user.
+#
+# Wrapped in block/rescue — errors are logged but NEVER block teardown.
+# The cluster is about to be destroyed; blocking here wastes time.
+#
+# Password lookup returns the same value generated during provisioning.
+# Some destroy roles need the password (e.g. to authenticate and
+# delete resources).
+# =========================================================================
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index c84273a..b893401 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -1,9 +1,39 @@
 ---
-# -------------------------------------------------------------------------
-# Run all tenant workloads for a single user.
-# No agnosticd_user_info calls — registration happens in phase 2.
-# -------------------------------------------------------------------------
+# =========================================================================
+# Run all tenant workloads for a single user (Phase 1).
+#
+# HOW VARIABLE OVERRIDE WORKS:
+#   include_role vars: sets ocp4_workload_tenant_keycloak_username and
+#   common_password for each role invocation. These have the highest
+#   Ansible precedence within the role execution.
+#
+#   All other tenant vars in the catalog item are Jinja2 expressions
+#   that reference these roots, e.g.:
+#     ocp4_workload_tenant_namespace_username:
+#       "{{ ocp4_workload_tenant_keycloak_username }}"
+#     ocp4_workload_showroom_namespace:
+#       "{{ ocp4_workload_tenant_keycloak_username }}-showroom"
+#
+#   Ansible evaluates Jinja2 lazily at use time, so overriding the
+#   root causes the entire variable tree to resolve correctly for
+#   this user. No need to remap every variable.
+#
+# WHAT NOT TO PASS:
+#   Do NOT pass _showroom_user — it triggers the showroom role's
+#   multi-user path which appends the username to the namespace,
+#   creating user1-showroom-user1 instead of user1-showroom.
+#
+#   Do NOT use module_defaults to inject user: into
+#   agnosticd_user_info — the showroom role uses
+#   user: {{ _showroom_user | default(omit) }} and omit overrides
+#   module_defaults, causing last-user-wins on lab_ui_url.
+#
+# No agnosticd_user_info calls here — registration is in phase 2.
+# =========================================================================
 
+# Password is deterministic per user number. The password lookup
+# writes to a file on first call and returns the same value on
+# re-runs — safe for idempotent re-provisioning.
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
index a5abaaa..802f606 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -1,8 +1,25 @@
 ---
-# -------------------------------------------------------------------------
-# Register a single user with Babylon (phase 2).
+# =========================================================================
+# Register a single user with Babylon (Phase 2).
+#
 # Called AFTER all tenant workloads complete for ALL users.
-# -------------------------------------------------------------------------
+# This is the ONLY agnosticd_user_info call with user: set.
+#
+# The catalog item defines tenant_user_info_data with lab-specific
+# data (e.g. showroom URL pattern). Jinja2 expressions in that dict
+# are evaluated per user. The bridge always adds 'user' and 'password'.
+#
+# Example tenant_user_info_data (in agnosticv common.yaml):
+#   tenant_user_info_data:
+#     lab_ui_url: >-
+#       https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+#
+# WHY set ocp4_workload_tenant_keycloak_username here?
+#   tenant_user_info_data references it via Jinja2. In phase 1
+#   it's set via include_role vars: (scoped to the role). Here
+#   we need it as a fact so the Jinja2 in tenant_user_info_data
+#   resolves correctly.
+# =========================================================================
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
index 7b6dbd6..07e447b 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
@@ -1,8 +1,22 @@
 ---
-# -------------------------------------------------------------------------
-# Destroy tenant workloads for each user (reverse user order).
-# All errors are ignored — destroy must never block cluster teardown.
-# -------------------------------------------------------------------------
+# =========================================================================
+# DESTROY: Clean up external resources per user.
+#
+# In a combined CI, the OCP cluster is destroyed after this role runs.
+# Everything inside the cluster (namespaces, pods, operators) is wiped
+# when the cluster goes away. Only EXTERNAL resources need explicit
+# cleanup — e.g. LiteMaaS virtual keys that live outside the cluster.
+#
+# The catalog item defines tenant_remove_workloads with only those
+# external cleanup roles. In-cluster workloads are NOT listed.
+#
+# Errors are caught and logged — destroy must NEVER block cluster
+# teardown. If LiteMaaS cleanup fails (key expired, API down),
+# the cluster still gets destroyed and the keys expire on their own.
+#
+# Users are destroyed in reverse order (LIFO) as a convention for
+# any cross-user dependencies.
+# =========================================================================
 
 - name: Display tenant loop destroy configuration
   ansible.builtin.debug:
@@ -10,7 +24,6 @@
     - "Tenant loop: destroying {{ num_users }} user(s)"
     - "Destroy order: {{ tenant_remove_workloads | default([]) | map('split', '.') | map('last') | list }}"
 
-# Skip entirely if nothing to clean up
 - name: Destroy tenant for each user (reverse order)
   when:
   - num_users | int > 0
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index e0dae45..ecd1123 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -1,7 +1,39 @@
 ---
-# -------------------------------------------------------------------------
-# Provision tenant workloads for each user
-# -------------------------------------------------------------------------
+# =========================================================================
+# PROVISION: Two-phase tenant provisioning
+#
+# This role is a generic bridge between cluster-level workloads (which
+# run once via the config's workloads: list) and tenant-level workloads
+# (which need to run once per user).
+#
+# The catalog item defines:
+#   workloads:         [..., ocp4_workload_multi_tenant_loop]  (cluster)
+#   tenant_workloads:  [keycloak, namespace, gitea, ...]       (per-user)
+#   num_users:         3
+#
+# Phase 1 — Provision:
+#   Loop all users sequentially. For each user, run every workload in
+#   tenant_workloads via include_role, overriding two root variables:
+#     - ocp4_workload_tenant_keycloak_username (e.g. user1)
+#     - common_password (auto-generated per user)
+#   All other tenant vars in the catalog item are Jinja2 expressions
+#   that derive from these two roots. Ansible evaluates them lazily,
+#   so the override cascades automatically.
+#
+# Phase 2 — Register:
+#   AFTER all users' workloads complete, loop again and call
+#   agnosticd_user_info per user. This registers each user with
+#   Babylon for workshop_user_mode: multi.
+#
+# WHY two phases?
+#   The showroom role reads user-data.yaml via agnosticd_user_data
+#   lookup. If per-user data exists (from agnosticd_user_info calls
+#   with user: set), the showroom role detects a 'users' dict and
+#   switches to multi-user mode — which appends the username to the
+#   namespace, creating wrong namespaces like user2-showroom-user1.
+#   By deferring all agnosticd_user_info calls to phase 2, the
+#   user-data.yaml stays clean during showroom deployment.
+# =========================================================================
 
 - name: Validate required variables
   ansible.builtin.assert:
@@ -20,18 +52,20 @@
     - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
     - "Pre-loop delay: {{ tenant_pre_loop_delay }}s"
 
-# Give async cluster resources time to settle (ArgoCD syncs,
-# operator CRDs, Keycloak realm init, etc.)
+# Some cluster workloads kick off async operations (ArgoCD syncs,
+# operator CRDs, Keycloak realm init) that aren't finished when
+# their Ansible tasks return. This pause gives them time to settle.
+# Set tenant_pre_loop_delay in the catalog item (default: 0).
 - name: Wait for cluster workloads to settle
   when: tenant_pre_loop_delay | int > 0
   ansible.builtin.pause:
     seconds: "{{ tenant_pre_loop_delay | int }}"
     prompt: "Waiting {{ tenant_pre_loop_delay }}s for async cluster resources to settle..."
 
-# Phase 1: Run all tenant workloads for each user.
-# No agnosticd_user_info calls here — showroom reads user-data.yaml
-# via agnosticd_user_data lookup. If per-user data exists at that
-# point, showroom triggers multi-user mode and creates wrong namespaces.
+# -----------------------------------------------------------------------
+# PHASE 1: Run tenant workloads for each user.
+# No agnosticd_user_info calls happen here — see note above.
+# -----------------------------------------------------------------------
 - name: "Phase 1: Provision tenant workloads"
   ansible.builtin.include_tasks: provision_user.yml
   loop: >-
@@ -43,8 +77,10 @@
     loop_var: _tenant_user_num
     label: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
 
-# Phase 2: Register all users with Babylon AFTER all workloads complete.
-# Now it's safe to write per-user data — showroom has already deployed.
+# -----------------------------------------------------------------------
+# PHASE 2: Register all users with Babylon.
+# Now safe to write per-user data — all showroom pods are deployed.
+# -----------------------------------------------------------------------
 - name: "Phase 2: Register users with Babylon"
   ansible.builtin.include_tasks: register_user.yml
   loop: >-

From 9c71f13d16b15f50ef65e056eb1c0546bea8f81b Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Tue, 2 Jun 2026 18:01:17 +1000
Subject: [PATCH 15/24] feat: GPTEINFRA-16720 add tenant_workload_vars support
 for per-workload overrides

When the same workload (e.g. gitops_bootstrap) runs at both cluster
and tenant level with different vars, the bridge needs to override
specific vars for the tenant invocation.

tenant_workload_vars is a dict in agnosticv mapping workload FQCNs
to var overrides. The bridge applies them via set_fact before
include_role. ocp4_workload_tenant_keycloak_username is also set
as a fact so Jinja2 expressions in tenant_workload_vars resolve.
---
 .../tasks/provision_user.yml                  | 47 ++++++-------------
 .../tasks/provision_workload.yml              | 33 +++++++++++++
 2 files changed, 47 insertions(+), 33 deletions(-)
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index b893401..37df678 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -2,38 +2,18 @@
 # =========================================================================
 # Run all tenant workloads for a single user (Phase 1).
 #
-# HOW VARIABLE OVERRIDE WORKS:
-#   include_role vars: sets ocp4_workload_tenant_keycloak_username and
-#   common_password for each role invocation. These have the highest
-#   Ansible precedence within the role execution.
+# Sets ocp4_workload_tenant_keycloak_username as a FACT (not just
+# include_role vars:) so Jinja2 expressions in tenant_workload_vars
+# resolve correctly when constructing the override dict.
 #
-#   All other tenant vars in the catalog item are Jinja2 expressions
-#   that reference these roots, e.g.:
-#     ocp4_workload_tenant_namespace_username:
-#       "{{ ocp4_workload_tenant_keycloak_username }}"
-#     ocp4_workload_showroom_namespace:
-#       "{{ ocp4_workload_tenant_keycloak_username }}-showroom"
-#
-#   Ansible evaluates Jinja2 lazily at use time, so overriding the
-#   root causes the entire variable tree to resolve correctly for
-#   this user. No need to remap every variable.
-#
-# WHAT NOT TO PASS:
-#   Do NOT pass _showroom_user — it triggers the showroom role's
-#   multi-user path which appends the username to the namespace,
-#   creating user1-showroom-user1 instead of user1-showroom.
-#
-#   Do NOT use module_defaults to inject user: into
-#   agnosticd_user_info — the showroom role uses
-#   user: {{ _showroom_user | default(omit) }} and omit overrides
-#   module_defaults, causing last-user-wins on lab_ui_url.
+# For each workload, applies per-workload overrides from
+# tenant_workload_vars (if defined) via set_fact before include_role.
+# This handles cases like gitops_bootstrap which runs at both cluster
+# and tenant level with different vars.
 #
 # No agnosticd_user_info calls here — registration is in phase 2.
 # =========================================================================
 
-# Password is deterministic per user number. The password lookup
-# writes to a file on first call and returns the same value on
-# re-runs — safe for idempotent re-provisioning.
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -43,14 +23,15 @@
            length=tenant_password_length | int,
            chars=['ascii_letters', 'digits']
          ) }}
+    # Set as fact so tenant_workload_vars Jinja2 expressions can
+    # reference it (e.g. bootstrap-tenant-{{ ocp4_workload_tenant_keycloak_username }})
+    ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
 
-- name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
-  ansible.builtin.include_role:
-    name: "{{ _tenant_workload }}"
+# Loop workloads via include_tasks (not direct include_role loop)
+# so we can apply per-workload set_fact overrides.
+- name: "Provision workloads for {{ _tenant_username }}"
+  ansible.builtin.include_tasks: provision_workload.yml
   loop: "{{ tenant_workloads }}"
   loop_control:
     loop_var: _tenant_workload
     label: "{{ _tenant_workload | split('.') | last }}"
-  vars:
-    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
-    common_password: "{{ _tenant_password }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
new file mode 100644
index 0000000..fc8a681
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -0,0 +1,33 @@
+---
+# =========================================================================
+# Run a single tenant workload for the current user.
+#
+# If tenant_workload_vars has overrides for this workload, apply them
+# via set_fact before include_role. This handles cases like
+# gitops_bootstrap which uses different repo_path, application_name,
+# and helm_values at cluster vs tenant level.
+#
+# set_fact overrides persist across workloads in the same user
+# iteration. This is safe because:
+#   - AgnosticD workloads use role-prefixed variable names
+#   - Overrides for gitops_bootstrap won't affect showroom vars
+#   - Each user iteration re-sets ocp4_workload_tenant_keycloak_username
+# =========================================================================
+
+# Apply per-workload variable overrides if defined in agnosticv.
+# These override the top-level (cluster) values for this workload.
+- name: "Apply overrides for {{ _tenant_workload | split('.') | last }}"
+  when: _tenant_workload in (tenant_workload_vars | default({}))
+  ansible.builtin.set_fact:
+    "{{ _override_item.key }}": "{{ _override_item.value }}"
+  loop: "{{ (tenant_workload_vars[_tenant_workload]) | dict2items }}"
+  loop_control:
+    loop_var: _override_item
+    label: "{{ _override_item.key }}"
+
+- name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+  ansible.builtin.include_role:
+    name: "{{ _tenant_workload }}"
+  vars:
+    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
+    common_password: "{{ _tenant_password }}"

From 4d106091d9b64d68ff0fec57d2df90aaf941bf06 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 08:02:50 +1000
Subject: [PATCH 16/24] fix: GPTEINFRA-16720 safe loop for tenant_workload_vars
 missing keys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

loop: evaluates before when: in Ansible — accessing
tenant_workload_vars[_tenant_workload] fails with KeyError
for workloads not in the dict. Use default({}) to return
empty list for missing keys.
---
 .../tasks/provision_workload.yml                 | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index fc8a681..8afe32e 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -6,24 +6,22 @@
 # via set_fact before include_role. This handles cases like
 # gitops_bootstrap which uses different repo_path, application_name,
 # and helm_values at cluster vs tenant level.
-#
-# set_fact overrides persist across workloads in the same user
-# iteration. This is safe because:
-#   - AgnosticD workloads use role-prefixed variable names
-#   - Overrides for gitops_bootstrap won't affect showroom vars
-#   - Each user iteration re-sets ocp4_workload_tenant_keycloak_username
 # =========================================================================
 
 # Apply per-workload variable overrides if defined in agnosticv.
-# These override the top-level (cluster) values for this workload.
+# Safe for workloads not in tenant_workload_vars — default({}) returns
+# empty dict, dict2items returns empty list, loop runs zero iterations.
 - name: "Apply overrides for {{ _tenant_workload | split('.') | last }}"
-  when: _tenant_workload in (tenant_workload_vars | default({}))
+  when: _tenant_workload_overrides | length > 0
   ansible.builtin.set_fact:
     "{{ _override_item.key }}": "{{ _override_item.value }}"
-  loop: "{{ (tenant_workload_vars[_tenant_workload]) | dict2items }}"
+  loop: "{{ _tenant_workload_overrides | dict2items }}"
   loop_control:
     loop_var: _override_item
     label: "{{ _override_item.key }}"
+  vars:
+    _tenant_workload_overrides: >-
+      {{ (tenant_workload_vars | default({}))[_tenant_workload] | default({}) }}
 
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:

From b021977f5e3f9a12932fe1db933ff4645a45c595 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 10:56:16 +1000
Subject: [PATCH 17/24] fix: GPTEINFRA-16720 use include_role vars: for
 tenant_workload_vars overrides
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

set_fact cannot override extra_vars (Ansible precedence).
include_role vars: CAN — it has higher precedence than extra_vars.

Merge tenant_workload_vars overrides into the include_role vars:
dict so they override the top-level (extra_var) values.

Also set common_password as a fact so Jinja2 chains in
tenant_workload_vars values resolve outside include_role context.
---
 .../tasks/provision_user.yml                  | 24 ++++-----
 .../tasks/provision_workload.yml              | 50 +++++++++++--------
 2 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 37df678..edfc8c2 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -2,14 +2,9 @@
 # =========================================================================
 # Run all tenant workloads for a single user (Phase 1).
 #
-# Sets ocp4_workload_tenant_keycloak_username as a FACT (not just
-# include_role vars:) so Jinja2 expressions in tenant_workload_vars
-# resolve correctly when constructing the override dict.
-#
-# For each workload, applies per-workload overrides from
-# tenant_workload_vars (if defined) via set_fact before include_role.
-# This handles cases like gitops_bootstrap which runs at both cluster
-# and tenant level with different vars.
+# Sets root variables as FACTS so they're available for Jinja2
+# resolution in tenant_workload_vars (which is evaluated outside
+# include_role context).
 #
 # No agnosticd_user_info calls here — registration is in phase 2.
 # =========================================================================
@@ -23,12 +18,17 @@
            length=tenant_password_length | int,
            chars=['ascii_letters', 'digits']
          ) }}
-    # Set as fact so tenant_workload_vars Jinja2 expressions can
-    # reference it (e.g. bootstrap-tenant-{{ ocp4_workload_tenant_keycloak_username }})
+    # These must be facts (not just include_role vars:) so that
+    # Jinja2 expressions in tenant_workload_vars can reference them
+    # when building the merged vars dict.
     ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
+    common_password: >-
+      {{ lookup('password',
+           output_dir ~ '/tenant_user_' ~ _tenant_user_num ~ '_password',
+           length=tenant_password_length | int,
+           chars=['ascii_letters', 'digits']
+         ) }}
 
-# Loop workloads via include_tasks (not direct include_role loop)
-# so we can apply per-workload set_fact overrides.
 - name: "Provision workloads for {{ _tenant_username }}"
   ansible.builtin.include_tasks: provision_workload.yml
   loop: "{{ tenant_workloads }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index 8afe32e..aed5f62 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -2,30 +2,40 @@
 # =========================================================================
 # Run a single tenant workload for the current user.
 #
-# If tenant_workload_vars has overrides for this workload, apply them
-# via set_fact before include_role. This handles cases like
-# gitops_bootstrap which uses different repo_path, application_name,
-# and helm_values at cluster vs tenant level.
+# ANSIBLE PRECEDENCE FIX:
+#   AgnosticV vars are passed to AAP as extra_vars (highest precedence).
+#   set_fact CANNOT override extra_vars.
+#   include_role vars: CAN override extra_vars.
+#
+#   So we merge tenant_workload_vars overrides into the include_role
+#   vars: dict. This ensures tenant values (bootstrap-tenant) override
+#   cluster values (bootstrap-infra) even though both are extra_vars.
+#
+# HOW IT WORKS:
+#   1. Build base vars dict (username + password)
+#   2. Lookup per-workload overrides from tenant_workload_vars
+#   3. Combine them (overrides win)
+#   4. Pass merged dict via include_role vars:
 # =========================================================================
 
-# Apply per-workload variable overrides if defined in agnosticv.
-# Safe for workloads not in tenant_workload_vars — default({}) returns
-# empty dict, dict2items returns empty list, loop runs zero iterations.
-- name: "Apply overrides for {{ _tenant_workload | split('.') | last }}"
-  when: _tenant_workload_overrides | length > 0
+# Build the merged vars dict as a fact so Jinja2 in the override
+# values resolves now (using facts set in provision_user.yml).
+- name: "Build vars for {{ _tenant_workload | split('.') | last }}"
   ansible.builtin.set_fact:
-    "{{ _override_item.key }}": "{{ _override_item.value }}"
-  loop: "{{ _tenant_workload_overrides | dict2items }}"
-  loop_control:
-    loop_var: _override_item
-    label: "{{ _override_item.key }}"
-  vars:
-    _tenant_workload_overrides: >-
-      {{ (tenant_workload_vars | default({}))[_tenant_workload] | default({}) }}
+    _role_vars: >-
+      {{
+        {
+          'ocp4_workload_tenant_keycloak_username': _tenant_username,
+          'common_password': _tenant_password
+        } | combine(
+          (tenant_workload_vars | default({}))[_tenant_workload] | default({})
+        )
+      }}
 
+# include_role vars: has higher precedence than extra_vars.
+# This is the ONLY way to override top-level vars that AAP
+# passes as extra_vars.
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
-  vars:
-    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
-    common_password: "{{ _tenant_password }}"
+  vars: "{{ _role_vars }}"

From 59965a7d1bd20bee0496c77d7270f5e855f52260 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 11:12:33 +1000
Subject: [PATCH 18/24] feat: GPTEINFRA-16720 add tenant_cluster_workloads for
 pre-loop cluster workloads
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the same workload (e.g. gitops_bootstrap) runs at both cluster
and tenant level with different vars, defining either set at the
top level makes them extra_vars (highest Ansible precedence — can't
be overridden by anything).

Fix: move the workload OUT of the workloads: list. Run it inside
the bridge via include_role vars: (which CAN override role defaults
when the vars are NOT extra_vars).

tenant_cluster_workloads: workloads to run ONCE before the user loop
tenant_cluster_workloads_vars: per-workload vars for those workloads
---
 .../defaults/main.yml                         | 37 +++++++-----
 .../tasks/provision_workload.yml              | 23 ++------
 .../tasks/run_cluster_workload.yml            | 21 +++++++
 .../tasks/workload.yml                        | 59 ++++++++-----------
 4 files changed, 70 insertions(+), 70 deletions(-)
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml

diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index f88d5b2..f397c41 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -24,27 +24,34 @@ tenant_user_prefix: user
 tenant_user_offset: 1
 
 # Length of auto-generated per-user passwords.
-# Stored at: {{ output_dir }}/tenant_user_{{ N }}_password
 tenant_password_length: 8
 
 # Seconds to wait before starting the tenant loop.
-# Gives async cluster resources (ArgoCD apps, operator CRDs, etc.)
-# time to settle after cluster workloads complete.
 tenant_pre_loop_delay: 0
 
 # Per-user data reported to Babylon via agnosticd_user_info.
-# Define in agnosticv with lab-specific data (showroom URL, etc.).
-# Jinja2 expressions are evaluated per user — use
-# ocp4_workload_tenant_keycloak_username for the current username.
-#
-# Example (in agnosticv common.yaml):
-#   tenant_user_info_data:
-#     lab_ui_url: "https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}"
-#
-# The bridge always adds 'user' and 'password' automatically.
 tenant_user_info_data: {}
 
-# Per-user message reported to Babylon via agnosticd_user_info.
-# Jinja2 expressions evaluated per user.
-# If empty, the bridge generates a default message with user/password.
+# Per-user message reported to Babylon.
 tenant_user_info_msg: ""
+
+# Per-workload variable overrides for tenant workloads.
+# Keys are workload FQCNs, values are dicts merged into include_role vars.
+tenant_workload_vars: {}
+
+# ---------------------------------------------------------------
+# Cluster workloads run ONCE inside the bridge, before the user loop.
+#
+# WHY: When the same role (e.g. gitops_bootstrap) runs at both
+# cluster and tenant level with different vars, defining either
+# set at the AgnosticV top level makes them extra_vars (highest
+# Ansible precedence — nothing can override them). By running
+# both invocations inside the bridge via include_role vars:,
+# we avoid the extra_vars conflict entirely.
+#
+# Workloads listed here are NOT in the config's workloads: list.
+# ---------------------------------------------------------------
+tenant_cluster_workloads: []
+
+# Per-workload variable overrides for cluster workloads.
+tenant_cluster_workloads_vars: {}
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index aed5f62..4548859 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -2,24 +2,12 @@
 # =========================================================================
 # Run a single tenant workload for the current user.
 #
-# ANSIBLE PRECEDENCE FIX:
-#   AgnosticV vars are passed to AAP as extra_vars (highest precedence).
-#   set_fact CANNOT override extra_vars.
-#   include_role vars: CAN override extra_vars.
-#
-#   So we merge tenant_workload_vars overrides into the include_role
-#   vars: dict. This ensures tenant values (bootstrap-tenant) override
-#   cluster values (bootstrap-infra) even though both are extra_vars.
-#
-# HOW IT WORKS:
-#   1. Build base vars dict (username + password)
-#   2. Lookup per-workload overrides from tenant_workload_vars
-#   3. Combine them (overrides win)
-#   4. Pass merged dict via include_role vars:
+# Merges base vars (username + password) with per-workload overrides
+# from tenant_workload_vars. Passes everything via include_role vars:
+# which overrides role defaults (but not extra_vars — so the
+# conflicting vars must NOT be at the AgnosticV top level).
 # =========================================================================
 
-# Build the merged vars dict as a fact so Jinja2 in the override
-# values resolves now (using facts set in provision_user.yml).
 - name: "Build vars for {{ _tenant_workload | split('.') | last }}"
   ansible.builtin.set_fact:
     _role_vars: >-
@@ -32,9 +20,6 @@
         )
       }}
 
-# include_role vars: has higher precedence than extra_vars.
-# This is the ONLY way to override top-level vars that AAP
-# passes as extra_vars.
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
new file mode 100644
index 0000000..ada57fc
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
@@ -0,0 +1,21 @@
+---
+# =========================================================================
+# Run a single cluster workload inside the bridge (Phase 0).
+#
+# These workloads can't be in the config's workloads: list because
+# their vars would become extra_vars and conflict with the tenant
+# invocation of the same role.
+#
+# By running them here via include_role vars:, the vars are NOT
+# extra_vars and include_role vars: overrides role defaults.
+# =========================================================================
+
+- name: "Build vars for {{ _cluster_workload | split('.') | last }}"
+  ansible.builtin.set_fact:
+    _cluster_role_vars: >-
+      {{ (tenant_cluster_workloads_vars | default({}))[_cluster_workload] | default({}) }}
+
+- name: "Run cluster workload: {{ _cluster_workload | split('.') | last }}"
+  ansible.builtin.include_role:
+    name: "{{ _cluster_workload }}"
+  vars: "{{ _cluster_role_vars }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index ecd1123..ba14d2d 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -1,38 +1,15 @@
 ---
 # =========================================================================
-# PROVISION: Two-phase tenant provisioning
+# PROVISION: Three-phase tenant provisioning
 #
-# This role is a generic bridge between cluster-level workloads (which
-# run once via the config's workloads: list) and tenant-level workloads
-# (which need to run once per user).
+# Phase 0: Run cluster workloads that can't be in the config's
+#          workloads: list (e.g. gitops_bootstrap when it also runs
+#          per-tenant with different vars — extra_vars conflict).
 #
-# The catalog item defines:
-#   workloads:         [..., ocp4_workload_multi_tenant_loop]  (cluster)
-#   tenant_workloads:  [keycloak, namespace, gitea, ...]       (per-user)
-#   num_users:         3
+# Phase 1: Loop all users, run tenant_workloads per user.
+#          No agnosticd_user_info calls (showroom compatibility).
 #
-# Phase 1 — Provision:
-#   Loop all users sequentially. For each user, run every workload in
-#   tenant_workloads via include_role, overriding two root variables:
-#     - ocp4_workload_tenant_keycloak_username (e.g. user1)
-#     - common_password (auto-generated per user)
-#   All other tenant vars in the catalog item are Jinja2 expressions
-#   that derive from these two roots. Ansible evaluates them lazily,
-#   so the override cascades automatically.
-#
-# Phase 2 — Register:
-#   AFTER all users' workloads complete, loop again and call
-#   agnosticd_user_info per user. This registers each user with
-#   Babylon for workshop_user_mode: multi.
-#
-# WHY two phases?
-#   The showroom role reads user-data.yaml via agnosticd_user_data
-#   lookup. If per-user data exists (from agnosticd_user_info calls
-#   with user: set), the showroom role detects a 'users' dict and
-#   switches to multi-user mode — which appends the username to the
-#   namespace, creating wrong namespaces like user2-showroom-user1.
-#   By deferring all agnosticd_user_info calls to phase 2, the
-#   user-data.yaml stays clean during showroom deployment.
+# Phase 2: Loop all users again, register each with Babylon.
 # =========================================================================
 
 - name: Validate required variables
@@ -49,13 +26,25 @@
     msg:
     - "Tenant loop: provisioning {{ num_users }} user(s)"
     - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
+    - "Cluster workloads: {{ tenant_cluster_workloads | default([]) | map('split', '.') | map('last') | list }}"
     - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
     - "Pre-loop delay: {{ tenant_pre_loop_delay }}s"
 
-# Some cluster workloads kick off async operations (ArgoCD syncs,
-# operator CRDs, Keycloak realm init) that aren't finished when
-# their Ansible tasks return. This pause gives them time to settle.
-# Set tenant_pre_loop_delay in the catalog item (default: 0).
+# -----------------------------------------------------------------------
+# PHASE 0: Run cluster workloads inside the bridge.
+#
+# These workloads can't be in the config's workloads: list because
+# their vars conflict with tenant invocations. Running them here
+# via include_role vars: avoids the extra_vars precedence issue.
+# -----------------------------------------------------------------------
+- name: "Phase 0: Run cluster workloads inside bridge"
+  when: tenant_cluster_workloads | default([]) | length > 0
+  ansible.builtin.include_tasks: run_cluster_workload.yml
+  loop: "{{ tenant_cluster_workloads }}"
+  loop_control:
+    loop_var: _cluster_workload
+    label: "{{ _cluster_workload | split('.') | last }}"
+
 - name: Wait for cluster workloads to settle
   when: tenant_pre_loop_delay | int > 0
   ansible.builtin.pause:
@@ -64,7 +53,6 @@
 
 # -----------------------------------------------------------------------
 # PHASE 1: Run tenant workloads for each user.
-# No agnosticd_user_info calls happen here — see note above.
 # -----------------------------------------------------------------------
 - name: "Phase 1: Provision tenant workloads"
   ansible.builtin.include_tasks: provision_user.yml
@@ -79,7 +67,6 @@
 
 # -----------------------------------------------------------------------
 # PHASE 2: Register all users with Babylon.
-# Now safe to write per-user data — all showroom pods are deployed.
 # -----------------------------------------------------------------------
 - name: "Phase 2: Register users with Babylon"
   ansible.builtin.include_tasks: register_user.yml

From bf082cbfd2a192ebc7ebdccb718e68f976534559 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 14:38:57 +1000
Subject: [PATCH 19/24] =?UTF-8?q?fix:=20GPTEINFRA-16720=20fix=20set=5Ffact?=
 =?UTF-8?q?=20YAML=20block=20scalar=20=E2=80=94=20must=20produce=20dict=20?=
 =?UTF-8?q?not=20string?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The >- block scalar makes set_fact store a STRING representation
of the dict instead of an actual dict. include_role vars: then
fails because it gets a string, not a mapping.

Remove >- so Jinja2 evaluates to an actual dict object.
---
 .../tasks/provision_workload.yml                  | 15 +--------------
 .../tasks/run_cluster_workload.yml                | 10 +---------
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index 4548859..bcc69bd 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -1,24 +1,11 @@
 ---
 # =========================================================================
 # Run a single tenant workload for the current user.
-#
-# Merges base vars (username + password) with per-workload overrides
-# from tenant_workload_vars. Passes everything via include_role vars:
-# which overrides role defaults (but not extra_vars — so the
-# conflicting vars must NOT be at the AgnosticV top level).
 # =========================================================================
 
 - name: "Build vars for {{ _tenant_workload | split('.') | last }}"
   ansible.builtin.set_fact:
-    _role_vars: >-
-      {{
-        {
-          'ocp4_workload_tenant_keycloak_username': _tenant_username,
-          'common_password': _tenant_password
-        } | combine(
-          (tenant_workload_vars | default({}))[_tenant_workload] | default({})
-        )
-      }}
+    _role_vars: "{{ {'ocp4_workload_tenant_keycloak_username': _tenant_username, 'common_password': _tenant_password} | combine((tenant_workload_vars | default({}))[_tenant_workload] | default({})) }}"
 
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
index ada57fc..5853073 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
@@ -1,19 +1,11 @@
 ---
 # =========================================================================
 # Run a single cluster workload inside the bridge (Phase 0).
-#
-# These workloads can't be in the config's workloads: list because
-# their vars would become extra_vars and conflict with the tenant
-# invocation of the same role.
-#
-# By running them here via include_role vars:, the vars are NOT
-# extra_vars and include_role vars: overrides role defaults.
 # =========================================================================
 
 - name: "Build vars for {{ _cluster_workload | split('.') | last }}"
   ansible.builtin.set_fact:
-    _cluster_role_vars: >-
-      {{ (tenant_cluster_workloads_vars | default({}))[_cluster_workload] | default({}) }}
+    _cluster_role_vars: "{{ (tenant_cluster_workloads_vars | default({}))[_cluster_workload] | default({}) }}"
 
 - name: "Run cluster workload: {{ _cluster_workload | split('.') | last }}"
   ansible.builtin.include_role:

From 17a42b3f2f4cc032c99981ed40e6852433264308 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 16:15:51 +1000
Subject: [PATCH 20/24] fix: GPTEINFRA-16720 use set_fact per-key instead of
 dynamic include_role vars

include_role vars: with a Jinja2 dict expression fails in AAP.
Switch to set_fact for each override key individually, then call
include_role without dynamic vars. Since these vars are NOT at
the top level (not extra_vars), set_fact CAN override role defaults.
---
 .../tasks/provision_workload.yml              | 20 ++++++++++++++++---
 .../tasks/run_cluster_workload.yml            | 14 ++++++++++---
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index bcc69bd..bb1ee7f 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -1,13 +1,27 @@
 ---
 # =========================================================================
 # Run a single tenant workload for the current user.
+#
+# Uses set_fact for per-workload overrides + include_role vars: for
+# the two root vars (username + password). The root vars MUST go
+# through include_role vars: because they change per user and per
+# workload invocation. The overrides use set_fact because they
+# persist for the user's iteration (which is fine — role-prefixed
+# var names don't collide across workloads).
 # =========================================================================
 
-- name: "Build vars for {{ _tenant_workload | split('.') | last }}"
+- name: "Apply tenant overrides for {{ _tenant_workload | split('.') | last }}"
+  when: _tenant_workload in (tenant_workload_vars | default({}))
   ansible.builtin.set_fact:
-    _role_vars: "{{ {'ocp4_workload_tenant_keycloak_username': _tenant_username, 'common_password': _tenant_password} | combine((tenant_workload_vars | default({}))[_tenant_workload] | default({})) }}"
+    "{{ _tw_item.key }}": "{{ _tw_item.value }}"
+  loop: "{{ (tenant_workload_vars[_tenant_workload]) | dict2items }}"
+  loop_control:
+    loop_var: _tw_item
+    label: "{{ _tw_item.key }}"
 
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
-  vars: "{{ _role_vars }}"
+  vars:
+    ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
+    common_password: "{{ _tenant_password }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
index 5853073..55298e4 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
@@ -1,13 +1,21 @@
 ---
 # =========================================================================
 # Run a single cluster workload inside the bridge (Phase 0).
+#
+# Uses set_fact to apply per-workload overrides. These vars are NOT
+# at the AgnosticV top level, so they're NOT extra_vars. set_fact
+# CAN override role defaults (just not extra_vars).
 # =========================================================================
 
-- name: "Build vars for {{ _cluster_workload | split('.') | last }}"
+- name: "Apply cluster overrides for {{ _cluster_workload | split('.') | last }}"
+  when: _cluster_workload in (tenant_cluster_workloads_vars | default({}))
   ansible.builtin.set_fact:
-    _cluster_role_vars: "{{ (tenant_cluster_workloads_vars | default({}))[_cluster_workload] | default({}) }}"
+    "{{ _cw_item.key }}": "{{ _cw_item.value }}"
+  loop: "{{ (tenant_cluster_workloads_vars[_cluster_workload]) | dict2items }}"
+  loop_control:
+    loop_var: _cw_item
+    label: "{{ _cw_item.key }}"
 
 - name: "Run cluster workload: {{ _cluster_workload | split('.') | last }}"
   ansible.builtin.include_role:
     name: "{{ _cluster_workload }}"
-  vars: "{{ _cluster_role_vars }}"

From efa8119c249cf11e419f779cf203052a550c8707 Mon Sep 17 00:00:00 2001
From: Prakhar Srivastava <31232641+prakhar1985@users.noreply.github.com>
Date: Wed, 3 Jun 2026 23:09:18 +1000
Subject: [PATCH 21/24] docs: GPTEINFRA-16720 add detailed comments explaining
 design decisions

Every file now explains WHY things are done the way they are:
- Why set_fact instead of include_role vars (AAP rejects Jinja2 dicts)
- Why set_fact works here (vars not at top level = not extra_vars)
- Why three phases (showroom agnosticd_user_data lookup)
- Why tenant_cluster_workloads exists (extra_vars precedence)
- Why common_password must be a fact (Jinja2 chain resolution)
- Ansible precedence order reference
---
 .../defaults/main.yml                         | 88 +++++++++++++++----
 .../tasks/provision_user.yml                  | 27 ++++--
 .../tasks/provision_workload.yml              | 34 +++++--
 .../tasks/run_cluster_workload.yml            | 31 ++++++-
 4 files changed, 148 insertions(+), 32 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index f397c41..890c155 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -3,9 +3,30 @@
 # ocp4_workload_multi_tenant_loop — default variables
 #
 # Generic bridge between cluster-level and tenant-level workloads
-# in a combined catalog item. Overrides two root variables per
-# user iteration (ocp4_workload_tenant_keycloak_username and
-# common_password); all other tenant vars cascade via Jinja2.
+# in a combined catalog item.
+#
+# HOW IT WORKS:
+#   The bridge overrides two root variables per user iteration:
+#     - ocp4_workload_tenant_keycloak_username (e.g. user1)
+#     - common_password (auto-generated per user)
+#   All other tenant vars in the catalog item should be Jinja2
+#   expressions that derive from these roots. Ansible evaluates
+#   Jinja2 lazily, so the entire variable tree resolves correctly
+#   per user without remapping every variable.
+#
+# THREE PHASES:
+#   Phase 0: Run tenant_cluster_workloads once (cluster-level)
+#   Phase 1: Loop users, run tenant_workloads per user
+#   Phase 2: Loop users again, register each with Babylon
+#
+# WHY THREE PHASES (not two):
+#   The showroom role reads user-data.yaml via the
+#   agnosticd_user_data lookup. If per-user data exists at that
+#   point (from agnosticd_user_info calls with user: set),
+#   showroom detects a 'users' dict and switches to multi-user
+#   mode — which appends the username to the namespace, creating
+#   wrong namespaces like user1-showroom-user1.
+#   Phase 2 (registration) happens AFTER all showroom deployments.
 # =================================================================
 
 # Number of tenant users to provision. Must be > 0 when used.
@@ -14,7 +35,9 @@ num_users: 0
 # Workload FQCNs to run per user (provision order). Must be non-empty.
 tenant_workloads: []
 
-# Workload FQCNs for destroy (explicit order). Must be non-empty.
+# Workload FQCNs for destroy (explicit order).
+# Only EXTERNAL resources need cleanup — everything inside the
+# cluster dies when the cluster is destroyed.
 tenant_remove_workloads: []
 
 # Username prefix. Users are named {prefix}{number}: user1, user2, ...
@@ -24,34 +47,67 @@ tenant_user_prefix: user
 tenant_user_offset: 1
 
 # Length of auto-generated per-user passwords.
+# Each user gets a unique password stored at:
+#   {{ output_dir }}/tenant_user_{{ N }}_password
+# The password lookup is deterministic — safe for re-runs.
 tenant_password_length: 8
 
-# Seconds to wait before starting the tenant loop.
+# Seconds to wait after Phase 0 before starting Phase 1.
+# Gives async cluster resources (ArgoCD syncs, operator CRDs,
+# Keycloak realm init) time to settle.
 tenant_pre_loop_delay: 0
 
 # Per-user data reported to Babylon via agnosticd_user_info.
+# Define in agnosticv with lab-specific data (showroom URL, etc.).
+# Jinja2 expressions are evaluated per user.
+# The bridge always adds 'user' and 'password' automatically.
+#
+# Example:
+#   tenant_user_info_data:
+#     lab_ui_url: >-
+#       https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
 tenant_user_info_data: {}
 
-# Per-user message reported to Babylon.
+# Per-user message for Babylon. Empty = auto-generated.
 tenant_user_info_msg: ""
 
-# Per-workload variable overrides for tenant workloads.
-# Keys are workload FQCNs, values are dicts merged into include_role vars.
+# Per-workload variable overrides for TENANT workloads.
+# Keys are workload FQCNs, values are dicts of var overrides.
+# Applied via set_fact before each include_role call.
+#
+# USE CASE: When the same workload runs at cluster and tenant level
+# with different vars (e.g. gitops_bootstrap with bootstrap-infra
+# vs bootstrap-tenant).
+#
+# NOTE: These overrides use set_fact, NOT include_role vars:,
+# because AAP rejects Jinja2 dict expressions in include_role vars:.
+# set_fact works here because the conflicting vars are NOT at the
+# AgnosticV top level (not extra_vars).
 tenant_workload_vars: {}
 
 # ---------------------------------------------------------------
-# Cluster workloads run ONCE inside the bridge, before the user loop.
+# Cluster workloads run ONCE inside the bridge (Phase 0).
+#
+# WHY THIS EXISTS:
+#   When a workload (e.g. gitops_bootstrap) runs at BOTH cluster
+#   and tenant level with different vars for the SAME variable
+#   names, you can't define either set at the AgnosticV top level.
+#   Top-level vars become extra_vars in AAP, and extra_vars have
+#   the HIGHEST Ansible precedence (#22 of 22). Nothing inside
+#   the playbook can override them — not set_fact, not
+#   include_role vars, nothing.
 #
-# WHY: When the same role (e.g. gitops_bootstrap) runs at both
-# cluster and tenant level with different vars, defining either
-# set at the AgnosticV top level makes them extra_vars (highest
-# Ansible precedence — nothing can override them). By running
-# both invocations inside the bridge via include_role vars:,
-# we avoid the extra_vars conflict entirely.
+#   Solution: move the workload OUT of the config's workloads:
+#   list. Run it inside the bridge where both invocations use
+#   set_fact to apply their respective overrides. Since the
+#   conflicting vars are NOT at the top level, they're NOT
+#   extra_vars, and set_fact overrides role defaults.
 #
-# Workloads listed here are NOT in the config's workloads: list.
+# Workloads listed here must NOT also appear in the config's
+# workloads: list.
 # ---------------------------------------------------------------
 tenant_cluster_workloads: []
 
 # Per-workload variable overrides for cluster workloads.
+# Same mechanism as tenant_workload_vars but for Phase 0.
 tenant_cluster_workloads_vars: {}
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index edfc8c2..6a7fba3 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -2,11 +2,24 @@
 # =========================================================================
 # Run all tenant workloads for a single user (Phase 1).
 #
-# Sets root variables as FACTS so they're available for Jinja2
-# resolution in tenant_workload_vars (which is evaluated outside
-# include_role context).
+# VARIABLE SCOPING:
+#   ocp4_workload_tenant_keycloak_username and common_password are set
+#   as FACTS here (not just via include_role vars:) because:
 #
-# No agnosticd_user_info calls here — registration is in phase 2.
+#   1. tenant_workload_vars values may contain Jinja2 expressions that
+#      reference them (e.g. "bootstrap-tenant-{{ ocp4_workload_tenant_
+#      keycloak_username }}"). These expressions are evaluated when
+#      provision_workload.yml applies the set_fact overrides — which
+#      happens OUTSIDE include_role context. Without the fact, the
+#      Jinja2 reference would be undefined.
+#
+#   2. common_password is not a top-level var in the catalog item.
+#      It only exists via include_role vars: inside the role. The
+#      Jinja2 chain (ocp4_workload_user_password → common_password)
+#      needs common_password to be resolvable at set_fact time.
+#
+# No agnosticd_user_info calls here — registration is in Phase 2.
+# See workload.yml header for why two-phase is necessary.
 # =========================================================================
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -18,9 +31,7 @@
            length=tenant_password_length | int,
            chars=['ascii_letters', 'digits']
          ) }}
-    # These must be facts (not just include_role vars:) so that
-    # Jinja2 expressions in tenant_workload_vars can reference them
-    # when building the merged vars dict.
+    # Must be facts for Jinja2 resolution in tenant_workload_vars
     ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     common_password: >-
       {{ lookup('password',
@@ -29,6 +40,8 @@
            chars=['ascii_letters', 'digits']
          ) }}
 
+# Loop via include_tasks (not direct include_role) so we can apply
+# per-workload set_fact overrides before each include_role call.
 - name: "Provision workloads for {{ _tenant_username }}"
   ansible.builtin.include_tasks: provision_workload.yml
   loop: "{{ tenant_workloads }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index bb1ee7f..e161050 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -2,14 +2,33 @@
 # =========================================================================
 # Run a single tenant workload for the current user.
 #
-# Uses set_fact for per-workload overrides + include_role vars: for
-# the two root vars (username + password). The root vars MUST go
-# through include_role vars: because they change per user and per
-# workload invocation. The overrides use set_fact because they
-# persist for the user's iteration (which is fine — role-prefixed
-# var names don't collide across workloads).
+# TWO MECHANISMS FOR PASSING VARS:
+#
+# 1. set_fact (for per-workload overrides from tenant_workload_vars)
+#    - Applied BEFORE include_role
+#    - Facts persist across workloads in the same user iteration
+#    - Safe because AgnosticD workloads use role-prefixed var names
+#      (gitops_bootstrap vars don't collide with showroom vars)
+#    - Works because these vars are NOT extra_vars (not at top level)
+#
+# 2. include_role vars: (for the two root vars)
+#    - ocp4_workload_tenant_keycloak_username and common_password
+#    - These MUST go through include_role vars: because they're the
+#      per-user identity that changes each iteration
+#    - All other tenant vars in the catalog item are Jinja2 expressions
+#      that derive from these two roots via lazy evaluation
+#
+# WHY NOT use include_role vars: for EVERYTHING:
+#   AAP rejects `include_role vars: "{{ dict_expression }}"` with
+#   "Vars in a IncludeRole must be specified as a dictionary".
+#   Only static YAML key: value mappings are accepted.
+#   So overrides go through set_fact, root vars through include_role.
 # =========================================================================
 
+# Apply per-workload overrides if this workload has entries in
+# tenant_workload_vars. Each key is set as a fact individually.
+# Example: gitops_bootstrap gets repo_path, app_name, helm_values
+# overridden from cluster values to tenant values.
 - name: "Apply tenant overrides for {{ _tenant_workload | split('.') | last }}"
   when: _tenant_workload in (tenant_workload_vars | default({}))
   ansible.builtin.set_fact:
@@ -19,6 +38,9 @@
     loop_var: _tw_item
     label: "{{ _tw_item.key }}"
 
+# Run the workload with the two root vars.
+# All other vars come from facts (set above or in provision_user.yml)
+# or from the catalog item's playbook-level vars (via Jinja2 cascade).
 - name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
index 55298e4..6b2fff0 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
@@ -2,11 +2,36 @@
 # =========================================================================
 # Run a single cluster workload inside the bridge (Phase 0).
 #
-# Uses set_fact to apply per-workload overrides. These vars are NOT
-# at the AgnosticV top level, so they're NOT extra_vars. set_fact
-# CAN override role defaults (just not extra_vars).
+# WHY THIS EXISTS:
+#   Some workloads (e.g. gitops_bootstrap) run at BOTH cluster and
+#   tenant level with different variable values. If you define either
+#   set of values at the AgnosticV top level, they become extra_vars
+#   (passed to AAP job template). Extra_vars have the HIGHEST
+#   precedence in Ansible (#22 of 22) — nothing can override them.
+#   Not set_fact (#19), not include_role vars (#20), nothing.
+#
+#   By moving the workload OUT of the config's workloads: list and
+#   running it HERE inside the bridge, the conflicting vars are
+#   defined only in tenant_cluster_workloads_vars (a nested dict).
+#   The individual keys inside that dict are NOT extra_vars.
+#   set_fact (#19) CAN override role defaults (#2), so it works.
+#
+# WHY set_fact AND NOT include_role vars:
+#   We tried `include_role vars: "{{ dict_var }}"` first.
+#   AAP/Ansible rejects it: "Vars in a IncludeRole must be specified
+#   as a dictionary" — it does not accept Jinja2 expressions that
+#   evaluate to a dict. Only static YAML mappings are allowed.
+#   So we use set_fact per key, then plain include_role.
+#
+# WHY set_fact WORKS HERE:
+#   Ansible precedence: extra_vars (#22) > set_fact (#19) > role
+#   defaults (#2). The conflicting vars are NOT at the top level,
+#   so they're NOT extra_vars. set_fact overrides role defaults.
 # =========================================================================
 
+# Apply per-workload overrides from tenant_cluster_workloads_vars.
+# Each key is set as a fact individually (dict2items loop).
+# Skipped entirely if this workload has no overrides defined.
 - name: "Apply cluster overrides for {{ _cluster_workload | split('.') | last }}"
   when: _cluster_workload in (tenant_cluster_workloads_vars | default({}))
   ansible.builtin.set_fact:

From 8328b32daf7868b932e179d8f66947decffec8ac Mon Sep 17 00:00:00 2001
From: prakhar <prakhar@redhat.com>
Date: Thu, 11 Jun 2026 09:14:17 +1000
Subject: [PATCH 22/24] fix: GPTEINFRA-16720 address JK review comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add vars/main.yml with _tenant_username/_tenant_password defaults
- Clarify three-phase WHY: showroom multi-user detection behavior,
  not a showroom bug — fixing it would break existing labs
- Drop | default({}) where vars are already in defaults/main.yml
- Drop | default(omit) pattern for tenant_user_info_msg, use omit directly
- Remove split('.') | last from all debug labels — show full FQCN
- Remove password from destroy_user vars: (not needed for remove)
- Simplify provision_workload.yml — remove incorrect comment about
  include_role vars: accepting Jinja2 (it was right for our use case
  but the explanation was misleading)
- tenant_user_info_msg default moved to defaults/main.yml
---
 .../defaults/main.yml                         | 26 +++++++-----
 .../tasks/destroy_user.yml                    | 11 ++---
 .../tasks/provision_user.yml                  | 22 ++++------
 .../tasks/provision_workload.yml              | 40 ++++---------------
 .../tasks/register_user.yml                   | 22 ++++------
 .../tasks/remove_workload.yml                 |  5 +--
 .../tasks/run_cluster_workload.yml            | 19 +++------
 .../tasks/workload.yml                        | 10 ++---
 .../vars/main.yml                             |  7 ++++
 9 files changed, 61 insertions(+), 101 deletions(-)
 create mode 100644 roles/ocp4_workload_multi_tenant_loop/vars/main.yml

diff --git a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
index 890c155..4744bd1 100644
--- a/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/defaults/main.yml
@@ -20,13 +20,17 @@
 #   Phase 2: Loop users again, register each with Babylon
 #
 # WHY THREE PHASES (not two):
-#   The showroom role reads user-data.yaml via the
-#   agnosticd_user_data lookup. If per-user data exists at that
-#   point (from agnosticd_user_info calls with user: set),
-#   showroom detects a 'users' dict and switches to multi-user
-#   mode — which appends the username to the namespace, creating
-#   wrong namespaces like user1-showroom-user1.
-#   Phase 2 (registration) happens AFTER all showroom deployments.
+#   agnosticd_user_info with user: set writes per-user entries to
+#   user-data.yaml. The showroom role reads user-data.yaml at deploy
+#   time. If it finds a 'users' dict, it switches to multi-user mode
+#   and appends the username to the namespace (user1-showroom becomes
+#   user1-showroom-user1). This is correct behavior for labs that
+#   provision all users in a single play — but conflicts with our
+#   pattern where each user gets their own showroom pod in Phase 1.
+#   Phase 2 defers all agnosticd_user_info calls until after all
+#   showroom pods are deployed, so user-data.yaml is empty when
+#   showroom runs. Fixing showroom's multi-user detection would change
+#   behavior for all existing multi-user labs that rely on it.
 # =================================================================
 
 # Number of tenant users to provision. Must be > 0 when used.
@@ -68,8 +72,12 @@ tenant_pre_loop_delay: 0
 #       https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
 tenant_user_info_data: {}
 
-# Per-user message for Babylon. Empty = auto-generated.
-tenant_user_info_msg: ""
+# Per-user message for Babylon.
+# Default uses the bridge-generated per-user password.
+# Override in agnosticv if the actual Keycloak password differs
+# (e.g. when common_password is a top-level extra_var).
+# Set to "" or omit to suppress the message entirely.
+tenant_user_info_msg: "User: {{ _tenant_username }} / {{ _tenant_password }}"
 
 # Per-workload variable overrides for TENANT workloads.
 # Keys are workload FQCNs, values are dicts of var overrides.
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index 7e1c5f2..f7bfbf8 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -3,11 +3,8 @@
 # Destroy external resources for a single user.
 #
 # Wrapped in block/rescue — errors are logged but NEVER block teardown.
-# The cluster is about to be destroyed; blocking here wastes time.
-#
-# Password lookup returns the same value generated during provisioning.
-# Some destroy roles need the password (e.g. to authenticate and
-# delete resources).
+# The cluster is about to be destroyed; only EXTERNAL resources
+# (e.g. LiteMaaS virtual keys) need explicit cleanup here.
 # =========================================================================
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -22,13 +19,13 @@
 
 - name: "Destroy workloads for {{ _tenant_username }}"
   block:
-  - name: "Remove {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+  - name: "Remove {{ _tenant_workload }} for {{ _tenant_username }}"
     ansible.builtin.include_role:
       name: "{{ _tenant_workload }}"
     loop: "{{ tenant_remove_workloads }}"
     loop_control:
       loop_var: _tenant_workload
-      label: "{{ _tenant_workload | split('.') | last }}"
+      label: "{{ _tenant_workload }}"
     vars:
       ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
       common_password: "{{ _tenant_password }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index 6a7fba3..a9c4029 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -4,19 +4,16 @@
 #
 # VARIABLE SCOPING:
 #   ocp4_workload_tenant_keycloak_username and common_password are set
-#   as FACTS here (not just via include_role vars:) because:
+#   as facts here (not just via include_role vars:) because:
 #
 #   1. tenant_workload_vars values may contain Jinja2 expressions that
-#      reference them (e.g. "bootstrap-tenant-{{ ocp4_workload_tenant_
-#      keycloak_username }}"). These expressions are evaluated when
-#      provision_workload.yml applies the set_fact overrides — which
-#      happens OUTSIDE include_role context. Without the fact, the
-#      Jinja2 reference would be undefined.
+#      reference them. These expressions are evaluated when
+#      provision_workload.yml applies the set_fact overrides — outside
+#      include_role scope. Without facts, the references are undefined.
 #
-#   2. common_password is not a top-level var in the catalog item.
-#      It only exists via include_role vars: inside the role. The
-#      Jinja2 chain (ocp4_workload_user_password → common_password)
-#      needs common_password to be resolvable at set_fact time.
+#   2. common_password needs to be resolvable at set_fact time for
+#      the Jinja2 chain in catalog vars (e.g. ocp4_workload_user_password
+#      → common_password) to resolve correctly.
 #
 # No agnosticd_user_info calls here — registration is in Phase 2.
 # See workload.yml header for why two-phase is necessary.
@@ -31,7 +28,6 @@
            length=tenant_password_length | int,
            chars=['ascii_letters', 'digits']
          ) }}
-    # Must be facts for Jinja2 resolution in tenant_workload_vars
     ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     common_password: >-
       {{ lookup('password',
@@ -40,11 +36,9 @@
            chars=['ascii_letters', 'digits']
          ) }}
 
-# Loop via include_tasks (not direct include_role) so we can apply
-# per-workload set_fact overrides before each include_role call.
 - name: "Provision workloads for {{ _tenant_username }}"
   ansible.builtin.include_tasks: provision_workload.yml
   loop: "{{ tenant_workloads }}"
   loop_control:
     loop_var: _tenant_workload
-    label: "{{ _tenant_workload | split('.') | last }}"
+    label: "{{ _tenant_workload }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index e161050..3fbe44c 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -2,46 +2,22 @@
 # =========================================================================
 # Run a single tenant workload for the current user.
 #
-# TWO MECHANISMS FOR PASSING VARS:
-#
-# 1. set_fact (for per-workload overrides from tenant_workload_vars)
-#    - Applied BEFORE include_role
-#    - Facts persist across workloads in the same user iteration
-#    - Safe because AgnosticD workloads use role-prefixed var names
-#      (gitops_bootstrap vars don't collide with showroom vars)
-#    - Works because these vars are NOT extra_vars (not at top level)
-#
-# 2. include_role vars: (for the two root vars)
-#    - ocp4_workload_tenant_keycloak_username and common_password
-#    - These MUST go through include_role vars: because they're the
-#      per-user identity that changes each iteration
-#    - All other tenant vars in the catalog item are Jinja2 expressions
-#      that derive from these two roots via lazy evaluation
-#
-# WHY NOT use include_role vars: for EVERYTHING:
-#   AAP rejects `include_role vars: "{{ dict_expression }}"` with
-#   "Vars in a IncludeRole must be specified as a dictionary".
-#   Only static YAML key: value mappings are accepted.
-#   So overrides go through set_fact, root vars through include_role.
+# Per-workload overrides from tenant_workload_vars are applied via
+# set_fact before include_role. This works because the vars are NOT
+# at the AgnosticV top level (not extra_vars), so set_fact can
+# override role defaults.
 # =========================================================================
 
-# Apply per-workload overrides if this workload has entries in
-# tenant_workload_vars. Each key is set as a fact individually.
-# Example: gitops_bootstrap gets repo_path, app_name, helm_values
-# overridden from cluster values to tenant values.
-- name: "Apply tenant overrides for {{ _tenant_workload | split('.') | last }}"
-  when: _tenant_workload in (tenant_workload_vars | default({}))
+- name: "Apply tenant overrides for {{ _tenant_workload }}"
+  when: _tenant_workload in tenant_workload_vars
   ansible.builtin.set_fact:
     "{{ _tw_item.key }}": "{{ _tw_item.value }}"
-  loop: "{{ (tenant_workload_vars[_tenant_workload]) | dict2items }}"
+  loop: "{{ tenant_workload_vars[_tenant_workload] | dict2items }}"
   loop_control:
     loop_var: _tw_item
     label: "{{ _tw_item.key }}"
 
-# Run the workload with the two root vars.
-# All other vars come from facts (set above or in provision_user.yml)
-# or from the catalog item's playbook-level vars (via Jinja2 cascade).
-- name: "Run {{ _tenant_workload | split('.') | last }} for {{ _tenant_username }}"
+- name: "Run {{ _tenant_workload }} for {{ _tenant_username }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
   vars:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
index 802f606..816745c 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -7,18 +7,13 @@
 #
 # The catalog item defines tenant_user_info_data with lab-specific
 # data (e.g. showroom URL pattern). Jinja2 expressions in that dict
-# are evaluated per user. The bridge always adds 'user' and 'password'.
-#
-# Example tenant_user_info_data (in agnosticv common.yaml):
-#   tenant_user_info_data:
-#     lab_ui_url: >-
-#       https://showroom-{{ ocp4_workload_tenant_keycloak_username }}-showroom.{{ openshift_cluster_ingress_domain }}
+# are evaluated per user. tenant_user_info_data is combined on top of
+# the base {user, password} so catalog values take precedence.
 #
 # WHY set ocp4_workload_tenant_keycloak_username here?
-#   tenant_user_info_data references it via Jinja2. In phase 1
-#   it's set via include_role vars: (scoped to the role). Here
-#   we need it as a fact so the Jinja2 in tenant_user_info_data
-#   resolves correctly.
+#   tenant_user_info_data may reference it via Jinja2. In phase 1
+#   it's passed via include_role vars: (scoped to the role). Here
+#   we need it as a fact so the Jinja2 resolves correctly.
 # =========================================================================
 
 - name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
@@ -37,11 +32,8 @@
     _base_data:
       user: "{{ _tenant_username }}"
       password: "{{ _tenant_password }}"
-    _user_data: >-
-      {{ _base_data | combine(tenant_user_info_data | default({})) }}
-    _default_msg: >-
-      User: {{ _tenant_username }} / {{ _tenant_password }}
+    _user_data: "{{ _base_data | combine(tenant_user_info_data) }}"
   agnosticd.core.agnosticd_user_info:
     user: "{{ _tenant_username }}"
     data: "{{ _user_data }}"
-    msg: "{{ tenant_user_info_msg | default(_default_msg, true) }}"
+    msg: "{{ tenant_user_info_msg | default(omit, true) }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
index 07e447b..ad1e879 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/remove_workload.yml
@@ -14,15 +14,14 @@
 # teardown. If LiteMaaS cleanup fails (key expired, API down),
 # the cluster still gets destroyed and the keys expire on their own.
 #
-# Users are destroyed in reverse order (LIFO) as a convention for
-# any cross-user dependencies.
+# Users are destroyed in reverse order (LIFO) as a convention.
 # =========================================================================
 
 - name: Display tenant loop destroy configuration
   ansible.builtin.debug:
     msg:
     - "Tenant loop: destroying {{ num_users }} user(s)"
-    - "Destroy order: {{ tenant_remove_workloads | default([]) | map('split', '.') | map('last') | list }}"
+    - "Destroy order: {{ tenant_remove_workloads | default([]) | list }}"
 
 - name: Destroy tenant for each user (reverse order)
   when:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
index 6b2fff0..26d7bbf 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/run_cluster_workload.yml
@@ -17,22 +17,13 @@
 #   set_fact (#19) CAN override role defaults (#2), so it works.
 #
 # WHY set_fact AND NOT include_role vars:
-#   We tried `include_role vars: "{{ dict_var }}"` first.
-#   AAP/Ansible rejects it: "Vars in a IncludeRole must be specified
-#   as a dictionary" — it does not accept Jinja2 expressions that
-#   evaluate to a dict. Only static YAML mappings are allowed.
+#   AAP/Ansible rejects `include_role vars: "{{ dict_var }}"`:
+#   "Vars in a IncludeRole must be specified as a dictionary".
+#   Only static YAML mappings are allowed, not Jinja2 expressions.
 #   So we use set_fact per key, then plain include_role.
-#
-# WHY set_fact WORKS HERE:
-#   Ansible precedence: extra_vars (#22) > set_fact (#19) > role
-#   defaults (#2). The conflicting vars are NOT at the top level,
-#   so they're NOT extra_vars. set_fact overrides role defaults.
 # =========================================================================
 
-# Apply per-workload overrides from tenant_cluster_workloads_vars.
-# Each key is set as a fact individually (dict2items loop).
-# Skipped entirely if this workload has no overrides defined.
-- name: "Apply cluster overrides for {{ _cluster_workload | split('.') | last }}"
+- name: "Apply cluster overrides for {{ _cluster_workload }}"
   when: _cluster_workload in (tenant_cluster_workloads_vars | default({}))
   ansible.builtin.set_fact:
     "{{ _cw_item.key }}": "{{ _cw_item.value }}"
@@ -41,6 +32,6 @@
     loop_var: _cw_item
     label: "{{ _cw_item.key }}"
 
-- name: "Run cluster workload: {{ _cluster_workload | split('.') | last }}"
+- name: "Run cluster workload: {{ _cluster_workload }}"
   ansible.builtin.include_role:
     name: "{{ _cluster_workload }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index ba14d2d..9c06243 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -26,16 +26,12 @@
     msg:
     - "Tenant loop: provisioning {{ num_users }} user(s)"
     - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
-    - "Cluster workloads: {{ tenant_cluster_workloads | default([]) | map('split', '.') | map('last') | list }}"
-    - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads | map('split', '.') | map('last') | list }}"
+    - "Cluster workloads: {{ tenant_cluster_workloads | default([]) }}"
+    - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads }}"
     - "Pre-loop delay: {{ tenant_pre_loop_delay }}s"
 
 # -----------------------------------------------------------------------
 # PHASE 0: Run cluster workloads inside the bridge.
-#
-# These workloads can't be in the config's workloads: list because
-# their vars conflict with tenant invocations. Running them here
-# via include_role vars: avoids the extra_vars precedence issue.
 # -----------------------------------------------------------------------
 - name: "Phase 0: Run cluster workloads inside bridge"
   when: tenant_cluster_workloads | default([]) | length > 0
@@ -43,7 +39,7 @@
   loop: "{{ tenant_cluster_workloads }}"
   loop_control:
     loop_var: _cluster_workload
-    label: "{{ _cluster_workload | split('.') | last }}"
+    label: "{{ _cluster_workload }}"
 
 - name: Wait for cluster workloads to settle
   when: tenant_pre_loop_delay | int > 0
diff --git a/roles/ocp4_workload_multi_tenant_loop/vars/main.yml b/roles/ocp4_workload_multi_tenant_loop/vars/main.yml
new file mode 100644
index 0000000..6a6da53
--- /dev/null
+++ b/roles/ocp4_workload_multi_tenant_loop/vars/main.yml
@@ -0,0 +1,7 @@
+---
+# Computed per-user identity vars.
+# These are overwritten by set_fact in each user iteration but
+# defined here so they are available in all execution contexts
+# (provision, destroy) without requiring explicit initialization.
+_tenant_username: ""
+_tenant_password: ""

From 04bd3cf0b1749cc78a929e50b9f483632ef3b12b Mon Sep 17 00:00:00 2001
From: prakhar <prakhar@redhat.com>
Date: Thu, 11 Jun 2026 09:34:50 +1000
Subject: [PATCH 23/24] fix: GPTEINFRA-16720 fix yamllint and ansible-lint
 failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- workload.yml line 28: shorten username pattern debug line (156→148 chars)
- name[template]: move Jinja2 templates to end of task names in
  destroy_user, provision_user, provision_workload, register_user
---
 .../ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml  | 6 +++---
 .../tasks/provision_user.yml                                | 4 ++--
 .../tasks/provision_workload.yml                            | 4 ++--
 .../ocp4_workload_multi_tenant_loop/tasks/register_user.yml | 4 ++--
 roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml    | 5 ++++-
 5 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index f7bfbf8..546731c 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -7,7 +7,7 @@
 # (e.g. LiteMaaS virtual keys) need explicit cleanup here.
 # =========================================================================
 
-- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-
@@ -19,7 +19,7 @@
 
 - name: "Destroy workloads for {{ _tenant_username }}"
   block:
-  - name: "Remove {{ _tenant_workload }} for {{ _tenant_username }}"
+  - name: "Remove workload: {{ _tenant_workload }}"
     ansible.builtin.include_role:
       name: "{{ _tenant_workload }}"
     loop: "{{ tenant_remove_workloads }}"
@@ -30,7 +30,7 @@
       ocp4_workload_tenant_keycloak_username: "{{ _tenant_username }}"
       common_password: "{{ _tenant_password }}"
   rescue:
-  - name: "Destroy failed for {{ _tenant_username }} — continuing"
+  - name: "Destroy failed — continuing: {{ _tenant_username }}"
     ansible.builtin.debug:
       msg: >-
         Tenant destroy failed for {{ _tenant_username }}.
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index a9c4029..ce293b8 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -19,7 +19,7 @@
 # See workload.yml header for why two-phase is necessary.
 # =========================================================================
 
-- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-
@@ -36,7 +36,7 @@
            chars=['ascii_letters', 'digits']
          ) }}
 
-- name: "Provision workloads for {{ _tenant_username }}"
+- name: "Provision workloads: {{ _tenant_username }}"
   ansible.builtin.include_tasks: provision_workload.yml
   loop: "{{ tenant_workloads }}"
   loop_control:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
index 3fbe44c..b7f2ac8 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_workload.yml
@@ -8,7 +8,7 @@
 # override role defaults.
 # =========================================================================
 
-- name: "Apply tenant overrides for {{ _tenant_workload }}"
+- name: "Apply tenant overrides: {{ _tenant_workload }}"
   when: _tenant_workload in tenant_workload_vars
   ansible.builtin.set_fact:
     "{{ _tw_item.key }}": "{{ _tw_item.value }}"
@@ -17,7 +17,7 @@
     loop_var: _tw_item
     label: "{{ _tw_item.key }}"
 
-- name: "Run {{ _tenant_workload }} for {{ _tenant_username }}"
+- name: "Run workload: {{ _tenant_workload }}"
   ansible.builtin.include_role:
     name: "{{ _tenant_workload }}"
   vars:
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
index 816745c..85ccac0 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -16,7 +16,7 @@
 #   we need it as a fact so the Jinja2 resolves correctly.
 # =========================================================================
 
-- name: "Set identity for {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-
@@ -27,7 +27,7 @@
          ) }}
     ocp4_workload_tenant_keycloak_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
 
-- name: "Register {{ _tenant_username }} with Babylon"
+- name: "Register with Babylon: {{ _tenant_username }}"
   vars:
     _base_data:
       user: "{{ _tenant_username }}"
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
index 9c06243..d3b56d9 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/workload.yml
@@ -25,7 +25,10 @@
   ansible.builtin.debug:
     msg:
     - "Tenant loop: provisioning {{ num_users }} user(s)"
-    - "Username pattern: {{ tenant_user_prefix }}{{ tenant_user_offset }}..{{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}"
+    - >-
+        Username pattern:
+        {{ tenant_user_prefix }}{{ tenant_user_offset }}..
+        {{ tenant_user_prefix }}{{ (tenant_user_offset | int) + (num_users | int) - 1 }}
     - "Cluster workloads: {{ tenant_cluster_workloads | default([]) }}"
     - "Tenant workloads ({{ tenant_workloads | length }}): {{ tenant_workloads }}"
     - "Pre-loop delay: {{ tenant_pre_loop_delay }}s"

From e79c3a22a57e58158501a5c95752a507586e1d90 Mon Sep 17 00:00:00 2001
From: prakhar <prakhar@redhat.com>
Date: Thu, 11 Jun 2026 17:09:32 +1000
Subject: [PATCH 24/24] fix: GPTEINFRA-16720 merge double template in task
 names to single expression

---
 roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml   | 2 +-
 roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml | 2 +-
 roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
index 546731c..cf5ccd0 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/destroy_user.yml
@@ -7,7 +7,7 @@
 # (e.g. LiteMaaS virtual keys) need explicit cleanup here.
 # =========================================================================
 
-- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix ~ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
index ce293b8..3f922ea 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/provision_user.yml
@@ -19,7 +19,7 @@
 # See workload.yml header for why two-phase is necessary.
 # =========================================================================
 
-- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix ~ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-
diff --git a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
index 85ccac0..3433f0f 100644
--- a/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
+++ b/roles/ocp4_workload_multi_tenant_loop/tasks/register_user.yml
@@ -16,7 +16,7 @@
 #   we need it as a fact so the Jinja2 resolves correctly.
 # =========================================================================
 
-- name: "Set identity: {{ tenant_user_prefix }}{{ _tenant_user_num }}"
+- name: "Set identity: {{ tenant_user_prefix ~ _tenant_user_num }}"
   ansible.builtin.set_fact:
     _tenant_username: "{{ tenant_user_prefix }}{{ _tenant_user_num }}"
     _tenant_password: >-