Let one repo deploy several components, each with its own state

.github/workflows/apply.yml

@@ -26,9 +26,13 @@ on:
         default: ${{ github.repository }}
         type: string
       github_repository_name:
-        description: "GitHub repository name (without the owner) used to locate the statefile and drift file. Defaults to the repository that initiated the run. Override when a workflow in one repository needs to operate on a different repository's Terraform code."
+        description: "GitHub repository name (without the owner) used to locate the statefile and drift file when component_name is empty. Defaults to the repository that initiated the run. Override when a workflow in one repository needs to operate on a different repository's Terraform code."
         default: ${{ github.event.repository.name }}
         type: string
+      component_name:
+        description: "Name of the thing being deployed. Used for the statefile path, the drift file path, and the Kosli flow name. Both paths are under terraform/<name>/. Empty keeps the repository-name defaults. Set it when one repository deploys several components, each with its own state."
+        default: ""
+        type: string
       ref:
         description: "Git reference (branch, tag or SHA) to check out. Defaults to the default branch of the checked-out repository."
         default: ""
@@ -94,10 +98,11 @@ jobs:
       github_repository_to_checkout: ${{ inputs.github_repository_to_checkout }}
       ref: ${{ inputs.ref }}
       tf_vars: ${{ inputs.tf_vars }}
+      component_name: ${{ inputs.component_name }}
       kosli_template_file: ${{ inputs.kosli_template_file }}
       kosli_host: ${{ inputs.kosli_host }}
       kosli_org: ${{ inputs.kosli_org }}
-      kosli_flow: terraform-apply-${{ inputs.environment }}-${{ github.event.repository.name }}
+      kosli_flow: terraform-apply-${{ inputs.environment }}-${{ inputs.component_name || github.event.repository.name }}
       kosli_cli_version: ${{ inputs.kosli_cli_version }}
       pass_commit_to_kosli_commands: ${{ inputs.pass_commit_to_kosli_commands }}
       job_timeout_minutes: ${{ inputs.job_timeout_minutes }}
@@ -120,6 +125,7 @@ jobs:
       KOSLI_TRAIL: ${{ needs.apply.outputs.kosli_trail }}
       COMMIT_ARG: ${{ inputs.pass_commit_to_kosli_commands && format('--commit {0}', inputs.ref) || '' }}
       GH_TOKEN: ${{ secrets.kosli_github_token }}
+      STATE_DIR: terraform/${{ inputs.component_name || inputs.github_repository_name }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -150,7 +156,7 @@ jobs:
 
       - name: Upload drift plan to S3
         run: |
-          aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/terraform/${{ inputs.github_repository_name }}/drift.plan.json"
+          aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json"
 
       - name: Upload drift plan artifact
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
@@ -169,7 +175,7 @@ jobs:
         if: inputs.kosli_template_file != ''
         run: |
           aws s3 cp \
-            "s3://${S3_BUCKET}/terraform/${{ inputs.github_repository_name }}/${{ inputs.tf_state_file_name }}" \
+            "s3://${S3_BUCKET}/${STATE_DIR}/${{ inputs.tf_state_file_name }}" \
             "/tmp/${{ inputs.tf_state_file_name }}"
 
       - name: Kosli attest state file

.github/workflows/base.yml

@@ -39,6 +39,10 @@ on:
         description: "Extra environment variables, one KEY=VALUE per line, exported before plan/apply. To set Terraform variable `foo`, use `TF_VAR_foo=...`. Single-line values only."
         default: ""
         type: string
+      component_name:
+        description: "Name of the thing being deployed, used for the statefile path terraform/<name>/. Empty uses the checked-out repository's name. Set it when one repository deploys several components, each with its own state."
+        default: ""
+        type: string
       kosli_template_file:
         description: "Path (in caller repo) to the Kosli trail template. Empty disables all Kosli steps."
         default: ""
@@ -147,6 +151,14 @@ jobs:
         with:
           version: ${{ inputs.kosli_cli_version }}
 
+      # Exported only when set, so an unset input leaves TF_COMPONENT_NAME
+      # free to come from the repo's tf.env, as it does when tf runs locally.
+      - name: Export component name
+        if: inputs.component_name != ''
+        env:
+          COMPONENT_NAME: ${{ inputs.component_name }}
+        run: printf 'TF_COMPONENT_NAME=%s\n' "$COMPONENT_NAME" >> "$GITHUB_ENV"
+
       - name: Export Terraform variables
         if: inputs.tf_vars != ''
         env:

.github/workflows/detect-drift.yml

@@ -29,11 +29,20 @@ on:
         description: "Minutes before GitHub cancels the drift plan job. Raise it together with aws_role_duration when a legitimate plan needs longer; see the README on timeouts."
         default: 30
         type: number
+      component_name:
+        description: "Name of the thing being deployed. Used for the statefile and drift file paths, terraform/<name>/. Empty uses the repository name. Must match the component_name the apply workflow was given."
+        default: ""
+        type: string
 
 concurrency:
-  group: detect-drift-${{ github.repository }}-${{ inputs.environment }}
+  # One group per component, so several components' drift runs in one
+  # repository do not replace each other's pending run.
+  group: detect-drift-${{ inputs.component_name || github.repository }}-${{ inputs.environment }}
   cancel-in-progress: false
 
+env:
+  STATE_DIR: terraform/${{ inputs.component_name || github.event.repository.name }}
+
 jobs:
   fetch-baseline:
     runs-on: ubuntu-latest
@@ -64,15 +73,15 @@ jobs:
 
       - name: Check baseline exists
         run: |
-          KEY="terraform/${{ github.event.repository.name }}/drift.plan.json"
+          KEY="${STATE_DIR}/drift.plan.json"
           if ! aws s3api head-object --bucket "$S3_BUCKET" --key "$KEY" >/dev/null 2>&1; then
             echo "::error::No drift baseline at s3://${S3_BUCKET}/${KEY} — run the apply workflow at least once to seed it."
             exit 1
           fi
 
       - name: Download baseline
         run: |
-          aws s3 cp "s3://${S3_BUCKET}/terraform/${{ github.event.repository.name }}/drift.plan.json" /tmp/drift.plan.json
+          aws s3 cp "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json" /tmp/drift.plan.json
 
       - name: Read baseline
         id: read
@@ -106,6 +115,7 @@ jobs:
       ref: ${{ needs.fetch-baseline.outputs.sha }}
       tf_apply: false
       tf_vars: ${{ inputs.tf_vars }}
+      component_name: ${{ inputs.component_name }}
       job_timeout_minutes: ${{ inputs.job_timeout_minutes }}
 
   flag-drift:
@@ -144,7 +154,7 @@ jobs:
 
       - name: Upload drift plan to S3
         run: |
-          aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/terraform/${{ github.event.repository.name }}/drift.plan.json"
+          aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json"
 
       - name: Drift summary
         run: |

.github/workflows/plan.yml

@@ -33,6 +33,10 @@ on:
         description: "Extra environment variables, one KEY=VALUE per line, exported before plan. To set Terraform variable `foo`, use `TF_VAR_foo=...`. Single-line values only."
         default: ""
         type: string
+      component_name:
+        description: "Name of the thing being deployed. Used for the statefile path (terraform/<name>/) and the Kosli flow name. Empty keeps the repository-name defaults. Set it when one repository deploys several components, each with its own state."
+        default: ""
+        type: string
       kosli_template_file:
         description: "Path (in caller repo) to the Kosli trail template. Empty disables all Kosli steps."
         default: ""
@@ -79,10 +83,11 @@ jobs:
       github_repository_to_checkout: ${{ inputs.github_repository_to_checkout }}
       ref: ${{ inputs.ref }}
       tf_vars: ${{ inputs.tf_vars }}
+      component_name: ${{ inputs.component_name }}
       kosli_template_file: ${{ inputs.kosli_template_file }}
       kosli_host: ${{ inputs.kosli_host }}
       kosli_org: ${{ inputs.kosli_org }}
-      kosli_flow: terraform-plan-${{ inputs.environment }}-${{ github.event.repository.name }}
+      kosli_flow: terraform-plan-${{ inputs.environment }}-${{ inputs.component_name || github.event.repository.name }}
       kosli_cli_version: ${{ inputs.kosli_cli_version }}
       job_timeout_minutes: ${{ inputs.job_timeout_minutes }}
     secrets:

README.md

@@ -101,6 +101,7 @@ Both `plan.yml` and `apply.yml` accept the same core inputs:
 | `tf_version` | no | `1.14.6` | Terraform version to install |
 | `tf_vars` | no | `""` | Extra env vars (one `KEY=VALUE` per line) exported before plan/apply; see [Supplying Terraform variables](#supplying-terraform-variables) |
 | `job_timeout_minutes` | no | `30` | Minutes before GitHub cancels the plan/apply job; see [Timeouts](#timeouts) |
+| `component_name` | no | `""` | Name of the thing being deployed; see [Component name](#component-name). Also accepted by `detect-drift.yml` |
 
 Plus, for opting into Kosli attestation (see [Kosli attestation](#kosli-attestation) below):
 
@@ -114,6 +115,10 @@ Plus, for opting into Kosli attestation (see [Kosli attestation](#kosli-attestat
 `apply.yml` also accepts `tf_state_file_name` (default `main.tfstate`) which names the state file
 under `terraform/<repo>/` in S3. This is used by `apply.yml`'s drift-plan housekeeping; see below.
 
+`apply.yml` also accepts `github_repository_name` (default: the calling repository's name), which
+names the drift file's directory when `component_name` is empty. Override it when a workflow in
+one repository operates on a different repository's Terraform code.
+
 ### Supplying Terraform variables
 
 A variable declared in `variables.tf` with no default and no `.tfvars` entry would normally make
@@ -336,6 +341,28 @@ be set in the environment or via `tf.env`:
 TF_STATE_FILE_NAME=environment-reporter.tfstate
 ```
 
+### Component name
+
+If a single repo deploys several components, each with its own state under its own name, set
+`TF_COMPONENT_NAME` to the component's name. The state is then stored at
+`terraform/<component-name>/main.tfstate` instead of `terraform/<repo-name>/main.tfstate`. Unset
+or empty, the repo name is used. The variable can be set in the environment or via `tf.env`:
+
+```
+TF_COMPONENT_NAME=creator
+```
+
+In GitHub Actions, pass the `component_name` input instead. `plan.yml`, `apply.yml` and
+`detect-drift.yml` all accept it, and each uses it in place of the repo name for:
+
+* the state file path, `terraform/<component-name>/`
+* the drift file, `terraform/<component-name>/drift.plan.json`
+* the Kosli flow, `terraform-plan-<env>-<component-name>` and `terraform-apply-<env>-<component-name>`
+* the drift-detection concurrency group
+
+Give `detect-drift.yml` the same `component_name` as `apply.yml`, so it reads the drift file
+`apply.yml` writes.
+
 ### State locking
 
 By default, `tf` uses Terraform's native S3 lockfile (`use_lockfile=true`), which writes a `.tflock`

bin/tf

@@ -163,6 +163,10 @@ class TfBackend:
             name = name[:-4]
         return name
 
+    @staticmethod
+    def _get_component_name() -> str:
+        return os.environ.get("TF_COMPONENT_NAME") or TfBackend._get_repo_name()
+
 
 SUBCOMMANDS_ACCEPTING_VAR_FILE = frozenset({
     "plan", "apply", "destroy", "import", "refresh", "console",
@@ -213,11 +217,11 @@ class TfRunner:
         account_id = TfVarsFiles._get_account_id()
         if not account_id:
             raise TfError("Cannot determine AWS account ID for backend config.")
-        repo_name = TfBackend._get_repo_name()
+        component_name = TfBackend._get_component_name()
         state_file_name = os.environ.get("TF_STATE_FILE_NAME", "main.tfstate")
         lock_backend = os.environ.get("TF_STATE_LOCK", "s3")
         return TfBackend(
-            account_id=account_id, region=context.region, repo_name=repo_name,
+            account_id=account_id, region=context.region, repo_name=component_name,
             state_file_name=state_file_name, lock_backend=lock_backend,
         )
 

tests/test_backend.py

@@ -156,3 +156,32 @@ def test_raises_when_not_a_git_repo(self):
 
             with pytest.raises(tf.TfError, match="not a git repo"):
                 tf.TfBackend._get_repo_name()
+
+
+class TestTfBackendComponentName:
+    def test_reads_component_name_from_env(self, monkeypatch):
+        monkeypatch.setenv("TF_COMPONENT_NAME", "creator")
+        with patch("tf.subprocess") as mock_subprocess:
+            mock_subprocess.run.return_value.returncode = 0
+            mock_subprocess.run.return_value.stdout = \
+                "https://github.com/cyber-dojo/web.git\n"
+
+            assert tf.TfBackend._get_component_name() == "creator"
+
+    def test_falls_back_to_repo_name_when_env_is_empty(self, monkeypatch):
+        monkeypatch.setenv("TF_COMPONENT_NAME", "")
+        with patch("tf.subprocess") as mock_subprocess:
+            mock_subprocess.run.return_value.returncode = 0
+            mock_subprocess.run.return_value.stdout = \
+                "https://github.com/cyber-dojo/web.git\n"
+
+            assert tf.TfBackend._get_component_name() == "web"
+
+    def test_falls_back_to_repo_name_when_env_is_unset(self, monkeypatch):
+        monkeypatch.delenv("TF_COMPONENT_NAME", raising=False)
+        with patch("tf.subprocess") as mock_subprocess:
+            mock_subprocess.run.return_value.returncode = 0
+            mock_subprocess.run.return_value.stdout = \
+                "https://github.com/cyber-dojo/web.git\n"
+
+            assert tf.TfBackend._get_component_name() == "web"

tests/test_main.py

@@ -474,3 +474,19 @@ def test_force_unlock_passes_lock_id_through(self, monkeypatch):
         mock_execvp.assert_called_once_with(
             "terraform", ["terraform", "force-unlock", "abc123"]
         )
+
+
+class TestTfRunnerStateKey:
+    def test_init_keys_state_on_component_name(self, monkeypatch):
+        monkeypatch.setenv("AWS_VAULT", "beta")
+        monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
+        monkeypatch.setenv("TF_COMPONENT_NAME", "creator")
+
+        with patch("tf.TfBackend._get_repo_name", return_value="web"), \
+             patch("tf.TfVarsFiles._get_account_id", return_value="244531986313"), \
+             patch("tf.subprocess.run", return_value=type("R", (), {"returncode": 0})()) as mock_run, \
+             patch("tf.os.execvp"):
+            tf.TfRunner(["force-unlock", "abc123"]).call()
+
+        init_command = mock_run.call_args[0][0]
+        assert "-backend-config=key=terraform/creator/main.tfstate" in init_command