diff --git a/.github/workflows/apply.yml b/.github/workflows/apply.yml index 36d33ac..934aed3 100644 --- a/.github/workflows/apply.yml +++ b/.github/workflows/apply.yml @@ -26,9 +26,13 @@ on: default: ${{ github.repository }} type: string github_repository_name: - description: "GitHub repository name (without the owner) used to locate the statefile and drift file. Defaults to the repository that initiated the run. Override when a workflow in one repository needs to operate on a different repository's Terraform code." + description: "GitHub repository name (without the owner) used to locate the statefile and drift file when component_name is empty. Defaults to the repository that initiated the run. Override when a workflow in one repository needs to operate on a different repository's Terraform code." default: ${{ github.event.repository.name }} type: string + component_name: + description: "Name of the thing being deployed. Used for the statefile path, the drift file path, and the Kosli flow name. Both paths are under terraform//. Empty keeps the repository-name defaults. Set it when one repository deploys several components, each with its own state." + default: "" + type: string ref: description: "Git reference (branch, tag or SHA) to check out. Defaults to the default branch of the checked-out repository." default: "" @@ -94,10 +98,11 @@ jobs: github_repository_to_checkout: ${{ inputs.github_repository_to_checkout }} ref: ${{ inputs.ref }} tf_vars: ${{ inputs.tf_vars }} + component_name: ${{ inputs.component_name }} kosli_template_file: ${{ inputs.kosli_template_file }} kosli_host: ${{ inputs.kosli_host }} kosli_org: ${{ inputs.kosli_org }} - kosli_flow: terraform-apply-${{ inputs.environment }}-${{ github.event.repository.name }} + kosli_flow: terraform-apply-${{ inputs.environment }}-${{ inputs.component_name || github.event.repository.name }} kosli_cli_version: ${{ inputs.kosli_cli_version }} pass_commit_to_kosli_commands: ${{ inputs.pass_commit_to_kosli_commands }} job_timeout_minutes: ${{ inputs.job_timeout_minutes }} @@ -120,6 +125,7 @@ jobs: KOSLI_TRAIL: ${{ needs.apply.outputs.kosli_trail }} COMMIT_ARG: ${{ inputs.pass_commit_to_kosli_commands && format('--commit {0}', inputs.ref) || '' }} GH_TOKEN: ${{ secrets.kosli_github_token }} + STATE_DIR: terraform/${{ inputs.component_name || inputs.github_repository_name }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -150,7 +156,7 @@ jobs: - name: Upload drift plan to S3 run: | - aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/terraform/${{ inputs.github_repository_name }}/drift.plan.json" + aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json" - name: Upload drift plan artifact uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 @@ -169,7 +175,7 @@ jobs: if: inputs.kosli_template_file != '' run: | aws s3 cp \ - "s3://${S3_BUCKET}/terraform/${{ inputs.github_repository_name }}/${{ inputs.tf_state_file_name }}" \ + "s3://${S3_BUCKET}/${STATE_DIR}/${{ inputs.tf_state_file_name }}" \ "/tmp/${{ inputs.tf_state_file_name }}" - name: Kosli attest state file diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index e8dfc4c..a9592c3 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -39,6 +39,10 @@ on: description: "Extra environment variables, one KEY=VALUE per line, exported before plan/apply. To set Terraform variable `foo`, use `TF_VAR_foo=...`. Single-line values only." default: "" type: string + component_name: + description: "Name of the thing being deployed, used for the statefile path terraform//. Empty uses the checked-out repository's name. Set it when one repository deploys several components, each with its own state." + default: "" + type: string kosli_template_file: description: "Path (in caller repo) to the Kosli trail template. Empty disables all Kosli steps." default: "" @@ -147,6 +151,14 @@ jobs: with: version: ${{ inputs.kosli_cli_version }} + # Exported only when set, so an unset input leaves TF_COMPONENT_NAME + # free to come from the repo's tf.env, as it does when tf runs locally. + - name: Export component name + if: inputs.component_name != '' + env: + COMPONENT_NAME: ${{ inputs.component_name }} + run: printf 'TF_COMPONENT_NAME=%s\n' "$COMPONENT_NAME" >> "$GITHUB_ENV" + - name: Export Terraform variables if: inputs.tf_vars != '' env: diff --git a/.github/workflows/detect-drift.yml b/.github/workflows/detect-drift.yml index 4ab3cc0..68a0f9d 100644 --- a/.github/workflows/detect-drift.yml +++ b/.github/workflows/detect-drift.yml @@ -29,11 +29,20 @@ on: description: "Minutes before GitHub cancels the drift plan job. Raise it together with aws_role_duration when a legitimate plan needs longer; see the README on timeouts." default: 30 type: number + component_name: + description: "Name of the thing being deployed. Used for the statefile and drift file paths, terraform//. Empty uses the repository name. Must match the component_name the apply workflow was given." + default: "" + type: string concurrency: - group: detect-drift-${{ github.repository }}-${{ inputs.environment }} + # One group per component, so several components' drift runs in one + # repository do not replace each other's pending run. + group: detect-drift-${{ inputs.component_name || github.repository }}-${{ inputs.environment }} cancel-in-progress: false +env: + STATE_DIR: terraform/${{ inputs.component_name || github.event.repository.name }} + jobs: fetch-baseline: runs-on: ubuntu-latest @@ -64,7 +73,7 @@ jobs: - name: Check baseline exists run: | - KEY="terraform/${{ github.event.repository.name }}/drift.plan.json" + KEY="${STATE_DIR}/drift.plan.json" if ! aws s3api head-object --bucket "$S3_BUCKET" --key "$KEY" >/dev/null 2>&1; then echo "::error::No drift baseline at s3://${S3_BUCKET}/${KEY} — run the apply workflow at least once to seed it." exit 1 @@ -72,7 +81,7 @@ jobs: - name: Download baseline run: | - aws s3 cp "s3://${S3_BUCKET}/terraform/${{ github.event.repository.name }}/drift.plan.json" /tmp/drift.plan.json + aws s3 cp "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json" /tmp/drift.plan.json - name: Read baseline id: read @@ -106,6 +115,7 @@ jobs: ref: ${{ needs.fetch-baseline.outputs.sha }} tf_apply: false tf_vars: ${{ inputs.tf_vars }} + component_name: ${{ inputs.component_name }} job_timeout_minutes: ${{ inputs.job_timeout_minutes }} flag-drift: @@ -144,7 +154,7 @@ jobs: - name: Upload drift plan to S3 run: | - aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/terraform/${{ github.event.repository.name }}/drift.plan.json" + aws s3 cp /tmp/drift.plan.json "s3://${S3_BUCKET}/${STATE_DIR}/drift.plan.json" - name: Drift summary run: | diff --git a/.github/workflows/plan.yml b/.github/workflows/plan.yml index 7053e5a..e3500ff 100644 --- a/.github/workflows/plan.yml +++ b/.github/workflows/plan.yml @@ -33,6 +33,10 @@ on: description: "Extra environment variables, one KEY=VALUE per line, exported before plan. To set Terraform variable `foo`, use `TF_VAR_foo=...`. Single-line values only." default: "" type: string + component_name: + description: "Name of the thing being deployed. Used for the statefile path (terraform//) and the Kosli flow name. Empty keeps the repository-name defaults. Set it when one repository deploys several components, each with its own state." + default: "" + type: string kosli_template_file: description: "Path (in caller repo) to the Kosli trail template. Empty disables all Kosli steps." default: "" @@ -79,10 +83,11 @@ jobs: github_repository_to_checkout: ${{ inputs.github_repository_to_checkout }} ref: ${{ inputs.ref }} tf_vars: ${{ inputs.tf_vars }} + component_name: ${{ inputs.component_name }} kosli_template_file: ${{ inputs.kosli_template_file }} kosli_host: ${{ inputs.kosli_host }} kosli_org: ${{ inputs.kosli_org }} - kosli_flow: terraform-plan-${{ inputs.environment }}-${{ github.event.repository.name }} + kosli_flow: terraform-plan-${{ inputs.environment }}-${{ inputs.component_name || github.event.repository.name }} kosli_cli_version: ${{ inputs.kosli_cli_version }} job_timeout_minutes: ${{ inputs.job_timeout_minutes }} secrets: diff --git a/README.md b/README.md index 9c619a0..ad49383 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Both `plan.yml` and `apply.yml` accept the same core inputs: | `tf_version` | no | `1.14.6` | Terraform version to install | | `tf_vars` | no | `""` | Extra env vars (one `KEY=VALUE` per line) exported before plan/apply; see [Supplying Terraform variables](#supplying-terraform-variables) | | `job_timeout_minutes` | no | `30` | Minutes before GitHub cancels the plan/apply job; see [Timeouts](#timeouts) | +| `component_name` | no | `""` | Name of the thing being deployed; see [Component name](#component-name). Also accepted by `detect-drift.yml` | Plus, for opting into Kosli attestation (see [Kosli attestation](#kosli-attestation) below): @@ -114,6 +115,10 @@ Plus, for opting into Kosli attestation (see [Kosli attestation](#kosli-attestat `apply.yml` also accepts `tf_state_file_name` (default `main.tfstate`) which names the state file under `terraform//` in S3. This is used by `apply.yml`'s drift-plan housekeeping; see below. +`apply.yml` also accepts `github_repository_name` (default: the calling repository's name), which +names the drift file's directory when `component_name` is empty. Override it when a workflow in +one repository operates on a different repository's Terraform code. + ### Supplying Terraform variables A variable declared in `variables.tf` with no default and no `.tfvars` entry would normally make @@ -336,6 +341,28 @@ be set in the environment or via `tf.env`: TF_STATE_FILE_NAME=environment-reporter.tfstate ``` +### Component name + +If a single repo deploys several components, each with its own state under its own name, set +`TF_COMPONENT_NAME` to the component's name. The state is then stored at +`terraform//main.tfstate` instead of `terraform//main.tfstate`. Unset +or empty, the repo name is used. The variable can be set in the environment or via `tf.env`: + +``` +TF_COMPONENT_NAME=creator +``` + +In GitHub Actions, pass the `component_name` input instead. `plan.yml`, `apply.yml` and +`detect-drift.yml` all accept it, and each uses it in place of the repo name for: + +* the state file path, `terraform//` +* the drift file, `terraform//drift.plan.json` +* the Kosli flow, `terraform-plan--` and `terraform-apply--` +* the drift-detection concurrency group + +Give `detect-drift.yml` the same `component_name` as `apply.yml`, so it reads the drift file +`apply.yml` writes. + ### State locking By default, `tf` uses Terraform's native S3 lockfile (`use_lockfile=true`), which writes a `.tflock` diff --git a/bin/tf b/bin/tf index e273c12..f84e31c 100755 --- a/bin/tf +++ b/bin/tf @@ -163,6 +163,10 @@ class TfBackend: name = name[:-4] return name + @staticmethod + def _get_component_name() -> str: + return os.environ.get("TF_COMPONENT_NAME") or TfBackend._get_repo_name() + SUBCOMMANDS_ACCEPTING_VAR_FILE = frozenset({ "plan", "apply", "destroy", "import", "refresh", "console", @@ -213,11 +217,11 @@ class TfRunner: account_id = TfVarsFiles._get_account_id() if not account_id: raise TfError("Cannot determine AWS account ID for backend config.") - repo_name = TfBackend._get_repo_name() + component_name = TfBackend._get_component_name() state_file_name = os.environ.get("TF_STATE_FILE_NAME", "main.tfstate") lock_backend = os.environ.get("TF_STATE_LOCK", "s3") return TfBackend( - account_id=account_id, region=context.region, repo_name=repo_name, + account_id=account_id, region=context.region, repo_name=component_name, state_file_name=state_file_name, lock_backend=lock_backend, ) diff --git a/tests/test_backend.py b/tests/test_backend.py index bad5360..5eb543d 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -156,3 +156,32 @@ def test_raises_when_not_a_git_repo(self): with pytest.raises(tf.TfError, match="not a git repo"): tf.TfBackend._get_repo_name() + + +class TestTfBackendComponentName: + def test_reads_component_name_from_env(self, monkeypatch): + monkeypatch.setenv("TF_COMPONENT_NAME", "creator") + with patch("tf.subprocess") as mock_subprocess: + mock_subprocess.run.return_value.returncode = 0 + mock_subprocess.run.return_value.stdout = \ + "https://github.com/cyber-dojo/web.git\n" + + assert tf.TfBackend._get_component_name() == "creator" + + def test_falls_back_to_repo_name_when_env_is_empty(self, monkeypatch): + monkeypatch.setenv("TF_COMPONENT_NAME", "") + with patch("tf.subprocess") as mock_subprocess: + mock_subprocess.run.return_value.returncode = 0 + mock_subprocess.run.return_value.stdout = \ + "https://github.com/cyber-dojo/web.git\n" + + assert tf.TfBackend._get_component_name() == "web" + + def test_falls_back_to_repo_name_when_env_is_unset(self, monkeypatch): + monkeypatch.delenv("TF_COMPONENT_NAME", raising=False) + with patch("tf.subprocess") as mock_subprocess: + mock_subprocess.run.return_value.returncode = 0 + mock_subprocess.run.return_value.stdout = \ + "https://github.com/cyber-dojo/web.git\n" + + assert tf.TfBackend._get_component_name() == "web" diff --git a/tests/test_main.py b/tests/test_main.py index 7aaf4af..854e803 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -474,3 +474,19 @@ def test_force_unlock_passes_lock_id_through(self, monkeypatch): mock_execvp.assert_called_once_with( "terraform", ["terraform", "force-unlock", "abc123"] ) + + +class TestTfRunnerStateKey: + def test_init_keys_state_on_component_name(self, monkeypatch): + monkeypatch.setenv("AWS_VAULT", "beta") + monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1") + monkeypatch.setenv("TF_COMPONENT_NAME", "creator") + + with patch("tf.TfBackend._get_repo_name", return_value="web"), \ + patch("tf.TfVarsFiles._get_account_id", return_value="244531986313"), \ + patch("tf.subprocess.run", return_value=type("R", (), {"returncode": 0})()) as mock_run, \ + patch("tf.os.execvp"): + tf.TfRunner(["force-unlock", "abc123"]).call() + + init_command = mock_run.call_args[0][0] + assert "-backend-config=key=terraform/creator/main.tfstate" in init_command