From fe4259b3038f74f8aff668b622cfb8b0a73a2b81 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Fri, 16 Oct 2026 12:48:43 +0200 Subject: [PATCH 1/3] fix(lambda): add an opt-in cleanup sweep Add enable_cleanup_schedule (default false) and cleanup_schedule to the Lambda module and the AWS environment, a cleanup entry in sweep_tasks (rule name -sweep-cleanup, clear of the unconsumed cleanup-lambda module's -cleanup), tftest assertions for the default and opt-in runs, and drop cleanup from the aws-lambda knownGaps. The server cleanup task deletes expired sessions and, via CleanupOldExecutions(30), completed executions (scheduled_date), canceled executions (updated_at) and executions outside failed/canceled (expires_at) older than 30 days, so it stays opt-in: the first run deletes the whole backlog. Part of #712 Co-Authored-By: Claude Opus 5.5 --- terraform/environments/aws/compute.tf | 2 ++ terraform/environments/aws/variables.tf | 12 ++++++++++++ terraform/modules/compute/aws/lambda/sweeps.tf | 1 + .../compute/aws/lambda/tests/sweeps.tftest.hcl | 15 ++++++++------- terraform/modules/compute/aws/lambda/variables.tf | 12 ++++++++++++ .../compute/scheduled_task_coverage_test.go | 1 - 6 files changed, 35 insertions(+), 8 deletions(-) diff --git a/terraform/environments/aws/compute.tf b/terraform/environments/aws/compute.tf index e87f573f0..33907a656 100644 --- a/terraform/environments/aws/compute.tf +++ b/terraform/environments/aws/compute.tf @@ -95,6 +95,8 @@ module "compute_lambda" { reap_stuck_purchases_schedule = var.reap_stuck_purchases_schedule enable_analytics_collect_schedule = var.enable_analytics_collect_schedule analytics_collect_schedule = var.analytics_collect_schedule + enable_cleanup_schedule = var.enable_cleanup_schedule + cleanup_schedule = var.cleanup_schedule # Additional environment variables archera_org_id = var.archera_org_id diff --git a/terraform/environments/aws/variables.tf b/terraform/environments/aws/variables.tf index c0acf8168..911d57f92 100644 --- a/terraform/environments/aws/variables.tf +++ b/terraform/environments/aws/variables.tf @@ -372,6 +372,18 @@ variable "analytics_collect_schedule" { default = "rate(1 day)" } +variable "enable_cleanup_schedule" { + description = "Enable the scheduled server cleanup sweep. Opt-in, default false: it deletes expired sessions and purchase_executions that are completed (scheduled_date), canceled (updated_at) or in a status outside failed/canceled (expires_at) and older than 30 days. The first run deletes the whole backlog; export any history you want to keep first." + type = bool + default = false +} + +variable "cleanup_schedule" { + description = "EventBridge schedule for the cleanup sweep (default rate(1 day))." + type = string + default = "rate(1 day)" +} + # ============================================== # Multi-Account Configuration # ============================================== diff --git a/terraform/modules/compute/aws/lambda/sweeps.tf b/terraform/modules/compute/aws/lambda/sweeps.tf index 0efb13553..96b789870 100644 --- a/terraform/modules/compute/aws/lambda/sweeps.tf +++ b/terraform/modules/compute/aws/lambda/sweeps.tf @@ -10,6 +10,7 @@ locals { process_scheduled_purchases = { rule = "process", enabled = var.enable_process_scheduled_purchases_schedule, schedule = var.process_scheduled_purchases_schedule } send_notifications = { rule = "notify", enabled = var.enable_send_notifications_schedule, schedule = var.send_notifications_schedule } finalize_revocations = { rule = "finalize", enabled = var.enable_finalize_revocations_schedule, schedule = var.finalize_revocations_schedule } + cleanup = { rule = "cleanup", enabled = var.enable_cleanup_schedule, schedule = var.cleanup_schedule } } active_sweeps = { for k, v in local.sweep_tasks : k => v if v.enabled } } diff --git a/terraform/modules/compute/aws/lambda/tests/sweeps.tftest.hcl b/terraform/modules/compute/aws/lambda/tests/sweeps.tftest.hcl index 2b808e893..716efc18e 100644 --- a/terraform/modules/compute/aws/lambda/tests/sweeps.tftest.hcl +++ b/terraform/modules/compute/aws/lambda/tests/sweeps.tftest.hcl @@ -23,7 +23,7 @@ run "defaults" { assert { condition = toset(keys(aws_cloudwatch_event_rule.sweep)) == toset(["finalize_revocations"]) - error_message = "Only finalize_revocations is a default-on sweep." + error_message = "Only finalize_revocations is a default-on sweep (cleanup is opt-in)." } assert { condition = length(aws_cloudwatch_event_rule.scheduled_recommendations) == 1 && length(aws_cloudwatch_event_rule.reap_stuck_purchases) == 1 && length(aws_cloudwatch_event_rule.analytics_collect) == 1 @@ -40,7 +40,7 @@ run "defaults" { assert { condition = alltrue([ for k, r in aws_cloudwatch_event_rule.sweep : - length(r.name) <= 64 && startswith(r.name, "cudly-") && endswith(r.name, "-sweep-${ { process_scheduled_purchases = "process", send_notifications = "notify", finalize_revocations = "finalize" }[k]}") + length(r.name) <= 64 && startswith(r.name, "cudly-") && endswith(r.name, "-sweep-${ { process_scheduled_purchases = "process", send_notifications = "notify", finalize_revocations = "finalize", cleanup = "cleanup" }[k]}") ]) error_message = "Rule names must be <= 64 chars, keep the cudly- prefix and end with -sweep-." } @@ -69,11 +69,12 @@ run "opt_in" { enable_process_scheduled_purchases_schedule = true enable_send_notifications_schedule = true enable_fire_scheduled_purchases_schedule = true + enable_cleanup_schedule = true } assert { - condition = toset(keys(aws_cloudwatch_event_rule.sweep)) == toset(["finalize_revocations", "process_scheduled_purchases", "send_notifications"]) - error_message = "Opting in must add process and send_notifications." + condition = toset(keys(aws_cloudwatch_event_rule.sweep)) == toset(["cleanup", "finalize_revocations", "process_scheduled_purchases", "send_notifications"]) + error_message = "Opting in must add process, send_notifications and cleanup." } assert { condition = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 1 @@ -82,7 +83,7 @@ run "opt_in" { assert { condition = alltrue([ for k, r in aws_cloudwatch_event_rule.sweep : - length(r.name) <= 64 && startswith(r.name, "cudly-") && endswith(r.name, "-sweep-${ { process_scheduled_purchases = "process", send_notifications = "notify", finalize_revocations = "finalize" }[k]}") + length(r.name) <= 64 && startswith(r.name, "cudly-") && endswith(r.name, "-sweep-${ { process_scheduled_purchases = "process", send_notifications = "notify", finalize_revocations = "finalize", cleanup = "cleanup" }[k]}") ]) error_message = "Rule names must be <= 64 chars, keep the cudly- prefix and end with -sweep-." } @@ -91,13 +92,13 @@ run "opt_in" { error_message = "Sweep rule names must be distinct." } assert { - condition = length(aws_cloudwatch_event_target.sweep) == 3 + condition = length(aws_cloudwatch_event_target.sweep) == 4 error_message = "Every opted-in sweep needs a target." } assert { condition = alltrue([ for k, r in aws_cloudwatch_event_rule.sweep : - r.schedule_expression == { process_scheduled_purchases = var.process_scheduled_purchases_schedule, send_notifications = var.send_notifications_schedule, finalize_revocations = var.finalize_revocations_schedule }[k] + r.schedule_expression == { process_scheduled_purchases = var.process_scheduled_purchases_schedule, send_notifications = var.send_notifications_schedule, finalize_revocations = var.finalize_revocations_schedule, cleanup = var.cleanup_schedule }[k] ]) error_message = "Schedule must come from the variable." } diff --git a/terraform/modules/compute/aws/lambda/variables.tf b/terraform/modules/compute/aws/lambda/variables.tf index 4d858da30..d645099b5 100644 --- a/terraform/modules/compute/aws/lambda/variables.tf +++ b/terraform/modules/compute/aws/lambda/variables.tf @@ -267,6 +267,18 @@ variable "finalize_revocations_schedule" { default = "rate(1 hour)" } +variable "enable_cleanup_schedule" { + description = "Enable the scheduled server cleanup sweep. Opt-in: default false. It deletes expired sessions and, via CleanupOldExecutions(30), purchase_executions that are completed with scheduled_date older than 30 days, canceled with updated_at older than 30 days, or in a status outside failed/canceled with expires_at older than 30 days. The first run deletes the whole backlog, so export any history you want to keep first." + type = bool + default = false +} + +variable "cleanup_schedule" { + description = "EventBridge schedule for the cleanup sweep." + type = string + default = "rate(1 day)" +} + variable "purchase_approved_reap_after" { description = "Threshold age for the stuck-purchase reaper. Any execution sitting in approved/running longer than this gets flipped to failed on the next sweep. Parsed via Go time.ParseDuration (e.g. \"10m\", \"15m\", \"1h\"). Empty string falls back to the in-code default (10m)." type = string diff --git a/terraform/modules/compute/scheduled_task_coverage_test.go b/terraform/modules/compute/scheduled_task_coverage_test.go index c9437dd41..722602c5f 100644 --- a/terraform/modules/compute/scheduled_task_coverage_test.go +++ b/terraform/modules/compute/scheduled_task_coverage_test.go @@ -52,7 +52,6 @@ var ( // aws/cleanup-lambda and gcp/cleanup-function run cleanup outside the server // task and are not scanned, so cleanup shows as a gap on AWS and GCP. var knownGaps = map[string]map[string][]string{ - "aws-lambda": {dimTask: {"cleanup"}}, "aws-fargate": {dimTask: {"analytics_collect", "cleanup", "finalize_revocations", "process_scheduled_purchases", "reap_stuck_purchases", "send_notifications"}}, "cloudformation": {dimTask: {"analytics_collect", "cleanup", "finalize_revocations", From 56bf8912f389a278449adcfa2423832b82b618bf Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Fri, 16 Oct 2026 12:49:44 +0200 Subject: [PATCH 2/3] docs: document the opt-in Lambda cleanup schedule State the exact CleanupOldExecutions(30) retention in DEPLOYMENT.md and add the AWS Lambda opt-in variables. In known_issues/33 correct the Status (retention is unscheduled by default on AWS, GCP and CFN), fix the handler line reference and note the Lambda opt-in. Part of #712 Co-Authored-By: Claude Opus 5.5 --- docs/DEPLOYMENT.md | 7 ++++--- known_issues/33_executions_retention_cleanup.md | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f2afe17a4..ea23d3744 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -592,13 +592,14 @@ Each cloud provider has lifecycle policies configured via Terraform (`terraform/ ### Database Cleanup Jobs -The server's `cleanup` task deletes expired sessions and old completed or canceled purchase executions. Today only Azure schedules it: +The server's `cleanup` task deletes expired sessions and, via `CleanupOldExecutions(30)`, purchase executions that are completed with `scheduled_date` older than 30 days, canceled with `updated_at` older than 30 days, or in any status other than failed or canceled with `expires_at` older than 30 days. Failed executions are kept. Azure Container Apps schedules it by default; AWS Lambda can schedule it on request: - **Azure Container Apps (when `enable_scheduled_tasks` is true):** a Logic App calls `/api/scheduled/cleanup` daily (`terraform/modules/compute/azure/container-apps/scheduled-tasks.tf`) -- **AWS:** nothing schedules it. The `terraform/modules/compute/aws/cleanup-lambda/` module is not consumed by any environment. +- **AWS Lambda (opt-in):** set `enable_cleanup_schedule = true` (default false; `cleanup_schedule` defaults to `rate(1 day)`) in `terraform/environments/aws`. The first run deletes the whole backlog older than 30 days, so export any history you want to keep first. +- **AWS Fargate and CloudFormation:** nothing schedules it yet. The `terraform/modules/compute/aws/cleanup-lambda/` module is not consumed by any environment. - **GCP:** nothing schedules it. The `terraform/modules/compute/gcp/cleanup-function/` module is not consumed by any environment. -AKS has no scheduler (#893). Scheduling for AWS (Lambda, Fargate, CloudFormation) and GCP is tracked by #712. +AKS has no scheduler (#893). Scheduling for AWS Fargate, CloudFormation and GCP is tracked by #712. Alternatively, use pg_cron for database-native scheduling. diff --git a/known_issues/33_executions_retention_cleanup.md b/known_issues/33_executions_retention_cleanup.md index 53deb447c..e5a0cfdac 100644 --- a/known_issues/33_executions_retention_cleanup.md +++ b/known_issues/33_executions_retention_cleanup.md @@ -2,23 +2,25 @@ **Surfaced during:** 2026-04-23 follow-up audit — Failed + Expired rows now persist and surface in History **Related commits:** `32f9e4ffc` (pending-in-history), `b44283746` (failed + expired states) -**Status:** ✔️ Resolved — verified 2026-04-25. +**Status:** Partially resolved. The cleanup task exists (verified 2026-04-25) but retention is unscheduled by default on AWS, GCP and CFN; Azure Container Apps schedules it. > **Amendment (#712):** the AWS trigger claimed below (`cleanup-lambda/main.tf:98`) > is false: no environment consumes the `cleanup-lambda` module, so nothing > schedules cleanup on AWS. The same holds for the Terraform behind the > "matches what the cleanup-lambda Terraform actually configures" sentence > under "Original problem". Only Azure Container Apps schedules cleanup today. Retention -> scheduling is tracked by #712. +> scheduling is tracked by #712; retention is scheduled via the server `cleanup` task on AWS Lambda when opted in. ## Resolution -`internal/server/handler.go::handleCleanupExpiredRecords` (line 153) calls +`internal/server/handler.go::handleCleanupExpiredRecords` (line 272) calls `CleanupOldExecutions(ctx, 30)` with a 30-day retention. Wired as the `TaskCleanupExpiredRecords` scheduled task type and triggered by: - **AWS:** ~~EventBridge rule `cleanup_schedule` in `terraform/modules/compute/aws/cleanup-lambda/main.tf:98`~~ (unconsumed module, see amendment) +- **AWS Lambda (opt-in):** the `enable_cleanup_schedule` / `cleanup_schedule` + sweep `-sweep-cleanup` runs the server cleanup task. - **Azure Container Apps:** a Logic App calls `/api/scheduled/cleanup` daily when `enable_scheduled_tasks` is true. (The `azure/cleanup-function` module is not consumed by any environment, and AKS has no scheduler, #893.) @@ -26,7 +28,7 @@ The `CleanupOldExecutions` Postgres implementation in `internal/config/store_postgres.go` runs the expected `DELETE FROM purchase_executions WHERE …`, so the table is bounded -only where that task is scheduled (Azure Container Apps today, see amendment). +only where that task is scheduled (Azure Container Apps by default, AWS Lambda when opted in, see amendment). ## Original problem (historical context) From d58ec6ed07e006e65626189c534f5bdbf8f85b01 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Fri, 16 Oct 2026 12:50:12 +0200 Subject: [PATCH 3/3] test(iac): match only false or "false" in the tfvars off scan The old pattern also matched an unterminated "false and values such as falsey. Add a negative fixture for both. Part of #712 Co-Authored-By: Claude Opus 5.5 --- terraform/modules/compute/scheduled_task_env_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform/modules/compute/scheduled_task_env_test.go b/terraform/modules/compute/scheduled_task_env_test.go index 0213969bb..1836272fb 100644 --- a/terraform/modules/compute/scheduled_task_env_test.go +++ b/terraform/modules/compute/scheduled_task_env_test.go @@ -37,7 +37,7 @@ const awsEnvDir = "../../environments/aws" var ( passLineRe = regexp.MustCompile(`(?m)^\s*(\w+)\s*=\s*var\.(\w+)\s*$`) - tfvarsOffRe = regexp.MustCompile(`(?m)^\s*(enable_\w+)\s*=\s*"?false\b`) + tfvarsOffRe = regexp.MustCompile(`(?m)^\s*(enable_\w+)\s*=\s*(?:false|"false")\s*(?:(?:#|//).*)?$`) defaultRe = regexp.MustCompile(`(?m)^\s*default\s*=\s*(\S+)`) ) @@ -293,6 +293,12 @@ func TestEnvChecksNegatives(t *testing.T) { if got := tfvarsProblems("a.tfvars", "enable_finalize_revocations_schedule = \"false\"\n"); len(got) != 1 { t.Errorf("quoted \"false\" must count as off: %v", got) } + if got := tfvarsProblems("a.tfvars", "enable_finalize_revocations_schedule = \"false\nenable_finalize_revocations_schedule = falsey\n"); len(got) != 0 { + t.Errorf("unterminated or longer values must not count as off: %v", got) + } + if got := tfvarsProblems("a.tfvars", "enable_finalize_revocations_schedule = false // note\n"); len(got) != 1 { + t.Errorf("a trailing // comment must not hide an off value: %v", got) + } if len(tfvarsProblems("a.tfvars.json", "{}")) != 1 { t.Error("JSON tfvars must fail loudly") }