feat: implement zombie turn management with internal API endpoints and add terminationGracePeriodSeconds

.changeset/20261019143019-regenerate-sdk-from-openapi.md

@@ -0,0 +1,5 @@
+---
+"@truefoundry/trueforge-sdk": patch
+---
+
+Regenerate SDK from updated OpenAPI spec.

.changeset/abandoned-turn-closer.md

@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge-core': patch
+'@truefoundry/trueforge': patch
+---
+
+Close stale running turns via a controller loop and internal list/close endpoints (abandoned reason).

.github/fern/openapi/openapi.json

@@ -2,6 +2,32 @@
   "components": {
     "parameters": {},
     "schemas": {
+      "AbandonedTurn": {
+        "additionalProperties": false,
+        "properties": {
+          "session_id": {
+            "description": "Session that owns the turn.",
+            "minLength": 1,
+            "type": "string"
+          },
+          "tenant_id": {
+            "description": "Tenant that owns the session.",
+            "minLength": 1,
+            "type": "string"
+          },
+          "turn_id": {
+            "description": "Running turn with stale last activity.",
+            "minLength": 1,
+            "type": "string"
+          }
+        },
+        "required": [
+          "tenant_id",
+          "session_id",
+          "turn_id"
+        ],
+        "type": "object"
+      },
       "ActionRequired": {
         "properties": {
           "id": {
@@ -861,6 +887,32 @@
         ],
         "type": "object"
       },
+      "CloseAbandonedTurnRequest": {
+        "additionalProperties": false,
+        "properties": {
+          "session_id": {
+            "description": "Session that owns the turn.",
+            "minLength": 1,
+            "type": "string"
+          },
+          "tenant_id": {
+            "description": "Tenant that owns the session.",
+            "minLength": 1,
+            "type": "string"
+          },
+          "turn_id": {
+            "description": "Turn to close.",
+            "minLength": 1,
+            "type": "string"
+          }
+        },
+        "required": [
+          "tenant_id",
+          "session_id",
+          "turn_id"
+        ],
+        "type": "object"
+      },
       "CompactionConfig": {
         "additionalProperties": false,
         "default": {
@@ -1890,6 +1942,21 @@
         },
         "type": "object"
       },
+      "ListAbandonedTurnsResponse": {
+        "additionalProperties": false,
+        "properties": {
+          "data": {
+            "items": {
+              "$ref": "#/components/schemas/AbandonedTurn"
+            },
+            "type": "array"
+          }
+        },
+        "required": [
+          "data"
+        ],
+        "type": "object"
+      },
       "ListAgentsResponse": {
         "properties": {
           "data": {
@@ -6700,6 +6767,93 @@
         "x-fern-sdk-method-name": "get_or_create_by_external_id"
       }
     },
+    "/api/internal/turns/abandoned": {
+      "get": {
+        "description": "Returns all running turns whose last activity is past the stale threshold.",
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ListAbandonedTurnsResponse"
+                }
+              }
+            },
+            "description": "Stale running turns oldest-first."
+          },
+          "401": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/RequestErrorResponse"
+                }
+              }
+            },
+            "description": "Invalid service credential."
+          }
+        },
+        "summary": "List abandoned running turns",
+        "tags": [
+          "Internal"
+        ],
+        "x-excluded": true,
+        "x-fern-sdk-group-name": [
+          "internal",
+          "turns"
+        ],
+        "x-fern-sdk-method-name": "list_abandoned"
+      }
+    },
+    "/api/internal/turns/close-abandoned": {
+      "post": {
+        "description": "Cancels a stale running turn (turn.done with abandoned reason).",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/CloseAbandonedTurnRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "Closed (or already terminal / missing turn)."
+          },
+          "401": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/RequestErrorResponse"
+                }
+              }
+            },
+            "description": "Invalid service credential."
+          },
+          "404": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/RequestErrorResponse"
+                }
+              }
+            },
+            "description": "Session not found."
+          }
+        },
+        "summary": "Close an abandoned running turn",
+        "tags": [
+          "Internal"
+        ],
+        "x-excluded": true,
+        "x-fern-sdk-group-name": [
+          "internal",
+          "turns"
+        ],
+        "x-fern-sdk-method-name": "close_abandoned"
+      }
+    },
     "/api/v1/agents": {
       "get": {
         "description": "List configured agents for the tenant, ordered by name. Optional `agent_name` filters by substring.",

charts/trueforge/templates/controller-deployment.yaml

@@ -44,7 +44,7 @@ spec:
       priorityClassName: {{ . | quote }}
       {{- end }}
       # Above the controller's GRACEFUL_TIMEOUT_SECONDS so SIGKILL does not cut off loop drain.
-      terminationGracePeriodSeconds: {{ add .Values.server.gracefulTimeoutSeconds 5 }}
+      terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }}
       {{- with (include "trueforge.customCA.initContainer" .) }}
       initContainers:
         {{- . | nindent 8 }}

charts/trueforge/templates/deployment.yaml

@@ -45,7 +45,7 @@ spec:
       priorityClassName: {{ . | quote }}
       {{- end }}
       # Above the server's GRACEFUL_TIMEOUT_SECONDS so SIGKILL does not cut off turn drain.
-      terminationGracePeriodSeconds: {{ add .Values.server.gracefulTimeoutSeconds 5 }}
+      terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }}
       {{- $caInit := include "trueforge.customCA.initContainer" . }}
       {{- if or $caInit .Values.initContainers }}
       initContainers:

charts/trueforge/values.yaml

@@ -120,6 +120,9 @@ server:
   publicBaseUrl: ""
   # Seconds the server is given to drain turns on SIGTERM (GRACEFUL_TIMEOUT_SECONDS).
   gracefulTimeoutSeconds: 30
+  # Seconds Kubernetes waits after SIGTERM before SIGKILL. Must exceed
+  # gracefulTimeoutSeconds so the process can finish draining.
+  terminationGracePeriodSeconds: 300
   # The server always runs peered (STANDALONE=false) against Postgres + Redis.
   # Escape hatch for knobs not covered by configs.*. Each entry is a full env
   # item: {name, value} or {name, valueFrom: {secretKeyRef|configMapKeyRef: ...}}.