From e8eb390b397e10df8966eabd18e12fc0724d5d9b Mon Sep 17 00:00:00 2001 From: Shashank Ram Date: Thu, 5 Nov 2026 08:45:02 -0800 Subject: [PATCH] charts/substrate: expose router timeouts with 60s defaults - Allow slow actor restores more time before the router cancels them. - Configure parking and API request limits across all ingress routes. - Keep a five-second parking budget for budget-exhaustion tests. /kind fix ```release-note Router parking and API request timeouts now default to 60s and are configurable through Helm. ``` (cherry picked from commit 4d8bf1ce7dadf584e9c9c0801ce8c18a76eb4308) Signed-off-by: Shashank Ram --- .github/workflows/helm-e2e.yaml | 1 + charts/substrate/README.md | 27 +++++++++- charts/substrate/templates/atenet-router.yaml | 2 + .../substrate/tests/control_plane_test.yaml | 53 ++++++++++++++++++- charts/substrate/values.yaml | 4 +- internal/e2e/suites/parking/parking_test.go | 6 +-- internal/e2e/suites/parking/values.yaml | 18 +++++++ 7 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 internal/e2e/suites/parking/values.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 4111523151..67d6c6aa52 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -96,6 +96,7 @@ jobs: --namespace ate-system \ --create-namespace \ -f internal/e2e/suites/credentials/values.yaml \ + -f internal/e2e/suites/parking/values.yaml \ --set postgres.clientCertificates.enabled=true \ --set image.registry=localhost:5001 \ --set image.tag=helm-e2e \ diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 7a2e93c655..45b9734287 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -60,7 +60,8 @@ See `values.yaml` for the full set; the important keys: | `postgres.schema` | `substrate` | Store the Substrate tables in this PostgreSQL schema | | `postgres.clientCertificates.enabled` | `false` | Project separate owner and runtime login certificates into the API server | | `.resources` | see `values.yaml` | Requests and limits for each container, e.g. `ateApi.resources` and `atenetRouter.agentgateway.resources` | -| `atenetRouter.requestParking.budget` | `5s` | Maximum wait for a suspended actor to become routable, e.g. `30s` for slower cold starts | +| `atenetRouter.requestParking.budget` | `60s` | Total budget for actor resolution and resume retries | +| `atenetRouter.requestTimeout` | `60s` | Timeout for each Substrate ingress API call, including actor resolution | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs:///` with bundled RustFS | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | @@ -83,6 +84,30 @@ See `values.yaml` for the full set; the important keys: | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateworkers, plus the router access log. Actor events go to stdout when OTLP logs are disabled | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | +## Router timeouts + +The router uses `ResumeActor` to resolve actor assignments, including actors +that already run. `atenetRouter.requestTimeout` sets the HTTP backend request +timeout for those API calls. `atenetRouter.requestParking.budget` bounds the +resume attempts and retry delays. Both apply to direct and CONNECT-tunneled +HTTP/gRPC routes. Both default to `60s`, allowing each attempt to use the +remaining parking budget. + +A resume must finish within both the per-call timeout and the remaining parking +budget. A shorter client deadline can still cancel it. Set both values as +duration strings when slow restores need more time, for example: + +```yaml +atenetRouter: + requestParking: + budget: "120s" + requestTimeout: "120s" +``` + +Choose limits using measured restore latency. A parking budget of `0s` uses +agentgateway's default budget; it does not disable the limit. These settings +govern API calls during routing, not the actor's application request duration. + ## Upgrading sandbox and snapshot configuration Public APIs now use versioned packages under `api/`. Upgrade clients, control diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index fc7b6b972a..34587b6274 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -114,6 +114,8 @@ data: # AgentGateway only uses atunnel's CONNECT listener. connectTargetPort: 8443 policies: + http: + requestTimeout: {{ .Values.atenetRouter.requestTimeout | quote }} backendTLS: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 126e2b729d..be9c55eb75 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -374,7 +374,7 @@ tests: - matchRegex: path: data["config.yaml"] pattern: 'source.spiffeId in \["spiffe://cluster.local/ns/client/sa/router-client"\]' -- it: keeps the router request parking budget at five seconds by default +- it: defaults router parking and API request timeouts to sixty seconds template: atenet-router.yaml documentSelector: path: kind @@ -382,7 +382,10 @@ tests: asserts: - matchRegex: path: data["config.yaml"] - pattern: 'requestParking:\n\s+budget: "5s"' + pattern: 'requestParking:\n\s+budget: "60s"' + - matchRegex: + path: data["config.yaml"] + pattern: 'policies:\n\s+http:\n\s+requestTimeout: "60s"' - it: configures the router request parking budget template: atenet-router.yaml set: @@ -394,3 +397,49 @@ tests: - matchRegex: path: data["config.yaml"] pattern: 'requestParking:\n\s+budget: "30s"' + - matchRegex: + path: data["config.yaml"] + pattern: 'policies:\n\s+http:\n\s+requestTimeout: "60s"' +- it: configures the router API request timeout independently + template: atenet-router.yaml + set: + atenetRouter.requestTimeout: 2m + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'requestParking:\n\s+budget: "60s"' + - matchRegex: + path: data["config.yaml"] + pattern: 'policies:\n\s+http:\n\s+requestTimeout: "2m"' +- it: configures both router timeouts + template: atenet-router.yaml + set: + atenetRouter.requestParking.budget: 90s + atenetRouter.requestTimeout: 45s + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'requestParking:\n\s+budget: "90s"' + - matchRegex: + path: data["config.yaml"] + pattern: 'policies:\n\s+http:\n\s+requestTimeout: "45s"' +- it: keeps a five-second parking budget in the E2E fixture + template: atenet-router.yaml + values: + - ../../../internal/e2e/suites/parking/values.yaml + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'requestParking:\n\s+budget: "5s"' + - matchRegex: + path: data["config.yaml"] + pattern: 'policies:\n\s+http:\n\s+requestTimeout: "60s"' diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index c4628edde4..26422252d9 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -145,7 +145,9 @@ ateController: atenetRouter: requestParking: # Maximum time to wait for a suspended actor to become routable. - budget: 5s + budget: "60s" + # Timeout for each Substrate ingress API call, including actor resolution. + requestTimeout: "60s" # mTLS client identities allowed to reach actors. Empty denies all ingress. ingressAllowedSPIFFEIDs: [] agentgateway: diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index 42df00756c..9299832a1f 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -16,8 +16,8 @@ // Envoy → ext_proc → ateapi → worker path: a deliberately 1-worker pool is // oversubscribed by two actors, so a request for the suspended actor parks // until the worker frees (ParkThenServed) or the park budget elapses -// (BudgetExhaustion). It runs with the router's default parking configuration -// (budget 5s); flag-dependent behavior (lot-full shed, parking disabled, +// (BudgetExhaustion). It requires a 5s parking budget. Flag-dependent behavior +// (lot-full shed, parking disabled, // custom budgets) is covered by unit tests instead, because the shared router // cannot be reconfigured per test. package parking @@ -39,7 +39,7 @@ import ( // k8s namespace holding its pool. var parkingAtespace = e2e.FixtureName("ate-e2e") + "-parking" -// The park budget the deployed router runs with (its flag default). The +// The park budget the deployed test router runs with. The // timing assertions below are windows around it, wide enough for scheduling // jitter but narrow enough to prove parking happened and that the router — // not an Envoy timeout — produced the verdict. diff --git a/internal/e2e/suites/parking/values.yaml b/internal/e2e/suites/parking/values.yaml new file mode 100644 index 0000000000..79247fbdde --- /dev/null +++ b/internal/e2e/suites/parking/values.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# BudgetExhaustion checks a five-second parking window. +atenetRouter: + requestParking: + budget: "5s"