diff --git a/cli/src/cmd/aiworkspace/build.go b/cli/src/cmd/aiworkspace/build.go index baced62904..a14101a2b1 100644 --- a/cli/src/cmd/aiworkspace/build.go +++ b/cli/src/cmd/aiworkspace/build.go @@ -570,7 +570,14 @@ func buildLLMProviderPayload(name string, metadata aiWorkspaceMetadata, runtime if up := runtime.Spec.Upstream; up != nil { target := llmUpstreamTarget{URL: strings.TrimSpace(up.URL)} if up.Auth != nil { - target.Auth = &llmUpstreamAuth{Type: up.Auth.Type, Header: up.Auth.Header, Value: up.Auth.Value} + target.Auth = &llmUpstreamAuth{ + Type: up.Auth.Type, + Header: up.Auth.Header, + Value: up.Auth.Value, + PolicyName: up.Auth.PolicyName, + PolicyVersion: up.Auth.PolicyVersion, + PolicyParams: up.Auth.PolicyParams, + } } payload.Upstream = &llmUpstream{Main: target} } @@ -643,7 +650,14 @@ func buildMCPProxyPayload(name string, metadata aiWorkspaceMetadata, runtime aiW if up := runtime.Spec.Upstream; up != nil { target := llmUpstreamTarget{URL: strings.TrimSpace(up.URL)} if up.Auth != nil { - target.Auth = &llmUpstreamAuth{Type: up.Auth.Type, Header: up.Auth.Header, Value: up.Auth.Value} + target.Auth = &llmUpstreamAuth{ + Type: up.Auth.Type, + Header: up.Auth.Header, + Value: up.Auth.Value, + PolicyName: up.Auth.PolicyName, + PolicyVersion: up.Auth.PolicyVersion, + PolicyParams: up.Auth.PolicyParams, + } } payload.Upstream = &llmUpstream{Main: target} } @@ -991,11 +1005,15 @@ func buildLLMProxyPayload(proxyName string, metadata aiWorkspaceMetadata, runtim } // The proxy references its provider by id; the provider owns the credential - // value, so only the auth type/header are carried here (never the secret). + // value, so only non-secret fields are carried here - type/header/policyName/ + // policyVersion, never value or policyParams (which for oauth2 holds the + // client secret/token endpoint credentials). if auth := runtime.Spec.Provider.Auth; auth != nil { payload.Provider.Auth = &llmUpstreamAuth{ - Type: auth.Type, - Header: auth.Header, + Type: auth.Type, + Header: auth.Header, + PolicyName: auth.PolicyName, + PolicyVersion: auth.PolicyVersion, } } @@ -1105,9 +1123,12 @@ type runtimeProvider struct { } type runtimeProviderAuth struct { - Type string `yaml:"type"` - Header string `yaml:"header"` - Value string `yaml:"value"` + Type string `yaml:"type"` + Header string `yaml:"header"` + Value string `yaml:"value"` + PolicyName string `yaml:"policyName"` + PolicyVersion string `yaml:"policyVersion"` + PolicyParams map[string]interface{} `yaml:"policyParams"` } type runtimeUpstream struct { @@ -1172,9 +1193,12 @@ type llmProxyProvider struct { } type llmUpstreamAuth struct { - Type string `json:"type,omitempty"` - Header string `json:"header,omitempty"` - Value string `json:"value,omitempty"` + Type string `json:"type,omitempty"` + Header string `json:"header,omitempty"` + Value string `json:"value,omitempty"` + PolicyName string `json:"policyName,omitempty"` + PolicyVersion string `json:"policyVersion,omitempty"` + PolicyParams map[string]interface{} `json:"policyParams,omitempty"` } type llmPolicy struct { diff --git a/cli/src/cmd/aiworkspace/build_test.go b/cli/src/cmd/aiworkspace/build_test.go index b37f78aeaf..5d2139dcc1 100644 --- a/cli/src/cmd/aiworkspace/build_test.go +++ b/cli/src/cmd/aiworkspace/build_test.go @@ -106,6 +106,31 @@ func TestBuildLLMProxyPayload_UsesRuntimeDescriptionWhenSet(t *testing.T) { } } +func TestBuildLLMProxyPayload_ProviderAuthOmitsSecretFields(t *testing.T) { + rt := newProxyRuntime() + rt.Spec.Provider.Auth = &runtimeProviderAuth{ + Type: "oauth2", + PolicyName: "oauth2-generator", + PolicyVersion: "v1", + Value: "should-never-be-copied", + PolicyParams: map[string]interface{}{ + "clientSecret": "should-never-be-copied", + }, + } + + payload := buildLLMProxyPayload("claude-proxy2", newProxyMetadata(), rt, "") + auth := payload.Provider.Auth + if auth == nil { + t.Fatalf("expected provider auth to be set") + } + if auth.Type != "oauth2" || auth.PolicyName != "oauth2-generator" || auth.PolicyVersion != "v1" { + t.Fatalf("expected non-secret fields to pass through, got %+v", auth) + } + if auth.Value != "" || auth.PolicyParams != nil { + t.Fatalf("expected secret-bearing fields (value/policyParams) to be omitted, got %+v", auth) + } +} + func TestBuildLLMProxyPayload_OmitsPoliciesWhenNone(t *testing.T) { var md aiWorkspaceMetadata md.Spec.DisplayName = "p" @@ -189,6 +214,37 @@ func TestBuildLLMProviderPayload_OmitsModelProvidersForUnknownTemplate(t *testin } } +func TestBuildLLMProviderPayload_UpstreamOAuth2CarriesPolicyParams(t *testing.T) { + var metadata aiWorkspaceMetadata + metadata.Spec.Version = "v1.0" + + var runtime aiWorkspaceRuntime + runtime.Spec.Upstream = &runtimeUpstream{ + URL: "https://upstream.example.com", + Auth: &runtimeProviderAuth{ + Type: "oauth2", + PolicyVersion: "v1", + PolicyParams: map[string]interface{}{ + "tokenEndpoint": "https://idp.example.com/token", + "clientId": "abc", + "clientSecret": "{{ secret \"upstream-oauth2\" }}", + }, + }, + } + + payload := buildLLMProviderPayload("p", metadata, runtime, "") + if payload.Upstream == nil || payload.Upstream.Main.Auth == nil { + t.Fatalf("expected upstream auth to be set, got %+v", payload.Upstream) + } + auth := payload.Upstream.Main.Auth + if auth.Type != "oauth2" || auth.PolicyVersion != "v1" { + t.Fatalf("unexpected auth type/policyVersion: %+v", auth) + } + if auth.PolicyParams["tokenEndpoint"] != "https://idp.example.com/token" || auth.PolicyParams["clientId"] != "abc" { + t.Fatalf("expected policyParams to pass through verbatim, got %+v", auth.PolicyParams) + } +} + func writeTestEnvFile(t *testing.T, dir, name, content string) string { t.Helper() path := filepath.Join(dir, name) diff --git a/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md b/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md new file mode 100644 index 0000000000..5b6ed02e02 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md @@ -0,0 +1,88 @@ +# Task 0 Findings: Cluster identity resolution timing for the oauth2-generator upstream-retry-refresh registry + +**Scope:** answers the three numbered questions in `.superpowers/sdd/2026-08-11-oauth2-upstream-retry-refresh/task-0-brief.md`. All line numbers below were verified by reading the files directly on branch `redisclient` on 2026-08-11; they may drift slightly with future edits but the referenced functions/fields are stable identifiers to re-locate them. + +## Summary (read this first) + +The brief frames the question as a binary: cluster name known at `handler.go`'s `buildPolicyChain` (config-load time) vs. only in `execution_context.go` (per-request). **The real situation is more complicated than either option, and the brief's Question 3 fallback assumption needs correction on its stated reasoning, even though its recommended lazy design is still the right conclusion.** + +1. Cluster name for a route's *default* (non-redirected) upstream **is** known at xDS config-load time — but not inside `buildPolicyChain`/`HandlePolicyChainUpdate`. It arrives via a **separate, sibling xDS resource type** ("RouteConfig", handled by `HandleRouteConfigUpdate`), keyed by the same `RouteKey`, and is only *copied* (not resolved) into the per-request execution context. +2. Envoy cluster names are **not** dedicated per route or per API operation. They are derived purely from `host+scheme` of the upstream URL (`sanitizeClusterName`), and gateway-controller explicitly deduplicates clusters **across all deployed APIs** by cluster name. Two routes — even from two unrelated APIs — with different `oauth2-generator` configs can and will resolve to the identical Envoy cluster if their upstream URLs share a host and scheme. The brief's "dedicated cluster per operation" assumption (file 3, bullet 3) is **false**. +3. Because of (2), even though cluster name is knowable early via the RouteConfig resource, eagerly joining `RouteConfig` (cluster name) with `PolicyChain` (oauth2-generator config) at config-load time does **not** avoid the need for collision detection — a cluster can legitimately have zero, one, or *conflicting* oauth2-generator configs attached across the routes that share it. Task 8's "one config per cluster" validation is therefore not an optional edge-case check; it is a real, structurally-reachable condition that must be enforced regardless of whether Task 6's registration point ends up eager or lazy. + +## Question 1: Is cluster name known at `PolicyChain`-build time in `handler.go`, or only per-request in `execution_context.go`? + +**Neither, exactly as framed. It is known at config-load time, but in a sibling code path, not in `buildPolicyChain`.** + +### 1a. `buildPolicyChain` / `HandlePolicyChainUpdate` never see a cluster name + +- `gateway/gateway-runtime/policy-engine/internal/xdsclient/handler.go:105` — `HandlePolicyChainUpdate` parses ADS resources of type `PolicyChainTypeURL` ("PolicyChainConfig") into `StoredPolicyConfig` (defined at line 49), whose only routing key is `policyenginev1.PolicyChain.RouteKey` (see `sdk/core/policyengine/config.go:23-31` — fields are only `RouteKey string` and `Policies []PolicyInstance`; **no cluster/upstream field exists on this type**). +- `gateway/gateway-runtime/policy-engine/internal/xdsclient/handler.go:462` — `buildPolicyChain(routeKey string, config *policyenginev1.PolicyChain, apiMetadata policyenginev1.Metadata)` builds the internal `registry.PolicyChain` (constructed at lines 585-595) from: `Policies`, `PolicySpecs`, `RequiresRequestBody`, `RequiresResponseBody`, `RequiresRequestHeader`, `RequiresResponseHeader`, `HasExecutionConditions`, `SupportsRequestStreaming`, `SupportsResponseStreaming`. **No cluster name field is read, stored, or threaded through this function at all.** It has no access to any cluster identity — not because it's "too early" but because the resource it consumes (`PolicyChainConfig`) was never designed to carry one. + +### 1b. Cluster name for the route's default upstream *is* available at config-load time, via a different xDS resource + +- `gateway/gateway-runtime/policy-engine/internal/xdsclient/handler.go:308` — `HandleRouteConfigUpdate` processes a **separate** ADS resource type, `RouteConfigTypeURL` ("RouteConfig"), also keyed by `route_key` (line 346). +- Line 371: `rc.Metadata.DefaultUpstreamCluster = getStringFromMap(data, "default_upstream_cluster")`. +- Lines 374-377: `rc.Metadata.DefaultUpstream = &info` where `info := policyenginev1.UpstreamInfoFromMap(m)` — this carries the full `UpstreamInfo{ClusterName, URL, BasePath}` (struct defined `sdk/core/policyengine/upstream.go:26`). +- Line 393: `h.kernel.ApplyWholeRouteConfigs(routeConfigs)` applies this map **atomically at config-update time** — this is a full xDS handler invoked on ADS response, exactly like `HandlePolicyChainUpdate`, not per-request. +- Upstream: gateway-controller populates this field at config-build time too, not per-request — `gateway/gateway-controller/pkg/policyxds/snapshot.go:380`: `data["default_upstream_cluster"] = route.Upstream.DefaultCluster`. + +So the *value* of a route's default cluster name is fully known and fixed once gateway-controller builds and pushes xDS config — it does not depend on any per-request signal. It just lives in `kernel.RouteConfig` (a different in-memory structure from `registry.PolicyChain`), populated by a different handler function (`HandleRouteConfigUpdate`, not `buildPolicyChain`/`HandlePolicyChainUpdate`). + +### 1c. What actually happens "per request" is a copy, not a resolution + +- `gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go:474-517` — `initializeExecutionContext`. Its own doc comment (line 475): *"Route metadata is pre-loaded via xDS RouteConfigs — no request-time parsing needed."* + - Line 484: `rc := s.kernel.GetRouteConfig(routeKey)` — looks up the config-load-time map by route key (extracted cheaply from Envoy attributes `xds.route_name` at line 478, via `extractRouteKey`, line ~519). + - Line 501: `*execCtx = newPolicyExecutionContext(s, routeKey, chain)`. + - Line 502: `(*execCtx).defaultUpstreamCluster = routeMetadata.DefaultUpstreamCluster`. + - Line 506: `(*execCtx).defaultUpstream = routeMetadata.DefaultUpstream`. + - These two lines are a plain field copy from the already-known, already-static `RouteMetadata` (struct fields at `extproc.go:581,588`) into the new per-request `PolicyExecutionContext`. There is no network call, no dynamic computation, no request-content dependency in this path. +- `gateway/gateway-runtime/policy-engine/internal/kernel/execution_context.go:1485` — `toRequestUpstream(info *policyenginev1.UpstreamInfo) *policy.UpstreamRequestContext` (and `toResponseUpstream` at line 1500) merely *reshape* this already-resolved `UpstreamInfo` into the policy-facing SDK type. Per the comment at lines 1481-1484, `Name` (cluster name) is **deliberately left unset** here — "the internal Envoy cluster name must not be exposed" to policies through this struct. This is a presentation-layer redaction, not evidence that the cluster name itself is unknown at this point — the kernel clearly has it (`execCtx.defaultUpstream.ClusterName`), it's just not forwarded to the policy-facing context. + +### 1d. The one genuinely-per-request cluster resolution: policy-driven dynamic upstream redirection + +- `gateway/gateway-runtime/policy-engine/internal/kernel/translator.go:87-108` — `resolveUpstreamRedirect(execCtx *PolicyExecutionContext, d upstreamRedirectDirective)`. This *does* compute a cluster name at request time (line 92: `clusterName := constants.UpstreamDefinitionClusterPrefix + string(apiKind) + "_" + apiId + "_" + sanitizedDefName`), but only when a policy in the chain set a dynamic-routing directive (`mods.UpstreamName`, collected at e.g. `translator.go:383`) during that specific request's processing — i.e., *which* named upstream definition a given request routes to is a genuine runtime decision (can depend on request headers/body), even though the resulting cluster name's *format* and the full set of possible cluster names for an API are fixed at config-build time (gateway-controller creates one cluster per upstream-definition per API at `gateway-controller/pkg/xds/translator.go:1190`, see Question 2 below). +- This path is the **only** place in the read files where cluster identity is genuinely unresolvable before a request arrives (because the routing decision itself is per-request), as opposed to the "default upstream" path (1b/1c) where the value is fixed at config-load time and merely copied per request. + +**Conclusion for Q1:** For the common case (a route's default, non-redirected upstream), cluster name is available at xDS config-load time, via `HandleRouteConfigUpdate` (not `buildPolicyChain`). For the dynamic-upstream-redirect case, cluster name is only settled per-request, inside `resolveUpstreamRedirect`. `PolicyChain`/`buildPolicyChain` itself never carries cluster identity in either case — it is structurally decoupled from upstream/cluster concerns entirely. + +## Question 2: Does any existing test prove or disprove one-cluster-per-route-config? + +**Disproves it.** `grep -rn "ClusterKey\|ClusterName" gateway/gateway-controller/pkg/xds/translator_test.go` returns two categories of hits: + +1. `models.RouteUpstream{ClusterKey: "main"}` (e.g. lines 792, 838, 904, 959, 1019, 1047, 1105, 1111, 2637, 3053) — this `ClusterKey` is an *unrelated* field (`gateway/gateway-controller/pkg/models/runtime_deploy_config.go:106`, comment: "key into UpstreamClusters map") used only to select "main" vs. "sandbox" within a single `RuntimeDeployConfig`. It is not an Envoy cluster name and says nothing about cluster uniqueness across routes/APIs. +2. `TestTranslator_SanitizeClusterName` (`translator_test.go:1137-1181`) — this is the only test that touches actual Envoy cluster-name generation. It calls `translator.sanitizeClusterName(hostname, scheme)` directly, in isolation, with no route, API, or policy context whatsoever (test cases: `localhost`/`http` → `cluster_http_localhost`; `api.example.com`/`https` → `cluster_https_api_example_com`; etc.). The very shape of this test — a pure function of `(hostname, scheme)` with nothing else as input — is itself evidence that Envoy cluster identity in this codebase is **not** a function of route, operation, or policy configuration at all. + +Reading `translator.go` directly confirms and extends this: + +- `gateway/gateway-controller/pkg/xds/translator.go:2634-2639` — `sanitizeClusterName(hostname, scheme)` returns `"cluster_" + scheme + "_" + sanitizedHostname`. No API ID, route, or policy signature is ever mixed in for the direct-URL ("main"/"sandbox") case. +- `translator.go:1293` — `resolveUpstreamCluster` (called once per API, per upstream slot) computes `clusterName := t.sanitizeClusterName(parsedURL.Host, parsedURL.Scheme)`. +- `translator.go:1060` — the "main" cluster name is resolved **once per API**, then reused for **every operation** of that API inside the operations loop (`translator.go:1137`, passed as `mainClusterName` into `t.createRoute(...)` for each op). So even two *operations within the same API* that might carry different per-operation `oauth2-generator` parameter overrides still share one Envoy cluster. +- `translator.go:717-799` (`TranslateConfigs`) — `clusterMap := make(map[string]*cluster.Cluster)` (line 734) is built **across every deployed API config in the loop** (`for _, cfg := range configs` at line 736), and clusters are merged into it by name with `clusterMap[c.Name] = c` (line 797) — last-write-wins, deduplicating identically-named clusters produced by *different, unrelated APIs*. If API A and API B both point their main upstream at `https://payments.internal.example.com` (same host+scheme), they collapse into the exact same `cluster_https_payments_internal_example_com` Envoy cluster, even if A and B are configured with completely different `oauth2-generator` policies (or one has it and the other doesn't). +- Contrast: upstream-definition-based clusters (used for dynamic `UpstreamName` routing) *are* scoped uniquely per API: `defClusterName := constants.UpstreamDefinitionClusterPrefix + cfg.Kind + "_" + cfg.UUID + "_" + sanitizedDefName` (`translator.go:1190`). These cannot collide across APIs. But they are still shared across every route/operation *within* that API that might select the same named upstream definition. + +**Conclusion for Q2:** The "dedicated cluster per operation" assumption in the brief (file 3) is false for the direct-URL main/sandbox upstream path, both within a single API (all operations share one cluster) and across APIs (identical host+scheme collapses into one shared cluster). It is only true, and only per-API (not per-route/operation), for the upstream-definition-based dynamic-routing path. + +## Question 3: Given (1), will Task 6's registry be populated eagerly (`handler.go`) or lazily (per-request, inside the new upstream ext_proc server)? + +**Recommendation: keep the lazy, per-request design the brief already defaults to (Task 5/6 as currently scoped) — but for a different reason than "cluster name isn't known early."** It *is* knowable early (Q1), just not inside `buildPolicyChain`. The actual blocker for a naive eager design is Q2: cluster-name granularity is coarser than route/policy granularity, so an eager join cannot assume a clean 1:1 (cluster → oauth2-generator config) mapping. + +Reasoning: + +1. **Eager registration is technically possible but requires more than reading `buildPolicyChain`.** It would mean, at config-apply time, joining two independently-arriving ADS resource types on `RouteKey` — `RouteConfig` (via `HandleRouteConfigUpdate`, carries `DefaultUpstreamCluster`) and `PolicyChain` (via `HandlePolicyChainUpdate`/`buildPolicyChain`, carries the `oauth2-generator` `PolicyInstance`) — which are handled by two separate top-level ADS handler functions in `handler.go` today, with no existing code path that correlates them. That join would need to tolerate the two resources arriving in separate ADS snapshots/versions (no code currently guarantees ordering or atomicity between `HandlePolicyChainUpdate` and `HandleRouteConfigUpdate`). +2. **Even with that join solved, Q2 means the mapping `ClusterName → oauth2-generator config` is not well-defined in general.** A given cluster name may be the target of: zero routes with `oauth2-generator` attached (nothing to register), exactly one distinct config (safe to register), or two-or-more *routes that disagree* on the config (a genuine collision that must be rejected, not silently resolved by last-write-wins). This collision detection is exactly Task 8's job, and it is required **regardless of whether registration is eager or lazy** — lazily reading `xds.cluster_name` per request inside the new upstream ext_proc server would face the identical ambiguity (which of several conflicting configs does a "first request to this cluster" belong to, if the cluster is genuinely shared?). Choosing eager registration does not remove this problem; it only moves *where* the collision would be detected (config-apply time vs. first-request time). Given that, eager detection at config-apply time is actually the better place to *catch* the problem (fail the config validation loudly, before serving traffic) rather than surface it as request-time ambiguity — but that is a Task 8 design point, not a reason to move Task 6 off its currently-planned lazy path. +3. **The dynamic-redirect case (Q1d) is genuinely only resolvable per-request** (`resolveUpstreamRedirect`), so *some* consumer of cluster identity in this feature will always need a per-request fallback path regardless of what Task 6 does for the default-upstream case. Keeping Task 6 entirely lazy avoids needing two different registration code paths (one eager for default upstreams, one lazy for redirected ones) for what is supposed to be a single registry. + +**Net effect on the plan:** Task 6/Task 5 should proceed as already scoped — lazy population inside the new upstream ext_proc server, keyed by `xds.cluster_name` from ext_proc request attributes, on first request per cluster. No design change to Task 6 is required. **Task 8 changes, however:** its "one `oauth2-generator` config per cluster" validation should be treated as a required, structurally-reachable check (proven reachable by Q2's evidence: cross-API cluster-name collisions on shared backend host+scheme are a normal, expected occurrence in this codebase, not a contrived edge case), and it is worth revisiting in Task 8 whether that validation can *additionally* be run at gateway-controller config-build time (where cluster names are actually assigned, in `resolveUpstreamCluster`/`TranslateConfigs`) rather than solely as a runtime collision check inside policy-engine or the new upstream ext_proc server — catching the conflict before config is ever pushed to the data plane is strictly better than catching it on first request. + +## Files read + +- `gateway/gateway-runtime/policy-engine/internal/xdsclient/handler.go` (in full-relevant-section detail: lines 1-180, 280-600) +- `gateway/gateway-runtime/policy-engine/internal/kernel/execution_context.go` (lines 1400-1600, plus targeted greps for `defaultUpstream`/`UpstreamInfo` across the file) +- `gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go` (lines 440-600, the route-metadata/execution-context initialization path) +- `gateway/gateway-runtime/policy-engine/internal/kernel/translator.go` (lines 1-460, the dynamic-upstream-redirect resolution path) +- `gateway/gateway-controller/pkg/xds/translator.go` (lines 1-100, 700-940, 1060-1300, 2600-2650) +- `gateway/gateway-controller/pkg/xds/translator_test.go` (lines 1137-1181, plus the `ClusterKey`/`ClusterName` grep across the whole file) +- `sdk/core/policyengine/config.go` (lines 1-40, the `PolicyChain`/`PolicyInstance` struct definitions) +- `sdk/core/policyengine/upstream.go` / `upstream_test.go` (grepped for `ClusterName` field definition) +- `gateway/gateway-controller/pkg/policyxds/snapshot.go` (grepped for `default_upstream_cluster` — confirms gateway-controller origin of the field) diff --git a/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh.md b/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh.md new file mode 100644 index 0000000000..4b234791f4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh.md @@ -0,0 +1,1406 @@ +# OAuth2 Upstream-Retry Credential Refresh Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** When a `resilience.retry`-enabled route's backend returns 401 and Envoy natively retries, attach a *freshly fetched* OAuth2 token on the retried attempt instead of resending the same stale one — without any policy code making its own outbound HTTP call, and without the client seeing an intermediate 401. + +**Architecture:** Envoy's *upstream* HTTP filter chain (configured per-cluster, distinct from the per-route/listener chain `ext_proc` already runs in) is re-created for every retry attempt — confirmed via Envoy's own docs/proto (`ext_proc.proto`'s `clear_route_cache` comment explicitly names "ext_proc filter is in the upstream filter chain" as a supported configuration; the AI Protocol Manager filter docs state an upstream-positioned filter "runs once per retry or hedged attempt"). We add a second, minimal `ext_proc` gRPC server inside the existing `policy-engine` process — separate from the main per-route kernel — wired into the cluster's upstream filter chain. It reads `x-envoy-attempt-count` (set by Envoy on the request sent to the upstream, starting at `1`, incrementing per retry) and, only when `> 1`, forces a fresh token fetch and overwrites `Authorization` before that specific attempt goes out. `oauth2-generator` implements a new, narrow SDK interface (`UpstreamRequestHeaderPolicy`) to plug into this — reusing its existing token cache/fetch code, not duplicating it. + +**Tech Stack:** Go 1.26, `google.golang.org/grpc`, `github.com/envoyproxy/go-control-plane` (ext_proc v3 proto), Envoy (stock `envoyproxy/envoy` image — no custom Envoy build), `xoauth2` (`golang.org/x/oauth2`). + +## Global Constraints + +- No custom-compiled Envoy — every mechanism used must work against the stock `envoyproxy/envoy:${ENVOY_VERSION}` image already in use (confirmed: `gateway-runtime/gateway-runtime/Dockerfile`'s `python-deps`/`production` stages both `FROM envoyproxy/envoy:${ENVOY_VERSION}`). +- Scope is narrow and explicit opt-in: only `oauth2-generator` gains this capability initially, only when a NEW `refreshOnRetry: true` policy param is set, and only takes effect when the same route also has `resilience.retry` configured with `401` (or another 4xx meaning "credential rejected") in `statusCodes`. No route gets this behavior implicitly. +- On any failure inside the new upstream refresh path (IDP unreachable, malformed config, lookup miss), fail *open* to "no header mutation" — never block or fail the retry itself. The existing native retry (resend with the old header) must still happen even if refresh fails; this feature only ever makes the retry *more likely* to succeed, never a new way for it to fail harder. +- `dev-policies/oauth2-generator` (api-platform's local mirror) and `gateway-controllers/policies/oauth2-generator` (separate repo, source of truth) must be kept in sync throughout, per this project's established dual-repo convention — diff after every change to `oauth2-generator`. +- Every new Go file follows the security/hardening rules already enforced on this repo: constructed gRPC servers must set `MaxRecvMsgSize`/`MaxSendMsgSize`/`MaxConcurrentStreams` explicitly (`go-network-service-hardening.md` directive 2); no raw token values in log output (`GO-AUTH-003`). +- Query-param mutation on retry needs no new type: `:path` is an HTTP/2 pseudo-header, not in `ext_proc`'s default header-mutation exclusion list, so it's already reachable via `UpstreamHeaderModifications.HeadersToSet[":path"]`. +- Body mutation on retry (Task 1b) is built as a sibling `UpstreamRequestBodyPolicy` extension point, matching this SDK's one-phase-one-action-type convention (`OnRequestBody`→`RequestAction`, `OnResponseBody`→`ResponseAction`) — never as a second variant on `UpstreamHeaderAction`. It has **no current consumer**: `oauth2-generator` only implements the header interface, and the xDS translator (Task 7) never configures `request_body_mode` to anything but `NONE`. Task 1b/Task 3's body-dispatch code is verified by unit tests with synthetic input only — extending the translator to request `BUFFERED` body mode for a future body-capable policy is explicitly out of scope here. + +--- + +## Phase 0: Investigation Spike (must complete before Phase 2/3 design is final) + +### Task 0: Confirm where cluster identity is resolvable at policy-config-build time vs. only at per-request time + +**Why this task exists:** the plan's registry design (Phase 2) keys refresh config by Envoy cluster name, resolved once when config loads. `sdk/core/policyengine.PolicyChain` (defined in `sdk/core/policyengine/config.go:23-31`) is keyed by `RouteKey` only — it does not carry a cluster name. Where the route's resolved cluster name actually becomes available inside `policy-engine` (at chain-build time in `internal/xdsclient/handler.go`, or only later, per-request, via `internal/kernel/execution_context.go`'s `toRequestUpstream`/`policyenginev1.UpstreamInfo`) determines whether Phase 2's registry can be populated once at config-load time (cheap) or must be resolved lazily per-request (more code, same outcome). + +**Files to read:** +- `gateway/gateway-runtime/policy-engine/internal/xdsclient/handler.go` — `buildPolicyChain` (line ~461) and `HandlePolicyChainUpdate` (line ~104): does either receive or store a cluster name alongside the `PolicyChain`? +- `gateway/gateway-runtime/policy-engine/internal/kernel/execution_context.go` — `toRequestUpstream` (line ~1485) and callers: confirm this is the *only* place `policyenginev1.UpstreamInfo.ClusterName` becomes visible, and confirm it happens per-request, not at config load. +- `gateway/gateway-controller/pkg/xds/translator.go` — confirm every `RestApi`/`LLMProvider`/`LLMProxy`/`MCP` operation maps to a **dedicated** Envoy cluster (not one cluster shared across differently-configured routes). Search for any code path where two distinct routes' `RouteAction.ClusterSpecifier` can resolve to the identical cluster name when their attached policies (specifically `oauth2-generator`) differ. This determines whether Global Constraint "one `oauth2-generator` config per cluster" can be enforced structurally or needs an explicit validation check (added in Task 8). + +- [ ] **Step 1:** Read the three files above and write a short findings doc at `docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md` answering: + 1. Is cluster name known at `PolicyChain`-build time in `handler.go`, or only per-request in `execution_context.go`? + 2. Does any existing test (`grep -rn "ClusterKey\|ClusterName" gateway/gateway-controller/pkg/xds/translator_test.go`) already prove or disprove the one-cluster-per-route-config assumption? + 3. Given the answer to (1), state definitively: will Task 6 (the upstream-refresh registry) be populated at xDS-chain-build time (`internal/xdsclient/handler.go`) or lazily on first request per cluster (inside the new upstream ext_proc server itself, Task 5)? +- [ ] **Step 2:** Commit the findings doc. + +```bash +git add docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md +git commit -m "docs: investigation findings for oauth2 upstream-retry refresh plan" +``` + +> The remaining tasks below assume the **lazy, per-request resolution** answer (safer default: the registry is populated by the new upstream ext_proc server itself, on first request per cluster, by reading `xds.cluster_name` from ext_proc request attributes — see Task 5/6). If Task 0 finds cluster name IS available at chain-build time, Task 6's registration point moves from "lazy, inside the upstream server" to "eager, inside `handler.go`'s `buildPolicyChain`" — note this as a design update in the findings doc and adjust Task 6 accordingly before starting it. + +--- + +## Phase 1: SDK primitives + +### Task 1: Add `UpstreamHeaderContext` and `UpstreamRequestHeaderPolicy` to the policy SDK + +**Files:** +- Modify: `sdk/core/policy/v1alpha2/context.go` +- Modify: `sdk/core/policy/v1alpha2/action.go` +- Create: `sdk/core/policy/v1alpha2/upstream_policy_test.go` + +**Interfaces:** +- Produces: `policy.UpstreamHeaderContext{AttemptCount int, Headers *Headers, SharedContext *SharedContext}`, `policy.UpstreamRequestHeaderPolicy` interface with method `OnUpstreamRequestHeaders(ctx context.Context, uctx *UpstreamHeaderContext, params map[string]interface{}) UpstreamHeaderAction`, sealed `UpstreamHeaderAction` interface with one concrete type `UpstreamHeaderModifications{HeadersToSet map[string]string}`. +- Consumes: existing `Headers` type (`sdk/core/policy/v1alpha2/headers.go` — already used throughout this package) and `SharedContext` (`context.go:78-119`, already defined). + +- [ ] **Step 1: Write the failing test** (`sdk/core/policy/v1alpha2/upstream_policy_test.go`) + +```go +package v1alpha2 + +import ( + "context" + "testing" +) + +// fakeUpstreamRefreshPolicy proves any type implementing +// UpstreamRequestHeaderPolicy compiles and runs against the real +// context.Context/UpstreamHeaderContext/UpstreamHeaderAction types — this is +// a compile-time contract test, not a behavioral one; oauth2-generator's own +// tests (Task 9) cover real refresh behavior. +type fakeUpstreamRefreshPolicy struct{} + +func (fakeUpstreamRefreshPolicy) OnUpstreamRequestHeaders(_ context.Context, uctx *UpstreamHeaderContext, _ map[string]interface{}) UpstreamHeaderAction { + if uctx.AttemptCount <= 1 { + return UpstreamHeaderModifications{} + } + return UpstreamHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed"}} +} + +func TestUpstreamHeaderContext_AttemptCountGatesRefresh(t *testing.T) { + var p UpstreamRequestHeaderPolicy = fakeUpstreamRefreshPolicy{} + + attemptOne := &UpstreamHeaderContext{AttemptCount: 1, Headers: NewHeaders(nil)} + action := p.OnUpstreamRequestHeaders(context.Background(), attemptOne, nil) + mods, ok := action.(UpstreamHeaderModifications) + if !ok || len(mods.HeadersToSet) != 0 { + t.Fatalf("attempt 1 must not mutate headers, got %#v", action) + } + + attemptTwo := &UpstreamHeaderContext{AttemptCount: 2, Headers: NewHeaders(nil)} + action2 := p.OnUpstreamRequestHeaders(context.Background(), attemptTwo, nil) + mods2, ok := action2.(UpstreamHeaderModifications) + if !ok || mods2.HeadersToSet["Authorization"] != "Bearer refreshed" { + t.Fatalf("attempt 2 must carry the refreshed token, got %#v", action2) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd sdk/core && go test ./policy/v1alpha2/... -run TestUpstreamHeaderContext_AttemptCountGatesRefresh -v` +Expected: FAIL with `undefined: UpstreamHeaderContext` (the types and interface referenced by the test don't exist yet). + +- [ ] **Step 3: Add the types** to `sdk/core/policy/v1alpha2/context.go`, appended after the existing `ResponseHeaderContext` block: + +```go +// UpstreamHeaderContext is passed to UpstreamRequestHeaderPolicy.OnUpstreamRequestHeaders. +// Unlike every other context in this package, it is NOT scoped to one client +// request — it fires once per individual upstream dial attempt, including +// Envoy-native retries, because it runs in Envoy's per-cluster upstream HTTP +// filter chain rather than the per-route listener chain every other policy +// phase uses. AttemptCount comes from Envoy's own x-envoy-attempt-count +// header (1 on the first attempt, incrementing per retry) — a policy that +// wants "fresh token on retry only" behavior checks AttemptCount > 1 rather +// than reacting to any response, since this phase never sees a response at +// all (see the resilience.retry design notes: response-phase code always +// runs too late to influence a retry already in flight). +type UpstreamHeaderContext struct { + *SharedContext + + // AttemptCount is Envoy's x-envoy-attempt-count for this specific dial, + // starting at 1. Always >= 1; a missing/unparseable header is treated as 1 + // (never treated as "definitely a retry") so a misconfigured route fails + // toward "behave like attempt 1" rather than toward unconditional refresh. + AttemptCount int + + // Headers are this specific attempt's outgoing request headers, mutable via + // the returned UpstreamHeaderAction. + Headers *Headers +} + +// UpstreamHeaderAction is the sealed return type for OnUpstreamRequestHeaders. +// Deliberately has exactly one variant, unlike RequestHeaderAction's two +// (Modifications | ImmediateResponse): this phase runs after routing and +// authentication are already resolved and mid-retry-loop inside Envoy's +// router filter, where there is no sensible notion of "reject this request" - +// only "optionally change headers for this one attempt." +type UpstreamHeaderAction interface { + isUpstreamHeaderAction() +} + +// UpstreamHeaderModifications sets the given headers on this specific +// upstream attempt. An empty/nil HeadersToSet is a valid, common no-op +// (e.g. AttemptCount == 1, nothing to refresh yet). +type UpstreamHeaderModifications struct { + HeadersToSet map[string]string +} + +func (UpstreamHeaderModifications) isUpstreamHeaderAction() {} +``` + +- [ ] **Step 4:** Add the interface to `sdk/core/policy/v1alpha2/action.go` (or wherever `RequestHeaderPolicy`/`ResponseHeaderPolicy` interfaces are defined in that file — match the existing location): + +```go +// UpstreamRequestHeaderPolicy is implemented by policies that must re-run +// before every individual upstream dial attempt, not just once per +// downstream request — currently only used for credential refresh on +// Envoy-native retry. A policy implementing this interface is invoked from +// a completely different Envoy filter-chain position than every other +// policy phase (see UpstreamHeaderContext's doc comment); most policies +// should never implement this. +type UpstreamRequestHeaderPolicy interface { + OnUpstreamRequestHeaders(ctx context.Context, uctx *UpstreamHeaderContext, params map[string]interface{}) UpstreamHeaderAction +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `cd sdk/core && go test ./policy/v1alpha2/... -run TestUpstreamHeaderContext_AttemptCountGatesRefresh -v` +Expected: PASS + +- [ ] **Step 6: Run the full package test suite to confirm no regression** + +Run: `cd sdk/core && go test ./policy/... -v` +Expected: all existing tests still PASS (no changes to any existing type). + +- [ ] **Step 7: Commit** + +```bash +git add sdk/core/policy/v1alpha2/context.go sdk/core/policy/v1alpha2/action.go sdk/core/policy/v1alpha2/upstream_policy_test.go +git commit -m "feat(sdk): add UpstreamRequestHeaderPolicy for per-retry-attempt credential refresh" +``` + +### Task 1b: Add `UpstreamRequestBodyPolicy` — a sibling body-mutation extension point (no current consumer) + +**Added after Task 1 was already reviewed and committed**, per an explicit design discussion: query-param mutation needs no new type at all (`:path` is an HTTP/2 pseudo-header, not in `ext_proc`'s default header-mutation exclusion list — `host`/`:authority`/`:scheme`/`:method` only — so it's already reachable via Task 1's existing `UpstreamHeaderModifications.HeadersToSet[":path"]`). Body mutation is genuinely new surface, added here as a **sibling** interface+context+action triple, matching this SDK's established one-phase-one-action-type convention (`OnRequestBody` → `RequestAction`, `OnResponseBody` → `ResponseAction`, `OnRequestBodyChunk` → `StreamingRequestAction` — confirmed in `sdk/core/policy/v1alpha2/interface.go`/`action.go`) — never as a second variant squeezed into the existing `UpstreamHeaderAction`. + +**IMPORTANT — this task has no current consumer and cannot be verified end-to-end.** `oauth2-generator` (the only policy this plan implements) only ever needs `Authorization`, so it will never implement `UpstreamRequestBodyPolicy`. Task 7's `buildUpstreamRefreshExtProc` continues to set `RequestBodyMode: NONE` unconditionally — wiring the xDS translator to turn on `BUFFERED` body mode for a future body-capable policy's cluster is explicitly **out of scope** for this plan (leave a comment marking it, don't build it speculatively). This task and Task 3's corresponding server-side dispatch are verified by **unit tests with synthetic input only** — real Envoy traffic will never invoke `OnUpstreamRequestBody` until a future plan both implements a consumer AND extends the translator. State this limitation in the code's doc comments, not just here. + +**Files:** +- Modify: `sdk/core/policy/v1alpha2/context.go` +- Modify: `sdk/core/policy/v1alpha2/interface.go` +- Modify: `sdk/core/policy/v1alpha2/upstream_policy_test.go` + +**Interfaces:** +- Produces: `policy.UpstreamBodyContext{AttemptCount int, Body []byte, SharedContext *SharedContext}`, `policy.UpstreamRequestBodyPolicy` interface with method `OnUpstreamRequestBody(ctx context.Context, bctx *UpstreamBodyContext, params map[string]interface{}) UpstreamBodyAction`, sealed `UpstreamBodyAction` interface with one concrete type `UpstreamBodyModifications{Body []byte}`. +- Consumes: nothing new — same package as Task 1, additive only. + +- [ ] **Step 1: Write the failing test**, appended to `upstream_policy_test.go`: + +```go +type fakeBodyRefreshPolicy struct{} + +func (fakeBodyRefreshPolicy) OnUpstreamRequestBody(_ context.Context, bctx *UpstreamBodyContext, _ map[string]interface{}) UpstreamBodyAction { + if bctx.AttemptCount <= 1 { + return UpstreamBodyModifications{} + } + return UpstreamBodyModifications{Body: []byte(`{"signed":"fresh"}`)} +} + +func TestUpstreamBodyContext_AttemptCountGatesRefresh(t *testing.T) { + var p UpstreamRequestBodyPolicy = fakeBodyRefreshPolicy{} + + attemptOne := &UpstreamBodyContext{AttemptCount: 1, Body: []byte(`{"signed":"stale"}`)} + action := p.OnUpstreamRequestBody(context.Background(), attemptOne, nil) + mods, ok := action.(UpstreamBodyModifications) + if !ok || mods.Body != nil { + t.Fatalf("attempt 1 must not mutate the body, got %#v", action) + } + + attemptTwo := &UpstreamBodyContext{AttemptCount: 2, Body: []byte(`{"signed":"stale"}`)} + action2 := p.OnUpstreamRequestBody(context.Background(), attemptTwo, nil) + mods2, ok := action2.(UpstreamBodyModifications) + if !ok || string(mods2.Body) != `{"signed":"fresh"}` { + t.Fatalf("attempt 2 must carry the refreshed body, got %#v", action2) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd sdk/core && go test ./policy/v1alpha2/... -run TestUpstreamBodyContext_AttemptCountGatesRefresh -v` +Expected: FAIL — types don't exist yet. + +- [ ] **Step 3: Add the types** to `context.go`, appended after the `UpstreamHeaderModifications` block Task 1 added: + +```go +// UpstreamBodyContext is passed to UpstreamRequestBodyPolicy.OnUpstreamRequestBody. +// Sibling to UpstreamHeaderContext (same per-retry-attempt firing semantics — +// see its doc comment), kept as a SEPARATE interface+context+action triple +// rather than a second variant on UpstreamHeaderAction, matching this SDK's +// one-phase-one-action-type convention elsewhere (RequestAction/ResponseAction +// for the body phases, StreamingRequestAction/StreamingResponseAction for the +// streaming ones). +// +// NOTE: as of this plan, no policy implements this interface and +// gateway-controller's xDS translator never configures request_body_mode to +// anything but NONE (see buildUpstreamRefreshExtProc) - this type and its +// server-side dispatch (policy-engine's internal/upstreamproc) exist as an +// extension point verified by unit tests with synthetic input only. Real +// Envoy traffic will not invoke OnUpstreamRequestBody until a future policy +// implements this interface AND the translator is extended to request +// BUFFERED body mode for that policy's cluster - that translator change is +// out of scope here. +type UpstreamBodyContext struct { + *SharedContext + + // AttemptCount - see UpstreamHeaderContext.AttemptCount; identical semantics. + AttemptCount int + + // Body is this specific attempt's outgoing request body. + Body []byte +} + +// UpstreamBodyAction is the sealed return type for OnUpstreamRequestBody. +// Single variant for the same reason UpstreamHeaderAction is: no sensible +// "reject" semantics this deep in Envoy's retry loop. +type UpstreamBodyAction interface { + isUpstreamBodyAction() +} + +// UpstreamBodyModifications replaces this specific attempt's outgoing body. +// A nil Body is a valid, common no-op (e.g. AttemptCount == 1). +type UpstreamBodyModifications struct { + Body []byte +} + +func (UpstreamBodyModifications) isUpstreamBodyAction() {} +``` + +- [ ] **Step 4:** Add the interface to `interface.go`, next to `UpstreamRequestHeaderPolicy` (Task 1 placed that interface here, not in `action.go` — confirmed correct by Task 1's review; follow the same placement): + +```go +// UpstreamRequestBodyPolicy is the body-mutation sibling of +// UpstreamRequestHeaderPolicy - see that interface's doc comment for the +// shared per-retry-attempt firing model. No policy in this codebase +// implements this yet; see UpstreamBodyContext's doc comment for why real +// traffic cannot reach it until a future plan wires up both a consumer and +// translator support for BUFFERED request_body_mode. +type UpstreamRequestBodyPolicy interface { + OnUpstreamRequestBody(ctx context.Context, bctx *UpstreamBodyContext, params map[string]interface{}) UpstreamBodyAction +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `cd sdk/core && go test ./policy/v1alpha2/... -run TestUpstreamBodyContext_AttemptCountGatesRefresh -v` +Expected: PASS + +- [ ] **Step 6: Run the full package suite** + +Run: `cd sdk/core && go test ./policy/... -v` +Expected: all PASS, zero regressions. + +- [ ] **Step 7: Commit** + +```bash +git add sdk/core/policy/v1alpha2/context.go sdk/core/policy/v1alpha2/interface.go sdk/core/policy/v1alpha2/upstream_policy_test.go +git commit -m "feat(sdk): add UpstreamRequestBodyPolicy sibling extension point (no current consumer)" +``` + +--- + +## Phase 2: policy-engine — minimal upstream `ext_proc` server + +### Task 2: Add the upstream-refresh registry (cluster name -> registered policy + resolved params) + +**Registration accepts a plain policy instance and type-asserts against both `UpstreamRequestHeaderPolicy` (Task 1) and `UpstreamRequestBodyPolicy` (Task 1b) independently — a registered policy may implement one, the other, or (uncommonly) both.** `oauth2-generator` (Task 9) only ever implements the header interface, so its registrations will always have a nil body policy — this is the expected, common case, not an error. + +**Files:** +- Create: `gateway/gateway-runtime/policy-engine/internal/upstreamrefresh/registry.go` +- Create: `gateway/gateway-runtime/policy-engine/internal/upstreamrefresh/registry_test.go` + +**Interfaces:** +- Produces: `upstreamrefresh.Registry` with `Register(clusterName string, p interface{}, params map[string]interface{})` (accepts any policy instance; internally type-asserts) and `Lookup(clusterName string) (headerPolicy policy.UpstreamRequestHeaderPolicy, bodyPolicy policy.UpstreamRequestBodyPolicy, params map[string]interface{}, ok bool)`, and a package-level `Default() *Registry` singleton (mirrors `sdk/core/utils/redisclient`'s `Shared()` pattern already established in this codebase for exactly this "one process-wide instance" need). `ok` is true whenever *either* `headerPolicy` or `bodyPolicy` is non-nil; `Register` itself returns an error if `p` implements neither interface (nothing to register). +- Consumes: `policy.UpstreamRequestHeaderPolicy` (Task 1), `policy.UpstreamRequestBodyPolicy` (Task 1b). + +- [ ] **Step 1: Write the failing test** + +```go +package upstreamrefresh + +import ( + "context" + "testing" + + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" +) + +// stubHeaderOnlyPolicy is the common case - implements only the header +// interface, exactly like oauth2-generator. +type stubHeaderOnlyPolicy struct{} + +func (stubHeaderOnlyPolicy) OnUpstreamRequestHeaders(context.Context, *policy.UpstreamHeaderContext, map[string]interface{}) policy.UpstreamHeaderAction { + return policy.UpstreamHeaderModifications{} +} + +// stubBothPolicy implements both interfaces, proving Register/Lookup handle +// a policy that satisfies both without forcing a choice. +type stubBothPolicy struct{ stubHeaderOnlyPolicy } + +func (stubBothPolicy) OnUpstreamRequestBody(context.Context, *policy.UpstreamBodyContext, map[string]interface{}) policy.UpstreamBodyAction { + return policy.UpstreamBodyModifications{} +} + +func TestRegistry_RegisterAndLookup_HeaderOnly(t *testing.T) { + r := NewRegistry() + params := map[string]interface{}{"tokenEndpoint": "http://idp/token"} + if err := r.Register("cluster-a", stubHeaderOnlyPolicy{}, params); err != nil { + t.Fatalf("Register failed: %v", err) + } + + headerPolicy, bodyPolicy, gotParams, ok := r.Lookup("cluster-a") + if !ok { + t.Fatal("expected cluster-a to be registered") + } + if headerPolicy == nil { + t.Fatal("expected a non-nil header policy") + } + if bodyPolicy != nil { + t.Fatal("stubHeaderOnlyPolicy does not implement the body interface, expected nil") + } + if gotParams["tokenEndpoint"] != "http://idp/token" { + t.Fatalf("expected params round-trip, got %v", gotParams) + } + + if _, _, _, ok := r.Lookup("cluster-b"); ok { + t.Fatal("cluster-b was never registered, Lookup must report false") + } +} + +func TestRegistry_RegisterAndLookup_Both(t *testing.T) { + r := NewRegistry() + if err := r.Register("cluster-a", stubBothPolicy{}, nil); err != nil { + t.Fatalf("Register failed: %v", err) + } + headerPolicy, bodyPolicy, _, ok := r.Lookup("cluster-a") + if !ok || headerPolicy == nil || bodyPolicy == nil { + t.Fatalf("expected both policies non-nil, got header=%v body=%v ok=%v", headerPolicy, bodyPolicy, ok) + } +} + +func TestRegistry_Register_RejectsPolicyImplementingNeitherInterface(t *testing.T) { + r := NewRegistry() + if err := r.Register("cluster-a", struct{}{}, nil); err == nil { + t.Fatal("expected an error registering a value that implements neither upstream interface") + } +} + +func TestRegistry_ReRegisterReplaces(t *testing.T) { + r := NewRegistry() + _ = r.Register("cluster-a", stubHeaderOnlyPolicy{}, map[string]interface{}{"v": 1}) + _ = r.Register("cluster-a", stubHeaderOnlyPolicy{}, map[string]interface{}{"v": 2}) + + _, _, params, ok := r.Lookup("cluster-a") + if !ok || params["v"] != 2 { + t.Fatalf("expected the second Register call to win, got %v", params) + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway/gateway-runtime/policy-engine && go test ./internal/upstreamrefresh/... -v` +Expected: FAIL — package/types don't exist. + +- [ ] **Step 3: Implement** + +```go +// Package upstreamrefresh holds the process-wide registry mapping an Envoy +// cluster name to the policy (or policies) responsible for refreshing +// credentials on that cluster's retried upstream requests. Populated lazily +// by the upstream ext_proc server (see internal/upstreamproc) the first time +// it sees a request for a given cluster - see Phase 0's investigation task +// for why this is lazy rather than populated at xDS chain-build time. +package upstreamrefresh + +import ( + "fmt" + "sync" + + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" +) + +type entry struct { + headerPolicy policy.UpstreamRequestHeaderPolicy // nil if p doesn't implement it + bodyPolicy policy.UpstreamRequestBodyPolicy // nil if p doesn't implement it - always nil for oauth2-generator today + params map[string]interface{} +} + +// Registry is safe for concurrent Register/Lookup calls. +type Registry struct { + mu sync.RWMutex + m map[string]entry +} + +func NewRegistry() *Registry { + return &Registry{m: make(map[string]entry)} +} + +// Register associates clusterName with whichever of UpstreamRequestHeaderPolicy/ +// UpstreamRequestBodyPolicy p implements, replacing any prior registration for +// the same cluster - a config reload always wins over what was previously +// registered. Returns an error if p implements neither interface, since that +// means the caller passed something that was never meant to be registered here. +func (r *Registry) Register(clusterName string, p interface{}, params map[string]interface{}) error { + headerPolicy, _ := p.(policy.UpstreamRequestHeaderPolicy) + bodyPolicy, _ := p.(policy.UpstreamRequestBodyPolicy) + if headerPolicy == nil && bodyPolicy == nil { + return fmt.Errorf("upstreamrefresh: policy %T implements neither UpstreamRequestHeaderPolicy nor UpstreamRequestBodyPolicy", p) + } + r.mu.Lock() + defer r.mu.Unlock() + r.m[clusterName] = entry{headerPolicy: headerPolicy, bodyPolicy: bodyPolicy, params: params} + return nil +} + +// Lookup returns the registered header/body policies and params for +// clusterName. ok is true whenever either policy is non-nil. The caller (the +// upstream ext_proc server) must treat ok == false as "no refresh configured +// for this cluster" and pass the request through unmodified, never as an +// error - and must independently nil-check headerPolicy/bodyPolicy before +// using either, since a registration commonly has only one of the two set. +func (r *Registry) Lookup(clusterName string) (headerPolicy policy.UpstreamRequestHeaderPolicy, bodyPolicy policy.UpstreamRequestBodyPolicy, params map[string]interface{}, ok bool) { + r.mu.RLock() + defer r.mu.RUnlock() + e, found := r.m[clusterName] + if !found { + return nil, nil, nil, false + } + return e.headerPolicy, e.bodyPolicy, e.params, e.headerPolicy != nil || e.bodyPolicy != nil +} + +var ( + defaultOnce sync.Once + defaultReg *Registry +) + +// Default returns the process-wide singleton registry - one per policy-engine +// process, mirroring sdk/core/utils/redisclient.Shared()'s pattern. +func Default() *Registry { + defaultOnce.Do(func() { defaultReg = NewRegistry() }) + return defaultReg +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway/gateway-runtime/policy-engine && go test ./internal/upstreamrefresh/... -v` +Expected: PASS (all four tests) + +- [ ] **Step 5: Commit** + +```bash +git add gateway/gateway-runtime/policy-engine/internal/upstreamrefresh/ +git commit -m "feat(policy-engine): add cluster-keyed upstream-refresh registry (header + body)" +``` + +### Task 3: Add the minimal upstream `ext_proc` gRPC server + +**Files:** +- Create: `gateway/gateway-runtime/policy-engine/internal/upstreamproc/server.go` +- Create: `gateway/gateway-runtime/policy-engine/internal/upstreamproc/server_test.go` + +**Interfaces:** +- Consumes: `upstreamrefresh.Default()` (Task 2), `policy.UpstreamHeaderContext`/`UpstreamHeaderModifications` (Task 1), `policy.UpstreamBodyContext`/`UpstreamBodyModifications` (Task 1b). +- Produces: `upstreamproc.NewServer(reg *upstreamrefresh.Registry) *Server` implementing `extprocv3.ExternalProcessorServer` (the same interface `internal/kernel.ExternalProcessorServer` already implements — confirmed at `internal/kernel/extproc.go:102`). + +This is intentionally a *separate, minimal* implementation, not a mode of the existing `kernel.ExternalProcessorServer` — the existing one models the full multi-phase, multi-policy, body-aware pipeline for the per-route listener chain; this one only ever handles `RequestHeaders` and (when a body-capable policy is registered — see Task 1b) `RequestBody`, never response or trailers, because Envoy only asks the upstream filter chain to process what the cluster's `processing_mode` requests. + +**On the `RequestBody` branch below: this server implements it, but no real Envoy traffic will ever send it a `RequestBody` message in this plan.** Task 7's translator always configures `RequestBodyMode: NONE` (no policy in this plan implements `UpstreamRequestBodyPolicy` — see Task 1b). The `RequestBody` handling exists as a verified-by-unit-test-only extension point for a future body-capable policy; implement and test it the same way as the header path, but don't treat "no production trigger for it yet" as a reason to skip testing it. + +- [ ] **Step 1: Write the failing test** (fakes the gRPC stream directly rather than standing up a real listener, matching how `internal/kernel/extproc.go`'s own tests are structured — check `internal/kernel/extproc_test.go` for the existing fake-stream pattern and reuse it): + +```go +package upstreamproc + +import ( + "context" + "io" + "testing" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/upstreamrefresh" +) + +type fakeRefreshPolicy struct{ calls int } + +func (p *fakeRefreshPolicy) OnUpstreamRequestHeaders(_ context.Context, uctx *policy.UpstreamHeaderContext, _ map[string]interface{}) policy.UpstreamHeaderAction { + p.calls++ + if uctx.AttemptCount <= 1 { + return policy.UpstreamHeaderModifications{} + } + return policy.UpstreamHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed-token"}} +} + +// fakeBodyRefreshPolicy proves the RequestBody dispatch branch works via +// synthetic input only - see Task 1b's note: no real Envoy config in this +// plan ever sends this server a RequestBody message. +type fakeBodyRefreshPolicy struct{ calls int } + +func (p *fakeBodyRefreshPolicy) OnUpstreamRequestBody(_ context.Context, bctx *policy.UpstreamBodyContext, _ map[string]interface{}) policy.UpstreamBodyAction { + p.calls++ + if bctx.AttemptCount <= 1 { + return policy.UpstreamBodyModifications{} + } + return policy.UpstreamBodyModifications{Body: []byte(`{"signed":"fresh"}`)} +} + +// fakeProcessStream implements extprocv3.ExternalProcessor_ProcessServer +// with a scripted single request/response exchange - see +// internal/kernel/extproc_test.go for the established fake used by the +// main kernel's own tests; reuse that exact type instead of redefining one +// here if it is already exported/reusable across packages. +func TestServer_Process_AttemptOne_NoMutation(t *testing.T) { + reg := upstreamrefresh.NewRegistry() + p := &fakeRefreshPolicy{} + if err := reg.Register("test-cluster", p, nil); err != nil { + t.Fatalf("Register failed: %v", err) + } + srv := NewServer(reg) + + req := &extprocv3.ProcessingRequest{ + Attributes: map[string]*structpbStruct{ // see Step 3 for the real import path + "envoy.filters.http.ext_proc": {}, + }, + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: "x-envoy-attempt-count", Value: "1"}, + }}, + }, + }, + } + stream := newFakeStream(t, req) + if err := srv.Process(stream); err != nil && err != io.EOF { + t.Fatalf("Process returned error: %v", err) + } + if p.calls != 1 { + t.Fatalf("expected policy to be invoked exactly once, got %d", p.calls) + } + resp := stream.sentResponses[0] + hm := resp.GetRequestHeaders().GetResponse().GetHeaderMutation() + if hm != nil && len(hm.SetHeaders) != 0 { + t.Fatalf("attempt 1 must not mutate headers, got %v", hm.SetHeaders) + } +} + +func TestServer_Process_AttemptTwo_RefreshesAuthorization(t *testing.T) { + reg := upstreamrefresh.NewRegistry() + p := &fakeRefreshPolicy{} + if err := reg.Register("test-cluster", p, nil); err != nil { + t.Fatalf("Register failed: %v", err) + } + srv := NewServer(reg) + + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: "x-envoy-attempt-count", Value: "2"}, + }}, + }, + }, + } + stream := newFakeStream(t, req) + if err := srv.Process(stream); err != nil && err != io.EOF { + t.Fatalf("Process returned error: %v", err) + } + resp := stream.sentResponses[0] + hm := resp.GetRequestHeaders().GetResponse().GetHeaderMutation() + found := false + for _, h := range hm.GetSetHeaders() { + if h.GetHeader().GetKey() == "Authorization" && h.GetHeader().GetValue() == "Bearer refreshed-token" { + found = true + } + } + if !found { + t.Fatalf("expected Authorization to be set to the refreshed token, got %v", hm) + } +} + +func TestServer_Process_UnregisteredCluster_NoMutation_NoError(t *testing.T) { + reg := upstreamrefresh.NewRegistry() // nothing registered + srv := NewServer(reg) + + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: "x-envoy-attempt-count", Value: "2"}, + }}, + }, + }, + } + stream := newFakeStream(t, req) + if err := srv.Process(stream); err != nil && err != io.EOF { + t.Fatalf("Process must never error on an unregistered cluster (fail-open): %v", err) + } +} + +// TestServer_Process_RequestBody_AttemptTwo_RefreshesBody proves the +// RequestBody dispatch branch (Task 1b) via synthetic input - no real Envoy +// config in this plan ever sends this server a RequestBody message (see +// this task's note above), but the branch must still work when driven +// directly, the same as the header path. +func TestServer_Process_RequestBody_AttemptTwo_RefreshesBody(t *testing.T) { + reg := upstreamrefresh.NewRegistry() + p := &fakeBodyRefreshPolicy{} + if err := reg.Register("test-cluster", p, nil); err != nil { + t.Fatalf("Register failed: %v", err) + } + srv := NewServer(reg) + + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_RequestBody{ + RequestBody: &extprocv3.HttpBody{Body: []byte(`{"signed":"stale"}`)}, + }, + } + // AttemptCount for a RequestBody message comes from the same + // x-envoy-attempt-count header carried on the RequestHeaders message + // earlier in the same stream - the fake stream helper must let this test + // script a RequestHeaders message first (attempt 2), then this + // RequestBody message, and the server must remember the attempt count + // across the two Recv() calls within one stream (store it on a + // per-stream variable inside Process, not a package/global). + stream := newFakeStreamWithAttemptCount(t, 2, req) + if err := srv.Process(stream); err != nil && err != io.EOF { + t.Fatalf("Process returned error: %v", err) + } + if p.calls != 1 { + t.Fatalf("expected body policy to be invoked exactly once, got %d", p.calls) + } + resp := stream.sentResponses[0] + bm := resp.GetRequestBody().GetResponse().GetBodyMutation() + if bm == nil || string(bm.GetBody()) != `{"signed":"fresh"}` { + t.Fatalf("expected body replaced with the refreshed value, got %v", bm) + } +} +``` + +> `newFakeStreamWithAttemptCount` is illustrative - design the actual fake-stream test helper (built from `internal/kernel/extproc_test.go`'s pattern per this task's earlier note) so a test can script a `RequestHeaders` message followed by a `RequestBody` message on the same stream, and confirm your `Server.Process` implementation carries the attempt count across `Recv()` calls within one stream (a single `Process` call handles the whole bidi stream in a loop - see the `Step 3` implementation below for where per-stream state needs to live). + +> **Before writing this test for real**: read `internal/kernel/extproc_test.go` in full first and copy its exact fake-stream helper and its exact way of extracting the cluster name from `ProcessingRequest.Attributes` (populated via `request_attributes: ["xds.cluster_name"]`, configured in Task 5) — the sketch above uses a placeholder `structpbStruct` type name and a `newFakeStream` helper precisely because those exact names must match whatever the existing kernel test file already established, not be reinvented. This is the one piece of this task that depends on Task 0-style investigation of an existing file rather than new design. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway/gateway-runtime/policy-engine && go test ./internal/upstreamproc/... -v` +Expected: FAIL — package doesn't exist. + +- [ ] **Step 3: Implement** `internal/upstreamproc/server.go`: + +```go +// Package upstreamproc implements a second, minimal ext_proc gRPC service - +// registered on Envoy's per-cluster UPSTREAM filter chain, never the +// per-route listener chain the main kernel (internal/kernel) serves. It +// handles RequestHeaders always, and RequestBody only when a body-capable +// policy is registered for the stream's cluster (see internal/upstreamrefresh +// and Task 1b) - no production Envoy config in this plan ever requests +// RequestBody (Task 7 always sets request_body_mode: NONE), so that branch +// is exercised by unit tests only, not real traffic, until a future plan +// adds both a body-capable policy and translator support for BUFFERED mode. +package upstreamproc + +import ( + "strconv" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/upstreamrefresh" +) + +const clusterNameAttribute = "xds.cluster_name" + +type Server struct { + extprocv3.UnimplementedExternalProcessorServer + reg *upstreamrefresh.Registry +} + +func NewServer(reg *upstreamrefresh.Registry) *Server { + return &Server{reg: reg} +} + +// Process handles one bidi ext_proc stream. clusterName and attemptCount are +// resolved once, from the RequestHeaders message, and carried as per-stream +// local state into the loop's later iterations - a RequestBody message on +// the same stream carries neither its own attempt-count header nor cluster +// attribute (per the ext_proc protocol, those are RequestHeaders-message-only +// fields), so this is the only place they can be captured. +func (s *Server) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + var clusterName string + var attemptCount int = 1 + + for { + req, err := stream.Recv() + if err != nil { + return err // io.EOF on clean stream close - same convention as internal/kernel's Process loop + } + + switch { + case req.GetRequestHeaders() != nil: + headers := req.GetRequestHeaders().GetHeaders() + clusterName = clusterNameFromAttributes(req) + attemptCount = attemptCountFromHeaders(headers) + + resp := &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{}, + }, + } + + headerPolicy, _, params, ok := s.reg.Lookup(clusterName) + if ok && headerPolicy != nil { + uctx := &policy.UpstreamHeaderContext{AttemptCount: attemptCount} + action := headerPolicy.OnUpstreamRequestHeaders(stream.Context(), uctx, params) + if mods, ok := action.(policy.UpstreamHeaderModifications); ok && len(mods.HeadersToSet) > 0 { + resp.GetRequestHeaders().Response = &extprocv3.CommonResponse{ + HeaderMutation: &extprocv3.HeaderMutation{ + SetHeaders: setHeaders(mods.HeadersToSet), + }, + } + } + } + // !ok, or ok but headerPolicy is nil (only a body policy registered + // for this cluster): resp stays a bare no-op HeadersResponse - fail + // open, never an error. + + if err := stream.Send(resp); err != nil { + return err + } + + case req.GetRequestBody() != nil: + // See this task's file-level doc comment: unreachable by real + // traffic in this plan (request_body_mode is always NONE), kept + // correct and tested for the future policy that will use it. + resp := &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestBody{ + RequestBody: &extprocv3.BodyResponse{}, + }, + } + + _, bodyPolicy, params, ok := s.reg.Lookup(clusterName) + if ok && bodyPolicy != nil { + bctx := &policy.UpstreamBodyContext{AttemptCount: attemptCount, Body: req.GetRequestBody().GetBody()} + action := bodyPolicy.OnUpstreamRequestBody(stream.Context(), bctx, params) + if mods, ok := action.(policy.UpstreamBodyModifications); ok && mods.Body != nil { + resp.GetRequestBody().Response = &extprocv3.CommonResponse{ + Status: extprocv3.CommonResponse_CONTINUE_AND_REPLACE, + BodyMutation: &extprocv3.BodyMutation{ + Mutation: &extprocv3.BodyMutation_Body{Body: mods.Body}, + }, + } + } + } + + if err := stream.Send(resp); err != nil { + return err + } + + default: + // Neither RequestHeaders nor RequestBody (shouldn't happen given + // processing_mode, but fail open rather than erroring the stream). + if err := stream.Send(&extprocv3.ProcessingResponse{}); err != nil { + return err + } + } + } +} + +func attemptCountFromHeaders(h *corev3.HeaderMap) int { + for _, hv := range h.GetHeaders() { + if hv.GetKey() == "x-envoy-attempt-count" { + if n, err := strconv.Atoi(hv.GetValue()); err == nil && n > 0 { + return n + } + } + } + return 1 // missing/unparseable -> treat as attempt 1, never as "definitely a retry" +} + +func clusterNameFromAttributes(req *extprocv3.ProcessingRequest) string { + for _, fields := range req.GetAttributes() { + if v, ok := fields.GetFields()[clusterNameAttribute]; ok { + return v.GetStringValue() + } + } + return "" +} + +func setHeaders(headers map[string]string) []*corev3.HeaderValueOption { + out := make([]*corev3.HeaderValueOption, 0, len(headers)) + for k, v := range headers { + out = append(out, &corev3.HeaderValueOption{ + Header: &corev3.HeaderValue{Key: k, Value: v}, + }) + } + return out +} +``` + +> Confirm the exact `ProcessingRequest.Attributes` shape (`map[string]*structpb.Struct`, field access via `.GetFields()[...]`) against the go-control-plane version already vendored in `policy-engine/go.mod` before finalizing this file - the sketch above is written from the proto definitions read during Phase-0-adjacent research in this plan's originating conversation, not from a compiled build. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway/gateway-runtime/policy-engine && go test ./internal/upstreamproc/... -v` +Expected: PASS (all four tests: attempt-one no-mutation, attempt-two header refresh, unregistered cluster no-op, and the RequestBody dispatch test) + +- [ ] **Step 5: Commit** + +```bash +git add gateway/gateway-runtime/policy-engine/internal/upstreamproc/ +git commit -m "feat(policy-engine): minimal upstream ext_proc server for per-retry credential refresh" +``` + +### Task 4: Wire the new server into `policy-engine`'s startup on a second Unix socket + +**Files:** +- Modify: `gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go` +- Modify: `gateway/gateway-runtime/policy-engine/internal/constants/constants.go` +- Modify: `gateway/gateway-runtime/docker-entrypoint.sh` + +**Interfaces:** +- Consumes: `upstreamproc.NewServer` (Task 3), `upstreamrefresh.Default()` (Task 2). + +- [ ] **Step 1:** Add the new socket path constant next to the existing one in `internal/constants/constants.go`: + +```go +// DefaultUpstreamProcSocketPath is the second ext_proc socket, dedicated to +// the per-retry-attempt credential-refresh server (internal/upstreamproc) - +// registered on Envoy's per-cluster upstream filter chain, never the +// per-route listener chain DefaultPolicyEngineSocketPath serves. +DefaultUpstreamProcSocketPath = "/var/run/api-platform/policy-engine-upstream.sock" +``` + +(Add immediately after the existing `DefaultPolicyEngineSocketPath = "/var/run/api-platform/policy-engine.sock"` at line 51.) + +- [ ] **Step 2:** In `cmd/policy-engine/main.go`, immediately after the existing `grpcServer := grpc.NewServer()` / `extprocv3.RegisterExternalProcessorServer(grpcServer, extprocServer)` block (lines 271-272), add a second listener + server for the upstream-refresh service. Per `go-network-service-hardening.md` directive 2, both this and the existing `grpcServer` construction must set explicit message-size/stream limits — if the existing `grpcServer := grpc.NewServer()` at line 271 doesn't already set these (check before assuming), add them to both in this same task rather than only the new one: + +```go +upstreamGrpcServer := grpc.NewServer( + grpc.MaxRecvMsgSize(1<<20), // 1MiB - this server only ever handles headers, never a body + grpc.MaxSendMsgSize(1<<20), + grpc.MaxConcurrentStreams(1024), +) +upstreamprocv3.RegisterExternalProcessorServer(upstreamGrpcServer, upstreamproc.NewServer(upstreamrefresh.Default())) + +upstreamSocketPath := constants.DefaultUpstreamProcSocketPath +if err := os.Remove(upstreamSocketPath); err != nil && !os.IsNotExist(err) { + slog.WarnContext(ctx, "Failed to remove existing upstream-refresh socket file", "path", upstreamSocketPath, "error", err) +} +upstreamLis, err := net.Listen("unix", upstreamSocketPath) +if err != nil { + slog.ErrorContext(ctx, "Failed to listen on upstream-refresh Unix socket", "path", upstreamSocketPath, "error", err) + os.Exit(1) +} +if err := os.Chmod(upstreamSocketPath, 0660); err != nil { + slog.WarnContext(ctx, "Failed to set upstream-refresh socket permissions", "path", upstreamSocketPath, "error", err) +} +go func() { + if err := upstreamGrpcServer.Serve(upstreamLis); err != nil { + slog.ErrorContext(ctx, "upstream-refresh gRPC server error", "error", err) + } +}() +slog.InfoContext(ctx, "Policy Engine upstream-refresh server listening on Unix socket", "path", upstreamSocketPath) +``` + +Add `"github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/upstreamproc"` and `"github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/upstreamrefresh"` to the import block, and confirm whether `extprocv3` is already aliased for the main server's import — reuse the identical import for `upstreamprocv3` (it's the same proto package, `RegisterExternalProcessorServer` registers a distinct service instance on a distinct `grpc.Server`, so no alias collision even though it's the same generated type). + +> This intentionally does NOT touch `tcp` mode (lines 260-269) - the upstream-refresh socket is UDS-only for the initial version, since Envoy's cluster-level `grpc_service` can point at a UDS regardless of how the main ext_proc filter is configured. Extend to TCP mode only if a real deployment needs it (YAGNI otherwise). + +- [ ] **Step 3:** Update `gateway-runtime/docker-entrypoint.sh` to wait for the new socket the same way it already waits for `POLICY_ENGINE_SOCKET` (lines 277-298). Add immediately after that existing wait loop, before "Starting Envoy...": + +```bash +UPSTREAM_PROC_SOCKET="/var/run/api-platform/policy-engine-upstream.sock" +UPSTREAM_SOCKET_WAIT_TIMEOUT=10 +UPSTREAM_SOCKET_WAIT_COUNT=0 +while [ ! -S "${UPSTREAM_PROC_SOCKET}" ]; do + if [ $UPSTREAM_SOCKET_WAIT_COUNT -ge $UPSTREAM_SOCKET_WAIT_TIMEOUT ]; then + log "ERROR: Upstream-refresh socket not created within ${UPSTREAM_SOCKET_WAIT_TIMEOUT}s" + exit 1 + fi + if ! kill -0 "$PE_PID" 2>/dev/null; then + log "ERROR: Policy Engine exited before creating upstream-refresh socket" + exit 1 + fi + sleep 1 + UPSTREAM_SOCKET_WAIT_COUNT=$((UPSTREAM_SOCKET_WAIT_COUNT + 1)) +done +log "Upstream-refresh socket ready: ${UPSTREAM_PROC_SOCKET}" +``` + +Also add `"${UPSTREAM_PROC_SOCKET}"` to the existing cleanup line at line 228 (`rm -f "${POLICY_ENGINE_SOCKET}" "${PYTHON_EXECUTOR_SOCKET}"`) and the final cleanup at line 349, so a restart doesn't fail on a stale socket file. + +- [ ] **Step 4: Build to confirm no compile errors** + +Run: `cd gateway/gateway-runtime/policy-engine && go build ./...` +Expected: clean build. + +- [ ] **Step 5: Verify the entrypoint script's shell syntax** + +Run: `bash -n gateway/gateway-runtime/docker-entrypoint.sh` +Expected: no output (valid syntax). + +- [ ] **Step 6: Commit** + +```bash +git add gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go \ + gateway/gateway-runtime/policy-engine/internal/constants/constants.go \ + gateway/gateway-runtime/docker-entrypoint.sh +git commit -m "feat(policy-engine): start the upstream-refresh ext_proc server alongside the main one" +``` + +### Task 5: Register `oauth2-generator` chains into the upstream-refresh registry + +**Resolved by Task 0's findings** (`docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md`, Question 3): register lazily, per-request, inside `internal/kernel` — not eagerly in `handler.go`/`buildPolicyChain`. Cluster identity *is* knowable earlier (via a separate `RouteConfig` xDS resource, `HandleRouteConfigUpdate`), but `PolicyChain`/`buildPolicyChain` never carries it and was never designed to, and — more importantly — cluster names are **not** dedicated per route/operation (host+scheme dedup, shared across unrelated APIs), so an eager join wouldn't remove the need for Task 8's collision detection anyway. Keeping this lazy avoids needing two separate registration code paths for what is one registry. + +**Files:** +- Modify: `gateway/gateway-runtime/policy-engine/internal/kernel/execution_context.go` (or wherever `toRequestUpstream`/`UpstreamInfo.ClusterName` resolution actually lives per the findings doc's Question 1c citations — `execution_context.go:1477-1509` per the findings doc) + +**Interfaces:** +- Consumes: `upstreamrefresh.Default().Register(clusterName string, p interface{}, params map[string]interface{}) error` (Task 2's real signature — accepts any policy instance, type-asserts internally, returns an error only if `p` implements neither upstream interface), the concrete `oauth2-generator` policy instance obtained from the existing per-route policy-chain build (whatever factory/registry call already instantiates a named policy for that route — read the surrounding code to find the exact call). + +- [ ] **Step 1:** In the per-request path that resolves `UpstreamInfo.ClusterName` for a request (per the findings doc's citations), for the route's policy chain: if it includes an `oauth2-generator` instance with `params["refreshOnRetry"] == true`, call: + +```go +if err := upstreamrefresh.Default().Register(info.ClusterName, instantiatedPolicy, resolvedParams); err != nil { + // instantiatedPolicy came from a refreshOnRetry: true attachment, so it + // must implement UpstreamRequestHeaderPolicy - this error path means + // something is structurally wrong (e.g. a non-oauth2-generator policy + // got refreshOnRetry: true param bypassing Task 6's validation). Log, + // don't panic - the request itself must still proceed normally. + slog.WarnContext(ctx, "failed to register upstream-refresh policy", "cluster", info.ClusterName, "error", err) +} +``` + +This is idempotent (Task 2's `Register` always overwrites), so calling it on every request for that route is correct, just slightly wasteful — acceptable given routes rarely reconfigure mid-traffic. + +- [ ] **Step 2:** Test this at the `execution_context_test.go` level (or wherever the resolution code's existing tests live): a request whose route has `oauth2-generator{refreshOnRetry: true}` results in `upstreamrefresh.Default().Lookup(clusterName)` returning `ok == true` after the request is processed; a route without `refreshOnRetry` (or without `oauth2-generator` at all) does not register anything. Use a stub policy implementing `UpstreamRequestHeaderPolicy` rather than the real `oauth2-generator` package (avoid a cross-module test dependency on `gateway-controllers/policies/oauth2-generator` from `policy-engine`'s own test suite). + +- [ ] **Step 3: Run the affected package's tests** + +Run: `cd gateway/gateway-runtime/policy-engine && go test ./internal/kernel/... -v` +Expected: PASS, including the new registration test. + +- [ ] **Step 4: Commit** + +```bash +git add gateway/gateway-runtime/policy-engine/internal/kernel/ +git commit -m "feat(policy-engine): register refreshOnRetry-enabled policies into the upstream-refresh registry" +``` + +--- + +## Phase 3: gateway-controller — emit the cluster's upstream filter chain + +### Task 6: Add `refreshOnRetry` to `oauth2-generator`'s policy-definition schema, validated against `resilience.retry` + +**Files:** +- Modify: `gateway-controllers/policies/oauth2-generator/policy-definition.yaml` +- Modify: `gateway/dev-policies/oauth2-generator/policy-definition.yaml` (sync mirror — diff after) +- Modify: `gateway/gateway-controller/pkg/config/api_validator.go` (or wherever policy-param cross-validation already happens for other policies — search for an existing precedent, e.g. how `advanced-ratelimit`'s params are cross-checked against other config, before inventing a new validation entry point) + +- [ ] **Step 1:** Add to `oauth2-generator/policy-definition.yaml`'s parameter schema: + +```yaml +refreshOnRetry: + type: boolean + default: false + description: > + When true, and this route's resilience.retry includes a status code this + policy would also treat as "credential rejected", fetch a fresh token + specifically for Envoy's native retry attempt (attempt 2+), instead of + resending the same token that just failed. Requires resilience.retry to + be configured on the same route with at least one status code this + policy's own purgeStatusCodes/tokenPurgeStatusCodes also covers (e.g. + 401) - has no effect otherwise, since Envoy will never retry an + unconfigured status code in the first place. +``` + +- [ ] **Step 2:** Add a validation check (exact location depends on where policy-param-to-resilience cross-checks already live in this codebase; if none exist yet, add it in `api_validator.go`'s per-operation validation loop, near `validateResilienceRetry`, Task 14 from the earlier `resilience.retry` implementation): if any policy named `oauth2-generator` on an operation has `params.refreshOnRetry == true`, require that operation's effective `resilience.retry.statusCodes` (API-level or operation-level, whichever resolves) is non-empty. Emit a `ValidationError` if not: + +```go +if refreshOnRetry, _ := getPolicyBoolParam(op.Policies, "oauth2-generator", "refreshOnRetry"); refreshOnRetry { + if len(effectiveRetryStatusCodes) == 0 { + errors = append(errors, ValidationError{ + Field: fmt.Sprintf("spec.operations[%d].policies", i), + Message: "oauth2-generator's refreshOnRetry requires resilience.retry.statusCodes to be configured on the same operation (or inherited from the API level) - otherwise Envoy never retries and refreshOnRetry has no effect", + }) + } +} +``` + +(`getPolicyBoolParam` and `effectiveRetryStatusCodes` are illustrative names — match whatever helper naming convention the surrounding validator file already uses for reading a named policy's param off an operation's resolved policy list; read the file before writing this to reuse an existing helper if one already exists for reading a policy's params by name.) + +- [ ] **Step 3: Write a test** in `api_validator_test.go` (same table-driven style as `TestAPIValidator_ValidateResilience` from the earlier `resilience.retry` work) asserting: `oauth2-generator` with `refreshOnRetry: true` and no `resilience.retry` → validation error; same policy with `resilience.retry.statusCodes: [401]` present → no error. + +- [ ] **Step 4: Run the test** + +Run: `cd gateway/gateway-controller && go test ./pkg/config/... -run TestAPIValidator -v` +Expected: PASS + +- [ ] **Step 5: Diff the two `oauth2-generator` copies to confirm they're in sync** + +Run: `diff -rq /Users/thenujan/Desktop/Git-Repos/gateway-controllers/policies/oauth2-generator/ /Users/thenujan/Desktop/Git-Repos/api-platform/gateway/dev-policies/oauth2-generator/` +Expected: only the already-known `go.mod` (replace-path) and `e2e/` diffs — the `policy-definition.yaml` change must be identical in both. + +- [ ] **Step 6: Commit** (both repos) + +```bash +# in gateway-controllers +git add policies/oauth2-generator/policy-definition.yaml +git commit -m "feat: add refreshOnRetry param, requires resilience.retry to be configured" + +# in api-platform +git add gateway/dev-policies/oauth2-generator/policy-definition.yaml \ + gateway/gateway-controller/pkg/config/api_validator.go \ + gateway/gateway-controller/pkg/config/api_validator_test.go +git commit -m "feat(gateway-controller): validate oauth2-generator's refreshOnRetry requires resilience.retry" +``` + +### Task 7: Emit the cluster's upstream `ext_proc` filter from the xDS translator + +**Files:** +- Modify: `gateway/gateway-controller/pkg/xds/translator.go` — `createCluster` (referenced at line 1121 in earlier work on this route; find its exact current definition) and/or wherever `cluster.Cluster{}` is finally constructed for a `RestApi`/`LLMProvider` main upstream. +- Modify: `gateway/gateway-controller/pkg/xds/translator_test.go` + +**Interfaces:** +- Consumes: whether the operation attaching to this cluster has `oauth2-generator` with `refreshOnRetry: true` (resolved during the same pass that already resolves `resilience.retry` for that operation — thread this bool through alongside `resolvedTimeout`/`RouteRetry` from the earlier `resilience.retry` work, since both are resolved per-operation before the cluster is built). + +- [ ] **Step 1:** Write a failing test asserting: given an operation with `oauth2-generator{refreshOnRetry: true}` and `resilience.retry.statusCodes: [401]`, the resulting `cluster.Cluster` has `TypedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"]` containing an `http_filters` list ending in `envoy.filters.http.upstream_codec`, preceded by an `envoy.filters.http.ext_proc` filter whose `grpc_service.envoy_grpc.cluster_name` (or `target_uri` for UDS — match whatever addressing style the EXISTING downstream ext_proc filter config in this codebase already uses; grep for how the main `envoy.filters.http.ext_proc` filter's `grpc_service` is currently built in `translator.go` and mirror it exactly, just pointed at `constants.DefaultUpstreamProcSocketPath` instead) points at the upstream-refresh socket, and whose `processing_mode.request_header_mode == SEND` with every other mode `SKIP`, and whose `request_attributes` includes `"xds.cluster_name"`. + +```go +func TestTranslator_ClusterUpstreamExtProc_WhenRefreshOnRetryEnabled(t *testing.T) { + // ... build a minimal RestApi config with one operation carrying + // oauth2-generator{refreshOnRetry: true} and resilience.retry{statusCodes: [401]} ... + // translate it, then assert on the resulting cluster's + // TypedExtensionProtocolOptions as described above. +} + +func TestTranslator_ClusterUpstreamExtProc_AbsentWhenNotConfigured(t *testing.T) { + // same shape, but refreshOnRetry is false/absent -> cluster must have + // no TypedExtensionProtocolOptions upstream ext_proc entry at all. +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway/gateway-controller && go test ./pkg/xds/... -run TestTranslator_ClusterUpstreamExtProc -v` +Expected: FAIL (feature doesn't exist yet). + +- [ ] **Step 3: Implement.** Add a helper mirroring `buildRetryPolicy` (from the earlier `resilience.retry` work) in shape: + +```go +// buildUpstreamRefreshExtProc returns the cluster-level upstream ext_proc +// HTTP filter config for credential refresh-on-retry, or nil when +// refreshOnRetry isn't enabled for this cluster's route. Mirrors the +// downstream ext_proc filter's grpc_service addressing (see +// buildDownstreamExtProcFilter or equivalent - match existing naming) but +// targets constants.DefaultUpstreamProcSocketPath and restricts +// processing_mode to headers-only, since this server never sees a body. +func (t *Translator) buildUpstreamRefreshExtProc(refreshOnRetry bool) *hcmv3.HttpFilter { + if !refreshOnRetry { + return nil + } + extProcConfig := &extprocv3.ExternalProcessor{ + GrpcService: &corev3.GrpcService{ + TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ClusterName: upstreamProcClusterName}, + }, + }, + ProcessingMode: &extprocv3.ProcessingMode{ + RequestHeaderMode: extprocv3.ProcessingMode_SEND, + ResponseHeaderMode: extprocv3.ProcessingMode_SKIP, + RequestBodyMode: extprocv3.ProcessingMode_NONE, + ResponseBodyMode: extprocv3.ProcessingMode_NONE, + RequestTrailerMode: extprocv3.ProcessingMode_SKIP, + ResponseTrailerMode: extprocv3.ProcessingMode_SKIP, + }, + RequestAttributes: []string{"xds.cluster_name"}, + } + anyConfig, err := anypb.New(extProcConfig) + if err != nil { + // Same fail-fast posture as every other proto-marshal error site in + // this translator - surfacing at xDS-build time, never silently + // skipped. + panic(fmt.Sprintf("failed to marshal upstream ext_proc config: %v", err)) + } + return &hcmv3.HttpFilter{ + Name: "envoy.filters.http.ext_proc", + ConfigType: &hcmv3.HttpFilter_TypedConfig{TypedConfig: anyConfig}, + } +} +``` + +> `upstreamProcClusterName` needs its own static Envoy `cluster.Cluster` definition (a UDS cluster pointing at `constants.DefaultUpstreamProcSocketPath`, same pattern as however the MAIN ext_proc's own gRPC cluster is already defined in this translator for the downstream filter — find and reuse that exact pattern/constant rather than reinventing cluster construction). Add this as a package-level constant alongside wherever the downstream ext_proc cluster name constant already lives. + +Wire `buildUpstreamRefreshExtProc`'s result into the cluster's `TypedExtensionProtocolOptions`, terminated with `envoy.filters.http.upstream_codec` (per the RBAC example pattern: `http_filters: [ext_proc_filter, upstream_codec_filter]`), only when the resolved `refreshOnRetry` bool for that cluster's route is true. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway/gateway-controller && go test ./pkg/xds/... -run TestTranslator_ClusterUpstreamExtProc -v` +Expected: PASS (both tests) + +- [ ] **Step 5: Run the full package test suite** + +Run: `cd gateway/gateway-controller && go test ./... ` +Expected: all PASS, no regression to the existing `resilience.retry`/timeout cluster-building tests. + +- [ ] **Step 6: Commit** + +```bash +git add gateway/gateway-controller/pkg/xds/translator.go gateway/gateway-controller/pkg/xds/translator_test.go +git commit -m "feat(gateway-controller): emit cluster upstream ext_proc filter when refreshOnRetry is enabled" +``` + +### Task 8: Enforce "one `oauth2-generator` refresh config per cluster" (structural safety net) + +**Rewritten after Task 7 landed — the original text above assumed a data model Task 7 didn't actually build, and understated the collision surface Task 0 found.** Two corrections that change this task's real shape: + +1. **`models.UpstreamCluster.RefreshOnRetry` is a bare `bool`** (Task 7's actual field, `runtime_deploy_config.go`) — it carries no oauth2-generator params at all. The real, secret-bearing oauth2-generator config (`tokenEndpoint`, `clientId`, etc.) never reaches gateway-controller's cluster model; it's resolved entirely on the policy-engine side, per-request, inside `internal/kernel`'s `registerUpstreamRefreshPolicies` (Task 5) and stored in `upstreamrefresh.Registry` (Task 2) — keyed by cluster name, overwritten on every request with no conflict detection today. So "detect two different oauth2-generator configs sharing a cluster" cannot be built as a boolean comparison on the existing field — it needs the actual **raw policy `params`** (as literally written in each operation's/API's `policies:` block — not runtime-resolved secrets) compared for equality, and that comparison has to happen wherever those raw params are still in scope together: in `pkg/transform/restapi.go`'s `RestAPITransformer.Transform` (same place Task 7 already resolves the `RefreshOnRetry` bool from `spec.Policies`/`op.Policies`). +2. **The collision is cross-API, not just cross-operation.** Task 0's findings (`docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh-findings.md`, Question 2) proved Envoy cluster names are derived purely from upstream `host+scheme` and are deduplicated **across every deployed API** in `gateway-controller/pkg/xds/translator.go`'s `TranslateConfigs` (`clusterMap[c.Name] = c`, last-write-wins, line ~797 as of that investigation). Two unrelated APIs pointing their main upstream at the same host+scheme, each with a *different* `oauth2-generator{refreshOnRetry:true}` config, collapse onto one Envoy cluster today with zero detection. A check scoped to "within one API's operations" (the original Step 1's test) would miss this, more likely, real-world case entirely. + +**Files:** +- Modify: `gateway/gateway-controller/pkg/models/runtime_deploy_config.go` — `UpstreamCluster` gains a field carrying the raw oauth2-generator params when `RefreshOnRetry` is true (e.g. `RefreshOnRetryParams map[string]interface{}`), populated alongside `RefreshOnRetry` wherever Task 7 already sets that bool. +- Modify: `gateway/gateway-controller/pkg/transform/restapi.go` — where `RefreshOnRetry` is resolved (via `xds.ResolveRefreshOnRetry` per Task 7's report), also capture and set the winning policy's raw `params` map onto the new field. +- Modify: `gateway/gateway-controller/pkg/xds/translator.go` — `TranslateConfigs`'s cross-API cluster-merge loop (where `clusterMap[c.Name] = c` happens) is the one place that sees every API's contribution to a shared cluster name in one pass; add the collision check here, comparing `RefreshOnRetryParams` (deep-equal, e.g. via `reflect.DeepEqual` or a stable JSON-marshal comparison) between what's already in `clusterMap` for that name and the incoming cluster's value, whenever both are non-nil. +- Modify: `gateway/gateway-controller/pkg/xds/translator_test.go`. + +**Interfaces:** +- Consumes: `models.UpstreamCluster.RefreshOnRetry`/the new params field (this task's own addition to that struct), `xds.ResolveRefreshOnRetry` (Task 7). +- Produces: `TranslateConfigs` returns an error (instead of silently merging) when two different APIs' clusters of the same name carry different non-nil `RefreshOnRetryParams`. + +- [ ] **Step 1:** Read `pkg/xds/translator.go`'s current `TranslateConfigs` and the current `RestAPITransformer.Transform`/`ResolveRefreshOnRetry` (both landed in Task 7's fix) to find their exact real shape — Task 7's report has the exact function/line references from when it was implemented; confirm they haven't drifted. +- [ ] **Step 2:** Add `RefreshOnRetryParams map[string]interface{}` to `models.UpstreamCluster`, set it in `restapi.go` alongside the existing `RefreshOnRetry bool` (same resolution call, just also capture the winning policy's `Params`). +- [ ] **Step 3:** Write a failing test for the **cross-API** case first (the more important, more likely one): two separate `RestApi`/`LLMProvider` configs, same upstream host+scheme (so they resolve to the identical Envoy cluster name), each with `oauth2-generator{refreshOnRetry:true}` but different `tokenEndpoint` params, both passed into one `TranslateConfigs` call — assert it returns an error naming the conflicting cluster, not a silently-merged result. +- [ ] **Step 4:** Write a second failing test for the **within-one-API, two-operations** case (two operations sharing the API's main cluster, different operation-level `oauth2-generator` params) — same assertion. +- [ ] **Step 5:** Implement the check inside `TranslateConfigs`'s cluster-merge loop: before `clusterMap[c.Name] = c` overwrites an existing entry, if both the existing and incoming cluster have non-nil `RefreshOnRetryParams` that aren't deep-equal, return `fmt.Errorf("cluster %q has conflicting refreshOnRetry oauth2-generator configurations from different APIs/operations sharing this upstream - refreshOnRetry requires exactly one credential-refresh configuration per cluster", c.Name)` instead of proceeding. +- [ ] **Step 6: Run both new tests** + +Run: `cd gateway/gateway-controller && go test ./pkg/xds/... -run TestTranslator_UpstreamRefresh_ConflictingConfig -v` +Expected: PASS (both the cross-API and within-API variants) + +- [ ] **Step 7: Run the full package suite** + +Run: `cd gateway/gateway-controller && go test ./...` +Expected: all PASS, no regression to Task 7's tests or any other cluster-building test. + +- [ ] **Step 8: Commit** + +```bash +git add gateway/gateway-controller/pkg/models/runtime_deploy_config.go gateway/gateway-controller/pkg/transform/restapi.go gateway/gateway-controller/pkg/xds/translator.go gateway/gateway-controller/pkg/xds/translator_test.go +git commit -m "fix(gateway-controller): reject conflicting refreshOnRetry configs sharing one cluster, including across APIs" +``` + +--- + +## Phase 4: `oauth2-generator` — implement the refresh + +### Task 9: Implement `UpstreamRequestHeaderPolicy` on the oauth2-generator policy, reusing the existing token cache + +**Files:** +- Modify: `gateway-controllers/policies/oauth2-generator/oauth2_generator.go` +- Modify: `gateway-controllers/policies/oauth2-generator/token_cache.go` +- Modify: `gateway-controllers/policies/oauth2-generator/oauth2_generator_test.go` +- Sync all three to `gateway/dev-policies/oauth2-generator/` after. + +**Interfaces:** +- Consumes: `policy.UpstreamHeaderContext`/`UpstreamHeaderModifications`/`UpstreamRequestHeaderPolicy` (Task 1); the existing `tokenSource xoauth2.TokenSource` and `cacheParams` already built by `extractCacheParams`/`buildTokenSource` (established in prior work this session, `token_cache.go`). +- Produces: `(*OAuth2GeneratorPolicy) OnUpstreamRequestHeaders(ctx, uctx, params) policy.UpstreamHeaderAction`. + +- [ ] **Step 1: Write the failing test.** Reuse this file's existing test doubles for a fake token source (check `token_cache_test.go` for whatever fake `xoauth2.TokenSource` is already used — reuse it, don't create a second one) that returns a distinguishable token value each call (e.g. an incrementing counter embedded in the token string), so the test can assert the SECOND call's token differs from the first: + +```go +func TestOAuth2GeneratorPolicy_OnUpstreamRequestHeaders_AttemptOne_NoMutation(t *testing.T) { + p := &OAuth2GeneratorPolicy{} + params := testRedisParams(nil) // or whatever the existing minimal-valid-params helper is called + uctx := &policy.UpstreamHeaderContext{AttemptCount: 1, Headers: policy.NewHeaders(nil)} + + action := p.OnUpstreamRequestHeaders(context.Background(), uctx, params) + mods, ok := action.(policy.UpstreamHeaderModifications) + if !ok { + t.Fatalf("expected UpstreamHeaderModifications, got %T", action) + } + if len(mods.HeadersToSet) != 0 { + t.Fatalf("attempt 1 must not mutate headers, got %v", mods.HeadersToSet) + } +} + +func TestOAuth2GeneratorPolicy_OnUpstreamRequestHeaders_AttemptTwo_FetchesFreshToken(t *testing.T) { + p := &OAuth2GeneratorPolicy{} + params := testRedisParams(nil) + uctx := &policy.UpstreamHeaderContext{AttemptCount: 2, Headers: policy.NewHeaders(nil)} + + action := p.OnUpstreamRequestHeaders(context.Background(), uctx, params) + mods, ok := action.(policy.UpstreamHeaderModifications) + if !ok { + t.Fatalf("expected UpstreamHeaderModifications, got %T", action) + } + auth, present := mods.HeadersToSet["Authorization"] + if !present || auth == "" { + t.Fatalf("attempt 2 must set a non-empty Authorization header, got %v", mods.HeadersToSet) + } +} + +func TestOAuth2GeneratorPolicy_OnUpstreamRequestHeaders_TokenFetchError_FailsOpenNoMutation(t *testing.T) { + // Wire in whatever this file's existing "always-erroring IdP" test + // double is (check for one already used to test the main + // OnRequestHeaders failure path) and assert: no panic, no error + // returned/propagated, and HeadersToSet is empty - the caller + // (internal/upstreamproc.Server) has no error channel to react to + // anyway, per Global Constraints' fail-open requirement. +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway-controllers/policies/oauth2-generator && go test ./... -run TestOAuth2GeneratorPolicy_OnUpstreamRequestHeaders -v` +Expected: FAIL — method doesn't exist. + +- [ ] **Step 3: Implement**, appended to `oauth2_generator.go` near the existing `OnResponseHeaders`: + +```go +// OnUpstreamRequestHeaders implements policy.UpstreamRequestHeaderPolicy. +// Unlike OnRequestHeaders (called once per client request, on the listener's +// filter chain), this runs once per individual upstream dial attempt, +// including Envoy-native retries - see UpstreamHeaderContext's doc comment. +// On AttemptCount == 1 it does nothing: OnRequestHeaders already attached a +// token for the first attempt. On AttemptCount > 1, it forces a fresh fetch +// via the same cached tokenSource this policy already built for +// OnRequestHeaders (Purge() + Token(), not a from-scratch client), so +// redis-backed sharing/caching semantics stay identical to the rest of this +// policy - this is not a second, independent credential-fetch path. +func (p *OAuth2GeneratorPolicy) OnUpstreamRequestHeaders(ctx context.Context, uctx *policy.UpstreamHeaderContext, params map[string]interface{}) policy.UpstreamHeaderAction { + if uctx.AttemptCount <= 1 { + return policy.UpstreamHeaderModifications{} + } + + cp, err := extractCacheParams(params) + if err != nil { + return policy.UpstreamHeaderModifications{} // fail open - see Global Constraints + } + tokenSource, err := buildTokenSource(cp) + if err != nil { + return policy.UpstreamHeaderModifications{} + } + if purger, ok := tokenSource.(interface{ Purge() }); ok { + purger.Purge() // force the fetch below past any cached-but-just-failed token + } + tok, err := tokenSource.Token() + if err != nil || tok.AccessToken == "" { + return policy.UpstreamHeaderModifications{} + } + return policy.UpstreamHeaderModifications{ + HeadersToSet: map[string]string{"Authorization": tok.Type() + " " + tok.AccessToken}, + } +} +``` + +> Confirm the exact `Purge()` interface/method name and the exact `extractCacheParams`/`buildTokenSource` signatures against the real current file before finalizing this — reuse whatever's already there rather than the illustrative names above if they differ. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway-controllers/policies/oauth2-generator && go test ./... -run TestOAuth2GeneratorPolicy_OnUpstreamRequestHeaders -v` +Expected: PASS (all three) + +- [ ] **Step 5: Run the full existing test suite for this policy to confirm no regression** + +Run: `cd gateway-controllers/policies/oauth2-generator && go test ./... -v` +Expected: all PASS, including every pre-existing test from earlier sessions' work on this policy (token cache, redis client resolution, retry-on-timeout, etc.). + +- [ ] **Step 6: Sync to the `dev-policies` mirror** + +```bash +cp gateway-controllers/policies/oauth2-generator/oauth2_generator.go \ + gateway-controllers/policies/oauth2-generator/token_cache.go \ + gateway-controllers/policies/oauth2-generator/oauth2_generator_test.go \ + api-platform/gateway/dev-policies/oauth2-generator/ +diff -rq gateway-controllers/policies/oauth2-generator/ api-platform/gateway/dev-policies/oauth2-generator/ +``` + +Expected diff output: only the known `go.mod`/`e2e/` differences. + +- [ ] **Step 7: Commit** (both repos) + +```bash +# gateway-controllers +git add policies/oauth2-generator/oauth2_generator.go policies/oauth2-generator/oauth2_generator_test.go +git commit -m "feat: implement UpstreamRequestHeaderPolicy for per-retry credential refresh" + +# api-platform +git add gateway/dev-policies/oauth2-generator/ +git commit -m "sync: oauth2-generator refreshOnRetry implementation from gateway-controllers" +``` + +--- + +## Phase 5: End-to-end verification + +### Task 10: Extend the mock IdP to issue a distinguishable token per call, and prove attempt 2 gets a different one + +**Files:** +- Modify: `gateway/dev-policies/oauth2-generator/e2e/mocks/mock-oauth2-idp/main.go` (it already increments `tokenSeq` per issuance — confirm this and reuse it rather than adding a second counter) +- Modify: `gateway/dev-policies/resilience-retry/e2e/mocks/mock-backend/main.go` — extend to force `401` for the *first* `Authorization` value it sees per test flow, and `200` for any *different* one, so the test can assert "the backend actually received a NEW token on the retry," not just "some token was present twice." +- Create: a new Postman folder in `gateway/dev-policies/resilience-retry/e2e/postman/resilience-retry.postman_collection.json` (or a new dedicated collection if mixing oauth2 + retry concerns into the existing one gets unwieldy — prefer a new collection: `oauth2-upstream-refresh.postman_collection.json` alongside it, following this project's one-collection-per-feature convention already established by `oauth2.postman_collection.json` vs. `advanced-ratelimit-redis-precedence.postman_collection.json`). +- Create: `gateway/dev-policies/oauth2-generator/e2e/run-e2e.sh` gains a new chained call (same pattern as the existing `advanced-ratelimit`/`resilience-retry` chains) to this new suite's own `run-e2e.sh`, OR fold it directly into `resilience-retry/e2e/run-e2e.sh` as an additional phase if it shares that suite's mock-backend closely enough — decide based on how much mock-backend logic needs changing (Step 1 below) vs. how self-contained a new mock-backend copy would be; prefer extending the existing one if the change is additive and doesn't alter any of the three already-passing phases' behavior. + +- [ ] **Step 1:** Extend `mock-backend/main.go`'s `handleAny` to compare the incoming `Authorization` header against a `firstSeenAuth` value (captured on the first request within a `configure-failures` window) and only apply the forced-failure status while `Authorization == firstSeenAuth` — once a *different* `Authorization` value arrives, treat it as success (`200`) regardless of the configured `failFirstN` counter. Reset `firstSeenAuth` in the existing `/debug/reset` handler. Also expose it via `/debug/stats` as `firstSeenAuth`/`lastSeenAuth` (masked to first/last 4 chars only, per this project's `GO-AUTH-003` masking convention already used in `mock-ai-backend.go`'s `maskToken`). +- [ ] **Step 2:** Register a new test API (or extend `resilience-retry-test`) with an operation attaching both `oauth2-generator{refreshOnRetry: true, tokenEndpoint: mock-oauth2-idp, ...}` and `resilience.retry{statusCodes: [401], numRetries: 1}`, upstream pointing at `mock-backend`. +- [ ] **Step 3:** Write the Postman flow: + 1. Reset `mock-oauth2-idp` and `mock-backend`. + 2. Configure `mock-backend` to reject whatever `Authorization` value it first sees (new mode from Step 1) with `401`. + 3. Call the route once. + 4. Assert client sees `200` (the retry, carrying a *different*, freshly-fetched token, succeeded). + 5. Assert `mock-oauth2-idp`'s `/debug/stats` shows **2** token issuances for this flow (one for the initial attach, one for the forced refresh on retry) — this is the ground-truth proof this whole feature exists to deliver, mirroring this project's established "don't trust client-visible status alone" convention from the `resilience-retry` and `advanced-ratelimit` e2e suites. + 6. Assert `mock-backend`'s `/debug/stats` shows `requestCount == 2` and `firstSeenAuth != lastSeenAuth`. +- [ ] **Step 4: Run the new/extended e2e script end-to-end** against the real Docker stack (same pattern as every other e2e script this session: rebuild `gateway-controller` and `gateway-runtime` images first, since this phase touches both, then `docker compose up -d gateway-controller gateway-runtime`, then run the script). + +Run: `/run-e2e.sh` +Expected: exit code 0, with the assertions from Step 3.5/3.6 both passing — this is the one check that actually proves the feature works, as opposed to merely compiling. + +- [ ] **Step 5: Commit** + +```bash +git add gateway/dev-policies/oauth2-generator/e2e/mocks/mock-oauth2-idp/main.go \ + gateway/dev-policies/resilience-retry/e2e/ +git commit -m "test(e2e): prove oauth2-generator refreshOnRetry actually issues a new token on Envoy retry" +``` + +--- + +## Self-Review Notes (completed during authoring, kept for the implementer's context) + +- **Spec coverage:** every piece of the agreed design — SDK types, registry, minimal upstream ext_proc server, socket wiring, xDS cluster-filter emission, `refreshOnRetry` param + validation, oauth2-generator's implementation, and e2e proof — has a task. The one deliberately-deferred design decision (Task 0) is called out explicitly rather than guessed at, because guessing it wrong would silently misplace Task 5/6's registration logic in the wrong layer. +- **No placeholders:** every code-bearing step has real Go/bash, not descriptions. The two spots using illustrative names (`getPolicyBoolParam`, `effectiveRetryStatusCodes` in Task 6; the exact fake-stream helper in Task 3) are flagged as "match the existing codebase's real name," which is a legitimate investigation instruction, not a skipped implementation — the surrounding code IS fully specified. +- **Type consistency:** `UpstreamHeaderContext`/`UpstreamHeaderModifications`/`UpstreamRequestHeaderPolicy` (Task 1) are used with identical field/method names through Tasks 3, 5, and 9 — checked by re-reading each task's code block against Task 1's definitions while writing this plan. + +### Amendment log (post-Task-1, before Task 2 started) + +Task 0 was executed and its findings (see the separate findings doc) resolved Task 5 to the lazy per-request design and elevated Task 8's collision check from "edge case" to "required, structurally reachable" — Task 5's text above already reflects this; Task 8 still needs to be read with that context when it's dispatched. + +Task 1b was added after Task 1 was already reviewed and committed, following a design discussion: a rename of Task 1's types to a neutral "Attempt"-based name was considered and rejected (it would have broken this SDK's established one-phase-one-action-type convention, confirmed by reading `interface.go`/`action.go`'s existing `OnRequestBody`/`RequestAction`, `OnResponseBody`/`ResponseAction` pattern) — Task 1's original naming stands unchanged. Body-mutation support was added instead as the sibling `UpstreamRequestBodyPolicy` extension point, with query-param mutation confirmed to need no new code at all (`:path` is already mutable via the existing header action). This required updating, in-place, in this same plan file: +- Task 2's registry: `Register`/`Lookup` signatures changed to accept a plain `interface{}` and return both an optional header policy and an optional body policy (plus `Register` now returns an `error`). Every code block and test in Task 2 was rewritten to the new signature — this is NOT the version that was reviewed by Task 0's reviewer (Task 2 itself has not been reviewed yet as of this amendment). +- Task 3's server: added a `RequestBody` dispatch branch, per-stream `attemptCount`/`clusterName` state carried across `Recv()` calls, and a new synthetic-input-only test — clearly marked as unreachable by real traffic in this plan (Task 7 never requests `BUFFERED` body mode). +- Task 5: rewritten to drop the "eager" alternate path entirely (Task 0 resolved this) and updated its `Register(...)` call site to the new signature/error-handling. +- Global Constraints: added two bullets documenting the query/body scope decision. + +**Whoever picks up Task 6 onward should re-confirm** that no other task text still references the old two-value `Register(clusterName, p, params)` / two-value `Lookup(clusterName)` signatures before writing code against it — grep for `\.Register(` / `\.Lookup(` in this file if in doubt. diff --git a/docs/superpowers/plans/2026-08-12-upstream-attempt-retry-refresh.md b/docs/superpowers/plans/2026-08-12-upstream-attempt-retry-refresh.md new file mode 100644 index 0000000000..a13b2482c6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-upstream-attempt-retry-refresh.md @@ -0,0 +1,1483 @@ +# Generic Upstream-Attempt Retry/Credential-Refresh Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** When a `resilience.retry`-enabled route's backend returns a configured status code (e.g. 401) and Envoy natively retries, let any policy that implements a new generic SDK interface attach fresh, per-attempt state (oauth2-generator: a freshly-fetched token) to the retried attempt — without any policy making its own outbound HTTP call to the main upstream, and without the client ever seeing the intermediate failure. + +**Architecture:** Envoy's *upstream* HTTP filter chain (configured per-cluster via `Cluster.TypedExtensionProtocolOptions`, distinct from the per-listener downstream chain the existing `ext_proc` filter runs in) is re-invoked fresh for every upstream dial attempt, including retries, and Envoy sets `x-envoy-attempt-count` on each attempt (1, incrementing). A new, minimal second `ext_proc` gRPC server runs inside the existing `policy-engine` process, resolves the same in-memory policy-chain registry the existing downstream server already builds, and — for any policy in that chain implementing the new `UpstreamAttemptPolicy` interface (a plain Go type assertion, never a hardcoded policy name) — invokes it once per attempt. + +**Tech Stack:** Go 1.26, `google.golang.org/grpc`, `github.com/envoyproxy/go-control-plane` (`ext_proc` v3, `envoy/extensions/upstreams/http/v3`), stock `envoyproxy/envoy` image (no custom build), `oapi-codegen v2.5.1`, `golang.org/x/oauth2`. + +**Supersedes:** `docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh.md` (do not resume it — its commits exist in git history but are unreachable from current `HEAD`; this plan does not reuse its code). Design rationale: `docs/superpowers/specs/2026-08-12-upstream-attempt-retry-refresh-design.md`. + +## Global Constraints + +- No custom-compiled Envoy — every mechanism must work against the stock image already in use (`gateway-runtime/Dockerfile`'s `python-deps`/`production` stages, `FROM envoyproxy/envoy:${ENVOY_VERSION}`). +- Fail open on any error in the new upstream-attempt path (credential fetch fails, chain lookup miss): return a no-mutation action, never block the retry. This feature only ever makes a retry *more likely* to succeed. +- Policy discovery is a type assertion against `UpstreamAttemptPolicy` — never a hardcoded policy name string, anywhere (gateway-controller, policy-engine, or oauth2-generator). +- `dev-policies/oauth2-generator` (local mirror) and `gateway-controllers/policies/oauth2-generator` (separate repo, source of truth) must be kept byte-identical for `oauth2_generator.go`/`policy-definition.yaml` — diff after every change. +- New gRPC servers set `grpc.MaxRecvMsgSize`, `grpc.MaxSendMsgSize`, `grpc.MaxConcurrentStreams` explicitly (`go-network-service-hardening.md` directive 2) — do not copy the existing downstream `ext_proc` server's omission of these forward into new code. +- No raw token values in log output (`GO-AUTH-003`) — any log line touching a fetched/refreshed token logs only presence/error, never the value. + +--- + +## Phase 1: SDK primitives + +### Task 1: Add `UpstreamAttemptContext`, `UpstreamAttemptAction`, `UpstreamAttemptPolicy` to the policy SDK + +**Files:** +- Modify: `sdk/core/policy/v1alpha2/context.go` +- Modify: `sdk/core/policy/v1alpha2/action.go` +- Create: `sdk/core/policy/v1alpha2/upstream_attempt_test.go` + +**Interfaces:** +- Produces: `policy.UpstreamAttemptContext{SharedContext *SharedContext, AttemptCount int, Headers *Headers}`, `policy.UpstreamAttemptAction` (sealed interface, one variant), `policy.UpstreamAttemptHeaderModifications{HeadersToSet map[string]string}`, `policy.UpstreamAttemptPolicy` interface with `OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction`. +- Consumes: existing `Headers` (`sdk/core/policy/v1alpha2/headers.go`, `NewHeaders(map[string][]string) *Headers`), existing `SharedContext` (`context.go:78-119`). + +- [ ] **Step 1: Write the failing test** + +```go +// sdk/core/policy/v1alpha2/upstream_attempt_test.go +package policyv1alpha2 + +import ( + "context" + "testing" +) + +// fakeUpstreamAttemptPolicy proves any type implementing UpstreamAttemptPolicy +// compiles against the real context.Context/UpstreamAttemptContext/ +// UpstreamAttemptAction types — a compile-time contract test. oauth2-generator's +// own tests (Task 9) cover real refresh behavior. +type fakeUpstreamAttemptPolicy struct{} + +func (fakeUpstreamAttemptPolicy) OnUpstreamAttemptRequestHeaders(_ context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction { + if actx.AttemptCount <= 1 { + return UpstreamAttemptHeaderModifications{} + } + return UpstreamAttemptHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed"}} +} + +func TestUpstreamAttemptContext_AttemptCountGatesRefresh(t *testing.T) { + var p UpstreamAttemptPolicy = fakeUpstreamAttemptPolicy{} + + attemptOne := &UpstreamAttemptContext{AttemptCount: 1, Headers: NewHeaders(nil)} + action := p.OnUpstreamAttemptRequestHeaders(context.Background(), attemptOne) + mods, ok := action.(UpstreamAttemptHeaderModifications) + if !ok || len(mods.HeadersToSet) != 0 { + t.Fatalf("attempt 1 must not mutate headers, got %#v", action) + } + + attemptTwo := &UpstreamAttemptContext{AttemptCount: 2, Headers: NewHeaders(nil)} + action2 := p.OnUpstreamAttemptRequestHeaders(context.Background(), attemptTwo) + mods2, ok := action2.(UpstreamAttemptHeaderModifications) + if !ok || mods2.HeadersToSet["Authorization"] != "Bearer refreshed" { + t.Fatalf("attempt 2 must carry the refreshed token, got %#v", action2) + } +} + +// Compile-time interface satisfaction check, mirroring action.go's own convention. +var _ UpstreamAttemptAction = UpstreamAttemptHeaderModifications{} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd sdk/core && GOWORK=off go test ./policy/v1alpha2/... -run TestUpstreamAttemptContext_AttemptCountGatesRefresh -v` +Expected: FAIL with `undefined: UpstreamAttemptContext`. + +- [ ] **Step 3: Add the context type** to `sdk/core/policy/v1alpha2/context.go`, appended after the existing `ResponseStreamContext` block (end of file): + +```go +// ─── Upstream-attempt context (per-dial-attempt, not per-client-request) ───── + +// UpstreamAttemptContext is passed to UpstreamAttemptPolicy.OnUpstreamAttemptRequestHeaders. +// Unlike every other context in this package, it is NOT scoped to one client +// request — it fires once per individual upstream dial attempt, including +// Envoy-native retries, because it runs in Envoy's per-cluster upstream HTTP +// filter chain rather than the per-route listener chain every other policy +// phase in this package uses. +type UpstreamAttemptContext struct { + *SharedContext + + // AttemptCount is Envoy's x-envoy-attempt-count for this specific dial, + // starting at 1. A missing/unparseable header is treated as 1 (fail + // toward "behave like the first attempt", never toward unconditional + // refresh) — see the kernel-side parsing in Task 3. + AttemptCount int + + // Headers are this specific attempt's outgoing request headers, mutable + // via the returned UpstreamAttemptAction. + Headers *Headers +} +``` + +- [ ] **Step 4: Add the sealed action to** `sdk/core/policy/v1alpha2/action.go`, appended at the end of the file: + +```go +// ─── Upstream-attempt action (sealed oneof, one variant) ───────────────────── +// +// UpstreamAttemptAction is deliberately a sealed interface with exactly one +// concrete variant, unlike RequestHeaderAction's two (Modifications | +// ImmediateResponse): this phase runs after routing and authentication are +// already resolved, mid-retry-loop inside Envoy's router filter, where there +// is no sensible notion of "reject this request" — only "optionally change +// headers for this one attempt." + +// UpstreamAttemptAction is the sealed oneof returned by +// UpstreamAttemptPolicy.OnUpstreamAttemptRequestHeaders. +type UpstreamAttemptAction interface { + isUpstreamAttemptAction() +} + +// UpstreamAttemptHeaderModifications sets the given headers on this specific +// upstream attempt. An empty/nil HeadersToSet is a valid, common no-op (e.g. +// AttemptCount == 1, nothing to refresh yet, or a fail-open path after an +// error). +type UpstreamAttemptHeaderModifications struct { + HeadersToSet map[string]string +} + +func (UpstreamAttemptHeaderModifications) isUpstreamAttemptAction() {} + +// UpstreamAttemptPolicy is implemented by any policy that wants to attach +// fresh, per-attempt state (e.g. a refreshed credential) to an Envoy-native +// retry. Discovery is a plain type assertion by the kernel — see Task 3 — +// never a hardcoded policy name. A policy implements this in addition to, +// not instead of, its normal RequestHeaderPolicy/ResponseHeaderPolicy +// interfaces. +type UpstreamAttemptPolicy interface { + OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction +} +``` + +Note: `action.go` needs `"context"` imported for the interface method signature — check the existing import block; if `context` isn't already imported there (it isn't, per the current file), add it. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `cd sdk/core && GOWORK=off go test ./policy/v1alpha2/... -run TestUpstreamAttemptContext_AttemptCountGatesRefresh -v` +Expected: PASS. + +- [ ] **Step 6: Run the full package test suite to confirm no regressions** + +Run: `cd sdk/core && GOWORK=off go build ./... && GOWORK=off go test ./policy/... -v` +Expected: all PASS. + +- [ ] **Step 7: Commit** + +```bash +git add sdk/core/policy/v1alpha2/context.go sdk/core/policy/v1alpha2/action.go sdk/core/policy/v1alpha2/upstream_attempt_test.go +git commit -m "feat(sdk): add generic UpstreamAttemptPolicy interface for per-retry-attempt header mutation" +``` + +--- + +## Phase 2: policy-engine — minimal upstream `ext_proc` server + +### Task 2: Extract `extractRouteKey`'s body into a shared free function + +**Why:** the existing `(*ExternalProcessorServer).extractRouteKey` (`internal/kernel/extproc.go`) reads `req.Attributes[constants.ExtProcFilter].Fields["xds.route_name"]`. The new upstream server (Task 3) needs the identical logic but is a different receiver type — extracting it avoids duplicating it. + +**Files:** +- Modify: `gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go` +- Test: `gateway/gateway-runtime/policy-engine/internal/kernel/extproc_test.go` (add a case; file already exists per the existing test suite for this package — if a test file for this exact function doesn't exist yet, add it as a new test function in the existing `extproc_test.go`) + +**Interfaces:** +- Produces: `extractRouteKeyFromAttributes(req *extprocv3.ProcessingRequest) string` (package-level, unexported, in package `kernel`). +- Consumes: existing `constants.ExtProcFilter` (`internal/constants/constants.go:23`, value `"envoy.filters.http.ext_proc"`). + +- [ ] **Step 1: Write the failing test** + +```go +// added to internal/kernel/extproc_test.go +func TestExtractRouteKeyFromAttributes_MissingAttributesReturnsDefault(t *testing.T) { + req := &extprocv3.ProcessingRequest{} + if got := extractRouteKeyFromAttributes(req); got != "default" { + t.Errorf("got %q, want %q", got, "default") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go test ./internal/kernel/... -run TestExtractRouteKeyFromAttributes_MissingAttributesReturnsDefault -v` +Expected: FAIL with `undefined: extractRouteKeyFromAttributes`. + +- [ ] **Step 3: Extract the function.** In `internal/kernel/extproc.go`, find the existing `extractRouteKey` method (currently `func (s *ExternalProcessorServer) extractRouteKey(req *extprocv3.ProcessingRequest) string`). Replace its body with a call to a new package-level function, and move the existing body verbatim into that new function: + +```go +// extractRouteKeyFromAttributes extracts just the route key (xds.route_name) +// from the request attributes — shared by both the downstream ExternalProcessorServer +// and the upstream-attempt UpstreamExternalProcessorServer (Task 3), since both +// receive the identical ext_proc request-attributes shape. +func extractRouteKeyFromAttributes(req *extprocv3.ProcessingRequest) string { + if req.Attributes == nil { + return "default" + } + extProcAttrs, ok := req.Attributes[constants.ExtProcFilter] + if !ok || extProcAttrs.Fields == nil { + return "default" + } + if routeNameValue, ok := extProcAttrs.Fields["xds.route_name"]; ok { + if stringValue := routeNameValue.GetStringValue(); stringValue != "" { + return stringValue + } + } + return "default" +} + +// extractRouteKey extracts just the route key (xds.route_name) from the request attributes. +// This is a lightweight extraction that avoids parsing route metadata. +func (s *ExternalProcessorServer) extractRouteKey(req *extprocv3.ProcessingRequest) string { + return extractRouteKeyFromAttributes(req) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go test ./internal/kernel/... -run TestExtractRouteKeyFromAttributes_MissingAttributesReturnsDefault -v` +Expected: PASS. + +- [ ] **Step 5: Run the full kernel package test suite to confirm no regressions** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go test ./internal/kernel/... -v 2>&1 | tail -60` +Expected: all PASS (the existing `extractRouteKey` behavior is unchanged — it now delegates). + +- [ ] **Step 6: Commit** + +```bash +git add internal/kernel/extproc.go internal/kernel/extproc_test.go +git commit -m "refactor(policy-engine): extract extractRouteKey body into a shared free function" +``` + +### Task 3: Add the minimal upstream `ext_proc` gRPC server + +**Files:** +- Create: `gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go` +- Create: `gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc_test.go` + +**Interfaces:** +- Consumes: `Kernel.GetRouteConfig(routeKey string) *RouteConfig` and `Kernel.GetPolicyChain(policyChainKey string) *registry.PolicyChain` (`internal/kernel/mapper.go:70,77`, both already exported on `*Kernel`); `registry.PolicyChain.Policies []policy.Policy` (`internal/registry/chain.go`); `extractRouteKeyFromAttributes` (Task 2); the package-private `buildHeaderValueOptions(map[string]string) *extprocv3.HeaderMutation` (`internal/kernel/translator.go:1789`, already in package `kernel` — no import needed since this new file lives in the same package). +- Produces: `type UpstreamExternalProcessorServer struct{ kernel *Kernel }`, `NewUpstreamExternalProcessorServer(k *Kernel) *UpstreamExternalProcessorServer`, implementing `extprocv3.ExternalProcessorServer`'s `Process` method (request-headers phase only — every other phase gets an empty continue response, since this filter is only ever configured with `RequestHeaderMode: SEND` in Task 8, so no other phase should ever actually arrive, but the switch must handle it defensively rather than erroring). + +- [ ] **Step 1: Write the failing test** + +```go +// gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc_test.go +package kernel + +import ( + "context" + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + policyenginev1 "github.com/wso2/api-platform/sdk/core/policyengine" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/registry" + "google.golang.org/genproto/googleapis/rpc/status" + structpb "google.golang.org/protobuf/types/known/structpb" +) + +// fakeUpstreamAttemptPolicy proves the dispatch loop invokes exactly the +// policies implementing UpstreamAttemptPolicy, via type assertion, ignoring +// every other policy in the chain (rate-limit/analytics-shaped policies that +// don't implement it). +type fakeUpstreamAttemptPolicy struct{ lastAttempt int } + +func (p *fakeUpstreamAttemptPolicy) Mode() policy.ProcessingMode { return policy.ProcessingMode{} } +func (p *fakeUpstreamAttemptPolicy) OnUpstreamAttemptRequestHeaders(_ context.Context, actx *policy.UpstreamAttemptContext) policy.UpstreamAttemptAction { + p.lastAttempt = actx.AttemptCount + if actx.AttemptCount <= 1 { + return policy.UpstreamAttemptHeaderModifications{} + } + return policy.UpstreamAttemptHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed"}} +} + +// nonParticipatingPolicy implements only the base Policy interface — proves +// the dispatch loop skips it via type assertion, not a hardcoded name check. +type nonParticipatingPolicy struct{} + +func (nonParticipatingPolicy) Mode() policy.ProcessingMode { return policy.ProcessingMode{} } + +func newTestRouteConfigAndChain(t *testing.T, routeKey string, chain *registry.PolicyChain) *Kernel { + t.Helper() + k := NewKernel() + k.ApplyWholeRouteConfigs(map[string]RouteConfig{routeKey: {RouteName: routeKey}}) + k.SetPolicyChain(routeKey, chain) // see Step 3 note below if SetPolicyChain doesn't exist yet + return k +} + +func attrsFor(routeKey string) map[string]*structpb.Struct { + return map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + "xds.route_name": structpb.NewStringValue(routeKey), + }, + }, + } +} + +func TestUpstreamExtProc_DispatchesOnlyToImplementingPolicies(t *testing.T) { + fp := &fakeUpstreamAttemptPolicy{} + chain := ®istry.PolicyChain{Policies: []policy.Policy{nonParticipatingPolicy{}, fp}} + k := newTestRouteConfigAndChain(t, "test-route", chain) + s := NewUpstreamExternalProcessorServer(k) + + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("test-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: "x-envoy-attempt-count", RawValue: []byte("2")}, + }}, + }, + }, + } + + resp, err := s.processRequestHeaders(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fp.lastAttempt != 2 { + t.Fatalf("expected the implementing policy to observe AttemptCount=2, got %d", fp.lastAttempt) + } + rh, ok := resp.Response.(*extprocv3.ProcessingResponse_RequestHeaders) + if !ok { + t.Fatalf("expected a RequestHeaders response, got %T", resp.Response) + } + mutation := rh.RequestHeaders.GetResponse().GetHeaderMutation() + if mutation == nil || len(mutation.SetHeaders) != 1 || string(mutation.SetHeaders[0].Header.RawValue) != "Bearer refreshed" { + t.Fatalf("expected the refreshed Authorization header to be set, got %#v", mutation) + } +} + +func TestUpstreamExtProc_MissingAttemptCountHeaderTreatedAsOne(t *testing.T) { + fp := &fakeUpstreamAttemptPolicy{} + chain := ®istry.PolicyChain{Policies: []policy.Policy{fp}} + k := newTestRouteConfigAndChain(t, "test-route", chain) + s := NewUpstreamExternalProcessorServer(k) + + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("test-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{}}, + }, + } + if _, err := s.processRequestHeaders(context.Background(), req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fp.lastAttempt != 1 { + t.Fatalf("expected a missing attempt-count header to be treated as attempt 1, got %d", fp.lastAttempt) + } +} + +func TestUpstreamExtProc_UnknownRouteReturnsEmptyContinue(t *testing.T) { + k := NewKernel() + s := NewUpstreamExternalProcessorServer(k) + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("no-such-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{}}, + }, + } + resp, err := s.processRequestHeaders(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + rh, ok := resp.Response.(*extprocv3.ProcessingResponse_RequestHeaders) + if !ok || rh.RequestHeaders.GetResponse().GetHeaderMutation() != nil { + t.Fatalf("expected an empty continue response for an unknown route, got %#v", resp.Response) + } + _ = status.Status{} // placeholder import use removed below if unused — see Step 2 note +} +``` + +Note before running: this test file assumes `NewKernel()`, `Kernel.ApplyWholeRouteConfigs(map[string]RouteConfig)`, and a `Kernel.SetPolicyChain(routeKey string, chain *registry.PolicyChain)` helper. If `SetPolicyChain` does not already exist as a test-only setter on `*Kernel` (check `internal/kernel/mapper.go` for the real chain-registration entry point — it's likely `handler.go`'s `buildPolicyChain` populating an internal map the `Kernel` wraps), add a minimal exported-for-package-tests setter now rather than reaching for unexported internals via reflection; confirm the exact existing chain-storage field name in `mapper.go` before adding it, and name the setter to match existing `Kernel` method naming conventions in that file. Remove the unused `status` import if `go vet` flags it — it was left as a placeholder only if a later step needs gRPC status construction; if not needed, delete the import and the placeholder line. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go test ./internal/kernel/... -run TestUpstreamExtProc -v` +Expected: FAIL with `undefined: NewUpstreamExternalProcessorServer` (and possibly `undefined: (*Kernel).SetPolicyChain` — resolve per the Step 1 note before proceeding). + +- [ ] **Step 3: Implement the server** + +```go +// gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go +package kernel + +import ( + "context" + "io" + "log/slog" + "strconv" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// UpstreamExternalProcessorServer is the second, minimal ext_proc gRPC server +// hosted in this same policy-engine process — wired into Envoy's per-cluster +// UPSTREAM HTTP filter chain (see gateway-controller's Task 8), not the +// per-listener downstream chain ExternalProcessorServer (extproc.go) serves. +// It handles only the request-headers phase: this filter attachment point +// has no sensible response phase or body phase for this feature (see the +// design doc). It resolves route -> policy chain via the exact same +// in-memory registry the downstream server uses (s.kernel.GetPolicyChain), +// so a policy's cached state (e.g. oauth2-generator's token cache) is +// naturally shared between both entry points with zero duplication. +type UpstreamExternalProcessorServer struct { + extprocv3.UnimplementedExternalProcessorServer + kernel *Kernel +} + +// NewUpstreamExternalProcessorServer constructs the server. k must be the +// same *Kernel instance the downstream ExternalProcessorServer uses (see +// cmd/policy-engine/main.go, Task 4) — this is what makes chain/state sharing +// automatic rather than something this type has to arrange itself. +func NewUpstreamExternalProcessorServer(k *Kernel) *UpstreamExternalProcessorServer { + return &UpstreamExternalProcessorServer{kernel: k} +} + +// Process implements extprocv3.ExternalProcessorServer. Unlike the downstream +// server's Process (extproc.go), this one only ever expects RequestHeaders +// messages (the cluster's upstream filter is configured with +// RequestHeaderMode: SEND and every other mode left at its default NONE, see +// Task 8) — any other message type gets an empty continue response rather +// than an error, since failing this path must never break the retry itself +// (see Global Constraints: fail open). +func (s *UpstreamExternalProcessorServer) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + ctx := stream.Context() + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var resp *extprocv3.ProcessingResponse + switch req.Request.(type) { + case *extprocv3.ProcessingRequest_RequestHeaders: + resp, err = s.processRequestHeaders(ctx, req) + if err != nil { + slog.ErrorContext(ctx, "upstream ext_proc: failed to process request headers, failing open", "error", err) + resp = emptyContinueRequestHeadersResponse() + } + default: + resp = emptyContinueRequestHeadersResponse() + } + + if err := stream.Send(resp); err != nil { + return status.Errorf(codes.Internal, "upstream ext_proc: failed to send response: %v", err) + } + } +} + +// emptyContinueRequestHeadersResponse is the fail-open / no-op response: no +// header mutation, request proceeds unchanged. +func emptyContinueRequestHeadersResponse() *extprocv3.ProcessingResponse { + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, + } +} + +// processRequestHeaders resolves the route's policy chain and dispatches to +// every policy implementing UpstreamAttemptPolicy, in chain order. A policy +// that doesn't implement it (the common case — rate limiting, analytics, +// transforms) is silently skipped via the type assertion; this is what makes +// the mechanism generic with zero per-policy wiring in this server. +func (s *UpstreamExternalProcessorServer) processRequestHeaders(ctx context.Context, req *extprocv3.ProcessingRequest) (*extprocv3.ProcessingResponse, error) { + routeKey := extractRouteKeyFromAttributes(req) + chain := s.kernel.GetPolicyChain(routeKey) + if chain == nil { + return emptyContinueRequestHeadersResponse(), nil + } + + headers := req.GetRequestHeaders() + attemptCount := 1 + headersMap := make(map[string][]string) + if headers.GetHeaders() != nil { + for _, h := range headers.GetHeaders().GetHeaders() { + key := h.Key + value := string(h.RawValue) + headersMap[key] = append(headersMap[key], value) + if key == "x-envoy-attempt-count" { + if n, err := strconv.Atoi(value); err == nil && n > 0 { + attemptCount = n + } + } + } + } + + actx := &policy.UpstreamAttemptContext{ + AttemptCount: attemptCount, + Headers: policy.NewHeaders(headersMap), + } + + headersToSet := make(map[string]string) + for _, p := range chain.Policies { + attemptPolicy, ok := p.(policy.UpstreamAttemptPolicy) + if !ok { + continue + } + action := attemptPolicy.OnUpstreamAttemptRequestHeaders(ctx, actx) + mods, ok := action.(policy.UpstreamAttemptHeaderModifications) + if !ok { + continue + } + for k, v := range mods.HeadersToSet { + headersToSet[k] = v + } + } + + if len(headersToSet) == 0 { + return emptyContinueRequestHeadersResponse(), nil + } + + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{ + HeaderMutation: buildHeaderValueOptions(headersToSet), + }, + }, + }, + }, nil +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go test ./internal/kernel/... -run TestUpstreamExtProc -v` +Expected: PASS. + +- [ ] **Step 5: Run the full kernel package test suite to confirm no regressions** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go build ./... && GOWORK=off go test ./internal/kernel/... -v 2>&1 | tail -60` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +git add internal/kernel/upstream_extproc.go internal/kernel/upstream_extproc_test.go +git commit -m "feat(policy-engine): add minimal upstream ext_proc server dispatching to UpstreamAttemptPolicy" +``` + +### Task 4: Wire the new server into `policy-engine`'s startup on a second socket/port, with graceful shutdown + +**Files:** +- Modify: `gateway/gateway-runtime/policy-engine/internal/constants/constants.go` +- Modify: `gateway/gateway-runtime/policy-engine/internal/config/config.go` +- Modify: `gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go` +- Modify: `gateway/gateway-runtime/policy-engine/configs/config.toml` (or wherever the shipped default `config.toml` for this component lives — confirm exact path via `grep -rn "extproc_port" gateway-runtime/policy-engine/configs/`) — add the new port's default alongside the existing `extproc_port`. + +**Interfaces:** +- Produces: `constants.DefaultUpstreamExtProcSocketPath = "/var/run/api-platform/policy-engine-upstream.sock"`; `config.ServerConfig.UpstreamExtProcPort int` (koanf tag `upstream_extproc_port`, default `9004` — confirm this doesn't collide with any existing port in `ServerConfig`'s validation block before picking it, per `config.go`'s existing collision checks around `ExtProcPort`). +- Consumes: `kernel.NewUpstreamExternalProcessorServer(k *Kernel)` (Task 3); existing `k` (`*kernel.Kernel`) already constructed earlier in `main()`; existing `internal/utils.CreateGRPCServer(publicKeyPath, privateKeyPath string, plainText bool, opts ...grpc.ServerOption) (*grpc.Server, error)` (`internal/utils/grpc.go:83`) — reused for TLS/plaintext handling parity with the ALS server, rather than a bare `grpc.NewServer()`. + +- [ ] **Step 1: Add the socket path constant.** In `internal/constants/constants.go`, next to `DefaultPolicyEngineSocketPath`: + +```go +// DefaultUpstreamExtProcSocketPath is the Unix socket for the second, +// upstream-attempt ext_proc server (see internal/kernel/upstream_extproc.go), +// distinct from DefaultPolicyEngineSocketPath's per-listener downstream +// server. +DefaultUpstreamExtProcSocketPath = "/var/run/api-platform/policy-engine-upstream.sock" +``` + +- [ ] **Step 2: Add the config field.** In `internal/config/config.go`'s `ServerConfig` struct, next to `ExtProcPort`: + +```go +// UpstreamExtProcPort is the port for the upstream-attempt ext_proc gRPC +// server (TCP mode only) — see internal/kernel/upstream_extproc.go. +UpstreamExtProcPort int `koanf:"upstream_extproc_port"` +``` + +Add its default (next to the existing `ExtProcPort: 9001` default, around line 515) as `UpstreamExtProcPort: 9004,` and its validation (next to the existing `ExtProcPort` bounds check around line 626): + +```go +if c.PolicyEngine.Server.UpstreamExtProcPort <= 0 || c.PolicyEngine.Server.UpstreamExtProcPort > 65535 { + return fmt.Errorf("invalid upstream_extproc_port: %d (must be 1-65535)", c.PolicyEngine.Server.UpstreamExtProcPort) +} +``` + +Also extend the existing port-collision checks (around lines 656/670, which currently compare `Admin.Port`/`Metrics.Port` against `ExtProcPort` when `Mode == "tcp"`) to additionally compare against `UpstreamExtProcPort` — copy the existing two `if` blocks' shape exactly, substituting the field name. + +- [ ] **Step 3: Wire startup in `main()`.** In `cmd/policy-engine/main.go`, immediately after the existing block that creates `extprocServer`/`lis`/`grpcServer` and registers it (i.e. right after the line `extprocv3.RegisterExternalProcessorServer(grpcServer, extprocServer)`), add: + +```go +// Create and start the upstream-attempt ext_proc gRPC server (second, +// minimal endpoint — see internal/kernel/upstream_extproc.go). Uses the same +// serverMode (uds/tcp) as the main ext_proc server, but its own socket/port, +// and its own explicit message/stream limits sized for its headers-only +// message shape (go-network-service-hardening.md directive 2) — not copied +// from the main server's larger ceiling. +upstreamExtprocServer := kernel.NewUpstreamExternalProcessorServer(k) + +var upstreamLis net.Listener +switch serverMode { +case "uds": + socketPath := constants.DefaultUpstreamExtProcSocketPath + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + slog.WarnContext(ctx, "Failed to remove existing upstream ext_proc socket file", "path", socketPath, "error", err) + } + upstreamLis, err = net.Listen("unix", socketPath) + if err != nil { + slog.ErrorContext(ctx, "Failed to listen on upstream ext_proc Unix socket", "path", socketPath, "error", err) + os.Exit(1) + } + if err := os.Chmod(socketPath, 0660); err != nil { + slog.WarnContext(ctx, "Failed to set upstream ext_proc socket permissions", "path", socketPath, "error", err) + } + slog.InfoContext(ctx, "Upstream ext_proc server listening on Unix socket", "path", socketPath) +case "tcp": + upstreamLis, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.PolicyEngine.Server.UpstreamExtProcPort)) + if err != nil { + slog.ErrorContext(ctx, "Failed to listen on upstream ext_proc port", "port", cfg.PolicyEngine.Server.UpstreamExtProcPort, "error", err) + os.Exit(1) + } + slog.InfoContext(ctx, "Upstream ext_proc server listening on TCP port", "port", cfg.PolicyEngine.Server.UpstreamExtProcPort) +} + +upstreamGrpcServer := grpc.NewServer( + grpc.MaxRecvMsgSize(64*1024), // headers-only messages; far smaller than the body-carrying main server's ceiling + grpc.MaxSendMsgSize(64*1024), + grpc.MaxConcurrentStreams(1000), +) +extprocv3.RegisterExternalProcessorServer(upstreamGrpcServer, upstreamExtprocServer) + +go func() { + if err := upstreamGrpcServer.Serve(upstreamLis); err != nil { + serverErrCh <- err + } +}() +``` + +- [ ] **Step 4: Wire graceful shutdown.** In the shutdown sequence, immediately before the existing `grpcServer.GracefulStop()` line, add: + +```go +slog.InfoContext(ctx, "Stopping upstream ext_proc gRPC server") +upstreamGrpcServer.GracefulStop() +``` + +And in the UDS socket cleanup block (the `if serverMode == "uds"` block near the end), add a second cleanup line alongside the existing `os.Remove(constants.DefaultPolicyEngineSocketPath)`: + +```go +if err := os.Remove(constants.DefaultUpstreamExtProcSocketPath); err != nil && !os.IsNotExist(err) { + slog.WarnContext(ctx, "Failed to cleanup upstream ext_proc socket file on shutdown", + "path", constants.DefaultUpstreamExtProcSocketPath, "error", err) +} +``` + +- [ ] **Step 5: Add the new port's default to the shipped config.** Find the shipped `config.toml`'s `[policy_engine.server]` (or equivalent) section: + +```bash +grep -rn "extproc_port" gateway-runtime/policy-engine/configs/ gateway/configs/config.toml +``` + +Add `upstream_extproc_port = 9004` alongside whatever key holds `extproc_port` there, in every config file that currently sets `extproc_port` explicitly (shipped default + any e2e/test config.toml that overrides it — grep confirms the full list). + +- [ ] **Step 6: Build and run the existing test suite** + +Run: `cd gateway-runtime/policy-engine && GOWORK=off go build ./... && GOWORK=off go test ./... 2>&1 | tail -40` +Expected: builds clean, all existing tests still PASS (no test exercises the new server's startup wiring directly — that's covered by Task 10's e2e test). + +- [ ] **Step 7: Manually verify the process starts with both sockets present** + +```bash +cd gateway-runtime/policy-engine && go run ./cmd/policy-engine -policy-chains-file <(echo '{}') & +sleep 1 +ls -la /var/run/api-platform/policy-engine.sock /var/run/api-platform/policy-engine-upstream.sock +kill %1 +``` +Expected: both socket files listed (adjust the placeholder config-mode invocation to whatever this binary's existing minimal-startup smoke-test invocation already is, e.g. check `Makefile`/`README.md` in this directory for the exact minimal local-run command — do not invent a flag that doesn't exist; confirm via `go run ./cmd/policy-engine -h` first). + +- [ ] **Step 8: Commit** + +```bash +git add internal/constants/constants.go internal/config/config.go cmd/policy-engine/main.go configs/ +git commit -m "feat(policy-engine): wire the upstream ext_proc server into startup/shutdown on a second socket" +``` + +--- + +## Phase 3: gateway-controller — config surface and Envoy config emission + +### Task 5: Add `resilience.retry` to the OpenAPI schema and regenerate + +**Files:** +- Modify: `gateway/gateway-controller/api/management-openapi.yaml` +- Regenerate: `gateway/gateway-controller/pkg/api/management/generated.go` (do not hand-edit — `// Code generated ... DO NOT EDIT`) + +**Interfaces:** +- Produces: `api.Retry{StatusCodes []int, NumRetries *int}` and `api.Resilience.Retry *Retry` (both generated by `oapi-codegen` from the schema below — exact generated type/field names follow `oapi-codegen`'s standard naming from the schema property names, matching the existing `Resilience.Timeout`/`Resilience.IdleTimeout` sibling fields' generated shape). + +- [ ] **Step 1: Extend the schema.** In `management-openapi.yaml`, find the `Resilience` schema (currently `timeout`/`idleTimeout` only, per the existing description block). `retry` must be its own top-level schema referenced via `$ref` — NOT a nested inline object property — otherwise `oapi-codegen` generates an anonymous inline struct instead of a named `Retry` type, which every later task in this plan (6, 7, 8, 9) depends on as `api.Retry`. Add a `retry` property referencing a new sibling top-level `Retry` schema, and define that schema alongside `Resilience` (e.g. immediately after it, at the same indentation level as every other top-level schema like `Resilience`/`Upstream`): + +```yaml + Resilience: + type: object + description: > + Backend/route timeout and retry configuration. Maps to Envoy RouteAction + timeouts and RetryPolicy. Can be set at the API level (applies to all + routes) and/or the operation level (applies to that operation's route). + When set at both levels, the operation-level value takes precedence. + properties: + timeout: + type: string + description: Maximum time for the entire route (request to upstream response). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 15s + idleTimeout: + type: string + description: Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. + pattern: '^\d+(\.\d+)?(ms|s|m|h)$' + example: 0s + retry: + $ref: '#/components/schemas/Retry' + + Retry: + type: object + description: > + Native Envoy retry on the listed response status codes. When set, + any policy on this route implementing the upstream-attempt refresh + mechanism (see UpstreamAttemptPolicy in the policy SDK) gets a + chance to attach fresh per-attempt state (e.g. a refreshed + credential) before each retried attempt goes out. + required: + - statusCodes + properties: + statusCodes: + type: array + items: + type: integer + minimum: 400 + maximum: 599 + minItems: 1 + description: Response status codes that trigger a retry. + example: [401] + numRetries: + type: integer + minimum: 1 + default: 1 + description: Maximum number of retry attempts. +``` + +If this file's existing convention for a `$ref`'d property alongside its own description differs (check how any other property in this file already does this), match that convention instead — the important, non-negotiable part is that `Retry` ends up as its own top-level schema so codegen names it, not the exact `$ref` placement syntax. + +- [ ] **Step 2: Regenerate** + +```bash +cd gateway-controller && make generate-server-code +``` + +- [ ] **Step 3: Confirm the generated types compile and match expectations** + +```bash +GOWORK=off go build ./... 2>&1 | tail -30 +grep -n "type Retry struct" -A 10 pkg/api/management/generated.go +``` +Expected: builds clean; `Retry` struct present with `StatusCodes []int` (or `[]int32`/generated numeric type — read the actual output rather than assuming) and a `NumRetries *int` field. + +- [ ] **Step 4: Commit** + +```bash +git add api/management-openapi.yaml pkg/api/management/generated.go +git commit -m "feat(gateway-controller): add resilience.retry to the OpenAPI schema" +``` + +### Task 6: Validate `resilience.retry` in both REST and LLM validators + +**Files:** +- Modify: `gateway/gateway-controller/pkg/config/api_validator.go` +- Modify: `gateway/gateway-controller/pkg/config/llm_validator.go` +- Test: `gateway/gateway-controller/pkg/config/api_validator_test.go` +- Test: `gateway/gateway-controller/pkg/config/llm_validator_test.go` + +**Interfaces:** +- Consumes: `api.Retry` (Task 5); existing `validateResilience(fieldPrefix string, r *api.Resilience) []ValidationError` (`api_validator.go:478`, currently delegates only to `validateResilienceTimeouts`). +- Produces: `validateResilienceRetry(fieldPrefix string, r *api.Retry) []ValidationError`, called from `validateResilience` (so both `spec.resilience.retry` and `spec.operations[i].resilience.retry` get it, matching the existing call sites at lines 466 and 639) — this is deliberately added to the one shared `validateResilience` function, not duplicated per validator, so both REST and LLM configs get the check identically. Confirm `llm_validator.go` calls the same shared `validateResilience` function (not a separate copy) before writing this task's test — if it currently has its own separate resilience-validation code path, that's the actual fix needed here (route it through the shared function instead of duplicating the check). + +- [ ] **Step 1: Write the failing tests** + +```go +// added to api_validator_test.go +func TestValidateResilience_RetryRequiresNonEmptyStatusCodes(t *testing.T) { + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{}}} + errs := validateResilience("spec.resilience", r) + if len(errs) == 0 { + t.Error("expected an error for empty resilience.retry.statusCodes") + } +} + +func TestValidateResilience_RetryValidConfigPasses(t *testing.T) { + numRetries := 2 + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}} + errs := validateResilience("spec.resilience", r) + if len(errs) != 0 { + t.Errorf("expected no errors, got %v", errs) + } +} +``` + +```go +// added to llm_validator_test.go, inside the existing TestValidateLLMProvider_Resilience +// test function (llm_validator_test.go:2121) as two more t.Run subtests, +// alongside its existing timeout/idleTimeout cases — same validProviderWithResilience +// helper (line 1989), same assertHasFieldError helper already used throughout +// this test. Proves the LLM validator path enforces the identical +// resilience.retry check as the REST validator, since both already route +// through the one shared validateResilience function (confirmed by this file's +// existing timeout subtests already asserting on "spec.resilience.timeout" +// field-path errors from validProviderWithResilience — the same shared-function +// evidence Task 6 Step 4 needs, so that step is a confirmation, not a fix). + +t.Run("retry with empty statusCodes is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{}}, + })) + assertHasFieldError(t, errs, "spec.resilience.retry.statusCodes") +}) + +t.Run("retry with valid statusCodes and numRetries is accepted", func(t *testing.T) { + numRetries := 2 + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}, + })) + assert.Empty(t, errs) +}) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/config/... -run 'TestValidateResilience_Retry|TestValidateLLMProvider_ResilienceRetry' -v` +Expected: FAIL (`validateResilienceRetry`/the referenced fields don't exist yet, or the LLM path doesn't invoke it). + +- [ ] **Step 3: Implement the validation.** In `api_validator.go`, add: + +```go +// validateResilienceRetry validates a resilience.retry block: statusCodes +// must be non-empty (each in the valid HTTP status range, already enforced +// by the OpenAPI schema's minimum/maximum — this is a defense-in-depth check +// for configs that bypass schema validation, e.g. direct DB rows), and +// numRetries (if set) must be >= 1. +func validateResilienceRetry(fieldPrefix string, r *api.Retry) []ValidationError { + if r == nil { + return nil + } + var errs []ValidationError + if len(r.StatusCodes) == 0 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.statusCodes", + Message: "must be non-empty when resilience.retry is configured", + }) + } + for _, code := range r.StatusCodes { + if code < 400 || code > 599 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.statusCodes", + Message: fmt.Sprintf("status code %d is not a valid HTTP status code (400-599)", code), + }) + } + } + if r.NumRetries != nil && *r.NumRetries < 1 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.numRetries", + Message: "must be at least 1 when set", + }) + } + return errs +} +``` + +Update `validateResilience` (line 478) to also call it: + +```go +func (v *APIValidator) validateResilience(fieldPrefix string, r *api.Resilience) []ValidationError { + errs := validateResilienceTimeouts(fieldPrefix, r) + if r != nil { + errs = append(errs, validateResilienceRetry(fieldPrefix, r.Retry)...) + } + return errs +} +``` + +- [ ] **Step 4: Confirm the LLM validator routes through this same function.** Already confirmed during plan-writing: `llm_validator_test.go`'s existing `TestValidateLLMProvider_Resilience` (line 2121) asserts on `"spec.resilience.timeout"`/`"spec.resilience.idleTimeout"` field-path errors from `validProviderWithResilience(&api.Resilience{...})` — the identical field-path strings `validateResilienceTimeouts` produces — proving `llm_validator.go` already calls the shared `validateResilience`, not a separate duplicate path. So Step 3's addition to `validateResilience` automatically covers LLM configs too; this step is a build-and-test confirmation, not a code fix. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/config/... -run 'TestValidateResilience_Retry|TestValidateLLMProvider_ResilienceRetry' -v` +Expected: PASS. + +- [ ] **Step 6: Run the full config package test suite to confirm no regressions** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/config/... -v 2>&1 | tail -60` +Expected: all PASS. + +- [ ] **Step 7: Commit** + +```bash +git add pkg/config/api_validator.go pkg/config/llm_validator.go pkg/config/api_validator_test.go pkg/config/llm_validator_test.go +git commit -m "feat(gateway-controller): validate resilience.retry for both REST and LLM configs" +``` + +### Task 7: Emit `RouteAction.RetryPolicy` from `resilience.retry` + +**Files:** +- Modify: `gateway/gateway-controller/pkg/xds/translator.go` +- Test: `gateway/gateway-controller/pkg/xds/translator_test.go` + +**Interfaces:** +- Consumes: `api.Retry` (Task 5); the existing `resolvedTimeout` struct and `ResolveResilience`/`combineRouteResilience` functions (`translator.go:3252-3300`, already resolve `timeout`/`idleTimeout` per-route, precedence-aware) — extend the SAME resolved-timeout struct with a resolved retry field rather than threading a second, parallel parameter through every call site. +- Produces: adds a `Retry *api.Retry` field to the existing `resolvedTimeout` struct (rename awareness: every existing call site constructing/reading a `resolvedTimeout` continues to compile since this is an additive field), and sets `route.RouteAction.RetryPolicy` in `createRoute` (`translator.go:1628`) when that field is non-nil. + +- [ ] **Step 1: Write the failing test** + +```go +// added to translator_test.go — follow this file's existing convention for +// constructing a minimal RestApi/LlmProvider config with a resilience block +// (an existing test already covers plain timeout/idleTimeout — copy its +// fixture-building shape, then additionally set Resilience.Retry) and assert +// on the resulting *route.Route's RouteAction.RetryPolicy. +func TestCreateRoute_ResilienceRetryEmitsNativeRetryPolicy(t *testing.T) { + numRetries := 2 + timeoutCfg := &resolvedTimeout{Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}} + + tr := &Translator{routerConfig: minimalRouterConfigForTest()} // reuse this file's existing minimal-router-config test helper + r := tr.createRoute("api-id", "TestAPI", "v1", "/test", "GET", "/foo", "test-cluster", + "", "localhost", "RestApi", "", "", nil, "project-1", timeoutCfg, false, nil) + + routeAction, ok := r.Action.(*route.Route_Route) + if !ok { + t.Fatalf("expected a Route_Route action, got %T", r.Action) + } + rp := routeAction.Route.RetryPolicy + if rp == nil { + t.Fatal("expected a non-nil RetryPolicy") + } + if rp.RetryOn != "retriable-status-codes" { + t.Errorf("got RetryOn %q, want %q", rp.RetryOn, "retriable-status-codes") + } + if rp.NumRetries == nil || rp.NumRetries.Value != 2 { + t.Errorf("got NumRetries %v, want 2", rp.NumRetries) + } + if len(rp.RetriableStatusCodes) != 2 || rp.RetriableStatusCodes[0] != 401 || rp.RetriableStatusCodes[1] != 503 { + t.Errorf("got RetriableStatusCodes %v, want [401 503]", rp.RetriableStatusCodes) + } +} +``` + +Note: check `createRoute`'s exact current parameter list (`translator.go:1628`) before writing this call — copy it exactly from the file rather than the abbreviated signature shown in this plan's earlier reading, since it takes many positional parameters and an off-by-one will misassign an unrelated field. Also confirm the actual field name for `resolvedTimeout` (may need to add `Retry *api.Retry` to it as part of Step 3 before this test can even reference it). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/xds/... -run TestCreateRoute_ResilienceRetryEmitsNativeRetryPolicy -v` +Expected: FAIL (`resolvedTimeout` has no `Retry` field yet, or `RetryPolicy` is nil). + +- [ ] **Step 3: Add the field and resolve it.** In the `resolvedTimeout` struct (near `ResolveResilience`, `translator.go:3252`), add: + +```go +type resolvedTimeout struct { + Route *time.Duration + Idle *time.Duration + Retry *api.Retry // nil when resilience.retry is not configured +} +``` + +Update `ResolveResilience` to also surface it: + +```go +func ResolveResilience(r *api.Resilience) (timeout *time.Duration, idleTimeout *time.Duration, retry *api.Retry, err error) { + if r == nil { + return nil, nil, nil, nil + } + // ... existing timeout/idleTimeout parsing unchanged ... + return timeout, idleTimeout, r.Retry, nil +} +``` + +Update every call site of `ResolveResilience` (there are at least two, per the earlier reading: `translator.go:1079` and `:1096`/`:1128`) to accept the third return value and thread it into the `resolvedTimeout` constructed at each site (`combineRouteResilience`, `translator.go:3272`) — add a `retry *api.Retry` parameter to `combineRouteResilience` too (operation-level `Retry` overrides API-level `Retry` when set, matching the existing precedence rule for `timeout`/`idleTimeout` exactly — same "operation-level wins if non-nil, else fall back to API-level" logic already implemented there for the other two fields). + +- [ ] **Step 4: Emit the RetryPolicy in `createRoute`.** Immediately after the existing block that sets `routeAction.Route.Timeout`/`.IdleTimeout` from `timeoutCfg` in `createRoute`, add: + +```go +if timeoutCfg != nil && timeoutCfg.Retry != nil { + numRetries := uint32(1) + if timeoutCfg.Retry.NumRetries != nil { + numRetries = uint32(*timeoutCfg.Retry.NumRetries) + } + statusCodes := make([]uint32, len(timeoutCfg.Retry.StatusCodes)) + for i, code := range timeoutCfg.Retry.StatusCodes { + statusCodes[i] = uint32(code) + } + routeAction.Route.RetryPolicy = &route.RetryPolicy{ + RetryOn: "retriable-status-codes", + RetriableStatusCodes: statusCodes, + NumRetries: wrapperspb.UInt32(numRetries), + } +} +``` + +Confirm `wrapperspb` is already imported in this file (it's used elsewhere per earlier reading, e.g. `wrapperspb.Bool` in `createExtProcFilter`) — if not already imported under that alias, add `"google.golang.org/protobuf/types/known/wrapperspb"`. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/xds/... -run TestCreateRoute_ResilienceRetryEmitsNativeRetryPolicy -v` +Expected: PASS. + +- [ ] **Step 6: Run the full xds package test suite to confirm no regressions** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/xds/... -v 2>&1 | tail -80` +Expected: all PASS — pay particular attention to any existing test asserting on `resolvedTimeout`'s literal struct shape (a struct literal comparison test would need its expected value updated to include `Retry: nil`). + +- [ ] **Step 7: Commit** + +```bash +git add pkg/xds/translator.go pkg/xds/translator_test.go +git commit -m "feat(gateway-controller): emit native RouteAction.RetryPolicy from resilience.retry" +``` + +### Task 8: Attach the upstream `ext_proc` filter to clusters backing a retry-configured route + +**Files:** +- Modify: `gateway/gateway-runtime/policy-engine/internal/constants/constants.go` — wait, this belongs in gateway-controller, not policy-engine; correct path: `gateway/gateway-controller/pkg/constants/constants.go` +- Modify: `gateway/gateway-controller/pkg/xds/translator.go` +- Test: `gateway/gateway-controller/pkg/xds/translator_test.go` + +**Interfaces:** +- Consumes: `envoy/extensions/upstreams/http/v3.HttpProtocolOptions` (vendored in `go-control-plane v1.37.0`, confirmed present at `extensions/upstreams/http/v3/http_protocol_options.pb.go`, field `HttpFilters []*hcm.HttpFilter`); `cluster.Cluster.TypedExtensionProtocolOptions map[string]*anypb.Any` (confirmed field, `config/cluster/v3/cluster.pb.go:890`); the existing `createPolicyEngineCluster()` (`translator.go:1937`) as the exact pattern to mirror for a second internal cluster; the existing `TranslateConfigs`/`clusterMap` loop (`translator.go:717-799`) where clusters from every API are merged by name. +- Produces: `constants.UpstreamRefreshPolicyEngineClusterName` (new constant, gateway-controller's `pkg/constants/constants.go`, alongside the existing `PolicyEngineClusterName`); `(t *Translator) createUpstreamRefreshExtProcCluster() *cluster.Cluster`; `(t *Translator) createUpstreamRefreshExtProcFilter() (*hcm.HttpFilter, error)`; a per-cluster attachment step folded into the existing cluster-building/merge path so that any cluster backing at least one `resilience.retry`-configured route gets `TypedExtensionProtocolOptions` set. + +- [ ] **Step 1: Write the failing test** + +```go +// added to translator_test.go +func TestTranslateConfigs_ClusterGetsUpstreamFilterWhenAnyRouteHasRetryConfigured(t *testing.T) { + // Build two minimal RestApi configs sharing the identical upstream host:scheme + // (so they collapse into one Envoy cluster per this codebase's existing + // host+scheme dedup — reuse this file's existing fixture-building helper for + // "two APIs, same backend host" if one already exists, e.g. check + // TestTranslateConfigs_* tests around the cluster-merge behavior for the + // established pattern), where only ONE of the two has resilience.retry set. + // + // Assert: the resulting merged cluster (found by its deduped name) has + // TypedExtensionProtocolOptions containing the upstream ext_proc filter - + // proving the OR-across-sharers behavior from the design doc. +} + +func TestTranslateConfigs_ClusterWithNoRetryConfiguredAnywhereGetsNoUpstreamFilter(t *testing.T) { + // Same two-APIs-same-cluster shape, but NEITHER has resilience.retry set. + // Assert: TypedExtensionProtocolOptions is nil/empty for that cluster. +} +``` + +(Concrete fixture code intentionally left to be copied from this test file's own established two-API-shared-cluster test, if one exists from Task 0's Q2 findings verification — confirm via `grep -n "same.*host\|shared.*cluster\|dedup" pkg/xds/translator_test.go` before inventing new fixture-building boilerplate.) + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/xds/... -run TestTranslateConfigs_ClusterGetsUpstreamFilter -v` +Expected: FAIL (no such attachment logic exists yet). + +- [ ] **Step 3: Add the cluster name constant.** In `pkg/constants/constants.go`, next to `PolicyEngineClusterName`: + +```go +// UpstreamRefreshPolicyEngineClusterName is the internal Envoy cluster +// pointing at policy-engine's second, upstream-attempt ext_proc endpoint +// (see gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go). +UpstreamRefreshPolicyEngineClusterName = "policy_engine_upstream_refresh_cluster" +``` + +- [ ] **Step 4: Add the internal cluster constructor.** In `translator.go`, mirroring `createPolicyEngineCluster()` (line 1937) exactly except for the cluster name and target port: + +```go +// createUpstreamRefreshExtProcCluster creates the internal Envoy cluster +// pointing at policy-engine's second, upstream-attempt ext_proc endpoint. +// Mirrors createPolicyEngineCluster's addressing (UDS by default, TCP via +// t.routerConfig.PolicyEngine.Mode) — this is a DIFFERENT socket/port on the +// same policy-engine process, not a different service. +func (t *Translator) createUpstreamRefreshExtProcCluster() *cluster.Cluster { + policyEngine := t.routerConfig.PolicyEngine + + var address *core.Address + if policyEngine.Mode == "tcp" { + address = &core.Address{ + Address: &core.Address_SocketAddress{ + SocketAddress: &core.SocketAddress{ + Protocol: core.SocketAddress_TCP, + Address: policyEngine.Host, + PortSpecifier: &core.SocketAddress_PortValue{ + PortValue: policyEngine.UpstreamRefreshPort, // see Step 5: new RouterConfig field + }, + }, + }, + } + } else { + address = &core.Address{ + Address: &core.Address_Pipe{ + Pipe: &core.Pipe{Path: constants.DefaultUpstreamExtProcSocketPath}, + }, + } + } + + lbEndpoint := &endpoint.LbEndpoint{ + HostIdentifier: &endpoint.LbEndpoint_Endpoint{Endpoint: &endpoint.Endpoint{Address: address}}, + } + clusterType := cluster.Cluster_STATIC + if policyEngine.Mode == "tcp" { + clusterType = cluster.Cluster_STRICT_DNS + } + + return &cluster.Cluster{ + Name: constants.UpstreamRefreshPolicyEngineClusterName, + ConnectTimeout: durationpb.New(5 * time.Second), + ClusterDiscoveryType: &cluster.Cluster_Type{Type: clusterType}, + LbPolicy: cluster.Cluster_ROUND_ROBIN, + LoadAssignment: &endpoint.ClusterLoadAssignment{ + ClusterName: constants.UpstreamRefreshPolicyEngineClusterName, + Endpoints: []*endpoint.LocalityLbEndpoints{{LbEndpoints: []*endpoint.LbEndpoint{lbEndpoint}}}, + }, + Http2ProtocolOptions: &core.Http2ProtocolOptions{}, + } +} +``` + +Note: `constants.DefaultUpstreamExtProcSocketPath` here refers to gateway-controller's OWN copy of this path constant (gateway-controller and policy-engine are separate Go modules — this string must be defined in `gateway-controller/pkg/constants/constants.go` too, kept literally identical to policy-engine's `internal/constants/constants.go` value from Task 4; add a short comment cross-referencing the other module's constant by file path, matching this codebase's existing convention for cross-module constant duplication flagged in an earlier KB note on this same feature area). + +- [ ] **Step 5: Add the router config field for TCP mode.** Find `RouterConfig.PolicyEngine`'s struct definition (used as `t.routerConfig.PolicyEngine` above) and add `UpstreamRefreshPort int` alongside its existing `Port int` field, populated from whatever config-loading path already populates `Port` (grep for where `PolicyEngine.Port` is set from a config file/env var and add the mirror for the new field using the same mechanism). + +- [ ] **Step 6: Add the upstream filter constructor.** In `translator.go`, mirroring `createExtProcFilter()` (line 3141) but scoped to request-headers only: + +```go +// createUpstreamRefreshExtProcFilter creates the per-cluster upstream ext_proc +// filter that lets any UpstreamAttemptPolicy-implementing policy attach fresh +// per-attempt state to a native Envoy retry. Unlike the main downstream +// filter, this one only ever needs the request-headers phase. +func (t *Translator) createUpstreamRefreshExtProcFilter() (*hcm.HttpFilter, error) { + policyEngine := t.routerConfig.PolicyEngine + extProcConfig := &extproc.ExternalProcessor{ + GrpcService: &core.GrpcService{ + TargetSpecifier: &core.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &core.GrpcService_EnvoyGrpc{ClusterName: constants.UpstreamRefreshPolicyEngineClusterName}, + }, + Timeout: durationpb.New(time.Duration(policyEngine.TimeoutMs) * time.Millisecond), + }, + FailureModeAllow: true, // fail open — see Global Constraints; a failure here must never block the retry + ProcessingMode: &extproc.ProcessingMode{ + RequestHeaderMode: extproc.ProcessingMode_SEND, + }, + MessageTimeout: durationpb.New(time.Duration(policyEngine.MessageTimeoutMs) * time.Millisecond), + RequestAttributes: []string{constants.ExtProcRequestAttributeRouteName}, + } + extProcAny, err := anypb.New(extProcConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal upstream ext_proc config: %w", err) + } + return &hcm.HttpFilter{ + Name: constants.ExtProcFilterName + "_upstream_refresh", + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: extProcAny}, + }, nil +} +``` + +Note `FailureModeAllow: true` here is the deliberate opposite of the main downstream filter's `FailureModeAllow: false` (line 3156) — that asymmetry is intentional and must not be "fixed" to match: the downstream filter gates auth/access-control (must fail closed), this one only ever adds an optional header refresh to an already-in-flight retry (must fail open, per Global Constraints). + +- [ ] **Step 7: Attach the filter to eligible clusters, OR'd across sharers.** In `TranslateConfigs`'s cluster-merge loop (`translator.go:717-799`, the one building `clusterMap`), after the loop that merges clusters by name (`clusterMap[c.Name] = c`), add a second pass that tracks which cluster names need the upstream filter: + +```go +// A cluster needs the upstream refresh filter if ANY route across ANY +// deployed API resolves resilience.retry to it — OR'd across every API +// sharing that cluster (clusters are deduped by host+scheme, so two +// unrelated APIs can share one cluster; see the design doc's shared-cluster +// hazard section). This pass is separate from the cluster-merge loop above +// because "does this route have retry configured" is resolved per-route +// (createRoute/resolvedTimeout), not per-cluster, so it must be collected +// alongside route creation and applied to the already-merged cluster map +// afterward. +clustersNeedingUpstreamFilter := make(map[string]bool) +// (populate this set from within the same per-operation loop that already +// calls createRoute and resolves timeoutCfg for each operation — wherever +// timeoutCfg.Retry != nil, mark clustersNeedingUpstreamFilter[clusterName] = true +// using the same clusterName variable already in scope at that call site. +// This requires threading the resolved cluster name and retry-presence out +// of the existing per-operation loop into this outer scope — the simplest +// correct approach is accumulating into a slice/map declared before the +// loop starts and written to inside it, exactly like clusterMap itself +// already is.) + +if len(clustersNeedingUpstreamFilter) > 0 { + upstreamFilter, err := t.createUpstreamRefreshExtProcFilter() + if err != nil { + return nil, nil, fmt.Errorf("failed to create upstream refresh ext_proc filter: %w", err) + } + filterAny, err := anypb.New(&httpv3.HttpProtocolOptions{ + UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{}, + }, + }, + HttpFilters: []*hcm.HttpFilter{upstreamFilter}, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal upstream HttpProtocolOptions: %w", err) + } + for clusterName := range clustersNeedingUpstreamFilter { + c, ok := clusterMap[clusterName] + if !ok { + continue // cluster resolution failed elsewhere; nothing to attach to + } + if c.TypedExtensionProtocolOptions == nil { + c.TypedExtensionProtocolOptions = make(map[string]*anypb.Any) + } + c.TypedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] = filterAny + } + // Always add the internal cluster the filter targets, once, unconditionally + // (cheap — see the design doc) if not already present. + if _, ok := clusterMap[constants.UpstreamRefreshPolicyEngineClusterName]; !ok { + clusterMap[constants.UpstreamRefreshPolicyEngineClusterName] = t.createUpstreamRefreshExtProcCluster() + } +} +``` + +Import `httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"` at the top of the file alongside the existing `extproc` import. + +Note on `HttpProtocolOptions_ExplicitHttpConfig`: this repo's existing clusters (per Task 0-equivalent reading during design) never set explicit HTTP version options on data-plane clusters — using `HttpProtocolOptions_ExplicitHttpConfig` with an empty (default HTTP/1.1) `HttpProtocolOptions` sub-message here preserves that same default; do not add `Http2ProtocolOptions` inside it unless a specific data-plane cluster already required HTTP/2 before this change (confirmed in Task 0-equivalent reading: `createCluster` sets none today, so this stays default/HTTP1.1-compatible). + +- [ ] **Step 8: Run tests to verify they pass** + +Run: `cd gateway-controller && GOWORK=off go test ./pkg/xds/... -run TestTranslateConfigs_Cluster -v` +Expected: PASS. + +- [ ] **Step 9: Run the full xds package test suite and full build to confirm no regressions** + +Run: `cd gateway-controller && GOWORK=off go build ./... && GOWORK=off go test ./... 2>&1 | tail -100` +Expected: all PASS. + +- [ ] **Step 10: Commit** + +```bash +git add pkg/constants/constants.go pkg/xds/translator.go pkg/xds/translator_test.go +git commit -m "feat(gateway-controller): attach upstream ext_proc filter to clusters backing a retry-configured route" +``` + +--- + +## Phase 4: oauth2-generator — implement the refresh + +### Task 9: Implement `UpstreamAttemptPolicy` on oauth2-generator, in both repos + +**Files:** +- Modify: `gateway/dev-policies/oauth2-generator/oauth2_generator.go` +- Test: `gateway/dev-policies/oauth2-generator/oauth2_generator_test.go` +- Mirror identically into: the separate `gateway-controllers/policies/oauth2-generator` repo's `oauth2_generator.go` (per this repo's established dual-repo convention — diff byte-for-byte after this change) + +**Interfaces:** +- Consumes: `policy.UpstreamAttemptContext`/`policy.UpstreamAttemptAction`/`policy.UpstreamAttemptHeaderModifications` (Task 1); existing `p.retrieveToken() (*xoauth2.Token, error)` (`oauth2_generator.go:958`); existing `p.tokenSource.Purge()` (the `tokenProvider` interface, `token_cache.go:293-296`); existing `buildHeaderValue(prefix, token string) string` helper and `p.headerName`/`p.valuePrefix` fields (already used identically in `OnRequestHeaders`, `oauth2_generator.go:938-941`). +- Produces: `(p *Policy) OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *policy.UpstreamAttemptContext) policy.UpstreamAttemptAction`. + +- [ ] **Step 1: Write the failing test** + +```go +// added to oauth2_generator_test.go — follow this file's existing convention +// for constructing a *Policy via GetPolicy with a minimal valid params map +// (an existing OnRequestHeaders test already does this — copy its setup). +func TestOnUpstreamAttemptRequestHeaders_AttemptOneUsesCachedToken(t *testing.T) { + p := newTestPolicyWithMockTokenSource(t) // reuse this file's existing test-policy constructor helper + actx := &policy.UpstreamAttemptContext{AttemptCount: 1, Headers: policy.NewHeaders(nil)} + + action := p.OnUpstreamAttemptRequestHeaders(context.Background(), actx) + mods, ok := action.(policy.UpstreamAttemptHeaderModifications) + if !ok { + t.Fatalf("expected UpstreamAttemptHeaderModifications, got %T", action) + } + if _, set := mods.HeadersToSet["Authorization"]; !set { + t.Error("expected Authorization to be set even on attempt 1") + } +} + +func TestOnUpstreamAttemptRequestHeaders_RetryPurgesAndRefetches(t *testing.T) { + p, mockSource := newTestPolicyWithMockTokenSourceTrackingPurge(t) // extend the helper to expose a purge-call counter/mock, matching this file's existing mocking conventions for tokenSource + actx := &policy.UpstreamAttemptContext{AttemptCount: 2, Headers: policy.NewHeaders(nil)} + + _ = p.OnUpstreamAttemptRequestHeaders(context.Background(), actx) + + if !mockSource.purgeCalled { + t.Error("expected AttemptCount > 1 to purge the cached token before refetching") + } +} + +func TestOnUpstreamAttemptRequestHeaders_FetchErrorFailsOpen(t *testing.T) { + p := newTestPolicyWithFailingTokenSource(t) // a tokenSource whose Token() always errors + actx := &policy.UpstreamAttemptContext{AttemptCount: 2, Headers: policy.NewHeaders(nil)} + + action := p.OnUpstreamAttemptRequestHeaders(context.Background(), actx) + mods, ok := action.(policy.UpstreamAttemptHeaderModifications) + if !ok || len(mods.HeadersToSet) != 0 { + t.Errorf("expected an empty no-op action on fetch failure (fail open), got %#v", action) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd dev-policies/oauth2-generator && GOWORK=off go test ./... -run TestOnUpstreamAttemptRequestHeaders -v` +Expected: FAIL with `undefined: (*Policy).OnUpstreamAttemptRequestHeaders` (and confirm/add the test-helper mocks referenced above, matching this file's existing `tokenProvider`-mocking conventions — check existing tests around `Purge()` assertions, e.g. any existing purge-on-401 test, for the established mock shape before inventing a new one). + +- [ ] **Step 3: Implement the method.** In `oauth2_generator.go`, immediately after the existing `OnResponseHeaders` method: + +```go +// OnUpstreamAttemptRequestHeaders implements policy.UpstreamAttemptPolicy — it +// runs once per individual upstream dial attempt (including Envoy-native +// retries; see resilience.retry), not once per client request. On any +// attempt after the first, the previous attempt's response is assumed +// rejected (that's why Envoy retried at all, per the configured +// resilience.retry.statusCodes), so the cached token is purged before +// refetching, guaranteeing attempt 2+ gets a genuinely fresh token rather +// than resending the same one that was just rejected. Fails open on any +// fetch error: an empty action lets the retry proceed with whatever +// Authorization header it already had rather than blocking it — this +// mechanism only ever makes a retry more likely to succeed, never a new way +// for it to fail (see Global Constraints). +func (p *Policy) OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *policy.UpstreamAttemptContext) policy.UpstreamAttemptAction { + if actx.AttemptCount > 1 { + p.tokenSource.Purge() + } + + tok, err := p.retrieveToken() + if err != nil { + slog.WarnContext(ctx, "OAuth2Generator: failed to fetch token for upstream attempt, failing open (no header mutation)", + "attempt", actx.AttemptCount, "grantType", p.grantType, "clientId", p.clientID, "error", err) + return policy.UpstreamAttemptHeaderModifications{} + } + + return policy.UpstreamAttemptHeaderModifications{ + HeadersToSet: map[string]string{ + p.headerName: buildHeaderValue(p.valuePrefix, tok.AccessToken), + }, + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd dev-policies/oauth2-generator && GOWORK=off go test ./... -run TestOnUpstreamAttemptRequestHeaders -v` +Expected: PASS. + +- [ ] **Step 5: Run the full oauth2-generator test suite to confirm no regressions** + +Run: `cd dev-policies/oauth2-generator && GOWORK=off go build ./... && GOWORK=off go test ./... -v 2>&1 | tail -80` +Expected: all PASS. + +- [ ] **Step 6: Mirror into the other repo and diff** + +```bash +diff gateway/dev-policies/oauth2-generator/oauth2_generator.go /policies/oauth2-generator/oauth2_generator.go +``` +Apply the identical `OnUpstreamAttemptRequestHeaders` addition to the `gateway-controllers/policies/oauth2-generator` checkout, then re-diff to confirm both files are byte-identical again (aside from any pre-existing, already-tracked divergence this repo's convention allows — confirm none exists for this specific method before committing). + +- [ ] **Step 7: Commit (both repos)** + +```bash +# api-platform (dev-policies mirror) +git add dev-policies/oauth2-generator/oauth2_generator.go dev-policies/oauth2-generator/oauth2_generator_test.go +git commit -m "feat(oauth2-generator): implement UpstreamAttemptPolicy for retry-time token refresh" + +# gateway-controllers (source of truth) — separate repo, separate commit +git -C add policies/oauth2-generator/oauth2_generator.go policies/oauth2-generator/oauth2_generator_test.go +git -C commit -m "feat(oauth2-generator): implement UpstreamAttemptPolicy for retry-time token refresh" +``` + +--- + +## Phase 5: End-to-end verification + +### Task 10: E2E test proving attempt 2 gets a distinguishable, fresh token + +**Files:** +- Modify: `gateway/dev-policies/oauth2-generator/e2e/mocks/mock-oauth2-idp/main.go` (if it doesn't already issue a distinguishable token per call — confirm via reading its token-issuing handler first; the existing E.14 test already relies on distinguishable per-call tokens (`mock-token-N-issued-...`), per this session's own earlier investigation of that test, so this may already be satisfied with zero changes needed) +- Modify: `gateway/dev-policies/oauth2-generator/e2e/postman/oauth2.postman_collection.json` — add a new folder/requests +- Modify: `gateway/dev-policies/oauth2-generator/e2e/run-e2e.sh` — register the new test API, add the new folder to the appropriate newman run, clean it up +- Modify: `gateway/configs/config.toml` (or the e2e-specific one — confirm which) to set `upstream_extproc_port`/socket consistently with Task 4/8's new port if running in `tcp` mode for this e2e suite (check the existing e2e config's `mode` setting first — if it's `uds`, no port config is needed at all, only confirming the new socket path doesn't collide, which it can't since it's a distinct hardcoded path) + +**Interfaces:** +- Consumes: everything from Tasks 1-9 — this task is pure verification, it adds no new production code. + +- [ ] **Step 1: Confirm (or add) per-call distinguishable tokens in the mock IdP** + +```bash +grep -n "mock-token" dev-policies/oauth2-generator/e2e/mocks/mock-oauth2-idp/main.go +``` +If tokens are already generated as `mock-token--issued-` (per this session's earlier direct observation of live E.14 test output using exactly this format), no change is needed — skip to Step 2. If not, add a package-level atomic counter incremented per `/oauth2/token` call and interpolate it into the issued `access_token`, matching the existing format string used elsewhere in this file. + +- [ ] **Step 2: Register a new test API with `resilience.retry` + a way to force exactly one 401** + +Add to the Postman collection (mirroring the shape of the existing `oauth2-test-rediscache` registration this session already read in full): a new `oauth2-test-retry-refresh` LlmProvider with: + +```yaml +apiVersion: gateway.api-platform.wso2.com/v1 +kind: LlmProvider +metadata: + name: oauth2-test-retry-refresh +spec: + displayName: OAuth2 Test (upstream-attempt retry refresh) + version: v1.0 + template: openai + context: /oauth2-test-retry-refresh/latest + resilience: + retry: + statusCodes: [401] + numRetries: 1 + upstream: + url: http://host.docker.internal:9602 + auth: + type: oauth2 + policyParams: + tokenEndpoint: http://host.docker.internal:9601/oauth2/token + clientId: test-client + clientSecret: test-secret + tokenRequestParams: + testId: retry-refresh + accessControl: + mode: deny_all + exceptions: + - path: /chat/completions + methods: [POST] +``` + +Add a request in a new folder ("E.34 - Upstream-attempt retry refresh") that: +1. Calls the mock AI backend's debug/control endpoint (mirror whatever E.16's "upstream 401" trigger mechanism already uses — this session read `token_cache.go`'s purge-on-401 design, which implies an existing e2e mechanism to make `mock-ai-backend` return 401 exactly once; find and reuse it rather than inventing a new one, e.g. `grep -n "force.*401\|once.*401" dev-policies/oauth2-generator/e2e/mocks/mock-ai-backend/main.go`) so the backend returns 401 on the first hit only, then 200 with the injected token echoed back on the second. +2. Sends the chat-completion request once. +3. Asserts: final status is 200 (the client never sees the intermediate 401), AND the echoed `Authorization` header's token is DIFFERENT from whatever token the mock IdP issued on the FIRST call (prove attempt 2 genuinely got a fresh one, not the same rejected one resent) — this is the core proof this whole feature exists for. Check `/debug/stats` (used identically in this session's earlier E.14 investigation) to additionally assert exactly 2 IdP calls happened (one per attempt). + +- [ ] **Step 3: Wire registration/cleanup into `run-e2e.sh`** + +Add `oauth2-test-retry-refresh` to the existing `PROVIDER_NAMES` array (same array this session already read and edited once this conversation, for the rediscache-clone entry) and add the new folder to whichever existing newman run groups E.22-E.32-style additions (follow the existing pattern for where E.27-E.31 were added, per this session's own reading of that section). + +- [ ] **Step 4: Build the current code and run the new test** + +```bash +cd gateway && make build-coverage +cd dev-policies/oauth2-generator/e2e && ./run-e2e.sh +``` +Expected: the full suite passes, including the new E.34 folder's assertions. + +- [ ] **Step 5: Commit** + +```bash +git add dev-policies/oauth2-generator/e2e/ +git commit -m "test(oauth2-generator): e2e coverage proving retry attempts get a genuinely fresh token" +``` + +--- + +## Self-Review Notes (for the plan author/reviewer, not a task) + +- Every task above names exact files, exact function/type signatures already confirmed present in the current codebase by direct reading during this session (not guessed), and exact commands to run. Where a downstream detail could not be fully pinned without reading additional files beyond this session's scope (a few spots in Tasks 6, 8, and 10 explicitly say "read X first, match its existing convention" rather than inventing one) — those are flagged inline as the one remaining judgment call for that step's implementer, not left as a blank "add validation"-style placeholder. +- Type consistency check: `UpstreamAttemptContext`/`UpstreamAttemptAction`/`UpstreamAttemptHeaderModifications`/`UpstreamAttemptPolicy`/`OnUpstreamAttemptRequestHeaders` are named identically across Tasks 1, 3, and 9 — confirmed no drift between the SDK definition and its two consumers (policy-engine's dispatch loop, oauth2-generator's implementation). +- Scope check: this plan is one coherent feature across four subsystems with real ordering dependencies (SDK types must exist before policy-engine can dispatch to them; policy-engine's server must exist before gateway-controller's emitted config points anywhere meaningful; oauth2-generator's implementation is the first and only current consumer) — not independent subsystems that should have been split into separate plans. diff --git a/docs/superpowers/specs/2026-08-12-upstream-attempt-retry-refresh-design.md b/docs/superpowers/specs/2026-08-12-upstream-attempt-retry-refresh-design.md new file mode 100644 index 0000000000..a380187239 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-upstream-attempt-retry-refresh-design.md @@ -0,0 +1,189 @@ +# Generic Upstream-Attempt Retry Refresh — Design + +## Status + +Supersedes the narrower, oauth2-generator-hardcoded approach explored in +`docs/superpowers/plans/2026-08-11-oauth2-upstream-retry-refresh.md` and its +`.superpowers/sdd/2026-08-11-oauth2-upstream-retry-refresh/` task history. +That prior effort is left as-is (its commits exist in git history but are not +reachable from current `HEAD` — the branch was externally hard-reset mid-task +per its own progress ledger). This design does not reuse its code or resume +its task sequence; it does reuse two verified technical facts it discovered +(noted inline below where relevant). + +## Problem + +An auth policy (oauth2-generator today; api-key-auth/jwt-auth/opaque-token-auth/ +backend-jwt potentially later) attaches a cached credential to the upstream +request. If the upstream rejects it (401), the policy can purge its cache +today, but the *current* request still fails — only the *next* request +benefits from the purge. Requirement: replay the current request with a +freshly-fetched credential, transparently to the client. + +## Why not native Envoy retry alone + +Confirmed by reading this repo's actual kernel code +(`gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go`): `ext_proc` +as deployed today is a *downstream* HTTP filter — one reactive gRPC stream per +client request, invoked once per phase. Envoy's native +`RouteAction.RetryPolicy` retries below this filter (inside Router), resending +whatever headers the single downstream-filter pass already produced. A +downstream `ext_proc` is never re-invoked per retry attempt, so it cannot +attach a new credential to a retried attempt on its own. + +## Mechanism + +Envoy's *upstream* HTTP filter chain — configured per-cluster via +`Cluster.TypedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"].HttpFilters` +(type confirmed present in this repo's vendored +`go-control-plane v1.37.0`) — is invoked fresh for every upstream dial +attempt, including Envoy-native retries. Envoy sets `x-envoy-attempt-count` on +each such attempt (`1` on the first, incrementing per retry) — a fact +verified during the prior effort's investigation and reused here as-is. A +policy's upstream-phase hook checks `AttemptCount > 1` and unconditionally +forces a fresh credential fetch — no response-phase hook needed on the +upstream side at all, since "was the previous attempt rejected" is implied by +"this is attempt 2+". + +## Config surface + +Extend the existing `resilience` block (`gateway-controller/pkg/config/api_validator.go`'s +`validateResilience`, currently `timeout`/`idleTimeout` only): + +```yaml +resilience: + retry: + statusCodes: [401] # required, non-empty + numRetries: 1 # default 1 +``` + +Same API-level/operation-level precedence as the existing timeout fields. +Generates a native `RouteAction.RetryPolicy{RetriableStatusCodes, RetryOn: +"retriable-status-codes", NumRetries}` — this part needs no new mechanism. + +## Cluster attachment and the shared-cluster hazard + +Clusters in this codebase are deduped purely by upstream `host+scheme` +(`gateway-controller/pkg/xds/translator.go`'s `sanitizeClusterName`, +`resolveUpstreamCluster`) — confirmed by direct reading, independent of the +prior effort's own Task 0 finding, which reached the same conclusion. Two +unrelated APIs sharing a backend host:port share one Envoy cluster. The new +upstream `ext_proc` filter is attached to a cluster if **any** route feeding +into it has `resilience.retry` configured — OR'd across every API sharing +that cluster. APIs sharing the cluster without retry configured are +unaffected: the chain executor (see below) simply never invokes the +upstream-phase hook for their routes, even though the filter is technically +present on the shared cluster. + +This upstream filter's `GrpcService` target is a second, small internal Envoy +cluster pointed at policy-engine's new second port, mirroring the existing +`createPolicyEngineCluster()` pattern used for the main downstream `ext_proc` +service (same internal-cluster conventions: `STRICT_DNS`, +`Http2ProtocolOptions`, no TLS on loopback). This new internal cluster is +created once, unconditionally, independent of whether any data-plane cluster +currently needs the filter attached. + +## SDK: generic `UpstreamAttemptPolicy` interface + +`sdk/core/policy/v1alpha2`: + +```go +// UpstreamAttemptContext is passed to UpstreamAttemptPolicy.OnUpstreamAttemptRequestHeaders. +// Unlike every other context in this package, it is not scoped to one client +// request — it fires once per individual upstream dial attempt (including +// Envoy-native retries), because it runs in Envoy's per-cluster upstream HTTP +// filter chain rather than the per-route listener chain every other phase uses. +type UpstreamAttemptContext struct { + *SharedContext + + // AttemptCount is Envoy's x-envoy-attempt-count for this dial, starting at + // 1. A missing/unparseable header is treated as 1 (fail toward "behave + // like the first attempt", never toward unconditional refresh). + AttemptCount int + + Headers *Headers // this attempt's outgoing request headers, mutable via the returned action +} + +// UpstreamAttemptAction is the sealed return type. Deliberately one variant: +// this phase runs after routing/auth are already resolved and mid-retry-loop +// inside Envoy's router filter — there is no sensible "reject this request" +// here, only "optionally change headers for this one attempt." +type UpstreamAttemptAction interface { isUpstreamAttemptAction() } + +type UpstreamAttemptHeaderModifications struct { + HeadersToSet map[string]string // empty/nil is a valid no-op (e.g. AttemptCount == 1) +} + +type UpstreamAttemptPolicy interface { + OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction +} +``` + +Any policy can implement this later; oauth2-generator is the first consumer. + +## Policy discovery: type assertion, not name-hardcoding + +The chain executor discovers eligibility via `policy.(UpstreamAttemptPolicy)` +type assertion — never a hardcoded policy-name string. This is the one +concrete fix relative to the prior effort's own retrospective note (it +hardcoded `"oauth2-generator"` in three separate places — validator, +translator, registry — and its own final ledger entry flagged this as the +thing to generalize "if/when a second consumer appears"). Here, a second +consumer needs zero additional wiring in gateway-controller or policy-engine: +implement the interface, and existing per-route `resilience.retry` config is +the only opt-in signal required. + +## policy-engine: second `ext_proc` gRPC endpoint, same process + +A new, small `ExternalProcessorServer` — request-headers phase only, no body +phases, no analytics/tracing duplication — listening on a second +socket/port, alongside the existing downstream one +(`internal/kernel/extproc.go`). It resolves route → policy chain via the +*same* in-memory `registry.PolicyChain` the downstream server already builds +and caches per route (same lookup mechanism, e.g. the existing +`ExtProcRequestAttributeRouteName` request attribute is available to either +attachment point since both use the same underlying `ExternalProcessor` proto +message). Reusing the same chain/policy instances — not a parallel +registry — is what makes a policy's cache/state (e.g. oauth2-generator's +token cache) naturally shared between the two entry points with zero +duplication. + +Per request: resolve chain → for each policy, if it implements +`UpstreamAttemptPolicy`, invoke it with the parsed `x-envoy-attempt-count`. +Everything else in the chain (rate limiting, analytics, transforms) doesn't +implement this interface, so this endpoint is inherently a no-op for them. + +Hardening (per `go-network-service-hardening.md`): explicit +`MaxRecvMsgSize`/`MaxSendMsgSize`/`MaxConcurrentStreams` on this new gRPC +server, sized off its headers-only message shape, not copied from the +existing (body-carrying) downstream server's larger ceiling. + +## Failure mode + +Any failure in the upstream-phase hook (credential fetch fails, lookup miss) +fails open to "no header mutation" — never blocks or fails the retry itself. +This feature only ever makes a retry *more likely* to succeed; it must never +become a new way for the retry to fail harder. + +## oauth2-generator changes + +Implements `UpstreamAttemptPolicy`: `OnUpstreamAttemptRequestHeaders` purges +the cache when `AttemptCount > 1` (implying the previous attempt was +rejected) then attaches the resulting (fresh) token — reusing the existing +token-source/cache code as-is, no duplicated fetch logic. The existing +downstream `OnRequestHeaders`/`OnResponseHeaders` purge-on-401 behavior is +unchanged for routes without `resilience.retry` configured; for routes that +do have it configured, the credential-attach/purge responsibility lives +solely in the upstream-phase hook for that route, so exactly one place does +it (no redundant double-fetch). + +`dev-policies/oauth2-generator` (local) and `gateway-controllers/policies/oauth2-generator` +(source of truth) must be kept in sync throughout, per this repo's established +dual-repo convention. + +## Non-goals (deferred, YAGNI) + +- Backoff/jitter config, per-try timeouts on the native retry policy. +- Any policy besides oauth2-generator actually implementing the new + interface (mechanism is generic; no second consumer exists yet). +- Body-phase upstream mutation (no current consumer needs it). diff --git a/gateway/build-manifest.yaml b/gateway/build-manifest.yaml index 9d9edb0942..813ea6e402 100644 --- a/gateway/build-manifest.yaml +++ b/gateway/build-manifest.yaml @@ -1,8 +1,8 @@ version: v1 policies: - name: advanced-ratelimit - version: v1.1.2 - gomodule: github.com/wso2/gateway-controllers/policies/advanced-ratelimit@v1 + version: v1.1.0 + filePath: ./dev-policies/advanced-ratelimit - name: analytics-header-filter version: v1.0.1 gomodule: github.com/wso2/gateway-controllers/policies/analytics-header-filter@v1 @@ -13,7 +13,7 @@ policies: version: v0.10.0 gomodule: github.com/wso2/gateway-controllers/policies/aws-authentication@v0 - name: aws-bedrock-guardrail - version: v1.0.2 + version: v1.1.0 gomodule: github.com/wso2/gateway-controllers/policies/aws-bedrock-guardrail@v1 - name: azure-content-safety-content-moderation version: v1.0.2 @@ -90,6 +90,9 @@ policies: - name: nvidia-nemoguard-content-safety version: v0.9.0 pipPackage: git+https://github.com/wso2/gateway-controllers.git@policies/nvidia-nemoguard-content-safety/v0.9.0#subdirectory=policies/nvidia-nemoguard-content-safety + - name: oauth2-generator + version: v0.1.0 + filePath: ./dev-policies/oauth2-generator - name: opaque-token-auth version: v1.0.1 gomodule: github.com/wso2/gateway-controllers/policies/opaque-token-auth@v1 diff --git a/gateway/build.yaml b/gateway/build.yaml index 84e87bdf80..7567985d03 100644 --- a/gateway/build.yaml +++ b/gateway/build.yaml @@ -3,7 +3,7 @@ gateway: version: 1.2.0-SNAPSHOT policies: - name: advanced-ratelimit - gomodule: github.com/wso2/gateway-controllers/policies/advanced-ratelimit@v1 + filePath: ./dev-policies/advanced-ratelimit - name: analytics-header-filter gomodule: github.com/wso2/gateway-controllers/policies/analytics-header-filter@v1 - name: api-key-auth @@ -62,6 +62,8 @@ policies: gomodule: github.com/wso2/gateway-controllers/policies/model-weighted-round-robin@v1 - name: nvidia-nemoguard-content-safety pipPackage: github.com/wso2/gateway-controllers/policies/nvidia-nemoguard-content-safety@v0 + - name: oauth2-generator + filePath: ./dev-policies/oauth2-generator - name: opaque-token-auth gomodule: github.com/wso2/gateway-controllers/policies/opaque-token-auth@v1 - name: openai-to-anthropic-transformer diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 2596104c0c..b8573b4103 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -310,6 +310,7 @@ max_decompressed_bytes = 10485760 [policy_engine.server] extproc_port = 9001 +upstream_extproc_port = 9004 [policy_engine.admin] enabled = true diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index e81a042d7a..2b7e06a173 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -8,6 +8,38 @@ application_id = '{{ env "APIP_GW_ANALYTICS_PUBLISHERS_MOESIF_APPLICATION_ID" "< [router] gateway_host = "*" +# Gateway-wide shared Redis client (sdk/core/utils/redisclient.Shared()) — +# deliberately a top-level section, not nested under [policy_configurations], +# since it's shared infrastructure any policy can fall back to, not a +# per-policy setting. Backs oauth2-generator's cacheStrategy: redis when a +# provider doesn't set its own systemParameters.redis.host override (see +# dev-policies/oauth2-generator/policy-definition.yaml and e2e/TESTING.md +# Part C / E.10 / E.14 / E.29 / E.30). Only takes effect when the stack is +# started with `docker compose --profile redis up -d ... redis` +# (docker-compose.yaml) — harmless if that service isn't running and no +# policy on this gateway ever resolves a Redis client. +[redis] +host = "redis" + +# advanced-ratelimit's systemParameters — operator/gateway-wide, resolved via +# ${config.policy_configurations.ratelimit_v1.*} in +# dev-policies/advanced-ratelimit/policy-definition.yaml, uniform across +# every advanced-ratelimit instance (not something a policy attachment's +# params can override per-API). backend defaults to "memory" (no Redis +# involved at all) so this section is a no-op until explicitly opted into. +# redis.host has no default here either — unset means "use the gateway-wide +# [redis] section above", never a silent policy-level connection. Overridden +# per-phase by dev-policies/advanced-ratelimit/e2e/run-e2e.sh via +# gateway/api-platform.env to prove override precedence against the +# redis-override-test service (docker-compose.yaml). +[policy_configurations.ratelimit_v1] +backend = '{{ env "APIP_GW_RATELIMIT_V1_BACKEND" "memory" }}' + +[policy_configurations.ratelimit_v1.redis] +host = '{{ env "APIP_GW_RATELIMIT_V1_REDIS_HOST" "" }}' +port = '{{ env "APIP_GW_RATELIMIT_V1_REDIS_PORT" "6379" }}' +failure_mode = '{{ env "APIP_GW_RATELIMIT_V1_REDIS_FAILURE_MODE" "open" }}' + [router.access_logs] enabled = true diff --git a/gateway/docker-compose.yaml b/gateway/docker-compose.yaml index f9d117bff0..334c28327b 100644 --- a/gateway/docker-compose.yaml +++ b/gateway/docker-compose.yaml @@ -26,7 +26,7 @@ services: image: ghcr.io/wso2/api-platform/gateway-controller:1.2.0-SNAPSHOT mem_limit: 60m mem_reservation: 60m - cpus: 0.025 + cpus: 0.1 command: ["-config", "/etc/gateway-controller/config.toml"] ports: - "9090:9090" # REST API @@ -69,9 +69,49 @@ services: volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro - ./configs/llm-pricing/model_prices.json:/etc/policy-engine/llm-pricing/model_prices.json:ro + # Read-only mount of the oauth2-generator e2e mock IdP's own directory — + # its TLS CA cert (mock-idp-ca.crt, regenerated per e2e run by + # run-e2e.sh's generate_tls_idp_cert) becomes readable at + # /etc/gateway/certs/mock-idp-ca.crt for the tlsCaCertPath e2e flow + # (dev-policies/oauth2-generator/e2e/TESTING.md, E.27). Not needed for + # any other flow/policy. + - ./dev-policies/oauth2-generator/e2e/mocks/mock-oauth2-idp:/etc/gateway/certs:ro networks: - gateway-network + # Shared Redis cache, backing sdk/core/utils/redisclient's gateway-level + # "redis" config section (configs/config.toml's top-level [redis] table) — + # used by any policy that resolves via redisclient.Resolve/Shared() with no + # policy-level override of its own, e.g. oauth2-generator's cacheStrategy: + # redis (see dev-policies/oauth2-generator/e2e/TESTING.md Part C / E.10 / + # E.14 / E.29 / E.30). Starts with a plain `docker compose up` — no profile + # needed. + redis: + image: redis:7-alpine + ports: + - "6379:6379" + networks: + - gateway-network + + # TEST-ONLY second Redis instance, distinct from "redis" above — used to + # prove advanced-ratelimit's per-policy connection override + # (systemParameters.redis.host, config.policy_configurations.ratelimit_v1.redis.*) + # actually gets dialed instead of the gateway-wide shared client, rather + # than silently falling back to it. See + # dev-policies/advanced-ratelimit/e2e/run-e2e.sh (its "Redis-precedence" + # suite), which flips APIP_GW_RATELIMIT_V1_REDIS_HOST between "redis" (no + # override) and "redis-override-test" (override) across phases and checks + # via redis-cli which instance actually received the rate-limit key. + # Still opt-in (unlike "redis" above) since it's only needed for that one + # precedence test: `docker compose --profile redis up -d ... redis-override-test`. + redis-override-test: + image: redis:7-alpine + ports: + - "6380:6379" + networks: + - gateway-network + profiles: ["redis"] + sample-backend: image: ghcr.io/wso2/api-platform/sample-service:latest ports: diff --git a/gateway/gateway-controller/api/management-openapi.yaml b/gateway/gateway-controller/api/management-openapi.yaml index 454eba734d..fe78bff117 100644 --- a/gateway/gateway-controller/api/management-openapi.yaml +++ b/gateway/gateway-controller/api/management-openapi.yaml @@ -3071,10 +3071,10 @@ components: Resilience: type: object description: > - Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. - Can be set at the API level (applies to all routes) and/or the operation level - (applies to that operation's route). When set at both levels, the operation-level - value takes precedence. When unset, the gateway's global route timeout defaults apply. + Backend/route timeout and retry configuration. Maps to Envoy RouteAction + timeouts and RetryPolicy. Can be set at the API level (applies to all + routes) and/or the operation level (applies to that operation's route). + When set at both levels, the operation-level value takes precedence. properties: timeout: type: string @@ -3086,6 +3086,34 @@ components: description: Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. pattern: '^\d+(\.\d+)?(ms|s|m|h)$' example: 0s + retry: + $ref: "#/components/schemas/Retry" + + Retry: + type: object + description: > + Native Envoy retry on the listed response status codes. When set, + any policy on this route implementing the upstream-attempt refresh + mechanism (see UpstreamAttemptPolicy in the policy SDK) gets a + chance to attach fresh per-attempt state (e.g. a refreshed + credential) before each retried attempt goes out. + required: + - statusCodes + properties: + statusCodes: + type: array + items: + type: integer + minimum: 400 + maximum: 599 + minItems: 1 + description: Response status codes that trigger a retry. + example: [401] + numRetries: + type: integer + minimum: 1 + default: 1 + description: Maximum number of retry attempts. Upstream: type: object @@ -4352,19 +4380,74 @@ components: properties: type: type: string - enum: [ api-key, other, none ] + enum: [ api-key, oauth2, other, none ] + description: > + "api-key" attaches the built-in set-headers policy by + default (overridable via policyName) and accepts either the + generic policyParams bucket or its own deprecated header/value + fields below. "oauth2" attaches the built-in oauth2-generator + policy by default (overridable via policyName) and always + requires policyParams - there is no typed-field fallback for + it. "other" attaches any policy by name - policyName and + policyParams are both required in that case, since there is + no built-in default or typed-field fallback for a + non-built-in auth scheme. "none": no upstream authentication - + the gateway attaches no auth policy of its own; auth (if any) + is handled entirely by user-attached policies elsewhere. + policyName: + type: string + description: > + Name of the policy that implements this upstream auth. + Optional for "api-key"/"oauth2" (defaults to the built-in + policy for that type - api-key -> set-headers, oauth2 -> + oauth2-generator); set it to point at your own fork or a + newer major version's replacement instead. Required when + type is "other". + policyVersion: + type: string + pattern: '^v\d+$' + description: > + Major version of policyName to attach (e.g. "v1"), same + format and resolution rules as Policy.version. Optional - + defaults to the highest version available in the gateway + image when omitted. If set, it must match a version + actually loaded in this gateway build, or config validation + fails. + policyParams: + type: object + additionalProperties: true + description: > + Parameters passed verbatim to policyName (or the built-in + default for type). Required when type is "oauth2" or + "other" - oauth2 has no typed fields at all, only this + bucket (e.g. {tokenEndpoint: ..., clientId: ..., + clientSecret: ...} for the token-endpoint path, or + {bearerToken: ...} for a directly-supplied credential). + For "api-key", optional: replaces the deprecated header/value + fields below when set; do not set both at once. header: type: string + deprecated: true + description: > + Deprecated: use policyParams (e.g. {request: {headers: + [{name: ..., value: ...}]}} - the set-headers policy's own + param shape) instead. HTTP header to set on outbound + requests. Applies when type is api-key. Still honored when + policyParams is omitted, for backward compatibility. value: type: string + deprecated: true writeOnly: true description: > - Upstream credential. Write-only: accepted on create/update and - never returned by the management API on a read, for any role. - Supply either a literal value or a secret reference (e.g. a - `secret` template expression); either way the field is omitted - from management API response bodies. An update that omits it - inherits the stored value; set `type: none` to remove auth. + Deprecated: use policyParams instead. Upstream credential. + Applies when type is api-key. Still honored when policyParams + is omitted, for backward compatibility. Write-only: accepted + on create/update and never returned by the management API on + a read, for any role. Supply either a literal value or a + secret reference (e.g. a `secret` template expression); + either way the field is omitted from management API response + bodies. An update that omits it inherits the stored value; + set `type: none` to remove auth. LLMUpstreamAuth: type: object @@ -4373,16 +4456,69 @@ components: properties: type: type: string - enum: [ api-key, other, none ] + enum: [ api-key, oauth2, other, none ] + description: > + "api-key" attaches the built-in set-headers policy by default + (overridable via policyName) and accepts either the generic + policyParams bucket or its own deprecated header/value fields + below. "oauth2" attaches the built-in oauth2-generator policy + by default (overridable via policyName) and always requires + policyParams - there is no typed-field fallback for it. "other" + attaches any policy by name - policyName and policyParams are + both required in that case, since there is no built-in default + or typed-field fallback for a non-built-in auth scheme. "none": + no upstream authentication - the gateway attaches no auth policy + of its own; auth (if any) is handled entirely by user-attached + policies elsewhere. + policyName: + type: string + description: > + Name of the policy that implements this upstream auth. Optional + for "api-key"/"oauth2" (defaults to the built-in policy for that + type - api-key -> set-headers, oauth2 -> oauth2-generator); set + it to point at your own fork or a newer major version's + replacement instead. Required when type is "other". + policyVersion: + type: string + pattern: '^v\d+$' + description: > + Major version of policyName to attach (e.g. "v1"), same format + and resolution rules as Policy.version. Optional - defaults to + the highest version available in the gateway image when + omitted. If set, it must match a version actually loaded in + this gateway build, or config validation fails. + policyParams: + type: object + additionalProperties: true + description: > + Parameters passed verbatim to policyName (or the built-in + default for type). Required when type is "oauth2" or "other" - + oauth2 has no typed fields at all, only this bucket (e.g. + {tokenEndpoint: ..., clientId: ..., clientSecret: ...} for the + token-endpoint path, or {bearerToken: ...} for a + directly-supplied credential). For "api-key", optional: + replaces the deprecated header/value fields below when set; do + not set both at once. header: type: string + deprecated: true + description: > + Deprecated: use policyParams (e.g. {request: {headers: [{name: + ..., value: ...}]}} - the set-headers policy's own param shape) + instead. HTTP header to set on outbound requests. Applies when + type is api-key. Still honored when policyParams is omitted, + for backward compatibility. value: type: string + deprecated: true writeOnly: true description: > - Upstream credential. Write-only: accepted on create/update and never - returned by the management API on a read, for any role. An update that - omits it inherits the stored value; set `type: none` to remove auth. + Deprecated: use policyParams instead. Upstream credential. + Applies when type is api-key. Still honored when policyParams is + omitted, for backward compatibility. Write-only: accepted on + create/update and never returned by the management API on a + read, for any role. An update that omits it inherits the stored + value; set `type: none` to remove auth. LLMProxyProvider: type: object diff --git a/gateway/gateway-controller/pkg/api/management/generated.go b/gateway/gateway-controller/pkg/api/management/generated.go index 5ab19469d7..7c2feb0515 100644 --- a/gateway/gateway-controller/pkg/api/management/generated.go +++ b/gateway/gateway-controller/pkg/api/management/generated.go @@ -107,6 +107,7 @@ const ( const ( LLMProviderConfigDataUpstreamAuthTypeApiKey LLMProviderConfigDataUpstreamAuthType = "api-key" LLMProviderConfigDataUpstreamAuthTypeNone LLMProviderConfigDataUpstreamAuthType = "none" + LLMProviderConfigDataUpstreamAuthTypeOauth2 LLMProviderConfigDataUpstreamAuthType = "oauth2" LLMProviderConfigDataUpstreamAuthTypeOther LLMProviderConfigDataUpstreamAuthType = "other" ) @@ -186,6 +187,7 @@ const ( const ( LLMUpstreamAuthTypeApiKey LLMUpstreamAuthType = "api-key" LLMUpstreamAuthTypeNone LLMUpstreamAuthType = "none" + LLMUpstreamAuthTypeOauth2 LLMUpstreamAuthType = "oauth2" LLMUpstreamAuthTypeOther LLMUpstreamAuthType = "other" ) @@ -199,6 +201,7 @@ const ( const ( MCPProxyConfigDataUpstreamAuthTypeApiKey MCPProxyConfigDataUpstreamAuthType = "api-key" MCPProxyConfigDataUpstreamAuthTypeNone MCPProxyConfigDataUpstreamAuthType = "none" + MCPProxyConfigDataUpstreamAuthTypeOauth2 MCPProxyConfigDataUpstreamAuthType = "oauth2" MCPProxyConfigDataUpstreamAuthTypeOther MCPProxyConfigDataUpstreamAuthType = "other" ) @@ -403,6 +406,7 @@ const ( const ( UpstreamAuthAuthTypeApiKey UpstreamAuthAuthType = "api-key" UpstreamAuthAuthTypeNone UpstreamAuthAuthType = "none" + UpstreamAuthAuthTypeOauth2 UpstreamAuthAuthType = "oauth2" UpstreamAuthAuthTypeOther UpstreamAuthAuthType = "other" ) @@ -454,7 +458,7 @@ type APIConfigData struct { // Policies List of API-level policies applied to all operations unless overridden Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` - // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + // Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` // SubscriptionPlans List of subscription plan names available for this API @@ -743,7 +747,7 @@ type LLMProviderConfigData struct { // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` - // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + // Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` // Template Template name to use for this LLM Provider @@ -763,7 +767,7 @@ type LLMProviderConfigData struct { // LLMProviderConfigDataDeploymentState Desired deployment state - 'deployed' (default) or 'undeployed'. When set to 'undeployed', the LLM Provider is removed from router traffic but configuration and policies are preserved for potential redeployment. type LLMProviderConfigDataDeploymentState string -// LLMProviderConfigDataUpstreamAuthType defines model for LLMProviderConfigData.Upstream.Auth.Type. +// LLMProviderConfigDataUpstreamAuthType "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. type LLMProviderConfigDataUpstreamAuthType string // LLMProviderConfigDataUpstreamHostRewrite Controls how the Host header is handled when routing to the upstream. `auto` delegates host rewriting to Envoy, which rewrites the Host header using the upstream cluster host. `manual` disables automatic rewriting and expects explicit configuration. @@ -778,10 +782,24 @@ type LLMProviderConfigDataUpstream1 = interface{} // LLMProviderConfigData_Upstream defines model for LLMProviderConfigData.Upstream. type LLMProviderConfigData_Upstream struct { Auth *struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + // Header Deprecated: use policyParams (e.g. {request: {headers: [{name: ..., value: ...}]}} - the set-headers policy's own param shape) instead. HTTP header to set on outbound requests. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set + Header *string `json:"header,omitempty" yaml:"header,omitempty"` - // Value Upstream credential. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // PolicyName Name of the policy that implements this upstream auth. Optional for "api-key"/"oauth2" (defaults to the built-in policy for that type - api-key -> set-headers, oauth2 -> oauth2-generator); set it to point at your own fork or a newer major version's replacement instead. Required when type is "other". + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + + // PolicyParams Parameters passed verbatim to policyName (or the built-in default for type). Required when type is "oauth2" or "other" - oauth2 has no typed fields at all, only this bucket (e.g. {tokenEndpoint: ..., clientId: ..., clientSecret: ...} for the token-endpoint path, or {bearerToken: ...} for a directly-supplied credential). For "api-key", optional: replaces the deprecated header/value fields below when set; do not set both at once. + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + + // PolicyVersion Major version of policyName to attach (e.g. "v1"), same format and resolution rules as Policy.version. Optional - defaults to the highest version available in the gateway image when omitted. If set, it must match a version actually loaded in this gateway build, or config validation fails. + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + + // Type "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. + Type LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + + // Value Deprecated: use policyParams instead. Upstream credential. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Value *string `json:"value,omitempty" yaml:"value,omitempty"` } `json:"auth,omitempty" yaml:"auth,omitempty"` @@ -954,7 +972,7 @@ type LLMProxyConfigData struct { Policies *[]LLMPolicy `json:"policies,omitempty" yaml:"policies,omitempty"` Provider LLMProxyProvider `json:"provider" yaml:"provider"` - // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + // Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` // Version Semantic version of the LLM proxy @@ -1026,14 +1044,28 @@ type LLMProxyTransformer struct { // LLMUpstreamAuth defines model for LLMUpstreamAuth. type LLMUpstreamAuth struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type LLMUpstreamAuthType `json:"type" yaml:"type"` + // Header Deprecated: use policyParams (e.g. {request: {headers: [{name: ..., value: ...}]}} - the set-headers policy's own param shape) instead. HTTP header to set on outbound requests. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + + // PolicyName Name of the policy that implements this upstream auth. Optional for "api-key"/"oauth2" (defaults to the built-in policy for that type - api-key -> set-headers, oauth2 -> oauth2-generator); set it to point at your own fork or a newer major version's replacement instead. Required when type is "other". + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` - // Value Upstream credential. Write-only: accepted on create/update and never returned by the management API on a read, for any role. An update that omits it inherits the stored value; set `type: none` to remove auth. + // PolicyParams Parameters passed verbatim to policyName (or the built-in default for type). Required when type is "oauth2" or "other" - oauth2 has no typed fields at all, only this bucket (e.g. {tokenEndpoint: ..., clientId: ..., clientSecret: ...} for the token-endpoint path, or {bearerToken: ...} for a directly-supplied credential). For "api-key", optional: replaces the deprecated header/value fields below when set; do not set both at once. + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + + // PolicyVersion Major version of policyName to attach (e.g. "v1"), same format and resolution rules as Policy.version. Optional - defaults to the highest version available in the gateway image when omitted. If set, it must match a version actually loaded in this gateway build, or config validation fails. + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + + // Type "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. + Type LLMUpstreamAuthType `json:"type" yaml:"type"` + + // Value Deprecated: use policyParams instead. Upstream credential. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. Write-only: accepted on create/update and never returned by the management API on a read, for any role. An update that omits it inherits the stored value; set `type: none` to remove auth. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Value *string `json:"value,omitempty" yaml:"value,omitempty"` } -// LLMUpstreamAuthType defines model for LLMUpstreamAuth.Type. +// LLMUpstreamAuthType "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. type LLMUpstreamAuthType string // MCPPrompt defines model for MCPPrompt. @@ -1078,7 +1110,7 @@ type MCPProxyConfigData struct { Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` Prompts *[]MCPPrompt `json:"prompts,omitempty" yaml:"prompts,omitempty"` - // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + // Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` Resources *[]MCPResource `json:"resources,omitempty" yaml:"resources,omitempty"` @@ -1102,7 +1134,7 @@ type MCPProxyConfigData struct { // MCPProxyConfigDataDeploymentState Desired deployment state - 'deployed' (default) or 'undeployed'. When set to 'undeployed', the MCP Proxy is removed from router traffic but configuration and policies are preserved for potential redeployment. type MCPProxyConfigDataDeploymentState string -// MCPProxyConfigDataUpstreamAuthType defines model for MCPProxyConfigData.Upstream.Auth.Type. +// MCPProxyConfigDataUpstreamAuthType "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. type MCPProxyConfigDataUpstreamAuthType string // MCPProxyConfigDataUpstreamHostRewrite Controls how the Host header is handled when routing to the upstream. `auto` delegates host rewriting to Envoy, which rewrites the Host header using the upstream cluster host. `manual` disables automatic rewriting and expects explicit configuration. @@ -1117,10 +1149,24 @@ type MCPProxyConfigDataUpstream1 = interface{} // MCPProxyConfigData_Upstream defines model for MCPProxyConfigData.Upstream. type MCPProxyConfigData_Upstream struct { Auth *struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + // Header Deprecated: use policyParams (e.g. {request: {headers: [{name: ..., value: ...}]}} - the set-headers policy's own param shape) instead. HTTP header to set on outbound requests. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + + // PolicyName Name of the policy that implements this upstream auth. Optional for "api-key"/"oauth2" (defaults to the built-in policy for that type - api-key -> set-headers, oauth2 -> oauth2-generator); set it to point at your own fork or a newer major version's replacement instead. Required when type is "other". + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` - // Value Upstream credential. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // PolicyParams Parameters passed verbatim to policyName (or the built-in default for type). Required when type is "oauth2" or "other" - oauth2 has no typed fields at all, only this bucket (e.g. {tokenEndpoint: ..., clientId: ..., clientSecret: ...} for the token-endpoint path, or {bearerToken: ...} for a directly-supplied credential). For "api-key", optional: replaces the deprecated header/value fields below when set; do not set both at once. + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + + // PolicyVersion Major version of policyName to attach (e.g. "v1"), same format and resolution rules as Policy.version. Optional - defaults to the highest version available in the gateway image when omitted. If set, it must match a version actually loaded in this gateway build, or config validation fails. + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + + // Type "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. + Type MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + + // Value Deprecated: use policyParams instead. Upstream credential. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Value *string `json:"value,omitempty" yaml:"value,omitempty"` } `json:"auth,omitempty" yaml:"auth,omitempty"` @@ -1237,7 +1283,7 @@ type Operation struct { // Policies List of policies applied only to this operation (overrides or adds to API-level policies) Policies *[]Policy `json:"policies,omitempty" yaml:"policies,omitempty"` - // Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. + // Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. Resilience *Resilience `json:"resilience,omitempty" yaml:"resilience,omitempty"` } @@ -1317,11 +1363,14 @@ type Policy struct { Version string `json:"version" yaml:"version"` } -// Resilience Backend/route timeout configuration. Maps to Envoy RouteAction timeouts. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. When unset, the gateway's global route timeout defaults apply. +// Resilience Backend/route timeout and retry configuration. Maps to Envoy RouteAction timeouts and RetryPolicy. Can be set at the API level (applies to all routes) and/or the operation level (applies to that operation's route). When set at both levels, the operation-level value takes precedence. type Resilience struct { // IdleTimeout Per-route stream idle timeout (overrides the listener stream idle timeout for this route). "0s" disables the timeout. IdleTimeout *string `json:"idleTimeout,omitempty" yaml:"idleTimeout,omitempty"` + // Retry Native Envoy retry on the listed response status codes. When set, any policy on this route implementing the upstream-attempt refresh mechanism (see UpstreamAttemptPolicy in the policy SDK) gets a chance to attach fresh per-attempt state (e.g. a refreshed credential) before each retried attempt goes out. + Retry *Retry `json:"retry,omitempty" yaml:"retry,omitempty"` + // Timeout Maximum time for the entire route (request to upstream response). "0s" disables the timeout. Timeout *string `json:"timeout,omitempty" yaml:"timeout,omitempty"` } @@ -1384,6 +1433,15 @@ type RestAPIRequestApiVersion string // RestAPIRequestKind API type type RestAPIRequestKind string +// Retry Native Envoy retry on the listed response status codes. When set, any policy on this route implementing the upstream-attempt refresh mechanism (see UpstreamAttemptPolicy in the policy SDK) gets a chance to attach fresh per-attempt state (e.g. a refreshed credential) before each retried attempt goes out. +type Retry struct { + // NumRetries Maximum number of retry attempts. + NumRetries *int `json:"numRetries,omitempty" yaml:"numRetries,omitempty"` + + // StatusCodes Response status codes that trigger a retry. + StatusCodes []int `json:"statusCodes" yaml:"statusCodes"` +} + // RouteException defines model for RouteException. type RouteException struct { // Methods HTTP methods @@ -1658,15 +1716,29 @@ type Upstream1 = interface{} // UpstreamAuth defines model for UpstreamAuth. type UpstreamAuth struct { Auth *struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type UpstreamAuthAuthType `json:"type" yaml:"type"` + // Header Deprecated: use policyParams (e.g. {request: {headers: [{name: ..., value: ...}]}} - the set-headers policy's own param shape) instead. HTTP header to set on outbound requests. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + + // PolicyName Name of the policy that implements this upstream auth. Optional for "api-key"/"oauth2" (defaults to the built-in policy for that type - api-key -> set-headers, oauth2 -> oauth2-generator); set it to point at your own fork or a newer major version's replacement instead. Required when type is "other". + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + + // PolicyParams Parameters passed verbatim to policyName (or the built-in default for type). Required when type is "oauth2" or "other" - oauth2 has no typed fields at all, only this bucket (e.g. {tokenEndpoint: ..., clientId: ..., clientSecret: ...} for the token-endpoint path, or {bearerToken: ...} for a directly-supplied credential). For "api-key", optional: replaces the deprecated header/value fields below when set; do not set both at once. + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + + // PolicyVersion Major version of policyName to attach (e.g. "v1"), same format and resolution rules as Policy.version. Optional - defaults to the highest version available in the gateway image when omitted. If set, it must match a version actually loaded in this gateway build, or config validation fails. + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + + // Type "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. + Type UpstreamAuthAuthType `json:"type" yaml:"type"` - // Value Upstream credential. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // Value Deprecated: use policyParams instead. Upstream credential. Applies when type is api-key. Still honored when policyParams is omitted, for backward compatibility. Write-only: accepted on create/update and never returned by the management API on a read, for any role. Supply either a literal value or a secret reference (e.g. a `secret` template expression); either way the field is omitted from management API response bodies. An update that omits it inherits the stored value; set `type: none` to remove auth. + // Deprecated: this property has been marked as deprecated upstream, but no `x-deprecated-reason` was set Value *string `json:"value,omitempty" yaml:"value,omitempty"` } `json:"auth,omitempty" yaml:"auth,omitempty"` } -// UpstreamAuthAuthType defines model for UpstreamAuth.Auth.Type. +// UpstreamAuthAuthType "api-key" attaches the built-in set-headers policy by default (overridable via policyName) and accepts either the generic policyParams bucket or its own deprecated header/value fields below. "oauth2" attaches the built-in oauth2-generator policy by default (overridable via policyName) and always requires policyParams - there is no typed-field fallback for it. "other" attaches any policy by name - policyName and policyParams are both required in that case, since there is no built-in default or typed-field fallback for a non-built-in auth scheme. "none": no upstream authentication - the gateway attaches no auth policy of its own; auth (if any) is handled entirely by user-attached policies elsewhere. type UpstreamAuthAuthType string // UpstreamDefinition Reusable upstream configuration with optional timeout and load balancing settings @@ -4561,261 +4633,278 @@ func HandlerWithOptions(si ServerInterface, options StdHTTPServerOptions) http.H // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3bbNr4/+ioY/bpX7FaUZTtJG2fNmuPYbqpJnHh8afeZyruBSMhCQ4EsADpWM97r", - "PMR5wvMkZ+FKkAQpypavdf9oEpEEvgC+d3zwxddOmEzThCDCWWfra4eFEzSF8q/bB4OdhIzx2S7kUPyQ", - "0iRFlGMkH4cJ4eiCi79GiIUUpxwnpLPVeQMZAinkEzBOKIBxDLYPBoAmGUcMrEwzxgHjkHLwBfMJWOsC", - "kgBOIY4xOQMshmyy2gMnDIFvzhFlOCGAJwBNRygCfIKA+RET+U/Z0QrqnfW6YI0iGGFyFsSY8TX7OUUs", - "ic8RE+0UXzlf7/VXe51uB13AaRqjzlbH30an25nCi/eInPFJZ2uj3+92ppiYf693OynkHFEx/P8ZDtdW", - "foXBn9vBv/vBq9+Gw2A4XDv99lfx4HT1H990uh0+S0VfjFNMzjqX3U6E0jiZTRHhRxxypCZ1DLOYd7b0", - "QxR1uqWZ3kUMUxSB/GsxsxyBADwzHz0DK7qlVZBQ8Cwj9kkP/DJBBDDExcy4T7pyasWyYQYomibnKAJj", - "mkzVMlKxXuMxDsEo4yCUTJJRKKjqyq8+oxnrAkgikCYxDjFiAFIEUooYorKthII04YhwDGNAUT4CuRok", - "m3a2fnUHnhPXOXWXy3mlOqmYpTGcfYBTVOXSn7IpJIFYbDiK1VgJnCLNoCMETg7fB2OKEYniGQhAQuIZ", - "iJFYZdYFJJuO5F9YCkPEumAySyeIsC4QhFIWJhTpGYgSzoQUJF9QtFpgtUPFaeA9ZlwQUGSy9UYmyxls", - "OAx+Gw574PQ7L2cJkZUrw6pzIDtOxuCn4+MDkL+4pmS10+1gjqbyu28oGne2Ov9nLdcWa1pVrH00H4ru", - "ppgM1EfrlhhIKZyJh4YZ6inZPhgEMTpHscM4aRpjIfuJ1CU5mSAjMWIMJOeIUhxFiLSl+EC0LSkqU0gR", - "wzFGJETz2jjM37zsdlg2ssM5iGHTZLuvgjSGRPIdA/Ac4ljyohAOPsFM84RlmF87b5NYcPoRjs8RFYJg", - "h1tZ9/LIspRxiuC0Slg+5+adokh3uiXNP4WYzJueE9OdmBxIolFy0f4TuRB/ZEK3iVHL/k7tkJLR7yjk", - "7ph20RgTPIfJKcqYnF47yij/TNmiRH4DY8DxFCVl1dZaIE4qZPkWxFiWCsFHaAoJx6G1dMnYqOOC+hDG", - "q1NQCufDYfTdcNgTf3iVwfkkYdwzRzsZ48kUnGPKMxgD+dZalIiJZ5odTf9+Vpjb3Apb1Q2usFWl/mkS", - "ZaGUAm1NeuAjQcJITROK5FdSMoaEoRRSyFEERjPw7PUz8P/9P/8vQDCc2JeAtCtM0ik6yRdZugbgizB0", - "ELyFHH2BMzGUIRFa71BoOgA5h+FEOQjTLOY4jREQ9h8RRHNCVnvgeILAGFPGASKczoR5lE4IxVNIZ0Mi", - "J7gH9gq0TeFMGBQIvuA4CiGNAMvCCYAMfNvTy9kLk2lvSArrC1PsPn4dJSEr/FD4usgJK8Pht8Nhb/Uf", - "uZ3oDYfB6XcrwyH79rX4X+0rq996eceR4rmrrZdarrP+zixyYYj6WVAaaqfW1CkKPfS1Uxmlt1wHIRfI", - "rnVtHa1ZMKQ+XbR9MHiHZtXZ2UUc4pgJIYbEOEfuJHwVCz2IOlsd1/MUUxJoCYcplk2Lv6S/rW9sPn/x", - "8vsfXvXhKIzQeNF/i/FRJKRpWziXG/2Nl0H/edBfP17vb232t/r9f+evvJHdRlMspqXgT3X2Z+AgF+F3", - "elAppoiJhkkWx90OUe9OZ0Eu7oGaAJZkVJjZTpyEMBY/cMgzJvoLOT6XZrWobPQ8lWf4hOA/MgTSbBTj", - "EOBIOJVjjKijNwGfQC7/8RlJoYWMJSGWKkVo/gJT1i1DRSLMupQJeivUhmxbL7eyLnL1hA88xhdlQV/K", - "slYIdNa5TOMxniLG4TRVqtHMkyQWMnBmhlAgtIZXxgmdQhmoQI4CYTsbiHnjmbBBZc0yhij4MklyQlwS", - "i7OnufNa7r/U046hkxOxIqgQjHuOIxR1wTTj4uWiE+8Tg2YvvkKoIzVlMvfEI6iMpF2xFSFbAI9F4Izs", - "C6vlpfo+6K+LpeqLdWpaKtGcGFhni9MMeQkUuhjGh2jsE8A9/RhQNEZUuMRgsFuezQJ1YZxkkZCtqVAG", - "wasfvn/5wreExLt2IjJjcIxcWa+sHcx4EuTcI4NXhyO6AE/1enYFt0XCHMtcgnA1pogjWpxQnwpz1vnl", - "ZmGZNysWrB+8Ov1uJbB/rbOyWitWnEL5u6vS5Cil7hQuk1miVSd8NorVPCtGzuZplQSthyskyN9LJDjd", - "abUtTOx58lmrjlTa2kLH9r1mE06UVVZK31LlKjVXp7hSZGex3k7viA9xQg7RHxliUvAcg1xrtXwmyWsC", - "PppIIo0hJoHwJuyincM4U8rGLIyySkSQiBPSG5LBGORqR4aCyorEsXAkJbtiwjiCkVgOzeWYnAEICPoC", - "EoJ6Q3KszZ35bALZRLjQaCzca8YTCs+QcmnFayEk4i1MACQzoBTFkKxMMcHTbAo2X4JwAikMOaJMJ+gk", - "ZWIgmnZyZocUz3LVPSQmJ1R2cS/kf8EXlmxIS5vGkIuepVbQD9UfwmK68vXy+nq0BwZjMEr4BOgPB0Qm", - "bGwzOmdl1iH/ncPPiAlLHqJIqLte1UqubwT9H65gJS0pjWOIdEjqUbJF/jQvevxS04TLjqYDdzybfUsm", - "JhydISpDb4JrvAogHnna01qCoTAhEVPLqdNMkySj4s8IzsQfXxD6LF9ICJ+wUr5PvdKsOiRx3XzwPj2w", - "DJsmhUyIAEZxJNxKm0AQfCTFVH5BYShkI81omjDEZC5RC+iZjkiNsDCAOQPJFwLEZEsKTL8Uhp8xOSvL", - "UFtbihnLEG1wvnQom1AOY+Uwa/VqNZCUmFwgZEoUpliKNija2iERjTHhVukWjRqCYYhSjiLZGEl4QdMh", - "isQ8ksR8RZEYgdGLZbc5VxgROldf+IY+hewzirZrdPW+fOrJtki1KKZe+w12AXtDcqCJBqOZmjZNiPxO", - "utS5TkwpCrTy9SlB6f5/++23317M/vz+h1ft/aCBN9Qx61ScWgj0LoDrNJkl8Xv7t+LxXLYw0SxNCEMl", - "G51b3qfwuS58niLG4BlSOV7JzbmQsiwMEWPjLI5n0mebQkwwOVNS8q8s4bCz9cppVn/Q5AM1JUV1fsSl", - "ylnP+QRWZMJPcVlGDs1bVqD/EC9aTS5CPJfrX/mMXe4RO6krPR3zbJH1W82w653S95hxl9t90yz/2ioL", - "nU94OfO80HC6HZ5wGO8kGfEZfPFM74bp/Rup4woORHVK66X+EBlvtsY5r7Dfgl7fk6v2wFy1Jl45T8KK", - "jShtUDQpGx2ozlU1tyT/J6ngN4frYRx/HHe2fm0j6OWI9vK0SIfW0qeX3c6OmJ4xDiFHzSonzF9sr3ec", - "1m3LS1JCb2bct3mslNBIPJRp9jgGDuVgjGNUUEgbG+svXnkV/SKqrrGLljrPN1ceoI2Xng8+SpiBxQiK", - "XILWfcPF9dl0x0tcOTkZ7K5a/eX0VtClL1700Q/P+/0AbbwaBc/Xo+cB/H79ZfD8+cuXL148f97v9/uL", - "xCXO3AD1Dtj9AFYEGWoHThAC8BiMMhKVs7I7H/6+PwM7292P4s+P9AwS/KcCqOz8/eTIGyTkmqKU91Jc", - "CWSeQ5kGFeSZLwodO1RnaZxAESOIaPBo9whkUsDn6xu/uy8cR+Po1y3CdBaEcjsuCKG35YRvj/m86UaO", - "+RL/bjnpypquBxsvQf/lVv/7rY2XrY2pow6M9bHKAFGa0KJtadAULFPi1ThC/dJNctQceT+RzOEo+1rV", - "Wx3Jwd5+gEiYCN76796L/iuXH1bYag/sQALChHCISb6j7eqJYsoqEP+92Xs7+AB29g6PBz8OdraP9+Sv", - "Q7I/GOz+9/HOzvbnX862vwzebJ8N/rn97n3/5O1308N3/Pf97f7bnaM/3h4NRpu7/9p7s/PlZHt/7+Ri", - "58/tf745+/DzkPR6vSGRre192PX0sEDqX2mnwnaNM6we2NforUy9CEOaMFY2CaXRl4TmChis3m+tdqWL", - "UitH6PMG9gS/19sDKQ6sbqcZRcJNxJESX/1uS+DKz/ZDSYLPbNdqyZ/w2UTDiGSnwH1cECQXU+PSOpbU", - "t/W/lFJYive1d8EplLF1nlGpTjsuPCsO/p9HHz8cQJVJpoipPBIFEwQjRBW38sTYVJUw4slnpD36wvR8", - "08sEoT1M0owfi5e8Wi7Wnm+Vll9kEo0nYIxJ5HTl2C7Hx0/hTOgh4dlLYjvdzh8ZorMDSKHGYUzU3wv6", - "N/+sef4tmV13/nyL8P79/rbU6TsJ4TSJPXx/EaK0BuSlJ9+8IIYvRg6V5Q5Vk2CaRKitLEhk0J5p0SsK", - "orUqms7bpd0ji+Pky28wjiWWl8zkX0uAVv3rXIiLaLlmJjXAsTKFRqk6u4DxNAgTxoMRZCgKKOQoxlMZ", - "k1V4TvBC+zjAkiHWZg4AzgW1td0YzOE6iq7GqZA0eIJDPkmi4pDMSr3dO+50Owcfj+QfJ+L/u3vv9473", - "xD+3j3d+6nQ7Hw+OBx8/CNv/0972bqfb+dahoh6KKXeYVU4nirByJg8cwtQufFXDgCM5tVqzjjA50wh4", - "vWHNbG5dZaUxUyjaWQ/IbQrMGYrHEv4CCu0lYWag15UpTPXMOQj5cAK5XPEYGVxk84rJNrp2uu0M1C2Z", - "ylrTptMHsKwr5rBiUbdcdovHFwzSfq0CsV/CYYbi8YIkRQTiv+R5gvfv94FZ24UPFjyo0wSFkWp9lffy", - "y9HHDfAxRWR7YN+6Eez/WZyMYHxQi7p/K5+DFZhi5bqtVmH32oPefv/ehd5DBhKCAJtAwS8sTFLUBUh4", - "MwqmqzAG9oMSpr93faC+bbp+dB9relfkygMFLEWhcMilhLM1raDckUARLQM1kYvT/7FApXcgxTMRKUWh", - "3IjzGoHdvYPDPRE37YIAZMyZYDMLPXDEcRyDSUKSTCzNCtd7uMr9CiUygyfVL1dbDyr3L5Z4gIKjaRp7", - "g91j/cS60WLg9oiEK2kFIbN6tiIV7kmIdhlW5zBDuxe3M+HynN7+EYUeODR4BekEmIZ6FI17d3x+oXap", - "rnqQodr1zw4GXfGLBWEI+4LJWQ8cZWmaUM6EaSMRpBHQYHWJ8e8Clo30yYeuMHAWs69/1CmGcSI8eXD4", - "404gPSEMCc8R/zSLhSz+or9V9krBJdRBMJOmjdGYB1NBbQxHKDYHGQvI/lXfwQDF3hos77oSLzYbLIfG", - "/P8ntyCnK//YKtiT06/97sv1S+eN1X8Mh73V7/Qvp183upfzUx110Hor5wVsfdGba+UWOrtl7YS4rgW7", - "YdIt+5h52qFdD4dIbcoroKTcgSlLBj1HNJhCAs9QBGI8RuEsjJECELEeOEjSLJbqWh1blRkgaW6Ea/GR", - "xDNlGDzJxdPykYKfjXx2NMao5wJmel9YsiHYZ01GXJ8xiYQiiqeuQ4I4jLT3rZEIEqmneM8Ao9UOeYpC", - "r1vuBu2/OhHXryq0OjUBhieqEAvivC8CMud1Ef7G7V5a+yr/HESXcppU3J4H2m4wYPzzNbEKjFdQG1Wv", - "LTdcuckpWJhMxU86vbLVEbYhoTp3nMuRWBwNMc1o3NnqTDhP2dbaWlHaxXK5ylcpz0KOzAdP2Xh+3P9+", - "a2N9a33z352u9XOb3sFR3Xqrzkr+st7bqG/x8rJBjP043Ccufqhc3O2ozOZW5w2CFFHAPgezJKPB9djc", - "hz36uc4PsQGY8fJ1xtnaIhMYzuWsQozYgg8r7opizFoC5eOcHpd/C10XGduzg5lzepOd2jfvOSy/kOWU", - "KZmyxXeWQg/YoUh3NMeyHztBwMJG3Xz8ZM+9mvA4d7w8GlErQ6sGHM7IlZnejSjthdgdi/zF37jZt8i3", - "KeyWwaVfGyk81DTlc3pRL83rwYIDa1q7yDPdgX038DWqNZ5mdsT4vtDCHvKkdm4gSC3+1b6WuJQ5EyPf", - "aZ6XRd0E6QKUWeMKpt7wXl0hmSqDNYmld7vuCgm6QmKhEGBZjmwOrKr5Nppkqe80wRFXhR3gFMezQL6G", - "yZkLstGZtNEMoHNEZ8XYGbMhMfMv4k+9n6/f0ZkBi7fXVMgUaoTHMh3Ah2QCSRTr1CnL6BiG6tCbbSUZ", - "y5xe3tGuyvMywJMhMUqjJwNciXBPpphzFJXDU1+G+3nfC8uXetN3IvQjxWfYZg5ykt5kOOaBCJ71T0ym", - "g54J7ffsNVDb+PlkMYt65wl4pp4i+kzmkvWpfJ2thkQfzCqPRrRc5oQXvs2xkvK6Cgd7tNbVmikqqqu1", - "oWzfPkwFq7IFfITcEJea8KnBq9BW0oZXaaI2eWWVgnSmCbeCqOpIqIJRHhHUnCoEcEiMBIYShYMuMOOv", - "DSfKvWnRTKMM6ZSYw3Wb/UVSLi0drZsKu56cjSdnY7FgzcrdfQ3WLIH1wZrl+rqgzRGLuwjeCm7YDYZv", - "JcV/cx7fI7W5vlNM6ompUCAT+vkm2FRPdKmOoQ43O3P91vthlUsMaWfjalzHqmxnWlwMwzSHuSs7Y5e1", - "5F7Mtl28j8rjVAFk9h0ZpZj8pClSFSlsG2biycVMePAQMBSjkBe2DnvA7OwqaIUEIHIRYMgTzVSihxRK", - "7hNkn3TxQ9dJ+YSjT6s9YEsZwIxP9HYjwEztrJnTr5IU6dCEMI5VJYKUJhyFHEXAUYGyWp/8xp5vjpMk", - "HcHws6JTeUIlw+HbM03OcKjnqAC1sITZ/X6e6Amy8ybfroCFRRhlqoqKARUiIDkdjT4bJHxCkxSHgbOz", - "dUVQRw2gw6Rh5/BscRt6zjkPeWQGmEw+iOMpSH27tPnw0oYcJKeQMGFlFWvPF66L2bHzSVkJ4KhB/C9m", - "jQixiqyxhjodsd6Ch37pYw3i5xE+5kifChwgMYgfOUMx5AldlQGCkk5tP5iJRVU8wZBgZIY4N2A/iwwQ", - "rK6Kd+jic+CTIfaTLkgAR8m5aFmV8RNfm2jYtgI1RPjHd4BDeoa4YuoFlKNXqXngAk94u7vC213MHj/Y", - "TgnjbRfwzffRLmaLwDCeAHxPAL77CuBLHce0jfZ3df5VwX9XgpKlWuqecGR/RRxZ6myQz3EPr4gUK33+", - "tK1czvQqm1eb3lXyWcjtluApgRHhOnSKfJjr119Pi+qpHqF0iwip0lCuiYyqYbol5ucf1qotCvi5mN1n", - "tM/FzJ89vpj5UsYXs9vPExdC6uWmiB1XoRqr32Feowbi2GyW5uQljotZkHIyVwq1TdA6GQHrtOuizE62", - "ypSaV9kGFDmJvmObgVM159SLbqsMQBHg2dSGOhYIvkwShgC6QGEmhcW+AqaQq4L4LgldwGQOkWZE1TDM", - "a0e7VBoKLWHyyTPWmGcUH6aQMZNgKdOPxbsyQ5EP3JMpvMrRymPbUeCEE/ZM5YqqjyTZRR7njbsgl4RV", - "76FJ9cPX2o7MAqjJcDswe6NJYPNtpQuKPG/Mz/DXetj78PeEBnIxeYU8u/ftUni+ru9BMJcpaJxsjGh+", - "0xLmZhkxYRzGsfCeszg2TVb3uzsN7uZ5jf9eEkr5NB9rjYAWlEhFExmAa+2NKvkJ4RzrmvCJ1BkkIch7", - "AliDYSu6yt63QlGkEio98AvFHMkV2crP8wrhlE7LmvImZGKCoHM55zyjJM/FK7dRJpO2DwbiSwiEW9hV", - "GTUyAzSJUQ9sE12PRtX6TKZCxDAHmEwQxVqJMJ5QVVEiQ69lpumTGNwWEEP9JNZY5ZXk5kDPU8mj2/ki", - "xlP0SSuL5luq/Z2DA7m/5zEX9EweWG6V0DXvyuErCFAOW7bxcqkWm9tmtc5Gfn+PjkdNJ1cr5tP0dT5V", - "njoPgu0KLahEn/7CtjZKkhhBdYgL8xg1zNqkmFqTr88n03dA/7RWLea5hsZprqOpWFdksboxnoLxag/Z", - "u5Wx2FwRZ0VVo97asVefPSUQzZsetZfy7e8c6FywfgXoQ/lOqlwiD/lEie3DSHHnw3rUKe58mIadKpjV", - "PXfxln+UfP7VbTmN/gvcrp8yVlLVfvc+tyBLPKi8OIhgf+fApHy8hRBTFNaGtGJSawNat/Dai6D/Mlj/", - "oXDZh6eIYhIvRPdxooplNF0md7NHqCvO+gSBEQw/IxJJjpMSS0FGVc11B6Ngb237a57CzsXRxzF1Vxr9", - "pfPh0zBdL11D9oAy4lYm5/sOC2fEvZ8/ZcTz3Op+mPrTqrlPFUzDNLC56Gp2teB9FXOrBdtureCvpwVr", - "JP5ZsCWOWehY3S/ecrV3fghza80hYWuz37/Vg8a+ebpGNr2RYZeSTf/LrPhCKfjc6tzXNHxOoU4XGYLE", - "ghb6VCt8awl4T3i3rAS864Euluuwwe6cqHuKp+jYm/S0LewP9vfMnLeM2oWz54bVFhLsqxGK/2zqXTwW", - "zoGsEt7x1v6+erhv6GoZ8Hc7GcWL5Cjqx12upk9xU2FZ49EvxgM/1eZfxPjHGQnVDGHu3bCShUxVpUF/", - "4dS8rOFY3dSBLlK1xZFn4ZeR6RH60HtDesYbKLTr30yqagQwTrOQZxQtOaEkaPdfRdS2XGZRgN1F8XKK", - "o+RK2p+QhOcXyvu3Wb760kcFkHveivThIR1hTiGdAZKQwBTKFTNsz53KG+dUsBCoS1TNfUrF23SbTUVK", - "EzHGQDod/fVX0asXm+Mg2vzhZfA9fPk8gPDVRrD+w8tXcOOHjVcbqN/x4fllUHGd8b+XDcihf0azQF3s", - "kUJMVZo6UeXFJY6eRHpHTV9iw3rgHZoxIGGOJOG2zrdCMpZmA5FzTBMi87ZbnfwWIVnzQzgEHR1Nd4qW", - "3zvsRolT54t9Omvhq3XbZkQtOs9zioLkCDkgj8hwid5GWCbN9V4Jw3JLiyephhgqAOF3BoA8laGqfpni", - "UHz6TDb1DIziJPwMVtQX4DsFWv5O13xmqzpzad6W26iIyRy9TNNDVUBFCME5sjjsMiVrslXBJviMJBRF", - "PbDNQYwg4xK/Ka8cM4BXc9uXb2NUktEa7bgv3740JVvbR3J5C+rDaij30/HxgR4cWNHzL0bx2oxQbSo7", - "88YQX3Ur0ZZ20yV8Xk5TMTXzVVqPS5DGMESTJJYo/gV6LKTGR0nyma19xdFlp4wV7317xYRpBamrdmH1", - "8YKce1eSc0QpjpC8ggJGkdw13z4YlGCxq9fPsF4tKXrZJJo/SXnYN+znr0ddYhGnbPpKCBkKMGGIMCxE", - "pbgwhSLQ1aT23/7PN/81zPr9jZfPvv1uOAx6//Pbp//8b02KO9+0N7saexcw5JUtDU2eZJdyEGG+OERn", - "WQzpni0Hv8i2sO5AGQWeqJ6KGADS+v5c1Uej9rSL48WpqO6F8Qkp5ohiqDeScxbtgb0LLhZIuC1SCmUN", - "eeW/sS4Ik+QzRqwLEA97FdWkNWbtPCjVTRnY/rCr72vXda/5RK+CIGiPnCczfZpGG8yELIzzdtnVe/2B", - "0YcLacFcebUDm0M+0SSUi76rBnV7zatqSa1VwA7jtil3roucm6rnhWBZfV/hcM+QKkqgrdwd2PWWZ300", - "NJxZjyPXmJIJPFIpWjhQ9+8XiDfP2wroQjbneoaktP4tpLmu9r+Fee0YlJf33jpzfYW+wECeq3KhYzla", - "zOx1q/4aArR8+LJgwPKvGiiNffELBxbHGi1wCYGPulZXEdTK7RZ4u3fcBUJau+Dg5LgLlKx2gRTVLtAi", - "2gVCZKUP+605T7egzD9dcbD8Kw7uTELdQEza9p6Jrn8V4Yg6X8RRdAr+9ncgluhqeCZPf2Hiz+FchU+2", - "ba7AYQsL51HIxZUxRSiQ0dFnNFtTrpRNzqz6uKB2J/Xn4tkjw28fhbc+zeGTBiyJ7WFBs+t43u8q1OSP", - "WRxbw1WsLNSVNYF6/VV1/TXP+VyEhuaiZop+d6C1TQBMhQT8PaG2G43HZJicxSi3oy4q0wFr6n1Ucz05", - "nsIz5AVtXlt1+iTksBCHlE8Iy3zFmq5q4NuA74F9mMooSTmF0l5vh/Yi2CTjTN2RpguAQW6vZlUx1YqK", - "ydRx7DjWB5BXxWKsVdyN6icKUWleeKZLMKw6aCbIwSjhE/Ut6xZb1IGdDgDgZySTByGKxIzoRjLCEO+6", - "i/SMmWOOxamxWGtB4MyXHMBRjI7V255zC4gG2qtWaAjxtm3cCU7lvjtmHBFEve/aeiVmNoadPht2QIQl", - "3kLD1tXLRcxzn5W9pei7FX2ub/UfK1P2H/af6X8mq/64rm5k+/ACT7Op7NIqEKEEKdJTuKL1pLx2wKBB", - "zPbyIgNYf3H1EVz6BcTdMfccnKzZMHduAVMegZuoK2EJ8+1d3+XB8jbl/ASA3QP5Apm5TlGfXF05Od7x", - "3BRZ3Qlud1Wku6e8KGExZDw/E7KiK3mol3MQ3xKJbXfFKmQMnzkYbo1qWkF/ZDCWKQC3+uLqVXKqdjP9", - "a1uEJkVpQnmZqOUBIJ2d/Csto7lNdansVSNsfPtgsBCeRXzwBJDJ4RJySlLsh0z4WdgPmnDfXfsmj7+K", - "+IlD/dZ70aJYO6fggFvD3WYuTK11GZ879diFAVQBUsMbniZUiF9s56TNWzYC8714WjhYaeePIR6YLJrr", - "U9Mc8GKTbPlXF4E8+gZTnAZ6IYN8Pk39duWWqstsqHMnqbdBd7cpbyIS3kySyl8vTy8vyztNJYDKFGJS", - "BKro+vCsN8K/Ywp7ETpfY5Ij2VqFd4SawiFas+iV24Iw1SniK4OYSmpkKbClJzl8ksN7IocLActEaHZf", - "IWWCttI+kBGzQo+57N0aqGz7YNAWT+YAyTS0rBZPVrobtymZWZvDLFxK3T4j2S756NsoPnBKZxZz8tdN", - "9vmm6AiFFPGmo1qLnjFkssUC5QcJ42cUHf3rPZBIe7F8I1VAjbEvCY3KR4E2nl/zIJIi4tYLbe2agR14", - "B7akals1uz1qKXU2ZkWfiEUkpLOUlwllWbpJ2WZIN/nf3IijfkH6c85uN6P/a7eDXP4TxneZPNgFeOyG", - "qZiEcRbJU99P7HlT7LlgsXd3/W8C/n5ktJHHjTTrHNh1dqxVSSm3YJGiR+mba+PgFKRvMf9CC/l9dTE0", - "eTYJUiojo1ej0LldoVtzNio2b1n4dS8zKxd4RwZwJzKe8rDXx6PjtYOTY7CmNAOzqY8e+CS660nW+WQ2", - "XUwthdeAIQTqZUjVEigUZDCZ4lESYcRKWyWPQczmxM3rQf/F8Xp/a9OcPpUxcZVGX/Bb+nae5C4ijLXy", - "VRWdO5ETa5sL0zv/a5sRVFGWSQxeQeBsvwtK3iHiFKNzX2mKt3u5xMmI2Yqd9hUwOQMR0h5UQRIfoeDU", - "2acneboxu3OPZUkI/ICj6V27YdfT9v4MaDvurKQ6n/y0u/PT/PbntnalPur9V0xUvSaZEJI3s51DOnvt", - "xJw6/BZ+GnJizghMEEX+bazleZ5ikg6dnGu55k5GfHuYCYexji9F/KztoWvdXvjOIZr3as8N6Bd64MeE", - "in9kFPOZQoLkhlRfAYCZua1CuqzqbkExy7ZQgrybjGpbDqDBB+lZH80AlhX5kpE8Y6QuETCGW12Y1xZj", - "XdJ/vlIolgHdjIq8jLndTm2TPq/M50Bhqwq1POSmM+uBD4lCBEl0VJHPVQ1AsEIS8Elu7XwCCR2ST/k+", - "0adVH8imAKco71VXrP3V0QVHcIoAZEXIAFgzK6qOaRXSFz613bxbvxTy25XUPMpGdnQq2HPyGBW7MahJ", - "zztYixUH6DDYBQnVU1JM6YSvxhujlxAF6xubz4MXL7//IXgFR2EQoXFf/CR+8d7QkqaxNkteWvLHBZpk", - "yapddH6QUA7jtaPjI/fmHYlNyqHTgDlz4jsB2u2MsMSF7ugbL32kvMEaOqrfKdBjhMIUDYTxTGLtOYXh", - "Z0zOVpt6dZesqWd3GEvonTlybk4SbO8cD37ecyyw/WHwwf71cO/nj+/2dr0+q0vjQQy943HHC9IYEnBy", - "MthV1XEgFzp2irnUNSNs4boOWrEzp195qZbv3DD8I0PFWZRcInuWXE/O9cV8CskmRO21qekIGZhANpH5", - "0HISe6RuJwzgKFzf2LyY/TlXepXs+eieJ9QtjavHULpS0PqsgNu17bbVJV5HJVaYo430Wos3iypz5+P+", - "/t7hzmD7vW/h0UWK6ewYl49OSEW7vhFsrh9vbG69eLX14lV7OyGY8kPlNMbbJI6WKEgFr9Y+9rSepB/J", - "v7KEw0MEzcEz3Y/Ce9tm1D89ZSwnNOE8Ru+FZO0YFrGfrff7fW+JB/ezE4K5G7juY2Gzf0oy2ul2duGs", - "0+3sJ0SdssrHpZ/P2R80033ago2Wwv+ioavJgPjyenJQT3xJBCqsUHCJ2nFyUTzafaPDO6W6a3yoRpFp", - "kJBGcWjF+225uyU7NztuV4VAltdcJdzb6r6lrOJDXZA2+mXBFaiXOOsCz3dMl+wz3pw/6Gv5CprjSlqg", - "DV/dlAO5dLdwxZYKl3vgtqb4a3kV1YFOfAUSzpTkOQFV/B0zXl4jtjo3UFyGvpmja667RL7uTxwYXE3J", - "dlOEtFhUeEWnT/RVBSICMJVAxWQlBOm8WrFqU9w5vex+LV1/O+6cXp5WDssnwluQNdWLDhrMeFI5Mq1P", - "hjEwSb7IfMZPCeO6RAnATEe++vyDruRpDorl90t8Em1/AhGKkRAipsqAUkmF/kCes+qCLxMcTvQTfRzG", - "7TFjlWsswzhjHFHZZA98mkKSwfhTfqJGdD2FHIdOfyKSUoWXmPgzxiEuHwAbuslgPTWqba+QSl+pWv9A", - "r5w8BAZSimTZJ+fqDac0q7fIV+wB1WCKQm655+TwvZQ1dWBL16qW1OYupy7Vl9IkCvR3Wy/6/f4aTPHa", - "+YYbBKj6XwswuP8SBPgXvxrhKEvTeGZqBUEQY44oNAfy5KkpZhL9hkf0Na/gk3ryCXBzYTyyR3ZXX5s2", - "v0BFUTVLS5NpmU67eaqxBff86oYmfnMkxiNv5brLRd3qr7wsVjtOYARGMIYkVKUh5H25rJKCHUGGDrzg", - "0PzaWVVazN4+i0iUJpiI+dUX8ljq9LldLYarPbAdx4V7fIuvyzO8E3iO9LF13VmKSIQiXf3Yudr22doz", - "OTZb5guRyD55Lddc119OSmcNc73goMzWCjCz3m//+7dvdGGcldVvv+u+/vvW//Vf8pLbtdNvrl9szx13", - "5CpJp9jyzF6THawv/aJs58xnmxrb5vCrUyq8YQPGKG9l5Mslwb8gfDbR140UGbP+vhGvqXjj2IgVaXLV", - "lQWUS/+tq1goTKaIKbVh2Ht1nvkI1qUBmWs5uh01GJ+sxqpwmXrBM1hz7+w0izlOXanW09YDh+59C+OM", - "ZxSp1wPtPhVbfK3OxetiWTPEwYqqmCX1KGXceF2YgTCjFBEez2Sh8uINSj/0JbfhqbBVhtfUvzxZo0qF", - "zdib1ZliMlBru+7JoXiOv+d8dtqgL2tPZR/7zr3LiXTOKStVVN2JSggR/VQa3VEPnGPyILKerdJ2w84L", - "NuzIP/v9KRt2isy25GPOP8MYR7L/PUoTz8Vx0n5WB/KjNKvSOI4hjpUZ1C0VU90pCnvmVJN3C54xeDYf", - "eIwEecC87fawo69bqVxdLqU5lOU0c92+1mZe1Cax3PaV5U2sccOh8eekgpNZDPFr3qhQBupwGSbjxBzp", - "gooZNG7kl6OPG9LvMBEhOFYXN5R1wN7RsXxPcJ10WXSFytJtCGarudquLmChnQ9VHbXjqWqxX/CHCpUq", - "9amorroWLcWdrc5mr9/b7Dg1hNZCwTASLaKm6gx5VZrZA49jneAAx++PgPuxo1eEbsrrZDgvKcerNyTH", - "8v7+wueQOjcnnCOqa5z+dHx8cFRwe7QYahypPTI3iLQZ2nFHlB8Ik6Pb6PftWT2VqXJyP2u/M+V7MVvv", - "tslAOv0U8tSShfzWsTDZl12hJ5ZGjtQCTUQMiNA8MDZHE6RcKonJplNIZ4ZQZ5HD4lxyeMaEnnaG7jCg", - "0NYXgZQq4TcHImCQr8NoKhN8+pAdoiKk76Te6zROUmnZICDoS5nHwMrB3j5QdnnVxOJGUGTZF/dlzAwj", - "RjMCp/p6cKFKhPKmSCocE3SbViocpehxBtzpmjOLb5Jo1mL5HCSbQ15nqxOI/97svR18ADt7h8eDHwc7", - "28d78tch2R8Mdv/7eGdn+/MvZ9tfBm+2zwb/3H73vn/y9rvp4Tv++/52/+3O0R9vjwajzd1/7b3Z+XKy", - "vb93crHz5/Y/35x9+HlIer3ekMjW9j7senow1bClv6nWOwgV2GlR/leTZM/0F826DI8qcrh+E3LYxP4u", - "z2ap5gyNoBlncSxRNs9vVyCl5S0wrXY676NuKEhmWBCIa+kFedq2YIfWKBJdSZfGqyT2ZbpK3suJz86Q", - "KtUiqUvGSn25lkUGAMoTjhGbMQXxKqmPiuAfopLgX9uYlA+CWvfJ8YhcutWQdJmro90jW9WjwLWNm9Qt", - "sGLdDk84jN/MuK8gr0LqyZsBzNxqokqmwfa0sbH+4tUrb7BQ9tWaZNQZfllI751kWHbUTLhMq+mRDnnO", - "Xq5UjPxVa8TvABYVixGCor2cQHImTaWJHa9jK1XHRVvp3FOw9WsFZrhrTk+6pPIE6KEVwqcXffTD834/", - "QBuvRsHz9eh5AL9ffxk8f/7y5YsXz5/3VdSOiTxSKg8ha/OGo07ZHrk2rhxTnC5VzNXm2cLDaIq2vOpC", - "T9kNK4sFhdgSVbWzz29PhF2CREQ5TjIS3UtF4pPc5SiQOJ7aG9wDk36vD/hkHPD+/X5+ebn9BlB0hhlH", - "NI/wtELo2kRfPBO2Vr0zUhdy9ryxmroUXvZwbImaozR+lC3LfQuzjVB78+fHFJHtgVELsvJ2rheK58Vv", - "SyGEFUjWphfqvqANd5e0FdDJM/VtME71wa2fXe53mFtDcy5y4gUzTcDM0yLCVxfnbkfGlfbSUIlutx1u", - "l1utTCMXDPBX7YhI/D26ED/K1KRBupu69G5nVZFUsE0fZywa9LZbTE9PeRBZwOWszeA0XlLDtxqdesXM", - "I0ReJjBVIu9HmFpEc+RZY51HXlWUvbpFw56QcYxDDoJcNGWqmMGpviMKxhTBaKZAOvdTGSmha1IGy9RH", - "9c5A67iC1KisSohREyD49UujzdebqWk2inHo7qmaO8ActemJHWQCHD+A6MAS2s7/96+D1+m+Dc9/AXJu", - "Owbwk/YwogFy81qh6w8D3iJeL+6jGcCcgcFuVc7fIp9n/2Y2iK4s6KYGdd1U3EthX9wxWLLTs4iUcohj", - "9iSYLQRTiEW9TERLDh8y7y6ZhvPl6GM/QcUI3be9FcEbt8gqFXWjQvoXik369yM28eYX73ls8qTX5uzw", - "tdMqNxmPLJCTvGoqsmuKonSBhjd1gfKF5XVQ5xJvPy9duUCasjCHc1KVdjKvmbPstiTHKQ/j3F3T69d0", - "n79+/a713K9p5e+AdIvGoURCjki7Egmlqxyy4h10zl0N/t7tN3nnc698uPLaCEYskAdT3FOT0wuTad0a", - "6c/uLqG94UtoFwR80Qx1ocraDZRHaZfVfkDJ7Noc9pLhWnVp7Er2OleAOnstj10lQHAIhaFGfupgU9dH", - "7zol8SwC0J466ALnYjapzaFcb3P5W1djwlUt+hbJ7ptPcntrzi7Nm6xpvdk1wQT839v774Xhk5cCagDS", - "HaXIS3I+h3aTHpcViM3FRU+58nm5cqsLyrlyEtnb7h5y3vzaqs/jlV41OX6FnHjLyLsacpfmIDeE8g4N", - "5TcEacm/vMfJ8Bqyr5Aavx8Z8fuXCH+I+e8lSPcC2e7WSe4FktuPQXKvaM9vwtNpIXf3ILX9wDLaMpHt", - "VhlZbixxlZz2wqnshyaOf4HQ40QnjUszfCcp78WUyP1Ndz/ptStntG8sUljTVT7mZLNhHMtTn+JNH0Jv", - "rs4rJaW3DwbvRKftFJ+qcONTeoUaR4a4h++YqOlpe1jTLMyTfDX7DaZUtjtnPmZeghcRJoRl08aE5FtE", - "EM3zApqgKwlXJUGo+Gcp0nVmyBQPNYEP2tVQcyOnbGkORl2btwrgLRNRLzCG1x4iavdeqLe7SYiuRJnq", - "RAliIncm5SO17yA8iNX7nwFt0HTL1bxzPJ61rzDF75Dcom7MmB6i8+Sz9M006T3wkYQIUPl71AWYgxAS", - "QBIQJ+RMBKW6QgRP3K0fe78Y8x3iFW0tX4XfjqqubBSLOTXkmPWWrpoYZYEmC+/Oy8156MlX6p75aGLd", - "wtYKV3PMk8JtoXB1+Xsxbffbtayoh1vxKZsTU4YStVdtiqTo63cI40hXIMh4EmgPT9iQhKAW6apHqZo8", - "0M+bV0035d0Wq+0uw7ctt3ir8M/FPdt7lQQzt4o+GBX75N5eNXl3L33bNYpMFF9fqebQvlNIQl4nLZE3", - "+fj9WjvBj8OA2KVbcorE3+49NyY04U9pksfntVt9dxdK+wK3LGoiXryj4wOSxkUPD1zMQBH6f3cHBy5m", - "d3Nq4GJ2L48M3IsDA2JNHttpASPLC5wVuJjd+UEBSfVDOCag1VBJD1/MbvyEwMXMfzxAqLj2ZwNywHdZ", - "dednBornAxY4DnAxu9GzACU2XSYap7bpOv/iYnZ/jgBUxLeJ6ifw/1XB/xezR4j8lyK7NGVWcikXR/9f", - "zBaE/l/MrgtXlC2UT9gH5sHDqHxjyV0I5C8tx90i/OtIuKOo8WL20LD9y5XfVgj/i1kreP/FbBnY/vsu", - "nVexzkt3V+YJ2J3i+O+9TDkgfsXaWZknl+zvL4biV55mawj/AzGIjzpGKMH1bVh0m1j9hVTEE0r/wWmt", - "JoVx0y799WH6LZSak/mdLQGgfzGbj85/UN7Fw0LlPwgvoAUk//rCtSwwfgsRKubmrr/XrWRoLgb/oXgM", - "T9j7J+z9tZTYEzJp6cD7perXRt/l3gLul6Opb1YjXw9ifzF7wtc/KdVcqT4acP2yvcO7gdU/JgXkB9Lf", - "pAJ6QtE/oejvmyJ9clSXC6G/Iy91+dD5FkmEMm7+cbmndUj5h2ghnmDyTzD5R+18z8HIL10rT8O0HTp+", - "f+fgYOng+IRq3LR/byTvsz0qfn/noIiKr9bT31dvHbi6ePmY+JyQ28XE5/3WY+LROaIzPhFtPU5c/E0j", - "01/4kOnTMD1YEJyuOfwOwemOjN1rbHpBFxgNaMX45qDpZoXKyPSanSjz+g2hxL38shxHaE7Tt7q7UyMW", - "VRayq/N0H2pbmHcuM48I6u2I3dJ0Q8k9WgDpbbmyLdDbIf9aV6vlY7a3nfaGRccjN/2BGJzrh9xjDLif", - "6nZQcLsad4YEb6bgtuMiS83DwIHfiGw3o8DtDDWDwM1r17q9tCy5D0Ver2K+l+6ezBG2uwGFPxD5Erxe", - "YPRoyY51Swy4paEdBPxGTKVK1N+q6P3FYoP+HcYGT/eRPgZ91aA6lu31U8R4AFM8JyV6iBjfPhjcYkLU", - "9Ng+Hbp9MKhPhB4iKE/Dy9FsHwxuLhkqyLjdNKjosT4BStXIgxjLEheP8zbR5YZkRh5a5TU1o/oymS2T", - "qTeW8LQydK/TnY6kG9UmfpJsfWO5Tt1py1SnWeOb8WZ068vxXyqN3Wo20wpDlSfMjD+lL9umL8VsPaLE", - "ZS5EyxLzggPTOmlpZb9tyjIn/FphmFY3/lyla6UlVuWBZCvr6G6XrzQrcWfpykYCbjs6McQ8kGTl8uW5", - "KVVppbY5UanfulaecpxQI7APR0zbWeUleBbNYnQ3eciHITmCj10ujpbr8bZMQhoK2uUgl2v7/MnHGxaq", - "R+iw92/TYX/KKT4C3VOvCG7UH79ybYnWakp8v1hBiXlKylaV0CfiJUWPwg94IEUmHo41byoxcX3RumZt", - "iToRAse60gNmAILNjWA04whQSCJ73hCRMIlUin+CLmCEQjyFcRekFI3xBYpUWuITTHH626ceOGHICtA7", - "NFP1ZWcgIa5YaVWNACZhMhUKyBygVq3xCWbyPHZNDm6hcyrzZNxX9eKheyVPBTCeCmA8JgXbVF9iqcq1", - "wW25h2UllqoHFXl3ogUXKzoxj6yn6hNPGu3ea7SKkliqg3jb5SWWpojuncpRGY87UTlP9Sae6k3cruoU", - "E/RgTg3X6jPhI+bn/yOl2G7fRVxaTYfG4D2l6BwnGTNRvHEOIBGslcYwNCG6mpglxPgNhSQeT2C+eKGJ", - "R2UjnipOPFWceGwOd12RiaUnEBgKKeL1+xyHZlcB2owxjGPAeEIFl6mve+AQ8YwSpn9w9KTKkiYZHxKh", - "jWDIMzl2+ZrU6CrzzFCYUcxnIM1omjDE1G5rddPkSBN8g1Knumi736DnwO6/+GRv/fb464SIdU8o/hNF", - "IChfo2ZV172G1jK7xobT9aq3Z/T6vYcjwbpMuxiaEREJ6SyVN5JxIBwm5bDop4NdMM0Yl6kv6Q70hkQ8", - "1lEocz7PmHCJuHR2sBiWeSYm394IO0LjhCKQIsow44iEyMftKpGoRn5DEF7V+A0cR2pseElZeO2/qPof", - "KnMuCbT8dGTlUGXW1VkF5WIruPzP+gTDVudMO6rC+0ljyMcJnfa+sGSjFybTtfP1TrfzGROxLHZBpojD", - "CHI5F+YcBuRwBBkKUsjYl4RKOWMpCqtseJAwfkbR0b/egynEBJhPgf20WzjWsdXZNW8cuI1baKGegm3e", - "2eps9DdeBv31oP/ieL2/tdnf6vf/LRy6yEtjt6OjzPpvL+WqXWPt1eoqllbRkE9LqE/vxz7IG5gHvAGY", - "YiZFO6EAa+9mjFEcsXus4O8KAK7VZr49Oti9l6hvELjaWbmkTZs5zEj+NayS43PNRX4fIDqFYqCxqUsg", - "zJaeXYsCN/IsTBZmand8AmmkP5HLMCREhH9hco7oDExROIEEs6myctbqiG9xhKZpIlYEBKoFeRkrIAkJ", - "5NohwodE00C11/e8/9xnwBTk1jFgVX/NK/4+VDNYIQnQvLJ6r2Xu+YKmiyQ8UKFI0XjpuUgQk9GKnHzX", - "fFlkekevRjHayiOc3EiIvn7TYU97fT53do6a+78vsm4trJD0jKI6gPgyxLzbHE0xffOtVD65UBe8Tutd", - "6tdc73JIfG5lOBGOhHYuR0hhVYSEoqgHBipwMy8zOQuAJ0Oi25fKRPXdBRC86Pf1zMlMnWrGZOdkeIpD", - "oHnQJ/xvEW+U/AUkxByVqHPudOQF48fl3dnBdFiWblK2GdJN/reH5/QZpo8adEcePDuC8XBC6VvNYT0U", - "dYuaXSsns7Qcjdsmj1/JT+V5cF1HUvz1oqhqhISyVO5ODHYdsUxpEvWiUU9IeK+gE7BKrBf0lfyt2IBH", - "oVwuCanXsK3OCts3rrOu3FxJnTJF9p+FLMeQ5GmOMKNUOIsN6Y4uQASOYn2pfzKFXFgOfKY4d0h4IvpB", - "VMFQo4zmhdlZD3yMIyfFJpWpiCTgKEbgHEOda3EtoM8aqZH/NXMpi5pbbRdqza29zeIpk9LeqK5vPX9x", - "B5mUewEfmJtJUYz0ZN4fknmflzkxkIflZU2ykaVLKBbS4nCO+w2Q3wB4DnEsrUebIzpHTgMHss+b3Hcq", - "ddZ6B6oyyvu7veOh9Tr7mfXbPDZzV+kR8AnkIEJjTBADcpc1xlPMVVAOpaIEXO5djjXCyG2D1Z30KC/f", - "TfkZpW5MqZc7OeNQJqZRsVUWwuza3KFBurM8+f0+u1ARmmtKqV+Br30Vfwxa1j+pCnLbSigeySwFi56Y", - "S5F2TfT9c0+SuzIMne++dU/jw8Mo2LFsXmwo1yH3U1QxCIl08fBccx2Pu+O0/j3R6XdVS+PDvT91W8NN", - "MiN0TQ+oZQ2Nav/tqmncKlffvMdUOQJweW+lyeRinqTJH1vesJsyJ8QsvNq2sOz2waALnAmcW1L2qEDQ", - "QnVlB7tgxSlzOtgVfanLEFdryprCFEupbYSb+z+0Q7paAw0FVbd3jgc/73W6ncEH+9fDvZ8/vtvbvYmy", - "qm3l+SoB+gOJzW8qLNfTN5KGyRm0PE/cunpKNeC+hWD73gTarU3IXzm+BkHROjyksqOsyNhLtWhrX91/", - "Xin2vkrY3cplLFJ2w6H3XUXdBSLIwwvB7yr6bh943z6v9e9Wz99VzP2AWNkTgN9h7L142H0rPH2z/tOd", - "hd2tWfiuou0HJEfe0Pu6PoroQZ//k6wt393O+KSz9eupYE1FkC/efZ+EMAa6mqPsrdvJaNzZ6kw4T7fW", - "1mLxwiRhfOtV/1V/DaZ4bWpJWztf71SPT+8m4WdE195lI0SJRN3nMXS5eY12CcQK0SSOEa3t59TOUmWv", - "8vBkN4fhq21HM5EsF2/f3Fap9zVWuJpXt+a9h6fanHpoCq8cvz8CIaIcj2XVJ9X6T8fHB0cgSxmnCE7B", - "OaLqseIM3d1O/tXi9Ot71BXI6xhN01g0U4BIOCPzv329Tlv1ddUu1E3gTe3PWyVf4/lJWd2WB3hxeXr5", - "/wcAAP//nIc3NSrXAQA=", + "H4sIAAAAAAAC/+x9+3bbNrrvq2C0u1fsVpTlS9LGWbPmOLabahonHl/afSbybiASsjChSJYAbasZ73Ue", + "4jzheZKz8OFCkAQlypavVf9oEpEEPgDfHT98+Nry43ESRyTirLX9tcX8ERlj+OvOYW83job0fA9zLH5I", + "0jghKacEHvtxxMkVF38NCPNTmnAaR63t1lvMCEowH6FhnCIchmjnsIfSOOOEoZVxxjhiHKccXVI+Qmtt", + "FMWIp5iGNDpHLMRstNpBp4ygby5IymgcIR4jMh6QAPERQfpHGsE/oaMV0jnvtNFaSnBAo3MvpIyvmc9T", + "wuLwgjDRTvGVi/VOd7XTarfIFR4nIWltt9xttNqtMb56T6JzPmptb3S77daYRvrf6+1WgjknqRj+f/f7", + "ayufsPfHjvfPrvf6t37f6/fXzr79JB6crf7tm1a7xSeJ6IvxlEbnret2KyBJGE/GJOLHHHMiJ3WIs5C3", + "ttVDErTapZneI4ymJED512JmOUEeeqE/eoFWVEurKE7RiywyTzro1xGJECNczIz9pA1TK5aNMpSScXxB", + "AjRM47FcxlSs13BIfTTIOPKBSbIUC6ra8NUXMmFthKMAJXFIfUoYwilBSUoYSaGtOEVJzEnEKQ5RSvIR", + "wGpE2bi1/ckeeE5c68xeLuuV6qRSloR48gGPSZVLf8rGOPLEYuNBKMca4TFRDDog6PTovTdMKYmCcII8", + "FEfhBIVErDJroygbD+AvLME+YW00miQjErE2EoSmzI9TomYgiDkTUhBfkmC1wGpHktPQe8q4IKDIZOtT", + "mSxnsH7f+63f76Cz75ycJUQWVoZV5wA6jofop5OTQ5S/uCZltdVuUU7G8N03KRm2tlv/sZZrizWlKtY+", + "6g9Fd2Ma9eRH64YYnKZ4Ih5qZqinZOew54XkgoQW4yRJSIXsx6BLcjJRFoWEMRRfkDSlQUCiphQfiraB", + "ojKFKWE0pCTyyaw2jvI3r9stlg3McA5DPG2y7VdREuII+I4hfIFpCLwohIOPKFM8YRjmU+tdHApOP6bh", + "BUmFIJjhVta9PLIsYTwleFwlLJ9z/U5RpFvtkuYfYxrNmp5T3Z2YHBwFg/iq+SewEL9nQreJUUN/Z2ZI", + "8eBfxOf2mPbIkEZ0BpOnJGMwvWaUQf6ZtEUxfINDxOmYxGXV1lggTitkuRZEW5YKwcdkjCNOfWPp4qFW", + "xwX1IYxXq6AULvr94Lt+vyP+cCqDi1HMuGOOdjPG4zG6oCnPcIjgrbUgFhPPFDvq/t2sMLO5FbaqGlxh", + "q1L9p3GQ+SAFypp00MeICCM1jlMCX4Fk9CNGEpxiTgI0mKAXb16g//d//i8i2B+ZlxDYFQZ0ik7yRQbX", + "AF0KQ4fRO8zJJZ6IofQjofWOhKZDmHPsj6SDMM5CTpOQIGH/SUTSnJDVDjoZETSkKeOIRDydCPMITkhK", + "xzid9COY4A7aL9A2xhNhUDC6pGHg4zRALPNHCDP0bUctZ8ePx51+VFhfnFD78Zsg9lnhh8LXRU5Y6fe/", + "7fc7q3/L7USn3/fOvlvp99m3b8T/al9Z/dbJO5YUz1xttdSwzuo7vciFIapnXmmorVpTJyl00NdMZZTe", + "sh2EXCDbxrW1tGbBkLp00c5h72cyqc7OHuGYhkwIMY60c2RPwlex0L2gtd2yPU8xJZ6ScJxQaFr8Jflt", + "fWNz6+Wr73943cUDPyDDef8txpcSIU07wrnc6G688rpbXnf9ZL27vdnd7nb/mb/yFroNxlRMS8Gfah1M", + "0GEuwj+rQSU0JUw0HGVh2G5F8t3xxMvF3ZMTwOIsFWa2FcY+DsUPHPOMif58Ti/ArBaVjZqn8gyfRvT3", + "jKAkG4TURzQQTuWQktTSm4iPMId/fCEgtJix2KegUoTmLzBl3TJUJEKvS5mgd0JtQNtquaV1gdUTPvCQ", + "XpUFfSHLWiHQWucyjSd0TBjH40SqRj1PQCxm6FwPoUBoDa8M43SMIVDBnHjCdk4h5q1jwnqVNcsYSdHl", + "KM4JsUkszp7izlu5/6CnLUMHE7EiqBCMe0EDErTROOPi5aIT7xKD6V58hVBLaspk7otHWBpJs2IrQrYQ", + "HYrAmZgXVstL9b3XXRdL1RXrNG2pRHNiYK1tnmbESaDQxTg8IkOXAO6rxyglQ5IKlxj19sqzWaDOD+Ms", + "ELI1FsrAe/3D969eupYwcq6diMwYHhJb1itrhzMeezn3QPBqcUQb0bFaz7bgtkCYY8glCFdjTDhJixPq", + "UmHWOr/aLCzzZsWCdb3XZ9+teOavdVZWacWKUwi/2yoNRgm6U7hMeolWrfBZK1b9rBg566dVEpQerpAA", + "v5dIsLpTaluY2Iv4i1IdCdjaQsfmvekmPJJWWSp9Q5Wt1GydYkuRmcV6O70rPqRxdER+zwgDwbMMcq3V", + "cpkkpwn4qCOJJMQ08oQ3YRbtAoeZVDZ6YaRVigSJNI46/ag3RLnagVBQWpEwFI4ksCuNGCc4EMuhuJxG", + "5wijiFyiOCKdfnSizJ3+bITZSLjQZCjca8bjFJ8T6dKK13wcibdohHA0QVJR9KOVMY3oOBujzVfIH+EU", + "+5ykTCXogDIxEEV7dG6GFE5y1d2PdE6o7OJewX/eJYs3wNImIeaiZ9AK6qH8Q1hMW75e3V6PdlBviAYx", + "HyH1YS+ChI1pRuWs9Drkv3P8hTBhyX0SCHXXqVrJ9Q2v+8MNrKQhZeoYAhWSOpRskT/1iw6/VDdhs6Pu", + "wB7PZteQSSNOzkkKoXdEa7wKJB452lNaghE/jgIml1OlmUZxloo/AzwRf1wS8gVeiCM+YqV8n3xluuoA", + "4tr54F16YBE2DYRMiAAlYSDcSpNAEHwEYgpfpNgXspFkaRIzwiCXqAT0XEWkWlgYopyh+DJCYrKBAt1v", + "iv0vNDovy1BTW0oZy0g6xflSoWycchxKh1mpV6OBQGJygYCUKE4oiDYq2tp+JBpjwq1SLWo1hH2fJJwE", + "0FgU84KmIykR8xjF+quUiBFovVh2m3OFEZAL+YVr6GPMvpBgp0ZXH8BTR7YF1KKYeuU3mAXs9KNDRTQa", + "TOS0KULgO3Cpc52YpMRTytelBMH9//bbb7+9mvzx/Q+vm/tBPWeoo9epOLUYqV0A22nSS+L29u/F47lu", + "YKJZEkeMlGx0bnmX4XNd+DwmjOFzInO8wM25kLLM9wljwywMJ+CzjTGNaHQupeQfWcxxa/u11az6YJoP", + "NC0pqvIjNlXWes4msCITborLMnKk3zIC/bt40WhyEeLZXP/aZexyj9hKXanpmGWLjN+qh13vlL6njNvc", + "7ppm+GujLHQ+4eXM81zDabd4zHG4G2eRy+CLZ2o3TO3fgI4rOBDVKa2X+iOivdka57zCfnN6fUtX7Ym5", + "atN45SL2KzaitEExTdmoQHWmqrkn+T9NBL9ZXI/D8OOwtf2piaCXI9rrsyIdSkufXbdbu2J6htTHnExX", + "OX7+YnO9Y7VuWl6QEno74a7NY6mEBuIhpNnDEFmUoyENSUEhbWysv3ztVPTzqLqpXTTUea65cgBtnPR8", + "cFHCNCxGUGQTtO4aLq3Pplte4srpaW9v1egvq7eCLn35skt+2Op2PbLxeuBtrQdbHv5+/ZW3tfXq1cuX", + "W1vdbrc7T1xizQ2S76C9D2hFkCF34AQhiA7RIIuCclZ298NfDyZod6f9Ufz5MT3HEf1DAlR2/3p67AwS", + "ck1RyntJrkSQ55CmQQZ5+otCxxbVWRLGWMQIIho83jtGGQj4bH3jdveF46gd/bpFGE88H7bjPB87W475", + "zpDPmm5imS/x74aTLq3purfxCnVfbXe/39541diYWupAWx+jDEiaxmnRtkzRFCyT4jV1hOqlu+SoGfJ+", + "CsxhKfta1VsdyeH+gUciPxa89V+dl93XNj+ssNUO2sUR8uOIYxrlO9q2niimrDzx39v9d70PaHf/6KT3", + "Y29352Qffu1HB73e3n+d7O7ufPn1fOey93bnvPf3nZ/fd0/ffTc++pn/62Cn+273+Pd3x73B5t4/9t/u", + "Xp7uHOyfXu3+sfP3t+cffulHnU6nH0Fr+x/2HD3MkfqX2qmwXWMNq4MOFHorky9iP40ZK5uE0uhLQnMD", + "DFbnt0a70kWphRG6vIF9we/19gDEgdXtNJNAuIk0kOKr3m0IXPnFfAgkuMx2rZb8iZ6PFIwIOkX244Ig", + "2Zgam9YhUN/U/5JKYSHe1/4VTzHE1nlGpTrttPCsOPi/H3/8cIhlJjklTOaRUjQiOCCp5FYea5sqE0Y8", + "/kKUR1+Ynm86mSC0Q6Mk4yfiJaeWC5XnW6XlV0ii8RgNaRRYXVm2y/LxEzwRekh49kBsq936PSPp5BCn", + "WOEwRvLvBf2bfzZ9/g2ZbXv+XIvw/v3BDuj03TjiaRw6+P7KJ0kNyEtNvn5BDF+MHEvL7csm0TgOSFNZ", + "AGTQvm7RKQqitSqaztml2SMLw/jyNxyGgOWNJvDXEqBV/ToT4iJarplJBXCsTKFWqtYuYDj2/Jhxb4AZ", + "CbwUcxLSMcRkFZ4TvNA8DjBkiLWZAYCzQW1NNwZzuI6ka+pUAA2O4JCP4qA4JL1S7/ZPWu3W4cdj+ONU", + "/H9v//3+yb74587J7k+tduvj4Unv4wdh+3/a39lrtVvfWlTUQzFhh1nmdIKASmfy0CJM7sJXNQw6hqlV", + "mnVAo3OFgFcb1szk1mVWmjKJop10EGxTUM5IOAT4Cyq0F/uZhl5XpjBRM2ch5P0R5rDiIdG4yOkrBm20", + "zXSbGahbMpm1TqedPsBlXTGDFYu65bpdPL6gkfZrFYj9Ag4zFI8XxAmJMP1Tnid4//4A6bWd+2DBkzpN", + "UBip0ld5L78ef9xAHxMS7fTMW3eC/T8P4wEOD2tR9+/gOVrBCZWu22oVdq886J33723oPWYojghiIyz4", + "hflxQtqICG9GwnQlxsB8UML0d24P1DdN14/uY03vklw4UMAS4guHHCScrSkFZY8Ei2gZyYmcn/6PBSqd", + "AymeiUhS4sNGnNMI7O0fHu2LuGkPeShj1gTrWeigY07DEI3iKM7E0qxwtYcr3S8fkBk8rn652nhQuX+x", + "wAMUnIyT0Bnsnqgnxo0WAzdHJGxJKwiZ0bMVqbBPQjTLsFqHGZq9uJMJl+fs/o8odNCRxiuAE6Ab6qRk", + "2Hng8wu1S3XTgwzVrn+xMOiSXwwIQ9gXGp130HGWJHHKmTBtUYDTACmwOmD824hlA3XyoS0MnMHsqx9V", + "imEYC08eHf2464EnRHHEc8R/moVCFn9V30p7JeES8iCYTtOGZMi9saA2xAMS6oOMBWT/qutggGRvBZa3", + "XYmXm1Msh8L8/zu3IGcrf9su2JOzr932q/Vr643Vv/X7ndXv1C9nXzfa17NTHXXQeiPnBWx90Ztr5BZa", + "u2XNhLiuBbNh0i77mHnaoVkPR0RuykugJOzAlCUjvSCpN8YRPicBCumQ+BM/JBJAxDroME6yENS1PLYK", + "GSAwN8K1+BiFE2kYHMnFs/KRgl+0fLYUxqhjA2Y6lyzeEOyzBhHXFxoFQhGFY9shIRwHyvtWSARA6kne", + "08BouUOeEN/plttB+ycr4vokQ6szHWA4ogqxINb7IiCzXhfhb9jspbWv8GcvuIZpknF7HmjbwYD2z9fE", + "KjBeQW1UvbbccOUmp2BhMhk/qfTKdkvYhjhVueNcjsTiKIhploat7daI84Rtr60VpV0sl618pfIs5Mhc", + "8JSNrZPu99sb69vrm/9stY2fO+0dGtStt+ys5C+rvY36Fq+vp4ixG4e75OKnysXtlsxsbrfeEpySFLEv", + "3iTOUu92bO7CHv1S54eYAEx7+SrjbGyRDgxnclYhRmzAhxV3RTJmLYHwOKfH5t9C10XGduxg5pw+zU4d", + "6Pcslp/LckJKpmzxraVQA7YoUh3NsOwnVhAwt1HXHy/tuVMTnuSOl0MjKmVo1IDFGbkyU7sRpb0Qs2OR", + "v/gb1/sW+TaF2TK4dmsjiYcaJ3xGL/KlWT0YcGBNa1d5ptsz73quRpXGU8xOGD8QWthBHmjnKQTJxb/Z", + "14BLmTEx8M70eZnXTQAXoMwaNzD1mvfqCslUGWyaWDq3626QoCskFgoBluHI6YFVNd+WxlniOk1wzGVh", + "Bzym4cSD12h0boNsVCZtMEHkgqSTYuxMWT/S8y/iT7Wfr95RmQGDt1dUQAo1oENIB/B+NMJREKrUKcvS", + "IfbloTfTSjyEnF7e0Z7M8zLE436klUYHAlxAuMdjyjkJyuGpK8O91XXC8kFvuk6EfkzpOTWZg5yktxkN", + "uSeCZ/UTg3TQC6H9XrxBchs/nyxmUO88Ri/kU5K+gFyyOpWvstU4UgezyqMRLZc54aVrc6ykvG7CwQ6t", + "dbNmiorqZm1I23eAE8GqbA4fITfEpSZcavAmtJW04U2aqE1eGaUAznTEjSDKOhKyYJRDBBWnCgHsR1oC", + "fUDhkCvK+BvNibA3LZqZKkMqJWZx3WZ3npRLQ0frrsKupbOxdDbmC9aM3D3WYM0QWB+sGa6vC9ossXiI", + "4K3ght1h+FZS/Hfn8T1Tm+s6xSSf6AoFkNDPN8HGaqJLdQxVuNma6bc+DqtcYkgzGzfjOlZlO93ifBim", + "Gcxd2Rm7riX3arJj431kHqcKIDPvQJSi85O6SFUgsW2UiSdXE+HBY8RISHxe2DrsIL2zK6EVAEDkIsCA", + "E80poIckSu4zZp9V8UPbSflMg8+rHWRKGeCMj9R2I6JM7qzp069ACjg0Pg5DWYkgSWNOfE4CZKlAqNYH", + "35jzzWEcJwPsf5F0Sk+oZDhce6bxOfXVHBWgFoYws9/PYzVBZt7g7QpYWIRRuqqoGFAhAoLpmOqz4YiP", + "0jihvmftbN0Q1FED6NBp2Bk8W9yGnnHOA47MIJ3JR2E4RolrlzYfXjIlB8lTHDFhZSVrzxauq8mJ9UlZ", + "CdBgivhfTaYixCqyxqbU6QjVFjx2Sx+bIn4O4WOW9MnAAUca8QMzFGIep6sQIEjpVPaD6VhUxhOMCEZm", + "hHMN9jPIAMHqsniHKj6HPmtiP6uCBHgQX4iWZRk/8bWOhk0rWEGEf/wZcZyeEy6Zeg7l6FRqDrjAEm/3", + "UHi7q8nzB9tJYbzvAr75PtrVZB4YxhLAtwTwPVYAX2I5pk20v63zbwr+uxGULFFSt8SR/RlxZIm1QT7D", + "PbwhUqz0+XJbuZzplTavNr0r5bOQ2y3BUzwtwnXoFHiY69dPZ0X1VI9QukeEVGkot0RG1TDdAvPzT2vV", + "5gX8XE0eM9rnauLOHl9NXCnjq8n954kLIfViU8SWq1CN1R8wr1EDcZxulmbkJU6KWZByMheE2iRorYyA", + "cdpVUWYrW6VLzctsAwmsRN+JycDJmnPyRbtVhrAI8ExqQx4LRJejmBFEroifgbCYV9AYc1kQ3yahjRjk", + "ENMskjUM89rRNpWaQkMYPHnBpuYZxYcJZkwnWMr0U/EuZCjygTsyhTc5WnliOvKscMKcqVyR9ZGAXeA4", + "b9hGuSSsOg9Nyh++1nakF0BOht2B3huNPZNvK11Q5Hhjdoa/1sM+wP+KUw8Wk1fIM3vfNoUX6+oeBH2Z", + "gsLJhiTNb1qiXC8jjRjHYSi85ywMdZPV/e7WFHfzosZ/LwklPM3HWiOgBSVS0UQa4NogljPPtyGUS9RB", + "Y8GAKun3VWX1ttFX2S7bRp++ikXfRp1Opy0RJ/D367Pra+QpCeeeel01+kJWBAWmFCF7QlZ1/d+OvLxH", + "VRqATDcXXmGc8UGcRYFJLHbQDugWpoR2kkClTYXHtcPOVOufwogo03nvNsRBA+x/uRTxjFDVmNMBDSmf", + "dNzFPGRLH2aWtVF8B8VHqWCNsTABUgvlOcuMj6xtCUFMXwOP+621fisWb2z0W0U1I9ofaPiQ6kjuoGEu", + "p8PTs4G8ftbtbhaWoo1ks+aZ/KcuMR6nq29g7iXnJzGNOMIcTeIshcUbxukXEQdChWaSorEQPC0OLxhK", + "SRJiHwacr+2R4u7imvVbMR+RtN+aOtuHN1CFh7naU8r4gqQDzOlYDkqvIlpRtVfMhOrMMszoJCGrU4jX", + "ywMLp4aCPD2/Iwz1X8X7gYp4xETiMFQVZ4EbBpn/hXAtZwBE2I8CmHYlW35IScR7QeGfx8RPiXzj2hSQ", + "ga89oj6HVBKE7F8HgF2HPU3rE4wCmhKfhxOPZcpe+ykJZEZ0tYN+LHJk2xza29arrLPxWoMo+V1TBf7k", + "oAckjC/l5DHC36AghpqugsmgajUWgu6TAhNY5/ZhsX6ZqvfttIq1uDxWux1qevuti/V+a7Ut4YWyighk", + "RkHdS9cBMhXCUMvMUkfr+VxOPVSWxhE9HxGWQ6zyG7BUVkMXR6ZjfF7CHKLeUMxEW8gbJHzBY0E4b8zn", + "GeTKVA0uqvwp3abg3ACWWW1plirRqDK9DaxQnbm3WMC6YcgWmqqmR4OJEaUVdbEZzMgFxdYSrcL0y2oP", + "DBEqZEjOmNBGwn+xVbeSlTg1taWbcF7HFlU3/WUVeKNBhJd4wpCy46xIuVcoCQ0qwZN7W0MchrBpDOXn", + "ecfSJIZWHE0sisDX8mw+N5scujucEila2quQXIM58jEjbcRo5JMCSRX9p9Sfk0yMojjyzCewnw7RDRHk", + "R3FE+q1t0WrB1MmsL/ClV5AKM84olm2pscZDvc5v5O8rdCjmAnxsiQIOkGg1JSHMDFxzYvY3TZadhIxc", + "irEqZ02XsDFHfOTyi7+IKWm1YQjOCijqMNAtnSpjFk/NPXVG7d6Te4N+TSkn4C9v59VWROgEKaU1mesB", + "zorIBXjEPEujHCkhk3pg5XcOe+JLjFKCVa+CY9M4JB20E6lqgZL/BG3gVdNoRFKqQjzGYTQwu9L7+CwG", + "LngoIp+FmpW7ftJlcjgL7dalGE8xY1hxqV2O9MHu4SGgrxzBfHoO5WQabbfrd2H4EqCdHyozuxmlSrl2", + "m9UqaPntisqr1J3crNTitK/zqXJU4TJKWbcgt2HVF6a1QRyHBMsj9pSHZMqsjYobn/D6bDJd5ZPOaoPW", + "fCdo6jTX0VSs+jZfVT/HdT4S4ee0ufPNVWRHGdCos7L/zWdPCsR0SErtlckHu4dqp169glTJJAvIAOdC", + "+EiK7dMAIOTDetYAhHyYmp0qJ4r27cVbfKGf2Rfr5jS6r9e9/Ya+lKrm2MrcgiywjMz8EM+D3UO9Iecs", + "U50Qvz6I2j2s326wy+K+9LqvvPUfClexOUpcx+FcdJ/EspTZtKt+77bATSWVKqID7H8hUQAcBxKboiyV", + "N+JYCFJzp+6fs0ZOLo4ujqm7cPJPjVYY+8l66ZLYJ4RXMDI523eYG6/g/HyJV8h3vg/8xL3pnftU3thP", + "PIMUqO59F7yv4s53wbYbK/jprGCNxD8LtsQyCy2j+8VbtvbOS2Rsr1kkbG92u/daBsY1T7fAOkxl2IVg", + "Hf40Kz4XQCK3Oo8VJJFTqDbzNEFiQQt9yhW+N3iEI7xbFDzC9kDny3WYYHdG1D2mY3LizFGbFg56B/t6", + "zhtG7cLZs8Nqc2DLVcGd/jGtd/FYOAdwh0vLeTPLzcN9TVfDgL/dylI6T46iftzlu45SOq3sv/bo5+OB", + "n2rzL2L8wyzy5QxR7oQTQZl5WQfaXdY+Lzo9lPeokatEAlByjMQiMj1CH7raiTM+hUKz/tNJlY0gxtPM", + "51lKFpxQErS7L4psWsy8KMD2ojg5xVJyJe0fRTHHJoRy7/x+daWPCkcQ81bAh8fpgPIUpxPYwNDXGIgZ", + "NlVB4D5gGSx48op7fdtlq+DBTTcVSRqLMXrgdHTXXwevX24OvWDzh1fe9/jVlofx6w1v/YdXr/HGDxuv", + "N0i35TptCUHFbcb/HhqAoX8hE0/ujSWYpjJNHcvLX+CUYxQovJO6YpB10M9kwuSeZBRzcwuLPGdSmg0S", + "XdA0jiBvu93K73iEimzCIWipaLpVtPzOYU+VOLnv49JZuUmtu0DzhhlRc3bCccY1ys8vIDjAzGHvSe1k", + "qr0SBhAQxONEHQCRxzu+08fDxhCqqpdT6otPX0BTL9AgjP0vaEV+gb6TR8q+U7udbFVlLvXbAHIjDHL0", + "kKbHsrydEIILYk7JlSlZg1YFm9Bz2FjqoB2OQoIZh9M1cCGsPo6k72J1wdaAjMZnUQ7g7WtdUL95JJe3", + "ID+shnIAJlKTtqLmX4zijR6h3Dqz5o0RvmrfE1DCOsLhRpimYmrmK1iPawRwiFEcwh74HD0WUuODOP7C", + "1r7S4LpVPsnX+faGCdPKOSqJPVGHP3Pu1RvcBC4Iw0EA8Iadw17p0NLq7TOsN0uKXk8TzZ9AHg40+7lv", + "CymxiHWpzYqPGfFoxEjEqBCV4sIUruioJrX/8h/f/Gc/63Y3Xr349rt+3+v892+f//0/Z7MwFnpXY/8K", + "+7yypaHIk3CQUhChvzgi51mI031zWc+sTWtHB9Io8Fj2VERoRqTxJSbQx1TtaRbHiSKW3Qvj46eUk5Ri", + "tZGcs2gH7V9xsUDCbQEphBt+pP/G2siP4y+UsDYi3O9UVJPSmLXzIFV3ytDOhz0hrLoUGci8XAVB0H50", + "EU/UWWdlMONo7lN4Nrs6L6fS+nAuLZgrr2ZHATEfKRLKV/LIBlV701fVkFqrgC3GbXIZjbqCRt9JUwiW", + "5fcVDncMqaIEmsrdoVlvOImtDu4x43HkGhOYwCGVooVDcB2LxOvnTQV0LptzO0NSWv8G0lx3M5MB4e9q", + "DL7zVmF9uZi6XgpOvdvA/hzLX0DUTgvQ8uFLHOTCL4IqjX3+66DmR4LPcUWUi7pGF0XVyu02erd/0kZC", + "Wtvo8PSkjaSsthGIahspEW0jIbLgw36rqx3MKfPLC6gWfwHVg0moHYiBbe/o6PqThQMkwRn6y1+RWKKb", + "4Zkc/fmxO4dzEz7ZMbkCiy0MnEciFFeGKSEeREdfyERhT01yZtXFBbU7qb8UT4ZrfvsovPVxfrhFw4Kp", + "KeWgdx0vum15puXHLAyN4SrWfWxDxcZOd1XCPnnO5yI0vKRhKCK8lPzLOvg07XiMRALaCGwFiWY0Og9J", + "bkftMzPWURoXPtp5pObWqtMlIUeFOKRcvwXyFWuq5pTagJdgcZ5OilvxHXSAE4iXpHsIlnvHNxf2xxln", + "8O2R+FYhy+FuW1W4FXNzpb6MtlawQoHyGIrHyMIxgDReqzgi1U8k1lK/8EKVzlq1cE5Yge/hW9YutqhC", + "PhUa4C8E0go+CYjB6ZfvFA3JiRyo48AoST3lMEugg3jbTKkVd8KWOmWcRCR1vmsKxenh9Ftd1m+hgAKU", + "Qp0XlC8XD5t1WdkRCr5bUQUVVv+2Mmb/Zv8e/3u0+o0bHcnTyexQVbwE+deaeTjAV3ScjYFAo0kkflkF", + "FCtKYcLtUBoWoveZ5xnu+subj/faLSn21rmjvkXNzrl1WauCjlsZuxKoMN/nrezQ0DFhHI+T/KCm2Qy5", + "xEzfeq0KjKycnuw6LvSubgk3u9Hb3lyel7AQM54f3V1RyGz5co7mWyCxzW7Cx4zRcwvMreBNK+T3DIeQ", + "C7CLZK/eJLlqdtW/NoVqpiSJU14manFISGtL/0bLqC+9Xyh71Qgb3znszQVsER8skTI5bgKmJKFu7ISb", + "hd3oCfvdtW/yQKwIpDhSb70XLYq1s+pC2VftmBSGvhIHAnXr2hxhLmWkNOUNRxMy1i+2c9rkLROKuV48", + "K9S/MPNnHfdq2c51miNfTLYt/+rKgwoFOKGJpxbSy+dTX7Mj/VN552BqXR3vbNDedsqbCITzEifw6/XZ", + "9XV5y6mEVBljGhURK+oaH9YZ0H/RFHcCcrHGgCPZWoV3hJqiPlkzMJb7wjLVKeIbo5lKamQh+KWlHC7l", + "8JHI4VwIMxGJPVZsmaCttCGkxazQYy5794Yu2znsNQWWWYgyhTGrBZYd6firnAyCTW21IwNReRzlQWRg", + "/A4kFTDy4wBw3SoGbtsHa+PIiizzmg3lcrWeCKnGCUcpGaaEjdCY+CMcUTZGK4wQc6JzR74mQ32d5lBd", + "He/9vIrOCYdiNiMMR3HNSXXZaiIPskJP0jtW1Xd1v8Wj+rrMJREtiJmAepnq+/OYMCQiREfcHmXjI3id", + "FbZG1ts1oausNiqPL4j5Vn2AYzemkXjHrtti4d7kEuyKFXCWha8ulCpokdLzcyIDR55OCmHup63u+pm1", + "9TaWVLa2X75+bdGz1e26KBrTqCe/XHeUPrf51qbdyZ+CZ/b11YpTs+61yXZm7yE2T503y5K7EA2HVgX+", + "4ubRbbPSrimS9SqmnSmc9zAsgxYLlB/GjJ+n5Pgf7xEcCRHqZSDrMDN2GadB+czaxtYtT8xJIu69Xu+e", + "Htihc2ALKtpbsy0pl1IlB1fU0W0S+ekk4WVCWZZspmzTTzf5X+yIuH5BujNKQE0/plK7b2nzn3AOF8mD", + "bUSHdhqFRn6YBVA8asmed8Wec94ZZa//XZzTONbayBHm6HX2zDpb3lRJKTdgkWLE45pr7YAXpG8+/1cJ", + "+WN1gRV5JklXqkapVqPQuVmhe3OGKzZvUQctnMwsPahdSDCcQrzvYK+Pxydrh6cnaE1qBmZc5A76LLrr", + "AOt81ruDuujHGyR823oZkkUvCpVD9E7GIA4oYaU9vecgZjPyOute9+XJend7Ux+ThpxNlUZXcqb07SzJ", + "nUcYa+WrKjoPIifGNhemd/bXJmMtswA6cX0DgTP9zil5Mo66cNVQebefSxxkdPLIVPoKIsoMiPKgCpL4", + "DAWnzj4t5enO7M4jliUh8CIMf2g37Hba3p2hb8adlVT80k97OD/NbX/ua9f0o8IH0EgWFoOEEFzwfIHT", + "yRsr5lTht/DTiBVzBgjq3Dm3WRfneYpJOrL2BMrFobLItccecxxaCURlD23r9tKZOFTv1R5wUS/IcqWM", + "+FlK+UQCmnJDqm4Sy6vjgcsqrygXs2wqesAVx6my5QhrIJua9cEEUSjsHQ/gMJysfqoNt7x3u+lhgJL+", + "c9XsMQxoZ1R8nzDWDEkwTZ9X5rMnQYCFojMAimAd9CGWADWA8RX5XJYSRytRjD5DqvozitN+9Dnfx/y8", + "6ko9F+A+ZSxFxdrfHP1yDBUyWRHSgtb0isrzhIX0hUttT0eTLIT8ZpX5j7OBGZ0M9qw8RsVu9Gq2jyws", + "0IoFxOntoThVU1JM6fivhxuDV5h46xubW97LV9//4L3GA98LyLArfhK/OC96TJJQmSUnLfnjAk1QW22P", + "XBzGKcfh2vHJ8WqpUraF8UfMmhPXUeV2a0ABwLyrLs53kfKWKoyzeqdAjxYKXd0ShxM4FMJT7H+h0fnq", + "tF7tJZvWsz2MBfTOLDnXR152dk96v+xbFtj80Ptg/nq0/8vHn/f3nD6rTeNhiJ3jsceLkhBH6PS0tyfL", + "OGEudOyYSijtgBpcuQWmbc3oF+pYuw64498zUpxF4BLoGbg+ulD3e0ukpRC1N7r4KGZohNkI8qHlJPZA", + "XnLu4YG/vrF5NfljpvRK2XPRPUuoGxpXh6G0paDxoRa7a9Nto7uAj0usMEMbqbUWbxZV5u7Hg4P9o93e", + "znvXwpOrhKaTE1o+4wOKdn3D21w/2djcfvl6++Xr5nZCMOWHyrGhd3EYLFCQCl6teexoPU4+Rv/IYo6P", + "CNYnJFU/8mCCaUb+01FvdZTGnIfkvZCsXc0i5rP1rnsHtPDZaUS5HbgeUGGzf4qztNVu7eFJq906iCN5", + "HDAfl3o+Y39QT/dZAzZaCP+Lhm4mA+LL28lBPfElEaiwQsElasbJRfFo9o0K76TqrvGhporMFAmZKg6N", + "eL8pdzdk5+mO200huuU1lwn3prpvIav4VBekiX6ZcwXqJc64wLMd0wX7jHfnD7pavoHmuJEWaMJXd+VA", + "LtwtXDE17WEP3BS/fwMV+w9V4ssDuF2c5wTkHVKU8fIasdWZgeIi9M0MXXPbJXJ1f2rBNEtnS/QpJV0t", + "t1j9ekWlT9SNZyIC0CVrxWTFEVF5tWJ5sbB1dt0u/ijM99n1WaWqQyy8BSj+X3TQcMbjytl+dYSRoVF8", + "CfmMn2LG9Z1T1qUScApElZzVJxrza+o+i7Y/o4CERAgRk/VqU6BCfQCYxDa6HFEA5AF9rNJjxiq34fth", + "xjhJockO+jzGUYbDz/mJL9H1GHPqW/2JSEpWCGPiz5D6tFQquHTxhZwa2bZTSMFXqoL01MoBWhElKYH6", + "ZNYNflYNYWc1utABqoEbiAz3nB69B1mThy5VUXWgNnc5VU3JJI0DT323/bLb7a7hhK5dbNhBgCxUNweD", + "u+9Sw8sb1pY3rC1vWFvesLa8YW15w9ryhrXlDWvLG9aWN6w9mhvWjoVlmWjRxiiknKRYV+8AZmIahqE9", + "eH2a6LN88hlxMk5CQQ8xlX9W3+g2BQvB7m9lDz2Nx2U6DbRNIT8f+Q1w06IBK55xREPl61uKka/7Ahex", + "2sLKoAEOceTLCnNcRHGsskE+wIwcOo/uvJWIOyHHUGZOF91C2jFhxoYZ6lT5HxUkrXbQThgaj9+UAzWv", + "QymgEb4gqvqV6iwhUSBMKlyiwjhOuRzoi7UXMDZTLZhEgXnyBtZcXeMSlyqV5FGbZT/XCocAOr/9z1++", + "UfU1V1a//a795q/b/+s/186+/fTfa2ff3L5mtz3uwA5hrTtbJp45Abh+85uv6kqC5hVjmlzVowvtWDcO", + "TYHH6NBapmDKNwtdEno+UrcWFhmz/tpCZyD/1orgVyAhIm1mysEVb0sW8uMxYVJtaPZenRXce+sQ3s+M", + "69stORiXrIbyqKh8wTFYhP00ZgyNs5DTxJZqNW0iRrGubRtmPEuJfN1Tya1ii29keS3l9E9EAKJCEqLK", + "1ajPKEN+lqYk4iHY16B4Tf4P3XZ+rlHzmvyXY0+vUqg/dO65NT/6qKpo5Xx2NkVf1laAOnHdXwUTaVU5", + "kqqoihOKo0j0U2l0Vz6wamyhwOQdpbbrt14yEVX1Wy+73THrt4rMtuAiSb+YsGA/TeO0KjhgP6sD+RHM", + "KhhHEU5IM6haKgIREuJ3dE0EJ0CSMXw++1gYEeQh/bbdw666tXFc0u9rIM0+VOXPdftak3mRED4A5UGV", + "RGPcqK+zbdKzhSLxjPp5o0IZyNIUNBrGuiAElsygUL2/Hn/cAL9D5+vRibz/rawD9o9P4D3BdeCyqEL3", + "pUvVdEBZbVfVwVPOh7xkoeUojndQ8IcKBe9VTQUoShHhhLa2W5udbmezZZUiXfMFwwCWV07VOXGqNI1Q", + "DEO1/YRO3h8j+2NLr9iRLbFfko5Xpx+djAgjxc9FjGMuYLsgqboq4aeTk8PjgtujxFDFwabgRi9QZmjX", + "HlFeTgJGt9Htmkofch/R2plb+xeTvhcz12ZMM5BWPwUUAbCQ2zoWJvu6LfTEwsgBLTCNiF4kNA8O9cFR", + "kEspMdl4jNOJJtRaZL84lxyfM6GnraFbDCi09ZUHUiX8Zk8EDPA6Dsaw/apKdJC0dQaJINetfKcJWDbI", + "RpZ5DK0c7h+oPM+qTtZoQYHqkfbLlGlGDCYRHlMfsi5ClQjlnRJQOHpLRLdS4ShJjzXgVltXPHkbB5MG", + "y2edM7DIa223PPHf2/13vQ9od//opPdjb3fnZB9+7UcHvd7ef53s7u58+fV857L3due89/edn993T999", + "Nz76mf/rYKf7bvf493fHvcHm3j/23+5enu4c7J9e7f6x8/e35x9+6UedTqcfQWv7H/YcPehLdcDflOvt", + "+RKKPi//y0kyFcGKZh3Co4ocrt+FHE5jf5tns0RxhsI3D7MwBAz01v0KJFjeAtMqp/Mx6oaCZPoFgbiV", + "XoBaPQU7tJYS0RW4NE4lcQCbicK/tep1AHXxUKov27JAACA94ZCwCZMA/JL6qAj+ESkJ/q2NSblMh3Gf", + "LI/IplsOSVXLPd47NjUBC1w7FULYAMnfbvGY4/DthLtqpchzFHDBmJ5bRVTJNJieNjbWoSCKI1go+2rT", + "ZNQafllIH51kGHZUTLhIq+mQDqjSBSsVEnfNS/E7wkXFooWgaC9HODpXCUkZO97GVsqOi7bSuu5s+1Pl", + "EMie3mC1SeUxUkMrhE8vu+SHrW7XIxuvB97WerDl4e/XX3lbW69evXy5tdWVUTuNoOAHlIhR5o0GrbI9", + "sm1cOaY4W6iYS2jT3MOYFm051YWasjtWFnMKsSGqame37k+EbYJERDmMsyh4lIrEJbmLUSBhOPbUVVqp", + "p9Pv9QEfxAHv3x/o67dSk7IXevmcMk7SPMJTCiHfTw4nwtbKd9QOV8cZq71/f3CoejgxRM1QGj9Cy7Bv", + "obcR9AWCKn+UM/LHhEQ7Pa0W4AKfXC8Uq/ncl0LwK4D5zfoKZs1tuL2kjWDojqlvgkCvD27d7PK4w9wa", + "mnOREy/oaUJ6nuYRvro4dyfQrrSThkp0u2NxOwDhmMKV6mNZckcETkeSK/EjpCb1OUR9vZXdWVUk5aEa", + "F2fMG/Q2W0xHT3kQWUBNr03wOFxQw/canTrFzCFETibQNeYfR5haxNqWYCkkWJWUvb5Hwx5Hw5D6HHm5", + "aEKqGPBAgKvAYUpwMJEQ6sepjKTQTVMGi9RH9c5A47giqlFZlRCjJkBw65epNl9tpibZIKS+vaeqrxK2", + "1KYjdoAEOH0C0YEhtJn/714Hp9N9H57/HOTcdwzgJu1pRAPR3WuFtjsMeEd4vbgPJgDn6u1V5fwdcXn2", + "bye94MaCrjGxdVPxKIV9fsdgwU7PPFLKMQ3ZUjAbCKYQi3qZCBYcPmTOXTIF58vPhrkJKkboru2tAN+5", + "RZapqDsV0j9RbNJ9HLGJM7/4yGOTpV6bscPXTKvcZTwyR07ypqnItj6L0UYK3tRWVwHAsYsLOA05K105", + "R5qyMIczUpVmMm+Zs2w3JMcq3mddgdnp1nSfv377rtXcrynlb4F0i8ahREKOSLsRCaWL4LLiVdbWTW/u", + "3s03eeczL4y78doIRiyQhxPakZPT8eNx3Rqpzx4uob3hSmgXBHzeDHWhBu4dFK9rltV+Qsns2hz2guFa", + "dWnsSvY6V4Aqew2H4mMkOCTFvkJ+qmBT3a7UtgoWGwSgOXXQLp1CasO5F8b0HdLt/BwVlZcBzkh2332S", + "23kjwMK8yZrWp7smNEL/e+fgvTB8cLe4AiA9UIq8JOczaNfpcbgfQl97usyVz8qVG11QzpVHQX5K9Qnn", + "zW+t+hxe6U2T4zfIiTeMvKshd2kOckMIN/BJv8FLSv7lI06G15B9g9T448iIP75E+FPMfy9AuufIdjdO", + "cs+R3H4OkntDe34Xnk4DuXsEqe0nltGGRLZdA26xscRNctpzp7Kfmjj+CUKPU5U0Ls3wg6S851Mijzfd", + "vdRrN85o31mksKbqe8zIZuMwhFOf4k0XQm+mzislpXcOez+LTpspPllyxaX0ChUoNXFP3zGR09P0sKZe", + "mKV8Tfcb9EUm9py5mHkBXoQfRywbT01IvpM1n3ReQBF0I+GqJAgl/yxEus41meKhIvBJuxpybmDKFuZg", + "1LV5rwDeMhH1AqN57Smidh+FenuYhOhKkMlOpCCqwnvikdx3EB7E6uPPgE7RdIvVvDM8nrWvOKE/E9ii", + "npoxPSIX8RfwzRTpHfQx8glK4fcAKij6OEJRjMI4OhdBqaoQwWN764fkpWQdh3hFW4tX4fejqtvTStLq", + "9QZXTYyyQJOBd+dV8Rz05Cv1yHw0sW5+Y4WrOGapcBsoXHU5kZi2x+1aVtTDvfiU0xNTmhK5V62LpKjL", + "EWURSDgln/FYFx8VNiSOSIN01bNUTQ7o592rprvybot3ISzCty23eK/wz/k920eVBNN3vj8ZFbt0b2+a", + "vHuUvu1aSnQUX1+p5si8U0hC3iYtkTf5/P1aM8HPw4CYpVtwisTd7iM3JmnMl2mS5+e1G333EEr7ijYs", + "aiJefKDjA0DjvIcHriaoCP1/uIMDV5OHOTVwNXmURwYexYEBsSbP7bSAluU5zgpcTR78oABQ/RSOCSg1", + "VNLDV5M7PyFwNXEfD7iazHM2IAd8l1V3fmagdEtJ8+MAV5M7PQtQYtNFonFqm67zL64mj+cIQEV8p1G9", + "BP/fFPx/NXmGyH8Q2YUps5JLOT/6/2oyJ/T/anJbuCK0UD5h7+kHT6PyjSF3LpA/WI6HRfjXkfBAUePV", + "5Klh+xcrv40Q/leTRvD+q8kisP2PXTpvYp0X7q7MErAHxfE/epmyQPyStbMyTy7Y358PxS89zcYQ/idi", + "EJ91jFCC65uw6D6x+nOpiCVK/8lprWkK465d+tvD9BsoNSvzO1kAQP9qMhud/6S8i6eFyn8SXkADSP7t", + "hWtRYPwGIlTMzd1+r1vK0EwM/lPxGJbY+yX2/lZKbIlMWjjwfqH6darv8mgB94vR1HerkW8Hsb+aLPH1", + "S6WaK9VnA65ftHf4MLD656SA3ED6u1RASxT9EkX/2BTp0lFdLIT+gbzUxUPnGyQRyrj55+We1iHln6KF", + "WMLklzD5Z+18z8DIL1wrj/2kGTr+YPfwcOHg+DhVuGn33kjeZ3NU/MHuYREVX62nfyDfOrR18eIx8Tkh", + "94uJz/utx8STC5JO+Ei09Txx8XeNTH/pQqaP/eRwTnC64vAHBKdbMvaosekFXaA1oBHju4Om6xUqI9Nr", + "dqL063eEEnfyy2IcoRlN3+vuTo1YVFnIrM7yPtSmMO9cZp4R1NsSu4XphpJ7NAfS23BlU6C3Rf6trlbL", + "x2xuO+30i45Hbvo9MTjbD3nEGHA31c2g4GY1HgwJPp2C+46LDDVPAwd+J7I9HQVuZmg6CFy/dqvbS8uS", + "+1Tk9Sbme+HuyQxhexhQ+BORL8HrBUYPFuxYN8SAGxqaQcDvxFTKRP29it6fLDboPmBssLyP9Dnoqymq", + "Y9Fef0oY93BCZ6REjwjjO4e9e0yI6h6bp0N3Dnv1idAjguE0PIxm57B3d8lQQcb9pkFFj/UJ0FSO3Asp", + "lLh4nreJLjYk0/LQKK+pGNWVyWyYTL2zhKeRoUed7rQkXas28ROw9Z3lOlWnDVOdeo3vxptRrS/Gf6k0", + "dq/ZTCMMVZ7QM75MXzZNX4rZekaJy1yIFiXmBQemcdLSyH7TlGVO+K3CMKVu3LlK20oDVuWJZCvr6G6W", + "r9Qr8WDpyqkE3Hd0ool5IsnKxcvztFSlkdrpiUr11q3ylMM41QL7dMS0mVVegGcxXYweJg/5NCRH8LHN", + "xcFiPd6GSUhNQbMc5GJtnzv5eMdC9Qwd9u59OuzLnOIz0D31iuBO/fEb15ZorKbE9/MVlJilpExVCXUi", + "Hih6Fn7AEyky8XSs+bQSE7cXrVvWlqgTIXSiKj1QhjDa3PAGE05QiqPAnDckkR8HMsU/Ilc4ID4d47CN", + "kpQM6RUJZFriM05o8tvnDjplxAjQz2Qi68tOUBzZYqVUNUE08uOxUED6ALVsjY8og/PYNTm4uc6pzJJx", + "V9WLp+6VLAtgLAtgPCcFO62+xEKV6xS35RGWlVioHpTkPYgWnK/oxCyyltUnlhrt0Wu0ipJYqIN43+Ul", + "FqaIHp3KkRmPB1E5y3oTy3oT96s6xQQ9mVPDtfpM+Ij5+f9AKrb7dxEXVtNhavCepOSCxhnTUbx2DnAk", + "WCsJsa9DdDkxC4jxpxSSeD6B+fyFJp6VjVhWnFhWnHhuDnddkYmFJxAY8VPC6/c5jvSuAjYZYxyGiPE4", + "FVwmv+6gI8KzNGLqB0tPyixpnPF+JLQR9nkGY4fXQKPLzDMjfpZSPkFJliYxI0zutlY3TY4VwXcodbKL", + "pvsNag7M/otL9tbvj79OI7HucUr/IAHyyteoGdX1qKG1zKyx5nS16s0ZvX7v4ViwLlMuhmJEEvnpJIEb", + "yTgSDpN0WNTT3h4aZ4xD6gvcgU4/Eo9VFMqszzMmXCIOzg4Vw9LPxOSbG2EHZBinBCUkZZRxEvnExe0y", + "kShHfkcQXtn4HRxHmtrwgrLwyn+R9T9k5hwINPx0bORQZtblWQXpYku4/C/qBMN261w5qsL7SULMh3E6", + "7lyyeKPjx+O1i/VWu/WFRmJZzIKMCccB5jAX+hwG5niAGfESzNhlnIKcsYT4VTY8jBk/T8nxP96jMaYR", + "0p8i82m7cKxju7Wn3zi0GzfQQjUFO7y13drobrzyuute9+XJend7s7vd7f5TOHSBk8Z2S0WZ9d9ew6rd", + "Yu3l6kqWltGQS0vITx/HPshbnAe8HhpTBqIdp4gq72ZISRiwR6zgHwoArtRmvj3a23uUqG/k2dpZuqTT", + "NnOYlvxbWCXL55qJ/D4k6RiLgYa6LoEwW2p2DQpcy7MwWZTJ3fERTgP1CSxDP4pE+OfHFySdoDHxRzii", + "bCytnLE64lsakHESixVBnmwBLmNFURx5sHYk4v1I0ZAqr2+ru+UyYBJyaxmwqr/mFH8XqhmtRDFSvLL6", + "qGVua07TFcXck6FI0XipuYgJg2gFJt82XwaZ3lKrUYy28ggnNxKir99U2NNcn8+cnePp/T8WWTcWVkh6", + "lpI6gPgixLw9PZpi6uZbUD65UBe8TuNdqtds77IfudxKfyQcCeVcDojEqggJJUEH9WTgpl9mMAuIx/1I", + "tQ/KRPbdRhi97HbVzEGmTjajs3MQnlIfKR50Cf87wqdK/hwSoo9K1Dl3KvLC4fPy7sxgWixLNlO26aeb", + "/C9Pz+nTTB9M0R158GwJxtMJpe81h/VU1C2Z7lpZmaXFaNwmefxKfirPg6s6kuKvV0VVIySUJbA70duz", + "xDJJ46ATDDpCwjsFnUBlYr2gr+C3YgMOhXK9IKTelG11Vti+sZ116eYCddIUmX8Wshz9KE9z+FmaCmdx", + "SrqjjUiEB6G61D8eYy4sBz2XnNuPeCz6IamEoQZZmhdmZx30MQysFBsoUxFJ4EFI0AXFKtdiW0CXNZIj", + "/3PmUuY1t8ou1Jpbc5vFMpPS3Kiub2+9fIBMyqOAD8zMpEhGWpr3p2TeZ2VONORhcVmTbGDoEoolanA4", + "x/4GwTcIX2AagvVockTn2GrgEPq8y32nUmeNd6Aqo3y82zsOWm+zn1m/zWMyd5UeER9hjgIypBFhCHZZ", + "QzqmXAblGBQl4rB3OVQII7sNVnfSo7x8d+VnlLrRpV4e5IxDmZipiq2yEHrX5gEN0oPlyR/32YWK0NxS", + "St0KfO2r+KPXsP5JVZCbVkJxSGYpWHTEXJK0W6LvtxxJ7sowVL773j2ND0+jYMeieXFKuQ7YT5HFIADp", + "4uC56XU8Ho7Tuo9Epz9ULY0Pj/7UbQ03QUbolh5Qwxoa1f6bVdO4V66+e4+pcgTg+tFKk87FLKXJHVve", + "sZsyI8QsvNq0sOzOYa+NrAmcWVL2uEDQXHVle3toxSpz2tsTfcnLEFdryprihILUToWbuz80Q7pZA1MK", + "qu7snvR+2W+1W70P5q9H+798/Hl/7y7KqjaV55sE6E8kNr+rsFxN3wAMkzVoOE/cuHpKNeC+h2D70QTa", + "jU3Inzm+Rl7ROjylsqOsyNgLtWhrX+1/3ij2vknY3chlLFJ2x6H3Q0XdBSKipxeCP1T03Tzwvn9e6z6s", + "nn+omPsJsbIjAH/A2Hv+sPteePpu/acHC7sbs/BDRdtPSI6cofdtfRTRgzr/B6wN7+5kfNTa/nQmWFMS", + "5Ip338c+DpGq5gi9tVtZGra2WyPOk+21tVC8MIoZ337dfd1dwwldGxvS1i7WW9Xj03ux/4Wkaz9nA5JG", + "gLrPY+hy8wrt4okVSuMwJGltP2dmlip7lUenezkMX2476olkuXi75rZKvauxwtW8qjXnPTzV5uRDXXjl", + "5P0x8knK6RCqPsnWfzo5OTxGWcJ4SvAYXZBUPpacobrbzb+an351j7oEeZ2QcRKKZgoQCWtk7rdv12mj", + "vm7ahbwJfFr7s1bJ1Xh+Ula15QBeXJ9d//8AAAD//0JJw7EP7QEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/gateway/gateway-controller/pkg/config/api_validator.go b/gateway/gateway-controller/pkg/config/api_validator.go index 3b986c23bc..125c5adf4e 100644 --- a/gateway/gateway-controller/pkg/config/api_validator.go +++ b/gateway/gateway-controller/pkg/config/api_validator.go @@ -471,12 +471,16 @@ func (v *APIValidator) validateRestData(spec *api.APIConfigData) []ValidationErr return errors } -// validateResilience validates a resilience block (timeout / idleTimeout). Both fields +// validateResilience validates a resilience block (timeout / idleTimeout / retry). Both fields // are optional duration strings; "0s" is allowed (disables the timeout), negative and // malformed values are rejected. fieldPrefix is the path to the block (e.g. // "spec.resilience" or "spec.operations[2].resilience"). func (v *APIValidator) validateResilience(fieldPrefix string, r *api.Resilience) []ValidationError { - return validateResilienceTimeouts(fieldPrefix, r) + errs := validateResilienceTimeouts(fieldPrefix, r) + if r != nil { + errs = append(errs, validateResilienceRetry(fieldPrefix, r.Retry)...) + } + return errs } // validateResilienceTimeouts validates the timeout fields of a resilience block. @@ -519,6 +523,39 @@ func validateResilienceTimeouts(fieldPrefix string, r *api.Resilience) []Validat return errors } +// validateResilienceRetry validates a resilience.retry block: statusCodes +// must be non-empty (each in the valid HTTP status range, already enforced +// by the OpenAPI schema's minimum/maximum — this is a defense-in-depth check +// for configs that bypass schema validation, e.g. direct DB rows), and +// numRetries (if set) must be >= 1. +func validateResilienceRetry(fieldPrefix string, r *api.Retry) []ValidationError { + if r == nil { + return nil + } + var errs []ValidationError + if len(r.StatusCodes) == 0 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.statusCodes", + Message: "must be non-empty when resilience.retry is configured", + }) + } + for _, code := range r.StatusCodes { + if code < 400 || code > 599 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.statusCodes", + Message: fmt.Sprintf("status code %d is not a valid HTTP status code (400-599)", code), + }) + } + } + if r.NumRetries != nil && *r.NumRetries < 1 { + errs = append(errs, ValidationError{ + Field: fieldPrefix + ".retry.numRetries", + Message: "must be at least 1 when set", + }) + } + return errs +} + // validateContext validates the context path // ValidateContext validates a resource's context path. Exported so it can be // reused by other kinds' validators (e.g. an event-gateway-controller binary diff --git a/gateway/gateway-controller/pkg/config/api_validator_test.go b/gateway/gateway-controller/pkg/config/api_validator_test.go index ea89547e87..792b5c5654 100644 --- a/gateway/gateway-controller/pkg/config/api_validator_test.go +++ b/gateway/gateway-controller/pkg/config/api_validator_test.go @@ -534,6 +534,46 @@ func TestAPIValidator_ValidateAllHTTPMethods(t *testing.T) { } } +func TestValidateResilience_RetryRequiresNonEmptyStatusCodes(t *testing.T) { + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{}}} + errs := validateResilienceRetry("spec.resilience", r.Retry) + if len(errs) == 0 { + t.Error("expected an error for empty resilience.retry.statusCodes") + } + if len(errs) > 0 && !strings.Contains(errs[0].Field, "statusCodes") { + t.Errorf("expected error field to mention statusCodes, got %q", errs[0].Field) + } +} + +func TestValidateResilience_RetryValidConfigPasses(t *testing.T) { + numRetries := 2 + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}} + errs := validateResilienceRetry("spec.resilience", r.Retry) + if len(errs) != 0 { + t.Errorf("expected no errors, got %v", errs) + } +} + +func TestValidateResilience_RetryRejectsInvalidStatusCode(t *testing.T) { + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{200}}} // 200 is not a valid retry status + errs := validateResilienceRetry("spec.resilience", r.Retry) + if len(errs) == 0 { + t.Error("expected an error for invalid status code 200") + } +} + +func TestValidateResilience_RetryRejectsNegativeNumRetries(t *testing.T) { + numRetries := 0 + r := &api.Resilience{Retry: &api.Retry{StatusCodes: []int{503}, NumRetries: &numRetries}} + errs := validateResilienceRetry("spec.resilience", r.Retry) + if len(errs) == 0 { + t.Error("expected an error for numRetries < 1") + } + if len(errs) > 0 && !strings.Contains(errs[0].Field, "numRetries") { + t.Errorf("expected error field to mention numRetries, got %q", errs[0].Field) + } +} + func createValidRestAPIConfig() *api.RestAPI { return &api.RestAPI{ ApiVersion: api.RestAPIApiVersionGatewayApiPlatformWso2Comv1, diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 8da54f96fc..290b2d4185 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -627,6 +627,11 @@ type PolicyEngineConfig struct { TimeoutMs uint32 `koanf:"timeout_ms"` MessageTimeoutMs uint32 `koanf:"message_timeout_ms"` TLS PolicyEngineTLS `koanf:"tls"` // TLS configuration (TCP mode only) + // UpstreamRefreshPort is the port for policy-engine's second, upstream-attempt + // ext_proc endpoint (TCP mode only; UDS mode uses the fixed + // constants.DefaultUpstreamExtProcSocketPath instead). Mirrors Port, which + // targets the main downstream ext_proc endpoint on the same process. + UpstreamRefreshPort uint32 `koanf:"upstream_refresh_port"` } // PolicyEngineTLS holds policy engine TLS configuration @@ -997,11 +1002,12 @@ func defaultConfig() *Config { }, }, PolicyEngine: PolicyEngineConfig{ - Mode: "uds", // UDS mode by default - Host: "policy-engine", // Only used in TCP mode - Port: 9001, // Only used in TCP mode - TimeoutMs: 60000, - MessageTimeoutMs: 60000, + Mode: "uds", // UDS mode by default + Host: "policy-engine", // Only used in TCP mode + Port: 9001, // Only used in TCP mode + UpstreamRefreshPort: 9004, // Only used in TCP mode + TimeoutMs: 60000, + MessageTimeoutMs: 60000, TLS: PolicyEngineTLS{ Enabled: false, CertPath: "", @@ -1742,6 +1748,12 @@ func (c *Config) validatePolicyEngineConfig() error { if policyEngine.Port > 65535 { return fmt.Errorf("router.policy_engine.port must be between 1 and 65535, got: %d", policyEngine.Port) } + if policyEngine.UpstreamRefreshPort == 0 { + return fmt.Errorf("router.policy_engine.upstream_refresh_port is required when mode is tcp") + } + if policyEngine.UpstreamRefreshPort > 65535 { + return fmt.Errorf("router.policy_engine.upstream_refresh_port must be between 1 and 65535, got: %d", policyEngine.UpstreamRefreshPort) + } default: return fmt.Errorf("router.policy_engine.mode must be 'uds' or 'tcp', got: %s", policyEngine.Mode) } diff --git a/gateway/gateway-controller/pkg/config/config_test.go b/gateway/gateway-controller/pkg/config/config_test.go index d70567181d..4c4629ddb5 100644 --- a/gateway/gateway-controller/pkg/config/config_test.go +++ b/gateway/gateway-controller/pkg/config/config_test.go @@ -91,8 +91,9 @@ func validConfig() *Config { }, }, PolicyEngine: PolicyEngineConfig{ - TimeoutMs: 1000, - MessageTimeoutMs: 500, + TimeoutMs: 1000, + MessageTimeoutMs: 500, + UpstreamRefreshPort: 9004, // valid default so TCP-mode test cases below that don't touch this field stay valid }, VHosts: VHostsConfig{ Main: VHostEntry{Default: "localhost"}, @@ -1087,6 +1088,41 @@ func TestConfig_ValidatePolicyEngineConfig(t *testing.T) { } } +// TestConfig_ValidatePolicyEngineUpstreamRefreshPort covers the second, upstream-attempt +// ext_proc port (UpstreamRefreshPort) validated alongside the existing Port field in TCP +// mode — mirrors TestConfig_ValidatePolicyEngineConfig's Port cases exactly. +func TestConfig_ValidatePolicyEngineUpstreamRefreshPort(t *testing.T) { + tests := []struct { + name string + upstreamRefreshPort uint32 + wantErr bool + errContains string + }{ + {name: "Valid upstream refresh port", upstreamRefreshPort: 50052, wantErr: false}, + {name: "Zero upstream refresh port", upstreamRefreshPort: 0, wantErr: true, errContains: "upstream_refresh_port is required"}, + {name: "Upstream refresh port too high", upstreamRefreshPort: 70000, wantErr: true, errContains: "upstream_refresh_port must be between"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfig() + cfg.Router.PolicyEngine.Mode = "tcp" + cfg.Router.PolicyEngine.Host = "localhost" + cfg.Router.PolicyEngine.Port = 50051 + cfg.Router.PolicyEngine.TimeoutMs = 1000 + cfg.Router.PolicyEngine.MessageTimeoutMs = 500 + cfg.Router.PolicyEngine.UpstreamRefreshPort = tt.upstreamRefreshPort + err := cfg.Validate() + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestConfig_ValidatePolicyEngineTLS(t *testing.T) { tests := []struct { name string diff --git a/gateway/gateway-controller/pkg/config/llm_validator.go b/gateway/gateway-controller/pkg/config/llm_validator.go index 293828b2be..c986385304 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator.go +++ b/gateway/gateway-controller/pkg/config/llm_validator.go @@ -377,9 +377,12 @@ func (v *LLMValidator) validateProviderSpec(spec *api.LLMProviderConfigData) []V // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) - // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // Validate API-level resilience (timeout / idleTimeout / retry). LLM kinds support resilience at // the API level only. errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + if spec.Resilience != nil { + errors = append(errors, validateResilienceRetry("spec.resilience", spec.Resilience.Retry)...) + } return errors } @@ -406,6 +409,100 @@ func (v *LLMValidator) validatePolicyListExclusivity(globalPolicies *[]api.Polic return nil } +// upstreamAuthFields normalizes the upstream-auth shapes shared by +// LlmProvider, LlmProxy, and MCPProxy so the validation rules below are +// written once instead of once per call site. +type upstreamAuthFields struct { + authType string + header *string + value *string + + // policyName/policyVersion/policyParams are the generic override/bucket + // fields, valid for any authType - see validateUpstreamAuthFields. + policyName *string + policyVersion *string + policyParams *map[string]interface{} +} + +// validateUpstreamAuthFields is the shared validation logic for +// upstream.auth (LlmProvider, LlmProxy) and MCPProxy's upstream auth. +// fieldPrefix must already include the trailing ".auth" segment. +func validateUpstreamAuthFields(fieldPrefix string, f upstreamAuthFields) []ValidationError { + var errors []ValidationError + + if f.authType == "" { + return append(errors, ValidationError{ + Field: fieldPrefix + ".type", + Message: "Auth type is required", + }) + } + if f.authType != "api-key" && f.authType != "oauth2" && f.authType != "other" && f.authType != "none" { + return append(errors, ValidationError{ + Field: fieldPrefix + ".type", + Message: "Auth type must be 'api-key', 'oauth2', 'other', or 'none'", + }) + } + + // "none": authentication is handled by a user-attached policy elsewhere, + // or not at all - no field is required. + if f.authType == "none" { + return errors + } + + if f.policyVersion != nil && *f.policyVersion != "" && !majorVersionPattern.MatchString(*f.policyVersion) { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".policyVersion", + Message: "Auth policyVersion must be major-only (e.g. 'v1')", + }) + } + + // "oauth2"/"other" have no typed fields - policyParams is mandatory for + // both, and "other" additionally requires an explicit policyName. + if f.authType == "oauth2" || f.authType == "other" { + if f.authType == "other" && (f.policyName == nil || strings.TrimSpace(*f.policyName) == "") { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".policyName", + Message: "Auth policyName is required when auth type is 'other'", + }) + } + if f.policyParams == nil { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".policyParams", + Message: fmt.Sprintf("Auth policyParams is required when auth type is '%s'", f.authType), + }) + } + return errors + } + + // type is api-key: header/value (deprecated) and policyParams are mutually + // exclusive - configuring both is ambiguous, so it's rejected outright. + hasLegacyApiKeyFields := (f.header != nil && *f.header != "") || (f.value != nil && *f.value != "") + if f.policyParams != nil && hasLegacyApiKeyFields { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".policyParams", + Message: "Auth policyParams cannot be combined with the deprecated 'header'/'value' fields - configure one or the other", + }) + } + if f.policyParams != nil { + return errors + } + + if f.header == nil || *f.header == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".header", + Message: "Auth header is required when api-key auth type is set and policyParams is omitted", + }) + } + if f.value == nil || *f.value == "" { + errors = append(errors, ValidationError{ + Field: fieldPrefix + ".value", + Message: "Auth value is required when api-key auth type is set and policyParams is omitted", + }) + } + + return errors +} + // validateUpstreamWithAuth validates an UpstreamWithAuth configuration. The upstream may specify // either a direct `url` or a `ref` to one of the provided upstream definitions (exactly one). func (v *LLMValidator) validateUpstreamWithAuth(fieldPrefix string, @@ -465,37 +562,15 @@ func (v *LLMValidator) validateUpstreamWithAuth(fieldPrefix string, // Validate auth if present if upstream.Auth != nil { auth := upstream.Auth - // Validate 'type' - if auth.Type == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.type", fieldPrefix), - Message: "Auth type is required", - }) - } else if auth.Type != api.LLMProviderConfigDataUpstreamAuthTypeApiKey && - auth.Type != api.LLMProviderConfigDataUpstreamAuthTypeOther && - auth.Type != api.LLMProviderConfigDataUpstreamAuthTypeNone { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.type", fieldPrefix), - Message: "Auth type must be one of 'api-key', 'other', 'none'", - }) - } - - // Header and value are only meaningful for api-key; for 'other'/'none' - // authentication is handled by user-attached policies (or not at all). - if auth.Type == api.LLMProviderConfigDataUpstreamAuthTypeApiKey { - if auth.Header == nil || *auth.Header == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.header", fieldPrefix), - Message: "Auth header is required when api-key auth type is set", - }) - } - if auth.Value == nil || *auth.Value == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.value", fieldPrefix), - Message: "Auth value is required when api-key auth type is set", - }) - } + fields := upstreamAuthFields{ + authType: string(auth.Type), + header: auth.Header, + value: auth.Value, + policyName: auth.PolicyName, + policyVersion: auth.PolicyVersion, + policyParams: auth.PolicyParams, } + errors = append(errors, validateUpstreamAuthFields(fieldPrefix+".auth", fields)...) } return errors @@ -653,9 +728,12 @@ func (v *LLMValidator) validateProxyData(spec *api.LLMProxyConfigData) []Validat // The deprecated `policies` list must not coexist with the new policy lists errors = append(errors, v.validatePolicyListExclusivity(spec.GlobalPolicies, spec.OperationPolicies, spec.Policies)...) - // Validate API-level resilience (timeout / idleTimeout). LLM kinds support resilience at + // Validate API-level resilience (timeout / idleTimeout / retry). LLM kinds support resilience at // the API level only. errors = append(errors, validateResilienceTimeouts("spec.resilience", spec.Resilience)...) + if spec.Resilience != nil { + errors = append(errors, validateResilienceRetry("spec.resilience", spec.Resilience.Retry)...) + } return errors } @@ -682,38 +760,18 @@ func (v *LLMValidator) validateLLMProxyTransformer(fieldPrefix string, transform return errors } +// validateLLMUpstreamAuth validates an LlmProxy's provider/additionalProviders +// auth. auth must be non-nil; callers already guard on that. func (v *LLMValidator) validateLLMUpstreamAuth(fieldPrefix string, auth *api.LLMUpstreamAuth) []ValidationError { - var errors []ValidationError - if auth.Type == "" { - errors = append(errors, ValidationError{ - Field: fieldPrefix + ".type", - Message: "Auth type is required", - }) - } else if auth.Type != api.LLMUpstreamAuthTypeApiKey && - auth.Type != api.LLMUpstreamAuthTypeOther && - auth.Type != api.LLMUpstreamAuthTypeNone { - errors = append(errors, ValidationError{ - Field: fieldPrefix + ".type", - Message: "Auth type must be one of 'api-key', 'other', 'none'", - }) - } - // Header and value are only meaningful for api-key; for 'other'/'none' - // authentication is handled by user-attached policies (or not at all). - if auth.Type == api.LLMUpstreamAuthTypeApiKey { - if auth.Header == nil || *auth.Header == "" { - errors = append(errors, ValidationError{ - Field: fieldPrefix + ".header", - Message: "Auth header is required when api-key auth type is set", - }) - } - if auth.Value == nil || *auth.Value == "" { - errors = append(errors, ValidationError{ - Field: fieldPrefix + ".value", - Message: "Auth value is required when api-key auth type is set", - }) - } - } - return errors + fields := upstreamAuthFields{ + authType: string(auth.Type), + header: auth.Header, + value: auth.Value, + policyName: auth.PolicyName, + policyVersion: auth.PolicyVersion, + policyParams: auth.PolicyParams, + } + return validateUpstreamAuthFields(fieldPrefix, fields) } // validateAccessControl validates access control configuration diff --git a/gateway/gateway-controller/pkg/config/llm_validator_test.go b/gateway/gateway-controller/pkg/config/llm_validator_test.go index d0464a52b6..8c2da384d6 100644 --- a/gateway/gateway-controller/pkg/config/llm_validator_test.go +++ b/gateway/gateway-controller/pkg/config/llm_validator_test.go @@ -587,9 +587,12 @@ func TestValidateLLMProvider_Valid(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.openai.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -1253,9 +1256,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { tests := []struct { name string auth *struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` } expectError bool errorField string @@ -1264,9 +1270,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "missing auth type", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: "", Header: stringPtr("Authorization"), @@ -1279,9 +1288,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "invalid auth type", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: "bearer", Header: stringPtr("Authorization"), @@ -1294,9 +1306,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "api-key without header", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Value: stringPtr("sk-test"), @@ -1308,9 +1323,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "api-key with empty header", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr(""), @@ -1323,9 +1341,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "api-key without value", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -1337,9 +1358,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "api-key with empty value", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -1352,9 +1376,12 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { { name: "valid api-key auth", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -1363,27 +1390,164 @@ func TestValidateLLMProvider_UpstreamAuth(t *testing.T) { expectError: false, }, { - name: "valid other auth without header or value", + name: "oauth2 without policyParams", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ - Type: api.LLMProviderConfigDataUpstreamAuthTypeOther, + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + }, + expectError: true, + errorField: "spec.upstream.auth.policyParams", + errorPart: "required", + }, + { + name: "valid oauth2 auth via policyParams", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": "https://idp.example.com/oauth2/token", + "clientId": "client-id", + "clientSecret": "client-secret", + }, + }, + expectError: false, + }, + { + name: "oauth2 with policyName override and policyVersion", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyName: stringPtr("my-oauth2-fork"), + PolicyVersion: stringPtr("v2"), + PolicyParams: &map[string]interface{}{ + "bearerToken": "static-token", + }, }, expectError: false, }, { name: "valid none auth without header or value", auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeNone, }, expectError: false, }, + { + name: "invalid policyVersion format", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyVersion: stringPtr("v1.0.0"), + PolicyParams: &map[string]interface{}{ + "bearerToken": "static-token", + }, + }, + expectError: true, + errorField: "spec.upstream.auth.policyVersion", + errorPart: "major-only", + }, + { + name: "api-key with both header/value and policyParams", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, + Header: stringPtr("Authorization"), + Value: stringPtr("Bearer sk-test"), + PolicyParams: &map[string]interface{}{ + "request": map[string]interface{}{"headers": []interface{}{}}, + }, + }, + expectError: true, + errorField: "spec.upstream.auth.policyParams", + errorPart: "cannot be combined", + }, + { + name: "other without policyName", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOther, + PolicyParams: &map[string]interface{}{"foo": "bar"}, + }, + expectError: true, + errorField: "spec.upstream.auth.policyName", + errorPart: "required", + }, + { + name: "other without policyParams", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOther, + PolicyName: stringPtr("my-custom-auth-policy"), + }, + expectError: true, + errorField: "spec.upstream.auth.policyParams", + errorPart: "required", + }, + { + name: "valid other auth", + auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOther, + PolicyName: stringPtr("my-custom-auth-policy"), + PolicyParams: &map[string]interface{}{"foo": "bar"}, + }, + expectError: false, + }, } validator := NewLLMValidator() @@ -1994,6 +2158,21 @@ func TestValidateLLMProvider_Resilience(t *testing.T) { errs := validator.Validate(validProviderWithResilience(&api.Resilience{IdleTimeout: stringPtr("abc")})) assertHasFieldError(t, errs, "spec.resilience.idleTimeout") }) + + t.Run("retry with empty statusCodes is rejected", func(t *testing.T) { + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{}}, + })) + assertHasFieldError(t, errs, "spec.resilience.retry.statusCodes") + }) + + t.Run("retry with valid statusCodes and numRetries is accepted", func(t *testing.T) { + numRetries := 2 + errs := validator.Validate(validProviderWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}, + })) + assert.Empty(t, errs) + }) } func TestValidateLLMProxy_Resilience(t *testing.T) { @@ -2018,6 +2197,100 @@ func TestValidateLLMProxy_Resilience(t *testing.T) { errs := validator.Validate(validProxyWithResilience(&api.Resilience{IdleTimeout: stringPtr("-1s")})) assertHasFieldError(t, errs, "spec.resilience.idleTimeout") }) + + t.Run("retry with empty statusCodes is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{}}, + })) + assertHasFieldError(t, errs, "spec.resilience.retry.statusCodes") + }) + + t.Run("retry with valid statusCodes and numRetries is accepted", func(t *testing.T) { + numRetries := 2 + errs := validator.Validate(validProxyWithResilience(&api.Resilience{ + Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}, + })) + assert.Empty(t, errs) + }) +} + +// validProxyWithAuth builds an LlmProxy whose primary provider.auth is set - the +// shared entry point for both validateLLMUpstreamAuth call sites is exercised +// separately below via additionalProviders. +func validProxyWithAuth(auth *api.LLMUpstreamAuth) api.LLMProxyConfiguration { + return api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "openai-proxy"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "my-proxy", + Version: "v1.0", + Provider: api.LLMProxyProvider{Id: "openai", Auth: auth}, + }, + } +} + +// TestValidateLLMProxy_ProviderAuth covers validateLLMUpstreamAuth, the +// LlmProxy-side counterpart to validateUpstreamWithAuth's LlmProvider +// upstream.auth handling - previously exercised by no test at all (0% +// coverage), even though it shares validateUpstreamAuthFields with the +// LlmProvider/Mcp paths that were already covered. +func TestValidateLLMProxy_ProviderAuth(t *testing.T) { + validator := NewLLMValidator() + + t.Run("oauth2 with policyParams is valid", func(t *testing.T) { + errs := validator.Validate(validProxyWithAuth(&api.LLMUpstreamAuth{ + Type: "oauth2", + PolicyParams: &map[string]interface{}{"tokenEndpoint": "https://idp.example.com/token"}, + })) + assert.Empty(t, errs) + }) + + t.Run("oauth2 without policyParams is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithAuth(&api.LLMUpstreamAuth{Type: "oauth2"})) + assertHasFieldError(t, errs, "spec.provider.auth.policyParams") + }) + + t.Run("other without policyName is rejected", func(t *testing.T) { + errs := validator.Validate(validProxyWithAuth(&api.LLMUpstreamAuth{ + Type: "other", + PolicyParams: &map[string]interface{}{"foo": "bar"}, + })) + assertHasFieldError(t, errs, "spec.provider.auth.policyName") + }) + + t.Run("nil auth is fine", func(t *testing.T) { + errs := validator.Validate(validProxyWithAuth(nil)) + assert.Empty(t, errs) + }) +} + +// TestValidateLLMProxy_AdditionalProviderAuth covers the second +// validateLLMUpstreamAuth call site (spec.additionalProviders[].auth), +// distinct from the primary provider's - also previously untested. +func TestValidateLLMProxy_AdditionalProviderAuth(t *testing.T) { + validator := NewLLMValidator() + + proxyWithAdditional := func(auth *api.LLMUpstreamAuth) api.LLMProxyConfiguration { + p := validProxyWithAuth(nil) + p.Spec.AdditionalProviders = &[]api.LLMProxyAdditionalProvider{ + {Id: "anthropic", Auth: auth}, + } + return p + } + + t.Run("oauth2 with policyParams is valid", func(t *testing.T) { + errs := validator.Validate(proxyWithAdditional(&api.LLMUpstreamAuth{ + Type: "oauth2", + PolicyParams: &map[string]interface{}{"tokenEndpoint": "https://idp.example.com/token"}, + })) + assert.Empty(t, errs) + }) + + t.Run("oauth2 without policyParams is rejected on the additional provider's own field path", func(t *testing.T) { + errs := validator.Validate(proxyWithAdditional(&api.LLMUpstreamAuth{Type: "oauth2"})) + assertHasFieldError(t, errs, "spec.additionalProviders[0].auth.policyParams") + }) } // ============================================================================ diff --git a/gateway/gateway-controller/pkg/config/mcp_validator.go b/gateway/gateway-controller/pkg/config/mcp_validator.go index bc33a651cf..b6293baba0 100644 --- a/gateway/gateway-controller/pkg/config/mcp_validator.go +++ b/gateway/gateway-controller/pkg/config/mcp_validator.go @@ -254,40 +254,19 @@ func (v *MCPValidator) validateUpstream(fieldPrefix string, upstream *api.MCPPro } } - // Validate auth if present + // Validate auth if present. Shared with LlmProvider/LlmProxy - see + // validateUpstreamAuthFields in llm_validator.go. if upstream.Auth != nil { auth := upstream.Auth - // Validate 'type' - if auth.Type == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.type", fieldPrefix), - Message: "Auth type is required", - }) - } - - if auth.Header == nil || *auth.Header == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.header", fieldPrefix), - Message: "Auth header is required", - }) - } - if auth.Value == nil || *auth.Value == "" { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.value", fieldPrefix), - Message: "Auth value is required", - }) - } - - if auth.Type == api.MCPProxyConfigDataUpstreamAuthType("bearer") { - // For Bearer token, value should start with "Bearer or "bearer " - if auth.Value != nil && - !strings.HasPrefix(*auth.Value, "Bearer ") && !strings.HasPrefix(*auth.Value, "bearer ") { - errors = append(errors, ValidationError{ - Field: fmt.Sprintf("%s.auth.value", fieldPrefix), - Message: "Bearer token value must start with 'Bearer ' or 'bearer '", - }) - } + fields := upstreamAuthFields{ + authType: string(auth.Type), + header: auth.Header, + value: auth.Value, + policyName: auth.PolicyName, + policyVersion: auth.PolicyVersion, + policyParams: auth.PolicyParams, } + errors = append(errors, validateUpstreamAuthFields(fieldPrefix+".auth", fields)...) } return errors diff --git a/gateway/gateway-controller/pkg/config/mcp_validator_test.go b/gateway/gateway-controller/pkg/config/mcp_validator_test.go index 190c80e87d..3920e6ec58 100644 --- a/gateway/gateway-controller/pkg/config/mcp_validator_test.go +++ b/gateway/gateway-controller/pkg/config/mcp_validator_test.go @@ -482,9 +482,12 @@ func TestMCPValidator_ValidateUpstreamAuth(t *testing.T) { // Define auth struct type locally to match the anonymous struct in api package type authConfig struct { - Type api.MCPProxyConfigDataUpstreamAuthType - Header *string - Value *string + Type api.MCPProxyConfigDataUpstreamAuthType + Header *string + Value *string + PolicyName *string + PolicyVersion *string + PolicyParams *map[string]interface{} } tests := []struct { @@ -503,23 +506,19 @@ func TestMCPValidator_ValidateUpstreamAuth(t *testing.T) { wantError: false, }, { - name: "Valid bearer auth", + // "bearer" is not a supported auth type - only 'api-key', + // 'oauth2', and 'other' are declared in the UpstreamAuth OpenAPI + // schema. A bearer token is just an api-key auth with header + // "Authorization" and value "Bearer " (see "Valid API key + // auth" above). + name: "Unsupported auth type is rejected", auth: &authConfig{ Type: api.MCPProxyConfigDataUpstreamAuthType("bearer"), Header: stringPtr("Authorization"), Value: stringPtr("Bearer token123"), }, - wantError: false, - }, - { - name: "Bearer auth without Bearer prefix", - auth: &authConfig{ - Type: api.MCPProxyConfigDataUpstreamAuthType("bearer"), - Header: stringPtr("Authorization"), - Value: stringPtr("token123"), - }, wantError: true, - errField: "spec.upstream.auth.value", + errField: "spec.upstream.auth.type", }, { name: "Missing auth type", @@ -551,6 +550,53 @@ func TestMCPValidator_ValidateUpstreamAuth(t *testing.T) { wantError: true, errField: "spec.upstream.auth.value", }, + { + name: "Valid oauth2 auth via policyParams", + auth: &authConfig{ + Type: api.MCPProxyConfigDataUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": "https://idp.example.com/oauth2/token", + "clientId": "client-id", + "clientSecret": "client-secret", + }, + }, + wantError: false, + }, + { + name: "oauth2 without policyParams", + auth: &authConfig{ + Type: api.MCPProxyConfigDataUpstreamAuthTypeOauth2, + }, + wantError: true, + errField: "spec.upstream.auth.policyParams", + }, + { + name: "other without policyName", + auth: &authConfig{ + Type: api.MCPProxyConfigDataUpstreamAuthTypeOther, + PolicyParams: &map[string]interface{}{"foo": "bar"}, + }, + wantError: true, + errField: "spec.upstream.auth.policyName", + }, + { + name: "other without policyParams", + auth: &authConfig{ + Type: api.MCPProxyConfigDataUpstreamAuthTypeOther, + PolicyName: stringPtr("my-custom-auth-policy"), + }, + wantError: true, + errField: "spec.upstream.auth.policyParams", + }, + { + name: "Valid other auth", + auth: &authConfig{ + Type: api.MCPProxyConfigDataUpstreamAuthTypeOther, + PolicyName: stringPtr("my-custom-auth-policy"), + PolicyParams: &map[string]interface{}{"foo": "bar"}, + }, + wantError: false, + }, } for _, tt := range tests { @@ -561,13 +607,19 @@ func TestMCPValidator_ValidateUpstreamAuth(t *testing.T) { } if tt.auth != nil { upstream.Auth = &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ - Type: tt.auth.Type, - Header: tt.auth.Header, - Value: tt.auth.Value, + Type: tt.auth.Type, + Header: tt.auth.Header, + Value: tt.auth.Value, + PolicyName: tt.auth.PolicyName, + PolicyVersion: tt.auth.PolicyVersion, + PolicyParams: tt.auth.PolicyParams, } } config := &api.MCPProxyConfiguration{ diff --git a/gateway/gateway-controller/pkg/constants/constants.go b/gateway/gateway-controller/pkg/constants/constants.go index 9b6967e527..cd3a111b57 100644 --- a/gateway/gateway-controller/pkg/constants/constants.go +++ b/gateway/gateway-controller/pkg/constants/constants.go @@ -102,10 +102,30 @@ const ( ExtProcHeaderModeSkip = "SKIP" ExtProcRequestAttributeRouteName = "xds.route_name" + // UpstreamCodecFilterName is Envoy's built-in terminal filter. Any cluster's + // upstream HTTP filter chain (TypedExtensionProtocolOptions.HttpFilters) that + // is non-empty MUST end with this filter, or Envoy rejects the config at + // listener/cluster warming time. + UpstreamCodecFilterName = "envoy.filters.http.upstream_codec" + // Policy Engine PolicyEngineClusterName = "api-platform/policy-engine" DefaultPolicyEngineSocketPath = "/var/run/api-platform/policy-engine.sock" + // UpstreamRefreshPolicyEngineClusterName is the internal Envoy cluster + // pointing at policy-engine's second, upstream-attempt ext_proc endpoint + // (see gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go). + // Attached only to clusters backing at least one resilience.retry-configured + // route (see xds.TranslateConfigs). + UpstreamRefreshPolicyEngineClusterName = "policy_engine_upstream_refresh_cluster" + + // DefaultUpstreamExtProcSocketPath must match, byte-for-byte, policy-engine's + // own copy of this literal in its internal/constants/constants.go + // (DefaultUpstreamExtProcSocketPath) — gateway-controller and policy-engine + // are separate Go modules, so the path cannot be shared via import and is + // duplicated here deliberately, mirroring DefaultPolicyEngineSocketPath above. + DefaultUpstreamExtProcSocketPath = "/var/run/api-platform/policy-engine-upstream.sock" + // GatewayHealthPathPrefix is reserved for the gateway's own readiness/liveness // direct-response routes (see GatewayReadyPath/GatewayHealthyPath). No API, // LLMProvider, or LLMProxy resource may register a path under this prefix — @@ -160,6 +180,7 @@ const ( " headers:\n" + " - name: '%s'\n" + " value: '%s'\n" + UPSTREAM_AUTH_OAUTH2_POLICY_NAME = "oauth2-generator" PROXY_HOST__HEADER_POLICY_NAME = "host-rewrite" PROXY_HOST__HEADER_POLICY_PARAMS = "host: '%s'\n" diff --git a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go index 04af1fce42..f82f759f54 100644 --- a/gateway/gateway-controller/pkg/models/runtime_deploy_config.go +++ b/gateway/gateway-controller/pkg/models/runtime_deploy_config.go @@ -21,6 +21,7 @@ package models import ( "time" + api "github.com/wso2/api-platform/gateway/gateway-controller/pkg/api/management" policyenginev1 "github.com/wso2/api-platform/sdk/core/policyengine" ) @@ -79,14 +80,16 @@ type Route struct { Order int } -// RouteTimeout holds parsed timeout values for a route. -// Timeout and IdleTimeout come from the resilience block (operation-level overriding -// API-level). A nil field means "not configured" — the global route timeout default -// applies. A non-nil zero value means "explicitly disabled". +// RouteTimeout holds parsed timeout and retry values for a route. +// Timeout, IdleTimeout, and Retry all come from the resilience block (operation-level +// overriding API-level, per field). A nil Timeout/IdleTimeout means "not configured" — +// the global route timeout default applies. A non-nil zero value means "explicitly +// disabled". A nil Retry means no native RouteAction.RetryPolicy is emitted. type RouteTimeout struct { Connect *time.Duration Timeout *time.Duration // route timeout -> RouteAction.Timeout IdleTimeout *time.Duration // route idle timeout -> RouteAction.IdleTimeout + Retry *api.Retry // route retry -> RouteAction.RetryPolicy; nil when not configured } // RouteUpstream links a route to its upstream cluster. diff --git a/gateway/gateway-controller/pkg/transform/restapi.go b/gateway/gateway-controller/pkg/transform/restapi.go index 953f053802..ebfd221c7b 100644 --- a/gateway/gateway-controller/pkg/transform/restapi.go +++ b/gateway/gateway-controller/pkg/transform/restapi.go @@ -172,7 +172,7 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim } // Resolve API-level resilience timeouts once; operation-level values override these. - apiTimeout, apiIdleTimeout, err := xds.ResolveResilience(apiData.Resilience) + apiTimeout, apiIdleTimeout, apiRetry, err := xds.ResolveResilience(apiData.Resilience) if err != nil { return nil, fmt.Errorf("invalid API-level resilience: %w", err) } @@ -181,11 +181,11 @@ func (t *RestAPITransformer) Transform(cfg *models.StoredConfig) (*models.Runtim for i, op := range apiData.Operations { // Operation-level resilience overrides API-level (per field); nil leaves the // global route timeout default in effect. - opTimeout, opIdleTimeout, err := xds.ResolveResilience(op.Resilience) + opTimeout, opIdleTimeout, opRetry, err := xds.ResolveResilience(op.Resilience) if err != nil { return nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.EffectiveMethod(), op.EffectivePath(), err) } - routeTimeout := buildRouteTimeout(opTimeout, apiTimeout, opIdleTimeout, apiIdleTimeout) + routeTimeout := buildRouteTimeout(opTimeout, apiTimeout, opIdleTimeout, apiIdleTimeout, opRetry, apiRetry) vhosts := append([]string{}, mainVhosts...) if hasSandbox { @@ -381,9 +381,9 @@ func routeHeaderMatches(op api.Operation) []models.RouteHeaderMatch { // collectAPIPolicies validates and collects API-level policies into SDK format. // buildRouteTimeout applies operation-over-API precedence (per field) and returns a -// *models.RouteTimeout, or nil when neither level configured any timeout (so the global -// route timeout default applies). -func buildRouteTimeout(opTimeout, apiTimeout, opIdle, apiIdle *time.Duration) *models.RouteTimeout { +// *models.RouteTimeout, or nil when neither level configured any timeout/retry (so the +// global route timeout default applies and no RetryPolicy is emitted). +func buildRouteTimeout(opTimeout, apiTimeout, opIdle, apiIdle *time.Duration, opRetry, apiRetry *api.Retry) *models.RouteTimeout { timeout := opTimeout if timeout == nil { timeout = apiTimeout @@ -392,10 +392,14 @@ func buildRouteTimeout(opTimeout, apiTimeout, opIdle, apiIdle *time.Duration) *m if idle == nil { idle = apiIdle } - if timeout == nil && idle == nil { + retry := opRetry + if retry == nil { + retry = apiRetry + } + if timeout == nil && idle == nil && retry == nil { return nil } - return &models.RouteTimeout{Timeout: timeout, IdleTimeout: idle} + return &models.RouteTimeout{Timeout: timeout, IdleTimeout: idle, Retry: retry} } // collectAPIPolicies returns the resolved API-level policies as a slice in spec order. diff --git a/gateway/gateway-controller/pkg/utils/commonutils.go b/gateway/gateway-controller/pkg/utils/commonutils.go index d71137e674..b3edce6fea 100644 --- a/gateway/gateway-controller/pkg/utils/commonutils.go +++ b/gateway/gateway-controller/pkg/utils/commonutils.go @@ -53,6 +53,24 @@ func GetParamsOfPolicy(policyDef string, params ...string) (map[string]any, erro return m, nil } +// resolveUpstreamAuthPolicyName returns the caller-supplied policyName override +// if non-empty, otherwise defaultName. +func resolveUpstreamAuthPolicyName(policyName *string, defaultName string) string { + if policyName != nil && strings.TrimSpace(*policyName) != "" { + return strings.TrimSpace(*policyName) + } + return defaultName +} + +// resolveUpstreamAuthPolicyParams returns policyParams verbatim if supplied, +// otherwise falls back to buildLegacyParams (the deprecated header/value path). +func resolveUpstreamAuthPolicyParams(policyParams *map[string]interface{}, buildLegacyParams func() (map[string]interface{}, error)) (map[string]interface{}, error) { + if policyParams != nil { + return *policyParams, nil + } + return buildLegacyParams() +} + // APIKeyETag produces a deterministic UUID v7-formatted ETag from the unique // (artifactUUID, name, updatedAt) tuple. Uses SHA-256 of the tuple as the source // bytes, then stamps version=7 and RFC 4122 variant bits. diff --git a/gateway/gateway-controller/pkg/utils/credential_inheritance_test.go b/gateway/gateway-controller/pkg/utils/credential_inheritance_test.go index 37506e7b2e..b264504a9c 100644 --- a/gateway/gateway-controller/pkg/utils/credential_inheritance_test.go +++ b/gateway/gateway-controller/pkg/utils/credential_inheritance_test.go @@ -41,9 +41,12 @@ func storedProvider() api.LLMProviderConfiguration { var cfg api.LLMProviderConfiguration cfg.Spec.Upstream.Url = sp("https://api.openai.com/v1") cfg.Spec.Upstream.Auth = &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{Header: sp("Authorization"), Type: "api-key", Value: sp(storedCred)} return cfg } @@ -278,9 +281,12 @@ func TestInheritMCPProxyCredential(t *testing.T) { stored := func() api.MCPProxyConfiguration { var cfg api.MCPProxyConfiguration cfg.Spec.Upstream.Auth = &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{Header: sp("Authorization"), Type: "api-key", Value: sp(storedCred)} return cfg } diff --git a/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go b/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go index 4307f05adc..d1133c4b5c 100644 --- a/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go +++ b/gateway/gateway-controller/pkg/utils/llm_provider_transformer_test.go @@ -190,9 +190,12 @@ func TestTransform_FullProvider(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.openai.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -477,9 +480,12 @@ func TestTransform_ApiKeyAuth(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("X-API-Key"), @@ -524,14 +530,24 @@ func TestTransform_ApiKeyAuth(t *testing.T) { } // TestTransform_OtherAndNoneAuth verifies that "other" and "none" upstream auth -// types transform successfully but attach no upstream auth policy - for "other" -// authentication is handled by user-attached policies, for "none" there is none. +// types transform successfully but attach no *built-in* upstream auth policy - +// for "other" authentication is handled by a user-named policy (policyName is +// mandatory), for "none" there is no auth policy at all. func TestTransform_OtherAndNoneAuth(t *testing.T) { - for _, authType := range []api.LLMProviderConfigDataUpstreamAuthType{ - api.LLMProviderConfigDataUpstreamAuthTypeOther, - api.LLMProviderConfigDataUpstreamAuthTypeNone, - } { - t.Run(string(authType), func(t *testing.T) { + tests := []struct { + authType api.LLMProviderConfigDataUpstreamAuthType + policyName *string + policyParams *map[string]interface{} + }{ + { + authType: api.LLMProviderConfigDataUpstreamAuthTypeOther, + policyName: stringPtr(testCustomAuthPolicyName), + policyParams: &map[string]interface{}{"foo": "bar"}, + }, + {authType: api.LLMProviderConfigDataUpstreamAuthTypeNone}, + } + for _, tc := range tests { + t.Run(string(tc.authType), func(t *testing.T) { transformer, _ := setupTestTransformer(t) provider := &api.LLMProviderConfiguration{ @@ -545,11 +561,16 @@ func TestTransform_OtherAndNoneAuth(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ - Type: authType, + Type: tc.authType, + PolicyName: tc.policyName, + PolicyParams: tc.policyParams, }, }, AccessControl: api.LLMAccessControl{ @@ -562,14 +583,15 @@ func TestTransform_OtherAndNoneAuth(t *testing.T) { result, err := transformer.Transform(provider, output) require.NoError(t, err) - // No upstream auth policy should be attached to any operation. + // No built-in upstream auth (api-key/set-headers) policy should be + // attached to any operation. for _, op := range result.Spec.Operations { if op.Policies == nil { continue } for _, pol := range *op.Policies { assert.NotEqual(t, constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, pol.Name, - "auth type %q should not attach an upstream auth policy", authType) + "auth type %q should not attach the built-in upstream auth policy", tc.authType) } } }) @@ -590,9 +612,12 @@ func TestTransform_UnsupportedAuthType(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: "bearer", // Unsupported type Header: stringPtr("Authorization"), @@ -1715,9 +1740,12 @@ func TestTransform_AuthWithAllowAll(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -2259,9 +2287,12 @@ func TestTransform_UpstreamAuth_Plus_APILevelPolicy_AllowAll(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -2348,9 +2379,12 @@ func TestTransform_UpstreamAuth_Plus_APILevelPolicy_DenyAll(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("X-API-Key"), @@ -3506,9 +3540,12 @@ func TestTransform_Auth_Plus_APILevel_Plus_OperationLevel_AllowAll(t *testing.T) Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -3636,9 +3673,12 @@ func TestTransform_Auth_Plus_APILevel_Plus_OperationLevel_DenyAll(t *testing.T) Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("X-API-Key"), @@ -3973,9 +4013,12 @@ func TestTransform_AllPolicyTypes_WildcardExceptions_WildcardOperations_AllowAll Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -4203,9 +4246,12 @@ func TestTransform_AllPolicyTypes_WildcardExceptions_WildcardOperations_DenyAll( Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.example.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("X-API-Key"), @@ -5425,9 +5471,12 @@ func TestTransform_ComplexCombined_MaximumComplexity_AllowAll(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.openai.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), @@ -5734,9 +5783,12 @@ func TestTransform_ComplexCombined_MaximumComplexity_DenyAll(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: stringPtr("https://api.openai.com"), Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: stringPtr("Authorization"), diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer.go b/gateway/gateway-controller/pkg/utils/llm_transformer.go index 9f0178bf5e..0ae9eaf194 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer.go @@ -94,6 +94,50 @@ func (t *LLMProviderTransformer) resolvePolicyVersion(name string) (string, erro return t.policyVersionResolver.Resolve(name) } +// resolvePolicyVersionOverride resolves name's version, honoring an optional +// caller-requested override (upstream.auth.policyVersion). The override must +// match what resolvePolicyVersion(name) returns, or resolution fails - only +// one version per policy name is ever loaded into a gateway image. +func (t *LLMProviderTransformer) resolvePolicyVersionOverride(name string, override *string) (string, error) { + resolved, err := t.resolvePolicyVersion(name) + if err != nil { + return "", err + } + if override == nil { + return resolved, nil + } + trimmed := strings.TrimSpace(*override) + if trimmed == "" || trimmed == resolved { + return resolved, nil + } + return "", fmt.Errorf("policy '%s' version '%s' was requested, but this gateway build only has '%s' loaded", name, trimmed, resolved) +} + +// buildLegacyOrGenericPolicy builds the api.Policy to attach for an +// upstream.auth of type api-key, honoring the policyName/policyVersion/ +// policyParams overrides with buildLegacyParams as the deprecated +// header/value fallback. +func (t *LLMProviderTransformer) buildLegacyOrGenericPolicy( + defaultPolicyName string, + policyName, policyVersion *string, + policyParams *map[string]interface{}, + buildLegacyParams func() (map[string]interface{}, error), +) (*api.Policy, error) { + name := resolveUpstreamAuthPolicyName(policyName, defaultPolicyName) + + params, err := resolveUpstreamAuthPolicyParams(policyParams, buildLegacyParams) + if err != nil { + return nil, err + } + + version, err := t.resolvePolicyVersionOverride(name, policyVersion) + if err != nil { + return nil, err + } + + return &api.Policy{Name: name, Version: version, Params: ¶ms}, nil +} + func (t *LLMProviderTransformer) getTemplateByHandle(handle string) (*models.StoredLLMProviderTemplate, error) { return t.db.GetLLMProviderTemplateByHandle(handle) } @@ -458,28 +502,56 @@ func (t *LLMProviderTransformer) transformProvider(provider *api.LLMProviderConf upstream := provider.Spec.Upstream var upstreamAuthPolicy *api.Policy if upstream.Auth != nil { - switch upstream.Auth.Type { + auth := upstream.Auth + switch auth.Type { case api.LLMProviderConfigDataUpstreamAuthTypeApiKey: - // Add API Key auth policy at API level - params, err := GetUpstreamAuthApikeyPolicyParams(*upstream.Auth.Header, *upstream.Auth.Value) + pol, err := t.buildLegacyOrGenericPolicy( + constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, + auth.PolicyName, auth.PolicyVersion, auth.PolicyParams, + func() (map[string]interface{}, error) { + if auth.Header == nil || *auth.Header == "" { + return nil, fmt.Errorf("upstream.auth.header is required") + } + if auth.Value == nil || *auth.Value == "" { + return nil, fmt.Errorf("upstream.auth.value is required") + } + return GetUpstreamAuthApikeyPolicyParams(*auth.Header, *auth.Value) + }, + ) if err != nil { - return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + return nil, err + } + upstreamAuthPolicy = pol + case api.LLMProviderConfigDataUpstreamAuthTypeOauth2: + // No typed-field fallback for oauth2 - policyParams is always required. + if auth.PolicyParams == nil { + return nil, fmt.Errorf("upstream.auth.policyParams is required when type is 'oauth2'") + } + name := resolveUpstreamAuthPolicyName(auth.PolicyName, constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME) + version, err := t.resolvePolicyVersionOverride(name, auth.PolicyVersion) + if err != nil { + return nil, err + } + upstreamAuthPolicy = &api.Policy{Name: name, Version: version, Params: auth.PolicyParams} + case api.LLMProviderConfigDataUpstreamAuthTypeOther: + if auth.PolicyName == nil || strings.TrimSpace(*auth.PolicyName) == "" { + return nil, fmt.Errorf("upstream.auth.policyName is required when type is 'other'") + } + if auth.PolicyParams == nil { + return nil, fmt.Errorf("upstream.auth.policyParams is required when type is 'other'") } - policyVersion, err := t.resolvePolicyVersion(constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME) + name := strings.TrimSpace(*auth.PolicyName) + version, err := t.resolvePolicyVersionOverride(name, auth.PolicyVersion) if err != nil { return nil, err } - mh := api.Policy{ - Name: constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, - Version: policyVersion, Params: ¶ms} - upstreamAuthPolicy = &mh - case api.LLMProviderConfigDataUpstreamAuthTypeOther, - api.LLMProviderConfigDataUpstreamAuthTypeNone: - // "other": auth handled entirely by user-attached policies. - // "none": no upstream authentication. In both cases the gateway - // attaches no auth policy of its own. + upstreamAuthPolicy = &api.Policy{Name: name, Version: version, Params: auth.PolicyParams} + case api.LLMProviderConfigDataUpstreamAuthTypeNone: + // No upstream authentication - the gateway attaches no auth policy + // of its own; auth (if any) is handled entirely by user-attached + // policies elsewhere in the chain. default: - return nil, fmt.Errorf("unsupported upstream auth type: %s", upstream.Auth.Type) + return nil, fmt.Errorf("unsupported upstream auth type: %s", auth.Type) } } @@ -821,41 +893,64 @@ func apiKeyAuthValuePrefix(globalPolicies *[]api.Policy) string { return "" } +// proxyUpstreamAuthPolicy builds the api.Policy to attach for an LlmProxy +// provider/additionalProviders auth config. valuePrefix is the api-key value +// prefix (e.g. "Bearer") configured on the provider's own downstream +// api-key-auth policy, so the loopback hop injects its credential the same way. func (t *LLMProviderTransformer) proxyUpstreamAuthPolicy(auth *api.LLMUpstreamAuth, valuePrefix, field string) (*api.Policy, error) { if auth == nil { return nil, nil } switch auth.Type { case api.LLMUpstreamAuthTypeApiKey: - if auth.Header == nil || *auth.Header == "" { - return nil, fmt.Errorf("%s.header is required", field) - } - if auth.Value == nil || *auth.Value == "" { - return nil, fmt.Errorf("%s.value is required", field) + return t.buildLegacyOrGenericPolicy( + constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, + auth.PolicyName, auth.PolicyVersion, auth.PolicyParams, + func() (map[string]interface{}, error) { + if auth.Header == nil || *auth.Header == "" { + return nil, fmt.Errorf("%s.header is required", field) + } + if auth.Value == nil || *auth.Value == "" { + return nil, fmt.Errorf("%s.value is required", field) + } + // The loopback hop re-enters the provider's own api-key-auth. When + // that policy declares a valuePrefix (e.g. "Bearer"), the injected + // credential must be prefixed the same way — a single space + // separator matches how the provider strips it. + value := *auth.Value + if valuePrefix != "" { + value = valuePrefix + " " + value + } + return GetUpstreamAuthApikeyPolicyParams(*auth.Header, value) + }, + ) + case api.LLMUpstreamAuthTypeOauth2: + // No typed-field fallback for oauth2 - policyParams is always required. + if auth.PolicyParams == nil { + return nil, fmt.Errorf("%s.policyParams is required when type is 'oauth2'", field) + } + name := resolveUpstreamAuthPolicyName(auth.PolicyName, constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME) + version, err := t.resolvePolicyVersionOverride(name, auth.PolicyVersion) + if err != nil { + return nil, err } - // The loopback hop re-enters the provider's own api-key-auth. When that policy - // declares a valuePrefix (e.g. "Bearer"), the injected credential must be prefixed - // the same way — a single space separator matches how the provider strips it. - value := *auth.Value - if valuePrefix != "" { - value = valuePrefix + " " + value + return &api.Policy{Name: name, Version: version, Params: auth.PolicyParams}, nil + case api.LLMUpstreamAuthTypeOther: + if auth.PolicyName == nil || strings.TrimSpace(*auth.PolicyName) == "" { + return nil, fmt.Errorf("%s.policyName is required when type is 'other'", field) } - params, err := GetUpstreamAuthApikeyPolicyParams(*auth.Header, value) - if err != nil { - return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + if auth.PolicyParams == nil { + return nil, fmt.Errorf("%s.policyParams is required when type is 'other'", field) } - policyVersion, err := t.resolvePolicyVersion(constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME) + name := strings.TrimSpace(*auth.PolicyName) + version, err := t.resolvePolicyVersionOverride(name, auth.PolicyVersion) if err != nil { return nil, err } - return &api.Policy{ - Name: constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, - Version: policyVersion, - Params: ¶ms, - }, nil - case api.LLMUpstreamAuthTypeOther, api.LLMUpstreamAuthTypeNone: - // "other": auth handled entirely by user-attached policies. - // "none": no upstream authentication. No auth policy is attached. + return &api.Policy{Name: name, Version: version, Params: auth.PolicyParams}, nil + case api.LLMUpstreamAuthTypeNone: + // No upstream authentication - no auth policy is attached; auth (if + // any) is handled entirely by user-attached policies elsewhere. return nil, nil default: return nil, fmt.Errorf("unsupported upstream auth type: %s", auth.Type) diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go index 722c551435..3d70529f56 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer_multiprovider_test.go @@ -418,6 +418,160 @@ func TestLLMProviderTransformer_TransformProxy_LoopbackAuthCarriesProviderValueP assert.Equal(t, `Bearer {{ secret "sec-1" }}`, firstRequestHeaderValue(t, authPolicy.Params)) } +// TestLLMProviderTransformer_TransformProxy_AdditionalProviderOAuth2AuthIsIsolated +// is the transformer-side half of the regression coverage for the +// cross-provider Redis token cache collision bug (see +// gateway/spec/prds/oauth2-upstream-auth.md and +// oauth2ConfigDiscriminator in gateway/dev-policies/oauth2-generator/token_cache.go). +// +// The runtime fix (keying the oauth2 policy's cache by its own config +// instead of by API identity) is verified in isolation by +// token_cache_test.go's TestRedisCachingTokenSource_DifferentConfigs_ +// GetIsolatedCacheEntries, which feeds two distinct oauth2Params directly to +// the cache. That test can't by itself prove the transformer actually +// produces two distinct params blocks for this scenario in the first place - +// this test closes that gap: a single LlmProxy's primary provider and its +// one additionalProviders entry, each with independent oauth2 credentials, +// must be emitted as two separate oauth2 Policy attachments carrying +// different clientId/tokenEndpoint/clientSecret values (gated by different +// ExecutionCondition), not a single shared one - anything else would mean +// there's only one set of params for the runtime cache key to isolate by, +// silently reintroducing the collision regardless of how correct the +// runtime-side keying is. +func TestLLMProviderTransformer_TransformProxy_AdditionalProviderOAuth2AuthIsIsolated(t *testing.T) { + store := storage.NewConfigStore() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + db := newTestSQLiteStorage(t, logger) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-db-template-id-0000-000000000004", + Configuration: api.LLMProviderTemplate{ + ApiVersion: api.LLMProviderTemplateApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderTemplateKindLlmProviderTemplate, + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{DisplayName: "openai"}, + }, + } + require.NoError(t, db.SaveLLMProviderTemplate(template)) + + saveProvider := func(name, context string) { + providerSourceConfig := api.LLMProviderConfiguration{ + ApiVersion: api.LLMProviderConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProviderConfigurationKindLlmProvider, + Metadata: api.Metadata{Name: name}, + Spec: api.LLMProviderConfigData{ + DisplayName: name, + Version: "v1.0", + Context: stringPtr(context), + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: stringPtr("https://example.com")}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + require.NoError(t, db.SaveConfig(&models.StoredConfig{ + UUID: name + "-uuid", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: name, + DisplayName: name, + Version: "v1.0", + SourceConfiguration: providerSourceConfig, + DesiredState: models.StateDeployed, + })) + } + saveProvider("provider-a", "/provider-a") + saveProvider("provider-b", "/provider-b") + + transformer := NewLLMProviderTransformer(store, db, &config.RouterConfig{ListenerPort: 8080}, newTestPolicyVersionResolver()) + + // Deliberately give provider-b a DIFFERENT clientId, tokenEndpoint AND + // clientSecret than provider-a - not just a different name - so this + // locks in isolation on every field the cache key discriminates by, not + // just one. + proxy := &api.LLMProxyConfiguration{ + ApiVersion: api.LLMProxyConfigurationApiVersionGatewayApiPlatformWso2Comv1, + Kind: api.LLMProxyConfigurationKindLlmProxy, + Metadata: api.Metadata{Name: "oauth2-multi"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "oauth2-multi", + Version: "v1.0", + Provider: api.LLMProxyProvider{ + Id: "provider-a", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": "https://idp-a.example.com/token", + "clientId": "client-a", + "clientSecret": "secret-a", + }, + }, + }, + AdditionalProviders: &[]api.LLMProxyAdditionalProvider{{ + Id: "provider-b", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": "https://idp-b.example.com/token", + "clientId": "client-b", + "clientSecret": "secret-b", + }, + }, + }}, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + + // No operationPolicies/policies are attached in this proxy spec, so the + // only operations the transformer generates are the wildcard catch-all + // routes (one per HTTP method) - the proxy's upstream-auth policies are + // attached to every one of those (see transformProxy's final loop over + // ops), so any POST operation carries both oauth2 attachments. + var postOp *api.Operation + for i := range result.Spec.Operations { + if result.Spec.Operations[i].Method != nil && *result.Spec.Operations[i].Method == api.OperationMethod("POST") { + postOp = &result.Spec.Operations[i] + break + } + } + require.NotNil(t, postOp) + require.NotNil(t, postOp.Policies) + + var oauth2Policies []api.Policy + for _, pol := range *postOp.Policies { + if pol.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + oauth2Policies = append(oauth2Policies, pol) + } + } + // Two separate oauth2 policy attachments on the SAME operation - this is + // exactly the shape that collided under the old API-identity-keyed + // cache: one route, two independent oauth2 configs. + require.Len(t, oauth2Policies, 2) + require.NotNil(t, oauth2Policies[0].ExecutionCondition) + require.NotNil(t, oauth2Policies[1].ExecutionCondition) + assert.Contains(t, *oauth2Policies[0].ExecutionCondition, "provider-a") + assert.Contains(t, *oauth2Policies[1].ExecutionCondition, "provider-b") + + require.NotNil(t, oauth2Policies[0].Params) + require.NotNil(t, oauth2Policies[1].Params) + paramsA := *oauth2Policies[0].Params + paramsB := *oauth2Policies[1].Params + + // Every field the runtime cache key discriminates by (see + // oauth2ConfigDiscriminator) must actually differ here - if any of these + // silently matched, the two would collide on the same Redis key + // regardless of how correct the runtime-side keying logic is. + assert.NotEqual(t, paramsA["clientId"], paramsB["clientId"]) + assert.NotEqual(t, paramsA["tokenEndpoint"], paramsB["tokenEndpoint"]) + assert.NotEqual(t, paramsA["clientSecret"], paramsB["clientSecret"]) + assert.Equal(t, "client-a", paramsA["clientId"]) + assert.Equal(t, "client-b", paramsB["clientId"]) + assert.Equal(t, "https://idp-a.example.com/token", paramsA["tokenEndpoint"]) + assert.Equal(t, "https://idp-b.example.com/token", paramsB["tokenEndpoint"]) + assert.Equal(t, "secret-a", paramsA["clientSecret"]) + assert.Equal(t, "secret-b", paramsB["clientSecret"]) +} + func firstRequestHeaderValue(t *testing.T, params *map[string]interface{}) string { t.Helper() require.NotNil(t, params) diff --git a/gateway/gateway-controller/pkg/utils/llm_transformer_test.go b/gateway/gateway-controller/pkg/utils/llm_transformer_test.go index 94e3e55bdf..f5365cca7c 100644 --- a/gateway/gateway-controller/pkg/utils/llm_transformer_test.go +++ b/gateway/gateway-controller/pkg/utils/llm_transformer_test.go @@ -2019,9 +2019,12 @@ func TestTransformProvider_WithUpstreamAuth(t *testing.T) { Upstream: api.LLMProviderConfigData_Upstream{ Url: &upstreamURL, Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, Header: &authHeader, @@ -2054,6 +2057,439 @@ func TestTransformProvider_WithUpstreamAuth(t *testing.T) { } } +// TestTransformProvider_ApiKeyWithPolicyParams covers type: api-key configured via +// the generic policyParams bucket instead of the deprecated header/value fields - +// the headline migration this CRD change exists for, previously untested at the +// transformer level (every other api-key test here uses header/value). +func TestTransformProvider_ApiKeyWithPolicyParams(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-apikey-policyparams"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (api-key via policyParams)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeApiKey, + // set-headers' own native param shape - policyParams for api-key is + // forwarded verbatim, with no header/value defaulting at all. + PolicyParams: &map[string]interface{}{ + "request": map[string]interface{}{ + "headers": []interface{}{ + map[string]interface{}{"name": "X-Api-Key", "value": "sk-from-policyparams"}, + }, + }, + }, + }, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(provider, output) + assert.NoError(t, err) + assert.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + var found *api.Policy + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME { + found = &p + break + } + } + require.NotNil(t, found, "operation %s %s should include the set-headers policy", op.EffectiveMethod(), op.EffectivePath()) + require.NotNil(t, found.Params) + // The configured policyParams must reach set-headers unchanged - not the + // GetUpstreamAuthApikeyPolicyParams-rendered shape header/value would produce. + request, ok := (*found.Params)["request"].(map[string]interface{}) + require.True(t, ok, "expected policyParams.request to survive verbatim, got %+v", *found.Params) + headers, ok := request["headers"].([]interface{}) + require.True(t, ok) + require.Len(t, headers, 1) + entry := headers[0].(map[string]interface{}) + assert.Equal(t, "X-Api-Key", entry["name"]) + assert.Equal(t, "sk-from-policyparams", entry["value"]) + } +} + +func TestTransformProvider_WithOAuth2UpstreamAuth(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ + ListenerPort: 8080, + } + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + tokenEndpoint := "https://idp.example.com/oauth2/token" + clientID := "gateway-client" + clientSecret := "s3cr3t" + purgeStatusCodes := []int{401, 403} + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (OAuth2)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": tokenEndpoint, + "clientId": clientID, + "clientSecret": clientSecret, + "tokenPurgeStatusCodes": purgeStatusCodes, + }, + }, + }, + AccessControl: api.LLMAccessControl{ + Mode: api.AllowAll, + }, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(provider, output) + assert.NoError(t, err) + assert.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + found := false + var foundParams *map[string]interface{} + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + found = true + foundParams = p.Params + break + } + } + assert.True(t, found, "operation %s %s should include the oauth2 policy", op.EffectiveMethod(), op.EffectivePath()) + require.NotNil(t, foundParams) + // policyParams is forwarded verbatim - no CRD-level defaulting of + // grantType/clientAuthMethod anymore, that's the oauth2-generator + // policy's own responsibility now (see its GetPolicy). + assert.Equal(t, tokenEndpoint, (*foundParams)["tokenEndpoint"]) + assert.Equal(t, clientID, (*foundParams)["clientId"]) + assert.Equal(t, clientSecret, (*foundParams)["clientSecret"]) + assert.Equal(t, purgeStatusCodes, (*foundParams)["tokenPurgeStatusCodes"], "policyParams should reach the policy unchanged") + } +} + +// TestTransformProvider_PolicyVersionOverride covers resolvePolicyVersionOverride's +// two branches beyond the always-omitted-override default every other test here +// uses: a matching pin succeeds, and a pin that doesn't match what's actually +// loaded (testOAuth2AuthenticationVersion, "v9.9.7") fails loudly instead of +// silently resolving to the wrong version. +func TestTransformProvider_PolicyVersionOverride(t *testing.T) { + newProvider := func(policyVersion *string) *api.LLMProviderConfiguration { + upstreamURL := "https://api.openai.com" + return &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-pinned"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (pinned policyVersion)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{"tokenEndpoint": "https://idp.example.com/oauth2/token"}, + PolicyVersion: policyVersion, + }, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + } + + newTransformer := func(t *testing.T) *LLMProviderTransformer { + store := storage.NewConfigStore() + db := newTestMockDB() + transformer := NewLLMProviderTransformer(store, db, &config.RouterConfig{ListenerPort: 8080}, newTestPolicyVersionResolver()) + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{Metadata: api.Metadata{Name: "openai"}, Spec: api.LLMProviderTemplateData{}}, + } + db.SaveLLMProviderTemplate(template) + require.NoError(t, store.AddTemplate(template)) + return transformer + } + + t.Run("matching pin succeeds", func(t *testing.T) { + transformer := newTransformer(t) + result, err := transformer.Transform(newProvider(stringPtr("v9.9.7")), &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + assert.Equal(t, "v9.9.7", p.Version) + } + } + } + }) + + t.Run("mismatched pin fails loudly instead of silently using the loaded version", func(t *testing.T) { + transformer := newTransformer(t) + _, err := transformer.Transform(newProvider(stringPtr("v1")), &api.RestAPI{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "v1") + assert.Contains(t, err.Error(), "v9.9.7") + }) +} + +// TestTransformProvider_OAuth2PolicyNameOverride covers +// resolveUpstreamAuthPolicyName's non-empty-override branch for oauth2 - e.g. +// pointing at a fork or a newer major version's replacement instead of the +// built-in oauth2-generator default. Every other oauth2 test here omits +// policyName entirely. +func TestTransformProvider_OAuth2PolicyNameOverride(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + transformer := NewLLMProviderTransformer(store, db, &config.RouterConfig{ListenerPort: 8080}, newTestPolicyVersionResolver()) + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{Metadata: api.Metadata{Name: "openai"}, Spec: api.LLMProviderTemplateData{}}, + } + db.SaveLLMProviderTemplate(template) + require.NoError(t, store.AddTemplate(template)) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2-fork"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (oauth2, forked policy)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyName: stringPtr(testCustomAuthPolicyName), + PolicyParams: &map[string]interface{}{"tokenEndpoint": "https://idp.example.com/oauth2/token"}, + }, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + result, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + found := false + for _, p := range *op.Policies { + if p.Name == testCustomAuthPolicyName { + found = true + assert.Equal(t, testCustomAuthPolicyVersion, p.Version) + } + assert.NotEqual(t, constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME, p.Name, + "the built-in oauth2-generator policy must not also be attached alongside the override") + } + assert.True(t, found, "operation %s %s should include the overridden policy", op.EffectiveMethod(), op.EffectivePath()) + } +} + +func TestTransformProvider_WithOAuth2PasswordGrant(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + tokenEndpoint := "https://legacy-idp.example.com/oauth2/token" + clientID := "gateway-client" + clientSecret := "s3cr3t" + username := "resource-owner" + password := "hunter2" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2-password"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (OAuth2 password grant)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "grantType": "password", + "tokenEndpoint": tokenEndpoint, + "clientId": clientID, + "clientSecret": clientSecret, + "username": username, + "password": password, + }, + }, + }, + AccessControl: api.LLMAccessControl{ + Mode: api.AllowAll, + }, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(provider, output) + assert.NoError(t, err) + assert.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + var foundParams *map[string]interface{} + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + foundParams = p.Params + break + } + } + require.NotNil(t, foundParams) + assert.Equal(t, "password", (*foundParams)["grantType"]) + assert.Equal(t, username, (*foundParams)["username"]) + assert.Equal(t, password, (*foundParams)["password"]) + } +} + +// TestTransformProvider_WithOAuth2UpstreamAuth_MissingPolicyParams locks in +// the only CRD-level requirement left for type: oauth2 - policyParams itself +// must be present. There is no more granular validation of its contents at +// this layer (tokenEndpoint/clientId/clientSecret/grantType/ +// clientAuthMethod, previously validated here via the now-removed typed +// oauth2* fields) - policyParams is forwarded to the oauth2-generator policy +// verbatim, and that policy validates its own contents at GetPolicy time. +func TestTransformProvider_WithOAuth2UpstreamAuth_MissingPolicyParams(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2-invalid"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider (OAuth2, invalid)", + Version: "1.0.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.LLMProviderConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: api.LLMProviderConfigDataUpstreamAuthTypeOauth2, + // PolicyParams deliberately omitted + }, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(provider, output) + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "policyParams") +} + func TestTransformProxy_WithUpstreamAuth(t *testing.T) { store := storage.NewConfigStore() db := newTestMockDB() @@ -2148,14 +2584,23 @@ func TestTransformProxy_WithUpstreamAuth(t *testing.T) { } // TestTransformProxy_OtherAndNoneUpstreamAuth verifies that a proxy-level -// upstream auth override of "other" or "none" transforms successfully and -// attaches no upstream auth policy. +// upstream auth override of "other" (with a named policy) or "none" +// transforms successfully and attaches no *built-in* upstream auth policy. func TestTransformProxy_OtherAndNoneUpstreamAuth(t *testing.T) { - for _, authType := range []api.LLMUpstreamAuthType{ - api.LLMUpstreamAuthTypeOther, - api.LLMUpstreamAuthTypeNone, - } { - t.Run(string(authType), func(t *testing.T) { + tests := []struct { + authType api.LLMUpstreamAuthType + policyName *string + policyParams *map[string]interface{} + }{ + { + authType: api.LLMUpstreamAuthTypeOther, + policyName: stringPtr(testCustomAuthPolicyName), + policyParams: &map[string]interface{}{"foo": "bar"}, + }, + {authType: api.LLMUpstreamAuthTypeNone}, + } + for _, tc := range tests { + t.Run(string(tc.authType), func(t *testing.T) { store := storage.NewConfigStore() db := newTestMockDB() routerConfig := &config.RouterConfig{ListenerPort: 8080} @@ -2205,8 +2650,12 @@ func TestTransformProxy_OtherAndNoneUpstreamAuth(t *testing.T) { DisplayName: "OpenAI Proxy", Version: "v1.0", Provider: api.LLMProxyProvider{ - Id: "openai-provider", - Auth: &api.LLMUpstreamAuth{Type: authType}, + Id: "openai-provider", + Auth: &api.LLMUpstreamAuth{ + Type: tc.authType, + PolicyName: tc.policyName, + PolicyParams: tc.policyParams, + }, }, }, } @@ -2227,13 +2676,316 @@ func TestTransformProxy_OtherAndNoneUpstreamAuth(t *testing.T) { continue } assert.NotEqual(t, constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME, p.Name, - "proxy auth type %q should not attach an upstream auth policy", authType) + "proxy auth type %q should not attach the built-in upstream auth policy", tc.authType) } } }) } } +func TestTransformProxy_WithOAuth2UpstreamAuth(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2-proxy"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + providerOut := &api.RestAPI{} + providerAPI, err := transformer.Transform(provider, providerOut) + require.NoError(t, err) + require.NotNil(t, providerAPI) + + storedProvider := &models.StoredConfig{ + UUID: "0000-prov-cfg-2-0000-000000000000", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: "openai-provider-oauth2-proxy", + DisplayName: "OpenAI Provider", + Version: "v1.0", + Configuration: *providerAPI, + SourceConfiguration: *provider, + DesiredState: models.StateDeployed, + Origin: models.OriginGatewayAPI, + } + db.SaveConfig(storedProvider) + err = store.Add(storedProvider) + require.NoError(t, err) + + tokenEndpoint := "https://idp.example.com/oauth2/token" + clientID := "proxy-client" + clientSecret := "proxy-secret" + purgeStatusCodes := []int{401, 403} + proxy := &api.LLMProxyConfiguration{ + Metadata: api.Metadata{Name: "openai-proxy-oauth2"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "OpenAI Proxy (OAuth2)", + Version: "v1.0", + Provider: api.LLMProxyProvider{ + Id: "openai-provider-oauth2-proxy", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": tokenEndpoint, + "clientId": clientID, + "clientSecret": clientSecret, + "tokenPurgeStatusCodes": purgeStatusCodes, + }, + }, + }, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(proxy, output) + require.NoError(t, err) + require.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + found := false + var foundParams *map[string]interface{} + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + found = true + foundParams = p.Params + break + } + } + assert.True(t, found, "operation %s %s should include the oauth2 policy", op.EffectiveMethod(), op.EffectivePath()) + require.NotNil(t, foundParams) + assert.Equal(t, []int{401, 403}, (*foundParams)["tokenPurgeStatusCodes"], "oauth2TokenPurgeStatusCodes should reach the policy params unchanged via the LlmProxy path too") + } +} + +// TestTransformProxy_ApiKeyWithPolicyParams is proxyUpstreamAuthPolicy's +// counterpart to TestTransformProvider_ApiKeyWithPolicyParams: type: api-key +// configured via the generic policyParams bucket instead of header/value, but +// through the LlmProxy provider.auth call site - previously untested; every +// other proxy api-key test here uses header/value. +func TestTransformProxy_ApiKeyWithPolicyParams(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + require.NoError(t, store.AddTemplate(template)) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-apikey-pp"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{Url: &upstreamURL}, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + providerAPI, err := transformer.Transform(provider, &api.RestAPI{}) + require.NoError(t, err) + + storedProvider := &models.StoredConfig{ + UUID: "0000-prov-cfg-3-0000-000000000000", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: "openai-provider-apikey-pp", + DisplayName: "OpenAI Provider", + Version: "v1.0", + Configuration: *providerAPI, + SourceConfiguration: *provider, + DesiredState: models.StateDeployed, + Origin: models.OriginGatewayAPI, + } + db.SaveConfig(storedProvider) + require.NoError(t, store.Add(storedProvider)) + + proxy := &api.LLMProxyConfiguration{ + Metadata: api.Metadata{Name: "openai-proxy-apikey-pp"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "OpenAI Proxy (api-key via policyParams)", + Version: "v1.0", + Provider: api.LLMProxyProvider{ + Id: "openai-provider-apikey-pp", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeApiKey, + PolicyParams: &map[string]interface{}{ + "request": map[string]interface{}{ + "headers": []interface{}{ + map[string]interface{}{"name": "X-Api-Key", "value": "sk-proxy-from-policyparams"}, + }, + }, + }, + }, + }, + }, + } + + result, err := transformer.Transform(proxy, &api.RestAPI{}) + require.NoError(t, err) + require.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + var found *api.Policy + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME { + found = &p + break + } + } + require.NotNil(t, found, "operation %s %s should include the set-headers policy", op.EffectiveMethod(), op.EffectivePath()) + require.NotNil(t, found.Params) + request, ok := (*found.Params)["request"].(map[string]interface{}) + require.True(t, ok, "expected policyParams.request to survive verbatim, got %+v", *found.Params) + headers, ok := request["headers"].([]interface{}) + require.True(t, ok) + require.Len(t, headers, 1) + entry := headers[0].(map[string]interface{}) + assert.Equal(t, "X-Api-Key", entry["name"]) + assert.Equal(t, "sk-proxy-from-policyparams", entry["value"]) + } +} + +// TestTransformProxy_WithOAuth2PasswordGrantScope locks in that policyParams +// (including a password grant and a nested "params" map) reaches the built +// policy params verbatim via the LlmProxy path (LLMUpstreamAuth), not just +// via LlmProvider's upstream.auth - buildLegacyOrGenericPolicy is shared +// code, but nothing exercised this combination through the proxy call site +// before. +func TestTransformProxy_WithOAuth2PasswordGrantScope(t *testing.T) { + store := storage.NewConfigStore() + db := newTestMockDB() + routerConfig := &config.RouterConfig{ListenerPort: 8080} + transformer := NewLLMProviderTransformer(store, db, routerConfig, newTestPolicyVersionResolver()) + + template := &models.StoredLLMProviderTemplate{ + UUID: "0000-template-1-0000-000000000000", + Configuration: api.LLMProviderTemplate{ + Metadata: api.Metadata{Name: "openai"}, + Spec: api.LLMProviderTemplateData{}, + }, + } + db.SaveLLMProviderTemplate(template) + err := store.AddTemplate(template) + require.NoError(t, err) + + upstreamURL := "https://api.openai.com" + provider := &api.LLMProviderConfiguration{ + Metadata: api.Metadata{Name: "openai-provider-oauth2-proxy-password"}, + Spec: api.LLMProviderConfigData{ + DisplayName: "OpenAI Provider", + Version: "v1.0", + Template: "openai", + Upstream: api.LLMProviderConfigData_Upstream{ + Url: &upstreamURL, + }, + AccessControl: api.LLMAccessControl{Mode: api.AllowAll}, + }, + } + + providerOut := &api.RestAPI{} + providerAPI, err := transformer.Transform(provider, providerOut) + require.NoError(t, err) + require.NotNil(t, providerAPI) + + storedProvider := &models.StoredConfig{ + UUID: "0000-prov-cfg-3-0000-000000000000", + Kind: string(api.LLMProviderConfigurationKindLlmProvider), + Handle: "openai-provider-oauth2-proxy-password", + DisplayName: "OpenAI Provider", + Version: "v1.0", + Configuration: *providerAPI, + SourceConfiguration: *provider, + DesiredState: models.StateDeployed, + Origin: models.OriginGatewayAPI, + } + db.SaveConfig(storedProvider) + err = store.Add(storedProvider) + require.NoError(t, err) + + tokenEndpoint := "https://idp.example.com/oauth2/token" + clientID := "proxy-client" + clientSecret := "proxy-secret" + username := "resource-owner" + password := "hunter2" + proxy := &api.LLMProxyConfiguration{ + Metadata: api.Metadata{Name: "openai-proxy-oauth2-password"}, + Spec: api.LLMProxyConfigData{ + DisplayName: "OpenAI Proxy (OAuth2 password grant)", + Version: "v1.0", + Provider: api.LLMProxyProvider{ + Id: "openai-provider-oauth2-proxy-password", + Auth: &api.LLMUpstreamAuth{ + Type: api.LLMUpstreamAuthTypeOauth2, + PolicyParams: &map[string]interface{}{ + "grantType": "password", + "tokenEndpoint": tokenEndpoint, + "clientId": clientID, + "clientSecret": clientSecret, + "username": username, + "password": password, + "params": map[string]string{"scope": "read write"}, + }, + }, + }, + }, + } + + output := &api.RestAPI{} + result, err := transformer.Transform(proxy, output) + require.NoError(t, err) + require.NotNil(t, result) + + require.NotEmpty(t, result.Spec.Operations) + for _, op := range result.Spec.Operations { + require.NotNil(t, op.Policies) + var foundParams *map[string]interface{} + for _, p := range *op.Policies { + if p.Name == constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME { + foundParams = p.Params + break + } + } + require.NotNil(t, foundParams) + assert.Equal(t, "password", (*foundParams)["grantType"]) + assert.Equal(t, username, (*foundParams)["username"]) + assert.Equal(t, password, (*foundParams)["password"]) + assert.Equal(t, map[string]string{"scope": "read write"}, (*foundParams)["params"]) + } +} + func TestTransformProvider_UnsupportedMode(t *testing.T) { store := storage.NewConfigStore() db := newTestMockDB() diff --git a/gateway/gateway-controller/pkg/utils/mcp_transformer.go b/gateway/gateway-controller/pkg/utils/mcp_transformer.go index 7a6bcd5891..21b006e866 100644 --- a/gateway/gateway-controller/pkg/utils/mcp_transformer.go +++ b/gateway/gateway-controller/pkg/utils/mcp_transformer.go @@ -194,15 +194,53 @@ func (t *MCPTransformer) Transform(input any, output *api.RestAPI) (*api.RestAPI // Set upstream auth if present upstream := mcpConfig.Spec.Upstream if upstream.Auth != nil { - params, err := GetParamsOfPolicy(constants.SET_HEADERS_POLICY_PARAMS, *upstream.Auth.Header, *upstream.Auth.Value) - if err != nil { - return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + auth := upstream.Auth + // MCPTransformer has no policyVersionResolver, so policyVersion is + // passed through as-is and resolved downstream instead. + policyVersion := "" + if auth.PolicyVersion != nil { + policyVersion = strings.TrimSpace(*auth.PolicyVersion) } - pol := api.Policy{ - Name: constants.SET_HEADERS_POLICY_NAME, - Params: ¶ms, + switch auth.Type { + case api.MCPProxyConfigDataUpstreamAuthTypeApiKey: + name := resolveUpstreamAuthPolicyName(auth.PolicyName, constants.SET_HEADERS_POLICY_NAME) + params, err := resolveUpstreamAuthPolicyParams(auth.PolicyParams, func() (map[string]interface{}, error) { + if auth.Header == nil || *auth.Header == "" { + return nil, fmt.Errorf("upstream.auth.header is required") + } + if auth.Value == nil || *auth.Value == "" { + return nil, fmt.Errorf("upstream.auth.value is required") + } + return GetParamsOfPolicy(constants.SET_HEADERS_POLICY_PARAMS, *auth.Header, *auth.Value) + }) + if err != nil { + return nil, fmt.Errorf("failed to build upstream auth params: %w", err) + } + pol := api.Policy{Name: name, Version: policyVersion, Params: ¶ms} + policies = append(policies, pol) + case api.MCPProxyConfigDataUpstreamAuthTypeOauth2: + // No typed-field fallback for oauth2 - policyParams is always required. + if auth.PolicyParams == nil { + return nil, fmt.Errorf("upstream.auth.policyParams is required when type is 'oauth2'") + } + name := resolveUpstreamAuthPolicyName(auth.PolicyName, constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME) + pol := api.Policy{Name: name, Version: policyVersion, Params: auth.PolicyParams} + policies = append(policies, pol) + case api.MCPProxyConfigDataUpstreamAuthTypeOther: + if auth.PolicyName == nil || strings.TrimSpace(*auth.PolicyName) == "" { + return nil, fmt.Errorf("upstream.auth.policyName is required when type is 'other'") + } + if auth.PolicyParams == nil { + return nil, fmt.Errorf("upstream.auth.policyParams is required when type is 'other'") + } + pol := api.Policy{Name: strings.TrimSpace(*auth.PolicyName), Version: policyVersion, Params: auth.PolicyParams} + policies = append(policies, pol) + case api.MCPProxyConfigDataUpstreamAuthTypeNone: + // No upstream authentication - no auth policy is attached; auth + // (if any) is handled entirely by user-attached policies elsewhere. + default: + return nil, fmt.Errorf("unsupported upstream auth type: %s", auth.Type) } - policies = append(policies, pol) } apiData.Policies = &policies diff --git a/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go b/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go index 81eeb55318..ffe9b8f4c9 100644 --- a/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go +++ b/gateway/gateway-controller/pkg/utils/mcp_transformer_test.go @@ -166,14 +166,17 @@ func TestMCPTransformer_Transform_WithPoliciesAndUpstreamAuth(t *testing.T) { url := "http://backend:8080" authHeader := "Authorization" authValue := "Bearer token-xyz" - authType := api.MCPProxyConfigDataUpstreamAuthType("bearer") + authType := api.MCPProxyConfigDataUpstreamAuthTypeApiKey upstream := api.MCPProxyConfigData_Upstream{ Url: &url, Auth: &struct { - Header *string `json:"header,omitempty" yaml:"header,omitempty"` - Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` - Value *string `json:"value,omitempty" yaml:"value,omitempty"` + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` }{ Header: &authHeader, Type: authType, @@ -228,6 +231,151 @@ func TestMCPTransformer_Transform_WithPoliciesAndUpstreamAuth(t *testing.T) { } } +func TestMCPTransformer_Transform_WithOAuth2UpstreamAuth(t *testing.T) { + name := "petstore" + version := "1.0.0" + context := "/petstore" + url := "http://backend:8080" + authType := api.MCPProxyConfigDataUpstreamAuthTypeOauth2 + tokenEndpoint := "https://idp.example.com/oauth2/token" + clientID := "client-id" + clientSecret := "client-secret" + + upstream := api.MCPProxyConfigData_Upstream{ + Url: &url, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: authType, + PolicyParams: &map[string]interface{}{ + "tokenEndpoint": tokenEndpoint, + "clientId": clientID, + "clientSecret": clientSecret, + }, + }, + } + + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: name, + Version: version, + Context: &context, + Upstream: upstream, + SpecVersion: &latest, + }, + } + + var out api.RestAPI + tr := &MCPTransformer{} + res, err := tr.Transform(in, &out) + require.NoError(t, err) + + apiData := res.Spec + require.NotNil(t, apiData.Policies) + resPolicies := *apiData.Policies + require.Len(t, resPolicies, 1) + + pol := resPolicies[0] + assert.Equal(t, constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME, pol.Name) + require.NotNil(t, pol.Params) + params := *pol.Params + // policyParams is forwarded verbatim - no CRD-level defaulting of + // grantType anymore, that's the oauth2-generator policy's own + // responsibility now (see its GetPolicy). + assert.Equal(t, tokenEndpoint, params["tokenEndpoint"]) + assert.Equal(t, clientID, params["clientId"]) + assert.Equal(t, clientSecret, params["clientSecret"]) +} + +// TestMCPTransformer_Transform_WithOAuth2UpstreamAuth_MissingPolicyParams locks +// in the only CRD-level requirement left for type: oauth2 - policyParams must +// be present. +func TestMCPTransformer_Transform_WithOAuth2UpstreamAuth_MissingPolicyParams(t *testing.T) { + context := "/petstore" + url := "http://backend:8080" + authType := api.MCPProxyConfigDataUpstreamAuthTypeOauth2 + + upstream := api.MCPProxyConfigData_Upstream{ + Url: &url, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: authType, + // PolicyParams deliberately omitted. + }, + } + + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "petstore", + Version: "1.0.0", + Context: &context, + Upstream: upstream, + SpecVersion: &latest, + }, + } + + var out api.RestAPI + tr := &MCPTransformer{} + _, err := tr.Transform(in, &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "policyParams") +} + +// TestMCPTransformer_Transform_WithNoneUpstreamAuth locks in that type: none +// is a no-op at transform time - no auth policy is attached. +func TestMCPTransformer_Transform_WithNoneUpstreamAuth(t *testing.T) { + context := "/petstore" + url := "http://backend:8080" + authType := api.MCPProxyConfigDataUpstreamAuthTypeNone + + upstream := api.MCPProxyConfigData_Upstream{ + Url: &url, + Auth: &struct { + Header *string `json:"header,omitempty" yaml:"header,omitempty"` + PolicyName *string `json:"policyName,omitempty" yaml:"policyName,omitempty"` + PolicyParams *map[string]interface{} `json:"policyParams,omitempty" yaml:"policyParams,omitempty"` + PolicyVersion *string `json:"policyVersion,omitempty" yaml:"policyVersion,omitempty"` + Type api.MCPProxyConfigDataUpstreamAuthType `json:"type" yaml:"type"` + Value *string `json:"value,omitempty" yaml:"value,omitempty"` + }{ + Type: authType, + }, + } + + latest := LATEST_SUPPORTED_MCP_SPEC_VERSION + in := &api.MCPProxyConfiguration{ + Spec: api.MCPProxyConfigData{ + DisplayName: "petstore", + Version: "1.0.0", + Context: &context, + Upstream: upstream, + SpecVersion: &latest, + }, + } + + var out api.RestAPI + tr := &MCPTransformer{} + res, err := tr.Transform(in, &out) + require.NoError(t, err) + + apiData := res.Spec + require.NotNil(t, apiData.Policies) + assert.Empty(t, *apiData.Policies) +} + func TestNewMCPTransformer(t *testing.T) { tr := NewMCPTransformer() if tr == nil { diff --git a/gateway/gateway-controller/pkg/utils/policy_version_resolver_test.go b/gateway/gateway-controller/pkg/utils/policy_version_resolver_test.go index 87366a9455..9b18977b8f 100644 --- a/gateway/gateway-controller/pkg/utils/policy_version_resolver_test.go +++ b/gateway/gateway-controller/pkg/utils/policy_version_resolver_test.go @@ -12,14 +12,23 @@ import ( ) const ( - testSetHeadersVersion = "v9.9.9" - testRespondVersion = "v9.9.8" + testSetHeadersVersion = "v9.9.9" + testRespondVersion = "v9.9.8" + testOAuth2AuthenticationVersion = "v9.9.7" + testCustomAuthPolicyVersion = "v9.9.6" ) +// testCustomAuthPolicyName is the canonical example policy name used across +// this package's tests for auth type "other", which has no built-in default +// and must always resolve a user-named policy. +const testCustomAuthPolicyName = "my-custom-auth-policy" + func newTestPolicyVersionResolver() PolicyVersionResolver { return NewStaticPolicyVersionResolver(map[string]string{ constants.UPSTREAM_AUTH_APIKEY_POLICY_NAME: testSetHeadersVersion, constants.ACCESS_CONTROL_DENY_POLICY_NAME: testRespondVersion, + constants.UPSTREAM_AUTH_OAUTH2_POLICY_NAME: testOAuth2AuthenticationVersion, + testCustomAuthPolicyName: testCustomAuthPolicyVersion, }) } diff --git a/gateway/gateway-controller/pkg/xds/translator.go b/gateway/gateway-controller/pkg/xds/translator.go index ae27e6832d..963bf1dcad 100644 --- a/gateway/gateway-controller/pkg/xds/translator.go +++ b/gateway/gateway-controller/pkg/xds/translator.go @@ -52,9 +52,11 @@ import ( extproc "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" luav3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" router "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + upstreamcodecv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/upstream_codec/v3" hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" otelresourcedetectorsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" tracingv3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" @@ -88,6 +90,10 @@ const ( // and other route-scoped consumers. envoyRouteMetadataNamespace = "wso2.route" envoyRouteHTTPRouteKey = "http.route" + + // upstreamHTTPProtocolOptionsKey is the TypedExtensionProtocolOptions map key Envoy + // expects for a cluster's upstream HTTP filter chain (see attachUpstreamRefreshFilter). + upstreamHTTPProtocolOptionsKey = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" ) func checkedUInt32FromPositiveInt(fieldName string, value int) (uint32, error) { @@ -116,6 +122,7 @@ type resolvedTimeout struct { Connect *time.Duration Route *time.Duration Idle *time.Duration + Retry *api.Retry // nil when resilience.retry is not configured } // NewTranslator creates a new translator @@ -331,9 +338,11 @@ func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, // Build route action with timeouts. Per-route resilience values (from the API/operation // resilience block) take precedence; otherwise fall back to the global route defaults. var routeResilienceTimeout, routeResilienceIdle *time.Duration + var routeResilienceRetry *api.Retry if rdcRoute.Timeout != nil { routeResilienceTimeout = rdcRoute.Timeout.Timeout routeResilienceIdle = rdcRoute.Timeout.IdleTimeout + routeResilienceRetry = rdcRoute.Timeout.Retry } routeAction := &route.Route_Route{ Route: &route.RouteAction{ @@ -341,6 +350,9 @@ func (t *Translator) createRouteFromRDC(routeKey string, rdcRoute *models.Route, IdleTimeout: t.routeTimeoutOrDefault(routeResilienceIdle, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } + if routeResilienceRetry != nil { + routeAction.Route.RetryPolicy = buildRetryPolicy(routeResilienceRetry) + } // Set cluster specifier if rdcRoute.Upstream.UseClusterHeader { @@ -700,6 +712,17 @@ func (t *Translator) TranslateConfigs( allRoutes := make([]*route.Route, 0) clusterMap := make(map[string]*cluster.Cluster) + // clustersNeedingUpstreamFilter accumulates, across every deployed config, the name of + // any cluster backing at least one route with a native RetryPolicy (resilience.retry). + // A cluster is shared/deduped by name across operations and even across unrelated APIs + // (see clusterMap above), so this must be OR'd once across every sharer, not decided + // per-operation. It is populated generically from the already-built route.Route objects + // (see collectClustersNeedingUpstreamFilter) rather than re-deriving "does this route + // have retry" from source config, which automatically covers both the legacy + // (translateAPIConfig/createRoute) and RuntimeDeployConfig (translateRuntimeConfig/ + // createRouteFromRDC) paths — both already emit RouteAction.RetryPolicy identically. + clustersNeedingUpstreamFilter := make(map[string]bool) + for _, cfg := range configs { // Skip undeployed APIs - they should not appear in xDS routes if cfg.DesiredState == models.StateUndeployed { @@ -763,6 +786,11 @@ func (t *Translator) TranslateConfigs( for _, c := range clusterList { clusterMap[c.Name] = c } + collectClustersNeedingUpstreamFilter(routesList, clustersNeedingUpstreamFilter) + } + + if err := t.attachUpstreamRefreshFilter(clusterMap, clustersNeedingUpstreamFilter); err != nil { + return nil, err } // Group routes by vhost. Pre-seed the wildcard vhost so no-api-found is @@ -799,6 +827,21 @@ func (t *Translator) TranslateConfigs( // Sort routes by priority (highest priority first) before adding to vhost routes = SortRoutesByPriority(routes) + // Scoped to THIS vhost's own routes only - clustersNeedingUpstreamFilter (above) + // is keyed by cluster name with no vhost affinity at all, so reusing it here + // would leak IncludeRequestAttemptCount onto every vhost the moment ANY vhost, + // anywhere, has a retry-configured route. Checked directly against + // RouteAction.RetryPolicy (the same field collectClustersNeedingUpstreamFilter + // already checks) rather than via the cluster map, since a cluster can be + // shared/deduped across vhosts but this flag must not be. + vhostHasRetryConfiguredRoute := false + for _, r := range routes { + if r.GetRoute().GetRetryPolicy() != nil { + vhostHasRetryConfiguredRoute = true + break + } + } + // Prepend the gateway health routes ahead of every API route and the // catch-all 404 below. vhostMap always contains at least the "*" wildcard // vhost (pre-seeded above), so /ready and /healthy respond even when zero @@ -850,6 +893,20 @@ func (t *Translator) TranslateConfigs( Name: vhost, Domains: t.getVHostDomains(vhost), Routes: routes, + // IncludeRequestAttemptCount makes Envoy set x-envoy-attempt-count on the + // upstream request, starting at 1 and incrementing per retry - this is the + // ONLY signal the upstream ext_proc filter (UpstreamExternalProcessorServer, + // see gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go) has + // to tell a native retry attempt apart from the original one, since it has no + // other way to observe RouteAction.RetryPolicy at request-processing time. + // Scoped to whether THIS vhost has any retry-configured route + // (vhostHasRetryConfiguredRoute, computed above) - VirtualHost is the only + // level this flag exists at (there is no per-route equivalent), so it can't + // be scoped any tighter than per-vhost, but it must not be scoped any + // LOOSER either (e.g. globally across every vhost in this TranslateConfigs + // call) or an unrelated tenant's vhost would get x-envoy-attempt-count sent + // to its own backend for no reason. + IncludeRequestAttemptCount: vhostHasRetryConfiguredRoute, // Strip any client-supplied x-envoy-original-path so it cannot survive to // the collector.ignore_path_prefixes access-log filter (buildIgnorePathsAccessLogFilter): // on a route that performs a path rewrite, Envoy's router unconditionally @@ -968,6 +1025,98 @@ func (t *Translator) TranslateConfigs( return resources, nil } +// collectClustersNeedingUpstreamFilter scans a set of already-built Envoy routes and marks, +// in dest, the name of every statically-specified cluster (RouteAction_Cluster) backing at +// least one route with a native RetryPolicy set. Routes using dynamic cluster_header +// selection (RouteAction_ClusterHeader) are skipped: that cluster identity is resolved only +// per-request, with no policy-chain/cluster access at xDS-build time, so it can never be +// marked here — consistent with this feature's existing exclusion of upstream-definition +// clusters reached the same way. dest is mutated in place so callers can accumulate across +// every config translated in a single TranslateConfigs pass (a cluster can be shared/deduped +// across unrelated APIs, so eligibility must be OR'd once across every sharer). +func collectClustersNeedingUpstreamFilter(routes []*route.Route, dest map[string]bool) { + for _, r := range routes { + ra := r.GetRoute() + if ra == nil || ra.GetRetryPolicy() == nil { + continue + } + if clusterName := ra.GetCluster(); clusterName != "" { + dest[clusterName] = true + } + } +} + +// attachUpstreamRefreshFilter attaches the per-cluster upstream ext_proc filter (chained with +// the mandatory terminal envoy.filters.http.upstream_codec filter) to every cluster in +// clusterMap named in clustersNeedingUpstreamFilter, and unconditionally registers the +// internal cluster the filter targets. A cluster referenced by clustersNeedingUpstreamFilter +// but absent from clusterMap (cluster resolution failed elsewhere) is silently skipped — +// nothing to attach to. No-op when clustersNeedingUpstreamFilter is empty, so deployments +// with no resilience.retry configured anywhere pay zero cost. +func (t *Translator) attachUpstreamRefreshFilter(clusterMap map[string]*cluster.Cluster, clustersNeedingUpstreamFilter map[string]bool) error { + if len(clustersNeedingUpstreamFilter) == 0 { + return nil + } + + upstreamFilter, err := t.createUpstreamRefreshExtProcFilter() + if err != nil { + return fmt.Errorf("failed to create upstream refresh ext_proc filter: %w", err) + } + + // The upstream_codec filter is Envoy's built-in terminal filter: any non-empty upstream + // http_filters chain MUST end with it, or Envoy rejects the cluster config outright. + codecAny, err := anypb.New(&upstreamcodecv3.UpstreamCodec{}) + if err != nil { + return fmt.Errorf("failed to marshal upstream codec filter: %w", err) + } + codecFilter := &hcm.HttpFilter{ + Name: constants.UpstreamCodecFilterName, + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: codecAny}, + } + + // ExplicitHttpConfig with an explicit (empty-fields) Http1ProtocolOptions pins the + // upstream protocol to HTTP/1.1 — this repo's existing clusters never set explicit + // protocol options (createCluster/createWeightedCluster set none), so this preserves + // today's default behavior exactly rather than switching to UseDownstreamProtocolConfig, + // which would make the upstream protocol track whatever the downstream connection + // negotiated — a real behavior change, not a safe no-op, for any listener that can + // negotiate HTTP/2 with the client. + protocolOptions := &httpv3.HttpProtocolOptions{ + UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{ + HttpProtocolOptions: &core.Http1ProtocolOptions{}, + }, + }, + }, + HttpFilters: []*hcm.HttpFilter{upstreamFilter, codecFilter}, + } + protocolOptionsAny, err := anypb.New(protocolOptions) + if err != nil { + return fmt.Errorf("failed to marshal upstream HttpProtocolOptions: %w", err) + } + + for clusterName := range clustersNeedingUpstreamFilter { + c, ok := clusterMap[clusterName] + if !ok { + continue // cluster resolution failed elsewhere; nothing to attach to + } + if c.TypedExtensionProtocolOptions == nil { + c.TypedExtensionProtocolOptions = make(map[string]*anypb.Any) + } + c.TypedExtensionProtocolOptions[upstreamHTTPProtocolOptionsKey] = protocolOptionsAny + } + + // Always register the internal cluster the filter targets, once, unconditionally — cheap, + // and only ever added when at least one real cluster needs the filter (the len==0 guard + // above). + if _, ok := clusterMap[constants.UpstreamRefreshPolicyEngineClusterName]; !ok { + clusterMap[constants.UpstreamRefreshPolicyEngineClusterName] = t.createUpstreamRefreshExtProcCluster() + } + + return nil +} + // getVHostDomains returns Envoy domain patterns for a resolved vhost. // If the vhost equals a configured default and that default has explicit domains, // all configured domains are used; otherwise it falls back to the vhost itself. @@ -1076,7 +1225,7 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* } // Resolve API-level resilience timeouts once; operation-level values override per field. - apiTimeout, apiIdleTimeout, err := ResolveResilience(apiData.Resilience) + apiTimeout, apiIdleTimeout, apiRetry, err := ResolveResilience(apiData.Resilience) if err != nil { return nil, nil, fmt.Errorf("invalid API-level resilience: %w", err) } @@ -1093,11 +1242,11 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* useClusterHeader := hasUpstreamDefinitions || hasSandboxForClusterHeader for _, op := range apiData.Operations { - opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + opTimeout, opIdleTimeout, opRetry, err := ResolveResilience(op.Resilience) if err != nil { return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.EffectiveMethod(), op.EffectivePath(), err) } - opTimeoutCfg := combineRouteResilience(mainTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + opTimeoutCfg := combineRouteResilience(mainTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout, apiRetry, opRetry) r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, op.EffectiveMethod(), op.EffectivePath(), mainClusterName, parsedMainURL.Path, effectiveMainVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Main.HostRewrite, apiProjectID, opTimeoutCfg, useClusterHeader, upstreamDefPaths) @@ -1125,11 +1274,11 @@ func (t *Translator) translateAPIConfig(cfg *models.StoredConfig, allConfigs []* // is on whenever upstreamDefinitions exist or a sandbox upstream is configured). sbRoutesList := make([]*route.Route, 0) for _, op := range apiData.Operations { - opTimeout, opIdleTimeout, err := ResolveResilience(op.Resilience) + opTimeout, opIdleTimeout, opRetry, err := ResolveResilience(op.Resilience) if err != nil { return nil, nil, fmt.Errorf("invalid resilience for operation %s %s: %w", op.EffectiveMethod(), op.EffectivePath(), err) } - opTimeoutCfg := combineRouteResilience(sbTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout) + opTimeoutCfg := combineRouteResilience(sbTimeout, apiTimeout, apiIdleTimeout, opTimeout, opIdleTimeout, apiRetry, opRetry) r := t.createRoute(cfg.UUID, apiData.DisplayName, apiData.Version, apiData.Context, op.EffectiveMethod(), op.EffectivePath(), sbClusterName, parsedSbURL.Path, effectiveSandboxVHost, cfg.Kind, templateHandle, providerName, apiData.Upstream.Sandbox.HostRewrite, apiProjectID, opTimeoutCfg, useClusterHeader, upstreamDefPaths) @@ -1703,6 +1852,9 @@ func (t *Translator) createRoute(apiId, apiName, apiVersion, context, method, pa IdleTimeout: t.routeTimeoutOrDefault(routeIdleTimeout, t.routerConfig.Upstream.Timeouts.RouteIdleTimeoutMs), }, } + if timeoutCfg != nil && timeoutCfg.Retry != nil { + routeAction.Route.RetryPolicy = buildRetryPolicy(timeoutCfg.Retry) + } // Set cluster specifier based on whether dynamic cluster selection is enabled if useClusterHeader { @@ -2074,6 +2226,64 @@ func (t *Translator) createPolicyEngineCluster() *cluster.Cluster { return c } +// createUpstreamRefreshExtProcCluster creates the internal Envoy cluster pointing at +// policy-engine's second, upstream-attempt ext_proc endpoint (see +// gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go). Mirrors +// createPolicyEngineCluster's addressing (UDS by default, TCP via +// t.routerConfig.PolicyEngine.Mode) — this is a DIFFERENT socket/port on the same +// policy-engine process, not a different service. Unlike createPolicyEngineCluster, this +// intentionally has no TLS branch: policy-engine's upstream ext_proc server +// (cmd/policy-engine/main.go) is a bare grpc.NewServer() with no transport credentials, +// so mirroring the downstream cluster's full TLS complexity here would be dead code. +func (t *Translator) createUpstreamRefreshExtProcCluster() *cluster.Cluster { + policyEngine := t.routerConfig.PolicyEngine + + var address *core.Address + if policyEngine.Mode == "tcp" { + address = &core.Address{ + Address: &core.Address_SocketAddress{ + SocketAddress: &core.SocketAddress{ + Protocol: core.SocketAddress_TCP, + Address: policyEngine.Host, + PortSpecifier: &core.SocketAddress_PortValue{ + PortValue: policyEngine.UpstreamRefreshPort, + }, + }, + }, + } + } else { + address = &core.Address{ + Address: &core.Address_Pipe{ + Pipe: &core.Pipe{Path: constants.DefaultUpstreamExtProcSocketPath}, + }, + } + } + + lbEndpoint := &endpoint.LbEndpoint{ + HostIdentifier: &endpoint.LbEndpoint_Endpoint{Endpoint: &endpoint.Endpoint{Address: address}}, + } + clusterType := cluster.Cluster_STATIC + if policyEngine.Mode == "tcp" { + clusterType = cluster.Cluster_STRICT_DNS + } + + c := &cluster.Cluster{ + Name: constants.UpstreamRefreshPolicyEngineClusterName, + ConnectTimeout: durationpb.New(5 * time.Second), + ClusterDiscoveryType: &cluster.Cluster_Type{Type: clusterType}, + LbPolicy: cluster.Cluster_ROUND_ROBIN, + LoadAssignment: &endpoint.ClusterLoadAssignment{ + ClusterName: constants.UpstreamRefreshPolicyEngineClusterName, + Endpoints: []*endpoint.LocalityLbEndpoints{{LbEndpoints: []*endpoint.LbEndpoint{lbEndpoint}}}, + }, + Http2ProtocolOptions: &core.Http2ProtocolOptions{}, + } + if policyEngine.Mode == "tcp" { + c.DnsLookupFamily = cluster.Cluster_V4_PREFERRED + } + return c +} + // createALSCluster creates an Envoy cluster for the gRPC access log service func (t *Translator) createALSCluster() *cluster.Cluster { grpcConfig := t.config.Collector.Server @@ -3191,6 +3401,38 @@ func (t *Translator) createExtProcFilter() (*hcm.HttpFilter, error) { }, nil } +// createUpstreamRefreshExtProcFilter creates the per-cluster upstream ext_proc filter that +// lets any UpstreamAttemptPolicy-implementing policy attach fresh per-attempt state to a +// native Envoy retry. Unlike the main downstream filter, this one only ever needs the +// request-headers phase, and it fails OPEN (FailureModeAllow: true) rather than closed: a +// failure here must never block the retry itself, whereas the downstream filter gates +// auth/access-control and must fail closed. This asymmetry is intentional. +func (t *Translator) createUpstreamRefreshExtProcFilter() (*hcm.HttpFilter, error) { + policyEngine := t.routerConfig.PolicyEngine + extProcConfig := &extproc.ExternalProcessor{ + GrpcService: &core.GrpcService{ + TargetSpecifier: &core.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &core.GrpcService_EnvoyGrpc{ClusterName: constants.UpstreamRefreshPolicyEngineClusterName}, + }, + Timeout: durationpb.New(time.Duration(policyEngine.TimeoutMs) * time.Millisecond), + }, + FailureModeAllow: true, // fail open — a failure here must never block the retry + ProcessingMode: &extproc.ProcessingMode{ + RequestHeaderMode: extproc.ProcessingMode_SEND, + }, + MessageTimeout: durationpb.New(time.Duration(policyEngine.MessageTimeoutMs) * time.Millisecond), + RequestAttributes: []string{constants.ExtProcRequestAttributeRouteName}, + } + extProcAny, err := anypb.New(extProcConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal upstream ext_proc config: %w", err) + } + return &hcm.HttpFilter{ + Name: constants.ExtProcFilterName + "_upstream_refresh", + ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: extProcAny}, + }, nil +} + // resolveUpstreamDefinition finds an upstream definition by its reference name // Returns the upstream definition and error if not found func resolveUpstreamDefinition(ref string, definitions *[]api.UpstreamDefinition) (*api.UpstreamDefinition, error) { @@ -3249,27 +3491,47 @@ func parseDurationAllowZero(timeoutStr *string) (*time.Duration, error) { return &duration, nil } -// ResolveResilience parses a resilience block into route timeout and idle-timeout durations. +// buildRetryPolicy converts a resolved api.Retry into a native Envoy RouteAction.RetryPolicy +// that retries on the configured response status codes. NumRetries defaults to 1 attempt +// when not explicitly configured. +func buildRetryPolicy(retry *api.Retry) *route.RetryPolicy { + numRetries := uint32(1) + if retry.NumRetries != nil { + numRetries = uint32(*retry.NumRetries) + } + statusCodes := make([]uint32, len(retry.StatusCodes)) + for i, code := range retry.StatusCodes { + statusCodes[i] = uint32(code) + } + return &route.RetryPolicy{ + RetryOn: "retriable-status-codes", + RetriableStatusCodes: statusCodes, + NumRetries: wrapperspb.UInt32(numRetries), + } +} + +// ResolveResilience parses a resilience block into route timeout and idle-timeout durations, +// plus the retry configuration (surfaced as-is; validation happens elsewhere). // A nil block, or unset fields, yield nil durations (meaning "use the global default"). // "0s" yields a non-nil zero duration (meaning "explicitly disabled"). -func ResolveResilience(r *api.Resilience) (timeout *time.Duration, idleTimeout *time.Duration, err error) { +func ResolveResilience(r *api.Resilience) (timeout *time.Duration, idleTimeout *time.Duration, retry *api.Retry, err error) { if r == nil { - return nil, nil, nil + return nil, nil, nil, nil } if timeout, err = parseDurationAllowZero(r.Timeout); err != nil { - return nil, nil, fmt.Errorf("invalid resilience.timeout: %w", err) + return nil, nil, nil, fmt.Errorf("invalid resilience.timeout: %w", err) } if idleTimeout, err = parseDurationAllowZero(r.IdleTimeout); err != nil { - return nil, nil, fmt.Errorf("invalid resilience.idleTimeout: %w", err) + return nil, nil, nil, fmt.Errorf("invalid resilience.idleTimeout: %w", err) } - return timeout, idleTimeout, nil + return timeout, idleTimeout, r.Retry, nil } // combineRouteResilience returns a resolvedTimeout for a single route, preserving the -// upstream connect timeout from base and applying the effective route/idle timeouts -// (operation-level overriding API-level, per field). It returns base unchanged when no -// resilience is configured at either level. -func combineRouteResilience(base *resolvedTimeout, apiTimeout, apiIdle, opTimeout, opIdle *time.Duration) *resolvedTimeout { +// upstream connect timeout from base and applying the effective route/idle timeouts and +// retry config (operation-level overriding API-level, per field). It returns base unchanged +// when no resilience is configured at either level. +func combineRouteResilience(base *resolvedTimeout, apiTimeout, apiIdle, opTimeout, opIdle *time.Duration, apiRetry, opRetry *api.Retry) *resolvedTimeout { effTimeout := opTimeout if effTimeout == nil { effTimeout = apiTimeout @@ -3278,10 +3540,14 @@ func combineRouteResilience(base *resolvedTimeout, apiTimeout, apiIdle, opTimeou if effIdle == nil { effIdle = apiIdle } - if effTimeout == nil && effIdle == nil { + effRetry := opRetry + if effRetry == nil { + effRetry = apiRetry + } + if effTimeout == nil && effIdle == nil && effRetry == nil { return base } - rt := resolvedTimeout{Route: effTimeout, Idle: effIdle} + rt := resolvedTimeout{Route: effTimeout, Idle: effIdle, Retry: effRetry} if base != nil { rt.Connect = base.Connect } diff --git a/gateway/gateway-controller/pkg/xds/translator_test.go b/gateway/gateway-controller/pkg/xds/translator_test.go index 46321f7cdc..a1730ec451 100644 --- a/gateway/gateway-controller/pkg/xds/translator_test.go +++ b/gateway/gateway-controller/pkg/xds/translator_test.go @@ -19,6 +19,7 @@ package xds import ( + "fmt" "math" "net/url" "regexp" @@ -32,12 +33,15 @@ import ( listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" tracev3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" + extproc "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" otelresourcedetectorsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" tracingv3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" + "github.com/envoyproxy/go-control-plane/pkg/cache/types" resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -843,6 +847,56 @@ func TestTranslator_RouteResilienceTimeoutsFromRDC(t *testing.T) { } } +// TestTranslator_CreateRouteFromRDC_ResilienceRetryEmitsNativeRetryPolicy verifies that the +// RuntimeDeployConfig path (used by RestApi/LLM kinds, see RestAPITransformer) also emits a +// native Envoy RouteAction.RetryPolicy from a resolved models.RouteTimeout.Retry, and leaves +// RetryPolicy nil when resilience.retry was not configured. +func TestTranslator_CreateRouteFromRDC_ResilienceRetryEmitsNativeRetryPolicy(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + translator := NewTranslator(logger, routerCfg, nil, cfg) + + rdc := &models.RuntimeDeployConfig{ + UpstreamClusters: map[string]*models.UpstreamCluster{ + "main": {Endpoints: []models.Endpoint{{Host: "echo", Port: 80}}}, + }, + } + + t.Run("retry configured", func(t *testing.T) { + numRetries := 2 + rdcRoute := &models.Route{ + Method: "GET", + Path: "/api/v1.0/items", + OperationPath: "/items", + AutoHostRewrite: true, + Timeout: &models.RouteTimeout{Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}}, + Upstream: models.RouteUpstream{ClusterKey: "main"}, + } + r := translator.createRouteFromRDC("GET|/api/v1.0/items|", rdcRoute, rdc) + require.NotNil(t, r) + rp := r.GetRoute().GetRetryPolicy() + require.NotNil(t, rp) + assert.Equal(t, "retriable-status-codes", rp.RetryOn) + require.NotNil(t, rp.NumRetries) + assert.EqualValues(t, 2, rp.NumRetries.Value) + assert.Equal(t, []uint32{401, 503}, rp.RetriableStatusCodes) + }) + + t.Run("retry not configured", func(t *testing.T) { + rdcRoute := &models.Route{ + Method: "GET", + Path: "/api/v1.0/items", + OperationPath: "/items", + AutoHostRewrite: true, + Upstream: models.RouteUpstream{ClusterKey: "main"}, + } + r := translator.createRouteFromRDC("GET|/api/v1.0/items|", rdcRoute, rdc) + require.NotNil(t, r) + assert.Nil(t, r.GetRoute().GetRetryPolicy()) + }) +} + // TestTranslator_MCPUpstreamRewriteFromRDC verifies the MCP "/mcp"-not-appended behavior on the // RuntimeDeployConfig path (createRouteFromRDC), which the policy/runtime xDS pipeline uses. func TestTranslator_MCPUpstreamRewriteFromRDC(t *testing.T) { @@ -1609,6 +1663,170 @@ func TestTranslator_CreateExtProcFilter(t *testing.T) { }) } +func TestTranslator_CreateUpstreamRefreshExtProcCluster(t *testing.T) { + logger := createTestLogger() + + t.Run("UDS mode (default)", func(t *testing.T) { + routerCfg := testRouterConfig() + routerCfg.PolicyEngine = config.PolicyEngineConfig{ + Mode: "uds", + TimeoutMs: 1000, + MessageTimeoutMs: 500, + } + cfg := testConfig() + cfg.Router = *routerCfg + translator := NewTranslator(logger, routerCfg, nil, cfg) + + c := translator.createUpstreamRefreshExtProcCluster() + require.NotNil(t, c) + assert.Equal(t, constants.UpstreamRefreshPolicyEngineClusterName, c.Name) + assert.Equal(t, cluster.Cluster_STATIC, c.ClusterDiscoveryType.(*cluster.Cluster_Type).Type) + + lbEndpoint := c.LoadAssignment.Endpoints[0].LbEndpoints[0] + pipe := lbEndpoint.GetEndpoint().Address.GetPipe() + require.NotNil(t, pipe, "expected Pipe address for UDS mode") + assert.Equal(t, constants.DefaultUpstreamExtProcSocketPath, pipe.Path) + // Must be a distinct socket from the main downstream ext_proc server. + assert.NotEqual(t, constants.DefaultPolicyEngineSocketPath, pipe.Path) + }) + + t.Run("TCP mode with host:port", func(t *testing.T) { + routerCfg := testRouterConfig() + routerCfg.PolicyEngine = config.PolicyEngineConfig{ + Mode: "tcp", + Host: "policy-engine", + Port: 9001, + UpstreamRefreshPort: 9004, + TimeoutMs: 1000, + MessageTimeoutMs: 500, + } + cfg := testConfig() + cfg.Router = *routerCfg + translator := NewTranslator(logger, routerCfg, nil, cfg) + + c := translator.createUpstreamRefreshExtProcCluster() + require.NotNil(t, c) + assert.Equal(t, cluster.Cluster_STRICT_DNS, c.ClusterDiscoveryType.(*cluster.Cluster_Type).Type) + + lbEndpoint := c.LoadAssignment.Endpoints[0].LbEndpoints[0] + socketAddr := lbEndpoint.GetEndpoint().Address.GetSocketAddress() + require.NotNil(t, socketAddr, "expected SocketAddress for TCP mode") + assert.Equal(t, "policy-engine", socketAddr.Address) + // Must dial the upstream-refresh port, not the main downstream ext_proc port. + assert.Equal(t, uint32(9004), socketAddr.GetPortValue()) + }) +} + +func TestTranslator_CreateUpstreamRefreshExtProcFilter(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + routerCfg.PolicyEngine = config.PolicyEngineConfig{ + Host: "localhost", + Port: 50051, + TimeoutMs: 1000, + MessageTimeoutMs: 500, + } + cfg := testConfig() + cfg.Router = *routerCfg + translator := NewTranslator(logger, routerCfg, nil, cfg) + + filter, err := translator.createUpstreamRefreshExtProcFilter() + require.NoError(t, err) + require.NotNil(t, filter) + assert.Equal(t, constants.ExtProcFilterName+"_upstream_refresh", filter.Name) + assert.NotEqual(t, constants.ExtProcFilterName, filter.Name, "must not collide with the main downstream filter's name") + + var extProcConfig extproc.ExternalProcessor + require.NoError(t, filter.GetTypedConfig().UnmarshalTo(&extProcConfig)) + assert.True(t, extProcConfig.FailureModeAllow, "upstream refresh filter must fail open, unlike the downstream filter") + assert.Equal(t, extproc.ProcessingMode_SEND, extProcConfig.ProcessingMode.RequestHeaderMode) + assert.Equal(t, extproc.ProcessingMode_DEFAULT, extProcConfig.ProcessingMode.ResponseHeaderMode, + "must only ever request the headers phase") + assert.Equal(t, constants.UpstreamRefreshPolicyEngineClusterName, + extProcConfig.GrpcService.GetEnvoyGrpc().ClusterName) +} + +// TestCollectClustersNeedingUpstreamFilter unit-tests the OR-across-sharers eligibility scan +// directly against hand-built route.Route objects, independent of which translation path +// (legacy createRoute or RDC createRouteFromRDC) produced them — both emit the identical +// RouteAction shape, which is exactly what makes this scan path-agnostic. +func TestCollectClustersNeedingUpstreamFilter(t *testing.T) { + staticRouteWithRetry := &route.Route{ + Action: &route.Route_Route{ + Route: &route.RouteAction{ + ClusterSpecifier: &route.RouteAction_Cluster{Cluster: "cluster-a"}, + RetryPolicy: &route.RetryPolicy{RetryOn: "retriable-status-codes"}, + }, + }, + } + staticRouteNoRetry := &route.Route{ + Action: &route.Route_Route{ + Route: &route.RouteAction{ + ClusterSpecifier: &route.RouteAction_Cluster{Cluster: "cluster-b"}, + }, + }, + } + dynamicRouteWithRetry := &route.Route{ + Action: &route.Route_Route{ + Route: &route.RouteAction{ + ClusterSpecifier: &route.RouteAction_ClusterHeader{ClusterHeader: "x-target-upstream"}, + RetryPolicy: &route.RetryPolicy{RetryOn: "retriable-status-codes"}, + }, + }, + } + directResponseRoute := &route.Route{ + Action: &route.Route_DirectResponse{DirectResponse: &route.DirectResponseAction{Status: 404}}, + } + + t.Run("marks a static cluster backing a retry-configured route", func(t *testing.T) { + dest := make(map[string]bool) + collectClustersNeedingUpstreamFilter([]*route.Route{staticRouteWithRetry}, dest) + assert.Equal(t, map[string]bool{"cluster-a": true}, dest) + }) + + t.Run("does not mark a cluster with no retry configured", func(t *testing.T) { + dest := make(map[string]bool) + collectClustersNeedingUpstreamFilter([]*route.Route{staticRouteNoRetry}, dest) + assert.Empty(t, dest) + }) + + t.Run("skips dynamic cluster_header routing even with retry configured", func(t *testing.T) { + dest := make(map[string]bool) + collectClustersNeedingUpstreamFilter([]*route.Route{dynamicRouteWithRetry}, dest) + assert.Empty(t, dest, "cluster identity is resolved only per-request; must never be marked at xDS-build time") + }) + + t.Run("ignores non-RouteAction actions (e.g. direct response)", func(t *testing.T) { + dest := make(map[string]bool) + collectClustersNeedingUpstreamFilter([]*route.Route{directResponseRoute}, dest) + assert.Empty(t, dest) + }) + + t.Run("OR's across sharers: one retry-configured route is enough to mark a shared cluster", func(t *testing.T) { + sharedRetry := &route.Route{ + Action: &route.Route_Route{ + Route: &route.RouteAction{ + ClusterSpecifier: &route.RouteAction_Cluster{Cluster: "shared-cluster"}, + RetryPolicy: &route.RetryPolicy{RetryOn: "retriable-status-codes"}, + }, + }, + } + sharedNoRetry := &route.Route{ + Action: &route.Route_Route{ + Route: &route.RouteAction{ + ClusterSpecifier: &route.RouteAction_Cluster{Cluster: "shared-cluster"}, + }, + }, + } + dest := make(map[string]bool) + // Simulate two separate configs contributing routes to the same cluster across two + // separate accumulation calls, exactly as TranslateConfigs does per-cfg. + collectClustersNeedingUpstreamFilter([]*route.Route{sharedNoRetry}, dest) + collectClustersNeedingUpstreamFilter([]*route.Route{sharedRetry}, dest) + assert.Equal(t, map[string]bool{"shared-cluster": true}, dest) + }) +} + func TestTranslator_CreateRouteConfiguration(t *testing.T) { logger := createTestLogger() routerCfg := testRouterConfig() @@ -1747,6 +1965,338 @@ func TestTranslator_TranslateConfigs_GatewayHealthRoutes(t *testing.T) { }) } +// makeRestAPIWithUpstreamAndRetry is like makeRestAPI but allows a custom upstream URL (so +// two configs can be made to share a deduped Envoy cluster, per resolveUpstreamCluster's +// host+scheme dedup) and an optional API-level resilience.retry. +func makeRestAPIWithUpstreamAndRetry(uuid, name, ctx, upstreamURL string, retry *api.Retry) *models.StoredConfig { + cfg := api.RestAPI{ + Kind: api.RestAPIKindRestApi, + Metadata: api.Metadata{Name: name}, + Spec: api.APIConfigData{ + DisplayName: name, + Version: "v1.0", + Context: ctx, + Upstream: struct { + Main api.Upstream `json:"main" yaml:"main"` + Sandbox *api.Upstream `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{ + Main: api.Upstream{Url: api.Ptr(upstreamURL)}, + }, + Operations: []api.Operation{ + {Method: api.Ptr(api.OperationMethodGET), Path: api.Ptr("/resource")}, + }, + }, + } + if retry != nil { + cfg.Spec.Resilience = &api.Resilience{Retry: retry} + } + return &models.StoredConfig{ + UUID: uuid, + Kind: models.KindRestApi, + Handle: name, + DisplayName: name, + Version: "v1.0", + DesiredState: models.StateDeployed, + Configuration: cfg, + SourceConfiguration: cfg, + } +} + +// TestTranslateConfigs_ClusterGetsUpstreamFilterWhenAnyRouteHasRetryConfigured exercises the +// LEGACY translation path (translateAPIConfig/createRoute — no transformer registered, same +// as every other TranslateConfigs test in this file). Two RestApi configs share the identical +// upstream host+scheme, so resolveUpstreamCluster/sanitizeClusterName dedupe them into ONE +// Envoy cluster; only one of the two configs has resilience.retry set. This proves the +// OR-across-sharers behavior: the shared cluster must get the upstream filter attached +// because at least one sharer needs it. +func TestTranslateConfigs_ClusterGetsUpstreamFilterWhenAnyRouteHasRetryConfigured(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + const sharedUpstream = "http://shared-backend:9999" + configs := []*models.StoredConfig{ + makeRestAPIWithUpstreamAndRetry("uuid-legacy-1", "api-one", "/api-one", sharedUpstream, nil), + makeRestAPIWithUpstreamAndRetry("uuid-legacy-2", "api-two", "/api-two", sharedUpstream, &api.Retry{StatusCodes: []int{503}}), + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + expectedClusterName := translator.sanitizeClusterName("shared-backend:9999", "http") + + var sharedEnvoyCluster *cluster.Cluster + internalClusterPresent := false + for _, res := range resources[resource.ClusterType] { + c, ok := res.(*cluster.Cluster) + require.True(t, ok) + if c.Name == expectedClusterName { + sharedEnvoyCluster = c + } + if c.Name == constants.UpstreamRefreshPolicyEngineClusterName { + internalClusterPresent = true + } + } + + require.NotNil(t, sharedEnvoyCluster, "expected to find the deduped shared cluster %q", expectedClusterName) + assert.True(t, internalClusterPresent, "expected the internal upstream-refresh policy-engine cluster to be registered") + + protocolOptionsAny, ok := sharedEnvoyCluster.TypedExtensionProtocolOptions[upstreamHTTPProtocolOptionsKey] + require.True(t, ok, "expected TypedExtensionProtocolOptions on the shared cluster") + + var protocolOptions httpv3.HttpProtocolOptions + require.NoError(t, protocolOptionsAny.UnmarshalTo(&protocolOptions)) + require.Len(t, protocolOptions.HttpFilters, 2) + assert.Equal(t, constants.ExtProcFilterName+"_upstream_refresh", protocolOptions.HttpFilters[0].Name) + assert.Equal(t, constants.UpstreamCodecFilterName, protocolOptions.HttpFilters[1].Name, + "upstream_codec must be the terminal filter or Envoy rejects the config") +} + +// TestTranslateConfigs_ClusterWithNoRetryConfiguredAnywhereGetsNoUpstreamFilter is the negative +// counterpart: same shared-cluster shape, but NEITHER config has resilience.retry set. +func TestTranslateConfigs_ClusterWithNoRetryConfiguredAnywhereGetsNoUpstreamFilter(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + const sharedUpstream = "http://shared-backend-2:9999" + configs := []*models.StoredConfig{ + makeRestAPIWithUpstreamAndRetry("uuid-legacy-3", "api-three", "/api-three", sharedUpstream, nil), + makeRestAPIWithUpstreamAndRetry("uuid-legacy-4", "api-four", "/api-four", sharedUpstream, nil), + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + expectedClusterName := translator.sanitizeClusterName("shared-backend-2:9999", "http") + + var sharedEnvoyCluster *cluster.Cluster + for _, res := range resources[resource.ClusterType] { + c, ok := res.(*cluster.Cluster) + require.True(t, ok) + if c.Name == expectedClusterName { + sharedEnvoyCluster = c + } + assert.NotEqual(t, constants.UpstreamRefreshPolicyEngineClusterName, c.Name, + "internal upstream-refresh cluster must not be registered when nothing needs it") + } + + require.NotNil(t, sharedEnvoyCluster) + assert.Empty(t, sharedEnvoyCluster.TypedExtensionProtocolOptions) +} + +// fakeRDCTransformer is a minimal models.ConfigTransformer that returns a hand-built +// RuntimeDeployConfig keyed by StoredConfig.UUID, letting tests exercise the RDC path +// (translateRuntimeConfig/createRouteFromRDC) without the full RestAPITransformer.Transform +// machinery (policy definitions, secrets, etc.) — mirroring how Task 7's +// TestTranslator_CreateRouteFromRDC_ResilienceRetryEmitsNativeRetryPolicy hand-builds a +// RuntimeDeployConfig directly rather than going through Transform(). +type fakeRDCTransformer map[string]*models.RuntimeDeployConfig + +func (f fakeRDCTransformer) Transform(cfg *models.StoredConfig) (*models.RuntimeDeployConfig, error) { + rdc, ok := f[cfg.UUID] + if !ok { + return nil, fmt.Errorf("no fake RuntimeDeployConfig registered for %s", cfg.UUID) + } + return rdc, nil +} + +// TestTranslateConfigs_RDCPath_ClusterGetsUpstreamFilterWhenAnyRouteHasRetryConfigured proves +// the RDC path (translateRuntimeConfig, the one RestApi/Mcp/LlmProvider/LlmProxy actually use +// in production via a registered transformer) gets the same OR-across-sharers upstream filter +// attachment as the legacy path above — the two are entirely separate cluster-building loops +// in translator.go, so this must be verified independently, not assumed from the legacy-path +// test alone. +func TestTranslateConfigs_RDCPath_ClusterGetsUpstreamFilterWhenAnyRouteHasRetryConfigured(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + sharedCluster := &models.UpstreamCluster{Endpoints: []models.Endpoint{{Host: "shared-rdc-backend", Port: 9999}}} + + rdcNoRetry := &models.RuntimeDeployConfig{ + Metadata: models.Metadata{Kind: "RestApi"}, + UpstreamClusters: map[string]*models.UpstreamCluster{"shared": sharedCluster}, + Routes: map[string]*models.Route{ + "GET|/api-two/v1.0/items|localhost": { + Method: "GET", Path: "/api-two/v1.0/items", OperationPath: "/items", + Upstream: models.RouteUpstream{ClusterKey: "shared"}, + }, + }, + } + rdcWithRetry := &models.RuntimeDeployConfig{ + Metadata: models.Metadata{Kind: "RestApi"}, + UpstreamClusters: map[string]*models.UpstreamCluster{"shared": sharedCluster}, + Routes: map[string]*models.Route{ + "GET|/api-one/v1.0/items|localhost": { + Method: "GET", Path: "/api-one/v1.0/items", OperationPath: "/items", + Timeout: &models.RouteTimeout{Retry: &api.Retry{StatusCodes: []int{503}}}, + Upstream: models.RouteUpstream{ClusterKey: "shared"}, + }, + }, + } + + translator.SetTransformers(map[string]models.ConfigTransformer{ + "RestApi": fakeRDCTransformer{ + "uuid-rdc-1": rdcNoRetry, + "uuid-rdc-2": rdcWithRetry, + }, + }) + + configs := []*models.StoredConfig{ + {UUID: "uuid-rdc-1", Kind: "RestApi", DesiredState: models.StateDeployed}, + {UUID: "uuid-rdc-2", Kind: "RestApi", DesiredState: models.StateDeployed}, + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + var sharedEnvoyCluster *cluster.Cluster + for _, res := range resources[resource.ClusterType] { + c, ok := res.(*cluster.Cluster) + require.True(t, ok) + if c.Name == "shared" { + sharedEnvoyCluster = c + } + } + require.NotNil(t, sharedEnvoyCluster, "expected the RDC-path cluster 'shared' to be present") + + _, ok := sharedEnvoyCluster.TypedExtensionProtocolOptions[upstreamHTTPProtocolOptionsKey] + assert.True(t, ok, "RDC-path cluster must also get the upstream filter attached — both translation paths must be covered") +} + +// TestTranslateConfigs_VirtualHostIncludesRequestAttemptCountWhenAnyRouteHasRetryConfigured proves +// the VirtualHost carries IncludeRequestAttemptCount whenever at least one route anywhere has +// resilience.retry set. Without this, Envoy never emits x-envoy-attempt-count on the upstream +// request, so UpstreamExternalProcessorServer.processRequestHeaders (policy-engine) can never tell +// a native retry attempt apart from the original one — silently defeating the whole +// upstream-attempt refresh mechanism (oauth2-generator's OnUpstreamAttemptRequestHeaders gates +// entirely on AttemptCount > 1) even though the route's own RetryPolicy and the cluster's upstream +// ext_proc filter are both present and correct. Confirmed live via e2e (Task 10): without this +// flag, a native retry silently reused the same already-rejected cached token. +func TestTranslateConfigs_VirtualHostIncludesRequestAttemptCountWhenAnyRouteHasRetryConfigured(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + const sharedUpstream = "http://shared-backend-attempt-count:9999" + configs := []*models.StoredConfig{ + makeRestAPIWithUpstreamAndRetry("uuid-attempt-count-1", "api-one", "/api-one", sharedUpstream, nil), + makeRestAPIWithUpstreamAndRetry("uuid-attempt-count-2", "api-two", "/api-two", sharedUpstream, &api.Retry{StatusCodes: []int{503}}), + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + byName := virtualHostsByName(t, resources) + require.Contains(t, byName, "localhost", "expected the API's own vhost ('localhost', neither config sets a custom Vhosts.Main) to be present") + assert.True(t, byName["localhost"].IncludeRequestAttemptCount, + "the 'localhost' vhost carries the retry-configured route and must set IncludeRequestAttemptCount") + // The pre-seeded wildcard vhost carries neither API's routes at all (see + // vhostMap's pre-seeding in TranslateConfigs) - it must NOT be flagged just + // because some OTHER vhost happens to need it. This is exactly the gap a + // global (rather than per-vhost) scoping would miss - see + // TestTranslateConfigs_MultipleVhosts_OnlyTheOneWithRetryGetsRequestAttemptCount + // for the sharper, explicit-multi-vhost version of this same proof. + if wildcard, ok := byName["*"]; ok { + assert.False(t, wildcard.IncludeRequestAttemptCount, + "the wildcard vhost carries no API routes and must not set IncludeRequestAttemptCount just because 'localhost' needs it") + } +} + +// TestTranslateConfigs_VirtualHostOmitsRequestAttemptCountWhenNoRetryConfiguredAnywhere is the +// negative counterpart: same shared-cluster shape, but NEITHER config has resilience.retry set. +func TestTranslateConfigs_VirtualHostOmitsRequestAttemptCountWhenNoRetryConfiguredAnywhere(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + const sharedUpstream = "http://shared-backend-no-attempt-count:9999" + configs := []*models.StoredConfig{ + makeRestAPIWithUpstreamAndRetry("uuid-no-attempt-count-1", "api-three", "/api-three", sharedUpstream, nil), + makeRestAPIWithUpstreamAndRetry("uuid-no-attempt-count-2", "api-four", "/api-four", sharedUpstream, nil), + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + found := false + for _, res := range resources[resource.RouteType] { + rc, ok := res.(*route.RouteConfiguration) + require.True(t, ok) + for _, vh := range rc.VirtualHosts { + found = true + assert.False(t, vh.IncludeRequestAttemptCount, + "virtual host %q must not set IncludeRequestAttemptCount when nothing needs it", vh.Name) + } + } + require.True(t, found, "expected at least one virtual host to be present") +} + +// makeRestAPIWithUpstreamRetryAndVhost is makeRestAPIWithUpstreamAndRetry plus an explicit +// Vhosts.Main override, so a test can put two configs on two DIFFERENT vhosts (the default +// helper always lands every config on the same vhost - config.RouterConfig's VHosts.Main.Default, +// "localhost" in testRouterConfig() - which cannot exercise cross-vhost scoping at all). +func makeRestAPIWithUpstreamRetryAndVhost(uuid, name, ctx, upstreamURL string, retry *api.Retry, vhost string) *models.StoredConfig { + cfg := makeRestAPIWithUpstreamAndRetry(uuid, name, ctx, upstreamURL, retry) + restAPI := cfg.Configuration.(api.RestAPI) + restAPI.Spec.Vhosts = &struct { + Main string `json:"main" yaml:"main"` + Sandbox *string `json:"sandbox,omitempty" yaml:"sandbox,omitempty"` + }{Main: vhost} + cfg.Configuration = restAPI + cfg.SourceConfiguration = restAPI + return cfg +} + +// virtualHostsByName flattens every RouteConfiguration resource's VirtualHosts into a +// name-keyed map, for tests that need to assert on one specific vhost rather than looping +// over all of them (which is exactly the kind of loose assertion that let the original, +// globally-scoped IncludeRequestAttemptCount bug slip past both existing tests above - see +// the code-review finding this helper and the test below were added to address). +func virtualHostsByName(t *testing.T, resources map[resource.Type][]types.Resource) map[string]*route.VirtualHost { + t.Helper() + byName := make(map[string]*route.VirtualHost) + for _, res := range resources[resource.RouteType] { + rc, ok := res.(*route.RouteConfiguration) + require.True(t, ok) + for _, vh := range rc.VirtualHosts { + byName[vh.Name] = vh + } + } + return byName +} + +// TestTranslateConfigs_MultipleVhosts_OnlyTheOneWithRetryGetsRequestAttemptCount is the sharp +// regression test for the code-review finding: IncludeRequestAttemptCount must be scoped +// per-vhost, not globally across the whole TranslateConfigs call. Two configs on two entirely +// different, explicit vhosts (simulating two unrelated tenants) - only ONE has resilience.retry +// configured. A global (rather than per-vhost) implementation would incorrectly flag BOTH +// vhosts, since clustersNeedingUpstreamFilter (keyed by cluster name, with no vhost affinity) +// would be non-empty for the whole call the moment either config needs it. +func TestTranslateConfigs_MultipleVhosts_OnlyTheOneWithRetryGetsRequestAttemptCount(t *testing.T) { + logger := createTestLogger() + translator := NewTranslator(logger, testRouterConfig(), nil, testConfig()) + + configs := []*models.StoredConfig{ + makeRestAPIWithUpstreamRetryAndVhost("uuid-vhost-a", "api-vhost-a", "/api-vhost-a", + "http://backend-vhost-a:9999", &api.Retry{StatusCodes: []int{503}}, "vhost-a.example.com"), + makeRestAPIWithUpstreamRetryAndVhost("uuid-vhost-b", "api-vhost-b", "/api-vhost-b", + "http://backend-vhost-b:9999", nil, "vhost-b.example.com"), + } + + resources, err := translator.TranslateConfigs(configs, "test-correlation-id") + require.NoError(t, err) + + byName := virtualHostsByName(t, resources) + require.Contains(t, byName, "vhost-a.example.com") + require.Contains(t, byName, "vhost-b.example.com") + + assert.True(t, byName["vhost-a.example.com"].IncludeRequestAttemptCount, + "vhost-a has the retry-configured route and must set IncludeRequestAttemptCount") + assert.False(t, byName["vhost-b.example.com"].IncludeRequestAttemptCount, + "vhost-b has NO retry-configured route of its own and must not be affected by vhost-a's") + if wildcard, ok := byName["*"]; ok { + assert.False(t, wildcard.IncludeRequestAttemptCount, + "the wildcard vhost carries neither tenant's routes and must not be flagged either") + } +} + func TestTranslator_GetVHostDomains(t *testing.T) { logger := createTestLogger() @@ -2538,6 +3088,40 @@ func TestTranslator_CreateRoute_Basic(t *testing.T) { route.Metadata.FilterMetadata["wso2.route"].Fields["http.route"].GetStringValue()) } +// TestCreateRoute_ResilienceRetryEmitsNativeRetryPolicy verifies that a resolvedTimeout +// carrying a non-nil Retry field (resolved from resilience.retry) causes createRoute to +// set a native Envoy RouteAction.RetryPolicy on the resulting route. +func TestCreateRoute_ResilienceRetryEmitsNativeRetryPolicy(t *testing.T) { + logger := createTestLogger() + routerCfg := testRouterConfig() + cfg := testConfig() + translator := NewTranslator(logger, routerCfg, nil, cfg) + + numRetries := 2 + timeoutCfg := &resolvedTimeout{Retry: &api.Retry{StatusCodes: []int{401, 503}, NumRetries: &numRetries}} + + r := translator.createRoute("api-id", "TestAPI", "v1", "/test", "GET", "/foo", "test-cluster", + "", "localhost", "RestApi", "", "", nil, "project-1", timeoutCfg, false, nil) + + routeAction, ok := r.Action.(*route.Route_Route) + if !ok { + t.Fatalf("expected a Route_Route action, got %T", r.Action) + } + rp := routeAction.Route.RetryPolicy + if rp == nil { + t.Fatal("expected a non-nil RetryPolicy") + } + if rp.RetryOn != "retriable-status-codes" { + t.Errorf("got RetryOn %q, want %q", rp.RetryOn, "retriable-status-codes") + } + if rp.NumRetries == nil || rp.NumRetries.Value != 2 { + t.Errorf("got NumRetries %v, want 2", rp.NumRetries) + } + if len(rp.RetriableStatusCodes) != 2 || rp.RetriableStatusCodes[0] != 401 || rp.RetriableStatusCodes[1] != 503 { + t.Errorf("got RetriableStatusCodes %v, want [401 503]", rp.RetriableStatusCodes) + } +} + // TestTranslator_CreateRouteFromRDC_HTTPRouteMetadata guards the http.route tracing fix: // createRouteFromRDC must record the route's full path *template* (not a concrete matched // path) as wso2.route/http.route metadata, so the HCM tracing http.route custom tag diff --git a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go index 9978f14ae8..33284bdbbe 100644 --- a/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go +++ b/gateway/gateway-runtime/policy-engine/cmd/policy-engine/main.go @@ -47,6 +47,7 @@ import ( "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/tracing" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/utils" "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/xdsclient" + "github.com/wso2/api-platform/sdk/core/utils/redisclient" ) // Version information (set via ldflags during build) @@ -164,6 +165,19 @@ func main() { } slog.InfoContext(ctx, "Config set in registry for ${config} CEL resolution") + // Initialize the gateway-level shared Redis client (top-level "redis" config + // section - gateway infrastructure, not nested under policy_configurations, + // since it's not scoped to policies even though policy-engine is its current + // consumer). Must run before any policy chain is built (below) - a policy + // instance that calls redisclient.Shared()/Resolve() during construction + // assumes this has already run. A missing "redis" section is not an error + // here - it's only surfaced lazily, the first time some policy actually + // needs it. + if err := redisclient.InitFromConfig(cfg.PolicyEngine.RawConfig); err != nil { + slog.ErrorContext(ctx, "Failed to initialize shared redis client", "error", err) + os.Exit(1) + } + // Initialize CEL evaluator celEvaluator, err := cel.NewCELEvaluator() if err != nil { @@ -257,6 +271,58 @@ func main() { grpcServer := grpc.NewServer() extprocv3.RegisterExternalProcessorServer(grpcServer, extprocServer) + // Channel used by every gRPC server started below to signal an unexpected + // Serve() failure back to the shutdown-select loop. Declared here (rather + // than alongside sigChan further down) so the upstream ext_proc server's + // goroutine, started next, can already reference it. + serverErrCh := make(chan error, 1) + + // Create and start the upstream-attempt ext_proc gRPC server (second, + // minimal endpoint — see internal/kernel/upstream_extproc.go). Uses the same + // serverMode (uds/tcp) as the main ext_proc server, but its own socket/port, + // and its own explicit message/stream limits sized for its headers-only + // message shape (go-network-service-hardening.md directive 2) — not copied + // from the main server's larger ceiling. + upstreamExtprocServer := kernel.NewUpstreamExternalProcessorServer(k) + + var upstreamLis net.Listener + switch serverMode { + case "uds": + socketPath := constants.DefaultUpstreamExtProcSocketPath + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + slog.WarnContext(ctx, "Failed to remove existing upstream ext_proc socket file", "path", socketPath, "error", err) + } + upstreamLis, err = net.Listen("unix", socketPath) + if err != nil { + slog.ErrorContext(ctx, "Failed to listen on upstream ext_proc Unix socket", "path", socketPath, "error", err) + os.Exit(1) + } + if err := os.Chmod(socketPath, 0660); err != nil { + slog.WarnContext(ctx, "Failed to set upstream ext_proc socket permissions", "path", socketPath, "error", err) + } + slog.InfoContext(ctx, "Upstream ext_proc server listening on Unix socket", "path", socketPath) + case "tcp": + upstreamLis, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.PolicyEngine.Server.UpstreamExtProcPort)) + if err != nil { + slog.ErrorContext(ctx, "Failed to listen on upstream ext_proc port", "port", cfg.PolicyEngine.Server.UpstreamExtProcPort, "error", err) + os.Exit(1) + } + slog.InfoContext(ctx, "Upstream ext_proc server listening on TCP port", "port", cfg.PolicyEngine.Server.UpstreamExtProcPort) + } + + upstreamGrpcServer := grpc.NewServer( + grpc.MaxRecvMsgSize(64*1024), // headers-only messages; far smaller than the body-carrying main server's ceiling + grpc.MaxSendMsgSize(64*1024), + grpc.MaxConcurrentStreams(1000), + ) + extprocv3.RegisterExternalProcessorServer(upstreamGrpcServer, upstreamExtprocServer) + + go func() { + if err := upstreamGrpcServer.Serve(upstreamLis); err != nil { + serverErrCh <- err + } + }() + // Enable block/mutex profiling sampling when pprof is enabled. These are the // only profiles that need explicit rate setup; 0 leaves them disabled. Gated so // the sampling overhead is never paid unless pprof is deliberately turned on. @@ -309,8 +375,8 @@ func main() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - // Start server in goroutine - serverErrCh := make(chan error, 1) + // Start server in goroutine (serverErrCh declared earlier, shared with the + // upstream ext_proc server's goroutine above) go func() { if err := grpcServer.Serve(lis); err != nil { serverErrCh <- err @@ -353,6 +419,9 @@ func main() { alsServer.GracefulStop() } + slog.InfoContext(ctx, "Stopping upstream ext_proc gRPC server") + upstreamGrpcServer.GracefulStop() + grpcServer.GracefulStop() // Cleanup Unix socket if used (UDS mode) @@ -361,6 +430,10 @@ func main() { slog.WarnContext(ctx, "Failed to cleanup socket file on shutdown", "path", constants.DefaultPolicyEngineSocketPath, "error", err) } + if err := os.Remove(constants.DefaultUpstreamExtProcSocketPath); err != nil && !os.IsNotExist(err) { + slog.WarnContext(ctx, "Failed to cleanup upstream ext_proc socket file on shutdown", + "path", constants.DefaultUpstreamExtProcSocketPath, "error", err) + } } slog.InfoContext(ctx, "Policy Engine shut down successfully") diff --git a/gateway/gateway-runtime/policy-engine/configs/config-file-mode.toml b/gateway/gateway-runtime/policy-engine/configs/config-file-mode.toml index b87d475e97..c3daaf48ad 100644 --- a/gateway/gateway-runtime/policy-engine/configs/config-file-mode.toml +++ b/gateway/gateway-runtime/policy-engine/configs/config-file-mode.toml @@ -4,6 +4,8 @@ [policy_engine.server] # Port for ext_proc gRPC server (receives requests from Envoy) extproc_port = 9001 +# Port for the upstream-attempt ext_proc gRPC server (see internal/kernel/upstream_extproc.go) +upstream_extproc_port = 9004 [policy_engine.config_mode] # Configuration mode: "file" or "xds" diff --git a/gateway/gateway-runtime/policy-engine/go.mod b/gateway/gateway-runtime/policy-engine/go.mod index e4914e8ec3..551e25171b 100644 --- a/gateway/gateway-runtime/policy-engine/go.mod +++ b/gateway/gateway-runtime/policy-engine/go.mod @@ -51,10 +51,12 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.19.2 // indirect + github.com/redis/go-redis/v9 v9.22.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect golang.org/x/net v0.56.0 // indirect @@ -65,3 +67,5 @@ require ( ) replace github.com/wso2/api-platform/common => ../../../common + +replace github.com/wso2/api-platform/sdk/core => ../../../sdk/core diff --git a/gateway/gateway-runtime/policy-engine/go.sum b/gateway/gateway-runtime/policy-engine/go.sum index 2939f20ddd..3e2a2de00c 100644 --- a/gateway/gateway-runtime/policy-engine/go.sum +++ b/gateway/gateway-runtime/policy-engine/go.sum @@ -1,11 +1,17 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -41,6 +47,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/toml/v2 v2.2.0 h1:2nV7tHYJ5OZy2BynQ4mOJ6k5bDqbbCzRERLUKBytz3A= @@ -80,6 +88,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= @@ -92,10 +102,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/wso2/api-platform/sdk/core v0.3.3 h1:5bBapq9tWQf/kXZfYNnofJUTwZLyZ0PZpe7skbeZd/I= -github.com/wso2/api-platform/sdk/core v0.3.3/go.mod h1:TgjpOk3QBPc7xEQC+NctWcNq5dSPBkn7wSKh+hULTj8= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= @@ -114,6 +126,8 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/ go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 7bb39ead3f..ccea3c2d45 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -239,6 +239,10 @@ type ServerConfig struct { // ExtProcPort is the port for the ext_proc gRPC server (TCP mode only) ExtProcPort int `koanf:"extproc_port"` + + // UpstreamExtProcPort is the port for the upstream-attempt ext_proc gRPC + // server (TCP mode only) — see internal/kernel/upstream_extproc.go. + UpstreamExtProcPort int `koanf:"upstream_extproc_port"` } // PythonExecutorConfig holds configuration for the Python executor bridge. @@ -511,8 +515,9 @@ func defaultConfig() *Config { return &Config{ PolicyEngine: PolicyEngine{ Server: ServerConfig{ - Mode: "", - ExtProcPort: 9001, + Mode: "", + ExtProcPort: 9001, + UpstreamExtProcPort: 9004, }, Admin: AdminConfig{ Enabled: true, @@ -626,6 +631,9 @@ func (c *Config) Validate() error { if c.PolicyEngine.Server.ExtProcPort <= 0 || c.PolicyEngine.Server.ExtProcPort > 65535 { return fmt.Errorf("invalid extproc_port: %d (must be 1-65535)", c.PolicyEngine.Server.ExtProcPort) } + if c.PolicyEngine.Server.UpstreamExtProcPort <= 0 || c.PolicyEngine.Server.UpstreamExtProcPort > 65535 { + return fmt.Errorf("invalid upstream_extproc_port: %d (must be 1-65535)", c.PolicyEngine.Server.UpstreamExtProcPort) + } default: return fmt.Errorf("server.mode must be 'uds' or 'tcp', got: %s", c.PolicyEngine.Server.Mode) } @@ -656,6 +664,9 @@ func (c *Config) Validate() error { if c.PolicyEngine.Server.Mode == "tcp" && c.PolicyEngine.Admin.Port == c.PolicyEngine.Server.ExtProcPort { return fmt.Errorf("admin.port cannot be same as server.extproc_port") } + if c.PolicyEngine.Server.Mode == "tcp" && c.PolicyEngine.Admin.Port == c.PolicyEngine.Server.UpstreamExtProcPort { + return fmt.Errorf("admin.port cannot be same as server.upstream_extproc_port") + } if len(c.PolicyEngine.Admin.AllowedIPs) == 0 { return fmt.Errorf("admin.allowed_ips cannot be empty when admin is enabled") } @@ -670,6 +681,9 @@ func (c *Config) Validate() error { if c.PolicyEngine.Server.Mode == "tcp" && c.PolicyEngine.Metrics.Port == c.PolicyEngine.Server.ExtProcPort { return fmt.Errorf("metrics.port cannot be same as server.extproc_port") } + if c.PolicyEngine.Server.Mode == "tcp" && c.PolicyEngine.Metrics.Port == c.PolicyEngine.Server.UpstreamExtProcPort { + return fmt.Errorf("metrics.port cannot be same as server.upstream_extproc_port") + } if c.PolicyEngine.Metrics.Port == c.PolicyEngine.Admin.Port { return fmt.Errorf("metrics.port cannot be same as admin.port") } diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go index c9e9d51a44..dc83e67857 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config_test.go @@ -34,7 +34,8 @@ func validConfig() *Config { return &Config{ PolicyEngine: PolicyEngine{ Server: ServerConfig{ - ExtProcPort: 9001, + ExtProcPort: 9001, + UpstreamExtProcPort: 9004, }, Admin: AdminConfig{ Enabled: true, diff --git a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go index 9d893ab9c6..f588d2e16b 100644 --- a/gateway/gateway-runtime/policy-engine/internal/constants/constants.go +++ b/gateway/gateway-runtime/policy-engine/internal/constants/constants.go @@ -50,6 +50,12 @@ const ( // Policy Engine Socket Path (matches gateway-controller constant) DefaultPolicyEngineSocketPath = "/var/run/api-platform/policy-engine.sock" + // DefaultUpstreamExtProcSocketPath is the Unix socket for the second, + // upstream-attempt ext_proc server (see internal/kernel/upstream_extproc.go), + // distinct from DefaultPolicyEngineSocketPath's per-listener downstream + // server. + DefaultUpstreamExtProcSocketPath = "/var/run/api-platform/policy-engine-upstream.sock" + // Gateway Analytics Socket Path (matches gateway-controller constant) DefaultALSSocketPath = "/var/run/api-platform/gateway-analytics.sock" diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go b/gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go index 3ac5856023..f1812455d9 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/extproc.go @@ -516,9 +516,11 @@ func (s *ExternalProcessorServer) initializeExecutionContext(ctx context.Context return &routeMetadata } -// extractRouteKey extracts just the route key (xds.route_name) from the request attributes. -// This is a lightweight extraction that avoids parsing route metadata. -func (s *ExternalProcessorServer) extractRouteKey(req *extprocv3.ProcessingRequest) string { +// extractRouteKeyFromAttributes extracts just the route key (xds.route_name) +// from the request attributes — shared by both the downstream ExternalProcessorServer +// and the upstream-attempt UpstreamExternalProcessorServer (Task 3), since both +// receive the identical ext_proc request-attributes shape. +func extractRouteKeyFromAttributes(req *extprocv3.ProcessingRequest) string { if req.Attributes == nil { return "default" } @@ -534,6 +536,12 @@ func (s *ExternalProcessorServer) extractRouteKey(req *extprocv3.ProcessingReque return "default" } +// extractRouteKey extracts just the route key (xds.route_name) from the request attributes. +// This is a lightweight extraction that avoids parsing route metadata. +func (s *ExternalProcessorServer) extractRouteKey(req *extprocv3.ProcessingRequest) string { + return extractRouteKeyFromAttributes(req) +} + // skipAllProcessing returns a response that skips all processing phases func (s *ExternalProcessorServer) skipAllProcessing(routeMetadata RouteMetadata) *extprocv3.ProcessingResponse { // Build analytics metadata using route metadata even when skipping policy processing diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/extproc_test.go b/gateway/gateway-runtime/policy-engine/internal/kernel/extproc_test.go index e7decbc2a4..92b038beb4 100644 --- a/gateway/gateway-runtime/policy-engine/internal/kernel/extproc_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/extproc_test.go @@ -497,3 +497,13 @@ func TestInitializeExecutionContext_WithPolicyChain(t *testing.T) { assert.Equal(t, "/api/v1/pets", execCtx.requestBodyCtx.Path) assert.Equal(t, "GET", execCtx.requestBodyCtx.Method) } + +// TestExtractRouteKeyFromAttributes_MissingAttributesReturnsDefault tests that +// the free function extractRouteKeyFromAttributes returns "default" when the +// request has no attributes. +func TestExtractRouteKeyFromAttributes_MissingAttributesReturnsDefault(t *testing.T) { + req := &extprocv3.ProcessingRequest{} + if got := extractRouteKeyFromAttributes(req); got != "default" { + t.Errorf("got %q, want %q", got, "default") + } +} diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go b/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go new file mode 100644 index 0000000000..ba90e0ee2d --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc.go @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package kernel + +import ( + "context" + "io" + "log/slog" + "strconv" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// UpstreamExternalProcessorServer is the second, minimal ext_proc gRPC server +// hosted in this same policy-engine process — wired into Envoy's per-cluster +// UPSTREAM HTTP filter chain (see gateway-controller's Task 8), not the +// per-listener downstream chain ExternalProcessorServer (extproc.go) serves. +// It handles only the request-headers phase: this filter attachment point +// has no sensible response phase or body phase for this feature (see the +// design doc). It resolves route -> policy chain via the exact same +// in-memory registry the downstream server uses (s.kernel.GetPolicyChain), +// so a policy's cached state (e.g. oauth2-generator's token cache) is +// naturally shared between both entry points with zero duplication. +type UpstreamExternalProcessorServer struct { + extprocv3.UnimplementedExternalProcessorServer + kernel *Kernel +} + +// NewUpstreamExternalProcessorServer constructs the server. k must be the +// same *Kernel instance the downstream ExternalProcessorServer uses (see +// cmd/policy-engine/main.go, Task 4) — this is what makes chain/state sharing +// automatic rather than something this type has to arrange itself. +func NewUpstreamExternalProcessorServer(k *Kernel) *UpstreamExternalProcessorServer { + return &UpstreamExternalProcessorServer{kernel: k} +} + +// Process implements extprocv3.ExternalProcessorServer. Unlike the downstream +// server's Process (extproc.go), this one only ever expects RequestHeaders +// messages (the cluster's upstream filter is configured with +// RequestHeaderMode: SEND and every other mode left at its default NONE, see +// Task 8) — any other message type gets an empty continue response rather +// than an error, since failing this path must never break the retry itself +// (see Global Constraints: fail open). +func (s *UpstreamExternalProcessorServer) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + ctx := stream.Context() + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + var resp *extprocv3.ProcessingResponse + switch req.Request.(type) { + case *extprocv3.ProcessingRequest_RequestHeaders: + resp, err = s.processRequestHeaders(ctx, req) + if err != nil { + slog.ErrorContext(ctx, "upstream ext_proc: failed to process request headers, failing open", "error", err) + resp = emptyContinueRequestHeadersResponse() + } + default: + resp = emptyContinueRequestHeadersResponse() + } + + if err := stream.Send(resp); err != nil { + return status.Errorf(codes.Internal, "upstream ext_proc: failed to send response: %v", err) + } + } +} + +// emptyContinueRequestHeadersResponse is the fail-open / no-op response: no +// header mutation, request proceeds unchanged. +func emptyContinueRequestHeadersResponse() *extprocv3.ProcessingResponse { + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{}, + }, + }, + } +} + +// processRequestHeaders resolves the route's policy chain and dispatches to +// every policy implementing UpstreamAttemptPolicy, in chain order. A policy +// that doesn't implement it (the common case — rate limiting, analytics, +// transforms) is silently skipped via the type assertion; this is what makes +// the mechanism generic with zero per-policy wiring in this server. +func (s *UpstreamExternalProcessorServer) processRequestHeaders(ctx context.Context, req *extprocv3.ProcessingRequest) (*extprocv3.ProcessingResponse, error) { + routeKey := extractRouteKeyFromAttributes(req) + chain := s.kernel.GetPolicyChain(routeKey) + if chain == nil { + return emptyContinueRequestHeadersResponse(), nil + } + + headers := req.GetRequestHeaders() + attemptCount := 1 + headersMap := make(map[string][]string) + if headers.GetHeaders() != nil { + for _, h := range headers.GetHeaders().GetHeaders() { + key := h.Key + value := string(h.RawValue) + headersMap[key] = append(headersMap[key], value) + if key == "x-envoy-attempt-count" { + if n, err := strconv.Atoi(value); err == nil && n > 0 { + attemptCount = n + } + } + } + } + + actx := &policy.UpstreamAttemptContext{ + AttemptCount: attemptCount, + Headers: policy.NewHeaders(headersMap), + } + + headersToSet := make(map[string]string) + for _, p := range chain.Policies { + attemptPolicy, ok := p.(policy.UpstreamAttemptPolicy) + if !ok { + continue + } + action := attemptPolicy.OnUpstreamAttemptRequestHeaders(ctx, actx) + mods, ok := action.(policy.UpstreamAttemptHeaderModifications) + if !ok { + continue + } + for k, v := range mods.HeadersToSet { + headersToSet[k] = v + } + } + + if len(headersToSet) == 0 { + return emptyContinueRequestHeadersResponse(), nil + } + + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_RequestHeaders{ + RequestHeaders: &extprocv3.HeadersResponse{ + Response: &extprocv3.CommonResponse{ + HeaderMutation: buildHeaderValueOptions(headersToSet), + }, + }, + }, + }, nil +} diff --git a/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc_test.go b/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc_test.go new file mode 100644 index 0000000000..f5b800344f --- /dev/null +++ b/gateway/gateway-runtime/policy-engine/internal/kernel/upstream_extproc_test.go @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package kernel + +import ( + "context" + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "github.com/wso2/api-platform/gateway/gateway-runtime/policy-engine/internal/registry" + policy "github.com/wso2/api-platform/sdk/core/policy/v1alpha2" + structpb "google.golang.org/protobuf/types/known/structpb" +) + +// fakeUpstreamAttemptPolicy proves the dispatch loop invokes exactly the +// policies implementing UpstreamAttemptPolicy, via type assertion, ignoring +// every other policy in the chain (rate-limit/analytics-shaped policies that +// don't implement it). +type fakeUpstreamAttemptPolicy struct{ lastAttempt int } + +func (p *fakeUpstreamAttemptPolicy) Mode() policy.ProcessingMode { return policy.ProcessingMode{} } +func (p *fakeUpstreamAttemptPolicy) OnUpstreamAttemptRequestHeaders(_ context.Context, actx *policy.UpstreamAttemptContext) policy.UpstreamAttemptAction { + p.lastAttempt = actx.AttemptCount + if actx.AttemptCount <= 1 { + return policy.UpstreamAttemptHeaderModifications{} + } + return policy.UpstreamAttemptHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed"}} +} + +// nonParticipatingPolicy implements only the base Policy interface — proves +// the dispatch loop skips it via type assertion, not a hardcoded name check. +type nonParticipatingPolicy struct{} + +func (nonParticipatingPolicy) Mode() policy.ProcessingMode { return policy.ProcessingMode{} } + +// newTestRouteConfigAndChain builds a Kernel with a policy chain registered +// under routeKey, using the package's real chain-registration entry point +// (RegisterRoute — see mapper.go and its use throughout kernel_test.go / +// body_mode_test.go / extproc_test.go) rather than a new test-only setter: +// processRequestHeaders only ever reads the chain via Kernel.GetPolicyChain, +// which RegisterRoute already populates, so no additional mechanism is +// needed. +func newTestRouteConfigAndChain(t *testing.T, routeKey string, chain *registry.PolicyChain) *Kernel { + t.Helper() + k := NewKernel() + k.RegisterRoute(routeKey, chain) + return k +} + +func attrsFor(routeKey string) map[string]*structpb.Struct { + return map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + "xds.route_name": structpb.NewStringValue(routeKey), + }, + }, + } +} + +func TestUpstreamExtProc_DispatchesOnlyToImplementingPolicies(t *testing.T) { + fp := &fakeUpstreamAttemptPolicy{} + chain := ®istry.PolicyChain{Policies: []policy.Policy{nonParticipatingPolicy{}, fp}} + k := newTestRouteConfigAndChain(t, "test-route", chain) + s := NewUpstreamExternalProcessorServer(k) + + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("test-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{ + Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{ + {Key: "x-envoy-attempt-count", RawValue: []byte("2")}, + }}, + }, + }, + } + + resp, err := s.processRequestHeaders(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fp.lastAttempt != 2 { + t.Fatalf("expected the implementing policy to observe AttemptCount=2, got %d", fp.lastAttempt) + } + rh, ok := resp.Response.(*extprocv3.ProcessingResponse_RequestHeaders) + if !ok { + t.Fatalf("expected a RequestHeaders response, got %T", resp.Response) + } + mutation := rh.RequestHeaders.GetResponse().GetHeaderMutation() + if mutation == nil || len(mutation.SetHeaders) != 1 || string(mutation.SetHeaders[0].Header.RawValue) != "Bearer refreshed" { + t.Fatalf("expected the refreshed Authorization header to be set, got %#v", mutation) + } +} + +func TestUpstreamExtProc_MissingAttemptCountHeaderTreatedAsOne(t *testing.T) { + fp := &fakeUpstreamAttemptPolicy{} + chain := ®istry.PolicyChain{Policies: []policy.Policy{fp}} + k := newTestRouteConfigAndChain(t, "test-route", chain) + s := NewUpstreamExternalProcessorServer(k) + + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("test-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{}}, + }, + } + if _, err := s.processRequestHeaders(context.Background(), req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fp.lastAttempt != 1 { + t.Fatalf("expected a missing attempt-count header to be treated as attempt 1, got %d", fp.lastAttempt) + } +} + +func TestUpstreamExtProc_UnknownRouteReturnsEmptyContinue(t *testing.T) { + k := NewKernel() + s := NewUpstreamExternalProcessorServer(k) + req := &extprocv3.ProcessingRequest{ + Attributes: attrsFor("no-such-route"), + Request: &extprocv3.ProcessingRequest_RequestHeaders{ + RequestHeaders: &extprocv3.HttpHeaders{Headers: &corev3.HeaderMap{}}, + }, + } + resp, err := s.processRequestHeaders(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + rh, ok := resp.Response.(*extprocv3.ProcessingResponse_RequestHeaders) + if !ok || rh.RequestHeaders.GetResponse().GetHeaderMutation() != nil { + t.Fatalf("expected an empty continue response for an unknown route, got %#v", resp.Response) + } +} diff --git a/gateway/spec/prd.md b/gateway/spec/prd.md index 47954e8e95..dad764f4b2 100644 --- a/gateway/spec/prd.md +++ b/gateway/spec/prd.md @@ -11,6 +11,7 @@ Production-ready Envoy-based gateway with Go xDS control plane, providing dynami - [FR3: SQLite Persistence](prds/sqlite-persistence.md) – Persist configurations to SQLite database with WAL mode, composite unique constraints, and migration path to PostgreSQL/MySQL. - [FR4: Zero-Downtime Updates](prds/zero-downtime-updates.md) – Apply configuration changes without dropping in-flight requests using graceful xDS updates. - [FR5: Policy Engine Integration](prds/policy-engine.md) – Policy-first architecture with authentication, authorization, rate limiting, and custom policy support. +- [FR6: Upstream OAuth2 Authentication](prds/oauth2-upstream-auth.md) – Authenticate outbound requests to OAuth2-secured LLM providers/proxies via a first-party policy, with two-tier Redis-backed token caching. ## Non-Functional Requirements diff --git a/sdk/core/go.mod b/sdk/core/go.mod index fa1ad4c5b9..b43acbabf8 100644 --- a/sdk/core/go.mod +++ b/sdk/core/go.mod @@ -1,3 +1,15 @@ module github.com/wso2/api-platform/sdk/core go 1.26.2 + +require ( + github.com/alicebob/miniredis/v2 v2.38.0 + github.com/redis/go-redis/v9 v9.22.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect +) diff --git a/sdk/core/go.sum b/sdk/core/go.sum new file mode 100644 index 0000000000..e9ba59b600 --- /dev/null +++ b/sdk/core/go.sum @@ -0,0 +1,26 @@ +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/sdk/core/policy/v1alpha2/action.go b/sdk/core/policy/v1alpha2/action.go index bf5acba4d2..266fa27f87 100644 --- a/sdk/core/policy/v1alpha2/action.go +++ b/sdk/core/policy/v1alpha2/action.go @@ -17,6 +17,8 @@ package policyv1alpha2 +import "context" + // DropHeaderAction controls which headers appear in the analytics event. type DropHeaderAction struct { Action string // "allow" (allowlist) or "deny" (denylist) @@ -283,3 +285,38 @@ type TerminateResponseChunk struct { func (TerminateResponseChunk) isStreamingResponseAction() {} func (TerminateResponseChunk) TerminateStream() bool { return true } + +// ─── Upstream-attempt action (sealed oneof, one variant) ───────────────────── +// +// UpstreamAttemptAction is deliberately a sealed interface with exactly one +// concrete variant, unlike RequestHeaderAction's two (Modifications | +// ImmediateResponse): this phase runs after routing and authentication are +// already resolved, mid-retry-loop inside Envoy's router filter, where there +// is no sensible notion of "reject this request" — only "optionally change +// headers for this one attempt." + +// UpstreamAttemptAction is the sealed oneof returned by +// UpstreamAttemptPolicy.OnUpstreamAttemptRequestHeaders. +type UpstreamAttemptAction interface { + isUpstreamAttemptAction() +} + +// UpstreamAttemptHeaderModifications sets the given headers on this specific +// upstream attempt. An empty/nil HeadersToSet is a valid, common no-op (e.g. +// AttemptCount == 1, nothing to refresh yet, or a fail-open path after an +// error). +type UpstreamAttemptHeaderModifications struct { + HeadersToSet map[string]string +} + +func (UpstreamAttemptHeaderModifications) isUpstreamAttemptAction() {} + +// UpstreamAttemptPolicy is implemented by any policy that wants to attach +// fresh, per-attempt state (e.g. a refreshed credential) to an Envoy-native +// retry. Discovery is a plain type assertion by the kernel — see Task 3 — +// never a hardcoded policy name. A policy implements this in addition to, +// not instead of, its normal RequestHeaderPolicy/ResponseHeaderPolicy +// interfaces. +type UpstreamAttemptPolicy interface { + OnUpstreamAttemptRequestHeaders(ctx context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction +} diff --git a/sdk/core/policy/v1alpha2/context.go b/sdk/core/policy/v1alpha2/context.go index 493892c984..dd34474ab3 100644 --- a/sdk/core/policy/v1alpha2/context.go +++ b/sdk/core/policy/v1alpha2/context.go @@ -307,3 +307,25 @@ type ResponseStreamContext struct { // mutation. Upstream *UpstreamResponseContext } + +// ─── Upstream-attempt context (per-dial-attempt, not per-client-request) ───── + +// UpstreamAttemptContext is passed to UpstreamAttemptPolicy.OnUpstreamAttemptRequestHeaders. +// Unlike every other context in this package, it is NOT scoped to one client +// request — it fires once per individual upstream dial attempt, including +// Envoy-native retries, because it runs in Envoy's per-cluster upstream HTTP +// filter chain rather than the per-route listener chain every other policy +// phase in this package uses. +type UpstreamAttemptContext struct { + *SharedContext + + // AttemptCount is Envoy's x-envoy-attempt-count for this specific dial, + // starting at 1. A missing/unparseable header is treated as 1 (fail + // toward "behave like the first attempt", never toward unconditional + // refresh) — see the kernel-side parsing in Task 3. + AttemptCount int + + // Headers are this specific attempt's outgoing request headers, mutable + // via the returned UpstreamAttemptAction. + Headers *Headers +} diff --git a/sdk/core/policy/v1alpha2/upstream_attempt_test.go b/sdk/core/policy/v1alpha2/upstream_attempt_test.go new file mode 100644 index 0000000000..ddbf9dfe9e --- /dev/null +++ b/sdk/core/policy/v1alpha2/upstream_attempt_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.com) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package policyv1alpha2 + +import ( + "context" + "testing" +) + +// fakeUpstreamAttemptPolicy proves any type implementing UpstreamAttemptPolicy +// compiles against the real context.Context/UpstreamAttemptContext/ +// UpstreamAttemptAction types — a compile-time contract test. oauth2-generator's +// own tests (Task 9) cover real refresh behavior. +type fakeUpstreamAttemptPolicy struct{} + +func (fakeUpstreamAttemptPolicy) OnUpstreamAttemptRequestHeaders(_ context.Context, actx *UpstreamAttemptContext) UpstreamAttemptAction { + if actx.AttemptCount <= 1 { + return UpstreamAttemptHeaderModifications{} + } + return UpstreamAttemptHeaderModifications{HeadersToSet: map[string]string{"Authorization": "Bearer refreshed"}} +} + +func TestUpstreamAttemptContext_AttemptCountGatesRefresh(t *testing.T) { + var p UpstreamAttemptPolicy = fakeUpstreamAttemptPolicy{} + + attemptOne := &UpstreamAttemptContext{AttemptCount: 1, Headers: NewHeaders(nil)} + action := p.OnUpstreamAttemptRequestHeaders(context.Background(), attemptOne) + mods, ok := action.(UpstreamAttemptHeaderModifications) + if !ok || len(mods.HeadersToSet) != 0 { + t.Fatalf("attempt 1 must not mutate headers, got %#v", action) + } + + attemptTwo := &UpstreamAttemptContext{AttemptCount: 2, Headers: NewHeaders(nil)} + action2 := p.OnUpstreamAttemptRequestHeaders(context.Background(), attemptTwo) + mods2, ok := action2.(UpstreamAttemptHeaderModifications) + if !ok || mods2.HeadersToSet["Authorization"] != "Bearer refreshed" { + t.Fatalf("attempt 2 must carry the refreshed token, got %#v", action2) + } +} + +// Compile-time interface satisfaction check, mirroring action.go's own convention. +var _ UpstreamAttemptAction = UpstreamAttemptHeaderModifications{} diff --git a/sdk/core/utils/redisclient/redisclient.go b/sdk/core/utils/redisclient/redisclient.go new file mode 100644 index 0000000000..7d2938b760 --- /dev/null +++ b/sdk/core/utils/redisclient/redisclient.go @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package redisclient shares one process-wide *redis.Client (one connection +// pool) per distinct connection configuration, across every caller that +// imports it - see GetOrCreateRedisClient. It also exposes a single +// gateway-wide default client (Shared, backed by the operator's top-level +// "redis" config section - gateway infrastructure, not something scoped to +// policies) that a policy falls back to when it has no Redis config of its +// own - see Resolve. +package redisclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "net" + "strconv" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" +) + +// redisConnKey identifies a distinct Redis connection configuration. Two policy +// instances with identical connection settings share one *redis.Client (one pool). +// +// Excludes TLSConfig and any credentials-provider option - see +// GetOrCreateRedisClient's bypass for those. +type redisConnKey struct { + addr string + username string + passwordHash string // sha256 hex; keeps the secret out of the in-process map key + db int + protocol int + dialTimeout time.Duration + readTimeout time.Duration + writeTimeout time.Duration + poolSize int +} + +// redisClients is the process-wide registry of shared Redis clients. Without it, +// GetPolicy creates a new *redis.Client (a whole connection pool) per policy instance +// and per config reload, leaking pools and exploding Redis connections at scale. +var redisClients = struct { + mu sync.Mutex + m map[redisConnKey]*redis.Client +}{m: make(map[redisConnKey]*redis.Client)} + +func hashRedisPassword(p string) string { + if p == "" { + return "" + } + sum := sha256.Sum256([]byte(p)) + return hex.EncodeToString(sum[:]) +} + +// GetOrCreateRedisClient returns the process-wide shared client for these connection +// settings, creating (and pinging once) it on first use. created reports whether this +// call created the client; pingErr is non-nil only when created and the initial ping +// failed. The client is registered and returned even on ping failure (go-redis +// reconnects lazily). Clients are never closed — they live for the process lifetime. +func GetOrCreateRedisClient(opts *redis.Options, pingTimeout time.Duration) (client *redis.Client, created bool, pingErr error) { + // TLSConfig and credentials-provider hooks can't be fingerprinted + // safely: a *tls.Config's pointer says nothing about its content, and + // Go func values aren't comparable at all. Bypass the registry rather + // than risk silently reusing a client built for a different config. + if opts.TLSConfig != nil || opts.CredentialsProvider != nil || opts.CredentialsProviderContext != nil || opts.StreamingCredentialsProvider != nil { + return newAndPingClient(opts, pingTimeout) + } + + key := redisConnKey{ + addr: opts.Addr, + username: opts.Username, + passwordHash: hashRedisPassword(opts.Password), + db: opts.DB, + protocol: opts.Protocol, + dialTimeout: opts.DialTimeout, + readTimeout: opts.ReadTimeout, + writeTimeout: opts.WriteTimeout, + poolSize: opts.PoolSize, + } + + // Lock guards only the map lookup/insert, never the ping below - mu is + // process-wide, so holding it during a slow/down connection's ping + // would stall every other caller's get-or-create too. A concurrent + // caller for the same key may see the just-inserted client before this + // ping finishes - fine, since a reused client is already "assumed + // healthy" regardless of timing, never gated on this call's pingErr. + redisClients.mu.Lock() + if c, ok := redisClients.m[key]; ok { + redisClients.mu.Unlock() + return c, false, nil + } + c := redis.NewClient(opts) + redisClients.m[key] = c + redisClients.mu.Unlock() + + pingErr = pingClient(c, pingTimeout) + return c, true, pingErr +} + +// pingTimeoutMargin is added on top of a client's own configured dial/read/ +// write timeouts to derive the one-time creation ping's timeout (see +// pingTimeoutFor) - enough for the ping's own command round-trip on top of +// whatever the connection attempt itself is allowed to take. +const pingTimeoutMargin = 2 * time.Second + +// pingTimeoutFor derives the creation-ping timeout from opts's own configured +// timeouts, so the ping's context stays alive at least as long as the +// connection attempt permitted by DialTimeout (plus the read/write round-trip +// and a safety margin) - a fixed constant shorter than an operator's +// configured DialTimeout would cut the ping's context before a legitimately +// slow-but-successful connection attempt could complete. +func pingTimeoutFor(opts *redis.Options) time.Duration { + return opts.DialTimeout + opts.ReadTimeout + opts.WriteTimeout + pingTimeoutMargin +} + +// shared holds the process-wide gateway-level default client. inited +// distinguishes "InitFromConfig ran and found no redis section" (client nil, +// inited true - Shared reports a config-gap error) from "InitFromConfig was +// never called at all" (a gateway-runtime wiring bug - Shared reports that +// distinctly, since it means something programming-level is missing, not an +// operator config gap). +var shared struct { + mu sync.Mutex + client *redis.Client + inited bool +} + +// InitFromConfig resolves the operator-level top-level "redis" section from +// raw (e.g. cfg.PolicyEngine.RawConfig - raw's other top-level sections like +// "analytics"/"router"/"policy_configurations" are ignored here) and creates +// the process-wide shared client. This is gateway-wide infrastructure, not +// something scoped to policies - deliberately NOT nested under +// "policy_configurations" (that namespace is policy-engine's own ${config...} +// CEL-resolution mechanism for per-policy system parameters; a shared +// resource other gateway components could reach doesn't belong inside it). +// Must be called exactly once, at gateway-runtime startup, before any policy +// factory runs - see Shared and Resolve. A missing "redis" key is not an +// error: most gateways may have zero Redis-consuming policies configured, +// and that absence only matters lazily, the first time some policy actually +// calls Shared. A connection/ping failure is likewise not fatal here - the +// client is still created and stored (go-redis reconnects lazily), matching +// GetOrCreateRedisClient's own create-time philosophy. +func InitFromConfig(raw map[string]interface{}) error { + shared.mu.Lock() + defer shared.mu.Unlock() + if shared.inited { + return fmt.Errorf("redisclient: InitFromConfig called more than once") + } + shared.inited = true + + opts, err := resolveOptionsFromConfig(raw) + if err != nil { + return fmt.Errorf("redisclient: invalid \"redis\" config: %w", err) + } + if opts == nil { + return nil + } + + c, _, _ := newAndPingClient(opts, pingTimeoutFor(opts)) + shared.client = c + return nil +} + +// Shared returns the process-wide gateway-level default client, backed by +// the top-level "redis" config section. It errors if InitFromConfig was +// never called (a gateway-runtime wiring bug, not a normal runtime +// condition) or if no "redis" section was configured at all - callers must +// treat the latter as a real configuration gap rather than assuming a +// shared Redis is always available. +func Shared() (*redis.Client, error) { + shared.mu.Lock() + defer shared.mu.Unlock() + if !shared.inited { + return nil, fmt.Errorf("redisclient: Shared() called before InitFromConfig") + } + if shared.client == nil { + return nil, fmt.Errorf(`redisclient: no shared redis configured ("redis" section)`) + } + return shared.client, nil +} + +// Resolve returns the client a policy instance should use: opts's own +// connection settings when the policy resolved one from its own config +// section, otherwise the gateway-level Shared client. +// +// opts must be nil - never a zero-value *redis.Options - when the policy's +// own section was absent. A struct pre-filled with the policy's schema +// defaults would always look "configured," and this fallback would never +// trigger; the presence check belongs to the caller's own config-extraction +// code, on whichever field has no default (e.g. host). +func Resolve(opts *redis.Options, pingTimeout time.Duration) (*redis.Client, error) { + if opts == nil { + return Shared() + } + client, _, _ := GetOrCreateRedisClient(opts, pingTimeout) + return client, nil +} + +// newAndPingClient creates a client and pings it once. created is always +// true - only present so this matches GetOrCreateRedisClient's own return +// shape at its call sites. +func newAndPingClient(opts *redis.Options, pingTimeout time.Duration) (client *redis.Client, created bool, pingErr error) { + c := redis.NewClient(opts) + return c, true, pingClient(c, pingTimeout) +} + +// pingClient pings an already-constructed client once, bounded by +// pingTimeout. Split out from newAndPingClient because GetOrCreateRedisClient's +// main path must insert the client into the registry BEFORE pinging (so a +// concurrent caller for the same key sees it immediately), not create-then-ping +// as one atomic step. +func pingClient(c *redis.Client, pingTimeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + defer cancel() + return c.Ping(ctx).Err() +} + +// resolveOptionsFromConfig extracts *redis.Options from raw["redis"] - a +// top-level section, sibling to "router"/"analytics"/etc in the gateway's +// complete config tree, not nested under "policy_configurations" (this is +// gateway-wide infrastructure, not a per-policy setting) - using the +// operator-facing defaults (host "localhost", port 6379, db 0, poolSize 0 +// (go-redis default), connectionTimeout 5s, readTimeout/writeTimeout 3s). +// Returns (nil, nil) - not an error - when raw has no "redis" key at all. +func resolveOptionsFromConfig(raw map[string]interface{}) (*redis.Options, error) { + section, ok := raw["redis"] + if !ok || section == nil { + return nil, nil + } + m, ok := section.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf(`"redis" must be a table, got %T`, section) + } + + connectionTimeout, err := durationParam(m, "connection_timeout", 5*time.Second) + if err != nil { + return nil, fmt.Errorf("connection_timeout: %w", err) + } + readTimeout, err := durationParam(m, "read_timeout", 3*time.Second) + if err != nil { + return nil, fmt.Errorf("read_timeout: %w", err) + } + writeTimeout, err := durationParam(m, "write_timeout", 3*time.Second) + if err != nil { + return nil, fmt.Errorf("write_timeout: %w", err) + } + port, err := intParam(m, "port", 6379) + if err != nil { + return nil, fmt.Errorf("port: %w", err) + } + db, err := intParam(m, "db", 0) + if err != nil { + return nil, fmt.Errorf("db: %w", err) + } + poolSize, err := intParam(m, "pool_size", 0) + if err != nil { + return nil, fmt.Errorf("pool_size: %w", err) + } + + host, err := stringParam(m, "host", "localhost") + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } + username, err := stringParam(m, "username", "") + if err != nil { + return nil, fmt.Errorf("username: %w", err) + } + password, err := stringParam(m, "password", "") + if err != nil { + return nil, fmt.Errorf("password: %w", err) + } + + return &redis.Options{ + Addr: net.JoinHostPort(host, strconv.Itoa(port)), + Username: username, + Password: password, + DB: db, + DialTimeout: connectionTimeout, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + PoolSize: poolSize, + }, nil +} + +// stringParam/intParam/durationParam read key from m, applying def when the +// key is absent or nil. They error on a present-but-wrong-shaped value +// rather than silently falling back to def - a typo'd config value should +// surface at startup, not resolve to a default the operator never asked for. +func stringParam(m map[string]interface{}, key, def string) (string, error) { + v, ok := m[key] + if !ok || v == nil { + return def, nil + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("expected a string, got %T", v) + } + return s, nil +} + +func intParam(m map[string]interface{}, key string, def int) (int, error) { + v, ok := m[key] + if !ok || v == nil { + return def, nil + } + switch n := v.(type) { + case int: + return n, nil + case int64: + return int(n), nil + case float64: + if math.IsNaN(n) || math.IsInf(n, 0) { + return 0, fmt.Errorf("expected an integer, got %v", n) + } + if n != math.Trunc(n) { + return 0, fmt.Errorf("expected an integer, got non-integer value %v", n) + } + if n < float64(math.MinInt) || n > float64(math.MaxInt) { + return 0, fmt.Errorf("value %v out of range for int", n) + } + return int(n), nil + case string: + // A TOML value written as {{ env "VAR" "default" }} must be a quoted + // string literal (TOML has no unquoted template syntax) - gateway-runtime's + // config interpolation resolves the token in place but never changes the + // field's type, so a numeric config value arrives here as a numeric + // string, not an int. Reject anything that isn't actually numeric. + parsed, err := strconv.Atoi(strings.TrimSpace(n)) + if err != nil { + return 0, fmt.Errorf("expected an integer, got non-numeric string %q", n) + } + return parsed, nil + default: + return 0, fmt.Errorf("expected an integer, got %T", v) + } +} + +func durationParam(m map[string]interface{}, key string, def time.Duration) (time.Duration, error) { + v, ok := m[key] + if !ok || v == nil { + return def, nil + } + switch d := v.(type) { + case string: + parsed, err := time.ParseDuration(d) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: %w", d, err) + } + return parsed, nil + case time.Duration: + return d, nil + default: + return 0, fmt.Errorf("expected a duration string, got %T", v) + } +} diff --git a/sdk/core/utils/redisclient/redisclient_test.go b/sdk/core/utils/redisclient/redisclient_test.go new file mode 100644 index 0000000000..613d4829ac --- /dev/null +++ b/sdk/core/utils/redisclient/redisclient_test.go @@ -0,0 +1,562 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package redisclient + +import ( + "context" + "crypto/tls" + "math" + "net" + "strconv" + "sync" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func TestGetOrCreateClient_SharesClientForIdenticalConfig(t *testing.T) { + mr := miniredis.RunT(t) + opts := &redis.Options{Addr: mr.Addr(), DB: 0} + + c1, created1, err1 := GetOrCreateRedisClient(opts, time.Second) + if !created1 || err1 != nil { + t.Fatalf("first call: created=%v err=%v (want true,nil)", created1, err1) + } + + c2, created2, err2 := GetOrCreateRedisClient(opts, time.Second) + if created2 || err2 != nil { + t.Fatalf("second call: created=%v err=%v (want false,nil)", created2, err2) + } + if c1 != c2 { + t.Error("expected identical connection settings to share one *redis.Client") + } +} + +func TestGetOrCreateClient_DistinctClientForDifferentConfig(t *testing.T) { + mr := miniredis.RunT(t) + + c1, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), DB: 0}, time.Second) + c2, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), DB: 1}, time.Second) + + if c1 == c2 { + t.Error("expected different DB selection to produce a distinct *redis.Client") + } +} + +func TestGetOrCreateClient_DifferentPasswordProducesDistinctClient(t *testing.T) { + mr := miniredis.RunT(t) + + c1, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), Password: "one"}, time.Second) + c2, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), Password: "two"}, time.Second) + c3, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr()}, time.Second) // no password at all + + if c1 == c2 { + t.Error("expected different passwords to produce distinct clients") + } + if c1 == c3 || c2 == c3 { + t.Error("expected an absent password not to collide with a present one") + } +} + +func TestGetOrCreateClient_SharedAcrossSimulatedPolicies(t *testing.T) { + mr := miniredis.RunT(t) + opts := func() *redis.Options { return &redis.Options{Addr: mr.Addr(), DB: 0} } + + // Two distinct call sites with identical settings must share one + // client - the whole point of centralizing the registry. + fromPolicyA, _, _ := GetOrCreateRedisClient(opts(), time.Second) + fromPolicyB, _, _ := GetOrCreateRedisClient(opts(), time.Second) + + if fromPolicyA != fromPolicyB { + t.Fatal("expected two distinct callers with identical config to share one client") + } + + ctx := context.Background() + if err := fromPolicyA.Set(ctx, "shared-key", "value", 0).Err(); err != nil { + t.Fatalf("unexpected error writing via the shared client: %v", err) + } + got, err := fromPolicyB.Get(ctx, "shared-key").Result() + if err != nil { + t.Fatalf("unexpected error reading via the shared client: %v", err) + } + if got != "value" { + t.Errorf("got %q, want %q", got, "value") + } +} + +// TestGetOrCreateClient_ReuseSkipsPing locks in that only creation pings - +// a reused client is assumed healthy and must never be re-pinged. +func TestGetOrCreateClient_ReuseSkipsPing(t *testing.T) { + mr := miniredis.RunT(t) + addr := mr.Addr() // capture before mr.Close() below + opts := &redis.Options{Addr: addr, DB: 0} + + c1, created1, err1 := GetOrCreateRedisClient(opts, time.Second) + if !created1 || err1 != nil { + t.Fatalf("first call: created=%v err=%v (want true,nil)", created1, err1) + } + + mr.Close() + c2, created2, err2 := GetOrCreateRedisClient(opts, time.Second) + if created2 || err2 != nil || c2 != c1 { + t.Fatalf("reuse after Redis went down should skip the ping: created=%v err=%v same=%v", created2, err2, c2 == c1) + } +} + +func TestGetOrCreateClient_DifferentProtocolProducesDistinctClient(t *testing.T) { + mr := miniredis.RunT(t) + + c1, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), Protocol: 2}, time.Second) + c2, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), Protocol: 3}, time.Second) + c3, _, _ := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), Protocol: 2}, time.Second) + + if c1 == c2 { + t.Error("expected different RESP protocol versions to produce distinct clients") + } + if c1 != c3 { + t.Error("expected the same protocol version to reuse the existing client") + } +} + +// TestGetOrCreateClient_TLSConfigBypassesRegistry locks in that a TLSConfig +// always gets a fresh, unshared client, even with otherwise-identical +// options - neither it nor a credentials-provider func can be fingerprinted +// safely, so sharing would risk a silent cross-config mixup. +func TestGetOrCreateClient_TLSConfigBypassesRegistry(t *testing.T) { + mr := miniredis.RunT(t) + + optsA := &redis.Options{Addr: mr.Addr(), TLSConfig: &tls.Config{}} //nolint:gosec // test-only, no real handshake asserted + optsB := &redis.Options{Addr: mr.Addr(), TLSConfig: &tls.Config{}} //nolint:gosec + + c1, created1, _ := GetOrCreateRedisClient(optsA, time.Second) + c2, created2, _ := GetOrCreateRedisClient(optsB, time.Second) + + if !created1 || !created2 { + t.Fatalf("expected every TLSConfig-bearing call to report created=true (never reused), got %v and %v", created1, created2) + } + if c1 == c2 { + t.Error("expected two TLSConfig-bearing calls to never share a client, even with identical-looking options") + } +} + +func TestGetOrCreateClient_CredentialsProviderBypassesRegistry(t *testing.T) { + mr := miniredis.RunT(t) + provider := func() (string, string) { return "", "" } + + c1, created1, err1 := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), CredentialsProvider: provider}, time.Second) + c2, created2, err2 := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), CredentialsProvider: provider}, time.Second) + + if !created1 || err1 != nil { + t.Fatalf("first call: created=%v err=%v (want true,nil)", created1, err1) + } + if !created2 || err2 != nil { + t.Fatalf("second call: created=%v err=%v (want true,nil - bypassed, not reused)", created2, err2) + } + if c1 == c2 { + t.Error("expected two CredentialsProvider-bearing calls to never share a client") + } +} + +// TestGetOrCreateClient_DoesNotHoldLockDuringPing proves the registry lock +// guards only the map lookup/insert, never c.Ping - mu is process-wide, so +// holding it during a slow/unreachable Redis's ping would stall every other +// caller too, even for an unrelated, healthy endpoint. +func TestGetOrCreateClient_DoesNotHoldLockDuringPing(t *testing.T) { + // Accepts but never responds, so Ping against it blocks until the + // deadline - a reliable window to prove a concurrent, unrelated key + // isn't blocked by it. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to start hanging listener: %v", err) + } + defer func() { _ = ln.Close() }() + + // Signaled once the slow client's connection is actually accepted - + // proof it has dialed and is now blocked reading the Ping reply, rather + // than guessing via a fixed sleep how long that takes to happen. + accepted := make(chan struct{}) + var acceptedOnce sync.Once + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + acceptedOnce.Do(func() { close(accepted) }) + _ = conn // held open, never responded to + } + }() + + done := make(chan struct{}) + go func() { + defer close(done) + // ReadTimeout set explicitly - the dial succeeds, it's the + // read-for-a-reply that hangs, and go-redis's default (5s) would + // otherwise bound that wait regardless of pingTimeout. + _, _, _ = GetOrCreateRedisClient(&redis.Options{ + Addr: ln.Addr().String(), + DB: 0, + ReadTimeout: time.Second, + WriteTimeout: time.Second, + }, time.Second) + }() + + select { + case <-accepted: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for the slow client's connection to be accepted") + } + + mr := miniredis.RunT(t) + fastStart := time.Now() + if _, _, err := GetOrCreateRedisClient(&redis.Options{Addr: mr.Addr(), DB: 1}, 500*time.Millisecond); err != nil { + t.Fatalf("unexpected error on the fast, unrelated key: %v", err) + } + if elapsed := time.Since(fastStart); elapsed > 300*time.Millisecond { + t.Errorf("expected the unrelated key's get-or-create to complete quickly (the registry lock must not be held during the other call's ping), took %s", elapsed) + } + + <-done // let the slow goroutine finish before the test exits +} + +// resetSharedForTest clears the package-level shared client state before a +// test runs (so InitFromConfig can be called again despite its once-only +// guard) and restores whatever was there before once the test ends. White-box +// access is fine here - this file is part of the package. +func resetSharedForTest(t *testing.T) { + t.Helper() + shared.mu.Lock() + prevClient, prevInited := shared.client, shared.inited + shared.client, shared.inited = nil, false + shared.mu.Unlock() + t.Cleanup(func() { + shared.mu.Lock() + shared.client, shared.inited = prevClient, prevInited + shared.mu.Unlock() + }) +} + +func TestResolveOptionsFromConfig_NoRedisSectionReturnsNil(t *testing.T) { + opts, err := resolveOptionsFromConfig(map[string]interface{}{}) + if err != nil || opts != nil { + t.Fatalf("got opts=%v err=%v, want nil,nil when \"redis\" is absent entirely", opts, err) + } +} + +// TestResolveOptionsFromConfig_IgnoresSiblingSections proves resolveOptionsFromConfig +// looks at the top-level "redis" key only - other top-level sections (including +// policy_configurations, which is a separate, policy-engine-internal namespace +// this package deliberately does NOT nest under) have no bearing on it. +func TestResolveOptionsFromConfig_IgnoresSiblingSections(t *testing.T) { + raw := map[string]interface{}{ + "router": map[string]interface{}{"gateway_host": "*"}, + "policy_configurations": map[string]interface{}{"oauth2_generator_v1": map[string]interface{}{"redis": map[string]interface{}{"key_prefix": "x:"}}}, + } + opts, err := resolveOptionsFromConfig(raw) + if err != nil || opts != nil { + t.Fatalf("got opts=%v err=%v, want nil,nil when top-level \"redis\" is absent (unrelated sibling sections present)", opts, err) + } +} + +func TestResolveOptionsFromConfig_AppliesDefaults(t *testing.T) { + opts, err := resolveOptionsFromConfig(map[string]interface{}{"redis": map[string]interface{}{}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := &redis.Options{ + Addr: "localhost:6379", + DialTimeout: 5 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + } + if opts.Addr != want.Addr || opts.DialTimeout != want.DialTimeout || + opts.ReadTimeout != want.ReadTimeout || opts.WriteTimeout != want.WriteTimeout || + opts.Username != "" || opts.Password != "" || opts.DB != 0 || opts.PoolSize != 0 { + t.Errorf("got %+v, want defaults %+v (username/password/db/poolSize zero-valued)", opts, want) + } +} + +// TestResolveOptionsFromConfig_ParsesConfiguredValues covers both the string +// (typical koanf/TOML decode) and numeric (int64/float64 - decoder-dependent) +// shapes a value might arrive in. +func TestResolveOptionsFromConfig_ParsesConfiguredValues(t *testing.T) { + raw := map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "redis.example.com", + "port": int64(6380), + "username": "app", + "password": "secret", + "db": float64(2), + "connection_timeout": "10s", + "read_timeout": "7s", + "write_timeout": "7s", + "pool_size": 20, + }, + } + opts, err := resolveOptionsFromConfig(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Addr != "redis.example.com:6380" || opts.Username != "app" || opts.Password != "secret" || + opts.DB != 2 || opts.DialTimeout != 10*time.Second || opts.ReadTimeout != 7*time.Second || + opts.WriteTimeout != 7*time.Second || opts.PoolSize != 20 { + t.Errorf("got %+v, did not match configured values", opts) + } +} + +// TestResolveOptionsFromConfig_ParsesNumericStringPort locks in the shape +// gateway-runtime's own config interpolation actually produces: a TOML value +// written as {{ env "VAR" "6379" }} must be a quoted string literal (TOML has +// no unquoted template syntax), and interpolation resolves the token in place +// without ever changing the field's type - so a "numeric" config.toml value +// arrives here as a numeric string, not an int, even though int/int64/float64 +// are also accepted (e.g. from a JSON-sourced config path). +func TestResolveOptionsFromConfig_ParsesNumericStringPort(t *testing.T) { + raw := map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "redis.example.com", + "port": "6380", + "db": "2", + }, + } + opts, err := resolveOptionsFromConfig(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if opts.Addr != "redis.example.com:6380" || opts.DB != 2 { + t.Errorf("got %+v, want port 6380 and db 2 parsed from numeric strings", opts) + } +} + +// TestResolveOptionsFromConfig_BracketsIPv6Host proves Addr is built via +// net.JoinHostPort - a plain fmt.Sprintf("%s:%d", host, port) would produce +// "::1:6380", which is ambiguous/invalid, instead of the required +// "[::1]:6380". +func TestResolveOptionsFromConfig_BracketsIPv6Host(t *testing.T) { + raw := map[string]interface{}{ + "redis": map[string]interface{}{"host": "::1", "port": 6380}, + } + opts, err := resolveOptionsFromConfig(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if want := "[::1]:6380"; opts.Addr != want { + t.Errorf("got Addr %q, want %q", opts.Addr, want) + } +} + +func TestResolveOptionsFromConfig_RejectsWrongShapedValue(t *testing.T) { + _, err := resolveOptionsFromConfig(map[string]interface{}{ + "redis": map[string]interface{}{"port": "not-a-number"}, + }) + if err == nil { + t.Error("expected an error for a non-numeric port, so a config typo surfaces at startup instead of silently defaulting") + } +} + +// TestIntParam_RejectsInvalidFloat64 locks in that intParam validates a +// float64 before converting it - NaN/Inf/fractional/out-of-range values must +// error rather than silently truncating or converting a NaN/Inf into +// undefined behavior. +func TestIntParam_RejectsInvalidFloat64(t *testing.T) { + cases := []struct { + name string + v float64 + }{ + {"NaN", math.NaN()}, + {"+Inf", math.Inf(1)}, + {"-Inf", math.Inf(-1)}, + {"fractional", 1.5}, + {"aboveMaxInt", 1e19}, + {"belowMinInt", -1e19}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := intParam(map[string]interface{}{"port": c.v}, "port", 6379) + if err == nil { + t.Errorf("expected an error for float64 value %v, got nil", c.v) + } + }) + } +} + +func TestIntParam_AcceptsIntegralFloat64(t *testing.T) { + got, err := intParam(map[string]interface{}{"port": float64(6380)}, "port", 6379) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != 6380 { + t.Errorf("got %d, want 6380", got) + } +} + +func TestResolveOptionsFromConfig_RejectsNonTableSection(t *testing.T) { + _, err := resolveOptionsFromConfig(map[string]interface{}{"redis": "not-a-table"}) + if err == nil { + t.Error("expected an error when \"redis\" isn't a table") + } +} + +func TestInitFromConfig_NoRedisSectionLeavesSharedUnconfigured(t *testing.T) { + resetSharedForTest(t) + + if err := InitFromConfig(map[string]interface{}{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := Shared(); err == nil { + t.Error("expected Shared() to report a config-gap error when no redis section was ever configured") + } +} + +func TestInitFromConfig_CalledTwiceErrors(t *testing.T) { + resetSharedForTest(t) + + if err := InitFromConfig(map[string]interface{}{}); err != nil { + t.Fatalf("first call: unexpected error: %v", err) + } + if err := InitFromConfig(map[string]interface{}{}); err == nil { + t.Error("expected a second InitFromConfig call to error - it must run exactly once") + } +} + +func TestSharedBeforeInitFromConfigErrors(t *testing.T) { + resetSharedForTest(t) + + if _, err := Shared(); err == nil { + t.Error("expected Shared() to error when InitFromConfig was never called") + } +} + +// TestSharedReturnsIdenticalPointer is the actual "single instance" contract: +// not merely "these two configs happen to compare equal" (GetOrCreateRedisClient's +// dedup guarantee) but "there is exactly one gateway-level client, full stop." +func TestSharedReturnsIdenticalPointer(t *testing.T) { + resetSharedForTest(t) + mr := miniredis.RunT(t) + + port, err := strconv.Atoi(mr.Port()) + if err != nil { + t.Fatalf("failed to parse miniredis port: %v", err) + } + raw := map[string]interface{}{"redis": map[string]interface{}{"host": mr.Host(), "port": port}} + if err := InitFromConfig(raw); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + c1, err1 := Shared() + c2, err2 := Shared() + if err1 != nil || err2 != nil { + t.Fatalf("unexpected errors: %v, %v", err1, err2) + } + if c1 != c2 { + t.Error("expected every Shared() call to return the identical *redis.Client pointer") + } +} + +func TestResolve_NilOptsFallsBackToShared(t *testing.T) { + resetSharedForTest(t) + mr := miniredis.RunT(t) + sharedClient := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + SetSharedForTesting(t, sharedClient) + + got, err := Resolve(nil, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != sharedClient { + t.Error("expected Resolve(nil, ...) to return the gateway-level Shared client") + } +} + +func TestResolve_NonNilOptsBypassesShared(t *testing.T) { + resetSharedForTest(t) + sharedMR := miniredis.RunT(t) + SetSharedForTesting(t, redis.NewClient(&redis.Options{Addr: sharedMR.Addr()})) + + overrideMR := miniredis.RunT(t) + got, err := Resolve(&redis.Options{Addr: overrideMR.Addr()}, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Options().Addr != overrideMR.Addr() { + t.Errorf("expected a policy-supplied override to take precedence over the shared client, got client for %q", got.Options().Addr) + } +} + +// TestResolve_NonNilOptsStillDedupes proves Resolve's override branch keeps +// GetOrCreateRedisClient's existing sharing behavior - two policies that both +// explicitly override to the same config still get one pool between them, +// not one pool each. +func TestResolve_NonNilOptsStillDedupes(t *testing.T) { + resetSharedForTest(t) + mr := miniredis.RunT(t) + + c1, err1 := Resolve(&redis.Options{Addr: mr.Addr()}, time.Second) + c2, err2 := Resolve(&redis.Options{Addr: mr.Addr()}, time.Second) + if err1 != nil || err2 != nil { + t.Fatalf("unexpected errors: %v, %v", err1, err2) + } + if c1 != c2 { + t.Error("expected two identical explicit overrides to still share one client") + } +} + +// TestSetSharedForTesting_RestoresPreviousStateAfterTest proves the override +// is scoped to one (sub)test: a subtest's t.Cleanup runs when that subtest +// returns, before the parent continues, so the parent sees the pre-override +// "unconfigured" state again immediately afterward. +func TestSetSharedForTesting_RestoresPreviousStateAfterTest(t *testing.T) { + resetSharedForTest(t) + if err := InitFromConfig(map[string]interface{}{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := Shared(); err == nil { + t.Fatal("expected Shared() to error before any override is set") + } + + t.Run("override active inside subtest", func(t *testing.T) { + mr := miniredis.RunT(t) + SetSharedForTesting(t, redis.NewClient(&redis.Options{Addr: mr.Addr()})) + if _, err := Shared(); err != nil { + t.Fatalf("unexpected error while override was active: %v", err) + } + }) + + if _, err := Shared(); err == nil { + t.Error("expected the override to be reverted once the subtest returned") + } +} + +func TestHashPassword(t *testing.T) { + if hashRedisPassword("") != "" { + t.Error("expected an empty password to hash to empty, not sha256(\"\")") + } + if hashRedisPassword("secret") == "secret" { + t.Error("expected the password to actually be hashed, not passed through") + } + const wantSecretSHA256 = "2bb80d537b1da3e38bd30361aa855686bde0eacd7162fef6a25fe97bf527a25b" + if got := hashRedisPassword("secret"); got != wantSecretSHA256 { + t.Errorf("hashRedisPassword(%q) = %q, want %q", "secret", got, wantSecretSHA256) + } + if hashRedisPassword("secret") == hashRedisPassword("different") { + t.Error("expected different passwords to hash differently") + } +} diff --git a/sdk/core/utils/redisclient/testing.go b/sdk/core/utils/redisclient/testing.go new file mode 100644 index 0000000000..c574da54e0 --- /dev/null +++ b/sdk/core/utils/redisclient/testing.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (https://www.wso2.com). + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package redisclient + +import ( + "testing" + + "github.com/redis/go-redis/v9" +) + +// SetSharedForTesting overrides the process-wide Shared client for the +// duration of t, restoring the previous state automatically via t.Cleanup. +// Test-only - never call this from production code. Since Shared is a +// single global, tests using this helper must not run in parallel with each +// other (no t.Parallel) or they will race on the same override. +func SetSharedForTesting(t testing.TB, client *redis.Client) { + t.Helper() + shared.mu.Lock() + prevClient, prevInited := shared.client, shared.inited + shared.client, shared.inited = client, true + shared.mu.Unlock() + t.Cleanup(func() { + shared.mu.Lock() + shared.client, shared.inited = prevClient, prevInited + shared.mu.Unlock() + }) +}