diff --git a/gateway/configs/config-template.toml b/gateway/configs/config-template.toml index 2596104c0c..9cf35f0e3f 100644 --- a/gateway/configs/config-template.toml +++ b/gateway/configs/config-template.toml @@ -14,11 +14,21 @@ shutdown_timeout = "15s" gateway_id = '{{ env "APIP_GW_CONTROLLER_SERVER_GATEWAY_ID" "platform-gateway-id" }}' [controller.admin_server] -# Dedicated admin/debug HTTP server for config dump and xDS sync endpoints +# Dedicated admin/debug HTTP server for config dump and xDS sync endpoints. +# Kept enabled by default because it also serves /health, used by Kubernetes +# liveness/readiness probes; see admin_server.config_dump below to gate the +# sensitive /config_dump route specifically. enabled = true port = 9092 allowed_ips = ["*"] +[controller.admin_server.config_dump] +# The /config_dump route returns a full snapshot of deployed APIs, policies, +# and resolved configuration. Off by default — enable only when needed for +# debugging, and prefer reaching it via `kubectl port-forward` over exposing +# the admin Service port. +enabled = false + [controller.admin_server.pprof] # Go runtime profiling (net/http/pprof) served on the admin server, off by default. # When profiling, also restrict admin_server.allowed_ips or reach it via port-forward. @@ -312,10 +322,20 @@ max_decompressed_bytes = 10485760 extproc_port = 9001 [policy_engine.admin] +# Kept enabled by default because it also serves /health, used by Kubernetes +# liveness/readiness probes; see admin.config_dump below to gate the +# sensitive /config_dump route specifically. enabled = true port = 9002 allowed_ips = ["*", "127.0.0.1"] +[policy_engine.admin.config_dump] +# The /config_dump route returns the resolved policy chain and route +# configuration. Off by default — enable only when needed for debugging, and +# prefer reaching it via `kubectl port-forward` over exposing the admin +# Service port. +enabled = false + [policy_engine.admin.pprof] # Go runtime profiling (net/http/pprof) served on the admin server, off by default. # When profiling, also restrict admin.allowed_ips or reach it via port-forward. diff --git a/gateway/configs/config.toml b/gateway/configs/config.toml index e81a042d7a..9d2bda2248 100644 --- a/gateway/configs/config.toml +++ b/gateway/configs/config.toml @@ -23,6 +23,11 @@ path = '{{ env "APIP_GW_CONTROLLER_STORAGE_SQLITE_PATH" "./data/gateway.db" }}' [policy_engine.logging] level = "info" +[policy_engine.admin.config_dump] +# Enabled for local development convenience. Off by default in production +# (see config-template.toml) — /config_dump returns 404 when disabled. +enabled = true + [controller.logging] level = '{{ env "APIP_GW_CONTROLLER_LOGGING_LEVEL" "info" }}' @@ -37,6 +42,11 @@ apim_oauth2_client_secret = '{{ env "APIP_GW_CONTROLLER_CONTROLPLANE_APIM_OAUTH2 [controller.policies] definitions_path = '{{ env "APIP_GW_CONTROLLER_POLICIES_DEFINITIONS_PATH" "./default-policies" }}' +[controller.admin_server.config_dump] +# Enabled for local development convenience. Off by default in production +# (see config-template.toml) — /config_dump returns 404 when disabled. +enabled = true + [controller.auth.basic] enabled = true diff --git a/gateway/distribution/docker-compose.yaml b/gateway/distribution/docker-compose.yaml index f423cbfa1f..a1ac1f7288 100644 --- a/gateway/distribution/docker-compose.yaml +++ b/gateway/distribution/docker-compose.yaml @@ -63,6 +63,9 @@ services: format: raw environment: - GATEWAY_CONTROLLER_HOST=gateway-controller + # Envoy admin is disabled by default in the image; enabled here for local + # dev convenience since the port is already mapped to the host above. + - ROUTER_ADMIN_ENABLED=true volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro - ./configs/llm-pricing/model_prices.json:/etc/policy-engine/llm-pricing/model_prices.json:ro diff --git a/gateway/docker-compose-perf.yaml b/gateway/docker-compose-perf.yaml index da1a26885c..2ce03890ba 100644 --- a/gateway/docker-compose-perf.yaml +++ b/gateway/docker-compose-perf.yaml @@ -66,6 +66,9 @@ services: environment: - GATEWAY_CONTROLLER_HOST=gateway-controller - LOG_LEVEL=info + # Envoy admin is disabled by default in the image; enabled here for local + # dev convenience since the port is already mapped to the host above. + - ROUTER_ADMIN_ENABLED=true volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro networks: diff --git a/gateway/docker-compose.debug.yaml b/gateway/docker-compose.debug.yaml index 469c713079..55bfaa1877 100644 --- a/gateway/docker-compose.debug.yaml +++ b/gateway/docker-compose.debug.yaml @@ -70,6 +70,9 @@ services: environment: - GATEWAY_CONTROLLER_HOST=gateway-controller - LOG_LEVEL=info + # Envoy admin is disabled by default in the image; enabled here for local + # dev convenience since the port is already mapped to the host above. + - ROUTER_ADMIN_ENABLED=true volumes: - ./configs/config.toml:/etc/policy-engine/config.toml:ro networks: diff --git a/gateway/gateway-controller/pkg/adminserver/server.go b/gateway/gateway-controller/pkg/adminserver/server.go index b28211b947..04e1ceb925 100644 --- a/gateway/gateway-controller/pkg/adminserver/server.go +++ b/gateway/gateway-controller/pkg/adminserver/server.go @@ -129,6 +129,11 @@ func (s *Server) Stop(ctx context.Context) error { // GetConfigDump implements adminapi.ServerInterface. func (s *Server) GetConfigDump(w http.ResponseWriter, r *http.Request) { + if !s.cfg.ConfigDump.Enabled { + http.NotFound(w, r) + return + } + resp, err := s.apiServer.BuildConfigDumpResponse(s.logger) if err != nil { http.Error(w, "Failed to retrieve configuration dump", http.StatusInternalServerError) diff --git a/gateway/gateway-controller/pkg/adminserver/server_test.go b/gateway/gateway-controller/pkg/adminserver/server_test.go index 35e44c3570..57816adab8 100644 --- a/gateway/gateway-controller/pkg/adminserver/server_test.go +++ b/gateway/gateway-controller/pkg/adminserver/server_test.go @@ -85,7 +85,11 @@ func TestAdminServer_ConfigDumpHandler(t *testing.T) { stub := &stubAPIServer{ configDump: adminapi.ConfigDumpResponse{Status: &status}, } - s := NewServer(&config.AdminServerConfig{Port: 9092, AllowedIPs: []string{"*"}}, stub, nil, slog.Default()) + s := NewServer(&config.AdminServerConfig{ + Port: 9092, + AllowedIPs: []string{"*"}, + ConfigDump: config.ConfigDumpConfig{Enabled: true}, + }, stub, nil, slog.Default()) req := httptest.NewRequest(http.MethodGet, AdminAPIBasePath+"/config_dump", nil) req.RemoteAddr = "127.0.0.1:12345" @@ -100,6 +104,22 @@ func TestAdminServer_ConfigDumpHandler(t *testing.T) { assert.Equal(t, "ok", *body.Status) } +func TestAdminServer_ConfigDumpHandler_DisabledByDefault(t *testing.T) { + status := "ok" + stub := &stubAPIServer{ + configDump: adminapi.ConfigDumpResponse{Status: &status}, + } + // ConfigDump.Enabled left at its zero value (false) — matches the production default. + s := NewServer(&config.AdminServerConfig{Port: 9092, AllowedIPs: []string{"*"}}, stub, nil, slog.Default()) + + req := httptest.NewRequest(http.MethodGet, AdminAPIBasePath+"/config_dump", nil) + req.RemoteAddr = "127.0.0.1:12345" + rr := httptest.NewRecorder() + + s.httpSrv.Handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusNotFound, rr.Code) +} + func TestAdminServer_XDSSyncStatusHandler(t *testing.T) { component := "gateway-controller" version := "12" @@ -221,7 +241,11 @@ func TestAdminServer_LegacyConfigDump(t *testing.T) { stub := &stubAPIServer{ configDump: adminapi.ConfigDumpResponse{Status: &status}, } - s := NewServer(&config.AdminServerConfig{Port: 9092, AllowedIPs: []string{"*"}}, stub, nil, slog.Default()) + s := NewServer(&config.AdminServerConfig{ + Port: 9092, + AllowedIPs: []string{"*"}, + ConfigDump: config.ConfigDumpConfig{Enabled: true}, + }, stub, nil, slog.Default()) req := httptest.NewRequest(http.MethodGet, "/config_dump", nil) req.RemoteAddr = "127.0.0.1:12345" @@ -307,7 +331,11 @@ func TestAdminServer_ConfigDump_WrongCredentials(t *testing.T) { func TestAdminServer_ConfigDump_WithValidAuth(t *testing.T) { status := "ok" stub := &stubAPIServer{configDump: adminapi.ConfigDumpResponse{Status: &status}} - s := NewServer(&config.AdminServerConfig{Port: 9092, AllowedIPs: []string{"*"}}, stub, newBasicAuthMiddleware(t), slog.Default()) + s := NewServer(&config.AdminServerConfig{ + Port: 9092, + AllowedIPs: []string{"*"}, + ConfigDump: config.ConfigDumpConfig{Enabled: true}, + }, stub, newBasicAuthMiddleware(t), slog.Default()) req := httptest.NewRequest(http.MethodGet, AdminAPIBasePath+"/config_dump", nil) req.SetBasicAuth(testAdminUser, testAdminPass) @@ -384,7 +412,11 @@ func TestAdminServer_LegacyConfigDump_RequiresAuth(t *testing.T) { func TestAdminServer_ConfigDump_AdminRoleAllowed(t *testing.T) { status := "ok" stub := &stubAPIServer{configDump: adminapi.ConfigDumpResponse{Status: &status}} - s := NewServer(&config.AdminServerConfig{Port: 9092, AllowedIPs: []string{"*"}}, stub, + s := NewServer(&config.AdminServerConfig{ + Port: 9092, + AllowedIPs: []string{"*"}, + ConfigDump: config.ConfigDumpConfig{Enabled: true}, + }, stub, newAdminProtectMiddleware(t, []string{"admin"}), slog.Default()) req := httptest.NewRequest(http.MethodGet, AdminAPIBasePath+"/config_dump", nil) diff --git a/gateway/gateway-controller/pkg/config/config.go b/gateway/gateway-controller/pkg/config/config.go index 8da54f96fc..289f358974 100644 --- a/gateway/gateway-controller/pkg/config/config.go +++ b/gateway/gateway-controller/pkg/config/config.go @@ -309,10 +309,18 @@ type ServerConfig struct { // AdminServerConfig holds controller admin HTTP server configuration. type AdminServerConfig struct { - Enabled bool `koanf:"enabled"` - Port int `koanf:"port"` - AllowedIPs []string `koanf:"allowed_ips"` - Pprof PprofConfig `koanf:"pprof"` + Enabled bool `koanf:"enabled"` + Port int `koanf:"port"` + AllowedIPs []string `koanf:"allowed_ips"` + Pprof PprofConfig `koanf:"pprof"` + ConfigDump ConfigDumpConfig `koanf:"config_dump"` +} + +// ConfigDumpConfig gates the /config_dump endpoint served on the admin HTTP +// server. Disabled by default — /health and other admin routes are unaffected +// by this flag; when disabled, /config_dump returns 404 rather than a payload. +type ConfigDumpConfig struct { + Enabled bool `koanf:"enabled"` } // PprofConfig gates the Go runtime profiling endpoints (net/http/pprof) served on @@ -834,6 +842,9 @@ func defaultConfig() *Config { BlockProfileRate: 0, MutexProfileFraction: 0, }, + ConfigDump: ConfigDumpConfig{ + Enabled: false, + }, }, PolicyServer: PolicyServerConfig{ Port: 18001, diff --git a/gateway/gateway-runtime/docker-entrypoint-debug.sh b/gateway/gateway-runtime/docker-entrypoint-debug.sh index 643c95d3f9..72aae6797d 100644 --- a/gateway/gateway-runtime/docker-entrypoint-debug.sh +++ b/gateway/gateway-runtime/docker-entrypoint-debug.sh @@ -94,12 +94,19 @@ export ROUTER_CONCURRENCY="${ROUTER_CONCURRENCY:-0}" export APIP_GW_POLICY_ENGINE_METRICS_ENABLED="${APIP_GW_POLICY_ENGINE_METRICS_ENABLED:-true}" export APIP_GW_ROUTER_RE2_MAX_PROGRAM_SIZE="${APIP_GW_ROUTER_RE2_MAX_PROGRAM_SIZE:-400}" +# Router (Envoy) admin interface configuration (see docker-entrypoint.sh for details). +# Disabled by default — not defined in the static envoy-bootstrap.yaml at all. Set +# ROUTER_ADMIN_ENABLED=true to inject it at startup, bound to ROUTER_ADMIN_HOST (loopback +# by default). +export ROUTER_ADMIN_ENABLED="${ROUTER_ADMIN_ENABLED:-false}" +export ROUTER_ADMIN_HOST="${ROUTER_ADMIN_HOST:-127.0.0.1}" +export ROUTER_ADMIN_PORT="${ROUTER_ADMIN_PORT:-9901}" + # Graceful shutdown configuration (see docker-entrypoint.sh for details). # On SIGTERM the Router (Envoy) is drained before processes are terminated so in-flight # requests finish and keep-alive connections close cleanly instead of being reset. +# Requires ROUTER_ADMIN_ENABLED=true — skipped otherwise. # Keep ROUTER_DRAIN_TIME_SECONDS < the pod terminationGracePeriodSeconds; 0 disables it. -export ROUTER_ADMIN_HOST="${ROUTER_ADMIN_HOST:-127.0.0.1}" -export ROUTER_ADMIN_PORT="${ROUTER_ADMIN_PORT:-9901}" export ROUTER_DRAIN_TIME_SECONDS="${ROUTER_DRAIN_TIME_SECONDS:-15}" # Derive Router (Envoy) xDS config — used by envsubst on config-override.yaml @@ -121,6 +128,11 @@ log " GOMAXPROCS: ${GOMAXPROCS}" log " Router Concurrency: ${ROUTER_CONCURRENCY}" log " Router RE2 Max Program Size: ${APIP_GW_ROUTER_RE2_MAX_PROGRAM_SIZE}" log " Policy Engine Metrics: ${APIP_GW_POLICY_ENGINE_METRICS_ENABLED}" +if [ "${ROUTER_ADMIN_ENABLED}" = "true" ]; then + log " Router Admin: enabled on ${ROUTER_ADMIN_HOST}:${ROUTER_ADMIN_PORT}" +else + log " Router Admin: disabled (set ROUTER_ADMIN_ENABLED=true to enable)" +fi [[ ${#ROUTER_ARGS[@]} -gt 0 ]] && log " Router extra args: ${ROUTER_ARGS[*]}" [[ ${#PE_ARGS[@]} -gt 0 ]] && log " Policy Engine extra args: ${PE_ARGS[*]}" @@ -130,6 +142,18 @@ rm -f "${POLICY_ENGINE_SOCKET}" # Generate Envoy config override by substituting environment variables CONFIG_OVERRIDE=$(envsubst < /etc/envoy/config-override.yaml) +# The admin interface has no entry in the static bootstrap at all, so it only exists when +# explicitly opted into here — bound to ROUTER_ADMIN_HOST (loopback by default), never 0.0.0.0. +if [ "${ROUTER_ADMIN_ENABLED}" = "true" ]; then + CONFIG_OVERRIDE="${CONFIG_OVERRIDE} +admin: + address: + socket_address: + address: ${ROUTER_ADMIN_HOST} + port_value: ${ROUTER_ADMIN_PORT} +" +fi + # Track child PIDs PE_PID="" ENVOY_PID="" @@ -167,7 +191,10 @@ shutdown() { # Drain the Router first so in-flight requests finish and keep-alive connections are # closed cleanly — prevents client-visible connection resets during rolling restarts. - if [ -n "$ENVOY_PID" ] && kill -0 "$ENVOY_PID" 2>/dev/null \ + # Requires ROUTER_ADMIN_ENABLED=true; skipped otherwise since draining needs an admin call. + if [ "${ROUTER_ADMIN_ENABLED}" != "true" ]; then + log "Router admin disabled (ROUTER_ADMIN_ENABLED=false); skipping graceful drain" + elif [ -n "$ENVOY_PID" ] && kill -0 "$ENVOY_PID" 2>/dev/null \ && [ "${ROUTER_DRAIN_TIME_SECONDS}" -gt 0 ] 2>/dev/null; then log "Draining Router (Envoy); waiting up to ${ROUTER_DRAIN_TIME_SECONDS}s for in-flight requests..." if drain_router; then diff --git a/gateway/gateway-runtime/docker-entrypoint.sh b/gateway/gateway-runtime/docker-entrypoint.sh index 6354895ff2..5804c335dc 100644 --- a/gateway/gateway-runtime/docker-entrypoint.sh +++ b/gateway/gateway-runtime/docker-entrypoint.sh @@ -120,17 +120,26 @@ export PYTHON_POLICY_WORKERS="${PYTHON_POLICY_WORKERS:-4}" export PYTHON_POLICY_MAX_CONCURRENT="${PYTHON_POLICY_MAX_CONCURRENT:-100}" export PYTHON_POLICY_TIMEOUT="${PYTHON_POLICY_TIMEOUT:-30}" +# Router (Envoy) admin interface configuration +# The admin interface (config_dump, stats, /runtime_modify, etc.) is disabled by default — +# it is not defined in the static envoy-bootstrap.yaml at all. Set ROUTER_ADMIN_ENABLED=true +# to have it injected at startup via `envoy --config-yaml`, bound to ROUTER_ADMIN_HOST (loopback +# by default) so it is never reachable outside the pod's network namespace regardless of this +# flag. Enabling it also restores the graceful-drain-on-SIGTERM behavior below, since draining +# is triggered via an admin API call. +export ROUTER_ADMIN_ENABLED="${ROUTER_ADMIN_ENABLED:-false}" +export ROUTER_ADMIN_HOST="${ROUTER_ADMIN_HOST:-127.0.0.1}" +export ROUTER_ADMIN_PORT="${ROUTER_ADMIN_PORT:-9901}" + # Graceful shutdown configuration # On SIGTERM the Router (Envoy) is drained before any process is terminated, so in-flight # requests complete and keep-alive connections are closed cleanly (Connection: close) # instead of being reset. This avoids connection-reset errors for clients during rolling -# restarts / pod evictions. -# ROUTER_ADMIN_HOST/PORT : Router (Envoy) admin endpoint used to trigger the drain +# restarts / pod evictions. Requires ROUTER_ADMIN_ENABLED=true — draining is skipped +# (with a clear log message, not a stalled attempt) when the admin interface is disabled. # ROUTER_DRAIN_TIME_SECONDS : how long to wait for in-flight requests to finish before # terminating. Keep this LESS than the pod's terminationGracePeriodSeconds (k8s # default 30s) or the container is SIGKILLed mid-drain. Set to 0 to disable draining. -export ROUTER_ADMIN_HOST="${ROUTER_ADMIN_HOST:-127.0.0.1}" -export ROUTER_ADMIN_PORT="${ROUTER_ADMIN_PORT:-9901}" export ROUTER_DRAIN_TIME_SECONDS="${ROUTER_DRAIN_TIME_SECONDS:-15}" # Derive Router (Envoy) xDS config — used by envsubst on config-override.yaml @@ -152,6 +161,11 @@ log " Policy Engine Socket: ${POLICY_ENGINE_SOCKET}" log " GOMAXPROCS: ${GOMAXPROCS}" log " Router Concurrency: ${ROUTER_CONCURRENCY}" log " Router RE2 Max Program Size: ${APIP_GW_ROUTER_RE2_MAX_PROGRAM_SIZE}" +if [ "${ROUTER_ADMIN_ENABLED}" = "true" ]; then + log " Router Admin: enabled on ${ROUTER_ADMIN_HOST}:${ROUTER_ADMIN_PORT}" +else + log " Router Admin: disabled (set ROUTER_ADMIN_ENABLED=true to enable)" +fi log " Policy Engine Metrics: ${APIP_GW_POLICY_ENGINE_METRICS_ENABLED}" log " Python Workers: ${PYTHON_POLICY_WORKERS}" log " Python Max Concurrent: ${PYTHON_POLICY_MAX_CONCURRENT}" @@ -167,6 +181,19 @@ rm -f "${PYTHON_EXECUTOR_SOCKET}" # Generate Envoy config override by substituting environment variables CONFIG_OVERRIDE=$(envsubst < /etc/envoy/config-override.yaml) +# The admin interface has no entry in the static bootstrap (envoy-bootstrap.yaml) at all, so +# it only exists when explicitly opted into here — bound to ROUTER_ADMIN_HOST (loopback by +# default), never 0.0.0.0, so it is unreachable outside the pod's network namespace regardless. +if [ "${ROUTER_ADMIN_ENABLED}" = "true" ]; then + CONFIG_OVERRIDE="${CONFIG_OVERRIDE} +admin: + address: + socket_address: + address: ${ROUTER_ADMIN_HOST} + port_value: ${ROUTER_ADMIN_PORT} +" +fi + # Track child PIDs PY_PID="" PE_PID="" @@ -207,7 +234,12 @@ shutdown() { # Drain the Router first so in-flight requests finish and keep-alive connections are # closed cleanly — prevents client-visible connection resets during rolling restarts. - if [ -n "$ENVOY_PID" ] && kill -0 "$ENVOY_PID" 2>/dev/null \ + # Requires the admin interface (ROUTER_ADMIN_ENABLED=true); draining is unavailable + # without it, so skip straight to termination rather than attempting an admin call + # that can never succeed. + if [ "${ROUTER_ADMIN_ENABLED}" != "true" ]; then + log "Router admin disabled (ROUTER_ADMIN_ENABLED=false); skipping graceful drain" + elif [ -n "$ENVOY_PID" ] && kill -0 "$ENVOY_PID" 2>/dev/null \ && [ "${ROUTER_DRAIN_TIME_SECONDS}" -gt 0 ] 2>/dev/null; then log "Draining Router (Envoy); waiting up to ${ROUTER_DRAIN_TIME_SECONDS}s for in-flight requests..." if drain_router; then diff --git a/gateway/gateway-runtime/health-check.sh b/gateway/gateway-runtime/health-check.sh index e79b9fe0dc..58e0fcb5f2 100644 --- a/gateway/gateway-runtime/health-check.sh +++ b/gateway/gateway-runtime/health-check.sh @@ -24,14 +24,27 @@ # # Exit 0 = healthy, Exit 1 = unhealthy +ROUTER_ADMIN_ENABLED="${ROUTER_ADMIN_ENABLED:-false}" ROUTER_ADMIN_PORT="${ROUTER_ADMIN_PORT:-9901}" +ROUTER_HTTP_PORT="${ROUTER_HTTP_PORT:-8080}" POLICY_ENGINE_ADMIN_PORT="${POLICY_ENGINE_ADMIN_PORT:-9002}" -# Check Router (Envoy) readiness — expect HTTP 200 -ROUTER_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:${ROUTER_ADMIN_PORT}/ready") -if [ "$ROUTER_STATUS" != "200" ]; then - echo "Router not ready (HTTP ${ROUTER_STATUS})" - exit 1 +# Check Router (Envoy) readiness. +# The admin interface (/ready) is disabled by default (see docker-entrypoint.sh / +# ROUTER_ADMIN_ENABLED) — fall back to a raw TCP check against the main listener, +# which confirms Envoy is up and accepting connections. +if [ "${ROUTER_ADMIN_ENABLED}" = "true" ]; then + ROUTER_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:${ROUTER_ADMIN_PORT}/ready") + if [ "$ROUTER_STATUS" != "200" ]; then + echo "Router not ready (HTTP ${ROUTER_STATUS})" + exit 1 + fi +else + if ! (exec 3<>"/dev/tcp/127.0.0.1/${ROUTER_HTTP_PORT}") 2>/dev/null; then + echo "Router not accepting connections on port ${ROUTER_HTTP_PORT}" + exit 1 + fi + exec 3<&- 3>&- 2>/dev/null || true fi # Check Policy Engine health — expect HTTP 200 diff --git a/gateway/gateway-runtime/policy-engine/internal/admin/server.go b/gateway/gateway-runtime/policy-engine/internal/admin/server.go index 408d53bc8a..ef377f6359 100644 --- a/gateway/gateway-runtime/policy-engine/internal/admin/server.go +++ b/gateway/gateway-runtime/policy-engine/internal/admin/server.go @@ -47,7 +47,8 @@ func NewServer(cfg *config.AdminConfig, k *kernel.Kernel, reg *registry.PolicyRe configDumpHandler := NewConfigDumpHandler(k, reg, xds) xdsSyncHandler := NewXDSSyncStatusHandler(xds) healthHandler := NewHealthHandler(health, pythonHealth) - mux.Handle("/config_dump", ipWhitelistMiddleware(cfg.AllowedIPs, configDumpHandler)) + mux.Handle("/config_dump", configDumpEnabledMiddleware(cfg.ConfigDump.Enabled, + ipWhitelistMiddleware(cfg.AllowedIPs, configDumpHandler))) mux.Handle("/xds_sync_status", ipWhitelistMiddleware(cfg.AllowedIPs, xdsSyncHandler)) // Health endpoint is registered without IP whitelist so Docker/k8s health probes can reach it mux.Handle("/health", healthHandler) @@ -93,6 +94,20 @@ func (s *Server) Stop(ctx context.Context) error { return s.httpServer.Shutdown(ctx) } +// configDumpEnabledMiddleware gates /config_dump behind an explicit enable flag +// that is independent of the admin server's own Enabled flag, so /health (relied +// on by Docker/k8s health probes) keeps working even when config_dump is off. +// Disabled by default; returns 404 rather than a payload when disabled. +func configDumpEnabledMiddleware(enabled bool, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !enabled { + http.NotFound(w, r) + return + } + next.ServeHTTP(w, r) + }) +} + // ipWhitelistMiddleware creates a middleware that checks if the request IP is in the allowed list func ipWhitelistMiddleware(allowedIPs []string, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/gateway/gateway-runtime/policy-engine/internal/admin/server_test.go b/gateway/gateway-runtime/policy-engine/internal/admin/server_test.go index b180075ccd..2ae838c629 100644 --- a/gateway/gateway-runtime/policy-engine/internal/admin/server_test.go +++ b/gateway/gateway-runtime/policy-engine/internal/admin/server_test.go @@ -76,6 +76,7 @@ func TestServer_StartAndStop(t *testing.T) { cfg := &config.AdminConfig{ Port: port, AllowedIPs: []string{"127.0.0.1", "*"}, + ConfigDump: config.ConfigDumpConfig{Enabled: true}, } k := kernel.NewKernel() reg := ®istry.PolicyRegistry{ diff --git a/gateway/gateway-runtime/policy-engine/internal/config/config.go b/gateway/gateway-runtime/policy-engine/internal/config/config.go index 7bb39ead3f..6d964587c7 100644 --- a/gateway/gateway-runtime/policy-engine/internal/config/config.go +++ b/gateway/gateway-runtime/policy-engine/internal/config/config.go @@ -277,6 +277,16 @@ type AdminConfig struct { // Pprof gates the Go runtime profiling endpoints served on this admin server. Pprof PprofConfig `koanf:"pprof"` + + // ConfigDump gates the /config_dump endpoint served on this admin server. + ConfigDump ConfigDumpConfig `koanf:"config_dump"` +} + +// ConfigDumpConfig gates the /config_dump endpoint served on the admin HTTP +// server. Disabled by default — /health and other admin routes are unaffected +// by this flag; when disabled, /config_dump returns 404 rather than a payload. +type ConfigDumpConfig struct { + Enabled bool `koanf:"enabled"` } // PprofConfig gates the Go runtime profiling endpoints (net/http/pprof) served on @@ -523,6 +533,9 @@ func defaultConfig() *Config { BlockProfileRate: 0, MutexProfileFraction: 0, }, + ConfigDump: ConfigDumpConfig{ + Enabled: false, + }, }, Metrics: MetricsConfig{ Enabled: false, diff --git a/gateway/gateway-runtime/router/config/envoy-bootstrap.yaml b/gateway/gateway-runtime/router/config/envoy-bootstrap.yaml index 2a4229e1d2..7742e8aafc 100644 --- a/gateway/gateway-runtime/router/config/envoy-bootstrap.yaml +++ b/gateway/gateway-runtime/router/config/envoy-bootstrap.yaml @@ -16,11 +16,12 @@ # under the License. # -------------------------------------------------------------------- -admin: - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 +# The admin interface (config_dump, stats, /runtime_modify, etc.) is intentionally +# NOT defined here. It is disabled by default and, when enabled via +# ROUTER_ADMIN_ENABLED, injected at startup by docker-entrypoint.sh through +# `envoy --config-yaml`, bound to loopback only. See docker-entrypoint.sh for +# the injected block and health-check.sh for how liveness/readiness are +# checked without depending on the admin interface. node: cluster: gateway-cluster diff --git a/gateway/it/docker-compose.test.postgres.yaml b/gateway/it/docker-compose.test.postgres.yaml index 96764f35c7..a45b3b5f25 100644 --- a/gateway/it/docker-compose.test.postgres.yaml +++ b/gateway/it/docker-compose.test.postgres.yaml @@ -193,6 +193,10 @@ services: environment: - GATEWAY_CONTROLLER_HOST=it-gateway-controller-xds - LOG_LEVEL=info + # The IT suite (steps_health.go) exercises Envoy's admin /ready directly on the + # host-published port, so admin must bind the container interface, not loopback. + - ROUTER_ADMIN_ENABLED=true + - ROUTER_ADMIN_HOST=0.0.0.0 # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage diff --git a/gateway/it/docker-compose.test.sqlserver.yaml b/gateway/it/docker-compose.test.sqlserver.yaml index 9810ce3a06..c0e16923d1 100644 --- a/gateway/it/docker-compose.test.sqlserver.yaml +++ b/gateway/it/docker-compose.test.sqlserver.yaml @@ -173,6 +173,10 @@ services: environment: - GATEWAY_CONTROLLER_HOST=it-gateway-controller - LOG_LEVEL=info + # The IT suite (steps_health.go) exercises Envoy's admin /ready directly on the + # host-published port, so admin must bind the container interface, not loopback. + - ROUTER_ADMIN_ENABLED=true + - ROUTER_ADMIN_HOST=0.0.0.0 # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage diff --git a/gateway/it/docker-compose.test.vhosts-multi.yaml b/gateway/it/docker-compose.test.vhosts-multi.yaml index ade327cb8e..ba4d7abac6 100644 --- a/gateway/it/docker-compose.test.vhosts-multi.yaml +++ b/gateway/it/docker-compose.test.vhosts-multi.yaml @@ -75,6 +75,10 @@ services: environment: - GATEWAY_CONTROLLER_HOST=it-gateway-controller - LOG_LEVEL=info + # The IT suite (steps_health.go) exercises Envoy's admin /ready directly on the + # host-published port, so admin must bind the container interface, not loopback. + - ROUTER_ADMIN_ENABLED=true + - ROUTER_ADMIN_HOST=0.0.0.0 # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage diff --git a/gateway/it/docker-compose.test.vhosts-single.yaml b/gateway/it/docker-compose.test.vhosts-single.yaml index 39e3626b46..9ba503967b 100644 --- a/gateway/it/docker-compose.test.vhosts-single.yaml +++ b/gateway/it/docker-compose.test.vhosts-single.yaml @@ -75,6 +75,10 @@ services: environment: - GATEWAY_CONTROLLER_HOST=it-gateway-controller - LOG_LEVEL=info + # The IT suite (steps_health.go) exercises Envoy's admin /ready directly on the + # host-published port, so admin must bind the container interface, not loopback. + - ROUTER_ADMIN_ENABLED=true + - ROUTER_ADMIN_HOST=0.0.0.0 # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage diff --git a/gateway/it/docker-compose.test.yaml b/gateway/it/docker-compose.test.yaml index 2452f0dd13..279ca81fda 100644 --- a/gateway/it/docker-compose.test.yaml +++ b/gateway/it/docker-compose.test.yaml @@ -105,6 +105,10 @@ services: environment: - GATEWAY_CONTROLLER_HOST=it-gateway-controller - LOG_LEVEL=info + # The IT suite (steps_health.go) exercises Envoy's admin /ready directly on the + # host-published port, so admin must bind the container interface, not loopback. + - ROUTER_ADMIN_ENABLED=true + - ROUTER_ADMIN_HOST=0.0.0.0 # Override AWS Bedrock Runtime endpoint for testing with mock service - AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://mock-aws-bedrock-guardrail:8080 - GOCOVERDIR=/coverage diff --git a/gateway/it/test-config.toml b/gateway/it/test-config.toml index 405682ead1..f24e69227a 100644 --- a/gateway/it/test-config.toml +++ b/gateway/it/test-config.toml @@ -88,6 +88,11 @@ level = '{{ env "APIP_GW_CONTROLLER_LOGGING_LEVEL" "debug" }}' [controller.metrics] enabled = true +[controller.admin_server.config_dump] +# config_dump is off by default in production; the IT suite (config-dump.feature) +# exercises this endpoint, so enable it explicitly here. +enabled = true + # ============================================================================= # ROUTER CONFIGURATION # ============================================================================= @@ -107,6 +112,10 @@ request_headers_timeout = "5s" [policy_engine.logging] level = "debug" +[policy_engine.admin.config_dump] +# config_dump is off by default in production; enabled here for the IT suite. +enabled = true + [policy_engine.metrics] enabled = true diff --git a/gateway/it/test-config.vhosts-multi.toml b/gateway/it/test-config.vhosts-multi.toml index 4953f9a79b..fbe649c60d 100644 --- a/gateway/it/test-config.vhosts-multi.toml +++ b/gateway/it/test-config.vhosts-multi.toml @@ -70,6 +70,10 @@ level = '{{ env "APIP_GW_CONTROLLER_LOGGING_LEVEL" "debug" }}' [controller.metrics] enabled = true +[controller.admin_server.config_dump] +# config_dump is off by default in production; enabled here for the IT suite. +enabled = true + # ============================================================================= # ROUTER CONFIGURATION # ============================================================================= @@ -93,6 +97,10 @@ default = "*-sandbox.wso2.com" [policy_engine.logging] level = "debug" +[policy_engine.admin.config_dump] +# config_dump is off by default in production; enabled here for the IT suite. +enabled = true + [policy_engine.metrics] enabled = true diff --git a/gateway/it/test-config.vhosts-single.toml b/gateway/it/test-config.vhosts-single.toml index edc8dc0914..eef60c6a08 100644 --- a/gateway/it/test-config.vhosts-single.toml +++ b/gateway/it/test-config.vhosts-single.toml @@ -70,6 +70,10 @@ level = '{{ env "APIP_GW_CONTROLLER_LOGGING_LEVEL" "debug" }}' [controller.metrics] enabled = true +[controller.admin_server.config_dump] +# config_dump is off by default in production; enabled here for the IT suite. +enabled = true + # ============================================================================= # ROUTER CONFIGURATION # ============================================================================= @@ -91,6 +95,10 @@ default = "*-sandbox.wso2.com" [policy_engine.logging] level = "debug" +[policy_engine.admin.config_dump] +# config_dump is off by default in production; enabled here for the IT suite. +enabled = true + [policy_engine.metrics] enabled = true diff --git a/kubernetes/gateway-operator/internal/controller/resources/api-platform-gateway-k8s-manifests.yaml b/kubernetes/gateway-operator/internal/controller/resources/api-platform-gateway-k8s-manifests.yaml index 0c1cd18e01..3664023bc4 100644 --- a/kubernetes/gateway-operator/internal/controller/resources/api-platform-gateway-k8s-manifests.yaml +++ b/kubernetes/gateway-operator/internal/controller/resources/api-platform-gateway-k8s-manifests.yaml @@ -34,9 +34,10 @@ spec: - name: "8080" port: 8080 targetPort: 8080 - - name: "9901" - port: 9901 - targetPort: 9901 + # Envoy admin (9901) is intentionally NOT published here — it's disabled by + # default in the image (ROUTER_ADMIN_ENABLED=false) and, even when enabled, + # binds loopback-only; publishing it on a Service would defeat that. See + # kubernetes/helm/gateway-helm-chart's expose.routerAdmin (default false). selector: io.kompose.service: router diff --git a/kubernetes/helm/gateway-helm-chart/README.md b/kubernetes/helm/gateway-helm-chart/README.md index b9b3be0934..f95020e19a 100644 --- a/kubernetes/helm/gateway-helm-chart/README.md +++ b/kubernetes/helm/gateway-helm-chart/README.md @@ -165,6 +165,7 @@ All configurable values are documented in `values.yaml`. Component blocks are fu - `gateway..deployment.*` – pod-level knobs (replicas, probes incl. optional `startupProbe`, scheduling via `nodeSelector`/`tolerations`/`affinity`/`topologySpreadConstraints`, update `strategy`, `terminationGracePeriodSeconds`, `hostAliases`, `dnsPolicy`/`dnsConfig`, `automountServiceAccountToken`, env overrides, extra volumes) and enable/disable switches. - `gateway..service.*` – service type/ports plus optional annotations and labels, and network tuning (`clusterIP`, `externalTrafficPolicy`, `loadBalancerClass`, `loadBalancerSourceRanges`, `ipFamilyPolicy`/`ipFamilies`, static `nodePorts.*`). - `gateway..service.expose.*` – per-port toggles for publishing admin/debug ports on the Service. **All default to `false`** so admin surfaces stay pod-internal (reach them with `kubectl port-forward`). Available toggles: `controller.service.expose.admin` (controller admin, 9092), `gatewayRuntime.service.expose.routerAdmin` (Router/Envoy admin, 9901 — includes mutating endpoints, leave off unless trusted), `gatewayRuntime.service.expose.policyEngineAdmin` (policy-engine admin, 9002). Probes are unaffected; they target container ports directly. Note: the controller admin port is no longer exposed by default — set `expose.admin=true` to restore prior behavior. The runtime port key was renamed `envoyAdmin` → `routerAdmin`. +- The sensitive `/config_dump` route on the controller and policy-engine admin servers is **off by default** (returns 404), independent of the `expose.*` Service toggles above — set `gateway.config.controller.admin_server.config_dump.enabled=true` / `gateway.config.policy_engine.admin.config_dump.enabled=true` to turn it on. `/health` and other admin routes are unaffected, so liveness/readiness probes keep working either way. Envoy's admin interface (port 9901) is likewise off by default at the process level — nothing is even listening on it unless `gateway.gatewayRuntime.deployment.env.routerAdminEnabled=true`, and even then it binds loopback-only inside the pod. Enabling it also restores the graceful-drain-on-SIGTERM behavior; `gateway-runtime`'s health probe falls back to a plain TCP check on the HTTP listener when it's off. - `gateway.controller.persistence` / `gateway.configMap` – PVC sizing/claims (plus PVC `labels`/`annotations`, e.g. `helm.sh/resource-policy: keep`) and component configuration payloads. - `commonLabels` / `commonAnnotations` – applied to every resource the chart renders; per-resource labels/annotations win on key conflicts. - `gateway.controller.controlPlane` – control-plane connectivity. The host is non-secret and rendered directly into `config.toml`; the token is injected from a Secret. The controller log level is `gateway.config.controller.logging.level`. diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml index 28de7f02b9..87153ef983 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-config.yaml @@ -29,6 +29,9 @@ data: port = {{ $gc.admin_server.port }} allowed_ips = [{{- range $i, $ip := $gc.admin_server.allowed_ips }}{{- if gt $i 0 }}, {{ end }}{{ $ip | quote }}{{- end }}] + [controller.admin_server.config_dump] + enabled = {{ $gc.admin_server.config_dump.enabled }} + [controller.admin_server.pprof] enabled = {{ $gc.admin_server.pprof.enabled }} block_profile_rate = {{ $gc.admin_server.pprof.block_profile_rate }} @@ -285,6 +288,9 @@ data: port = {{ $pe.admin.port }} allowed_ips = [{{- range $i, $ip := $pe.admin.allowed_ips }}{{- if gt $i 0 }}, {{ end }}{{ $ip | quote }}{{- end }}] + [policy_engine.admin.config_dump] + enabled = {{ $pe.admin.config_dump.enabled }} + [policy_engine.admin.pprof] enabled = {{ $pe.admin.pprof.enabled }} block_profile_rate = {{ $pe.admin.pprof.block_profile_rate }} diff --git a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-runtime/deployment.yaml b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-runtime/deployment.yaml index 3b9ffc9f02..5a6ca4688e 100644 --- a/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-runtime/deployment.yaml +++ b/kubernetes/helm/gateway-helm-chart/templates/gateway/gateway-runtime/deployment.yaml @@ -97,6 +97,8 @@ spec: - name: MOESIF_KEY value: {{ $deployment.env.moesifKey | quote }} {{- end }} + - name: ROUTER_ADMIN_ENABLED + value: {{ default false $deployment.env.routerAdminEnabled | quote }} {{- range $deployment.extraEnv }} - {{- toYaml . | nindent 14 }} {{- end }} diff --git a/kubernetes/helm/gateway-helm-chart/values.yaml b/kubernetes/helm/gateway-helm-chart/values.yaml index 4d06677af4..3e808277f0 100644 --- a/kubernetes/helm/gateway-helm-chart/values.yaml +++ b/kubernetes/helm/gateway-helm-chart/values.yaml @@ -83,9 +83,20 @@ gateway: # config.toml — see defaultConfig() in pkg/config/config.go. Adding it here # only makes the existing implicit default explicit and overridable. admin_server: + # Kept enabled by default because it also serves /health, used by the + # controller's livenessProbe/readinessProbe (see deployment.livenessProbe + # above) — disabling this would crash-loop the pod. Gate the sensitive + # /config_dump route specifically via config_dump.enabled below instead. enabled: true port: 9092 allowed_ips: ["*"] + config_dump: + # The /config_dump route returns a full snapshot of deployed APIs, + # policies, and resolved configuration. Off by default — enable only + # when needed for debugging, and prefer reaching it via + # `kubectl port-forward` over exposing the admin Service port + # (see gateway.controller.service.expose.admin). + enabled: false pprof: # Go runtime profiling (net/http/pprof) served on the admin server, off by # default. When enabling, also restrict allowed_ips above or reach it via @@ -448,7 +459,10 @@ gateway: # Admin HTTP server configuration admin: - # Enable admin HTTP server for debugging endpoints + # Kept enabled by default because it also serves /health, used by the + # gateway-runtime's exec health-check.sh probe — disabling this would + # crash-loop the pod. Gate the sensitive /config_dump route + # specifically via config_dump.enabled below instead. enabled: true # Port for admin HTTP server @@ -460,6 +474,14 @@ gateway: - "*" - "127.0.0.1" + config_dump: + # The /config_dump route returns the resolved policy chain and route + # configuration. Off by default — enable only when needed for + # debugging, and prefer reaching it via `kubectl port-forward` over + # exposing the admin Service port (see + # gateway.gatewayRuntime.service.expose.policyEngineAdmin). + enabled: false + pprof: # Go runtime profiling (net/http/pprof), off by default. When enabling, # also restrict allowed_ips above or reach it via port-forward. @@ -596,6 +618,9 @@ gateway: # server serves config_dump and xds_sync_status, so keep it pod-internal (reach it # via `kubectl port-forward`) unless you need in-cluster Service access. Liveness/ # readiness probes are unaffected: they hit the container port directly, not the Service. + # Independent of gateway.config.controller.admin_server.config_dump.enabled, which + # gates whether /config_dump responds at all (off by default) rather than whether + # the Service publishes the port. expose: admin: false # Static cluster IP (e.g. "None" for a headless Service). Empty = auto-assign. @@ -856,6 +881,12 @@ gateway: policyEngineMetrics: 9003 # Publish the runtime admin/debug ports on the Service. Off by default — these serve # config_dump and health; keep them pod-internal unless needed. Probes are unaffected. + # This is independent of, and in addition to, the process-level defaults: Envoy's + # admin interface itself is off unless deployment.env.routerAdminEnabled=true (and + # even then binds loopback-only), and policy-engine's /config_dump route is off + # unless gateway.config.policy_engine.admin.config_dump.enabled=true. Leave both + # OFF here regardless of those flags — this toggle controls k8s Service/network + # reachability, not the admin interface's own enablement. expose: # Router (Envoy) admin: exposes MUTATING endpoints (/quitquitquit, /healthcheck/fail, # /runtime_modify) — leave OFF unless you fully trust in-cluster network access. @@ -902,6 +933,15 @@ gateway: gatewayControllerHost: "" logLevel: info moesifKey: "" + # Router (Envoy) admin interface (config_dump, stats, /runtime_modify, etc.). + # Off by default — it isn't even present in the static Envoy bootstrap unless + # this is true, and is bound to loopback-only when enabled, so it's never + # reachable outside the pod's network namespace either way. Enabling it also + # restores graceful-drain-on-SIGTERM (see docker-entrypoint.sh), at the cost of + # health-check.sh falling back to a raw TCP check instead of Envoy's own + # /ready when this is off. See also gateway.gatewayRuntime.service.expose.routerAdmin + # to control whether the port is published on the k8s Service (separate concern). + routerAdminEnabled: false extraEnv: [] # extraEnvFrom: # - configMapRef: