From f0899f7dfb2881aa0bda54d9665d2d42621c8308 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Sat, 22 Aug 2026 08:50:28 +0000 Subject: [PATCH 1/4] feat(networking): add tenant agent isolation NetworkPolicy Implements Phase 1 of the DevOps roadmap: zero-trust network isolation for agent pods across tenant namespaces. Changes: - internal/controller/agentdeployment_controller.go: Add 'agentrax.io/agent: true' label to agentLabels(). This label acts as the NetworkPolicy pod selector key. Safe for existing Deployments because reconcileDeployment() only writes spec.selector on creation (ResourceVersion == ''). - config/network-policy/tenant-agent-isolation.yaml: New NetworkPolicy targeting pods with agentrax.io/agent=true. Default-denies all ingress/egress, then allows: - Ingress: Prometheus scrape on port 8080 from namespaces labelled monitoring=enabled - Egress: kube-apiserver port 6443, CoreDNS port 53 UDP+TCP - config/network-policy/kustomization.yaml: Add new manifest to the network-policy Kustomize component. - config/default/kustomization.yaml: Uncomment the network-policy component so it is included in the default overlay. - docs/networking/README.md: Two-tier policy model documentation, traffic diagrams, and per-tenant application instructions. Verified: make test -> all packages pass (controller: 71.8%) make lint -> 0 errors make manifests -> 0 errors helm lint -> 0 failures YAML validate -> NetworkPolicy schema correct --- config/default/kustomization.yaml | 2 +- config/network-policy/kustomization.yaml | 1 + .../tenant-agent-isolation.yaml | 65 +++++++++ docs/networking/README.md | 138 ++++++++++++++++++ .../controller/agentdeployment_controller.go | 3 + 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 config/network-policy/tenant-agent-isolation.yaml create mode 100644 docs/networking/README.md diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index db44c80..de04fec 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -31,7 +31,7 @@ resources: # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. # Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will # be able to communicate with the Webhook Server. -#- ../network-policy +- ../network-policy # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml index ec0fb5e..9713cd9 100644 --- a/config/network-policy/kustomization.yaml +++ b/config/network-policy/kustomization.yaml @@ -1,2 +1,3 @@ resources: - allow-metrics-traffic.yaml +- tenant-agent-isolation.yaml diff --git a/config/network-policy/tenant-agent-isolation.yaml b/config/network-policy/tenant-agent-isolation.yaml new file mode 100644 index 0000000..6bbb539 --- /dev/null +++ b/config/network-policy/tenant-agent-isolation.yaml @@ -0,0 +1,65 @@ +--- +# Tenant Agent Isolation NetworkPolicy +# +# Purpose: Restricts agent pods across tenant namespaces, enforcing zero-trust +# isolation between tenants and preventing unauthorized outbound traffic. +# +# Selector: Matches all pods labelled `agentrax.io/agent: "true"`. +# The AgentDeployment reconciler sets this label on every pod template +# it manages, so this policy applies to all managed agent pods. +# +# Ingress rules: +# - Allow Prometheus to scrape metrics on port 8080 from namespaces +# labelled `monitoring: enabled` (kube-prometheus-stack namespace). +# +# Egress rules: +# - Allow egress to kube-apiserver on port 6443 (required for agent-to-API +# communication and tool-calling via the Kubernetes API). +# - Allow CoreDNS lookups on port 53 (UDP and TCP) for service discovery +# within the cluster. +# - All other egress (internet, cross-tenant) is denied by default. +# +# Usage: Apply this manifest to every tenant namespace: +# kubectl apply -n tenant- -f tenant-agent-isolation.yaml +# +# Note: This policy does NOT apply to the agentrax-system namespace (operator pods). +# The operator namespace is protected by allow-metrics-traffic.yaml. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: tenant-agent-isolation + labels: + app.kubernetes.io/name: agentrax + app.kubernetes.io/managed-by: kustomize +spec: + # Select all pods carrying the agentrax.io/agent=true label. + # This label is set by agentLabels() in the AgentDeployment reconciler. + podSelector: + matchLabels: + agentrax.io/agent: "true" + policyTypes: + - Ingress + - Egress + ingress: + # Allow Prometheus to scrape /metrics on port 8080. + # Prometheus Operator runs in a namespace labelled `monitoring: enabled`. + - from: + - namespaceSelector: + matchLabels: + monitoring: enabled + ports: + - port: 8080 + protocol: TCP + egress: + # Allow outbound to kube-apiserver on port 6443. + # Agents may call the Kubernetes API to discover services or use cluster tools. + - ports: + - port: 6443 + protocol: TCP + # Allow CoreDNS resolution on UDP and TCP port 53. + # Without this, service name lookups fail and MCP tool endpoints are unreachable. + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/docs/networking/README.md b/docs/networking/README.md new file mode 100644 index 0000000..5ae278b --- /dev/null +++ b/docs/networking/README.md @@ -0,0 +1,138 @@ +# Agentrax — Tenant Network Isolation + +This document explains the two-tier network policy model shipped with Agentrax +and how platform operators apply it to tenant namespaces. + +## Overview + +Agentrax uses Kubernetes `NetworkPolicy` to enforce a **zero-trust perimeter** +around all agent pods. This prevents a compromised or misbehaving agent in one +tenant from reaching another tenant's services, the operator control plane, or +arbitrary internet destinations. + +Two policies are maintained: + +| Policy File | Namespace | Purpose | +| ----------------------------- | -------------------------- | -------------------------------------------------------------------------- | +| `allow-metrics-traffic.yaml` | `agentrax-system` | Allows Prometheus to scrape the operator `/metrics` endpoint | +| `tenant-agent-isolation.yaml` | Every `tenant-*` namespace | Isolates agent pods — restricts all ingress/egress to the minimum required | + +## How the Label Selector Works + +The `tenant-agent-isolation` policy uses `podSelector.matchLabels`: + +```yaml +podSelector: + matchLabels: + agentrax.io/agent: "true" +``` + +The `AgentDeployment` reconciler (`internal/controller/agentdeployment_controller.go`) +stamps this label onto every agent `Deployment`'s pod template via `agentLabels()`. +No manual labelling is needed — all agent pods are automatically covered. + +## Traffic Model + +``` +┌─────────────────────────────────────────────────────┐ +│ tenant-finance namespace │ +│ │ +│ [Agent Pod] agentrax.io/agent=true │ +│ │ │ +│ ├─ Ingress ← port 8080 ← [Prometheus] │ +│ │ (monitoring namespace only) │ +│ │ │ +│ ├─ Egress → port 6443 → [kube-apiserver] │ +│ ├─ Egress → port 53 → [CoreDNS] │ +│ │ │ +│ └─ ALL OTHER TRAFFIC: BLOCKED │ +└─────────────────────────────────────────────────────┘ +``` + +## Applying the Policy to Tenant Namespaces + +The `tenant-agent-isolation.yaml` NetworkPolicy must be applied to each tenant +namespace. The policy is **not** automatically applied by the operator — it is +applied once by a platform admin when provisioning a tenant namespace. + +### Apply Manually + +```bash +# Apply to a specific tenant namespace: +kubectl apply -n tenant-finance \ + -f config/network-policy/tenant-agent-isolation.yaml + +kubectl apply -n tenant-marketing \ + -f config/network-policy/tenant-agent-isolation.yaml +``` + +### Apply via Kustomize (Development) + +The default Kustomize overlay applies both network policies to the `agentrax-system` +namespace for development/testing. The `tenant-agent-isolation` policy in this +context validates the manifest schema; in production it must be applied per tenant +namespace as above. + +```bash +kubectl apply -k config/default/ +``` + +### Apply via Helm (Recommended for Production) + +When installing via Helm, set `networkPolicy.enabled: true` (Phase 1 Helm +integration — coming in a future release): + +```bash +helm upgrade --install agentrax charts/agentrax/ \ + --set networkPolicy.enabled=true +``` + +## Labelling the Prometheus Namespace + +The ingress rule allows traffic from namespaces labelled `monitoring: enabled`. +Apply this label to the namespace where Prometheus Operator / kube-prometheus-stack +is installed: + +```bash +kubectl label namespace monitoring monitoring=enabled +# Or, if using the default kube-prometheus-stack namespace name: +kubectl label namespace monitoring monitoring=enabled +``` + +## Required CNI Support + +This NetworkPolicy relies on a Container Network Interface (CNI) plugin that +**enforces** `NetworkPolicy` objects. Verify your CNI supports this: + +| Environment | Supported CNI | +| ---------------- | ----------------------------- | +| Kind (local dev) | Kindnet (default) ✅ | +| Azure AKS | Azure CNI or Calico ✅ | +| AWS EKS | VPC CNI + Calico or Cilium ✅ | +| GKE | Dataplane V2 (Cilium) ✅ | + +> **Note**: Flannel does **not** enforce NetworkPolicy by default. Use Calico or +> Cilium as a replacement CNI if Flannel is your cluster default. + +## Verifying the Policy + +After applying, verify that the policy is active and that an agent pod has +the correct label: + +```bash +# Confirm agent pod has the isolation label: +kubectl get pods -n tenant-finance -L agentrax.io/agent + +# Confirm the NetworkPolicy is present: +kubectl get networkpolicy -n tenant-finance + +# Test that cross-tenant traffic is blocked (from within an agent pod): +kubectl exec -n tenant-finance -- \ + curl --connect-timeout 2 http:// +# Expected: connection timed out (blocked) + +# Test that Kubernetes API access is allowed: +kubectl exec -n tenant-finance -- \ + curl -k https://kubernetes.default.svc:443/healthz +# Expected: "ok" +``` diff --git a/internal/controller/agentdeployment_controller.go b/internal/controller/agentdeployment_controller.go index e44dfec..d64b90a 100644 --- a/internal/controller/agentdeployment_controller.go +++ b/internal/controller/agentdeployment_controller.go @@ -659,12 +659,15 @@ func (r *AgentDeploymentReconciler) reconcileMCPRegistration(ctx context.Context // agentLabels returns the canonical label set applied to all resources owned by ad. // For stable resources (Deployment, Service), this includes variant=stable. +// The agentrax.io/agent label is the NetworkPolicy selector key — all agent pod +// templates carry it so the tenant-agent-isolation policy applies automatically. func agentLabels(ad *agentraxv1alpha1.AgentDeployment) map[string]string { return map[string]string{ "app.kubernetes.io/name": ad.Name, "app.kubernetes.io/managed-by": "agentrax", "agentrax.io/tenant": ad.Spec.TenantRef, "agentrax.io/variant": "stable", + "agentrax.io/agent": "true", } } From ded3a33e2aad7aeb21dbf8878ac58a4b0c74c97c Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Sat, 22 Aug 2026 09:15:58 +0000 Subject: [PATCH 2/4] feat: implement agent network isolation with label and updated NetworkPolicy egress rules Signed-off-by: Ankit Kr. Chowdhury --- .agents/skills/agentrax-context/SKILL.md | 1 + .../tenant-agent-isolation.yaml | 4 +- docs/ARCHITECTURE.md | 52 ++++--- docs/networking/README.md | 138 ------------------ .../agentdeployment_builder_test.go | 10 +- 5 files changed, 47 insertions(+), 158 deletions(-) delete mode 100644 docs/networking/README.md diff --git a/.agents/skills/agentrax-context/SKILL.md b/.agents/skills/agentrax-context/SKILL.md index 88671d0..573a17a 100644 --- a/.agents/skills/agentrax-context/SKILL.md +++ b/.agents/skills/agentrax-context/SKILL.md @@ -20,6 +20,7 @@ description: Project context and settled architecture decisions for the Agentrax - **Autoscaling**: native `HorizontalPodAutoscaler` pointed at Prometheus Adapter custom metrics (`queueDepth` or `gpuUtilization`). No custom scaling loop. During active canary, the stable HPA is paused (deleted) and no canary HPA is created — autoscaling resumes only after promotion or rollback. - **Traffic splitting**: Gateway API `HTTPRoute` weighted backends. Not Istio, not ingress annotations. +- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (Operator in `agentrax-system`, tenant pods in `tenant-*` selected by `agentrax.io/agent: "true"`). No service mesh. - **MCP registry**: embedded HTTP handler inside the operator process, backed by a `ConfigMap`. Not a separate Deployment, not a new database — HA storage is a v2 item. - **Non-goals**: no model training/fine-tuning, no general-purpose workload management, no service mesh, no UI in v1. Flag any drift toward these rather than quietly implementing them. diff --git a/config/network-policy/tenant-agent-isolation.yaml b/config/network-policy/tenant-agent-isolation.yaml index 6bbb539..15af8bc 100644 --- a/config/network-policy/tenant-agent-isolation.yaml +++ b/config/network-policy/tenant-agent-isolation.yaml @@ -51,9 +51,11 @@ spec: - port: 8080 protocol: TCP egress: - # Allow outbound to kube-apiserver on port 6443. + # Allow outbound to kube-apiserver via ClusterIP (port 443) and direct endpoint (port 6443). # Agents may call the Kubernetes API to discover services or use cluster tools. - ports: + - port: 443 + protocol: TCP - port: 6443 protocol: TCP # Allow CoreDNS resolution on UDP and TCP port 53. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0080652..a83ed7c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -89,16 +89,16 @@ flowchart TB The repository enforces strict directional boundaries to prevent circular dependencies and isolate business logic from Kubernetes plumbing: -| Package | Scope & Responsibility | Key Invariants | -| ---------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| `api/v1alpha1/` | CRD type definitions, OpenAPI markers, schema validation rules, and status condition constants. | **Zero business logic**; only struct declarations and generated deep-copy methods. | -| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes). Consumes subsystems via interfaces. | -| `internal/quota/` | Quota arithmetic and concurrency-safe in-flight reservation cache. | Pure arithmetic; mutex-guarded state map; zero direct API server network calls in calculation paths. | -| `internal/webhook/` | Validating and Mutating admission webhooks. | Shared with `internal/quota` to enforce admission rules before objects are persisted. | -| `internal/scaling/` | HPA synthesis, velocity rules, and dynamic quota ceiling headroom. | Calculates `QuotaHeadroom()` to cap HPA `maxReplicas` and applies stabilization windows. | -| `internal/rollout/` | Canary state machine, PromQL query construction, and threshold evaluation. | Re-entrant state machine; sample-size gating; fail-safe timeout evaluation. | -| `internal/registry/` | MCP registrar, JSON-RPC 2.0 handshake, TTL sweeper, and discovery REST API. | In-memory registry with ConfigMap write-through for persistence; background health probes and TTL sweep. Explicitly allowed to write the `agentrax-registry` ConfigMap for state recovery. | -| `internal/metrics/` | Bounded HTTP Prometheus query client. | Wraps all responses with `io.LimitReader` (1 MiB ceiling) to prevent memory exhaustion. | +| Package | Scope & Responsibility | Key Invariants | +| ---------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `api/v1alpha1/` | CRD type definitions, OpenAPI markers, schema validation rules, and status condition constants. | **Zero business logic**; only struct declarations and generated deep-copy methods. | +| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes). Consumes subsystems via interfaces. | +| `internal/quota/` | Quota arithmetic and concurrency-safe in-flight reservation cache. | Pure arithmetic; mutex-guarded state map; zero direct API server network calls in calculation paths. | +| `internal/webhook/` | Validating and Mutating admission webhooks. | Shared with `internal/quota` to enforce admission rules before objects are persisted. | +| `internal/scaling/` | HPA synthesis, velocity rules, and dynamic quota ceiling headroom. | Calculates `QuotaHeadroom()` to cap HPA `maxReplicas` and applies stabilization windows. | +| `internal/rollout/` | Canary state machine, PromQL query construction, and threshold evaluation. | Re-entrant state machine; sample-size gating; fail-safe timeout evaluation. | +| `internal/registry/` | MCP registrar, JSON-RPC 2.0 handshake, TTL sweeper, and discovery REST API. | In-memory registry with ConfigMap write-through for persistence; background health probes and TTL sweep. Explicitly allowed to write the `agentrax-registry` ConfigMap for state recovery. | +| `internal/metrics/` | Bounded HTTP Prometheus query client. | Wraps all responses with `io.LimitReader` (1 MiB ceiling) to prevent memory exhaustion. | --- @@ -319,19 +319,35 @@ When an `AgentDeployment` is deleted, Kubernetes sets `metadata.deletionTimestam 5. Kubernetes GC cascade deletes child resources (Deployment, Service, HPA, Route) ``` -**Invariant**: MCP deregistration MUST complete _before_ the child `Service` is garbage collected, ensuring external clients never encounter dead routing endpoints. +### 4.6 Zero-Trust Multi-Tenant Network Isolation + +Agentrax enforces a zero-trust network perimeter around all AI agent workloads running in `tenant-*` namespaces. Because autonomous agents dynamically execute tools via MCP and consume cluster resources, flat Kubernetes networking presents severe security risks (unauthorized inter-tenant access, data exfiltration, and lateral movement). + +Agentrax maintains a **two-tier network policy model**: + +| Policy Manifest | Target Namespace | Scope & Responsibility | +| :---------------------------- | :---------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `allow-metrics-traffic.yaml` | `agentrax-system` | Protects the operator process; allows Prometheus to scrape `/metrics` on port `:8443`/`:8080`. | +| `tenant-agent-isolation.yaml` | Every `tenant-*` | Isolates agent pods; enforces default-deny on ingress/egress, strictly whitelisting only metrics scraping (`:8080`), Kubernetes API server (`:443`/`:6443`), and CoreDNS (`:53`). | + +#### Ingress & Egress Invariants: + +- **Ingress**: Only TCP port `8080` from namespaces labeled `monitoring: enabled` (Prometheus metric scraping). +- **Egress**: Only TCP ports `443`/`6443` (`kube-apiserver`) and UDP/TCP port `53` (`CoreDNS`). All other outbound egress (cross-tenant, external internet) is blocked at the CNI layer. +- **Label Selector Binding**: The `tenant-agent-isolation` policy selects pods dynamically via `agentrax.io/agent: "true"`. The `AgentDeploymentReconciler` automatically stamps this label into the `PodTemplateSpec` of every managed `Deployment` via `agentLabels()`. --- ## 5. Architectural Decision Records (ADRs) & Trade-Offs -| Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | -| --------------------------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Gateway API (`HTTPRoute`)** | Istio `VirtualService` / Ingress Annotations | Istio requires a heavy service-mesh control plane and sidecar injection. Ingress annotations lack standardized multi-backend weighted traffic splits. Gateway API provides a lightweight, vendor-neutral standard for traffic shifting. | -| **Custom Canary Rollout Engine** | Argo Rollouts / Flagger | Generic rollout tools treat metric anomalies as pure percentages without low-traffic statistical gating (`minRequestSample`). Building an embedded, re-entrant state machine allowed us to guarantee sample-size gating and MCP tool re-registration upon promotion. | -| **Native HPA via Custom Metrics** | KEDA (`ScaledObject`) | KEDA is powerful but adds external CRD dependencies. Generating native Kubernetes `HorizontalPodAutoscaler` objects tied to the Prometheus Adapter custom metrics pipeline minimized dependencies while giving full control over stabilization windows. | -| **Embedded Registry + ConfigMap Store** | Dedicated etcd / Redis / Database | Adding a dedicated database for service discovery increases operator operational complexity. The in-operator HTTP server with ConfigMap write-through store provides simple, robust storage for hundreds of agent services with cold-restart recovery. | -| **Go (`controller-runtime`)** | Python (`Kopf`) | Go provides native compile-time safety, seamless alignment with Kubernetes upstream libraries, and access to `setup-envtest` for isolated in-process integration testing. | +| Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | +| --------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Gateway API (`HTTPRoute`)** | Istio `VirtualService` / Ingress Annotations | Istio requires a heavy service-mesh control plane and sidecar injection. Ingress annotations lack standardized multi-backend weighted traffic splits. Gateway API provides a lightweight, vendor-neutral standard for traffic shifting. | +| **Custom Canary Rollout Engine** | Argo Rollouts / Flagger | Generic rollout tools treat metric anomalies as pure percentages without low-traffic statistical gating (`minRequestSample`). Building an embedded, re-entrant state machine allowed us to guarantee sample-size gating and MCP tool re-registration upon promotion. | +| **Native HPA via Custom Metrics** | KEDA (`ScaledObject`) | KEDA is powerful but adds external CRD dependencies. Generating native Kubernetes `HorizontalPodAutoscaler` objects tied to the Prometheus Adapter custom metrics pipeline minimized dependencies while giving full control over stabilization windows. | +| **Embedded Registry + ConfigMap Store** | Dedicated etcd / Redis / Database | Adding a dedicated database for service discovery increases operator operational complexity. The in-operator HTTP server with ConfigMap write-through store provides simple, robust storage for hundreds of agent services with cold-restart recovery. | +| **Two-Tier NetworkPolicy** | Istio / Linkerd Service Mesh | Service mesh requires sidecar injection and significant control plane memory overhead. Native Kubernetes NetworkPolicy with label-selector binding (`agentrax.io/agent: "true"`) provides lightweight, CNI-enforced zero-trust tenant isolation with default-deny rules. | +| **Go (`controller-runtime`)** | Python (`Kopf`) | Go provides native compile-time safety, seamless alignment with Kubernetes upstream libraries, and access to `setup-envtest` for isolated in-process integration testing. | --- diff --git a/docs/networking/README.md b/docs/networking/README.md deleted file mode 100644 index 5ae278b..0000000 --- a/docs/networking/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Agentrax — Tenant Network Isolation - -This document explains the two-tier network policy model shipped with Agentrax -and how platform operators apply it to tenant namespaces. - -## Overview - -Agentrax uses Kubernetes `NetworkPolicy` to enforce a **zero-trust perimeter** -around all agent pods. This prevents a compromised or misbehaving agent in one -tenant from reaching another tenant's services, the operator control plane, or -arbitrary internet destinations. - -Two policies are maintained: - -| Policy File | Namespace | Purpose | -| ----------------------------- | -------------------------- | -------------------------------------------------------------------------- | -| `allow-metrics-traffic.yaml` | `agentrax-system` | Allows Prometheus to scrape the operator `/metrics` endpoint | -| `tenant-agent-isolation.yaml` | Every `tenant-*` namespace | Isolates agent pods — restricts all ingress/egress to the minimum required | - -## How the Label Selector Works - -The `tenant-agent-isolation` policy uses `podSelector.matchLabels`: - -```yaml -podSelector: - matchLabels: - agentrax.io/agent: "true" -``` - -The `AgentDeployment` reconciler (`internal/controller/agentdeployment_controller.go`) -stamps this label onto every agent `Deployment`'s pod template via `agentLabels()`. -No manual labelling is needed — all agent pods are automatically covered. - -## Traffic Model - -``` -┌─────────────────────────────────────────────────────┐ -│ tenant-finance namespace │ -│ │ -│ [Agent Pod] agentrax.io/agent=true │ -│ │ │ -│ ├─ Ingress ← port 8080 ← [Prometheus] │ -│ │ (monitoring namespace only) │ -│ │ │ -│ ├─ Egress → port 6443 → [kube-apiserver] │ -│ ├─ Egress → port 53 → [CoreDNS] │ -│ │ │ -│ └─ ALL OTHER TRAFFIC: BLOCKED │ -└─────────────────────────────────────────────────────┘ -``` - -## Applying the Policy to Tenant Namespaces - -The `tenant-agent-isolation.yaml` NetworkPolicy must be applied to each tenant -namespace. The policy is **not** automatically applied by the operator — it is -applied once by a platform admin when provisioning a tenant namespace. - -### Apply Manually - -```bash -# Apply to a specific tenant namespace: -kubectl apply -n tenant-finance \ - -f config/network-policy/tenant-agent-isolation.yaml - -kubectl apply -n tenant-marketing \ - -f config/network-policy/tenant-agent-isolation.yaml -``` - -### Apply via Kustomize (Development) - -The default Kustomize overlay applies both network policies to the `agentrax-system` -namespace for development/testing. The `tenant-agent-isolation` policy in this -context validates the manifest schema; in production it must be applied per tenant -namespace as above. - -```bash -kubectl apply -k config/default/ -``` - -### Apply via Helm (Recommended for Production) - -When installing via Helm, set `networkPolicy.enabled: true` (Phase 1 Helm -integration — coming in a future release): - -```bash -helm upgrade --install agentrax charts/agentrax/ \ - --set networkPolicy.enabled=true -``` - -## Labelling the Prometheus Namespace - -The ingress rule allows traffic from namespaces labelled `monitoring: enabled`. -Apply this label to the namespace where Prometheus Operator / kube-prometheus-stack -is installed: - -```bash -kubectl label namespace monitoring monitoring=enabled -# Or, if using the default kube-prometheus-stack namespace name: -kubectl label namespace monitoring monitoring=enabled -``` - -## Required CNI Support - -This NetworkPolicy relies on a Container Network Interface (CNI) plugin that -**enforces** `NetworkPolicy` objects. Verify your CNI supports this: - -| Environment | Supported CNI | -| ---------------- | ----------------------------- | -| Kind (local dev) | Kindnet (default) ✅ | -| Azure AKS | Azure CNI or Calico ✅ | -| AWS EKS | VPC CNI + Calico or Cilium ✅ | -| GKE | Dataplane V2 (Cilium) ✅ | - -> **Note**: Flannel does **not** enforce NetworkPolicy by default. Use Calico or -> Cilium as a replacement CNI if Flannel is your cluster default. - -## Verifying the Policy - -After applying, verify that the policy is active and that an agent pod has -the correct label: - -```bash -# Confirm agent pod has the isolation label: -kubectl get pods -n tenant-finance -L agentrax.io/agent - -# Confirm the NetworkPolicy is present: -kubectl get networkpolicy -n tenant-finance - -# Test that cross-tenant traffic is blocked (from within an agent pod): -kubectl exec -n tenant-finance -- \ - curl --connect-timeout 2 http:// -# Expected: connection timed out (blocked) - -# Test that Kubernetes API access is allowed: -kubectl exec -n tenant-finance -- \ - curl -k https://kubernetes.default.svc:443/healthz -# Expected: "ok" -``` diff --git a/internal/controller/agentdeployment_builder_test.go b/internal/controller/agentdeployment_builder_test.go index 848f43c..9f8270f 100644 --- a/internal/controller/agentdeployment_builder_test.go +++ b/internal/controller/agentdeployment_builder_test.go @@ -240,7 +240,8 @@ func TestDesiredService_ClusterIPType(t *testing.T) { // ── agentLabels ─────────────────────────────────────────────────────────────── -// TestAgentLabels verifies standard label generation for an AgentDeployment. +// TestAgentLabels verifies standard label generation for an AgentDeployment, +// including the agentrax.io/agent selector key used by NetworkPolicies. func TestAgentLabels(t *testing.T) { ad := &agentraxv1alpha1.AgentDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, @@ -252,7 +253,14 @@ func TestAgentLabels(t *testing.T) { "app.kubernetes.io/name": "foo", "app.kubernetes.io/managed-by": "agentrax", "agentrax.io/tenant": "bar", + "agentrax.io/variant": "stable", + "agentrax.io/agent": "true", } + + if len(labels) != len(expected) { + t.Errorf("expected %d labels, got %d: %v", len(expected), len(labels), labels) + } + for k, v := range expected { if labels[k] != v { t.Errorf("agentLabels[%s] = %q, want %q", k, labels[k], v) From 300d68033b3c33def4fad3e12ab9192bb53b418c Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Sun, 23 Aug 2026 06:59:51 +0000 Subject: [PATCH 3/4] refactor: tighten tenant network policy egress to specific DNS pods and update architecture documentation Signed-off-by: Ankit Kr. Chowdhury --- .agents/skills/agentrax-context/SKILL.md | 2 +- .../network-policy/tenant-agent-isolation.yaml | 17 ++++++++++++----- docs/ARCHITECTURE.md | 12 ++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.agents/skills/agentrax-context/SKILL.md b/.agents/skills/agentrax-context/SKILL.md index 573a17a..7285fe1 100644 --- a/.agents/skills/agentrax-context/SKILL.md +++ b/.agents/skills/agentrax-context/SKILL.md @@ -20,7 +20,7 @@ description: Project context and settled architecture decisions for the Agentrax - **Autoscaling**: native `HorizontalPodAutoscaler` pointed at Prometheus Adapter custom metrics (`queueDepth` or `gpuUtilization`). No custom scaling loop. During active canary, the stable HPA is paused (deleted) and no canary HPA is created — autoscaling resumes only after promotion or rollback. - **Traffic splitting**: Gateway API `HTTPRoute` weighted backends. Not Istio, not ingress annotations. -- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (Operator in `agentrax-system`, tenant pods in `tenant-*` selected by `agentrax.io/agent: "true"`). No service mesh. +- **Network Isolation**: Two-tier Kubernetes `NetworkPolicy` (`allow-metrics-traffic` in `agentrax-system` allowing operator metrics on TCP 8443; `tenant-agent-isolation` rendered into every `tenant-*` namespace selecting agent pods with `agentrax.io/agent: "true"` for scraping on TCP 8080 and egress to API server/CoreDNS). No service mesh. - **MCP registry**: embedded HTTP handler inside the operator process, backed by a `ConfigMap`. Not a separate Deployment, not a new database — HA storage is a v2 item. - **Non-goals**: no model training/fine-tuning, no general-purpose workload management, no service mesh, no UI in v1. Flag any drift toward these rather than quietly implementing them. diff --git a/config/network-policy/tenant-agent-isolation.yaml b/config/network-policy/tenant-agent-isolation.yaml index 15af8bc..75619c6 100644 --- a/config/network-policy/tenant-agent-isolation.yaml +++ b/config/network-policy/tenant-agent-isolation.yaml @@ -22,8 +22,8 @@ # Usage: Apply this manifest to every tenant namespace: # kubectl apply -n tenant- -f tenant-agent-isolation.yaml # -# Note: This policy does NOT apply to the agentrax-system namespace (operator pods). -# The operator namespace is protected by allow-metrics-traffic.yaml. +# Note: This policy applies only to tenant-* namespaces (managed agent pods). +# The operator namespace (agentrax-system) is protected by allow-metrics-traffic.yaml (TCP 8443). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -41,7 +41,7 @@ spec: - Ingress - Egress ingress: - # Allow Prometheus to scrape /metrics on port 8080. + # Allow Prometheus to scrape agent /metrics on port 8080. # Prometheus Operator runs in a namespace labelled `monitoring: enabled`. - from: - namespaceSelector: @@ -59,8 +59,15 @@ spec: - port: 6443 protocol: TCP # Allow CoreDNS resolution on UDP and TCP port 53. - # Without this, service name lookups fail and MCP tool endpoints are unreachable. - - ports: + # Matches cluster DNS pods in any namespace (e.g. kube-system). + - to: + - namespaceSelector: {} + podSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["kube-dns", "coredns"] + ports: - port: 53 protocol: UDP - port: 53 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a83ed7c..f0d7abd 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -92,7 +92,7 @@ The repository enforces strict directional boundaries to prevent circular depend | Package | Scope & Responsibility | Key Invariants | | ---------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `api/v1alpha1/` | CRD type definitions, OpenAPI markers, schema validation rules, and status condition constants. | **Zero business logic**; only struct declarations and generated deep-copy methods. | -| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes). Consumes subsystems via interfaces. | +| `internal/controller/` | Controller-runtime reconcile loops (`AgentDeployment`, `TenantQuota`). | Only layer that executes write calls against the Kubernetes API for core-owned resources (Deployments, Services, HPAs, HTTPRoutes, ServiceMonitors). Consumes subsystems via interfaces. | | `internal/quota/` | Quota arithmetic and concurrency-safe in-flight reservation cache. | Pure arithmetic; mutex-guarded state map; zero direct API server network calls in calculation paths. | | `internal/webhook/` | Validating and Mutating admission webhooks. | Shared with `internal/quota` to enforce admission rules before objects are persisted. | | `internal/scaling/` | HPA synthesis, velocity rules, and dynamic quota ceiling headroom. | Calculates `QuotaHeadroom()` to cap HPA `maxReplicas` and applies stabilization windows. | @@ -325,15 +325,15 @@ Agentrax enforces a zero-trust network perimeter around all AI agent workloads r Agentrax maintains a **two-tier network policy model**: -| Policy Manifest | Target Namespace | Scope & Responsibility | -| :---------------------------- | :---------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `allow-metrics-traffic.yaml` | `agentrax-system` | Protects the operator process; allows Prometheus to scrape `/metrics` on port `:8443`/`:8080`. | +| Policy Manifest | Target Namespace | Scope & Responsibility | +| :---------------------------- | :---------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `allow-metrics-traffic.yaml` | `agentrax-system` | Protects the operator process; allows Prometheus to scrape operator `/metrics` on port `:8443` (HTTPS). | | `tenant-agent-isolation.yaml` | Every `tenant-*` | Isolates agent pods; enforces default-deny on ingress/egress, strictly whitelisting only metrics scraping (`:8080`), Kubernetes API server (`:443`/`:6443`), and CoreDNS (`:53`). | #### Ingress & Egress Invariants: -- **Ingress**: Only TCP port `8080` from namespaces labeled `monitoring: enabled` (Prometheus metric scraping). -- **Egress**: Only TCP ports `443`/`6443` (`kube-apiserver`) and UDP/TCP port `53` (`CoreDNS`). All other outbound egress (cross-tenant, external internet) is blocked at the CNI layer. +- **Ingress**: Only TCP port `8080` from namespaces labeled `monitoring: enabled` (Prometheus scraping tenant agent metrics). +- **Egress**: Only to the Kubernetes API server (`kube-apiserver` on TCP ports `443`/`6443`) and cluster CoreDNS (`UDP/TCP :53` in DNS pods). All cross-tenant and arbitrary external internet egress destinations remain blocked at the CNI layer. - **Label Selector Binding**: The `tenant-agent-isolation` policy selects pods dynamically via `agentrax.io/agent: "true"`. The `AgentDeploymentReconciler` automatically stamps this label into the `PodTemplateSpec` of every managed `Deployment` via `agentLabels()`. --- From 39bb184db2067bed6af7399aafdad4e6c47652e5 Mon Sep 17 00:00:00 2001 From: "Ankit Kr. Chowdhury" Date: Sun, 23 Aug 2026 07:09:37 +0000 Subject: [PATCH 4/4] refactor: improve test reliability with conflict retries and constrain network egress DNS to kube-system namespace Signed-off-by: Ankit Kr. Chowdhury --- .../tenant-agent-isolation.yaml | 6 ++-- docs/ARCHITECTURE.md | 2 +- .../controller/tenantquota_controller_test.go | 34 +++++++++++++------ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/config/network-policy/tenant-agent-isolation.yaml b/config/network-policy/tenant-agent-isolation.yaml index 75619c6..bdc4740 100644 --- a/config/network-policy/tenant-agent-isolation.yaml +++ b/config/network-policy/tenant-agent-isolation.yaml @@ -59,9 +59,11 @@ spec: - port: 6443 protocol: TCP # Allow CoreDNS resolution on UDP and TCP port 53. - # Matches cluster DNS pods in any namespace (e.g. kube-system). + # Matches cluster DNS pods in the kube-system namespace. - to: - - namespaceSelector: {} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system podSelector: matchExpressions: - key: k8s-app diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f0d7abd..98a73e9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -333,7 +333,7 @@ Agentrax maintains a **two-tier network policy model**: #### Ingress & Egress Invariants: - **Ingress**: Only TCP port `8080` from namespaces labeled `monitoring: enabled` (Prometheus scraping tenant agent metrics). -- **Egress**: Only to the Kubernetes API server (`kube-apiserver` on TCP ports `443`/`6443`) and cluster CoreDNS (`UDP/TCP :53` in DNS pods). All cross-tenant and arbitrary external internet egress destinations remain blocked at the CNI layer. +- **Egress**: Only to the Kubernetes API server (`kube-apiserver` on TCP ports `443`/`6443`) and cluster CoreDNS (`UDP/TCP :53` in `kube-system` DNS pods). All cross-tenant and arbitrary external internet egress destinations remain blocked at the CNI layer. - **Label Selector Binding**: The `tenant-agent-isolation` policy selects pods dynamically via `agentrax.io/agent: "true"`. The `AgentDeploymentReconciler` automatically stamps this label into the `PodTemplateSpec` of every managed `Deployment` via `agentLabels()`. --- diff --git a/internal/controller/tenantquota_controller_test.go b/internal/controller/tenantquota_controller_test.go index 0455ab7..33db6e0 100644 --- a/internal/controller/tenantquota_controller_test.go +++ b/internal/controller/tenantquota_controller_test.go @@ -136,12 +136,18 @@ var _ = Describe("TenantQuota Controller", func() { g.Expect(fetched.Status.UsedAgents).To(BeNumerically("==", 1)) }, timeout, interval).Should(Succeed()) - // Remove finalizer so we can delete immediately. - fetched := &agentraxv1alpha1.AgentDeployment{} - Expect(k8sClient.Get(ctx, namespacedName("ad-del-1", tqNS), fetched)).To(Succeed()) - fetched.Finalizers = nil - Expect(k8sClient.Update(ctx, fetched)).To(Succeed()) - Expect(k8sClient.Delete(ctx, fetched)).To(Succeed()) + // Remove finalizer with conflict retry so deletion is not raced by the controller. + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &agentraxv1alpha1.AgentDeployment{} + if err := k8sClient.Get(ctx, namespacedName("ad-del-1", tqNS), latest); err != nil { + return err + } + latest.Finalizers = nil + return k8sClient.Update(ctx, latest) + })).To(Succeed()) + Expect(k8sClient.Delete(ctx, &agentraxv1alpha1.AgentDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ad-del-1", Namespace: tqNS}, + })).To(Succeed()) Eventually(func(g Gomega) { tqFetched := &agentraxv1alpha1.TenantQuota{} @@ -217,11 +223,17 @@ var _ = Describe("TenantQuota Controller", func() { }, timeout, interval).Should(Succeed()) // Delete one AD → usage falls back to 1 == maxAgents; OverQuota clears. - ad1 := &agentraxv1alpha1.AgentDeployment{} - Expect(k8sClient.Get(ctx, namespacedName("ad-clearoq-1", tqNS), ad1)).To(Succeed()) - ad1.Finalizers = nil - Expect(k8sClient.Update(ctx, ad1)).To(Succeed()) - Expect(k8sClient.Delete(ctx, ad1)).To(Succeed()) + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &agentraxv1alpha1.AgentDeployment{} + if err := k8sClient.Get(ctx, namespacedName("ad-clearoq-1", tqNS), latest); err != nil { + return err + } + latest.Finalizers = nil + return k8sClient.Update(ctx, latest) + })).To(Succeed()) + Expect(k8sClient.Delete(ctx, &agentraxv1alpha1.AgentDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "ad-clearoq-1", Namespace: tqNS}, + })).To(Succeed()) Eventually(func(g Gomega) { f := &agentraxv1alpha1.TenantQuota{}