feat: implement canary rollout logic with PromQL-based threshold eval…

.agents/AGENTS.md

@@ -9,7 +9,7 @@
 2. **Reconciler Pattern**:
    - Fetch the object first; if not found (`apierrors.IsNotFound`), return `ctrl.Result{}` immediately — it was deleted.
    - Always update `status` last, after all child resources are reconciled. Never update status mid-reconcile.
-   - Use `controllerutil.CreateOrUpdate` for all owned child resources (Deployment, Service, ServiceMonitor, HPA).
+   - Use `controllerutil.CreateOrUpdate` for all owned child resources (Deployment, Service, ServiceMonitor, HPA, HTTPRoute).
    - Requeue transient errors with `ctrl.Result{RequeueAfter: ...}`, not `ctrl.Result{Requeue: true}`.
 
 3. **Owner References & Finalizers**:

.agents/skills/agentrax-context/SKILL.md

@@ -56,8 +56,11 @@ description: Project context and settled architecture decisions for the Agentrax
 | Canary    | Prometheus unreachable during pause             | Fail-safe rollback after 60 s; never hang                                        |
 | Canary    | Second rollout triggered mid-rollout            | Webhook rejects; never run two concurrent canaries on the same `AgentDeployment` |
 | Canary    | HPA during active canary                        | Pause stable HPA; no canary HPA; resume only after promote/rollback              |
+| Canary    | Out-of-band deletion of HTTPRoute/Deployment    | Self-healed on next reconcile cycle in `Step()` preserving active traffic split  |
+| Canary    | Rollout failed or aborted                       | Operator sets `RolloutFailed`, preserves `status.canaryVersion`, no retry loop   |
 | Quota     | Two concurrent near-limit creates               | In-flight reservation; one wins, one is rejected                                 |
 | Quota     | Quota lowered below current usage               | Set `OverQuota` condition; never forcibly delete existing resources              |
+| Quota     | `TenantQuota` deleted while agents exist        | Surface `TenantQuotaNotFound` on `QuotaLimited` condition; never crash           |
 | MCP       | Ungraceful pod termination                      | TTL/heartbeat expires the entry within one TTL window (default 90 s)             |
 | MCP       | Pod `Ready` but MCP handshake fails             | Do not register; surface `MCPHandshakeFailed` condition                          |
 | Deletion  | `AgentDeployment` deleted                       | Finalizer ensures MCP deregistration before `Service` is GC'd                    |

api/v1alpha1/agentdeployment_types.go

@@ -162,6 +162,21 @@ type AgentDeploymentStatus struct {
 	// CanaryWeight is the current percentage of traffic routed to the canary (0–100).
 	CanaryWeight int32 `json:"canaryWeight,omitempty"`
 
+	// CanaryStepIndex is the index of the currently executing rollout step.
+	// Persisted so the state machine survives operator restarts.
+	// +optional
+	CanaryStepIndex int `json:"canaryStepIndex,omitempty"`
+
+	// PauseStartedAt records when the current pause step began.
+	// Used to enforce maximum pause extensions and the fail-safe rollback timeout.
+	// +optional
+	PauseStartedAt *metav1.Time `json:"pauseStartedAt,omitempty"`
+
+	// PromUnreachableSince records when Prometheus last became unreachable.
+	// When non-nil and age exceeds FailSafeTimeout, a fail-safe rollback fires.
+	// +optional
+	PromUnreachableSince *metav1.Time `json:"promUnreachableSince,omitempty"`
+
 	// Registered is true when this agent is currently registered in the MCP registry.
 	Registered bool `json:"registered,omitempty"`
 

cmd/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -39,10 +40,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
 	"github.com/gitcommitankit/agentrax/internal/controller"
+	"github.com/gitcommitankit/agentrax/internal/metrics"
 	"github.com/gitcommitankit/agentrax/internal/quota"
+	"github.com/gitcommitankit/agentrax/internal/rollout"
 	agentraxwebhook "github.com/gitcommitankit/agentrax/internal/webhook"
 	// +kubebuilder:scaffold:imports
 )
@@ -58,6 +62,7 @@ func init() {
 	utilruntime.Must(autoscalingv2.AddToScheme(scheme))
 	utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
 	utilruntime.Must(monitoringv1.AddToScheme(scheme))
+	utilruntime.Must(gatewayv1.Install(scheme))
 
 	utilruntime.Must(agentraxv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
@@ -71,6 +76,9 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var gpuResourceName string
+	var prometheusURL string
+	var gatewayName string
+	var gatewayNamespace string
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -84,6 +92,13 @@ func main() {
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&gpuResourceName, "gpu-resource-name", quota.DefaultGPUResourceName,
 		"Kubernetes resource name used to count GPU units in AgentDeployment resource limits.")
+	flag.StringVar(&prometheusURL, "prometheus-url", "",
+		"URL of the Prometheus HTTP API (e.g. http://prometheus-operated.monitoring.svc:9090). "+
+			"Required for Canary rollout strategy; if empty, canary is unavailable.")
+	flag.StringVar(&gatewayName, "gateway-name", "agentrax-gateway",
+		"Name of the Gateway API Gateway object used for canary traffic splitting.")
+	flag.StringVar(&gatewayNamespace, "gateway-namespace", "agentrax-system",
+		"Namespace of the Gateway API Gateway object used for canary traffic splitting.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -171,10 +186,29 @@ func main() {
 	// Shared quota enforcer used by both the webhook validator and TenantQuota reconciler.
 	quotaEnforcer := quota.NewEnforcer(gpuResourceName)
 
+	// Build the CanaryController when --prometheus-url is provided.
+	// When nil, AgentDeployments with strategy=Canary behave as Recreate.
+	var canaryController *rollout.Controller
+	if prometheusURL != "" {
+		setupLog.Info("canary rollout enabled", "prometheusURL", prometheusURL,
+			"gatewayName", gatewayName, "gatewayNamespace", gatewayNamespace)
+		canaryController = &rollout.Controller{
+			Client:           mgr.GetClient(),
+			Scheme:           mgr.GetScheme(),
+			PromClient:       metrics.NewClient(prometheusURL),
+			GatewayName:      gatewayName,
+			GatewayNamespace: gatewayNamespace,
+			FailSafeTimeout:  60 * time.Second,
+		}
+	} else {
+		setupLog.Info("canary rollout disabled (no --prometheus-url)")
+	}
+
 	if err = (&controller.AgentDeploymentReconciler{
-		Client:          mgr.GetClient(),
-		Scheme:          mgr.GetScheme(),
-		GPUResourceName: gpuResourceName,
+		Client:           mgr.GetClient(),
+		Scheme:           mgr.GetScheme(),
+		GPUResourceName:  gpuResourceName,
+		CanaryController: canaryController,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "AgentDeployment")
 		os.Exit(1)

config/crd/bases/agentrax.io_agentdeployments.yaml

@@ -377,6 +377,11 @@ spec:
           status:
             description: AgentDeploymentStatus defines the observed state of an AgentDeployment.
             properties:
+              canaryStepIndex:
+                description: |-
+                  CanaryStepIndex is the index of the currently executing rollout step.
+                  Persisted so the state machine survives operator restarts.
+                type: integer
               canaryVersion:
                 description: CanaryVersion is the container image tag of the canary
                   deployment, if one is in progress.
@@ -447,6 +452,12 @@ spec:
                 description: CurrentReplicas is the number of replicas currently running.
                 format: int32
                 type: integer
+              pauseStartedAt:
+                description: |-
+                  PauseStartedAt records when the current pause step began.
+                  Used to enforce maximum pause extensions and the fail-safe rollback timeout.
+                format: date-time
+                type: string
               phase:
                 description: Phase is the high-level lifecycle phase of this deployment.
                 enum:
@@ -456,6 +467,12 @@ spec:
                 - RolloutFailed
                 - Degraded
                 type: string
+              promUnreachableSince:
+                description: |-
+                  PromUnreachableSince records when Prometheus last became unreachable.
+                  When non-nil and age exceeds FailSafeTimeout, a fail-safe rollback fires.
+                format: date-time
+                type: string
               registered:
                 description: Registered is true when this agent is currently registered
                   in the MCP registry.