diff --git a/README.md b/README.md index 9e60ae7..0838991 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ configurable values. ```sh helm upgrade hyperbytedb-operator \ oci://ghcr.io/hyperbyte-cloud/charts/hyperbytedb-operator \ - --version 0.3.1 \ + --version 0.3.2 \ --namespace hyperbytedb-operator-system \ --reuse-values ``` diff --git a/dist/chart/Chart.yaml b/dist/chart/Chart.yaml index dc1b195..64d42d4 100644 --- a/dist/chart/Chart.yaml +++ b/dist/chart/Chart.yaml @@ -7,8 +7,8 @@ type: application # release workflow (.github/workflows/release.yml) stamps them with the git # tag (or a `0.0.0-sha.` snapshot for main branch builds) when the # chart is packaged and pushed to ghcr.io. -version: 0.3.1 -appVersion: "0.3.1" +version: 0.3.2 +appVersion: "0.3.2" home: https://hyperbyte-cloud.github.io/hyperbytedb/ sources: diff --git a/docs/compatibility.md b/docs/compatibility.md index 75c68b2..d3ae8a8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -32,12 +32,13 @@ Reasons: `Supported`, `Assumed` (no version found), `NewerThanKnown` | Operator | Renders for HyperbyteDB | Newest known | |---|---|---| +| v0.3.2 | 0.8.0 and later | 0.9.1-beta | | v0.3.1 | 0.8.0 and later | 0.9.1-beta | | v0.3.0 | 0.8.0 and later | 0.9.1-beta | | v0.2.0 | 0.8.0 and later | 0.9.0-beta | | v0.1.x | not version-aware: renders the newest keys for every cluster (0.8.x ignores the ones it does not know) | — | -What v0.3.1 renders per version: +What v0.3.2 renders per version: | Capability | HyperbyteDB | For older versions the operator | |---|---|---| @@ -55,6 +56,15 @@ The source of truth is `CompatibilityMatrix` in ## Upgrading +- v0.3.1 → v0.3.2: a pool stuck in a rolling restart recovers by itself. + The proxy Deployment is applied before the rolling restart, so a broken + proxy is fixed even while a restart waits on it; proxies are excluded + from and included in a restart when their process runs, Ready or not + (a one-node pool's proxies are not Ready while its only backend is + excluded), and proxies that are not running are skipped; a restart that + has excluded nothing while no update is pending is cleared. A `Degraded` + condition is set back to `False` once a reconcile completes. No pod + template changes. - v0.3.0 → v0.3.1: a proxy on 0.9.0-beta or later is probed on its admin listener (port 8087, or 8088 when the proxy's client port is 8087). v0.3.0 probed `/healthz` and `/readyz` on the client port, where those diff --git a/internal/controller/hyperbytedbcluster_controller.go b/internal/controller/hyperbytedbcluster_controller.go index 91de1c3..c528233 100644 --- a/internal/controller/hyperbytedbcluster_controller.go +++ b/internal/controller/hyperbytedbcluster_controller.go @@ -205,6 +205,17 @@ func (r *HyperbytedbClusterReconciler) Reconcile(ctx context.Context, req ctrl.R return r.setFailedStatus(ctx, cluster, "ScaleDownFailed", err) } + // 6a'. Proxy (optional; enabled by default, spec.proxy.enabled=false to + // skip). Before the rolling restart, which coordinates through the + // proxies and can return early on every pass: applied after it, a + // broken proxy Deployment (v0.3.0 probed 0.9 proxies on the wrong port) + // was never fixed while a restart was waiting on that proxy. + if hyperbytedb.ProxyEnabled(cluster) { + if err := r.reconcileProxy(ctx, cluster); err != nil { + log.Error(err, "Failed to reconcile proxy") + } + } + // 6b. Operator-driven rolling restart coordination. When the StatefulSet // has a pending rolling update and the proxy is enabled, the operator // orchestrates pod-by-pod exclusion from the proxy, drain wait, pod @@ -254,14 +265,6 @@ func (r *HyperbytedbClusterReconciler) Reconcile(ctx context.Context, req ctrl.R log.Error(err, "Failed to reconcile HPA") } - // 11b. Proxy (optional). Enabled by default; set spec.proxy.enabled=false - // to skip reconciliation. - if hyperbytedb.ProxyEnabled(cluster) { - if err := r.reconcileProxy(ctx, cluster); err != nil { - log.Error(err, "Failed to reconcile proxy") - } - } - // 12. Collect member statuses memberStatuses := r.Members.CollectMemberStatuses(ctx, cluster, cluster.Namespace, replicas) cluster.Status.Members = memberStatuses @@ -298,7 +301,16 @@ func (r *HyperbytedbClusterReconciler) Reconcile(ctx context.Context, req ctrl.R log.Error(err, "Auto-failover handling failed") } - // 16. Update status + // 16. Update status. Reaching here means every step succeeded, so a + // Degraded condition from an earlier failure no longer holds; it stayed + // True on healthy clusters because nothing cleared it. + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: "Degraded", + Status: metav1.ConditionFalse, + Reason: "Reconciled", + Message: "the last reconcile completed", + ObservedGeneration: cluster.Generation, + }) r.updateStatus(ctx, cluster, stsResult, configHash) return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -562,6 +574,20 @@ func (r *HyperbytedbClusterReconciler) reconcileRollingRestart( return nil, nil } + // A restart that has not excluded anything yet while no update is + // pending any more has nothing to do. Such a state held a pool's + // rolling restart (and the proxy fix behind it) until it was cleared + // by hand. + if state := cluster.Status.RollingRestart; !rollingUpgrade && state != nil && + state.Phase == hyperbytedbv1alpha1.RollingRestartExcluding && !state.ExcludeConfirmed { + cluster.Status.RollingRestart = nil + if err := r.Status().Update(ctx, cluster); err != nil { + return nil, err + } + logf.FromContext(ctx).Info("Cleared a rolling restart that had not started and has no pending update") + return nil, nil + } + // The proxies' admin API: its own listener from proxy 0.9.0-beta, else // the proxy's client port (spec.proxy.port, not the database's). proxyPort := hyperbytedb.ProxyAdminPort(cluster) @@ -812,49 +838,50 @@ func isPodReady(pod *corev1.Pod) bool { return false } -// excludeFromAllProxies sends the exclude command to every proxy pod -// individually, so the exclusion takes effect on all proxy instances (the -// ClusterIP Service only round-robins to one). Returns an error if any -// proxy pod is not ready so the caller retries. +// excludeFromAllProxies tells every running proxy pod to stop routing to +// backendIP (each pod individually: the ClusterIP Service only reaches one). func (r *HyperbytedbClusterReconciler) excludeFromAllProxies( ctx context.Context, cluster *hyperbytedbv1alpha1.HyperbytedbCluster, backendIP string, proxyPort int32, ) error { - proxyPods := &corev1.PodList{} - if err := r.List(ctx, proxyPods, - client.InNamespace(cluster.Namespace), - client.MatchingLabels{ - "app.kubernetes.io/name": "hyperbytedb-proxy", - "app.kubernetes.io/instance": cluster.Name, - "app.kubernetes.io/managed-by": "hyperbytedb-operator", - "app.kubernetes.io/component": "proxy", - }, - ); err != nil { - return fmt.Errorf("listing proxy pods: %w", err) - } - for i := range proxyPods.Items { - pod := &proxyPods.Items[i] - if pod.Status.PodIP == "" || !isPodReady(pod) { - return fmt.Errorf("proxy pod %s not ready (IP=%s, ready=%v), will retry", pod.Name, pod.Status.PodIP, isPodReady(pod)) - } - if err := r.Members.Client.ExcludeProxyBackend(ctx, pod.Status.PodIP, proxyPort, backendIP); err != nil { - return fmt.Errorf("exclude on proxy pod %s: %w", pod.Name, err) - } - } - return nil + return r.forEachRunningProxy(ctx, cluster, func(pod *corev1.Pod) error { + return r.Members.Client.ExcludeProxyBackend(ctx, pod.Status.PodIP, proxyPort, backendIP) + }) } -// includeFromAllProxies sends the include command to every proxy pod. -// Returns an error if any proxy pod is not ready, so the caller retries — -// skipping a proxy would leave a stale exclusion on that instance. +// includeFromAllProxies tells every running proxy pod to route to backendIP +// again. func (r *HyperbytedbClusterReconciler) includeFromAllProxies( ctx context.Context, cluster *hyperbytedbv1alpha1.HyperbytedbCluster, backendIP string, proxyPort int32, ) error { + return r.forEachRunningProxy(ctx, cluster, func(pod *corev1.Pod) error { + return r.Members.Client.IncludeProxyBackend(ctx, pod.Status.PodIP, proxyPort, backendIP) + }) +} + +// forEachRunningProxy calls send for every proxy pod whose proxy process is +// running, ready or not, and skips the rest. +// +// It used to require every proxy pod to be Ready and retried until they +// were, which deadlocked twice over. A proxy that is down (crash-looping, +// or a Deployment the operator had not yet fixed) never became Ready, so a +// rolling restart waited on it forever. And a proxy is Ready only with a +// routable backend: on a one-node pool, excluding the only backend makes +// every proxy un-Ready, so the include that would make them Ready again +// never went out. The admin API answers whenever the process runs, and a +// proxy that is not running holds no exclusions (they are in memory): it +// starts with none. +func (r *HyperbytedbClusterReconciler) forEachRunningProxy( + ctx context.Context, + cluster *hyperbytedbv1alpha1.HyperbytedbCluster, + send func(*corev1.Pod) error, +) error { + log := logf.FromContext(ctx) proxyPods := &corev1.PodList{} if err := r.List(ctx, proxyPods, client.InNamespace(cluster.Namespace), @@ -869,16 +896,32 @@ func (r *HyperbytedbClusterReconciler) includeFromAllProxies( } for i := range proxyPods.Items { pod := &proxyPods.Items[i] - if pod.Status.PodIP == "" || !isPodReady(pod) { - return fmt.Errorf("proxy pod %s not ready (IP=%s, ready=%v), will retry", pod.Name, pod.Status.PodIP, isPodReady(pod)) + if !proxyRunning(pod) { + log.Info("Skipping proxy pod that is not running", "pod", pod.Name, "ip", pod.Status.PodIP) + continue } - if err := r.Members.Client.IncludeProxyBackend(ctx, pod.Status.PodIP, proxyPort, backendIP); err != nil { - return fmt.Errorf("include on proxy pod %s: %w", pod.Name, err) + if err := send(pod); err != nil { + return fmt.Errorf("proxy pod %s: %w", pod.Name, err) } } return nil } +// proxyRunning reports whether the pod's proxy container is running, so its +// admin API can answer: it has an IP, is not being deleted, and the "proxy" +// container is in the Running state. +func proxyRunning(pod *corev1.Pod) bool { + if pod.Status.PodIP == "" || pod.DeletionTimestamp != nil { + return false + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.Name == "proxy" { + return cs.State.Running != nil + } + } + return false +} + // ---------- Scale-down handling ---------- // runScaleDownClusterHooks drains pods that will be removed (highest ordinals first) and asks diff --git a/internal/controller/proxy_restart_test.go b/internal/controller/proxy_restart_test.go new file mode 100644 index 0000000..d347711 --- /dev/null +++ b/internal/controller/proxy_restart_test.go @@ -0,0 +1,220 @@ +package controller + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strconv" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + hyperbytedbv1alpha1 "github.com/hyperbyte-cloud/hyperbytedb-operator/api/v1alpha1" + "github.com/hyperbyte-cloud/hyperbytedb-operator/internal/hyperbytedb" +) + +// A pool whose proxy was down when a rolling restart began: the restart +// retried excluding pod 0 from the proxies forever, and the proxy +// Deployment, applied after the restart step, was never fixed. +var _ = Describe("Rolling restart with the proxy down", func() { + ctx := context.Background() + // A fresh pool per test: the reconciler adds a finalizer that nothing + // removes here, so a deleted pool lingers. + var name string + var key types.NamespacedName + pools := 0 + + reconciler := func() *HyperbytedbClusterReconciler { + return &HyperbytedbClusterReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Recorder: record.NewFakeRecorder(64), + Members: hyperbytedb.NewMemberManager(), + } + } + + BeforeEach(func() { + pools++ + name = fmt.Sprintf("stuck-pool-%d", pools) + key = types.NamespacedName{Name: name, Namespace: "default"} + Expect(k8sClient.Create(ctx, &hyperbytedbv1alpha1.HyperbytedbCluster{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: hyperbytedbv1alpha1.HyperbytedbClusterSpec{ + Replicas: ptr.To(int32(1)), + Version: "0.9.1-beta", + Proxy: &hyperbytedbv1alpha1.ProxySpec{Enabled: true}, + }, + })).To(Succeed()) + // First pass creates the StatefulSet. + _, _ = reconciler().Reconcile(ctx, reconcile.Request{NamespacedName: key}) + }) + + AfterEach(func() { + _ = k8sClient.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace("default")) + }) + + // Seed the stuck state: a restart waiting to exclude pod 0, an update + // pending on the StatefulSet, and the only proxy pod crash-looping. + seedStuckRestart := func() { + sts := &appsv1.StatefulSet{} + Expect(k8sClient.Get(ctx, key, sts)).To(Succeed()) + sts.Status.Replicas = 1 + sts.Status.CurrentRevision = "old" + sts.Status.UpdateRevision = "new" + Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) + + cluster := &hyperbytedbv1alpha1.HyperbytedbCluster{} + Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed()) + cluster.Status.RollingRestart = &hyperbytedbv1alpha1.RollingRestartState{ + TotalOrdinals: 1, + Phase: hyperbytedbv1alpha1.RollingRestartExcluding, + PhaseStartedAt: metav1.Now(), + } + Expect(k8sClient.Status().Update(ctx, cluster)).To(Succeed()) + + db := testPod(name+"-0", map[string]string{"app.kubernetes.io/name": "hyperbytedb"}, "10.0.0.10", "hyperbytedb", true) + Expect(createWithStatus(ctx, db)).To(Succeed()) + proxy := testPod(name+"-proxy-x", proxyPodLabels(name), "10.0.0.20", "proxy", false) + Expect(createWithStatus(ctx, proxy)).To(Succeed()) + } + + It("applies the proxy Deployment and gets past the exclusion", func() { + seedStuckRestart() + _, err := reconciler().Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + + dep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name + "-proxy", Namespace: "default"}, dep)). + To(Succeed(), "the proxy Deployment is applied before the restart step") + probe := dep.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port.String() + Expect(probe).To(Equal("admin")) + + cluster := &hyperbytedbv1alpha1.HyperbytedbCluster{} + Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed()) + Expect(cluster.Status.RollingRestart).NotTo(BeNil()) + Expect(cluster.Status.RollingRestart.ExcludeConfirmed).To(BeTrue(), + "no proxy is running, so there is nothing to exclude from and the restart moves on") + }) + + It("clears a restart that never started once no update is pending", func() { + seedStuckRestart() + sts := &appsv1.StatefulSet{} + Expect(k8sClient.Get(ctx, key, sts)).To(Succeed()) + sts.Status.UpdateRevision = sts.Status.CurrentRevision + Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) + + _, err := reconciler().Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + cluster := &hyperbytedbv1alpha1.HyperbytedbCluster{} + Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed()) + Expect(cluster.Status.RollingRestart).To(BeNil()) + }) + + It("clears a stale Degraded condition once a reconcile completes", func() { + cluster := &hyperbytedbv1alpha1.HyperbytedbCluster{} + Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed()) + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: "Degraded", Status: metav1.ConditionTrue, + Reason: "RollingRestartFailed", Message: "from an earlier failure", + }) + Expect(k8sClient.Status().Update(ctx, cluster)).To(Succeed()) + + _, err := reconciler().Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, key, cluster)).To(Succeed()) + cond := meta.FindStatusCondition(cluster.Status.Conditions, "Degraded") + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal("Reconciled")) + }) +}) + +// One proxy pod whose process runs but is not Ready (its only backend is +// excluded) and one that is crash-looping: exclude and include go to the +// running one only, and are not held for readiness. Requiring Ready made the +// include that would make a one-node pool's proxies Ready again never go out. +var _ = Describe("Proxy exclude and include", func() { + ctx := context.Background() + const name = "one-node-pool" + + It("reach every running proxy, Ready or not, and skip the rest", func() { + var mu sync.Mutex + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + got = append(got, r.URL.Path) + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + host, portStr, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + + running := testPod(name+"-proxy-a", proxyPodLabels(name), host, "proxy", false) + running.Status.ContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + Expect(createWithStatus(ctx, running)).To(Succeed()) + down := testPod(name+"-proxy-b", proxyPodLabels(name), "10.0.0.31", "proxy", false) + Expect(createWithStatus(ctx, down)).To(Succeed()) + defer func() { _ = k8sClient.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace("default")) }() + + r := &HyperbytedbClusterReconciler{Client: k8sClient, Members: hyperbytedb.NewMemberManager()} + cluster := &hyperbytedbv1alpha1.HyperbytedbCluster{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}} + Expect(r.excludeFromAllProxies(ctx, cluster, "10.0.0.10", int32(port))).To(Succeed()) + Expect(r.includeFromAllProxies(ctx, cluster, "10.0.0.10", int32(port))).To(Succeed()) + Expect(got).To(Equal([]string{ + "/admin/backends/10.0.0.10/exclude", + "/admin/backends/10.0.0.10/include", + })) + }) +}) + +func proxyPodLabels(cluster string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "hyperbytedb-proxy", + "app.kubernetes.io/instance": cluster, + "app.kubernetes.io/managed-by": "hyperbytedb-operator", + "app.kubernetes.io/component": "proxy", + } +} + +// testPod is a pod with one container: Ready and running when ready is +// true, else crash-looping (waiting, not Ready). +func testPod(name string, labels map[string]string, ip, container string, ready bool) *corev1.Pod { + state := corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}} + readyStatus := corev1.ConditionFalse + if ready { + state = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + readyStatus = corev1.ConditionTrue + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: container, Image: "x"}}}, + Status: corev1.PodStatus{ + PodIP: ip, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: readyStatus}}, + ContainerStatuses: []corev1.ContainerStatus{{Name: container, Ready: ready, State: state}}, + }, + } +} + +// createWithStatus creates the pod, then writes its status (create ignores it). +func createWithStatus(ctx context.Context, pod *corev1.Pod) error { + status := pod.Status + if err := k8sClient.Create(ctx, pod); err != nil { + return fmt.Errorf("create %s: %w", pod.Name, err) + } + pod.Status = status + return k8sClient.Status().Update(ctx, pod) +} diff --git a/internal/hyperbytedb/client.go b/internal/hyperbytedb/client.go index 871c949..e55a446 100644 --- a/internal/hyperbytedb/client.go +++ b/internal/hyperbytedb/client.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strings" "time" ) @@ -511,11 +512,19 @@ func (c *Client) ExcludeProxyBackend(ctx context.Context, proxyHost string, port return fmt.Errorf("exclude backend %s: %w", ip, err) } defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - respBody, _ := io.ReadAll(resp.Body) + respBody, _ := io.ReadAll(resp.Body) + switch { + // 409: already excluded (a retry after another proxy failed). As an + // error it made every retry fail on the proxies that had done it. + case resp.StatusCode == http.StatusOK, resp.StatusCode == http.StatusConflict: + return nil + // The proxy's own 404 for a backend not in its pool: it does not route + // there either. Any other 404 (a wrong port or path) stays an error. + case resp.StatusCode == http.StatusNotFound && strings.Contains(string(respBody), "not found in pool"): + return nil + default: return fmt.Errorf("exclude backend %s returned %d: %s", ip, resp.StatusCode, string(respBody)) } - return nil } // IncludeProxyBackend tells the proxy to resume routing to the given backend IP. diff --git a/internal/hyperbytedb/proxy_test.go b/internal/hyperbytedb/proxy_test.go index d8b002b..fb9c250 100644 --- a/internal/hyperbytedb/proxy_test.go +++ b/internal/hyperbytedb/proxy_test.go @@ -1,6 +1,11 @@ package hyperbytedb import ( + "context" + "net" + "net/http" + "net/http/httptest" + "strconv" "testing" corev1 "k8s.io/api/core/v1" @@ -134,3 +139,33 @@ func TestTheProxyServiceExposesOnlyTheClientPort(t *testing.T) { t.Errorf("service ports = %v, want only %s", svc.Spec.Ports, httpPortName) } } + +// Exclude is asked again after a partial failure: a proxy that already did it +// answers 409, and one that has no such backend answers its own 404. Both +// are done; any other 404 (a wrong port or path, as v0.3.0's) is not. +func TestExcludeTreatsAlreadyDoneAsDone(t *testing.T) { + cases := []struct { + status int + body string + ok bool + }{ + {http.StatusOK, `{"status":"excluded"}`, true}, + {http.StatusConflict, `{"status":"already_excluded"}`, true}, + {http.StatusNotFound, `{"status":"error","message":"backend 10.0.0.9 not found in pool"}`, true}, + {http.StatusNotFound, `not found`, false}, + {http.StatusInternalServerError, `boom`, false}, + } + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + host, portStr, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + err := NewClient().ExcludeProxyBackend(context.Background(), host, int32(port), "10.0.0.9") + if (err == nil) != tc.ok { + t.Errorf("%d %q: err = %v, want ok=%v", tc.status, tc.body, err, tc.ok) + } + srv.Close() + } +}