From 1408c8ed32f476d342449c83d1d1b57ff0a53d7d Mon Sep 17 00:00:00 2001 From: Exonical Date: Fri, 29 May 2026 04:42:34 +0000 Subject: [PATCH 1/2] feat: auto-set LB IP-pinning and sharing-key annotations from allocation IP When lb_ip_annotation and lb_sharing_key config keys are set, Wings automatically populates them on each game server's LoadBalancer Service using the server's allocation IP from the Panel. This enables multiple game servers to share a single external IP (on different ports) without manual annotation patching. Example config for Cilium LB-IPAM: kubernetes: lb_ip_annotation: "lbipam.cilium.io/ips" lb_sharing_key: "lbipam.cilium.io/sharing-key" lb_annotations: io.cilium/lb-ipam-pool: "game-servers" Co-Authored-By: Bryce Anglin --- chart/pelican-wings/templates/configmap.yaml | 6 ++ chart/pelican-wings/values.yaml | 8 ++ config/config_kubernetes.go | 23 +++++ environment/kubernetes/network.go | 32 ++++++- environment/kubernetes/network_test.go | 88 ++++++++++++++++++++ 5 files changed, 155 insertions(+), 2 deletions(-) diff --git a/chart/pelican-wings/templates/configmap.yaml b/chart/pelican-wings/templates/configmap.yaml index 565bdd52..0128d4fc 100644 --- a/chart/pelican-wings/templates/configmap.yaml +++ b/chart/pelican-wings/templates/configmap.yaml @@ -32,6 +32,12 @@ data: lb_annotations: {{- toYaml .Values.wings.kubernetes.lbAnnotations | nindent 8 }} {{- end }} + {{- if .Values.wings.kubernetes.lbIPAnnotation }} + lb_ip_annotation: {{ .Values.wings.kubernetes.lbIPAnnotation | quote }} + {{- end }} + {{- if .Values.wings.kubernetes.lbSharingKey }} + lb_sharing_key: {{ .Values.wings.kubernetes.lbSharingKey | quote }} + {{- end }} storage_mode: {{ .Values.wings.kubernetes.storageMode | quote }} {{- if .Values.wings.kubernetes.storageClass }} storage_class: {{ .Values.wings.kubernetes.storageClass | quote }} diff --git a/chart/pelican-wings/values.yaml b/chart/pelican-wings/values.yaml index 1178a72d..f211fa65 100644 --- a/chart/pelican-wings/values.yaml +++ b/chart/pelican-wings/values.yaml @@ -85,6 +85,14 @@ wings: # lbAnnotations: # io.cilium/lb-ipam-pool: "game-servers" lbAnnotations: {} + # -- Annotation key Wings sets to the server's allocation IP on each LB Service. + # This pins the LB to the IP selected in the Panel so multiple servers share one IP. + # Cilium: "lbipam.cilium.io/ips" | MetalLB: "metallb.universe.tf/loadBalancerIPs" + lbIPAnnotation: "" + # -- Annotation key for LB IP sharing. Wings sets the value to the allocation IP, + # grouping all servers on the same IP under one sharing key. + # Cilium: "lbipam.cilium.io/sharing-key" | MetalLB: "metallb.universe.tf/allow-shared-ip" + lbSharingKey: "" # -- Storage mode: "hostpath" or "pvc" storageMode: pvc # -- StorageClass for PVCs (empty = cluster default) diff --git a/config/config_kubernetes.go b/config/config_kubernetes.go index e7423ae6..319ef4a5 100644 --- a/config/config_kubernetes.go +++ b/config/config_kubernetes.go @@ -99,6 +99,29 @@ type KubernetesConfiguration struct { // metallb.universe.tf/address-pool: "game-servers" LBAnnotations map[string]string `json:"lb_annotations" yaml:"lb_annotations"` + // LBIPAnnotation is the annotation key that Wings sets to the server's + // allocation IP on each LoadBalancer Service. This pins the LB to the IP + // the user selected in the Panel, enabling multiple game servers to share + // the same external IP (on different ports). + // + // Common values: + // Cilium: "lbipam.cilium.io/ips" + // MetalLB: "metallb.universe.tf/loadBalancerIPs" + // + // When empty (default), no IP-pinning annotation is added. + LBIPAnnotation string `default:"" json:"lb_ip_annotation" yaml:"lb_ip_annotation"` + + // LBSharingKey is the annotation key used to allow multiple Services to + // share the same external IP. Wings sets this annotation to the allocation + // IP value, grouping all game servers on the same IP under one sharing key. + // + // Common values: + // Cilium: "lbipam.cilium.io/sharing-key" + // MetalLB: "metallb.universe.tf/allow-shared-ip" + // + // When empty (default), no sharing-key annotation is added. + LBSharingKey string `default:"" json:"lb_sharing_key" yaml:"lb_sharing_key"` + // Tolerations allow game server Pods to be scheduled on tainted nodes. Tolerations []KubeToleration `json:"tolerations" yaml:"tolerations"` diff --git a/environment/kubernetes/network.go b/environment/kubernetes/network.go index eb9e3cf8..b7444890 100644 --- a/environment/kubernetes/network.go +++ b/environment/kubernetes/network.go @@ -91,8 +91,21 @@ func (e *Environment) EnsureService(ctx context.Context) error { var annotations map[string]string if isLB { svcType = corev1.ServiceTypeLoadBalancer - if len(cfg.Kubernetes.LBAnnotations) > 0 { - annotations = cfg.Kubernetes.LBAnnotations + annotations = make(map[string]string) + for k, v := range cfg.Kubernetes.LBAnnotations { + annotations[k] = v + } + + // Auto-set IP-pinning and sharing-key annotations from the + // server's allocation IP so the LB is bound to the IP the user + // selected in the Panel. + if allocIP := e.allocationIP(); allocIP != "" { + if cfg.Kubernetes.LBIPAnnotation != "" { + annotations[cfg.Kubernetes.LBIPAnnotation] = allocIP + } + if cfg.Kubernetes.LBSharingKey != "" { + annotations[cfg.Kubernetes.LBSharingKey] = allocIP + } } } @@ -342,3 +355,18 @@ func sanitizePortName(name string) string { func portName(proto string, port int) string { return sanitizePortName(fmt.Sprintf("%s-%d", proto, port)) } + +// allocationIP returns the server's default allocation IP if it is a usable +// public/external address. Returns empty for 0.0.0.0, 127.0.0.1, or when +// no default allocation is configured. +func (e *Environment) allocationIP() string { + allocs := e.Configuration.Allocations() + if allocs.DefaultMapping == nil { + return "" + } + ip := allocs.DefaultMapping.Ip + if ip == "" || ip == "0.0.0.0" || ip == "127.0.0.1" { + return "" + } + return ip +} diff --git a/environment/kubernetes/network_test.go b/environment/kubernetes/network_test.go index 1781940e..5bbf5af9 100644 --- a/environment/kubernetes/network_test.go +++ b/environment/kubernetes/network_test.go @@ -147,6 +147,94 @@ func TestNetwork(t *testing.T) { svcs, _ := client.CoreV1().Services("pelican").List(context.Background(), metav1.ListOptions{}) g.Assert(len(svcs.Items)).Equal(0) }) + + g.It("should auto-set LB IP and sharing-key annotations from allocation IP", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + g.Assert(svc.Spec.Type).Equal(corev1.ServiceTypeLoadBalancer) + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + g.Assert(svc.Annotations["lbipam.cilium.io/ips"]).Equal("23.227.184.222") + g.Assert(svc.Annotations["lbipam.cilium.io/sharing-key"]).Equal("23.227.184.222") + }) + + g.It("should not set IP annotations when allocation IP is 0.0.0.0", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) + + g.It("should not set IP annotations when config keys are empty", func() { + client := fake.NewSimpleClientset() + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "23.227.184.222", + Port: 27015, + }, + Mappings: map[string][]int{"23.227.184.222": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBIPAnnotation = "" + c.Kubernetes.LBSharingKey = "" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + }) }) g.Describe("DeleteService", func() { From 7457b9d972a9f231a318701ce30feede39331008 Mon Sep 17 00:00:00 2001 From: Exonical Date: Fri, 29 May 2026 04:48:43 +0000 Subject: [PATCH 2/2] fix: remove stale LB IP-pinning annotations on Service update When allocation IP becomes invalid (0.0.0.0, empty), the update path now replaces annotations entirely instead of merging, ensuring stale IP-pinning and sharing-key annotations are cleaned up. Co-Authored-By: Bryce Anglin --- environment/kubernetes/network.go | 9 +---- environment/kubernetes/network_test.go | 54 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/environment/kubernetes/network.go b/environment/kubernetes/network.go index b7444890..e0459a97 100644 --- a/environment/kubernetes/network.go +++ b/environment/kubernetes/network.go @@ -145,13 +145,8 @@ func (e *Environment) EnsureService(ctx context.Context) error { existing.Labels = labels e.log().WithField("service", svcName).Infof("updating %s service for server", svcType) - if isLB && len(annotations) > 0 { - if existing.Annotations == nil { - existing.Annotations = make(map[string]string) - } - for k, v := range annotations { - existing.Annotations[k] = v - } + if isLB { + existing.Annotations = annotations } _, err = e.client.CoreV1().Services(ns).Update(ctx, existing, metav1.UpdateOptions{}) if err != nil { diff --git a/environment/kubernetes/network_test.go b/environment/kubernetes/network_test.go index 5bbf5af9..7df5d379 100644 --- a/environment/kubernetes/network_test.go +++ b/environment/kubernetes/network_test.go @@ -235,6 +235,60 @@ func TestNetwork(t *testing.T) { _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] g.Assert(hasIP).IsFalse() }) + + g.It("should remove stale IP annotations when allocation IP becomes invalid", func() { + existingSvc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gs-test-server-uuid", + Namespace: "pelican", + Annotations: map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + "lbipam.cilium.io/ips": "23.227.184.222", + "lbipam.cilium.io/sharing-key": "23.227.184.222", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + {Name: "tcp-27015", Protocol: corev1.ProtocolTCP, Port: 27015}, + {Name: "udp-27015", Protocol: corev1.ProtocolUDP, Port: 27015}, + }, + Selector: map[string]string{"pelican.dev/server-id": "test-server-uuid"}, + }, + } + client := fake.NewSimpleClientset(existingSvc) + allocs := environment.Allocations{ + DefaultMapping: &environment.DefaultAllocationMapping{ + Ip: "0.0.0.0", + Port: 27015, + }, + Mappings: map[string][]int{"0.0.0.0": {27015}}, + } + env := newTestEnv(client, allocs) + + config.Update(func(c *config.Configuration) { + c.Kubernetes.NetworkMode = config.KubeNetworkLoadBalancer + c.Kubernetes.Namespace = "pelican" + c.Kubernetes.LBAnnotations = map[string]string{ + "io.cilium/lb-ipam-pool": "game-servers", + } + c.Kubernetes.LBIPAnnotation = "lbipam.cilium.io/ips" + c.Kubernetes.LBSharingKey = "lbipam.cilium.io/sharing-key" + }) + + err := env.EnsureService(context.Background()) + g.Assert(err).IsNil() + + svc, err := client.CoreV1().Services("pelican").Get(context.Background(), "gs-test-server-uuid", metav1.GetOptions{}) + g.Assert(err).IsNil() + // Pool annotation should remain. + g.Assert(svc.Annotations["io.cilium/lb-ipam-pool"]).Equal("game-servers") + // IP-pinning annotations should be removed. + _, hasIP := svc.Annotations["lbipam.cilium.io/ips"] + g.Assert(hasIP).IsFalse() + _, hasKey := svc.Annotations["lbipam.cilium.io/sharing-key"] + g.Assert(hasKey).IsFalse() + }) }) g.Describe("DeleteService", func() {