From 1be9213c38058c816623c944547bed7d6274725b Mon Sep 17 00:00:00 2001 From: as673366 Date: Mon, 22 Dec 2025 21:49:20 +0530 Subject: [PATCH 1/8] adding changes to support dual gateway requirements in otk --- api/v1/gateway_types.go | 17 + .../bases/security.brcmlabs.com_gateways.yaml | 32 +- example/gateway/otk/otk-ssg-dmz.yaml | 204 +++++ example/gateway/otk/otk-ssg-internal.yaml | 201 +++++ internal/controller/gateway/controller.go | 5 + pkg/api/reconcile/l7api.go | 4 +- pkg/gateway/reconcile/cron.go | 39 +- pkg/gateway/reconcile/externalkeys.go | 765 ++++++++++++++++++ pkg/gateway/reconcile/gateway.go | 13 +- pkg/gateway/reconcile/l7otkcertificates.go | 117 --- 10 files changed, 1262 insertions(+), 135 deletions(-) create mode 100644 example/gateway/otk/otk-ssg-dmz.yaml create mode 100644 example/gateway/otk/otk-ssg-internal.yaml diff --git a/api/v1/gateway_types.go b/api/v1/gateway_types.go index 41fcad93..646d9929 100644 --- a/api/v1/gateway_types.go +++ b/api/v1/gateway_types.go @@ -360,17 +360,32 @@ type Otk struct { // This configures a relationship between DMZ and Internal Gateways. InternalOtkGatewayReference string `json:"internalGatewayReference,omitempty"` // InternalGatewayPort defaults to 9443 or graphmanDynamicSync port + // This port is used when the Internal gateway is external (not managed by operator) InternalGatewayPort int `json:"internalGatewayPort,omitempty"` // OTKPort is used in Single mode - sets the otk.port cluster-wide property and in Dual-Mode // sets host_oauth2_auth_server port in #OTK Client Context Variables // TODO: Make this an array for many dmz deployments to one internal DmzOtkGatewayReference string `json:"dmzGatewayReference,omitempty"` + // DmzGatewayPort defaults to 9443 or graphmanDynamicSync port + // This port is used when the DMZ gateway is external (not managed by operator) + DmzGatewayPort int `json:"dmzGatewayPort,omitempty"` // OTKPort defaults to 8443 OTKPort int `json:"port,omitempty"` // MaintenanceTasks for the OTK database are disabled by default MaintenanceTasks OtkMaintenanceTasks `json:"maintenanceTasks,omitempty"` // RuntimeSyncIntervalSeconds how often OTK Gateways should be updated in internal/dmz mode RuntimeSyncIntervalSeconds int `json:"runtimeSyncIntervalSeconds,omitempty"` + // SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + // Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + SyncIntervalSeconds int `json:"syncIntervalSeconds,omitempty"` + // DmzKeySecret is a reference to a kubernetes.io/tls Secret containing the DMZ private key and certificate + DmzKeySecret string `json:"dmzKeySecret,omitempty"` + // InternalKeySecret is a reference to a kubernetes.io/tls Secret containing the Internal private key and certificate + InternalKeySecret string `json:"internalKeySecret,omitempty"` + // DmzAuthSecret is a reference to a Secret containing username and password for DMZ authentication + DmzAuthSecret string `json:"dmzAuthSecret,omitempty"` + // InternalAuthSecret is a reference to a Secret containing username and password for Internal authentication + InternalAuthSecret string `json:"internalAuthSecret,omitempty"` } // OtkMaintenanceTasks are included in the install bundle as disabled scheduled tasks @@ -893,6 +908,8 @@ type ExternalKey struct { // only one key usage type is allowed // SSL | CA | AUDIT_SIGNING | AUDIT_VIEWER KeyUsageType KeyUsageType `json:"keyUsageType,omitempty"` + // Otk indicates that this key usage was specific for OTK + Otk bool `json:"otk,omitempty"` } type KeyUsageType string diff --git a/config/crd/bases/security.brcmlabs.com_gateways.yaml b/config/crd/bases/security.brcmlabs.com_gateways.yaml index a473a499..0ebb6746 100644 --- a/config/crd/bases/security.brcmlabs.com_gateways.yaml +++ b/config/crd/bases/security.brcmlabs.com_gateways.yaml @@ -1589,6 +1589,10 @@ spec: description: Name of the kubernetes.io/tls Secret which already exists in Kubernetes type: string + otk: + description: Otk indicates that this key usage was specific + for OTK + type: boolean type: object type: array externalSecrets: @@ -4003,9 +4007,21 @@ spec: description: Type of OTK Database type: string type: object + dmzAuthSecret: + description: DmzAuthSecret is a reference to a Secret containing + username and password... + type: string + dmzGatewayPort: + description: |- + DmzGatewayPort defaults to 9443 or graphmanDynamicSync port + This port is... + type: integer dmzGatewayReference: description: OTKPort is used in Single mode - sets the otk. type: string + dmzKeySecret: + description: DmzKeySecret is a reference to a kubernetes. + type: string enabled: description: Enable or disable the OTK initContainer type: boolean @@ -4142,14 +4158,22 @@ spec: type: string type: object type: object + internalAuthSecret: + description: InternalAuthSecret is a reference to a Secret + containing username and... + type: string internalGatewayPort: - description: InternalGatewayPort defaults to 9443 or graphmanDynamicSync - port + description: |- + InternalGatewayPort defaults to 9443 or graphmanDynamicSync port + This port... type: integer internalGatewayReference: description: InternalOtkGatewayReference to an Operator managed Gateway deployment that... type: string + internalKeySecret: + description: InternalKeySecret is a reference to a kubernetes. + type: string maintenanceTasks: description: MaintenanceTasks for the OTK database are disabled by default @@ -4202,6 +4226,10 @@ spec: items: type: string type: array + syncIntervalSeconds: + description: SyncIntervalSeconds determines how often DMZ + and Internal gateways should... + type: integer type: description: Type of OTK installation single, internal or dmz diff --git a/example/gateway/otk/otk-ssg-dmz.yaml b/example/gateway/otk/otk-ssg-dmz.yaml new file mode 100644 index 00000000..9bae180e --- /dev/null +++ b/example/gateway/otk/otk-ssg-dmz.yaml @@ -0,0 +1,204 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-dmz +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + updateStrategy: + type: rollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 2 + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + externalSecrets: + - name: gateway-secret + enabled: true + variableReferencable: true + description: Gateway Secret + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-dmz-tls-secret + enabled: true + alias: otk-dmz-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: dmz + internalAuthSecret: otk-internal-auth-secret + internalGatewayReference: as673366-gw-upgrade-0.apim.labs.broadcom.net + # InternalGatewayPort is used when the Internal gateway is external (not managed by operator) + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + internalGatewayPort: 8443 + # SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for DMZ key (used by OTK reconciliation) + dmzKeySecret: otk-dmz-tls-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + # A single secret containing all of the values defined here will be created + # if existingSecret is set the corresponding gateway, readOnly or admin will be omitted from the secret + # if no values are set, a secret will not be created or referenced and the deployment will be invalidated. + # existingSecret: otk-db-secret + gateway: + username: otk_user + password: otkUserPass + readOnly: + # username: readonly_user + username: readonly_user + password: readonly_userPass + admin: + # username: admin + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + #jdbcUrl: jdbc:mysql://:/ + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + autoscaling: + enabled: false + bundle: [] + repositoryReferences: [] + bootstrap: + script: + enabled: true + initContainers: [] + hazelcast: + external: false + endpoint: hazelcast.example.com:5701 + management: + secretName: gateway-secret + #username: admin + #password: 7layer + # Management port requires a separate service... + service: + enabled: true + #annotations: + # cloud.google.com/load-balancer-type: "Internal" + type: LoadBalancer + ports: + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + restman: + enabled: false + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + dynamicSyncPort: 9443 + cluster: + #password: 7layer + hostname: gateway.brcmlabs.com + database: + enabled: false # this runs the gateway in dbbacked/ephemeral mode + # jdbcUrl: "jdbc:mysql://cluster1-haproxy.pxc.svc.cluster.local:3306/ssg" + # username: "gateway" + # password: "ACm8BDr3Rfk2Flx9V" + java: + jvmHeap: + calculate: true + percentage: 50 + default: 4g + extraArgs: + - -Dcom.l7tech.server.audit.message.saveToInternal=false + - -Dcom.l7tech.server.audit.admin.saveToInternal=false + - -Dcom.l7tech.server.audit.system.saveToInternal=false + - -Dcom.l7tech.server.audit.log.format=json + - -Djava.util.logging.config.file=/opt/SecureSpan/Gateway/node/default/etc/conf/log-override.properties + - -Dcom.l7tech.security.ssl.hostAllowWildcard=true + - -Dcom.l7tech.server.pkix.useDefaultTrustAnchors=true + #- -Dcom.l7tech.bootstrap.autoTrustSslKey=trustAnchor,TrustedFor.SSL,TrustedFor.SAML_ISSUER + listenPorts: + harden: true + custom: + enabled: false + ports: [] + cwp: + enabled: true + properties: + - name: io.httpsHostAllowWildcard + value: "true" + - name: log.levels + value: | + com.l7tech.level = CONFIG + com.l7tech.server.policy.variable.ServerVariables.level = SEVERE + com.l7tech.external.assertions.odata.server.producer.jdbc.GenerateSqlQuery.level = SEVERE + com.l7tech.server.policy.assertion.ServerSetVariableAssertion.level = SEVERE + com.l7tech.external.assertions.comparison.server.ServerComparisonAssertion.level = SEVERE + - name: audit.setDetailLevel.FINE + value: 152 7101 7103 9648 9645 7026 7027 4155 150 4716 4114 6306 4100 9655 150 151 11000 4104 + system: + properties: |- + # Default Gateway system properties + # Configuration properties for shared state extensions. + com.l7tech.server.extension.sharedKeyValueStoreProvider=embeddedhazelcast + com.l7tech.server.extension.sharedCounterProvider=ssgdb + com.l7tech.server.extension.sharedClusterInfoProvider=ssgdb + # By default, FIPS module will block an RSA modulus from being used for encryption if it has been used for + # signing, or visa-versa. Set true to disable this default behaviour and remain backwards compatible. + com.l7tech.org.bouncycastle.rsa.allow_multi_use=true + # Specifies the type of Trust Store (JKS/PKCS12) provided by AdoptOpenJDK that is used by Gateway. + # Must be set correctly when Gateway is running in FIPS mode. If not specified it will default to PKCS12. + javax.net.ssl.trustStoreType=jks + com.l7tech.server.clusterStaleNodeCleanupTimeoutSeconds=86400 + # Additional properties go here + service: + # annotations: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + ingress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/ssl-passthrough: "true" + tls: + - hosts: + - otk-ssg-dmz.brcmlabs.com + secretName: brcmlabs + rules: + - host: otk-ssg-dmz.brcmlabs.com + # containerSecurityContext: + # runAsNonRoot: true + # runAsUser: 1000669998 + # capabilities: + # drop: + # - ALL + # allowPrivilegeEscalation: false + # podSecurityContext: + # runAsUser: 1000669998 + # runAsGroup: 1000669998 + + diff --git a/example/gateway/otk/otk-ssg-internal.yaml b/example/gateway/otk/otk-ssg-internal.yaml new file mode 100644 index 00000000..786d60d4 --- /dev/null +++ b/example/gateway/otk/otk-ssg-internal.yaml @@ -0,0 +1,201 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-internal +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + updateStrategy: + type: rollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 2 + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-internal-tls-secret + enabled: true + alias: otk-internal-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: internal + dmzGatewayReference: otk-ssg-dmz + # DmzGatewayPort is used when the DMZ gateway is external (not managed by operator) + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + dmzGatewayPort: 9443 + # SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for Internal key (used by OTK reconciliation) + internalKeySecret: otk-internal-tls-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + # A single secret containing all of the values defined here will be created + # if existingSecret is set the corresponding gateway, readOnly or admin will be omitted from the secret + # if no values are set, a secret will not be created or referenced and the deployment will be invalidated. + # existingSecret: otk-db-secret + gateway: + username: otk_user + password: otkUserPass + readOnly: + # username: readonly_user + username: readonly_user + password: readonly_userPass + admin: + # username: admin + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + #jdbcUrl: jdbc:mysql://:/ + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + autoscaling: + enabled: false + bundle: [] + repositoryReferences: [] + bootstrap: + script: + enabled: true + initContainers: [] + hazelcast: + external: false + endpoint: hazelcast.example.com:5701 + management: + secretName: gateway-secret + #username: admin + #password: 7layer + # Management port requires a separate service... + service: + enabled: false + #annotations: + # cloud.google.com/load-balancer-type: "Internal" + type: LoadBalancer + ports: + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + restman: + enabled: false + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + cluster: + #password: 7layer + hostname: gateway.brcmlabs.com + database: + enabled: false # this runs the gateway in dbbacked/ephemeral mode + # jdbcUrl: "jdbc:mysql://cluster1-haproxy.pxc.svc.cluster.local:3306/ssg" + # username: "gateway" + # password: "ACm8BDr3Rfk2Flx9V" + java: + jvmHeap: + calculate: true + percentage: 50 + default: 4g + extraArgs: + - -Dcom.l7tech.server.audit.message.saveToInternal=false + - -Dcom.l7tech.server.audit.admin.saveToInternal=false + - -Dcom.l7tech.server.audit.system.saveToInternal=false + - -Dcom.l7tech.server.audit.log.format=json + - -Djava.util.logging.config.file=/opt/SecureSpan/Gateway/node/default/etc/conf/log-override.properties + - -Dcom.l7tech.security.ssl.hostAllowWildcard=true + - -Dcom.l7tech.server.pkix.useDefaultTrustAnchors=true + #- -Dcom.l7tech.bootstrap.autoTrustSslKey=trustAnchor,TrustedFor.SSL,TrustedFor.SAML_ISSUER + listenPorts: + harden: true + custom: + enabled: false + ports: [] + cwp: + enabled: true + properties: + - name: io.httpsHostAllowWildcard + value: "true" + - name: log.levels + value: | + com.l7tech.level = CONFIG + com.l7tech.server.policy.variable.ServerVariables.level = SEVERE + com.l7tech.external.assertions.odata.server.producer.jdbc.GenerateSqlQuery.level = SEVERE + com.l7tech.server.policy.assertion.ServerSetVariableAssertion.level = SEVERE + com.l7tech.external.assertions.comparison.server.ServerComparisonAssertion.level = SEVERE + - name: audit.setDetailLevel.FINE + value: 152 7101 7103 9648 9645 7026 7027 4155 150 4716 4114 6306 4100 9655 150 151 11000 4104 + system: + properties: |- + # Default Gateway system properties + # Configuration properties for shared state extensions. + com.l7tech.server.extension.sharedKeyValueStoreProvider=embeddedhazelcast + com.l7tech.server.extension.sharedCounterProvider=ssgdb + com.l7tech.server.extension.sharedClusterInfoProvider=ssgdb + # By default, FIPS module will block an RSA modulus from being used for encryption if it has been used for + # signing, or visa-versa. Set true to disable this default behaviour and remain backwards compatible. + com.l7tech.org.bouncycastle.rsa.allow_multi_use=true + # Specifies the type of Trust Store (JKS/PKCS12) provided by AdoptOpenJDK that is used by Gateway. + # Must be set correctly when Gateway is running in FIPS mode. If not specified it will default to PKCS12. + javax.net.ssl.trustStoreType=jks + com.l7tech.server.clusterStaleNodeCleanupTimeoutSeconds=86400 + # Additional properties go here + service: + # annotations: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + ingress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/ssl-passthrough: "true" + tls: + - hosts: + - otk-ssg-internal.brcmlabs.com + secretName: brcmlabs + rules: + - host: otk-ssg-internal.brcmlabs.com + # containerSecurityContext: + # runAsNonRoot: true + # runAsUser: 1000669998 + # capabilities: + # drop: + # - ALL + # allowPrivilegeEscalation: false + # podSecurityContext: + # runAsUser: 1000669998 + # runAsGroup: 1000669998 + + diff --git a/internal/controller/gateway/controller.go b/internal/controller/gateway/controller.go index 1ee34b0b..af83bb95 100644 --- a/internal/controller/gateway/controller.go +++ b/internal/controller/gateway/controller.go @@ -236,6 +236,11 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { req = append(req, creconcile.Request{NamespacedName: types.NamespacedName{Namespace: gateway.Namespace, Name: gateway.Name}}) } } + if gateway.Spec.App.Otk.Enabled { + if gateway.Spec.App.Otk.DmzKeySecret == a.GetName() || gateway.Spec.App.Otk.InternalKeySecret == a.GetName() { + req = append(req, creconcile.Request{NamespacedName: types.NamespacedName{Namespace: gateway.Namespace, Name: gateway.Name}}) + } + } } return req }), diff --git a/pkg/api/reconcile/l7api.go b/pkg/api/reconcile/l7api.go index e5a4d12b..c516b8e1 100644 --- a/pkg/api/reconcile/l7api.go +++ b/pkg/api/reconcile/l7api.go @@ -188,7 +188,7 @@ func deployL7ApiToGateway(ctx context.Context, params Params, gateway *v1.Gatewa } if tryRequest { - endpoint := pod.Status.PodIP + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" var errorMessage string status := SUCCESS name := gateway.Name @@ -264,7 +264,7 @@ func undeployL7ApiToGateway(ctx context.Context, params Params, gateway *v1.Gate } if tryRequest { - endpoint := pod.Status.PodIP + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" status := SUCCESS name := gateway.Name if gateway.Spec.App.Management.SecretName != "" { diff --git a/pkg/gateway/reconcile/cron.go b/pkg/gateway/reconcile/cron.go index 979c54f7..2de5da1c 100644 --- a/pkg/gateway/reconcile/cron.go +++ b/pkg/gateway/reconcile/cron.go @@ -72,16 +72,37 @@ func registerJobs(ctx context.Context, params Params) { if err != nil { params.Log.V(2).Info("otk policy sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) } - if params.Instance.Spec.App.Otk.Type == securityv1.OtkTypeDMZ || params.Instance.Spec.App.Otk.Type == securityv1.OtkTypeInternal { - _, err = s.Every(otkSyncInterval).Seconds().Tag(params.Instance.Name+"-"+params.Instance.Namespace+"-sync-otk-certificates").Do(syncOtkCertificates, ctx, params) - if err != nil { - params.Log.V(2).Info("otk certificate sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - } - _, err = s.Every(otkSyncInterval).Seconds().Tag(params.Instance.Name+"-"+params.Instance.Namespace+"-sync-otk-certificate-secret").Do(manageCertificateSecrets, ctx, params) - if err != nil { - params.Log.V(2).Info("otk certificate secret sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - } + + // Register certificate sync job for DMZ and Internal gateways + // Use SyncIntervalSeconds if specified, otherwise fall back to RuntimeSyncIntervalSeconds or default + certSyncInterval := otkSyncInterval + if params.Instance.Spec.App.Otk.SyncIntervalSeconds != 0 { + certSyncInterval = params.Instance.Spec.App.Otk.SyncIntervalSeconds + } + + _, err = s.Every(certSyncInterval).Seconds().Tag(params.Instance.Name+"-sync-otk-certificates").Do(syncOtkCertificates, ctx, params) + + if err != nil { + params.Log.V(2).Info("otk certificate sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) } + + // Register external keys sync job for certificate publishing between DMZ and Internal + _, err = s.Every(certSyncInterval).Seconds().Tag(params.Instance.Name+"-sync-otk-external-keys").Do(syncOtkExternalKeys, ctx, params) + + if err != nil { + params.Log.V(2).Info("otk external keys sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) + } + } +} + +func syncOtkExternalKeys(ctx context.Context, params Params) { + // Sync certificates between DMZ and Internal gateways via ExternalKeys + // This handles OTK certificate publishing (publishDmzCertToInternal, publishInternalCertToDmz) + err := ExternalKeys(ctx, params) + if err != nil { + params.Log.Error(err, "failed to sync OTK external keys certificates", "name", params.Instance.Name, "namespace", params.Instance.Namespace) + } else { + params.Log.V(2).Info("OTK external keys certificates synced", "name", params.Instance.Name, "namespace", params.Instance.Namespace, "interval", params.Instance.Spec.App.Otk.SyncIntervalSeconds) } } diff --git a/pkg/gateway/reconcile/externalkeys.go b/pkg/gateway/reconcile/externalkeys.go index de07af50..ddacf3ef 100644 --- a/pkg/gateway/reconcile/externalkeys.go +++ b/pkg/gateway/reconcile/externalkeys.go @@ -28,10 +28,37 @@ package reconcile import ( "context" + "crypto/sha1" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "strings" + + securityv1 "github.com/caapim/layer7-operator/api/v1" + "github.com/caapim/layer7-operator/internal/graphman" + "github.com/caapim/layer7-operator/pkg/util" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) func ExternalKeys(ctx context.Context, params Params) error { gateway := params.Instance + + // Handle OTK keys if OTK is enabled + if gateway.Spec.App.Otk.Enabled { + err := handleOtkKeys(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle OTK keys", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } + + // Handle regular external keys if len(gateway.Spec.App.ExternalKeys) == 0 && len(gateway.Status.LastAppliedExternalKeys) == 0 { return nil } @@ -66,3 +93,741 @@ func ExternalKeys(ctx context.Context, params Params) error { return nil } + +func handleOtkKeys(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Handle DMZ key updates only if there's an externalKey with otk: true referencing the same secret + if gateway.Spec.App.Otk.DmzKeySecret != "" && gateway.Spec.App.Otk.Type == securityv1.OtkTypeDMZ { + // Check if there's an externalKey with otk: true that references this secret + hasOtkExternalKey := false + for _, ek := range gateway.Spec.App.ExternalKeys { + if ek.Enabled && ek.Otk && ek.Name == gateway.Spec.App.Otk.DmzKeySecret { + hasOtkExternalKey = true + break + } + } + + // Only process if there's an externalKey with otk: true + if hasOtkExternalKey { + err := handleDmzKeyUpdate(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle DMZ key update", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } else { + params.Log.V(2).Info("Skipping DMZ key update - no externalKey with otk: true found", "secret", gateway.Spec.App.Otk.DmzKeySecret) + } + } + + // Handle Internal key updates only if there's an externalKey with otk: true referencing the same secret + if gateway.Spec.App.Otk.InternalKeySecret != "" && gateway.Spec.App.Otk.Type == securityv1.OtkTypeInternal { + // Check if there's an externalKey with otk: true that references this secret + hasOtkExternalKey := false + for _, ek := range gateway.Spec.App.ExternalKeys { + if ek.Enabled && ek.Otk && ek.Name == gateway.Spec.App.Otk.InternalKeySecret { + hasOtkExternalKey = true + break + } + } + + // Only process if there's an externalKey with otk: true + if hasOtkExternalKey { + err := handleInternalKeyUpdate(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle Internal key update", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } else { + params.Log.V(2).Info("Skipping Internal key update - no externalKey with otk: true found", "secret", gateway.Spec.App.Otk.InternalKeySecret) + } + } + + return nil +} + +func handleDmzKeyUpdate(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Get DMZ key secret + dmzKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("DMZ key secret not found, skipping", "secret", gateway.Spec.App.Otk.DmzKeySecret) + return nil + } + return err + } + + // Check if operator managed (ephemeral mode) + isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled + + if isOperatorManaged { + // Update DMZ with the new key + err = updateDmzWithKey(ctx, params, gateway, dmzKeySecret) + if err != nil { + return fmt.Errorf("failed to update DMZ with key: %w", err) + } + } + + // Publish DMZ cert to Internal for Truststore & FIP User + if gateway.Spec.App.Otk.InternalOtkGatewayReference != "" { + err = publishDmzCertToInternal(ctx, params, gateway, dmzKeySecret) + if err != nil { + return fmt.Errorf("failed to publish DMZ cert to Internal: %w", err) + } + } + + return nil +} + +func handleInternalKeyUpdate(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Get Internal key secret + internalKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("Internal key secret not found, skipping", "secret", gateway.Spec.App.Otk.InternalKeySecret) + return nil + } + return err + } + + // Check if operator managed (ephemeral mode) + isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled + + if isOperatorManaged { + // Update Internal with the new key + err = updateInternalWithKey(ctx, params, gateway, internalKeySecret) + if err != nil { + return fmt.Errorf("failed to update Internal with key: %w", err) + } + } + + // Publish Internal cert to DMZ for Truststore + if gateway.Spec.App.Otk.DmzOtkGatewayReference != "" { + err = publishInternalCertToDmz(ctx, params, gateway, internalKeySecret) + if err != nil { + return fmt.Errorf("failed to publish Internal cert to DMZ: %w", err) + } + } + + return nil +} + +func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Gateway, keySecret *corev1.Secret) error { + if keySecret.Type != corev1.SecretTypeTLS { + return fmt.Errorf("DMZ key secret must be of type kubernetes.io/tls") + } + + certData := keySecret.Data["tls.crt"] + keyData := keySecret.Data["tls.key"] + + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt and tls.key") + } + + // Extract certificate from chain + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format in DMZ key secret") + } + + // Use first certificate in chain + firstCert := crtStrings[0] + b, _ := pem.Decode([]byte(firstCert)) + if b == nil { + return fmt.Errorf("failed to decode certificate") + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + + // Create Graphman key bundle + keySecretMap := []util.GraphmanKey{ + { + Name: crtX509.Subject.CommonName, + Crt: string(certData), + Key: string(keyData), + Alias: "otk-dmz-key", + UsageType: "SSL", + }, + } + + bundleBytes, err := util.ConvertX509ToGraphmanBundle(keySecretMap, []string{}) + if err != nil { + return fmt.Errorf("failed to convert key to bundle: %w", err) + } + + // Calculate checksum + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Get gateway secret for authentication + name := gateway.Name + if gateway.Spec.App.Management.SecretName != "" { + name = gateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/otk-dmz-key" + + if !gateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + err = ReconcileEphemeralGateway(ctx, params, "otk dmz key", *podList, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, params, "otk dmz key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) + if err != nil { + return err + } + } + + // Update cluster property otk.dmz.private_key.name after DMZ key is updated + //if err := updateDmzPrivateKeyClusterProperty(ctx, params, gateway, "otk-dmz-key"); err != nil { + // params.Log.V(2).Info("Failed to update DMZ private key cluster property", "error", err, "gateway", gateway.Name) + // // Don't fail the entire operation if cluster property update fails + //} + + return nil +} + +func updateInternalWithKey(ctx context.Context, params Params, gateway *securityv1.Gateway, keySecret *corev1.Secret) error { + if keySecret.Type != corev1.SecretTypeTLS { + return fmt.Errorf("Internal key secret must be of type kubernetes.io/tls") + } + + certData := keySecret.Data["tls.crt"] + keyData := keySecret.Data["tls.key"] + + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt and tls.key") + } + + // Extract certificate from chain + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format in Internal key secret") + } + + // Use first certificate in chain + firstCert := crtStrings[0] + b, _ := pem.Decode([]byte(firstCert)) + if b == nil { + return fmt.Errorf("failed to decode certificate") + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + + // Create Graphman key bundle + keySecretMap := []util.GraphmanKey{ + { + Name: crtX509.Subject.CommonName, + Crt: string(certData), + Key: string(keyData), + Alias: "otk-internal-key", + UsageType: "SSL", + }, + } + + bundleBytes, err := util.ConvertX509ToGraphmanBundle(keySecretMap, []string{}) + if err != nil { + return fmt.Errorf("failed to convert key to bundle: %w", err) + } + + // Calculate checksum + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Get gateway secret for authentication + name := gateway.Name + if gateway.Spec.App.Management.SecretName != "" { + name = gateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/otk-internal-key" + + if !gateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + err = ReconcileEphemeralGateway(ctx, params, "otk internal key", *podList, gateway, gwSecret, "", annotation, sha1Sum, false, "otk internal key", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, params, "otk internal key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk internal key", bundleBytes) + if err != nil { + return err + } + } + + return nil +} + +func publishDmzCertToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret) error { + // Get Internal gateway + internalGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.InternalOtkGatewayReference, + Namespace: gateway.Namespace, + }, internalGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.InternalGatewayPort != 0 { + params.Log.V(2).Info("Internal gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, + "port", gateway.Spec.App.Otk.InternalGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("Internal gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference) + return nil + } + } else { + return err + } + } + + certData := dmzKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt") + } + + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") + } + + bundle := graphman.Bundle{} + + // Add to TrustedCerts + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + + // Add to FIP Users + bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ + Name: crtX509.Subject.CommonName, + ProviderName: "otk-fips-provider", + SubjectDn: "cn=" + crtX509.Subject.CommonName, + CertBase64: base64.RawStdEncoding.EncodeToString(crtX509.Raw), + }) + } + + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return err + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port + if isExternalGateway { + // For external gateways, we can't use the standard reconciliation + // Log that external gateway support requires additional configuration + params.Log.V(2).Info("External Internal gateway detected, port will be used for connection", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, + "port", gateway.Spec.App.Otk.InternalGatewayPort) + // Note: External gateway connection would require additional implementation + // For now, we skip reconciliation for external gateways + return nil + } + + // Get Internal gateway secret + name := internalGateway.Name + if internalGateway.Spec.App.Management.SecretName != "" { + name = internalGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" + + internalParams := params + internalParams.Instance = internalGateway + + // Note: InternalGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !internalGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, internalParams) + if err != nil { + return err + } + err = ReconcileEphemeralGateway(ctx, internalParams, "otk certificates", *podList, internalGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, internalParams) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, internalParams, "otk certificates", *gatewayDeployment, internalGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + if err != nil { + return err + } + } + + return nil +} + +func publishInternalCertToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway, internalKeySecret *corev1.Secret) error { + // Get DMZ gateway + dmzGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.DmzOtkGatewayReference, + Namespace: gateway.Namespace, + }, dmzGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.DmzGatewayPort != 0 { + params.Log.V(2).Info("DMZ gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, + "port", gateway.Spec.App.Otk.DmzGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("DMZ gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference) + return nil + } + } else { + return err + } + } + + certData := internalKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt") + } + + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") + } + + bundle := graphman.Bundle{} + + // Add to TrustedCerts + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + } + + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return err + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port + if isExternalGateway { + // For external gateways, we can't use the standard reconciliation + // Log that external gateway support requires additional configuration + params.Log.V(2).Info("External DMZ gateway detected, port will be used for connection", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, + "port", gateway.Spec.App.Otk.DmzGatewayPort) + // Note: External gateway connection would require additional implementation + // For now, we skip reconciliation for external gateways + return nil + } + + // Get DMZ gateway secret + name := dmzGateway.Name + if dmzGateway.Spec.App.Management.SecretName != "" { + name = dmzGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + + dmzParams := params + dmzParams.Instance = dmzGateway + + // Note: DmzGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !dmzGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, dmzParams) + if err != nil { + return err + } + err = ReconcileEphemeralGateway(ctx, dmzParams, "otk certificates", *podList, dmzGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, dmzParams, "otk certificates", *gatewayDeployment, dmzGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + if err != nil { + return err + } + } + + return nil +} + +// updateDmzPrivateKeyClusterProperty updates the cluster property otk.dmz.private_key.name +// with the DMZ private key name. This is called after the DMZ key is successfully updated. +func updateDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gateway *securityv1.Gateway, keyName string) error { + // Only update cluster property for DMZ gateway type + if gateway.Spec.App.Otk.Type != securityv1.OtkTypeDMZ { + return nil + } + + // Get or create the cluster properties ConfigMap + cmName := gateway.Name + "-cwp-bundle" + cm, err := getGatewayConfigMap(ctx, params, cmName) + if err != nil { + if !k8serrors.IsNotFound(err) { + return fmt.Errorf("failed to get cluster properties ConfigMap: %w", err) + } + // ConfigMap doesn't exist, create it with the property + return createDmzPrivateKeyClusterProperty(ctx, params, gateway, keyName, cmName) + } + + // Parse existing bundle + bundle := graphman.Bundle{} + bundleJSON := cm.Data["cwp.json"] + if bundleJSON == "" { + // Empty bundle, create new one + return createDmzPrivateKeyClusterProperty(ctx, params, gateway, keyName, cmName) + } + + err = json.Unmarshal([]byte(bundleJSON), &bundle) + if err != nil { + return fmt.Errorf("failed to parse cluster properties bundle: %w", err) + } + + // Initialize bundle properties if nil + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + + // Check if property already exists and update it, or add new one + propertyName := "otk.dmz.private_key.name" + found := false + for _, cwp := range bundle.ClusterProperties { + if cwp.Name == propertyName { + cwp.Value = keyName + found = true + break + } + } + + if !found { + // Add new cluster property + bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ + Name: propertyName, + Value: keyName, + }) + } + + // Marshal bundle back to JSON + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return fmt.Errorf("failed to marshal cluster properties bundle: %w", err) + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Update ConfigMap + cm.Data["cwp.json"] = string(bundleBytes) + if cm.ObjectMeta.Annotations == nil { + cm.ObjectMeta.Annotations = make(map[string]string) + } + cm.ObjectMeta.Annotations["checksum/data"] = sha1Sum + + err = params.Client.Update(ctx, cm) + if err != nil { + return fmt.Errorf("failed to update cluster properties ConfigMap: %w", err) + } + + params.Log.V(2).Info("Updated cluster property ConfigMap", "property", propertyName, "value", keyName, "gateway", gateway.Name) + + // Apply the cluster property using the existing mechanism + gwUpdReq, err := NewGwUpdateRequest( + ctx, + gateway, + params, + WithBundleType(BundleTypeClusterProp), + ) + if err != nil { + return fmt.Errorf("failed to create gateway update request: %w", err) + } + + err = SyncGateway(ctx, params, *gwUpdReq) + if err != nil { + return fmt.Errorf("failed to sync cluster property: %w", err) + } + + params.Log.V(2).Info("Applied cluster property", "property", propertyName, "value", keyName, "gateway", gateway.Name) + + return nil +} + +// createDmzPrivateKeyClusterProperty creates a new cluster properties ConfigMap with the DMZ private key property +func createDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gateway *securityv1.Gateway, keyName string, cmName string) error { + // Create new bundle with the property + bundle := graphman.Bundle{ + ClusterProperties: []*graphman.ClusterPropertyInput{ + { + Name: "otk.dmz.private_key.name", + Value: keyName, + }, + }, + Properties: &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + }, + } + + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return fmt.Errorf("failed to marshal cluster properties bundle: %w", err) + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Create ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: gateway.Namespace, + Annotations: map[string]string{ + "checksum/data": sha1Sum, + }, + }, + Data: map[string]string{ + "cwp.json": string(bundleBytes), + }, + } + + // Set controller reference + if err := controllerutil.SetControllerReference(gateway, cm, params.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + err = params.Client.Create(ctx, cm) + if err != nil { + return fmt.Errorf("failed to create cluster properties ConfigMap: %w", err) + } + + params.Log.V(2).Info("Created cluster property ConfigMap", "property", "otk.dmz.private_key.name", "value", keyName, "gateway", gateway.Name) + + // Apply the cluster property using the existing mechanism + gwUpdReq, err := NewGwUpdateRequest( + ctx, + gateway, + params, + WithBundleType(BundleTypeClusterProp), + ) + if err != nil { + return fmt.Errorf("failed to create gateway update request: %w", err) + } + + err = SyncGateway(ctx, params, *gwUpdReq) + if err != nil { + return fmt.Errorf("failed to sync cluster property: %w", err) + } + + params.Log.V(2).Info("Applied cluster property", "property", "otk.dmz.private_key.name", "value", keyName, "gateway", gateway.Name) + + return nil +} diff --git a/pkg/gateway/reconcile/gateway.go b/pkg/gateway/reconcile/gateway.go index f01acbc3..24a1ed28 100644 --- a/pkg/gateway/reconcile/gateway.go +++ b/pkg/gateway/reconcile/gateway.go @@ -499,7 +499,8 @@ func NewGwUpdateRequest(ctx context.Context, gateway *securityv1.Gateway, params for _, k := range gateway.Status.LastAppliedExternalKeys { found := false for _, ek := range gateway.Spec.App.ExternalKeys { - if k == ek.Alias && ek.Enabled { + if k == ek.Alias && ek.Enabled && !ek.Otk { + // Only process non-OTK keys in regular external keys flow found = true } } @@ -511,7 +512,8 @@ func NewGwUpdateRequest(ctx context.Context, gateway *securityv1.Gateway, params var sha1Sum string for _, externalKey := range gateway.Spec.App.ExternalKeys { - if externalKey.Enabled { + if externalKey.Enabled && !externalKey.Otk { + // Skip keys with otk: true - they are handled separately by OTK reconciliation secret, err := getGatewaySecret(ctx, params, externalKey.Name) if err != nil { return nil, err @@ -1532,7 +1534,7 @@ func updateGatewayDeployment(ctx context.Context, params Params, gwUpdReq *Gatew leaderAvailable := false for _, pod := range gwUpdReq.podList.Items { if pod.ObjectMeta.Labels["management-access"] == "leader" { - endpoint = podIP(pod.Status.PodIP) + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" + endpoint = "127.0.0.1" + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" leaderAvailable = true } } @@ -1838,7 +1840,8 @@ func updateGatewayPods(ctx context.Context, params Params, gwUpdReq *GatewayUpda if update && ready { updateStatus = true - endpoint := podIP(pod.Status.PodIP) + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" + endpoint := "127.0.0.1" + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" + requestCacheEntry := pod.Name + "-" + gwUpdReq.cacheEntry syncRequest, err := syncCache.Read(requestCacheEntry) tryRequest := true @@ -2117,7 +2120,7 @@ func ReconcileEphemeralGateway(ctx context.Context, params Params, kind string, if update && ready { updateStatus = true - endpoint := podIP(pod.Status.PodIP) + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" requestCacheEntry := pod.Name + "-" + gateway.Name + "-" + name + "-" + sha1Sum syncRequest, err := syncCache.Read(requestCacheEntry) diff --git a/pkg/gateway/reconcile/l7otkcertificates.go b/pkg/gateway/reconcile/l7otkcertificates.go index c290f3f5..1f664f80 100644 --- a/pkg/gateway/reconcile/l7otkcertificates.go +++ b/pkg/gateway/reconcile/l7otkcertificates.go @@ -27,15 +27,10 @@ package reconcile import ( "context" - "crypto/tls" "encoding/base64" "encoding/json" - "strconv" - securityv1 "github.com/caapim/layer7-operator/api/v1" "github.com/caapim/layer7-operator/internal/graphman" - "github.com/caapim/layer7-operator/pkg/gateway" - corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ) @@ -143,115 +138,3 @@ func applyOtkCertificates(ctx context.Context, params Params, gateway *securityv return nil } - -func manageCertificateSecrets(ctx context.Context, params Params) { - gw := &securityv1.Gateway{} - err := params.Client.Get(ctx, types.NamespacedName{Name: params.Instance.Name, Namespace: params.Instance.Namespace}, gw) - if err != nil && k8serrors.IsNotFound(err) { - params.Log.Error(err, "gateway not found", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - _ = removeJob(params.Instance.Name + "-sync-otk-certificate-secret") - return - } - - if !gw.Spec.App.Otk.Enabled { - _ = removeJob(params.Instance.Name + "-sync-otk-certificate-secret") - return - } - params.Instance = gw - internalGatewayPort := 9443 - defaultOtkPort := 8443 - rawInternalCertList := map[string][]byte{} - rawDMZCertList := map[string][]byte{} - desiredSecrets := []*corev1.Secret{} - if gw.Spec.App.Management.Graphman.DynamicSyncPort != 0 { - internalGatewayPort = gw.Spec.App.Management.Graphman.DynamicSyncPort - - } - - if gw.Spec.App.Otk.InternalGatewayPort != 0 { - internalGatewayPort = gw.Spec.App.Otk.InternalGatewayPort - } - - if gw.Spec.App.Otk.OTKPort != 0 { - defaultOtkPort = gw.Spec.App.Otk.OTKPort - } - podList, err := getGatewayPods(ctx, params) - if err != nil { - params.Log.Error(err, "failed to retrieve gateway pods", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - - for _, pod := range podList.Items { - for _, containerStatus := range pod.Status.ContainerStatuses { - if containerStatus.Name == "gateway" { - if !containerStatus.Ready { - params.Log.V(2).Info("pod not ready", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - } - } - - switch gw.Spec.App.Otk.Type { - case securityv1.OtkTypeDMZ: - rawCert, err := retrieveCertificate(pod.Status.PodIP, strconv.Itoa(defaultOtkPort)) - if err != nil { - params.Log.Error(err, "failed to retrieve certificate", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - if len(rawDMZCertList) > 0 { - for _, cert := range rawDMZCertList { - if string(rawCert) != string(cert) { - rawDMZCertList[pod.Name] = rawCert - } - } - } else { - rawDMZCertList[pod.Name] = rawCert - } - case securityv1.OtkTypeInternal: - rawCert, err := retrieveCertificate(pod.Status.PodIP, strconv.Itoa(internalGatewayPort)) - if err != nil { - params.Log.Error(err, "failed to retrieve certificate", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - if len(rawInternalCertList) > 0 { - for _, cert := range rawInternalCertList { - if string(rawCert) != string(cert) { - rawInternalCertList[pod.Name] = rawCert - } - } - } else { - rawInternalCertList[pod.Name] = rawCert - } - - } - } - - if gw.Spec.App.Otk.Type == securityv1.OtkTypeDMZ && len(rawDMZCertList) > 0 { - desiredSecrets = append(desiredSecrets, gateway.NewOtkCertificateSecret(gw, gw.Name+"-otk-dmz-certificates", rawDMZCertList)) - } - - if gw.Spec.App.Otk.Type == securityv1.OtkTypeInternal && len(rawInternalCertList) > 0 { - desiredSecrets = append(desiredSecrets, gateway.NewOtkCertificateSecret(gw, gw.Name+"-otk-internal-certificates", rawInternalCertList)) - } - - err = reconcileSecrets(ctx, params, desiredSecrets) - if err != nil { - params.Log.Error(err, "failed to reconcile otk certificates", "Name", gw.Name, "namespace", gw.Namespace) - return - } - -} - -func retrieveCertificate(host string, port string) ([]byte, error) { - conf := &tls.Config{ - InsecureSkipVerify: true, - } - - conn, err := tls.Dial("tcp", host+":"+port, conf) - if err != nil { - return nil, err - } - defer conn.Close() - cert := conn.ConnectionState().PeerCertificates[0].Raw - return cert, nil -} From a53e1077d24117b2de959afd6a87a24afde83af4 Mon Sep 17 00:00:00 2001 From: as673366 Date: Mon, 22 Dec 2025 22:08:18 +0530 Subject: [PATCH 2/8] reverting unintenteded changes --- pkg/api/reconcile/l7api.go | 4 ++-- pkg/gateway/reconcile/gateway.go | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/api/reconcile/l7api.go b/pkg/api/reconcile/l7api.go index c516b8e1..e5a4d12b 100644 --- a/pkg/api/reconcile/l7api.go +++ b/pkg/api/reconcile/l7api.go @@ -188,7 +188,7 @@ func deployL7ApiToGateway(ctx context.Context, params Params, gateway *v1.Gatewa } if tryRequest { - endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := pod.Status.PodIP + ":" + strconv.Itoa(graphmanPort) + "/graphman" var errorMessage string status := SUCCESS name := gateway.Name @@ -264,7 +264,7 @@ func undeployL7ApiToGateway(ctx context.Context, params Params, gateway *v1.Gate } if tryRequest { - endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := pod.Status.PodIP + ":" + strconv.Itoa(graphmanPort) + "/graphman" status := SUCCESS name := gateway.Name if gateway.Spec.App.Management.SecretName != "" { diff --git a/pkg/gateway/reconcile/gateway.go b/pkg/gateway/reconcile/gateway.go index 24a1ed28..f2ca4434 100644 --- a/pkg/gateway/reconcile/gateway.go +++ b/pkg/gateway/reconcile/gateway.go @@ -1534,7 +1534,7 @@ func updateGatewayDeployment(ctx context.Context, params Params, gwUpdReq *Gatew leaderAvailable := false for _, pod := range gwUpdReq.podList.Items { if pod.ObjectMeta.Labels["management-access"] == "leader" { - endpoint = "127.0.0.1" + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" + endpoint = podIP(pod.Status.PodIP) + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" leaderAvailable = true } } @@ -1840,8 +1840,7 @@ func updateGatewayPods(ctx context.Context, params Params, gwUpdReq *GatewayUpda if update && ready { updateStatus = true - endpoint := "127.0.0.1" + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" - + endpoint := podIP(pod.Status.PodIP) + ":" + strconv.Itoa(gwUpdReq.graphmanPort) + "/graphman" requestCacheEntry := pod.Name + "-" + gwUpdReq.cacheEntry syncRequest, err := syncCache.Read(requestCacheEntry) tryRequest := true @@ -2120,7 +2119,7 @@ func ReconcileEphemeralGateway(ctx context.Context, params Params, kind string, if update && ready { updateStatus = true - endpoint := "127.0.0.1" + ":" + strconv.Itoa(graphmanPort) + "/graphman" + endpoint := podIP(pod.Status.PodIP) + ":" + strconv.Itoa(graphmanPort) + "/graphman" requestCacheEntry := pod.Name + "-" + gateway.Name + "-" + name + "-" + sha1Sum syncRequest, err := syncCache.Read(requestCacheEntry) From 6ef94c1b1cfba9f28147bbeff54d0f19a82ec894 Mon Sep 17 00:00:00 2001 From: as673366 Date: Tue, 23 Dec 2025 10:39:05 +0530 Subject: [PATCH 3/8] segregating certificate & key sync operations --- pkg/gateway/reconcile/externalkeys.go | 346 ++-------- pkg/gateway/reconcile/l7otkcertificates.go | 727 +++++++++++++++++++-- 2 files changed, 742 insertions(+), 331 deletions(-) diff --git a/pkg/gateway/reconcile/externalkeys.go b/pkg/gateway/reconcile/externalkeys.go index ddacf3ef..8a76eca1 100644 --- a/pkg/gateway/reconcile/externalkeys.go +++ b/pkg/gateway/reconcile/externalkeys.go @@ -30,7 +30,6 @@ import ( "context" "crypto/sha1" "crypto/x509" - "encoding/base64" "encoding/json" "encoding/pem" "fmt" @@ -42,7 +41,6 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -159,21 +157,13 @@ func handleDmzKeyUpdate(ctx context.Context, params Params, gateway *securityv1. isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled if isOperatorManaged { - // Update DMZ with the new key + // Update DMZ with the new key (key sync only, cert publishing handled by syncOtkCertificates) err = updateDmzWithKey(ctx, params, gateway, dmzKeySecret) if err != nil { return fmt.Errorf("failed to update DMZ with key: %w", err) } } - // Publish DMZ cert to Internal for Truststore & FIP User - if gateway.Spec.App.Otk.InternalOtkGatewayReference != "" { - err = publishDmzCertToInternal(ctx, params, gateway, dmzKeySecret) - if err != nil { - return fmt.Errorf("failed to publish DMZ cert to Internal: %w", err) - } - } - return nil } @@ -192,21 +182,13 @@ func handleInternalKeyUpdate(ctx context.Context, params Params, gateway *securi isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled if isOperatorManaged { - // Update Internal with the new key + // Update Internal with the new key (key sync only, cert publishing handled by syncOtkCertificates) err = updateInternalWithKey(ctx, params, gateway, internalKeySecret) if err != nil { return fmt.Errorf("failed to update Internal with key: %w", err) } } - // Publish Internal cert to DMZ for Truststore - if gateway.Spec.App.Otk.DmzOtkGatewayReference != "" { - err = publishInternalCertToDmz(ctx, params, gateway, internalKeySecret) - if err != nil { - return fmt.Errorf("failed to publish Internal cert to DMZ: %w", err) - } - } - return nil } @@ -273,31 +255,51 @@ func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Ga annotation := "security.brcmlabs.com/otk-dmz-key" + // Check if DMZ key was already applied (to determine if update is needed) + keyWasUpdated := false if !gateway.Spec.App.Management.Database.Enabled { podList, err := getGatewayPods(ctx, params) if err != nil { return err } + // Check current annotation value before update + currentSha1Sum := "" + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } err = ReconcileEphemeralGateway(ctx, params, "otk dmz key", *podList, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) if err != nil { return err } + // Key was updated if sha1Sum changed + keyWasUpdated = (currentSha1Sum != sha1Sum) } else { gatewayDeployment, err := getGatewayDeployment(ctx, params) if err != nil { return err } + // Check current annotation value before update + currentSha1Sum := gatewayDeployment.ObjectMeta.Annotations[annotation] err = ReconcileDBGateway(ctx, params, "otk dmz key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) if err != nil { return err } + // Key was updated if sha1Sum changed (ReconcileDBGateway returns early if already applied) + keyWasUpdated = (currentSha1Sum != sha1Sum) } - // Update cluster property otk.dmz.private_key.name after DMZ key is updated - //if err := updateDmzPrivateKeyClusterProperty(ctx, params, gateway, "otk-dmz-key"); err != nil { - // params.Log.V(2).Info("Failed to update DMZ private key cluster property", "error", err, "gateway", gateway.Name) - // // Don't fail the entire operation if cluster property update fails - //} + // Update cluster property only if DMZ key was updated + if keyWasUpdated { + if err := updateDmzPrivateKeyClusterProperty(ctx, params, gateway, "otk-dmz-key"); err != nil { + params.Log.V(2).Info("Failed to update DMZ private key cluster property", "error", err, "gateway", gateway.Name) + // Don't fail the entire operation if cluster property update fails + } + } else if !keyWasUpdated { + params.Log.V(2).Info("DMZ key was not updated, skipping cluster property update", "gateway", gateway.Name) + } return nil } @@ -388,299 +390,68 @@ func updateInternalWithKey(ctx context.Context, params Params, gateway *security return nil } -func publishDmzCertToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret) error { - // Get Internal gateway - internalGateway := &securityv1.Gateway{} - err := params.Client.Get(ctx, types.NamespacedName{ - Name: gateway.Spec.App.Otk.InternalOtkGatewayReference, - Namespace: gateway.Namespace, - }, internalGateway) - - isExternalGateway := false - if err != nil { - if k8serrors.IsNotFound(err) { - // Gateway not found - check if it's external (port specified) - if gateway.Spec.App.Otk.InternalGatewayPort != 0 { - params.Log.V(2).Info("Internal gateway not found but port specified, treating as external", - "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, - "port", gateway.Spec.App.Otk.InternalGatewayPort) - isExternalGateway = true - } else { - params.Log.V(2).Info("Internal gateway not found and no port specified, skipping cert publish", - "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference) - return nil - } - } else { - return err - } - } - - certData := dmzKeySecret.Data["tls.crt"] - if len(certData) == 0 { - return fmt.Errorf("DMZ key secret must contain tls.crt") - } - - // Parse certificate - crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") - if len(crtStrings) == 0 { - return fmt.Errorf("invalid certificate format") - } - - bundle := graphman.Bundle{} - - // Add to TrustedCerts - for _, certStr := range crtStrings { - if certStr == "" { - continue - } - b, _ := pem.Decode([]byte(certStr)) - if b == nil { - continue - } - crtX509, err := x509.ParseCertificate(b.Bytes) - if err != nil { - continue - } - - bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ - Name: crtX509.Subject.CommonName, - CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), - TrustAnchor: true, - VerifyHostname: false, - RevocationCheckPolicyType: "USE_DEFAULT", - TrustedFor: []graphman.TrustedForType{ - "SSL", - "SIGNING_SERVER_CERTS", - }, - }) - - // Add to FIP Users - bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ - Name: crtX509.Subject.CommonName, - ProviderName: "otk-fips-provider", - SubjectDn: "cn=" + crtX509.Subject.CommonName, - CertBase64: base64.RawStdEncoding.EncodeToString(crtX509.Raw), - }) - } - - bundleBytes, err := json.Marshal(bundle) - if err != nil { - return err - } - - // Calculate checksum - h := sha1.New() - h.Write(bundleBytes) - sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) - - // If gateway is external (not managed by operator), use specified port - if isExternalGateway { - // For external gateways, we can't use the standard reconciliation - // Log that external gateway support requires additional configuration - params.Log.V(2).Info("External Internal gateway detected, port will be used for connection", - "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, - "port", gateway.Spec.App.Otk.InternalGatewayPort) - // Note: External gateway connection would require additional implementation - // For now, we skip reconciliation for external gateways - return nil - } - - // Get Internal gateway secret - name := internalGateway.Name - if internalGateway.Spec.App.Management.SecretName != "" { - name = internalGateway.Spec.App.Management.SecretName - } - gwSecret, err := getGatewaySecret(ctx, params, name) - if err != nil { - return err - } - - annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" - - internalParams := params - internalParams.Instance = internalGateway - - // Note: InternalGatewayPort is used when the gateway is external (not found in cluster) - // For operator-managed gateways, the gateway's own graphman port configuration is used - - if !internalGateway.Spec.App.Management.Database.Enabled { - podList, err := getGatewayPods(ctx, internalParams) - if err != nil { - return err - } - err = ReconcileEphemeralGateway(ctx, internalParams, "otk certificates", *podList, internalGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) - if err != nil { - return err - } - } else { - gatewayDeployment, err := getGatewayDeployment(ctx, internalParams) - if err != nil { - return err - } - err = ReconcileDBGateway(ctx, internalParams, "otk certificates", *gatewayDeployment, internalGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) - if err != nil { - return err - } +// checkClusterPropertyExists checks if the cluster property exists in the ConfigMap +func checkClusterPropertyExists(ctx context.Context, params Params, gateway *securityv1.Gateway, propertyName string) bool { + // Only check for DMZ gateway type + if gateway.Spec.App.Otk.Type != securityv1.OtkTypeDMZ { + return false } - return nil -} - -func publishInternalCertToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway, internalKeySecret *corev1.Secret) error { - // Get DMZ gateway - dmzGateway := &securityv1.Gateway{} - err := params.Client.Get(ctx, types.NamespacedName{ - Name: gateway.Spec.App.Otk.DmzOtkGatewayReference, - Namespace: gateway.Namespace, - }, dmzGateway) - - isExternalGateway := false + // Get the cluster properties ConfigMap + cmName := gateway.Name + "-cwp-bundle" + cm, err := getGatewayConfigMap(ctx, params, cmName) if err != nil { - if k8serrors.IsNotFound(err) { - // Gateway not found - check if it's external (port specified) - if gateway.Spec.App.Otk.DmzGatewayPort != 0 { - params.Log.V(2).Info("DMZ gateway not found but port specified, treating as external", - "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, - "port", gateway.Spec.App.Otk.DmzGatewayPort) - isExternalGateway = true - } else { - params.Log.V(2).Info("DMZ gateway not found and no port specified, skipping cert publish", - "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference) - return nil - } - } else { - return err - } + return false } - certData := internalKeySecret.Data["tls.crt"] - if len(certData) == 0 { - return fmt.Errorf("Internal key secret must contain tls.crt") - } - - // Parse certificate - crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") - if len(crtStrings) == 0 { - return fmt.Errorf("invalid certificate format") + // Parse existing bundle + bundleJSON := cm.Data["cwp.json"] + if bundleJSON == "" { + return false } bundle := graphman.Bundle{} - - // Add to TrustedCerts - for _, certStr := range crtStrings { - if certStr == "" { - continue - } - b, _ := pem.Decode([]byte(certStr)) - if b == nil { - continue - } - crtX509, err := x509.ParseCertificate(b.Bytes) - if err != nil { - continue - } - - bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ - Name: crtX509.Subject.CommonName, - CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), - TrustAnchor: true, - VerifyHostname: false, - RevocationCheckPolicyType: "USE_DEFAULT", - TrustedFor: []graphman.TrustedForType{ - "SSL", - "SIGNING_SERVER_CERTS", - }, - }) - } - - bundleBytes, err := json.Marshal(bundle) - if err != nil { - return err - } - - // Calculate checksum - h := sha1.New() - h.Write(bundleBytes) - sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) - - // If gateway is external (not managed by operator), use specified port - if isExternalGateway { - // For external gateways, we can't use the standard reconciliation - // Log that external gateway support requires additional configuration - params.Log.V(2).Info("External DMZ gateway detected, port will be used for connection", - "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, - "port", gateway.Spec.App.Otk.DmzGatewayPort) - // Note: External gateway connection would require additional implementation - // For now, we skip reconciliation for external gateways - return nil - } - - // Get DMZ gateway secret - name := dmzGateway.Name - if dmzGateway.Spec.App.Management.SecretName != "" { - name = dmzGateway.Spec.App.Management.SecretName - } - gwSecret, err := getGatewaySecret(ctx, params, name) + err = json.Unmarshal([]byte(bundleJSON), &bundle) if err != nil { - return err + return false } - annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" - - dmzParams := params - dmzParams.Instance = dmzGateway - - // Note: DmzGatewayPort is used when the gateway is external (not found in cluster) - // For operator-managed gateways, the gateway's own graphman port configuration is used - - if !dmzGateway.Spec.App.Management.Database.Enabled { - podList, err := getGatewayPods(ctx, dmzParams) - if err != nil { - return err - } - err = ReconcileEphemeralGateway(ctx, dmzParams, "otk certificates", *podList, dmzGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) - if err != nil { - return err - } - } else { - gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) - if err != nil { - return err - } - err = ReconcileDBGateway(ctx, dmzParams, "otk certificates", *gatewayDeployment, dmzGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) - if err != nil { - return err + // Check if property exists + for _, cwp := range bundle.ClusterProperties { + if cwp.Name == propertyName { + return true } } - return nil + return false } // updateDmzPrivateKeyClusterProperty updates the cluster property otk.dmz.private_key.name -// with the DMZ private key name. This is called after the DMZ key is successfully updated. +// with the DMZ private key name. This function only updates the property if it exists. +// It does not create the property if it doesn't exist. func updateDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gateway *securityv1.Gateway, keyName string) error { // Only update cluster property for DMZ gateway type if gateway.Spec.App.Otk.Type != securityv1.OtkTypeDMZ { return nil } - // Get or create the cluster properties ConfigMap + // Get the cluster properties ConfigMap cmName := gateway.Name + "-cwp-bundle" cm, err := getGatewayConfigMap(ctx, params, cmName) if err != nil { if !k8serrors.IsNotFound(err) { return fmt.Errorf("failed to get cluster properties ConfigMap: %w", err) } - // ConfigMap doesn't exist, create it with the property - return createDmzPrivateKeyClusterProperty(ctx, params, gateway, keyName, cmName) + // ConfigMap doesn't exist, property doesn't exist - skip update + return fmt.Errorf("cluster property ConfigMap does not exist") } // Parse existing bundle bundle := graphman.Bundle{} bundleJSON := cm.Data["cwp.json"] if bundleJSON == "" { - // Empty bundle, create new one - return createDmzPrivateKeyClusterProperty(ctx, params, gateway, keyName, cmName) + // Empty bundle, property doesn't exist - skip update + return fmt.Errorf("cluster property bundle is empty") } err = json.Unmarshal([]byte(bundleJSON), &bundle) @@ -695,7 +466,7 @@ func updateDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gate } } - // Check if property already exists and update it, or add new one + // Check if property exists and update it propertyName := "otk.dmz.private_key.name" found := false for _, cwp := range bundle.ClusterProperties { @@ -707,11 +478,8 @@ func updateDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gate } if !found { - // Add new cluster property - bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ - Name: propertyName, - Value: keyName, - }) + // Property doesn't exist - skip update (only update, don't create) + return fmt.Errorf("cluster property %s does not exist", propertyName) } // Marshal bundle back to JSON diff --git a/pkg/gateway/reconcile/l7otkcertificates.go b/pkg/gateway/reconcile/l7otkcertificates.go index 1f664f80..d8703d70 100644 --- a/pkg/gateway/reconcile/l7otkcertificates.go +++ b/pkg/gateway/reconcile/l7otkcertificates.go @@ -27,10 +27,18 @@ package reconcile import ( "context" + "crypto/sha1" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" + "fmt" + "strings" + securityv1 "github.com/caapim/layer7-operator/api/v1" "github.com/caapim/layer7-operator/internal/graphman" + "github.com/caapim/layer7-operator/pkg/util" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ) @@ -49,88 +57,465 @@ func syncOtkCertificates(ctx context.Context, params Params) { return } - err = applyOtkCertificates(ctx, params, gateway) + // Publish DMZ certs to Internal Gateway when DMZ key is updated + if gateway.Spec.App.Otk.Type == securityv1.OtkTypeDMZ && gateway.Spec.App.Otk.DmzKeySecret != "" { + err = publishDmzCertificatesToInternal(ctx, params, gateway) + if err != nil { + params.Log.V(2).Info("failed to publish DMZ certificates to Internal", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + } + } + + // Publish Internal certs to DMZ Gateway when Internal key is updated + if gateway.Spec.App.Otk.Type == securityv1.OtkTypeInternal && gateway.Spec.App.Otk.InternalKeySecret != "" { + err = publishInternalCertificatesToDmz(ctx, params, gateway) + if err != nil { + params.Log.V(2).Info("failed to publish Internal certificates to DMZ", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + } + } +} + +// publishDmzCertificatesToInternal publishes DMZ certificates to Internal gateway when DMZ key is updated +// Handles ephemeral, DB-backed, and external gateways +func publishDmzCertificatesToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Check if Internal gateway reference is specified + if gateway.Spec.App.Otk.InternalOtkGatewayReference == "" { + return nil + } + + // Get DMZ key secret + dmzKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzKeySecret) if err != nil { - params.Log.Info("failed to reconcile otk certificates", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("DMZ key secret not found, skipping cert publish", "secret", gateway.Spec.App.Otk.DmzKeySecret) + return nil + } + return err + } + + // Check if key was updated by comparing annotation + annotation := "security.brcmlabs.com/otk-dmz-key" + currentSha1Sum := "" + + if !gateway.Spec.App.Management.Database.Enabled { + // Ephemeral gateway - check pod annotations + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } + } else { + // DB-backed gateway - check deployment annotations + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err + } + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] + } + + // Calculate current key checksum + certData := dmzKeySecret.Data["tls.crt"] + keyData := dmzKeySecret.Data["tls.key"] + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt and tls.key") } + + keySecretMap := []struct { + Name string + Crt string + Key string + Alias string + UsageType string + }{ + { + Name: "dmz-key", + Crt: string(certData), + Key: string(keyData), + Alias: "otk-dmz-key", + UsageType: "SSL", + }, + } + + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + newSha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Only publish if key was updated + if currentSha1Sum == newSha1Sum { + params.Log.V(2).Info("DMZ key not updated, skipping cert publish", "gateway", gateway.Name) + return nil + } + + // Publish DMZ cert to Internal (handles ephemeral, DB-backed, and external gateways) + return publishDmzCertToInternal(ctx, params, gateway, dmzKeySecret) } -func applyOtkCertificates(ctx context.Context, params Params, gateway *securityv1.Gateway) error { +// publishInternalCertificatesToDmz publishes Internal certificates to DMZ gateway when Internal key is updated +// Handles ephemeral, DB-backed, and external gateways +func publishInternalCertificatesToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Check if DMZ gateway reference is specified + if gateway.Spec.App.Otk.DmzOtkGatewayReference == "" { + return nil + } - bundle := graphman.Bundle{} - annotation := "" - sha1Sum := "" + // Get Internal key secret + internalKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("Internal key secret not found, skipping cert publish", "secret", gateway.Spec.App.Otk.InternalKeySecret) + return nil + } + return err + } - switch gateway.Spec.App.Otk.Type { - case securityv1.OtkTypeDMZ: - internalSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalOtkGatewayReference+"-otk-internal-certificates") - sha1Sum = internalSecret.ObjectMeta.Annotations["checksum/data"] + // Check if key was updated by comparing annotation + annotation := "security.brcmlabs.com/otk-internal-key" + currentSha1Sum := "" + if !gateway.Spec.App.Management.Database.Enabled { + // Ephemeral gateway - check pod annotations + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } + } else { + // DB-backed gateway - check deployment annotations + gatewayDeployment, err := getGatewayDeployment(ctx, params) if err != nil { return err } - annotation = "security.brcmlabs.com/" + gateway.Name + "-" + string(gateway.Spec.App.Otk.Type) + "-certificates" - for k, v := range internalSecret.Data { - bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ - Name: k, - CertBase64: base64.StdEncoding.EncodeToString(v), - TrustAnchor: true, - VerifyHostname: false, - RevocationCheckPolicyType: "USE_DEFAULT", - TrustedFor: []graphman.TrustedForType{ - "SSL", - "SIGNING_SERVER_CERTS", - }, + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] + } + + // Calculate current key checksum + certData := internalKeySecret.Data["tls.crt"] + keyData := internalKeySecret.Data["tls.key"] + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt and tls.key") + } + + keySecretMap := []struct { + Name string + Crt string + Key string + Alias string + UsageType string + }{ + { + Name: "internal-key", + Crt: string(certData), + Key: string(keyData), + Alias: "otk-internal-key", + UsageType: "SSL", + }, + } + + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + newSha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Only publish if key was updated + if currentSha1Sum == newSha1Sum { + params.Log.V(2).Info("Internal key not updated, skipping cert publish", "gateway", gateway.Name) + return nil + } + + // Publish Internal cert to DMZ (handles ephemeral, DB-backed, and external gateways) + return publishInternalCertToDmz(ctx, params, gateway, internalKeySecret) +} + +func publishDmzCertToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret) error { + // Get Internal gateway + internalGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.InternalOtkGatewayReference, + Namespace: gateway.Namespace, + }, internalGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.InternalGatewayPort != 0 { + params.Log.V(2).Info("Internal gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, + "port", gateway.Spec.App.Otk.InternalGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("Internal gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference) + return nil + } + } else { + return err + } + } + + certData := dmzKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt") + } + + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") + } + + bundle := graphman.Bundle{} + + // Add to TrustedCerts + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + + // Add to FIP Users + bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ + Name: crtX509.Subject.CommonName, + ProviderName: "otk-fips-provider", + SubjectDn: "cn=" + crtX509.Subject.CommonName, + CertBase64: base64.RawStdEncoding.EncodeToString(crtX509.Raw), + }) + } + + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return err + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port and auth secret + if isExternalGateway { + // Parse certificates to extract information for FIP user creation + var certInfo []struct { + commonName string + subjectDn string + certRaw []byte + } + + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + // Extract full Subject DN from certificate + subjectDn := extractSubjectDN(crtX509) + + certInfo = append(certInfo, struct { + commonName string + subjectDn string + certRaw []byte + }{ + commonName: crtX509.Subject.CommonName, + subjectDn: subjectDn, + certRaw: crtX509.Raw, }) } - case securityv1.OtkTypeInternal: - dmzSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzOtkGatewayReference+"-otk-dmz-certificates") - sha1Sum = dmzSecret.ObjectMeta.Annotations["checksum/data"] + return syncDmzCertToExternalInternalGateway(ctx, params, gateway, dmzKeySecret, certInfo) + } + + // Get Internal gateway secret + name := internalGateway.Name + if internalGateway.Spec.App.Management.SecretName != "" { + name = internalGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" + + internalParams := params + internalParams.Instance = internalGateway + + // Note: InternalGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !internalGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, internalParams) if err != nil { return err } - annotation = "security.brcmlabs.com/" + gateway.Name + "-" + string(gateway.Spec.App.Otk.Type) + "-fips-users" - for k, v := range dmzSecret.Data { - bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ - Name: k, - ProviderName: "otk-fips-provider", - SubjectDn: "cn=" + k, - CertBase64: base64.RawStdEncoding.EncodeToString(v), - }) + err = ReconcileEphemeralGateway(ctx, internalParams, "otk certificates", *podList, internalGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, internalParams) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, internalParams, "otk certificates", *gatewayDeployment, internalGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + if err != nil { + return err } } + return nil +} + +func publishInternalCertToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway, internalKeySecret *corev1.Secret) error { + // Get DMZ gateway + dmzGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.DmzOtkGatewayReference, + Namespace: gateway.Namespace, + }, dmzGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.DmzGatewayPort != 0 { + params.Log.V(2).Info("DMZ gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, + "port", gateway.Spec.App.Otk.DmzGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("DMZ gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference) + return nil + } + } else { + return err + } + } + + certData := internalKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt") + } + + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") + } + + bundle := graphman.Bundle{} + + // Add to TrustedCerts + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + } + bundleBytes, err := json.Marshal(bundle) if err != nil { return err } - name := gateway.Name - if gateway.Spec.App.Management.SecretName != "" { - name = gateway.Spec.App.Management.SecretName + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port and auth secret + if isExternalGateway { + return syncInternalCertToExternalDmzGateway(ctx, params, gateway, bundleBytes, sha1Sum) } - gwSecret, err := getGatewaySecret(ctx, params, name) + // Get DMZ gateway secret + name := dmzGateway.Name + if dmzGateway.Spec.App.Management.SecretName != "" { + name = dmzGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) if err != nil { return err } - if !gateway.Spec.App.Management.Database.Enabled { - podList, err := getGatewayPods(ctx, params) + annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + + dmzParams := params + dmzParams.Instance = dmzGateway + + // Note: DmzGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !dmzGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, dmzParams) if err != nil { return err } - err = ReconcileEphemeralGateway(ctx, params, "otk certificates", *podList, gateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + err = ReconcileEphemeralGateway(ctx, dmzParams, "otk certificates", *podList, dmzGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) if err != nil { return err } } else { - gatewayDeployment, err := getGatewayDeployment(ctx, params) + gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) if err != nil { return err } - err = ReconcileDBGateway(ctx, params, "otk certificates", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + err = ReconcileDBGateway(ctx, dmzParams, "otk certificates", *gatewayDeployment, dmzGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) if err != nil { return err } @@ -138,3 +523,261 @@ func applyOtkCertificates(ctx context.Context, params Params, gateway *securityv return nil } + +// syncDmzCertToExternalInternalGateway syncs DMZ certificate to an external Internal gateway +// using graphman. First it adds the certificate as a trusted cert, then creates a FIP user. +func syncDmzCertToExternalInternalGateway(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret, certInfo []struct { + commonName string + subjectDn string + certRaw []byte +}) error { + // Get auth secret for external Internal gateway + if gateway.Spec.App.Otk.InternalAuthSecret == "" { + return fmt.Errorf("internalAuthSecret is required for external Internal gateway") + } + + authSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalAuthSecret) + if err != nil { + return fmt.Errorf("failed to get auth secret for external Internal gateway: %w", err) + } + + // Parse username and password from auth secret + username, password := parseGatewaySecret(authSecret) + if username == "" || password == "" { + return fmt.Errorf("could not retrieve gateway credentials from auth secret: %s", gateway.Spec.App.Otk.InternalAuthSecret) + } + + // Build endpoint URL for external gateway + // Format: :/graphman + // ApplyGraphmanBundle expects format: host:port/path (without https://) + gatewayReference := gateway.Spec.App.Otk.InternalOtkGatewayReference + port := gateway.Spec.App.Otk.InternalGatewayPort + if port == 0 { + port = 9443 // Default graphman port + } + + // For external gateways, the reference might be a hostname or IP + // If it's just a name without domain, we might need to construct a full hostname + // For now, use the reference as-is (could be FQDN, hostname, or IP) + endpoint := fmt.Sprintf("%s:%d/graphman", gatewayReference, port) + + // Step 1: Sync DMZ certificate as TrustedCert first + certData := dmzKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt") + } + + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + trustedCertBundle := graphman.Bundle{} + + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + trustedCertBundle.TrustedCerts = append(trustedCertBundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + } + + trustedCertBundleBytes, err := json.Marshal(trustedCertBundle) + if err != nil { + return fmt.Errorf("failed to marshal trusted cert bundle: %w", err) + } + + params.Log.V(2).Info("Syncing DMZ certificate as TrustedCert to external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Apply trusted cert bundle first + err = util.ApplyGraphmanBundle(username, password, endpoint, "", trustedCertBundleBytes) + if err != nil { + return fmt.Errorf("failed to sync DMZ certificate as TrustedCert to external Internal gateway: %w", err) + } + + params.Log.Info("Successfully synced DMZ certificate as TrustedCert to external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Step 2: Create FIP user with DMZ certificate + if len(certInfo) == 0 { + params.Log.V(2).Info("No certificate info available, skipping FIP user creation") + return nil + } + + fipUserBundle := graphman.Bundle{} + + for _, info := range certInfo { + // Use the extracted Subject DN (not just "cn=" + CommonName) + // Since FIP identity provider doesn't have a default subject dn, we must provide it + fipUserBundle.FipUsers = append(fipUserBundle.FipUsers, &graphman.FipUserInput{ + Name: info.commonName, + ProviderName: "otk-fips-provider", + SubjectDn: info.subjectDn, // Full Subject DN from certificate + CertBase64: base64.RawStdEncoding.EncodeToString(info.certRaw), + }) + } + + fipUserBundleBytes, err := json.Marshal(fipUserBundle) + if err != nil { + return fmt.Errorf("failed to marshal FIP user bundle: %w", err) + } + + params.Log.V(2).Info("Creating FIP user with DMZ certificate in external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Apply FIP user bundle after certificate is synced + err = util.ApplyGraphmanBundle(username, password, endpoint, "", fipUserBundleBytes) + if err != nil { + return fmt.Errorf("failed to create FIP user with DMZ certificate in external Internal gateway: %w", err) + } + + params.Log.Info("Successfully created FIP user with DMZ certificate in external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + return nil +} + +// extractSubjectDN extracts the full Subject DN from an x509 certificate +// Format: CN=name,OU=org unit,O=org,C=country, etc. +func extractSubjectDN(cert *x509.Certificate) string { + var parts []string + + // Add CommonName + if cert.Subject.CommonName != "" { + parts = append(parts, "CN="+cert.Subject.CommonName) + } + + // Add Country + for _, c := range cert.Subject.Country { + if c != "" { + parts = append(parts, "C="+c) + } + } + + // Add Organization + for _, o := range cert.Subject.Organization { + if o != "" { + parts = append(parts, "O="+o) + } + } + + // Add Organizational Unit + for _, ou := range cert.Subject.OrganizationalUnit { + if ou != "" { + parts = append(parts, "OU="+ou) + } + } + + // Add Locality + for _, l := range cert.Subject.Locality { + if l != "" { + parts = append(parts, "L="+l) + } + } + + // Add Province/State + for _, p := range cert.Subject.Province { + if p != "" { + parts = append(parts, "ST="+p) + } + } + + // Add Street Address + for _, s := range cert.Subject.StreetAddress { + if s != "" { + parts = append(parts, "STREET="+s) + } + } + + // Add Postal Code + for _, pc := range cert.Subject.PostalCode { + if pc != "" { + parts = append(parts, "POSTALCODE="+pc) + } + } + + // Add Serial Number + if cert.Subject.SerialNumber != "" { + parts = append(parts, "SERIALNUMBER="+cert.Subject.SerialNumber) + } + + // Join all parts with comma + if len(parts) == 0 { + // Fallback to CN if nothing else is available + if cert.Subject.CommonName != "" { + return "CN=" + cert.Subject.CommonName + } + return "" + } + + return strings.Join(parts, ",") +} + +// syncInternalCertToExternalDmzGateway syncs Internal certificate to an external DMZ gateway +// using graphman. It adds the certificate as a trusted cert. +func syncInternalCertToExternalDmzGateway(ctx context.Context, params Params, gateway *securityv1.Gateway, bundleBytes []byte, sha1Sum string) error { + // Get auth secret for external DMZ gateway + if gateway.Spec.App.Otk.DmzAuthSecret == "" { + return fmt.Errorf("dmzAuthSecret is required for external DMZ gateway") + } + + authSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzAuthSecret) + if err != nil { + return fmt.Errorf("failed to get auth secret for external DMZ gateway: %w", err) + } + + // Parse username and password from auth secret + username, password := parseGatewaySecret(authSecret) + if username == "" || password == "" { + return fmt.Errorf("could not retrieve gateway credentials from auth secret: %s", gateway.Spec.App.Otk.DmzAuthSecret) + } + + // Build endpoint URL for external gateway + // Format: :/graphman + // ApplyGraphmanBundle expects format: host:port/path (without https://) + gatewayReference := gateway.Spec.App.Otk.DmzOtkGatewayReference + port := gateway.Spec.App.Otk.DmzGatewayPort + if port == 0 { + port = 9443 // Default graphman port + } + + // For external gateways, the reference might be a hostname or IP + endpoint := fmt.Sprintf("%s:%d/graphman", gatewayReference, port) + + params.Log.V(2).Info("Syncing Internal certificate to external DMZ gateway", + "gateway", gatewayReference, + "endpoint", endpoint, + "sha1Sum", sha1Sum) + + // Apply bundle to external gateway using graphman + err = util.ApplyGraphmanBundle(username, password, endpoint, "", bundleBytes) + if err != nil { + return fmt.Errorf("failed to sync Internal certificate to external DMZ gateway: %w", err) + } + + params.Log.Info("Successfully synced Internal certificate to external DMZ gateway", + "gateway", gatewayReference, + "endpoint", endpoint, + "sha1Sum", sha1Sum) + + return nil +} From 089de635c3e38dfcc378b85bf1c5cc8882c69b92 Mon Sep 17 00:00:00 2001 From: as673366 Date: Tue, 23 Dec 2025 12:55:48 +0530 Subject: [PATCH 4/8] adding readme with in depth details --- .../gateway/otk/README-DUAL-GATEWAY-OTK.md | 498 ++++++++++++++++++ pkg/gateway/reconcile/externalkeys.go | 19 +- pkg/gateway/reconcile/l7otkcertificates.go | 181 ++++++- 3 files changed, 682 insertions(+), 16 deletions(-) create mode 100644 example/gateway/otk/README-DUAL-GATEWAY-OTK.md diff --git a/example/gateway/otk/README-DUAL-GATEWAY-OTK.md b/example/gateway/otk/README-DUAL-GATEWAY-OTK.md new file mode 100644 index 00000000..7e42ea61 --- /dev/null +++ b/example/gateway/otk/README-DUAL-GATEWAY-OTK.md @@ -0,0 +1,498 @@ +# Dual Gateway OTK Configuration Guide + +This guide describes how to configure a Dual Gateway OAuth Toolkit (OTK) deployment using the Layer7 Gateway Operator. In a dual gateway setup, one gateway acts as the DMZ (Demilitarized Zone) gateway and another acts as the Internal gateway, providing enhanced security and separation of concerns. + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Configuration Overview](#configuration-overview) +- [Step 1: Create Required Secrets](#step-1-create-required-secrets) +- [Step 2: Configure DMZ Gateway](#step-2-configure-dmz-gateway) +- [Step 3: Configure Internal Gateway](#step-3-configure-internal-gateway) +- [Key Configuration Fields](#key-configuration-fields) +- [Deployment](#deployment) +- [Certificate Synchronization](#certificate-synchronization) +- [External Gateway Support](#external-gateway-support) +- [Troubleshooting](#troubleshooting) + +## Overview + +The Dual Gateway OTK deployment consists of: + +- **DMZ Gateway**: Handles external client requests and acts as the OAuth authorization server +- **Internal Gateway**: Handles token validation and resource server operations + +The operator automatically synchronizes certificates and keys between the two gateways, ensuring secure communication and proper OAuth flow. + +## Configuration Overview + +The dual gateway setup requires: + +1. **TLS Secrets**: For DMZ and Internal gateway keys/certificates +2. **Auth Secrets**: For gateway authentication credentials +3. **DMZ Gateway Configuration**: With `type: dmz` +4. **Internal Gateway Configuration**: With `type: internal` + +## Step 1: Create Required Secrets + +### Create TLS Secrets + +You need to create TLS secrets for both DMZ and Internal gateways. These secrets must be of type `kubernetes.io/tls` and contain: +- `tls.crt`: The certificate +- `tls.key`: The private key + +#### Option 1: Using the Provided Script + +A helper script is available to generate self-signed certificates and create all required secrets: + +```bash +cd example/gateway/otk/secrets +./create-secrets.sh +``` + +This script creates: +- `otk-dmz-tls-secret`: TLS secret for DMZ gateway +- `otk-internal-tls-secret`: TLS secret for Internal gateway +- `otk-dmz-auth-secret`: Authentication secret for DMZ gateway (username: `admin`, password: `7layer`) +- `otk-internal-auth-secret`: Authentication secret for Internal gateway (username: `admin`, password: `7layer`) + +#### Option 2: Manual Secret Creation + +**DMZ TLS Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-dmz-tls-secret + namespace: default +type: kubernetes.io/tls +data: + tls.crt: + tls.key: +``` + +**Internal TLS Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-internal-tls-secret + namespace: default +type: kubernetes.io/tls +data: + tls.crt: + tls.key: +``` + +**DMZ Auth Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-dmz-auth-secret + namespace: default +type: Opaque +stringData: + SSG_ADMIN_USERNAME: admin + SSG_ADMIN_PASSWORD: 7layer +``` + +**Internal Auth Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-internal-auth-secret + namespace: default +type: Opaque +stringData: + SSG_ADMIN_USERNAME: admin + SSG_ADMIN_PASSWORD: 7layer +``` + +## Step 2: Configure DMZ Gateway + +The DMZ gateway configuration should include: + +- `otk.type: dmz` +- Reference to Internal gateway +- DMZ key secret reference +- Internal auth secret for communication with Internal gateway +- Database configuration + +### Sample DMZ Gateway Configuration + +```yaml +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-dmz +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-dmz-tls-secret + enabled: true + alias: otk-dmz-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: dmz + # Reference to Internal gateway (can be Gateway name or external hostname) + internalGatewayReference: otk-ssg-internal + # InternalGatewayPort is used when the Internal gateway is external + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + internalGatewayPort: 9443 + # SyncIntervalSeconds determines how often certificates are synchronized + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for DMZ key + dmzKeySecret: otk-dmz-tls-secret + # Auth secret for Internal gateway communication + internalAuthSecret: otk-internal-auth-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + gateway: + username: otk_user + password: otkUserPass + readOnly: + username: readonly_user + password: readonly_userPass + admin: + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + management: + secretName: gateway-secret + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + dynamicSyncPort: 9443 + cluster: + hostname: gateway.brcmlabs.com + service: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP +``` + +## Step 3: Configure Internal Gateway + +The Internal gateway configuration should include: + +- `otk.type: internal` +- Reference to DMZ gateway +- Internal key secret reference +- DMZ auth secret for communication with DMZ gateway +- Database configuration (shared with DMZ) + +### Sample Internal Gateway Configuration + +```yaml +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-internal +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-internal-tls-secret + enabled: true + alias: otk-internal-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: internal + # Reference to DMZ gateway (can be Gateway name or external hostname) + dmzGatewayReference: otk-ssg-dmz + # DmzGatewayPort is used when the DMZ gateway is external + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + dmzGatewayPort: 9443 + # SyncIntervalSeconds determines how often certificates are synchronized + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for Internal key + internalKeySecret: otk-internal-tls-secret + # Auth secret for DMZ gateway communication + dmzAuthSecret: otk-dmz-auth-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + gateway: + username: otk_user + password: otkUserPass + readOnly: + username: readonly_user + password: readonly_userPass + admin: + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + management: + secretName: gateway-secret + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + cluster: + hostname: gateway.brcmlabs.com + service: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP +``` + +## Key Configuration Fields + +### OTK-Specific Fields + +| Field | Description | Required | Default | +|-------|-------------|----------|---------| +| `otk.enabled` | Enable OTK installation | Yes | `false` | +| `otk.type` | OTK type: `dmz`, `internal`, or `single` | Yes | - | +| `otk.initContainerImage` | OTK init container image | Yes | - | +| `otk.dmzKeySecret` | Reference to TLS secret containing DMZ key/cert | Yes (DMZ) | - | +| `otk.internalKeySecret` | Reference to TLS secret containing Internal key/cert | Yes (Internal) | - | +| `otk.dmzAuthSecret` | Reference to secret with DMZ gateway credentials | Yes (Internal) | - | +| `otk.internalAuthSecret` | Reference to secret with Internal gateway credentials | Yes (DMZ) | - | +| `otk.dmzGatewayReference` | Reference to DMZ gateway (name or hostname) | Yes (Internal) | - | +| `otk.internalGatewayReference` | Reference to Internal gateway (name or hostname) | Yes (DMZ) | - | +| `otk.dmzGatewayPort` | Port for DMZ gateway (when external) | No | `9443` or `graphmanDynamicSync` port | +| `otk.internalGatewayPort` | Port for Internal gateway (when external) | No | `9443` or `graphmanDynamicSync` port | +| `otk.syncIntervalSeconds` | Certificate sync interval in seconds | No | `RuntimeSyncIntervalSeconds` or `10` | +| `otk.port` | OTK port (defaults to 8443) | No | `8443` | + +### External Keys Configuration + +Both gateways must have `externalKeys` configured with the `otk: true` flag: + +```yaml +externalKeys: +- name: otk-dmz-tls-secret # or otk-internal-tls-secret + enabled: true + alias: otk-dmz-key # or otk-internal-key + keyUsageType: SSL + otk: true # Required for OTK key handling +``` + +## Deployment + +### 1. Create Secrets + +```bash +cd example/gateway/otk/secrets +./create-secrets.sh default +``` + +### 2. Deploy DMZ Gateway + +```bash +kubectl apply -f example/gateway/otk/otk-ssg-dmz.yaml +``` + +### 3. Deploy Internal Gateway + +```bash +kubectl apply -f example/gateway/otk/otk-ssg-internal.yaml +``` + +### 4. Verify Deployment + +```bash +# Check gateway pods +kubectl get pods -l app=gateway + +# Check gateway status +kubectl get gateway otk-ssg-dmz +kubectl get gateway otk-ssg-internal + +# Check logs +kubectl logs -l app=gateway,gateway-name=otk-ssg-dmz +kubectl logs -l app=gateway,gateway-name=otk-ssg-internal +``` + +## Certificate Synchronization + +The operator automatically synchronizes certificates between DMZ and Internal gateways: + +1. **DMZ Certificate → Internal Gateway**: When the DMZ certificate is updated, it's automatically published to the Internal gateway as a trusted certificate and used for FIP (Federated Identity Provider) user creation. + +2. **Internal Certificate → DMZ Gateway**: When the Internal certificate is updated, it's automatically published to the DMZ gateway as a trusted certificate. + +3. **Sync Interval**: Controlled by `syncIntervalSeconds` (default: 10 seconds or `RuntimeSyncIntervalSeconds`). + +4. **Key Updates**: When DMZ or Internal keys are updated: + - The key is synchronized to the respective gateway + - The DMZ private key name is updated in the cluster-wide property `otk.dmz.private_key.name` (DMZ gateway only) + - Old certificates are removed before new ones are published + +## External Gateway Support + +The operator supports scenarios where one or both gateways are external (not managed by the operator): + +### External DMZ Gateway + +If the DMZ gateway is external, configure the Internal gateway with: + +```yaml +otk: + type: internal + dmzGatewayReference: external-dmz-gateway.example.com + dmzGatewayPort: 9443 # Port for Graphman API + dmzAuthSecret: otk-dmz-auth-secret +``` + +### External Internal Gateway + +If the Internal gateway is external, configure the DMZ gateway with: + +```yaml +otk: + type: dmz + internalGatewayReference: external-internal-gateway.example.com + internalGatewayPort: 9443 # Port for Graphman API + internalAuthSecret: otk-internal-auth-secret +``` + +### External Gateway Requirements + +- Graphman API must be enabled and accessible +- Authentication credentials must be provided via auth secrets +- The correct port must be specified if different from default (9443) +- The gateway must be reachable from the operator's network + +## Troubleshooting + +### Common Issues + +1. **Certificates not synchronizing** + - Verify `syncIntervalSeconds` is set appropriately + - Check that Graphman is enabled on both gateways + - Verify auth secrets are correctly configured + - Check operator logs for errors + +2. **Gateway communication failures** + - Verify gateway references are correct (name or hostname) + - Check network connectivity between gateways + - Verify ports are correctly configured + - Ensure auth secrets contain valid credentials + +3. **Key update failures** + - Verify TLS secrets are of type `kubernetes.io/tls` + - Check that secrets contain both `tls.crt` and `tls.key` + - Ensure `externalKeys` have `otk: true` flag + - Verify alias matches the expected value + +4. **Database connection issues** + - Verify database credentials in `otk.database.auth` + - Check JDBC URL is correct and accessible + - Ensure database exists or `create: true` is set + - Verify database wait timeout is sufficient + +### Checking Certificate Sync Status + +```bash +# Check annotations for certificate thumbprints +kubectl get gateway otk-ssg-dmz -o jsonpath='{.metadata.annotations}' +kubectl get gateway otk-ssg-internal -o jsonpath='{.metadata.annotations}' + +# Check cluster-wide properties +kubectl exec -it -- /opt/SecureSpan/Gateway/node/default/bin/ssgconfig \ + get cluster-wide-properties | grep otk.dmz.private_key.name +``` + +### Operator Logs + +```bash +# View operator logs +kubectl logs -n -l control-plane=controller-manager + +# Filter for OTK-related logs +kubectl logs -n -l control-plane=controller-manager | grep -i otk +``` + +## Additional Resources + +- [Layer7 Gateway Operator Documentation](https://github.com/broadcom/layer7-operator) +- [OAuth Toolkit Documentation](https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/11-1.html) +- Example configurations: `example/gateway/otk/` + +--- + +**Note**: This configuration guide assumes you have a working Kubernetes cluster with the Layer7 Gateway Operator installed. Adjust namespaces, hostnames, and other values according to your environment. + diff --git a/pkg/gateway/reconcile/externalkeys.go b/pkg/gateway/reconcile/externalkeys.go index 8a76eca1..a19e3300 100644 --- a/pkg/gateway/reconcile/externalkeys.go +++ b/pkg/gateway/reconcile/externalkeys.go @@ -228,7 +228,7 @@ func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Ga Crt: string(certData), Key: string(keyData), Alias: "otk-dmz-key", - UsageType: "SSL", + UsageType: "", }, } @@ -257,13 +257,13 @@ func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Ga // Check if DMZ key was already applied (to determine if update is needed) keyWasUpdated := false + currentSha1Sum := "" if !gateway.Spec.App.Management.Database.Enabled { podList, err := getGatewayPods(ctx, params) if err != nil { return err } // Check current annotation value before update - currentSha1Sum := "" for _, pod := range podList.Items { if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { currentSha1Sum = val @@ -275,29 +275,30 @@ func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Ga return err } // Key was updated if sha1Sum changed - keyWasUpdated = (currentSha1Sum != sha1Sum) + keyWasUpdated = currentSha1Sum != sha1Sum } else { gatewayDeployment, err := getGatewayDeployment(ctx, params) if err != nil { return err } // Check current annotation value before update - currentSha1Sum := gatewayDeployment.ObjectMeta.Annotations[annotation] + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] err = ReconcileDBGateway(ctx, params, "otk dmz key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) if err != nil { return err } // Key was updated if sha1Sum changed (ReconcileDBGateway returns early if already applied) - keyWasUpdated = (currentSha1Sum != sha1Sum) + keyWasUpdated = currentSha1Sum != sha1Sum } - // Update cluster property only if DMZ key was updated - if keyWasUpdated { + // Update cluster property if DMZ key was updated OR if this is the first reconciliation (currentSha1Sum is empty) + // This ensures the CWP is set on first reconciliation, not just on key updates + if keyWasUpdated || currentSha1Sum == "" { if err := updateDmzPrivateKeyClusterProperty(ctx, params, gateway, "otk-dmz-key"); err != nil { params.Log.V(2).Info("Failed to update DMZ private key cluster property", "error", err, "gateway", gateway.Name) // Don't fail the entire operation if cluster property update fails } - } else if !keyWasUpdated { + } else { params.Log.V(2).Info("DMZ key was not updated, skipping cluster property update", "gateway", gateway.Name) } @@ -340,7 +341,7 @@ func updateInternalWithKey(ctx context.Context, params Params, gateway *security Crt: string(certData), Key: string(keyData), Alias: "otk-internal-key", - UsageType: "SSL", + UsageType: "", }, } diff --git a/pkg/gateway/reconcile/l7otkcertificates.go b/pkg/gateway/reconcile/l7otkcertificates.go index d8703d70..04de4470 100644 --- a/pkg/gateway/reconcile/l7otkcertificates.go +++ b/pkg/gateway/reconcile/l7otkcertificates.go @@ -26,10 +26,12 @@ package reconcile import ( + "bytes" "context" "crypto/sha1" "crypto/x509" "encoding/base64" + "encoding/hex" "encoding/json" "encoding/pem" "fmt" @@ -136,7 +138,7 @@ func publishDmzCertificatesToInternal(ctx context.Context, params Params, gatewa Crt: string(certData), Key: string(keyData), Alias: "otk-dmz-key", - UsageType: "SSL", + UsageType: "", }, } @@ -217,7 +219,7 @@ func publishInternalCertificatesToDmz(ctx context.Context, params Params, gatewa Crt: string(certData), Key: string(keyData), Alias: "otk-internal-key", - UsageType: "SSL", + UsageType: "", }, } @@ -274,9 +276,82 @@ func publishDmzCertToInternal(ctx context.Context, params Params, gateway *secur return fmt.Errorf("invalid certificate format") } + // Before adding new certs, remove existing ones if they were previously applied + // Check if certs were previously applied by checking the annotation + annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" + thumbprintAnnotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates-thumbprints" + previousCertChecksum := "" + var oldThumbprints []string + if !isExternalGateway { + if !internalGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err == nil { + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + previousCertChecksum = val + } + if val, ok := pod.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + // Parse comma-separated thumbprints + oldThumbprints = strings.Split(val, ",") + } + if previousCertChecksum != "" { + break + } + } + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err == nil { + previousCertChecksum = gatewayDeployment.ObjectMeta.Annotations[annotation] + if val, ok := gatewayDeployment.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + oldThumbprints = strings.Split(val, ",") + } + } + } + } + bundle := graphman.Bundle{} - // Add to TrustedCerts + // If we have old thumbprints, add deletion mappings before adding new certs + if len(oldThumbprints) > 0 && previousCertChecksum != "" { + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + for _, thumbprint := range oldThumbprints { + thumbprint = strings.TrimSpace(thumbprint) + if thumbprint != "" { + bundle.Properties.Mappings.TrustedCerts = append(bundle.Properties.Mappings.TrustedCerts, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{ThumbprintSha1: thumbprint}, + }) + } + } + // Also remove old FIP users with the same names + // We'll identify them by the cert CommonName pattern + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + // Remove FIP user by name (CommonName) + bundle.Properties.Mappings.FipUsers = append(bundle.Properties.Mappings.FipUsers, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{Name: crtX509.Subject.CommonName}, + }) + } + } + + // Calculate thumbprints for new certs and add to TrustedCerts + var newThumbprints []string for _, certStr := range crtStrings { if certStr == "" { continue @@ -290,9 +365,19 @@ func publishDmzCertToInternal(ctx context.Context, params Params, gateway *secur continue } + // Calculate thumbprint for this cert + thumbprint, err := calculateCertThumbprint(crtX509.Raw) + if err != nil { + params.Log.V(2).Info("Failed to calculate cert thumbprint", "error", err, "cert", crtX509.Subject.CommonName) + thumbprint = "" // Continue without thumbprint + } else { + newThumbprints = append(newThumbprints, thumbprint) + } + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ Name: crtX509.Subject.CommonName, CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + ThumbprintSha1: thumbprint, TrustAnchor: true, VerifyHostname: false, RevocationCheckPolicyType: "USE_DEFAULT", @@ -370,8 +455,6 @@ func publishDmzCertToInternal(ctx context.Context, params Params, gateway *secur return err } - annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" - internalParams := params internalParams.Instance = internalGateway @@ -439,9 +522,66 @@ func publishInternalCertToDmz(ctx context.Context, params Params, gateway *secur return fmt.Errorf("invalid certificate format") } + // Before adding new certs, remove existing ones if they were previously applied + // Check if certs were previously applied by checking the annotation + annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + thumbprintAnnotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates-thumbprints" + previousCertChecksum := "" + var oldThumbprints []string + if !isExternalGateway { + if !dmzGateway.Spec.App.Management.Database.Enabled { + dmzParams := params + dmzParams.Instance = dmzGateway + podList, err := getGatewayPods(ctx, dmzParams) + if err == nil { + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + previousCertChecksum = val + } + if val, ok := pod.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + // Parse comma-separated thumbprints + oldThumbprints = strings.Split(val, ",") + } + if previousCertChecksum != "" { + break + } + } + } + } else { + dmzParams := params + dmzParams.Instance = dmzGateway + gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) + if err == nil { + previousCertChecksum = gatewayDeployment.ObjectMeta.Annotations[annotation] + if val, ok := gatewayDeployment.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + oldThumbprints = strings.Split(val, ",") + } + } + } + } + bundle := graphman.Bundle{} - // Add to TrustedCerts + // If we have old thumbprints, add deletion mappings before adding new certs + if len(oldThumbprints) > 0 && previousCertChecksum != "" { + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + for _, thumbprint := range oldThumbprints { + thumbprint = strings.TrimSpace(thumbprint) + if thumbprint != "" { + bundle.Properties.Mappings.TrustedCerts = append(bundle.Properties.Mappings.TrustedCerts, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{ThumbprintSha1: thumbprint}, + }) + } + } + } + + // Calculate thumbprints for new certs and add to TrustedCerts + var newThumbprints []string for _, certStr := range crtStrings { if certStr == "" { continue @@ -455,9 +595,19 @@ func publishInternalCertToDmz(ctx context.Context, params Params, gateway *secur continue } + // Calculate thumbprint for this cert + thumbprint, err := calculateCertThumbprint(crtX509.Raw) + if err != nil { + params.Log.V(2).Info("Failed to calculate cert thumbprint", "error", err, "cert", crtX509.Subject.CommonName) + thumbprint = "" // Continue without thumbprint + } else { + newThumbprints = append(newThumbprints, thumbprint) + } + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ Name: crtX509.Subject.CommonName, CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + ThumbprintSha1: thumbprint, TrustAnchor: true, VerifyHostname: false, RevocationCheckPolicyType: "USE_DEFAULT", @@ -493,7 +643,8 @@ func publishInternalCertToDmz(ctx context.Context, params Params, gateway *secur return err } - annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + // annotation is already declared above, reuse it + // annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" dmzParams := params dmzParams.Instance = dmzGateway @@ -781,3 +932,19 @@ func syncInternalCertToExternalDmzGateway(ctx context.Context, params Params, ga return nil } + +// calculateCertThumbprint calculates the SHA1 thumbprint of a certificate in the format expected by Graphman +// Format: base64-encoded hex string of SHA1 fingerprint +func calculateCertThumbprint(rawCert []byte) (string, error) { + fingerprint := sha1.Sum(rawCert) + var buf bytes.Buffer + for _, f := range fingerprint { + fmt.Fprintf(&buf, "%02X", f) + } + hexDump, err := hex.DecodeString(buf.String()) + if err != nil { + return "", err + } + buf.Reset() + return base64.StdEncoding.EncodeToString(hexDump), nil +} From 1846943f776ec17f8e2c0632cdeb7ea7139c22dc Mon Sep 17 00:00:00 2001 From: as673366 Date: Tue, 23 Dec 2025 14:37:47 +0530 Subject: [PATCH 5/8] some additional changes wrt customization repos --- example/gateway/otk/otk-ssg-dmz.yaml | 36 +++++++++++++++++++++-- example/gateway/otk/otk-ssg-internal.yaml | 36 +++++++++++++++++++++-- example/repositories/kustomization.yaml | 2 ++ pkg/util/graphman.go | 10 +++++++ 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/example/gateway/otk/otk-ssg-dmz.yaml b/example/gateway/otk/otk-ssg-dmz.yaml index 9bae180e..967b2eed 100644 --- a/example/gateway/otk/otk-ssg-dmz.yaml +++ b/example/gateway/otk/otk-ssg-dmz.yaml @@ -84,8 +84,40 @@ spec: databaseWaitTimeout: 60 autoscaling: enabled: false - bundle: [] - repositoryReferences: [] + bundle: + - type: restman + source: secret + name: restman-bootstrap-bundle + - type: graphman + source: secret + name: graphman-bootstrap-bundle + repositoryReferences: + - name: l7-gw-myframework + enabled: true + type: static + encryption: + existingSecret: graphman-encryption-secret + key: FRAMEWORK_ENCRYPTION_PASSPHRASE + - name: l7-gw-myapis + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: APIS_ENCRYPTION_PASSPHRASE + - name: l7-gw-mysubscriptions + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: SUBSCRIPTIONS_ENCRYPTION_PASSPHRASE + - name: local-reference-repository + enabled: true + type: dynamic + encryption: { } + - name: otk-customizations-dmz + enabled: true + type: dynamic + encryption: { } bootstrap: script: enabled: true diff --git a/example/gateway/otk/otk-ssg-internal.yaml b/example/gateway/otk/otk-ssg-internal.yaml index 786d60d4..c0cd32dd 100644 --- a/example/gateway/otk/otk-ssg-internal.yaml +++ b/example/gateway/otk/otk-ssg-internal.yaml @@ -78,8 +78,40 @@ spec: databaseWaitTimeout: 60 autoscaling: enabled: false - bundle: [] - repositoryReferences: [] + bundle: + - type: restman + source: secret + name: restman-bootstrap-bundle + - type: graphman + source: secret + name: graphman-bootstrap-bundle + repositoryReferences: + - name: l7-gw-myframework + enabled: true + type: static + encryption: + existingSecret: graphman-encryption-secret + key: FRAMEWORK_ENCRYPTION_PASSPHRASE + - name: l7-gw-myapis + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: APIS_ENCRYPTION_PASSPHRASE + - name: l7-gw-mysubscriptions + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: SUBSCRIPTIONS_ENCRYPTION_PASSPHRASE + - name: local-reference-repository + enabled: true + type: dynamic + encryption: { } + - name: otk-customizations-internal + enabled: true + type: dynamic + encryption: { } bootstrap: script: enabled: true diff --git a/example/repositories/kustomization.yaml b/example/repositories/kustomization.yaml index 46194a77..834d6c23 100644 --- a/example/repositories/kustomization.yaml +++ b/example/repositories/kustomization.yaml @@ -6,4 +6,6 @@ resources: - ./apis-repository.yaml - ./local-reference-repository.yaml - ./otk-customizations-single.yaml + - ./otk-customizations-dmz.yaml + - ./otk-customizations-internal.yaml - ../base diff --git a/pkg/util/graphman.go b/pkg/util/graphman.go index cabd1049..02e0c799 100644 --- a/pkg/util/graphman.go +++ b/pkg/util/graphman.go @@ -595,6 +595,11 @@ func BuildOtkOverrideBundle(mode string, gatewayHost string, otkPort int) ([]byt Soap: false, }) } + bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ + Name: "otk.port", + Value: strconv.Itoa(otkPort), + Description: "OTK Port", + }) } bundle.FederatedIdps = append(bundle.FederatedIdps, &graphman.FederatedIdpInput{ @@ -629,6 +634,11 @@ func BuildOtkOverrideBundle(mode string, gatewayHost string, otkPort int) ([]byt Soap: false, }) } + bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ + Name: "otk.port", + Value: strconv.Itoa(otkPort), + Description: "OTK Port", + }) } case "SINGLE": bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ From bec7421b5bfa60312bf691e63b22bab37058ebca Mon Sep 17 00:00:00 2001 From: as673366 Date: Fri, 23 Jan 2026 12:33:25 +0530 Subject: [PATCH 6/8] adding dmz & internal customization repo & commenting reference as of now --- example/repositories/kustomization.yaml | 4 ++-- example/repositories/otk-customizations-dmz.yaml | 10 ++++++++++ example/repositories/otk-customizations-internal.yaml | 10 ++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 example/repositories/otk-customizations-dmz.yaml create mode 100644 example/repositories/otk-customizations-internal.yaml diff --git a/example/repositories/kustomization.yaml b/example/repositories/kustomization.yaml index 834d6c23..903aa370 100644 --- a/example/repositories/kustomization.yaml +++ b/example/repositories/kustomization.yaml @@ -6,6 +6,6 @@ resources: - ./apis-repository.yaml - ./local-reference-repository.yaml - ./otk-customizations-single.yaml - - ./otk-customizations-dmz.yaml - - ./otk-customizations-internal.yaml +# - ./otk-customizations-dmz.yaml +# - ./otk-customizations-internal.yaml - ../base diff --git a/example/repositories/otk-customizations-dmz.yaml b/example/repositories/otk-customizations-dmz.yaml new file mode 100644 index 00000000..0b9b755b --- /dev/null +++ b/example/repositories/otk-customizations-dmz.yaml @@ -0,0 +1,10 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Repository +metadata: + name: otk-customizations-dmz +spec: + enabled: true + endpoint: https://github.com/Layer7-Community/otk-customizations-dmz + branch: main + type: git + auth: {} \ No newline at end of file diff --git a/example/repositories/otk-customizations-internal.yaml b/example/repositories/otk-customizations-internal.yaml new file mode 100644 index 00000000..2cfca944 --- /dev/null +++ b/example/repositories/otk-customizations-internal.yaml @@ -0,0 +1,10 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Repository +metadata: + name: otk-customizations-internal +spec: + enabled: true + endpoint: https://github.com/Layer7-Community/otk-customizations-internal + branch: main + type: git + auth: {} \ No newline at end of file From bf051ea954e19b58884a2ca92bdda206cb8376ea Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Fri, 13 Mar 2026 07:23:17 +0000 Subject: [PATCH 7/8] removing old ci actions --- .github/check_image_tags.sh | 13 ------------- .github/workflows/ci.yaml | 38 ------------------------------------- .github/workflows/pr.yaml | 38 ------------------------------------- 3 files changed, 89 deletions(-) delete mode 100755 .github/check_image_tags.sh delete mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/pr.yaml diff --git a/.github/check_image_tags.sh b/.github/check_image_tags.sh deleted file mode 100755 index e39b5e8c..00000000 --- a/.github/check_image_tags.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -## Determine if image tags already exist -## Do not allow overwrites -function tag_exists() { - IMAGE_TAG_BASE=${1#*/} - curl -s -f -lSL https://hub.docker.com/v2/repositories/${IMAGE_TAG_BASE}/tags/$2 &> /dev/null -} - -if tag_exists $1 $2; then - echo tag $2 already exists on $1, exiting. - exit 1 -fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 7a6c4940..00000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build Images -on: - push: - branches: - - main - - develop - - experimental -jobs: - build-and-push-images: - name: Build and Push Images - runs-on: ubuntu-latest - env: - VERSION: ${{ github.ref_name }} - IMAGE_TAG_BASE: docker.io/layer7api/layer7-operator - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Prepare Bases - run: make version - - name: Build and push Operator - run: make docker-build docker-push - - name: Build and push Operator Bundle - run: make bundle-build bundle-push \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml deleted file mode 100644 index 8f52953b..00000000 --- a/.github/workflows/pr.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build PR Images -on: - pull_request: - branches: - - main - - develop - - experimental -jobs: - build-and-push-images: - name: Build and Push Images - runs-on: ubuntu-latest - env: - VERSION: ${GITHUB_HEAD_REF} - IMAGE_TAG_BASE: docker.io/layer7api/layer7-operator - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Prepare Bases - run: make version - - name: Build and push Operator - run: make docker-build docker-push - - name: Build and push Operator Bundle - run: make bundle-build bundle-push \ No newline at end of file From 307b6b12520581e37e97f36d7b1489f87e818ab6 Mon Sep 17 00:00:00 2001 From: Gary Vermeulen Date: Fri, 13 Mar 2026 09:31:16 +0000 Subject: [PATCH 8/8] updated examples --- example/Makefile | 16 ++++++++-------- example/basic/README.md | 4 ++-- example/otel-elastic/README.md | 16 ++++++++-------- example/otel-elastic/collector.yaml | 4 ++-- example/otel-elastic/components/agent.yaml | 4 ++-- example/otel-elastic/components/apm.yaml | 2 +- example/otel-elastic/components/es.yaml | 2 +- example/otel-elastic/components/filebeat.yaml | 2 +- example/otel-elastic/components/kibana.yaml | 2 +- example/otel-elastic/components/metricbeat.yaml | 2 +- .../dashboard/apim-dashboard.ndjson | 4 ++-- example/otel-elastic/instrumentation.yaml | 2 +- example/otel-lgtm/readme.md | 8 ++++---- example/otel-prometheus/README.md | 17 +++++++++-------- 14 files changed, 43 insertions(+), 42 deletions(-) diff --git a/example/Makefile b/example/Makefile index 9548e996..515ccf9e 100644 --- a/example/Makefile +++ b/example/Makefile @@ -150,15 +150,15 @@ otel-elastic-example: install cert-manager open-telemetry elastic @echo "#####################################################\n" cert-manager: - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml @$(MAKE) --silent t=10 wait kubectl wait --for=condition=ready --timeout=600s pod -l app=cert-manager -n cert-manager kubectl wait --for=condition=ready --timeout=600s pod -l app=cainjector -n cert-manager kubectl wait --for=condition=ready --timeout=600s pod -l app=webhook -n cert-manager elastic: - kubectl create -f https://download.elastic.co/downloads/eck/2.8.0/crds.yaml - kubectl apply -f https://download.elastic.co/downloads/eck/2.8.0/operator.yaml + kubectl create -f https://download.elastic.co/downloads/eck/3.3.1/crds.yaml + kubectl apply -f https://download.elastic.co/downloads/eck/3.3.1/operator.yaml @$(MAKE) --silent t=10 wait kubectl wait --for=condition=ready --timeout=600s pod -l control-plane=elastic-operator -n elastic-system kubectl apply -f ./otel-elastic/components @@ -174,7 +174,7 @@ metrics-server: kubectl apply -f ./metrics-server/metrics-server-0-6-3.yaml open-telemetry: - kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.97.1/opentelemetry-operator.yaml + kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml @$(MAKE) --silent t=10 wait kubectl wait --for=condition=ready --timeout=600s pod -l app.kubernetes.io/name=opentelemetry-operator -n opentelemetry-operator-system @@ -195,7 +195,7 @@ prometheus-lgtm: jaeger: -kubectl create namespace observability - kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.44.0/jaeger-operator.yaml -n observability + kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.65.0/jaeger-operator.yaml -n observability @$(MAKE) --silent t=10 wait kubectl wait --for=condition=ready --timeout=600s pod -l name=jaeger-operator -n observability @@ -228,11 +228,11 @@ uninstall: -kubectl delete -f ./otel-prometheus/instrumentation.yaml -kubectl delete -f ./otel-elastic/instrumentation.yaml -kubectl delete -f ./otel-prometheus/observability/jaeger - -kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.97.1/opentelemetry-operator.yaml + -kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml -helm uninstall prometheus -n monitoring -kubectl delete -k ./otel-prometheus/monitoring/grafana/ - -kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml - -kubectl delete -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.44.0/jaeger-operator.yaml -n observability + -kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml + -kubectl delete -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.65.0/jaeger-operator.yaml -n observability -kubectl delete -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml -kubectl delete -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/cloud/deploy.yaml -kubectl delete ns observability diff --git a/example/basic/README.md b/example/basic/README.md index abe2ae75..15a758a3 100644 --- a/example/basic/README.md +++ b/example/basic/README.md @@ -170,8 +170,8 @@ You will see an init container starting with graphman-static-init. Init Containers: graphman-static-init-c1b58adb6d: Container ID: containerd://21924ae85d25437d3634ea5da1415c9bb58d678600f9fd67d4f0b0360857d7c5 - Image: docker.io/layer7api/graphman-static-init:1.0.0 - Image ID: docker.io/layer7api/graphman-static-init@sha256:24189a432c0283845664c6fd54c3e8d9f86ad9d35ef12714bb3a18b7aba85aa4 + Image: docker.io/caapim/graphman-static-init:1.0.4 + Image ID: docker.io/caapim/graphman-static-init@sha256:8cb1035035b18fa9dc2c95e2b584c758e78909b3f615ee5f49dce166e8aae213 Port: Host Port: State: Terminated diff --git a/example/otel-elastic/README.md b/example/otel-elastic/README.md index 1f9544ac..7d744214 100644 --- a/example/otel-elastic/README.md +++ b/example/otel-elastic/README.md @@ -149,7 +149,7 @@ If you use Quickstart you do not need to install/deploy any additional resources The container gateway configuration required for this integration is relatively simple. We will set some environment variables in our OTel [instrumentation](./instrumentation.yaml) that the Otel Agent present on the Container Gateway will use to send logs, traces and metrics to the Otel Collector sidecar. ``` -apiVersion: opentelemetry.io/v1alpha1 +apiVersion: opentelemetry.io/v1beta1 kind: Instrumentation metadata: name: otel-instrumentation @@ -398,7 +398,7 @@ export elasticPass=$(kubectl get secret quickstart-es-elastic-user -o go-templat ``` Create Dashboard ``` -curl -XPOST https://kibana.brcmlabs.com/api/saved_objects/_import?createNewCopies=false -H "kbn-xsrf: true" -k -uelastic:$elasticPass -F "file=@./otel-elastic/dashboard/apim-dashboard.ndjson" +curl -XPOST "https://kibana.brcmlabs.com/api/saved_objects/_import?createNewCopies=false" -H "kbn-xsrf: true" -k -uelastic:$elasticPass -F "file=@./otel-elastic/dashboard/apim-dashboard.ndjson" ``` - wait for all components to be ready @@ -428,7 +428,7 @@ You can now move on to test your gateway deployment! ### Install Cert Manager These steps are based the official documentation for installing Cert-Manager [here](https://cert-manager.io/docs/installation/). Cert-Manager is a pre-requisite for the Open Telemetry Operator. ``` -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml ``` #### View CertManager Components @@ -462,7 +462,7 @@ These steps are based the official documentation for installing Open Telemetry [ - Install the Open Telemetry Operator. ``` -kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.76.1/opentelemetry-operator.yaml +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml ``` ##### View Open Telemetry Components @@ -677,8 +677,8 @@ kubectl describe pods ssg-57d96567cb-n24g9 Init Containers: graphman-static-init-c1b58adb6d: Container ID: containerd://21924ae85d25437d3634ea5da1415c9bb58d678600f9fd67d4f0b0360857d7c5 - Image: docker.io/layer7api/graphman-static-init:1.0.0 - Image ID: docker.io/layer7api/graphman-static-init@sha256:24189a432c0283845664c6fd54c3e8d9f86ad9d35ef12714bb3a18b7aba85aa4 + Image: docker.io/caapim/graphman-static-init:1.0.4 + Image ID: docker.io/caapim/graphman-static-init@sha256:8cb1035035b18fa9dc2c95e2b584c758e78909b3f615ee5f49dce166e8aae213 Port: Host Port: State: Terminated @@ -857,8 +857,8 @@ kubectl delete -f ./example/otel-elastic/collector.yaml kubectl delete -f ./example/otel-elastic/instrumentation.yaml kubectl delete -f https://download.elastic.co/downloads/eck/2.8.0/crds.yaml kubectl delete -f https://download.elastic.co/downloads/eck/2.8.0/operator.yaml -kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.76.1/opentelemetry-operator.yaml -kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml +kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml +kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml kubectl delete -k ./example/repositories/ kubectl delete -f ./example/gateway/otel-elastic-gateway.yaml diff --git a/example/otel-elastic/collector.yaml b/example/otel-elastic/collector.yaml index bb714338..1888f9ad 100644 --- a/example/otel-elastic/collector.yaml +++ b/example/otel-elastic/collector.yaml @@ -3,9 +3,9 @@ kind: OpenTelemetryCollector metadata: name: ssg-eck spec: - image: otel/opentelemetry-collector-contrib:0.97.0 + image: otel/opentelemetry-collector-contrib:0.147.0 mode: sidecar - config: | + config: receivers: otlp: protocols: diff --git a/example/otel-elastic/components/agent.yaml b/example/otel-elastic/components/agent.yaml index a87cf6fc..3f5efdd1 100644 --- a/example/otel-elastic/components/agent.yaml +++ b/example/otel-elastic/components/agent.yaml @@ -3,7 +3,7 @@ kind: Agent metadata: name: elastic-agent spec: - version: 8.8.2 + version: 9.3.1 elasticsearchRefs: - name: quickstart daemonSet: @@ -38,7 +38,7 @@ spec: meta: package: name: kubernetes - version: 0.2.8 + version: 1.35.2 data_stream: namespace: k8s streams: diff --git a/example/otel-elastic/components/apm.yaml b/example/otel-elastic/components/apm.yaml index 092b1f82..95648831 100644 --- a/example/otel-elastic/components/apm.yaml +++ b/example/otel-elastic/components/apm.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server-quickstart spec: - version: 8.8.2 + version: 9.3.1 count: 1 elasticsearchRef: name: quickstart diff --git a/example/otel-elastic/components/es.yaml b/example/otel-elastic/components/es.yaml index 3c966781..5a5236e5 100644 --- a/example/otel-elastic/components/es.yaml +++ b/example/otel-elastic/components/es.yaml @@ -57,4 +57,4 @@ spec: certificate: {} updateStrategy: changeBudget: {} - version: 8.8.2 + version: 9.3.1 diff --git a/example/otel-elastic/components/filebeat.yaml b/example/otel-elastic/components/filebeat.yaml index cfc1347a..f6771a4f 100644 --- a/example/otel-elastic/components/filebeat.yaml +++ b/example/otel-elastic/components/filebeat.yaml @@ -4,7 +4,7 @@ metadata: name: filebeat spec: type: filebeat - version: 8.8.2 + version: 9.3.1 elasticsearchRef: name: quickstart kibanaRef: diff --git a/example/otel-elastic/components/kibana.yaml b/example/otel-elastic/components/kibana.yaml index 7e0062f6..6077a77a 100644 --- a/example/otel-elastic/components/kibana.yaml +++ b/example/otel-elastic/components/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: quickstart spec: - version: 8.8.2 + version: 9.3.1 count: 1 elasticsearchRef: name: quickstart diff --git a/example/otel-elastic/components/metricbeat.yaml b/example/otel-elastic/components/metricbeat.yaml index 16d3216f..b558d318 100644 --- a/example/otel-elastic/components/metricbeat.yaml +++ b/example/otel-elastic/components/metricbeat.yaml @@ -4,7 +4,7 @@ metadata: name: metricbeat spec: type: metricbeat - version: 8.8.2 + version: 9.3.1 elasticsearchRef: name: quickstart kibanaRef: diff --git a/example/otel-elastic/dashboard/apim-dashboard.ndjson b/example/otel-elastic/dashboard/apim-dashboard.ndjson index 5321762c..85cce660 100644 --- a/example/otel-elastic/dashboard/apim-dashboard.ndjson +++ b/example/otel-elastic/dashboard/apim-dashboard.ndjson @@ -1,3 +1,3 @@ -{"attributes":{"allowNoIndex":true,"fieldAttrs":"{\"layer7_service_routing_latency\":{\"count\":5},\"broadcom_totalBackendLatency\":{\"count\":1},\"@timestamp\":{\"count\":1},\"layer7_totalBackendLatency\":{\"count\":2},\"layer7_totalBackendLatencyV1\":{\"count\":1},\"layer7_totalFrontendLatency\":{\"count\":1},\"db.client.connections.usage\":{\"count\":2},\"labels.pool_name\":{\"count\":1},\"labels.state\":{\"count\":1},\"process.runtime.jvm.classes.loaded\":{\"count\":2},\"db.client.connections.pending_requests\":{\"count\":1},\"labels.process_runtime_description\":{\"count\":2},\"process.runtime.jvm.gc.duration\":{\"count\":1},\"layer7_service_total\":{\"count\":1},\"process.runtime.jvm.memory.limit\":{\"count\":2},\"process.runtime.jvm.memory.usage\":{\"count\":1},\"process.runtime.jvm.system.cpu.load_1m\":{\"count\":1},\"layer7_routing_failues\":{\"count\":1},\"layer7_service_success\":{\"count\":1},\"layer7_service_latency\":{\"count\":2},\"labels.serviceName\":{\"count\":1},\"labels.gc\":{\"count\":1},\"labels.method\":{\"count\":1},\"numeric_labels.code\":{\"count\":1},\"process.runtime.jvm.threads.count\":{\"count\":1},\"span.name\":{\"count\":3},\"l7_service_latency\":{\"count\":1},\"l7_service_policy_violations\":{\"count\":1},\"labels.l7_serviceName\":{\"count\":1},\"labels.layer7gw_name\":{\"count\":1}}","fieldFormatMap":"{\"trace.id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"apm/link-to/trace/{{value}}\",\"labelTemplate\":\"{{value}}\"}},\"transaction.id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"apm/link-to/transaction/{{value}}\",\"labelTemplate\":\"{{value}}\"}},\"transaction.duration.us\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"microseconds\",\"outputFormat\":\"asMilliseconds\",\"showSuffix\":true,\"useShortSuffix\":true,\"outputPrecision\":2,\"includeSpaceWithSuffix\":true}}}","fields":"[]","name":"APM","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*","typeMeta":"{}"},"coreMigrationVersion":"8.8.0","created_at":"2024-05-14T07:23:11.980Z","id":"apm_static_index_pattern_id","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-05-14T07:23:11.980Z","version":"WzQyMiwxXQ=="} -{"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"twoLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{\"d3e0d4d0-b1de-465c-8772-806a49213c2b\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"d3e0d4d0-b1de-465c-8772-806a49213c2b\",\"fieldName\":\"labels.layer7gw_name\",\"title\":\"Gateway\",\"selectedOptions\":[],\"enhancements\":{},\"existsSelected\":false}}}"},"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":22,\"h\":11,\"i\":\"675c235a-535c-48d9-a5ed-848810986e5a\"},\"panelIndex\":\"675c235a-535c-48d9-a5ed-848810986e5a\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0\"}],\"state\":{\"visualization\":{\"columns\":[{\"columnId\":\"82dab02b-8829-488e-bd30-aa8c45996590\",\"isTransposed\":false,\"summaryRow\":\"sum\"},{\"columnId\":\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\",\"isTransposed\":false,\"summaryRow\":\"sum\",\"summaryLabel\":\"\"},{\"columnId\":\"ff0032fe-94ec-4bf8-a50f-9905301868f9\",\"isTransposed\":false,\"summaryRow\":\"sum\",\"summaryLabel\":\"\"},{\"columnId\":\"ed1df4b3-0bcd-4059-965d-39bc63eef215\",\"isTransposed\":false}],\"layerId\":\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"layerType\":\"data\",\"paging\":{\"size\":10,\"enabled\":true}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\":{\"columns\":{\"ed1df4b3-0bcd-4059-965d-39bc63eef215\":{\"label\":\"Service Name\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.l7_serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"column\",\"columnId\":\"82dab02b-8829-488e-bd30-aa8c45996590\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]},\"customLabel\":true},\"82dab02b-8829-488e-bd30-aa8c45996590\":{\"label\":\"Policy Violations\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"l7_service_policy_violations\",\"filter\":{\"query\":\"l7_service_policy_violations: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\":{\"label\":\"Success\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"l7_service_success\",\"filter\":{\"query\":\"l7_service_success: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"ff0032fe-94ec-4bf8-a50f-9905301868f9\":{\"label\":\"Routing Failures\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"l7_service_routing_failures\",\"filter\":{\"query\":\"l7_service_routing_failures: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"ed1df4b3-0bcd-4059-965d-39bc63eef215\",\"82dab02b-8829-488e-bd30-aa8c45996590\",\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\",\"ff0032fe-94ec-4bf8-a50f-9905301868f9\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Usage\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":0,\"w\":26,\"h\":11,\"i\":\"0ae50fd8-f924-4207-b546-5d600ac701b3\"},\"panelIndex\":\"0ae50fd8-f924-4207-b546-5d600ac701b3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-cad7aae7-308f-4c4f-abbd-b03792fbec56\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"title\":\"Empty XY chart\",\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"cad7aae7-308f-4c4f-abbd-b03792fbec56\",\"accessors\":[\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\",\"splitAccessor\":\"2bffabd5-d07e-4187-a67d-c1d085ea598a\"}],\"curveType\":\"CURVE_MONOTONE_X\",\"fittingFunction\":\"Zero\",\"emphasizeFitting\":true,\"endValue\":\"Zero\",\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"yTitle\":\"Requests\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"cad7aae7-308f-4c4f-abbd-b03792fbec56\":{\"columns\":{\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"2bffabd5-d07e-4187-a67d-c1d085ea598a\":{\"label\":\"Top 10 values of labels.l7_serviceName\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.l7_serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]}},\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\":{\"label\":\"Requests\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"82148cde-9560-4544-a841-38ccaf7b66f5\"],\"filter\":{\"query\":\"\",\"language\":\"kuery\"},\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"82148cde-9560-4544-a841-38ccaf7b66f5\":{\"label\":\"Maximum of l7_service_attempted\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"l7_service_attempted\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"2bffabd5-d07e-4187-a67d-c1d085ea598a\",\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\",\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\",\"82148cde-9560-4544-a841-38ccaf7b66f5\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Message Rate\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":11,\"w\":22,\"h\":11,\"i\":\"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7\"},\"panelIndex\":\"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-fd66544d-5ee7-4f36-8966-c7aa38197646\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yTitle\":\"Latency (ms)\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"fd66544d-5ee7-4f36-8966-c7aa38197646\",\"accessors\":[\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"c973fd1a-9645-46a1-ad45-483cf1ff0441\",\"splitAccessor\":\"4ef8265b-6ee1-449f-9ced-ae065efc7296\"}],\"showCurrentTimeMarker\":false,\"yLeftScale\":\"linear\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"fd66544d-5ee7-4f36-8966-c7aa38197646\":{\"columns\":{\"c973fd1a-9645-46a1-ad45-483cf1ff0441\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"l7_service_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"l7_service_routing_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"subtract\",\"args\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\"],\"location\":{\"min\":0,\"max\":63},\"text\":\"average(l7_service_latency)-average(l7_service_routing_latency)\"}},\"references\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\"],\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\":{\"label\":\"Frontend\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"average(l7_service_latency)-average(l7_service_routing_latency)\",\"isFormulaBroken\":false},\"references\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\"],\"customLabel\":true},\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\":{\"label\":\"Part of Backend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"l7_service_routing_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\":{\"label\":\"Backend\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"average(l7_service_routing_latency)\",\"isFormulaBroken\":false},\"references\":[\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\"],\"customLabel\":true},\"4ef8265b-6ee1-449f-9ced-ae065efc7296\":{\"label\":\"Top 10 values of labels.l7_serviceName\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.l7_serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]}}},\"columnOrder\":[\"4ef8265b-6ee1-449f-9ced-ae065efc7296\",\"c973fd1a-9645-46a1-ad45-483cf1ff0441\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\",\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\",\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Latency\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":11,\"w\":26,\"h\":11,\"i\":\"d9305573-c761-4802-9c0b-6f6326266125\"},\"panelIndex\":\"d9305573-c761-4802-9c0b-6f6326266125\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yTitle\":\"Number of Messages\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"seriesType\":\"bar\",\"accessors\":[\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"231179ca-b88a-4495-b857-8006bca2b409\"],\"layerType\":\"data\",\"yConfig\":[{\"forAccessor\":\"231179ca-b88a-4495-b857-8006bca2b409\",\"color\":\"#8b8c2b\",\"axisMode\":\"left\"},{\"forAccessor\":\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"color\":\"#2da275\",\"axisMode\":\"left\"},{\"forAccessor\":\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"color\":\"#781032\",\"axisMode\":\"left\"}],\"xAccessor\":\"227364f6-80d8-4d15-89ca-7166dc900804\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\":{\"columns\":{\"227364f6-80d8-4d15-89ca-7166dc900804\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"m\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"231179ca-b88a-4495-b857-8006bca2b409\":{\"label\":\"Routing failues\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"6e76dd16-427c-4267-a977-522b4f1a8eba\"],\"customLabel\":true,\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}}},\"6e76dd16-427c-4267-a977-522b4f1a8eba\":{\"label\":\"Maximum of l7_service_routing_failures\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"l7_service_routing_failures\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}},\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\":{\"label\":\"Policy Violations\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"84104e54-bd82-47f6-b34d-b04a1db60694\"],\"timeScale\":\"s\",\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"84104e54-bd82-47f6-b34d-b04a1db60694\":{\"label\":\"Maximum of l7_service_policy_violations\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"l7_service_policy_violations\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}},\"28a49db2-d0b2-47ea-bd58-12e827e9851d\":{\"label\":\"Success\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"3da518c4-50c1-415f-8248-b11b5be8873f\"],\"timeScale\":\"s\",\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"3da518c4-50c1-415f-8248-b11b5be8873f\":{\"label\":\"Maximum of l7_service_attempted\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"l7_service_attempted\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"227364f6-80d8-4d15-89ca-7166dc900804\",\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"231179ca-b88a-4495-b857-8006bca2b409\",\"6e76dd16-427c-4267-a977-522b4f1a8eba\",\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"84104e54-bd82-47f6-b34d-b04a1db60694\",\"3da518c4-50c1-415f-8248-b11b5be8873f\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Service Status\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":22,\"w\":48,\"h\":15,\"i\":\"868932e9-487a-408f-89fd-14f9324534e9\"},\"panelIndex\":\"868932e9-487a-408f-89fd-14f9324534e9\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-2381ace6-24bb-4e47-b57b-6fe06c980216\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-06696642-3076-4612-8291-189dcb25dd9e\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":false,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Zero\",\"curveType\":\"CURVE_MONOTONE_X\",\"yLeftScale\":\"sqrt\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar\",\"layers\":[{\"layerId\":\"2381ace6-24bb-4e47-b57b-6fe06c980216\",\"accessors\":[\"b13dba51-c5c9-4c20-9212-e8607d048660\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"209b6a96-6641-42d5-b243-cd7a68dc55be\",\"splitAccessor\":\"b8a1138e-65ed-4595-8c72-23c294313fc5\",\"collapseFn\":\"\"},{\"layerId\":\"06696642-3076-4612-8291-189dcb25dd9e\",\"layerType\":\"data\",\"accessors\":[\"74470d07-35e7-40c2-bf78-7bd282323271\"],\"seriesType\":\"line\",\"xAccessor\":\"96a4345f-06e4-4252-9ed1-31ed39ee347c\",\"splitAccessor\":\"8756242d-1a48-4550-9fd5-66057b194995\"}],\"emphasizeFitting\":true,\"yTitle\":\"Connections\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"2381ace6-24bb-4e47-b57b-6fe06c980216\":{\"columns\":{\"b8a1138e-65ed-4595-8c72-23c294313fc5\":{\"label\":\"Top values of labels.pool_name + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool_name\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"b13dba51-c5c9-4c20-9212-e8607d048660\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"labels.state\"]}},\"209b6a96-6641-42d5-b243-cd7a68dc55be\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"b13dba51-c5c9-4c20-9212-e8607d048660\":{\"label\":\"conns\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"db.client.connections.usage\",\"filter\":{\"query\":\"db.client.connections.usage: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"b8a1138e-65ed-4595-8c72-23c294313fc5\",\"209b6a96-6641-42d5-b243-cd7a68dc55be\",\"b13dba51-c5c9-4c20-9212-e8607d048660\"],\"incompleteColumns\":{},\"sampling\":1},\"06696642-3076-4612-8291-189dcb25dd9e\":{\"linkToLayers\":[],\"columns\":{\"74470d07-35e7-40c2-bf78-7bd282323271\":{\"label\":\"pending requests\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"db.client.connections.pending_requests\",\"filter\":{\"query\":\"db.client.connections.pending_requests: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"96a4345f-06e4-4252-9ed1-31ed39ee347c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"8756242d-1a48-4550-9fd5-66057b194995\":{\"label\":\"Top 3 values of labels.pool_name\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool_name\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"74470d07-35e7-40c2-bf78-7bd282323271\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false}}},\"columnOrder\":[\"8756242d-1a48-4550-9fd5-66057b194995\",\"96a4345f-06e4-4252-9ed1-31ed39ee347c\",\"74470d07-35e7-40c2-bf78-7bd282323271\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"DB Connections\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":37,\"w\":22,\"h\":10,\"i\":\"9a54b939-5371-49f6-8888-ab92e165e625\"},\"panelIndex\":\"9a54b939-5371-49f6-8888-ab92e165e625\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a57f2de0-c4c8-4a18-9b3a-533ad5572097\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yLeftExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a57f2de0-c4c8-4a18-9b3a-533ad5572097\",\"accessors\":[\"f0c37a83-1464-4350-bb91-e57c2198b927\",\"cdcae1da-c18f-40a5-aeed-1be00a243149\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\",\"yConfig\":[{\"forAccessor\":\"cdcae1da-c18f-40a5-aeed-1be00a243149\",\"color\":\"#e7664c\"}]}],\"yTitle\":\"CPU utilization (0-100)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a57f2de0-c4c8-4a18-9b3a-533ad5572097\":{\"columns\":{\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"f0c37a83-1464-4350-bb91-e57c2198b927X0\":{\"label\":\"Part of System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.system.cpu.utilization\",\"filter\":{\"query\":\"process.runtime.jvm.system.cpu.utilization: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"f0c37a83-1464-4350-bb91-e57c2198b927X1\":{\"label\":\"Part of System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"multiply\",\"args\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X0\",100],\"location\":{\"min\":0,\"max\":121},\"text\":\"multiply(last_value(process.runtime.jvm.system.cpu.utilization, kql='process.runtime.jvm.system.cpu.utilization: *'),100)\"}},\"references\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X0\"],\"customLabel\":true},\"f0c37a83-1464-4350-bb91-e57c2198b927\":{\"label\":\"System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"multiply(last_value(process.runtime.jvm.system.cpu.utilization, kql='process.runtime.jvm.system.cpu.utilization: *'),100)\",\"isFormulaBroken\":false},\"references\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X1\"],\"customLabel\":true},\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\":{\"label\":\"Part of JVM CPU utilization\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.cpu.utilization\",\"filter\":{\"query\":\"process.runtime.jvm.cpu.utilization: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"cdcae1da-c18f-40a5-aeed-1be00a243149\":{\"label\":\"JVM CPU utilization\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"last_value(process.runtime.jvm.cpu.utilization, kql='process.runtime.jvm.cpu.utilization: *')\",\"isFormulaBroken\":false},\"references\":[\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\"],\"customLabel\":true}},\"columnOrder\":[\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\",\"f0c37a83-1464-4350-bb91-e57c2198b927\",\"f0c37a83-1464-4350-bb91-e57c2198b927X0\",\"f0c37a83-1464-4350-bb91-e57c2198b927X1\",\"cdcae1da-c18f-40a5-aeed-1be00a243149\",\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"CPU/System Utilization\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":37,\"w\":26,\"h\":10,\"i\":\"2a2057e2-ce45-4804-8ce0-3d4ed94679d7\"},\"panelIndex\":\"2a2057e2-ce45-4804-8ce0-3d4ed94679d7\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-66ce3428-e822-4b54-afc8-69bbffd9c106\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\",\"accessors\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"7e7413d7-db3d-4473-b46f-d76132f403b0\",\"splitAccessor\":\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\",\"palette\":{\"type\":\"palette\",\"name\":\"status\"},\"collapseFn\":\"\",\"yConfig\":[]},{\"layerId\":\"b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\",\"layerType\":\"referenceLine\",\"accessors\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81\"],\"yConfig\":[{\"forAccessor\":\"81fcf196-8145-4246-9d3a-f1679cb62c81\",\"axisMode\":\"left\",\"icon\":\"bolt\",\"iconPosition\":\"auto\",\"lineWidth\":2,\"color\":\"#9d2022\"}]},{\"layerId\":\"66ce3428-e822-4b54-afc8-69bbffd9c106\",\"layerType\":\"data\",\"accessors\":[\"323e403c-e4a1-481d-897e-907192f1cc4a\"],\"seriesType\":\"line\",\"xAccessor\":\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\",\"yConfig\":[{\"forAccessor\":\"323e403c-e4a1-481d-897e-907192f1cc4a\",\"color\":\"#040f12\"}]}],\"yLeftExtent\":{\"mode\":\"full\"},\"yTitle\":\"Megabytes\",\"yLeftScale\":\"log\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\":{\"columns\":{\"7e7413d7-db3d-4473-b46f-d76132f403b0\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\":{\"label\":\"Part of Heap\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\":{\"label\":\"Part of Heap\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\",1048576],\"location\":{\"min\":0,\"max\":126},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: heap'), 1048576)\"}},\"references\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\"],\"customLabel\":true},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\":{\"label\":\"Heap\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\"],\"customLabel\":true},\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\":{\"label\":\"Top 10 values of labels.pool\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":true}}},\"columnOrder\":[\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\",\"7e7413d7-db3d-4473-b46f-d76132f403b0\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\"],\"incompleteColumns\":{},\"sampling\":1},\"b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\":{\"linkToLayers\":[],\"columns\":{\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\":{\"label\":\"Part of Max\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.limit\",\"filter\":{\"query\":\"process.runtime.jvm.memory.limit: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\":{\"label\":\"Part of Max\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\",1048576],\"location\":{\"min\":0,\"max\":126},\"text\":\"divide(last_value(process.runtime.jvm.memory.limit, kql='process.runtime.jvm.memory.limit: * and labels.type: heap'), 1048576)\"}},\"references\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\"],\"customLabel\":true},\"81fcf196-8145-4246-9d3a-f1679cb62c81\":{\"label\":\"Max\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.limit, kql='process.runtime.jvm.memory.limit: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"references\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\"],\"customLabel\":true}},\"columnOrder\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81\",\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\",\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\"],\"sampling\":1,\"incompleteColumns\":{}},\"66ce3428-e822-4b54-afc8-69bbffd9c106\":{\"linkToLayers\":[],\"columns\":{\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"323e403c-e4a1-481d-897e-907192f1cc4aX0\":{\"label\":\"Part of Non-Heap\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage: * and labels.type: non_heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"323e403c-e4a1-481d-897e-907192f1cc4aX1\":{\"label\":\"Part of Non-Heap\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX0\",1048576],\"location\":{\"min\":0,\"max\":130},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: non_heap'), 1048576)\"}},\"references\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX0\"],\"customLabel\":true},\"323e403c-e4a1-481d-897e-907192f1cc4a\":{\"label\":\"Non-Heap\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: non_heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX1\"],\"customLabel\":true}},\"columnOrder\":[\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\",\"323e403c-e4a1-481d-897e-907192f1cc4a\",\"323e403c-e4a1-481d-897e-907192f1cc4aX1\",\"323e403c-e4a1-481d-897e-907192f1cc4aX0\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Memory Usage\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":47,\"w\":22,\"h\":11,\"i\":\"542275ac-682e-43b8-b1ac-cf919d206e02\"},\"panelIndex\":\"542275ac-682e-43b8-b1ac-cf919d206e02\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-28f4234a-7411-4d15-8ad9-36ac978f7e19\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yLeftExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"28f4234a-7411-4d15-8ad9-36ac978f7e19\",\"accessors\":[\"04481efd-b3d8-44bc-be6c-5a0af56e241b\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\"}],\"yTitle\":\"System Load (0-100)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"28f4234a-7411-4d15-8ad9-36ac978f7e19\":{\"columns\":{\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\":{\"label\":\"Part of System Load\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.system.cpu.load_1m\",\"filter\":{\"query\":\"process.runtime.jvm.system.cpu.load_1m: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"04481efd-b3d8-44bc-be6c-5a0af56e241b\":{\"label\":\"System Load\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"last_value(process.runtime.jvm.system.cpu.load_1m, kql='process.runtime.jvm.system.cpu.load_1m: *')\",\"isFormulaBroken\":false},\"references\":[\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\"],\"customLabel\":true}},\"columnOrder\":[\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\",\"04481efd-b3d8-44bc-be6c-5a0af56e241b\",\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"System Load\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":47,\"w\":26,\"h\":11,\"i\":\"a3aace75-debc-4370-9954-e5a6e8ac6259\"},\"panelIndex\":\"a3aace75-debc-4370-9954-e5a6e8ac6259\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\",\"accessors\":[\"603e1430-a496-4249-b2da-73a5205b964e\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\",\"splitAccessor\":\"a9ec0126-a45f-4f86-b783-a4820badd7d8\",\"yConfig\":[{\"forAccessor\":\"603e1430-a496-4249-b2da-73a5205b964e\",\"axisMode\":\"left\"}],\"palette\":{\"type\":\"palette\",\"name\":\"status\"}}],\"yLeftScale\":\"log\",\"yTitle\":\"Megabytes\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\":{\"columns\":{\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"603e1430-a496-4249-b2da-73a5205b964eX0\":{\"label\":\"Part of divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage_after_last_gc\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"603e1430-a496-4249-b2da-73a5205b964eX1\":{\"label\":\"Part of divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"603e1430-a496-4249-b2da-73a5205b964eX0\",1048576],\"location\":{\"min\":0,\"max\":154},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\"}},\"references\":[\"603e1430-a496-4249-b2da-73a5205b964eX0\"],\"customLabel\":true},\"603e1430-a496-4249-b2da-73a5205b964e\":{\"label\":\"MB\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"603e1430-a496-4249-b2da-73a5205b964eX1\"],\"filter\":{\"query\":\"\",\"language\":\"kuery\"},\"customLabel\":true},\"a9ec0126-a45f-4f86-b783-a4820badd7d8\":{\"label\":\"Top 10 values of labels.pool\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false}}},\"columnOrder\":[\"a9ec0126-a45f-4f86-b783-a4820badd7d8\",\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\",\"603e1430-a496-4249-b2da-73a5205b964e\",\"603e1430-a496-4249-b2da-73a5205b964eX0\",\"603e1430-a496-4249-b2da-73a5205b964eX1\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Memory after GC\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":58,\"w\":22,\"h\":10,\"i\":\"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367\"},\"panelIndex\":\"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a3ee7019-98bb-45cd-8536-e36a8e58297e\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a3ee7019-98bb-45cd-8536-e36a8e58297e\",\"accessors\":[\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"9c350955-95a1-4726-93fd-717b104a218c\",\"yConfig\":[{\"forAccessor\":\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\",\"color\":\"#54b399\"}]}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a3ee7019-98bb-45cd-8536-e36a8e58297e\":{\"columns\":{\"9c350955-95a1-4726-93fd-717b104a218c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\":{\"label\":\"Threads count\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.threads.count\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"process.runtime.jvm.threads.count: *\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true},\"customLabel\":true}},\"columnOrder\":[\"9c350955-95a1-4726-93fd-717b104a218c\",\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Threads\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":58,\"w\":26,\"h\":10,\"i\":\"4f987373-1b85-4e80-a7cc-c1c000349c5e\"},\"panelIndex\":\"4f987373-1b85-4e80-a7cc-c1c000349c5e\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a45cd02a-4c62-4b34-bdce-c702b87b4246\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yRightTitle\":\"Duration (ms)\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a45cd02a-4c62-4b34-bdce-c702b87b4246\",\"accessors\":[\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"4e7a6993-fcee-47ae-9599-0047809bb29e\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"57050bde-2c14-47d2-9dde-6e35326cf5cf\",\"yConfig\":[{\"forAccessor\":\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"axisMode\":\"left\"},{\"forAccessor\":\"4e7a6993-fcee-47ae-9599-0047809bb29e\",\"color\":\"#e7664c\",\"axisMode\":\"left\"}]}],\"yTitle\":\"Duration (ms)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a45cd02a-4c62-4b34-bdce-c702b87b4246\":{\"columns\":{\"57050bde-2c14-47d2-9dde-6e35326cf5cf\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\":{\"label\":\"Minor GC\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.gc.duration\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.action: \\\"end of minor GC\\\" \",\"language\":\"kuery\"},\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":2}},\"emptyAsNull\":true},\"customLabel\":true},\"4e7a6993-fcee-47ae-9599-0047809bb29e\":{\"label\":\"Major GC\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.gc.duration\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.action : \\\"end of major GC\\\"\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true},\"customLabel\":true}},\"columnOrder\":[\"57050bde-2c14-47d2-9dde-6e35326cf5cf\",\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"4e7a6993-fcee-47ae-9599-0047809bb29e\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Garbage Collection\"}]","timeRestore":false,"title":"Layer7 Gateway Dashboard","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-05-14T07:28:39.093Z","id":"c29a9900-2161-11ee-be6c-57cbef857d54","managed":false,"references":[{"id":"apm_static_index_pattern_id","name":"675c235a-535c-48d9-a5ed-848810986e5a:indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"0ae50fd8-f924-4207-b546-5d600ac701b3:indexpattern-datasource-layer-cad7aae7-308f-4c4f-abbd-b03792fbec56","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7:indexpattern-datasource-layer-fd66544d-5ee7-4f36-8966-c7aa38197646","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"d9305573-c761-4802-9c0b-6f6326266125:indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"868932e9-487a-408f-89fd-14f9324534e9:indexpattern-datasource-layer-2381ace6-24bb-4e47-b57b-6fe06c980216","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"868932e9-487a-408f-89fd-14f9324534e9:indexpattern-datasource-layer-06696642-3076-4612-8291-189dcb25dd9e","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"9a54b939-5371-49f6-8888-ab92e165e625:indexpattern-datasource-layer-a57f2de0-c4c8-4a18-9b3a-533ad5572097","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-04cd0fc2-f652-4583-a7d4-c1a8f6f8818f","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-66ce3428-e822-4b54-afc8-69bbffd9c106","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"542275ac-682e-43b8-b1ac-cf919d206e02:indexpattern-datasource-layer-28f4234a-7411-4d15-8ad9-36ac978f7e19","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"a3aace75-debc-4370-9954-e5a6e8ac6259:indexpattern-datasource-layer-067c1db7-2942-4bf9-8fd6-4ca0a42a23fa","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367:indexpattern-datasource-layer-a3ee7019-98bb-45cd-8536-e36a8e58297e","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"4f987373-1b85-4e80-a7cc-c1c000349c5e:indexpattern-datasource-layer-a45cd02a-4c62-4b34-bdce-c702b87b4246","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"controlGroup_d3e0d4d0-b1de-465c-8772-806a49213c2b:optionsListDataView","type":"index-pattern"}],"type":"dashboard","typeMigrationVersion":"8.7.0","updated_at":"2024-05-14T07:28:39.093Z","version":"WzE2ODksMV0="} +{"attributes":{"allowNoIndex":true,"fieldAttrs":"{\"layer7_service_routing_latency\":{\"count\":5},\"broadcom_totalBackendLatency\":{\"count\":1},\"@timestamp\":{\"count\":1},\"layer7_totalBackendLatency\":{\"count\":2},\"layer7_totalBackendLatencyV1\":{\"count\":1},\"layer7_totalFrontendLatency\":{\"count\":1},\"db.client.connections.usage\":{\"count\":2},\"labels.pool_name\":{\"count\":1},\"labels.state\":{\"count\":1},\"process.runtime.jvm.classes.loaded\":{\"count\":2},\"db.client.connections.pending_requests\":{\"count\":1},\"labels.process_runtime_description\":{\"count\":2},\"process.runtime.jvm.gc.duration\":{\"count\":1},\"layer7_service_total\":{\"count\":1},\"process.runtime.jvm.memory.limit\":{\"count\":2},\"process.runtime.jvm.memory.usage\":{\"count\":1},\"process.runtime.jvm.system.cpu.load_1m\":{\"count\":1},\"layer7_routing_failues\":{\"count\":1},\"layer7_service_success\":{\"count\":1},\"layer7_service_latency\":{\"count\":2},\"labels.serviceName\":{\"count\":1},\"labels.gc\":{\"count\":1},\"labels.method\":{\"count\":1},\"numeric_labels.code\":{\"count\":1},\"process.runtime.jvm.threads.count\":{\"count\":1},\"span.name\":{\"count\":3},\"layer7_service_latency\":{\"count\":1},\"layer7_service_policy_violations\":{\"count\":1},\"labels.serviceName\":{\"count\":1},\"labels.layer7gw_name\":{\"count\":1}}","fieldFormatMap":"{\"trace.id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"apm/link-to/trace/{{value}}\",\"labelTemplate\":\"{{value}}\"}},\"transaction.id\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"apm/link-to/transaction/{{value}}\",\"labelTemplate\":\"{{value}}\"}},\"transaction.duration.us\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"microseconds\",\"outputFormat\":\"asMilliseconds\",\"showSuffix\":true,\"useShortSuffix\":true,\"outputPrecision\":2,\"includeSpaceWithSuffix\":true}}}","fields":"[]","name":"APM","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*","typeMeta":"{}"},"coreMigrationVersion":"8.8.0","created_at":"2024-05-14T07:23:11.980Z","id":"apm_static_index_pattern_id","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-05-14T07:23:11.980Z","version":"WzQyMiwxXQ=="} +{"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"twoLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{\"d3e0d4d0-b1de-465c-8772-806a49213c2b\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"d3e0d4d0-b1de-465c-8772-806a49213c2b\",\"fieldName\":\"labels.layer7gw_name\",\"title\":\"Gateway\",\"selectedOptions\":[],\"enhancements\":{},\"existsSelected\":false}}}"},"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":22,\"h\":11,\"i\":\"675c235a-535c-48d9-a5ed-848810986e5a\"},\"panelIndex\":\"675c235a-535c-48d9-a5ed-848810986e5a\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0\"}],\"state\":{\"visualization\":{\"columns\":[{\"columnId\":\"82dab02b-8829-488e-bd30-aa8c45996590\",\"isTransposed\":false,\"summaryRow\":\"sum\"},{\"columnId\":\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\",\"isTransposed\":false,\"summaryRow\":\"sum\",\"summaryLabel\":\"\"},{\"columnId\":\"ff0032fe-94ec-4bf8-a50f-9905301868f9\",\"isTransposed\":false,\"summaryRow\":\"sum\",\"summaryLabel\":\"\"},{\"columnId\":\"ed1df4b3-0bcd-4059-965d-39bc63eef215\",\"isTransposed\":false}],\"layerId\":\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"layerType\":\"data\",\"paging\":{\"size\":10,\"enabled\":true}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\":{\"columns\":{\"ed1df4b3-0bcd-4059-965d-39bc63eef215\":{\"label\":\"Service Name\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"column\",\"columnId\":\"82dab02b-8829-488e-bd30-aa8c45996590\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]},\"customLabel\":true},\"82dab02b-8829-488e-bd30-aa8c45996590\":{\"label\":\"Policy Violations\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"layer7_service_policy_violations\",\"filter\":{\"query\":\"layer7_service_policy_violations: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\":{\"label\":\"Success\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"layer7_service_success\",\"filter\":{\"query\":\"layer7_service_success: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"ff0032fe-94ec-4bf8-a50f-9905301868f9\":{\"label\":\"Routing Failures\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"layer7_service_routing_failures\",\"filter\":{\"query\":\"layer7_service_routing_failures: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"ed1df4b3-0bcd-4059-965d-39bc63eef215\",\"82dab02b-8829-488e-bd30-aa8c45996590\",\"ae1e2f3d-6ec8-49c9-a444-8249ed271df8\",\"ff0032fe-94ec-4bf8-a50f-9905301868f9\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Usage\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":0,\"w\":26,\"h\":11,\"i\":\"0ae50fd8-f924-4207-b546-5d600ac701b3\"},\"panelIndex\":\"0ae50fd8-f924-4207-b546-5d600ac701b3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-cad7aae7-308f-4c4f-abbd-b03792fbec56\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"title\":\"Empty XY chart\",\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"cad7aae7-308f-4c4f-abbd-b03792fbec56\",\"accessors\":[\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\",\"splitAccessor\":\"2bffabd5-d07e-4187-a67d-c1d085ea598a\"}],\"curveType\":\"CURVE_MONOTONE_X\",\"fittingFunction\":\"Zero\",\"emphasizeFitting\":true,\"endValue\":\"Zero\",\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"yTitle\":\"Requests\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"cad7aae7-308f-4c4f-abbd-b03792fbec56\":{\"columns\":{\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"2bffabd5-d07e-4187-a67d-c1d085ea598a\":{\"label\":\"Top 10 values of labels.serviceName\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]}},\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\":{\"label\":\"Requests\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"82148cde-9560-4544-a841-38ccaf7b66f5\"],\"filter\":{\"query\":\"\",\"language\":\"kuery\"},\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"82148cde-9560-4544-a841-38ccaf7b66f5\":{\"label\":\"Maximum of layer7_service_attempted\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"layer7_service_attempted\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"2bffabd5-d07e-4187-a67d-c1d085ea598a\",\"bf646547-1ad3-4810-8d0c-ac6d35be61ec\",\"4ee0c11f-c34e-48b9-ba2b-3a855498cd44\",\"82148cde-9560-4544-a841-38ccaf7b66f5\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Message Rate\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":11,\"w\":22,\"h\":11,\"i\":\"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7\"},\"panelIndex\":\"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-fd66544d-5ee7-4f36-8966-c7aa38197646\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yTitle\":\"Latency (ms)\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"fd66544d-5ee7-4f36-8966-c7aa38197646\",\"accessors\":[\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"c973fd1a-9645-46a1-ad45-483cf1ff0441\",\"splitAccessor\":\"4ef8265b-6ee1-449f-9ced-ae065efc7296\"}],\"showCurrentTimeMarker\":false,\"yLeftScale\":\"linear\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"fd66544d-5ee7-4f36-8966-c7aa38197646\":{\"columns\":{\"c973fd1a-9645-46a1-ad45-483cf1ff0441\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"layer7_service_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"layer7_service_routing_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\":{\"label\":\"Part of Frontend\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"subtract\",\"args\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\"],\"location\":{\"min\":0,\"max\":63},\"text\":\"average(layer7_service_latency)-average(layer7_service_routing_latency)\"}},\"references\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\"],\"customLabel\":true},\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\":{\"label\":\"Frontend\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"average(layer7_service_latency)-average(layer7_service_routing_latency)\",\"isFormulaBroken\":false},\"references\":[\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\"],\"customLabel\":true},\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\":{\"label\":\"Part of Backend\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"layer7_service_routing_latency\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\":{\"label\":\"Backend\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"average(layer7_service_routing_latency)\",\"isFormulaBroken\":false},\"references\":[\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\"],\"customLabel\":true},\"4ef8265b-6ee1-449f-9ced-ae065efc7296\":{\"label\":\"Top 10 values of labels.serviceName\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.serviceName\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[]}}},\"columnOrder\":[\"4ef8265b-6ee1-449f-9ced-ae065efc7296\",\"c973fd1a-9645-46a1-ad45-483cf1ff0441\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5\",\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4b\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X0\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X2\",\"f4fc3664-ef1d-44f5-b9ce-edb2168d5ce5X1\",\"6da1dadd-1f1a-42d6-b954-e8bd5a318f4bX0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Latency\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":11,\"w\":26,\"h\":11,\"i\":\"d9305573-c761-4802-9c0b-6f6326266125\"},\"panelIndex\":\"d9305573-c761-4802-9c0b-6f6326266125\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yTitle\":\"Number of Messages\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\",\"seriesType\":\"bar\",\"accessors\":[\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"231179ca-b88a-4495-b857-8006bca2b409\"],\"layerType\":\"data\",\"yConfig\":[{\"forAccessor\":\"231179ca-b88a-4495-b857-8006bca2b409\",\"color\":\"#8b8c2b\",\"axisMode\":\"left\"},{\"forAccessor\":\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"color\":\"#2da275\",\"axisMode\":\"left\"},{\"forAccessor\":\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"color\":\"#781032\",\"axisMode\":\"left\"}],\"xAccessor\":\"227364f6-80d8-4d15-89ca-7166dc900804\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"8638d8b0-f850-42f0-9e64-35e6f239e0d0\":{\"columns\":{\"227364f6-80d8-4d15-89ca-7166dc900804\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"m\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"231179ca-b88a-4495-b857-8006bca2b409\":{\"label\":\"Routing failues\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"6e76dd16-427c-4267-a977-522b4f1a8eba\"],\"customLabel\":true,\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}}},\"6e76dd16-427c-4267-a977-522b4f1a8eba\":{\"label\":\"Maximum of layer7_service_routing_failures\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"layer7_service_routing_failures\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}},\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\":{\"label\":\"Policy Violations\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"84104e54-bd82-47f6-b34d-b04a1db60694\"],\"timeScale\":\"s\",\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"84104e54-bd82-47f6-b34d-b04a1db60694\":{\"label\":\"Maximum of layer7_service_policy_violations\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"layer7_service_policy_violations\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}},\"28a49db2-d0b2-47ea-bd58-12e827e9851d\":{\"label\":\"Success\",\"dataType\":\"number\",\"operationType\":\"counter_rate\",\"isBucketed\":false,\"scale\":\"ratio\",\"references\":[\"3da518c4-50c1-415f-8248-b11b5be8873f\"],\"timeScale\":\"s\",\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"3da518c4-50c1-415f-8248-b11b5be8873f\":{\"label\":\"Maximum of layer7_service_attempted\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"layer7_service_attempted\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"227364f6-80d8-4d15-89ca-7166dc900804\",\"28a49db2-d0b2-47ea-bd58-12e827e9851d\",\"231179ca-b88a-4495-b857-8006bca2b409\",\"6e76dd16-427c-4267-a977-522b4f1a8eba\",\"6d6fe07e-c712-4d47-89c6-1ebdeaa23304\",\"84104e54-bd82-47f6-b34d-b04a1db60694\",\"3da518c4-50c1-415f-8248-b11b5be8873f\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Service Status\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":22,\"w\":48,\"h\":15,\"i\":\"868932e9-487a-408f-89fd-14f9324534e9\"},\"panelIndex\":\"868932e9-487a-408f-89fd-14f9324534e9\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-2381ace6-24bb-4e47-b57b-6fe06c980216\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-06696642-3076-4612-8291-189dcb25dd9e\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":false,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Zero\",\"curveType\":\"CURVE_MONOTONE_X\",\"yLeftScale\":\"sqrt\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar\",\"layers\":[{\"layerId\":\"2381ace6-24bb-4e47-b57b-6fe06c980216\",\"accessors\":[\"b13dba51-c5c9-4c20-9212-e8607d048660\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"209b6a96-6641-42d5-b243-cd7a68dc55be\",\"splitAccessor\":\"b8a1138e-65ed-4595-8c72-23c294313fc5\",\"collapseFn\":\"\"},{\"layerId\":\"06696642-3076-4612-8291-189dcb25dd9e\",\"layerType\":\"data\",\"accessors\":[\"74470d07-35e7-40c2-bf78-7bd282323271\"],\"seriesType\":\"line\",\"xAccessor\":\"96a4345f-06e4-4252-9ed1-31ed39ee347c\",\"splitAccessor\":\"8756242d-1a48-4550-9fd5-66057b194995\"}],\"emphasizeFitting\":true,\"yTitle\":\"Connections\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"2381ace6-24bb-4e47-b57b-6fe06c980216\":{\"columns\":{\"b8a1138e-65ed-4595-8c72-23c294313fc5\":{\"label\":\"Top values of labels.pool_name + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool_name\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"b13dba51-c5c9-4c20-9212-e8607d048660\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"labels.state\"]}},\"209b6a96-6641-42d5-b243-cd7a68dc55be\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"b13dba51-c5c9-4c20-9212-e8607d048660\":{\"label\":\"conns\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"db.client.connections.usage\",\"filter\":{\"query\":\"db.client.connections.usage: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"b8a1138e-65ed-4595-8c72-23c294313fc5\",\"209b6a96-6641-42d5-b243-cd7a68dc55be\",\"b13dba51-c5c9-4c20-9212-e8607d048660\"],\"incompleteColumns\":{},\"sampling\":1},\"06696642-3076-4612-8291-189dcb25dd9e\":{\"linkToLayers\":[],\"columns\":{\"74470d07-35e7-40c2-bf78-7bd282323271\":{\"label\":\"pending requests\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"db.client.connections.pending_requests\",\"filter\":{\"query\":\"db.client.connections.pending_requests: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"96a4345f-06e4-4252-9ed1-31ed39ee347c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"8756242d-1a48-4550-9fd5-66057b194995\":{\"label\":\"Top 3 values of labels.pool_name\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool_name\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"74470d07-35e7-40c2-bf78-7bd282323271\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false}}},\"columnOrder\":[\"8756242d-1a48-4550-9fd5-66057b194995\",\"96a4345f-06e4-4252-9ed1-31ed39ee347c\",\"74470d07-35e7-40c2-bf78-7bd282323271\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"DB Connections\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":37,\"w\":22,\"h\":10,\"i\":\"9a54b939-5371-49f6-8888-ab92e165e625\"},\"panelIndex\":\"9a54b939-5371-49f6-8888-ab92e165e625\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a57f2de0-c4c8-4a18-9b3a-533ad5572097\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yLeftExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a57f2de0-c4c8-4a18-9b3a-533ad5572097\",\"accessors\":[\"f0c37a83-1464-4350-bb91-e57c2198b927\",\"cdcae1da-c18f-40a5-aeed-1be00a243149\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\",\"yConfig\":[{\"forAccessor\":\"cdcae1da-c18f-40a5-aeed-1be00a243149\",\"color\":\"#e7664c\"}]}],\"yTitle\":\"CPU utilization (0-100)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a57f2de0-c4c8-4a18-9b3a-533ad5572097\":{\"columns\":{\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"f0c37a83-1464-4350-bb91-e57c2198b927X0\":{\"label\":\"Part of System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.system.cpu.utilization\",\"filter\":{\"query\":\"process.runtime.jvm.system.cpu.utilization: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"f0c37a83-1464-4350-bb91-e57c2198b927X1\":{\"label\":\"Part of System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"multiply\",\"args\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X0\",100],\"location\":{\"min\":0,\"max\":121},\"text\":\"multiply(last_value(process.runtime.jvm.system.cpu.utilization, kql='process.runtime.jvm.system.cpu.utilization: *'),100)\"}},\"references\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X0\"],\"customLabel\":true},\"f0c37a83-1464-4350-bb91-e57c2198b927\":{\"label\":\"System CPU utilization\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"multiply(last_value(process.runtime.jvm.system.cpu.utilization, kql='process.runtime.jvm.system.cpu.utilization: *'),100)\",\"isFormulaBroken\":false},\"references\":[\"f0c37a83-1464-4350-bb91-e57c2198b927X1\"],\"customLabel\":true},\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\":{\"label\":\"Part of JVM CPU utilization\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.cpu.utilization\",\"filter\":{\"query\":\"process.runtime.jvm.cpu.utilization: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"cdcae1da-c18f-40a5-aeed-1be00a243149\":{\"label\":\"JVM CPU utilization\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"last_value(process.runtime.jvm.cpu.utilization, kql='process.runtime.jvm.cpu.utilization: *')\",\"isFormulaBroken\":false},\"references\":[\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\"],\"customLabel\":true}},\"columnOrder\":[\"deb7bb17-0f00-4a18-b2be-e6a5a2becb7e\",\"f0c37a83-1464-4350-bb91-e57c2198b927\",\"f0c37a83-1464-4350-bb91-e57c2198b927X0\",\"f0c37a83-1464-4350-bb91-e57c2198b927X1\",\"cdcae1da-c18f-40a5-aeed-1be00a243149\",\"cdcae1da-c18f-40a5-aeed-1be00a243149X0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"CPU/System Utilization\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":37,\"w\":26,\"h\":10,\"i\":\"2a2057e2-ce45-4804-8ce0-3d4ed94679d7\"},\"panelIndex\":\"2a2057e2-ce45-4804-8ce0-3d4ed94679d7\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\",\"type\":\"index-pattern\"},{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-66ce3428-e822-4b54-afc8-69bbffd9c106\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\",\"accessors\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"7e7413d7-db3d-4473-b46f-d76132f403b0\",\"splitAccessor\":\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\",\"palette\":{\"type\":\"palette\",\"name\":\"status\"},\"collapseFn\":\"\",\"yConfig\":[]},{\"layerId\":\"b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\",\"layerType\":\"referenceLine\",\"accessors\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81\"],\"yConfig\":[{\"forAccessor\":\"81fcf196-8145-4246-9d3a-f1679cb62c81\",\"axisMode\":\"left\",\"icon\":\"bolt\",\"iconPosition\":\"auto\",\"lineWidth\":2,\"color\":\"#9d2022\"}]},{\"layerId\":\"66ce3428-e822-4b54-afc8-69bbffd9c106\",\"layerType\":\"data\",\"accessors\":[\"323e403c-e4a1-481d-897e-907192f1cc4a\"],\"seriesType\":\"line\",\"xAccessor\":\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\",\"yConfig\":[{\"forAccessor\":\"323e403c-e4a1-481d-897e-907192f1cc4a\",\"color\":\"#040f12\"}]}],\"yLeftExtent\":{\"mode\":\"full\"},\"yTitle\":\"Megabytes\",\"yLeftScale\":\"log\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"04cd0fc2-f652-4583-a7d4-c1a8f6f8818f\":{\"columns\":{\"7e7413d7-db3d-4473-b46f-d76132f403b0\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\":{\"label\":\"Part of Heap\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\":{\"label\":\"Part of Heap\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\",1048576],\"location\":{\"min\":0,\"max\":126},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: heap'), 1048576)\"}},\"references\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\"],\"customLabel\":true},\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\":{\"label\":\"Heap\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\"],\"customLabel\":true},\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\":{\"label\":\"Top 10 values of labels.pool\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":true}}},\"columnOrder\":[\"06b7cd00-d4bf-4c98-a2e1-2932a19a92ae\",\"7e7413d7-db3d-4473-b46f-d76132f403b0\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X0\",\"371f4c5e-dca1-4de3-b495-b29d0c51ad42X1\"],\"incompleteColumns\":{},\"sampling\":1},\"b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d\":{\"linkToLayers\":[],\"columns\":{\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\":{\"label\":\"Part of Max\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.limit\",\"filter\":{\"query\":\"process.runtime.jvm.memory.limit: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\":{\"label\":\"Part of Max\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\",1048576],\"location\":{\"min\":0,\"max\":126},\"text\":\"divide(last_value(process.runtime.jvm.memory.limit, kql='process.runtime.jvm.memory.limit: * and labels.type: heap'), 1048576)\"}},\"references\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\"],\"customLabel\":true},\"81fcf196-8145-4246-9d3a-f1679cb62c81\":{\"label\":\"Max\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.limit, kql='process.runtime.jvm.memory.limit: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"references\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\"],\"customLabel\":true}},\"columnOrder\":[\"81fcf196-8145-4246-9d3a-f1679cb62c81\",\"81fcf196-8145-4246-9d3a-f1679cb62c81X0\",\"81fcf196-8145-4246-9d3a-f1679cb62c81X1\"],\"sampling\":1,\"incompleteColumns\":{}},\"66ce3428-e822-4b54-afc8-69bbffd9c106\":{\"linkToLayers\":[],\"columns\":{\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"323e403c-e4a1-481d-897e-907192f1cc4aX0\":{\"label\":\"Part of Non-Heap\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage: * and labels.type: non_heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"323e403c-e4a1-481d-897e-907192f1cc4aX1\":{\"label\":\"Part of Non-Heap\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX0\",1048576],\"location\":{\"min\":0,\"max\":130},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: non_heap'), 1048576)\"}},\"references\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX0\"],\"customLabel\":true},\"323e403c-e4a1-481d-897e-907192f1cc4a\":{\"label\":\"Non-Heap\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage, kql='process.runtime.jvm.memory.usage: * and labels.type: non_heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"323e403c-e4a1-481d-897e-907192f1cc4aX1\"],\"customLabel\":true}},\"columnOrder\":[\"f8721fc3-00f3-4106-96eb-0121ce00e6aa\",\"323e403c-e4a1-481d-897e-907192f1cc4a\",\"323e403c-e4a1-481d-897e-907192f1cc4aX1\",\"323e403c-e4a1-481d-897e-907192f1cc4aX0\"],\"sampling\":1,\"incompleteColumns\":{}}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Memory Usage\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":47,\"w\":22,\"h\":11,\"i\":\"542275ac-682e-43b8-b1ac-cf919d206e02\"},\"panelIndex\":\"542275ac-682e-43b8-b1ac-cf919d206e02\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-28f4234a-7411-4d15-8ad9-36ac978f7e19\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yLeftExtent\":{\"mode\":\"full\"},\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"28f4234a-7411-4d15-8ad9-36ac978f7e19\",\"accessors\":[\"04481efd-b3d8-44bc-be6c-5a0af56e241b\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\"}],\"yTitle\":\"System Load (0-100)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"28f4234a-7411-4d15-8ad9-36ac978f7e19\":{\"columns\":{\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\":{\"label\":\"Part of System Load\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.system.cpu.load_1m\",\"filter\":{\"query\":\"process.runtime.jvm.system.cpu.load_1m: *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"04481efd-b3d8-44bc-be6c-5a0af56e241b\":{\"label\":\"System Load\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"last_value(process.runtime.jvm.system.cpu.load_1m, kql='process.runtime.jvm.system.cpu.load_1m: *')\",\"isFormulaBroken\":false},\"references\":[\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\"],\"customLabel\":true}},\"columnOrder\":[\"5e07b038-8927-4bdb-a933-ddbd1f7d0374\",\"04481efd-b3d8-44bc-be6c-5a0af56e241b\",\"04481efd-b3d8-44bc-be6c-5a0af56e241bX0\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"System Load\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":47,\"w\":26,\"h\":11,\"i\":\"a3aace75-debc-4370-9954-e5a6e8ac6259\"},\"panelIndex\":\"a3aace75-debc-4370-9954-e5a6e8ac6259\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\",\"accessors\":[\"603e1430-a496-4249-b2da-73a5205b964e\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\",\"splitAccessor\":\"a9ec0126-a45f-4f86-b783-a4820badd7d8\",\"yConfig\":[{\"forAccessor\":\"603e1430-a496-4249-b2da-73a5205b964e\",\"axisMode\":\"left\"}],\"palette\":{\"type\":\"palette\",\"name\":\"status\"}}],\"yLeftScale\":\"log\",\"yTitle\":\"Megabytes\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"067c1db7-2942-4bf9-8fd6-4ca0a42a23fa\":{\"columns\":{\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"603e1430-a496-4249-b2da-73a5205b964eX0\":{\"label\":\"Part of divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.jvm.memory.usage_after_last_gc\",\"filter\":{\"query\":\"process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"603e1430-a496-4249-b2da-73a5205b964eX1\":{\"label\":\"Part of divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"divide\",\"args\":[\"603e1430-a496-4249-b2da-73a5205b964eX0\",1048576],\"location\":{\"min\":0,\"max\":154},\"text\":\"divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\"}},\"references\":[\"603e1430-a496-4249-b2da-73a5205b964eX0\"],\"customLabel\":true},\"603e1430-a496-4249-b2da-73a5205b964e\":{\"label\":\"MB\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"divide(last_value(process.runtime.jvm.memory.usage_after_last_gc, kql='process.runtime.jvm.memory.usage_after_last_gc: * and labels.type: heap'), 1048576)\",\"isFormulaBroken\":false},\"references\":[\"603e1430-a496-4249-b2da-73a5205b964eX1\"],\"filter\":{\"query\":\"\",\"language\":\"kuery\"},\"customLabel\":true},\"a9ec0126-a45f-4f86-b783-a4820badd7d8\":{\"label\":\"Top 10 values of labels.pool\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"labels.pool\",\"isBucketed\":true,\"params\":{\"size\":10,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false}}},\"columnOrder\":[\"a9ec0126-a45f-4f86-b783-a4820badd7d8\",\"2ab4d7f4-49aa-4e45-9ad7-73256400df18\",\"603e1430-a496-4249-b2da-73a5205b964e\",\"603e1430-a496-4249-b2da-73a5205b964eX0\",\"603e1430-a496-4249-b2da-73a5205b964eX1\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Memory after GC\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":58,\"w\":22,\"h\":10,\"i\":\"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367\"},\"panelIndex\":\"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a3ee7019-98bb-45cd-8536-e36a8e58297e\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a3ee7019-98bb-45cd-8536-e36a8e58297e\",\"accessors\":[\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"9c350955-95a1-4726-93fd-717b104a218c\",\"yConfig\":[{\"forAccessor\":\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\",\"color\":\"#54b399\"}]}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a3ee7019-98bb-45cd-8536-e36a8e58297e\":{\"columns\":{\"9c350955-95a1-4726-93fd-717b104a218c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\":{\"label\":\"Threads count\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.threads.count\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"process.runtime.jvm.threads.count: *\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true},\"customLabel\":true}},\"columnOrder\":[\"9c350955-95a1-4726-93fd-717b104a218c\",\"ebf7c494-f676-4642-b2cf-ead2e8e93dee\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Threads\"},{\"version\":\"8.8.2\",\"type\":\"lens\",\"gridData\":{\"x\":22,\"y\":58,\"w\":26,\"h\":10,\"i\":\"4f987373-1b85-4e80-a7cc-c1c000349c5e\"},\"panelIndex\":\"4f987373-1b85-4e80-a7cc-c1c000349c5e\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"id\":\"apm_static_index_pattern_id\",\"name\":\"indexpattern-datasource-layer-a45cd02a-4c62-4b34-bdce-c702b87b4246\",\"type\":\"index-pattern\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yRightTitle\":\"Duration (ms)\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"a45cd02a-4c62-4b34-bdce-c702b87b4246\",\"accessors\":[\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"4e7a6993-fcee-47ae-9599-0047809bb29e\"],\"position\":\"top\",\"seriesType\":\"bar\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"57050bde-2c14-47d2-9dde-6e35326cf5cf\",\"yConfig\":[{\"forAccessor\":\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"axisMode\":\"left\"},{\"forAccessor\":\"4e7a6993-fcee-47ae-9599-0047809bb29e\",\"color\":\"#e7664c\",\"axisMode\":\"left\"}]}],\"yTitle\":\"Duration (ms)\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"a45cd02a-4c62-4b34-bdce-c702b87b4246\":{\"columns\":{\"57050bde-2c14-47d2-9dde-6e35326cf5cf\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":false,\"dropPartials\":false}},\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\":{\"label\":\"Minor GC\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.gc.duration\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.action: \\\"end of minor GC\\\" \",\"language\":\"kuery\"},\"params\":{\"format\":{\"id\":\"number\",\"params\":{\"decimals\":2}},\"emptyAsNull\":true},\"customLabel\":true},\"4e7a6993-fcee-47ae-9599-0047809bb29e\":{\"label\":\"Major GC\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.jvm.gc.duration\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.action : \\\"end of major GC\\\"\",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true},\"customLabel\":true}},\"columnOrder\":[\"57050bde-2c14-47d2-9dde-6e35326cf5cf\",\"e3c6a7fb-d980-43d4-bb84-ae22018a0710\",\"4e7a6993-fcee-47ae-9599-0047809bb29e\"],\"incompleteColumns\":{},\"sampling\":1}}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Garbage Collection\"}]","timeRestore":false,"title":"Layer7 Gateway Dashboard","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-05-14T07:28:39.093Z","id":"c29a9900-2161-11ee-be6c-57cbef857d54","managed":false,"references":[{"id":"apm_static_index_pattern_id","name":"675c235a-535c-48d9-a5ed-848810986e5a:indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"0ae50fd8-f924-4207-b546-5d600ac701b3:indexpattern-datasource-layer-cad7aae7-308f-4c4f-abbd-b03792fbec56","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"ee3d33f8-243e-42d9-87fa-09c00dbfd1b7:indexpattern-datasource-layer-fd66544d-5ee7-4f36-8966-c7aa38197646","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"d9305573-c761-4802-9c0b-6f6326266125:indexpattern-datasource-layer-8638d8b0-f850-42f0-9e64-35e6f239e0d0","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"868932e9-487a-408f-89fd-14f9324534e9:indexpattern-datasource-layer-2381ace6-24bb-4e47-b57b-6fe06c980216","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"868932e9-487a-408f-89fd-14f9324534e9:indexpattern-datasource-layer-06696642-3076-4612-8291-189dcb25dd9e","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"9a54b939-5371-49f6-8888-ab92e165e625:indexpattern-datasource-layer-a57f2de0-c4c8-4a18-9b3a-533ad5572097","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-04cd0fc2-f652-4583-a7d4-c1a8f6f8818f","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-b369a9a0-f9ee-4b7e-ba8f-97d2722cd89d","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"2a2057e2-ce45-4804-8ce0-3d4ed94679d7:indexpattern-datasource-layer-66ce3428-e822-4b54-afc8-69bbffd9c106","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"542275ac-682e-43b8-b1ac-cf919d206e02:indexpattern-datasource-layer-28f4234a-7411-4d15-8ad9-36ac978f7e19","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"a3aace75-debc-4370-9954-e5a6e8ac6259:indexpattern-datasource-layer-067c1db7-2942-4bf9-8fd6-4ca0a42a23fa","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"7f1f017e-d42b-4cf4-8cb2-3f4bb3de5367:indexpattern-datasource-layer-a3ee7019-98bb-45cd-8536-e36a8e58297e","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"4f987373-1b85-4e80-a7cc-c1c000349c5e:indexpattern-datasource-layer-a45cd02a-4c62-4b34-bdce-c702b87b4246","type":"index-pattern"},{"id":"apm_static_index_pattern_id","name":"controlGroup_d3e0d4d0-b1de-465c-8772-806a49213c2b:optionsListDataView","type":"index-pattern"}],"type":"dashboard","typeMigrationVersion":"8.7.0","updated_at":"2024-05-14T07:28:39.093Z","version":"WzE2ODksMV0="} {"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":2,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/example/otel-elastic/instrumentation.yaml b/example/otel-elastic/instrumentation.yaml index f3add671..09550e28 100644 --- a/example/otel-elastic/instrumentation.yaml +++ b/example/otel-elastic/instrumentation.yaml @@ -1,4 +1,4 @@ -apiVersion: opentelemetry.io/v1alpha1 +apiVersion: opentelemetry.io/v1beta1 kind: Instrumentation metadata: name: otel-instrumentation diff --git a/example/otel-lgtm/readme.md b/example/otel-lgtm/readme.md index 36d8e3b3..f4ccd07a 100644 --- a/example/otel-lgtm/readme.md +++ b/example/otel-lgtm/readme.md @@ -260,7 +260,7 @@ the OpenTelemetry Operator is not a requirement, it makes configuring the Gatewa - Deploy Cert-Manager (OTel Operator dependency) ``` -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml ``` verify cert-manager has been successfully deployed ``` @@ -271,7 +271,7 @@ kubectl wait --for=condition=ready --timeout=600s pod -l app=webhook -n cert-man - Deploy the OpenTelemetry Operator ``` -kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.97.1/opentelemetry-operator.yaml +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml ``` verify that the OpenTelemetry Operator has been successfully deployed ``` @@ -407,8 +407,8 @@ kubectl describe pods ssg-57d96567cb-n24g9 Init Containers: graphman-static-init-c1b58adb6d: Container ID: containerd://21924ae85d25437d3634ea5da1415c9bb58d678600f9fd67d4f0b0360857d7c5 - Image: docker.io/layer7api/graphman-static-init:1.0.0 - Image ID: docker.io/layer7api/graphman-static-init@sha256:24189a432c0283845664c6fd54c3e8d9f86ad9d35ef12714bb3a18b7aba85aa4 + Image: docker.io/caapim/graphman-static-init:1.0.4 + Image ID: docker.io/caapim/graphman-static-init@sha256:8cb1035035b18fa9dc2c95e2b584c758e78909b3f615ee5f49dce166e8aae213 Port: Host Port: State: Terminated diff --git a/example/otel-prometheus/README.md b/example/otel-prometheus/README.md index 264133a6..ed179e56 100644 --- a/example/otel-prometheus/README.md +++ b/example/otel-prometheus/README.md @@ -409,7 +409,7 @@ You can now move on to test your gateway deployment! ### Install Cert Manager These steps are based the official documentation for installing Cert-Manager [here](https://cert-manager.io/docs/installation/). Cert-Manager is a pre-requisite for the Open Telemetry Operator. ``` - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml ``` #### View CertManager Components @@ -443,7 +443,7 @@ These steps are based the official documentation for installing Open Telemetry [ - Install the Open Telemetry Operator. ``` -kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.97.1/opentelemetry-operator.yaml +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml ``` #### View Open Telemetry Components @@ -500,7 +500,7 @@ kubectl apply -f ./example/otel-prometheus/servicemonitor.yaml ``` ### Install Jaeger -These steps are based on instructions that can be found in the Jaeger [documentation](https://www.jaegertracing.io/docs/1.44/operator/) +These steps are based on instructions that can be found in the Jaeger [documentation](https://www.jaegertracing.io/docs/1.65/operator/) - Create a namespace called observability ``` @@ -508,7 +508,7 @@ kubectl create namespace observability ``` - Install the Jaeger Operator ``` -kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.44.0/jaeger-operator.yaml -n observability +kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.65.0/jaeger-operator.yaml -n observability ``` #### View Jaeger Components @@ -658,8 +658,8 @@ kubectl describe pods ssg-57d96567cb-n24g9 Init Containers: graphman-static-init-c1b58adb6d: Container ID: containerd://21924ae85d25437d3634ea5da1415c9bb58d678600f9fd67d4f0b0360857d7c5 - Image: docker.io/layer7api/graphman-static-init:1.0.0 - Image ID: docker.io/layer7api/graphman-static-init@sha256:24189a432c0283845664c6fd54c3e8d9f86ad9d35ef12714bb3a18b7aba85aa4 + Image: docker.io/caapim/graphman-static-init:1.0.4 + Image ID: docker.io/caapim/graphman-static-init@sha256:8cb1035035b18fa9dc2c95e2b584c758e78909b3f615ee5f49dce166e8aae213 Port: Host Port: State: Terminated @@ -835,8 +835,9 @@ kubectl delete -f ./example/otel-prometheus/collector.yaml kubectl delete -f ./example/otel-prometheus/instrumentation.yaml kubectl delete -f ./example/otel-prometheus/observability/jaeger/jaeger.yaml kubectl delete -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/cloud/deploy.yaml -kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.76.1/opentelemetry-operator.yaml -kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml +kubectl delete -f https://github.com/open-telemetry/opentelemetry-operator/releases/download/v0.146.0/opentelemetry-operator.yaml +kubectl delete -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.65.0/jaeger-operator.yaml -n observability +kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.20.0/cert-manager.yaml helm uninstall prometheus -n monitoring kubectl delete ns monitoring kubectl delete ns observability