diff --git a/deploy/Makefile b/deploy/Makefile index 1d62f6cb..43789c69 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -11,6 +11,16 @@ REMOTE_SSH_KEY_ARG = $(if $(strip $(REMOTE_SSH_KEY)),-i $(REMOTE_SSH_KEY),) HELM_OPENSEARCH_RELEASE ?= cogstack-opensearch HELM_OPENSEARCH_NAMESPACE ?= cogstack HELM_OPENSEARCH_CHART ?= ./charts/opensearch +HELM_OPENSEARCH_VALUES_FILE ?= ./helm/opensearch.values.yaml +HELM_OPENSEARCH_VALUES_ARG = -f $(HELM_OPENSEARCH_VALUES_FILE) +HELM_POSTGRESQL_RELEASE ?= cogstack-postgresql +HELM_POSTGRESQL_NAMESPACE ?= cogstack +HELM_POSTGRESQL_CHART ?= ./charts/postgresql +HELM_POSTGRESQL_VALUES_FILE ?= ./helm/postgresql.values.yaml +HELM_POSTGRESQL_VALUES_ARG = -f $(HELM_POSTGRESQL_VALUES_FILE) +CNPG_OPERATOR_MINOR ?= 1.28 +CNPG_OPERATOR_VERSION ?= 1.28.1 +CNPG_OPERATOR_MANIFEST ?= https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-$(CNPG_OPERATOR_MINOR)/releases/cnpg-$(CNPG_OPERATOR_VERSION).yaml define WITH_ENV set -a && source ./export_env_vars.sh; @@ -72,11 +82,20 @@ load-env: ## Load variables from export_env_vars.sh in a subshell show-env: ## Print sorted environment variables after loading export_env_vars.sh ${WITH_ENV} >/dev/null 2>&1; printenv | sort -helm-template-opensearch: ## Render OpenSearch chart using deploy/elasticsearch.env - helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) --set-file envFile.raw=./elasticsearch.env +helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml + helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) -helm-install-opensearch: ## Install/upgrade OpenSearch chart using deploy/elasticsearch.env - helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) --set-file envFile.raw=./elasticsearch.env --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace +helm-install-opensearch: ## Install/upgrade OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml + helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace + +kube-install-cnpg-operator: ## Install CloudNativePG operator/CRDs from the pinned upstream manifest + kubectl apply --server-side -f $(CNPG_OPERATOR_MANIFEST) + +helm-template-postgresql: ## Render CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml + helm template $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) + +helm-install-postgresql: ## Install/upgrade CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml + helm upgrade --install $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) --create-namespace remote-deploy-service: ## Deploy one or more services to a remote machine via SSH + docker compose @@ -105,7 +124,7 @@ _check-remote-params: exit 1; \ fi -.PHONY: remote-deploy-service remote-stop-service remote-delete-service _check-remote-params helm-template-opensearch helm-install-opensearch +.PHONY: remote-deploy-service remote-stop-service remote-delete-service _check-remote-params helm-template-opensearch helm-install-opensearch kube-install-cnpg-operator helm-template-postgresql helm-install-postgresql # start services diff --git a/deploy/charts/README.md b/deploy/charts/README.md index db3a4bfe..0a790922 100644 --- a/deploy/charts/README.md +++ b/deploy/charts/README.md @@ -9,19 +9,22 @@ This directory contains Helm charts owned by this repository's deployment layer. ## Current charts -- `opensearch/` - OpenSearch + OpenSearch Dashboards chart used by this repo. +- `opensearch/` - OpenSearch and/or OpenSearch Dashboards chart used by this repo. +- `postgresql/` - CloudNativePG-backed PostgreSQL cluster chart used by this repo. ## Quick usage ```bash # Render manifests helm template cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env + -f ./deploy/helm/opensearch.values.yaml # Install/upgrade helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env \ + -f ./deploy/helm/opensearch.values.yaml \ --namespace cogstack --create-namespace ``` -Only keys in `envFile.includeKeys` are imported from the env file. +The OpenSearch and Dashboards config files should come from `services/`, and the security and env files from `security/` and `deploy/`, so Docker and Kubernetes use the same source files. +The values file is for cluster-specific overrides only; it does not need to repeat the shared YAML or env file paths. +Only keys in `envFile.includeKeys`, `usersEnvFile.includeKeys`, and `certificatesEnvFile.includeKeys` are imported. diff --git a/deploy/charts/opensearch/README.md b/deploy/charts/opensearch/README.md index 2192f814..0f4b6499 100644 --- a/deploy/charts/opensearch/README.md +++ b/deploy/charts/opensearch/README.md @@ -1,44 +1,91 @@ # cogstack-opensearch Helm Chart -Helm chart for deploying OpenSearch plus OpenSearch Dashboards using the CogStack configuration baseline. +Helm chart for deploying OpenSearch and/or OpenSearch Dashboards using the CogStack configuration baseline. ## What this chart deploys -- OpenSearch `StatefulSet` (default: 3 replicas) -- OpenSearch client + headless Services +- OpenSearch `StatefulSet` (default: 3 replicas, controlled by `opensearch.enabled`) +- OpenSearch client + headless Services (when `opensearch.enabled=true`) - OpenSearch Dashboards `Deployment` + Service (enabled by default) - ConfigMaps for: - - `opensearch.yml` - - `log4j2.properties` - - OpenSearch Security files (`config.yml`, `internal_users.yml`, `roles.yml`, `roles_mapping.yml`) + - `opensearch.yml` (when `opensearch.enabled=true`) + - `log4j2.properties` (when `opensearch.enabled=true`) + - OpenSearch Security files (`config.yml`, `internal_users.yml`, `roles.yml`, `roles_mapping.yml`) (when `opensearch.enabled=true`) - `opensearch_dashboards.yml` +- PVC-backed `data`, `logs`, and performance-analyzer storage for OpenSearch by default ## Prerequisites -1. Kubernetes cluster with dynamic PV provisioning (or set `persistence.enabled=false`). -2. Kubernetes Secrets containing TLS materials. +1. Kubernetes cluster with dynamic PV provisioning (if `opensearch.enabled=true` and `persistence.enabled=true`). +2. Kubernetes Secrets containing TLS materials for enabled components. 3. If `credentials.create=false`, an existing Secret with: - - `OPENSEARCH_INITIAL_ADMIN_PASSWORD` - - `KIBANA_USER` - - `KIBANA_PASSWORD` + - `OPENSEARCH_INITIAL_ADMIN_PASSWORD` when `opensearch.enabled=true` + - `KIBANA_USER` and `KIBANA_PASSWORD` when `dashboards.enabled=true` ## Required certificate secrets -Set these in `values.yaml`: +Set these in `values.yaml` for enabled components: -- `certificates.opensearchSecretName` -- `certificates.dashboardsSecretName` +- `certificates.opensearchSecretName` (if `opensearch.enabled=true`) +- `certificates.dashboardsSecretName` (if `dashboards.enabled=true`) Secret keys are mapped via: -- `certificates.opensearchFiles.*` -- `certificates.dashboardsFiles.*` +- `certificates.opensearchFiles.*` (if `opensearch.enabled=true`) +- `certificates.opensearchNodeFiles[*]` for per-pod node cert/key selection (if `opensearch.enabled=true`) +- `certificates.dashboardsFiles.*` (if `dashboards.enabled=true`) + +Repo-aligned certificate source paths: + +- OpenSearch shared certs: + - `security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem` + - `security/certificates/elastic/opensearch/admin.crt` + - `security/certificates/elastic/opensearch/admin.key.pem` +- OpenSearch node certs: + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt` + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.key` + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.crt` + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.key` + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.crt` + - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.key` +- Dashboards certs: + - `security/certificates/elastic/opensearch/es_kibana_client.pem` + - `security/certificates/elastic/opensearch/es_kibana_client.key` + +Example secret creation from the repo layout: + +```bash +kubectl create secret generic opensearch-certs \ + --from-file=elastic-stack-ca.crt.pem=./security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \ + --from-file=admin.crt=./security/certificates/elastic/opensearch/admin.crt \ + --from-file=admin.key.pem=./security/certificates/elastic/opensearch/admin.key.pem \ + --from-file=elasticsearch-1.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt \ + --from-file=elasticsearch-1.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.key \ + --from-file=elasticsearch-2.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.crt \ + --from-file=elasticsearch-2.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.key \ + --from-file=elasticsearch-3.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.crt \ + --from-file=elasticsearch-3.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.key \ + --from-file=es_kibana_client.pem=./security/certificates/elastic/opensearch/es_kibana_client.pem \ + --from-file=es_kibana_client.key=./security/certificates/elastic/opensearch/es_kibana_client.key +``` ## Install ```bash helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env \ + -f ./deploy/helm/opensearch.values.yaml \ + --namespace cogstack --create-namespace +``` + +## Dashboards-only install + +Use this mode when OpenSearch is managed externally: + +```bash +helm upgrade --install cogstack-dashboards ./deploy/charts/opensearch \ + --set opensearch.enabled=false \ + --set dashboards.enabled=true \ + --set 'dashboards.opensearchHosts[0]=https://opensearch-client.cogstack.svc:9200' \ --namespace cogstack --create-namespace ``` @@ -46,12 +93,24 @@ helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \ ```bash helm template cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env + -f ./deploy/helm/opensearch.values.yaml ``` ## Notes -- The chart packages current repository config files under `files/`. -- `envFile.raw` can be set from `deploy/elasticsearch.env` and is loaded via `envFrom` into OpenSearch and Dashboards. -- Only keys listed in `envFile.includeKeys` are imported (to avoid leaking secrets from env files into ConfigMaps). +- Helm templates cannot read arbitrary `../../...` paths directly; `.Files.Get` only sees files packaged inside the chart. +- In this repo, the chart `files/` entries are symlinked to the shared `deploy/`, `services/`, and `security/` sources so Docker and Kubernetes stay aligned. +- The standard install/render commands now use `-f ./deploy/helm/opensearch.values.yaml`; that file is for cluster-specific overrides only. +- The shared `services/`, `security/`, and selected `deploy/` env files are consumed automatically by the chart defaults; you do not need to repeat those paths in the values file. +- `envFile.raw` defaults to `deploy/elasticsearch.env` and can still be overridden; the chart reads only `ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, and `KIBANA_SERVER_NAME`, while pod IP and discovery hosts remain Kubernetes-specific. +- `usersEnvFile.raw` defaults to `security/env/users_elasticsearch.env` and can still be overridden; only the credential keys required by the enabled components are imported. +- `certificatesEnvFile.raw` defaults to `security/env/certificates_elasticsearch.env` and can still be overridden; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`.pem` / `.key`). +- `deploy/elasticsearch.env` shared values are used where they make sense on Kubernetes (`ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, `KIBANA_SERVER_NAME`), while pod IP and discovery hosts remain Kubernetes-specific. +- By default, `certificates.opensearchNodeFiles[*]` maps pod ordinals `0/1/2` to repo-style node cert keys `elasticsearch-1/2/3`. +- `opensearch.logPersistence` and `opensearch.performanceAnalyzerPersistence` default to PVC-backed storage to stay closer to the Docker Compose deployment. +- `opensearch.snapshotBackups` adds shared PVC-backed mounts for `/mnt/es_data_backups` and `/mnt/es_config_backups`; use RWX storage or set `existingClaim` values, and still set `path.repo` in the shared OpenSearch config if you want the cluster to use them. +- `configFiles.opensearchRaw` can be set from `services/elasticsearch/config/opensearch.yml`. +- `configFiles.log4jRaw` can be set from `services/elasticsearch/config/log4j2_opensearch.properties`. +- `configFiles.dashboardsRaw` can be set from `services/kibana/config/opensearch.yml`. +- `securityFiles.*Raw` can be set from `security/es_roles/opensearch/*.yml` and overrides the chart-bundled OpenSearch security files. - Review security and certificate settings before production use. diff --git a/deploy/charts/opensearch/files/certificates-elasticsearch.envfile b/deploy/charts/opensearch/files/certificates-elasticsearch.envfile new file mode 120000 index 00000000..7b16ab4c --- /dev/null +++ b/deploy/charts/opensearch/files/certificates-elasticsearch.envfile @@ -0,0 +1 @@ +../../../../security/env/certificates_elasticsearch.env \ No newline at end of file diff --git a/deploy/charts/opensearch/files/deploy-elasticsearch.envfile b/deploy/charts/opensearch/files/deploy-elasticsearch.envfile new file mode 120000 index 00000000..e4a00255 --- /dev/null +++ b/deploy/charts/opensearch/files/deploy-elasticsearch.envfile @@ -0,0 +1 @@ +../../../../deploy/elasticsearch.env \ No newline at end of file diff --git a/deploy/charts/opensearch/files/log4j2.properties b/deploy/charts/opensearch/files/log4j2.properties deleted file mode 100644 index 1fcce23c..00000000 --- a/deploy/charts/opensearch/files/log4j2.properties +++ /dev/null @@ -1,9 +0,0 @@ -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console \ No newline at end of file diff --git a/deploy/charts/opensearch/files/log4j2.properties b/deploy/charts/opensearch/files/log4j2.properties new file mode 120000 index 00000000..9c36ba0a --- /dev/null +++ b/deploy/charts/opensearch/files/log4j2.properties @@ -0,0 +1 @@ +../../../../services/elasticsearch/config/log4j2_opensearch.properties \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch-security/config.yml b/deploy/charts/opensearch/files/opensearch-security/config.yml deleted file mode 100644 index e54a7a08..00000000 --- a/deploy/charts/opensearch/files/opensearch-security/config.yml +++ /dev/null @@ -1,265 +0,0 @@ ---- - -# This is the main OpenSearch Security configuration file where authentication -# and authorization is defined. -# -# You need to configure at least one authentication domain in the authc of this file. -# An authentication domain is responsible for extracting the user credentials from -# the request and for validating them against an authentication backend like Active Directory for example. -# -# If more than one authentication domain is configured the first one which succeeds wins. -# If all authentication domains fail then the request is unauthenticated. -# In this case an exception is thrown and/or the HTTP status is set to 401. -# -# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect -# the roles from a given backend for the authenticated user. -# -# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both. -# http_enabled: true -# transport_enabled: true -# -# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to -# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated. -# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous" -# and one role named "anonymous_backendrole". -# If you enable anonymous authentication all HTTP authenticators will not challenge. -# -# -# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert" -# first and the challenging one last. -# Because it's not possible to challenge a client with two different authentication methods (for example -# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation -# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request. -# -# Default value of the challenge flag is true. -# -# -# HTTP -# basic (challenging) -# proxy (not challenging, needs xff) -# kerberos (challenging) -# clientcert (not challenging, needs https) -# jwt (not challenging) -# host (not challenging) #DEPRECATED, will be removed in a future version. -# host based authentication is configurable in roles_mapping - -# Authc -# internal -# noop -# ldap - -# Authz -# ldap -# noop - - - -_meta: - type: "config" - config_version: 2 - -config: - dynamic: - # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index - # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default) - # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently - #filtered_alias_mode: warn - #do_not_fail_on_forbidden: false - kibana: - # this must match whatver index name is in the 'opensearchDashboards.index' property in : services/kibana/config/opensearch.yml - index: '.opensearch_dashboards' - # Kibana multitenancy - #multitenancy_enabled: true - #private_tenant_enabled: true - #default_tenant: "" - #server_username: kibanaserver - http: - anonymous_auth_enabled: false - xff: - enabled: false - internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern - #internalProxies: '.*' # trust all internal proxies, regex pattern - #remoteIpHeader: 'x-forwarded-for' - ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help - ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For - ###### and here https://tools.ietf.org/html/rfc7239 - ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve - authc: - openid_auth_domain: - http_enabled: true - transport_enabled: true - order: 0 - http_authenticator: - type: openid - challenge: false - config: - subject_key: preferred_username - roles_key: roles - openid_connect_url: https://keycloak.example.com:8080/auth/realms/master/.well-known/openid-configuration - required_audience: your-openid-client-id - authentication_backend: - type: noop - kerberos_auth_domain: - http_enabled: false - transport_enabled: false - order: 6 - http_authenticator: - type: kerberos - challenge: true - config: - # If true a lot of kerberos/security related debugging output will be logged to standard out - krb_debug: false - # If true then the realm will be stripped from the user name - strip_realm_from_principal: true - authentication_backend: - type: noop - basic_internal_auth_domain: - description: "Authenticate via HTTP Basic against internal users database" - http_enabled: true - transport_enabled: true - order: 4 - http_authenticator: - type: basic - challenge: true - authentication_backend: - type: intern - proxy_auth_domain: - description: "Authenticate via proxy" - http_enabled: false - transport_enabled: false - order: 3 - http_authenticator: - type: proxy - challenge: false - config: - user_header: "x-proxy-user" - roles_header: "x-proxy-roles" - authentication_backend: - type: noop - jwt_auth_domain: - description: "Authenticate via Json Web Token" - http_enabled: false - transport_enabled: false - order: 0 - http_authenticator: - type: jwt - challenge: false - config: - signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key" - jwt_header: "Authorization" - jwt_url_parameter: null - jwt_clock_skew_tolerance_seconds: 30 - roles_key: null - subject_key: null - authentication_backend: - type: noop - clientcert_auth_domain: - description: "Authenticate via SSL client certificates" - http_enabled: false - transport_enabled: false - order: 2 - http_authenticator: - type: clientcert - config: - username_attribute: cn #optional, if omitted DN becomes username - challenge: false - authentication_backend: - type: noop - ldap: - description: "Authenticate via LDAP or Active Directory" - http_enabled: false - transport_enabled: false - order: 5 - http_authenticator: - type: basic - challenge: false - authentication_backend: - # LDAP authentication backend (authenticate users against a LDAP or Active Directory) - type: ldap - config: - # enable ldaps - enable_ssl: false - # enable start tls, enable_ssl should be false - enable_start_tls: false - # send client certificate - enable_ssl_client_auth: false - # verify ldap hostname - verify_hostnames: true - hosts: - - localhost:8389 - bind_dn: null - password: null - userbase: 'ou=people,dc=example,dc=com' - # Filter to search for users (currently in the whole subtree beneath userbase) - # {0} is substituted with the username - usersearch: '(sAMAccountName={0})' - # Use this attribute from the user as username (if not set then DN is used) - username_attribute: null - authz: - roles_from_myldap: - description: "Authorize via LDAP or Active Directory" - http_enabled: false - transport_enabled: false - authorization_backend: - # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too) - type: ldap - config: - # enable ldaps - enable_ssl: false - # enable start tls, enable_ssl should be false - enable_start_tls: false - # send client certificate - enable_ssl_client_auth: false - # verify ldap hostname - verify_hostnames: true - hosts: - - localhost:8389 - bind_dn: null - password: null - rolebase: 'ou=groups,dc=example,dc=com' - # Filter to search for roles (currently in the whole subtree beneath rolebase) - # {0} is substituted with the DN of the user - # {1} is substituted with the username - # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute - rolesearch: '(member={0})' - # Specify the name of the attribute which value should be substituted with {2} above - userroleattribute: null - # Roles as an attribute of the user entry - userrolename: disabled - #userrolename: memberOf - # The attribute in a role entry containing the name of that role, Default is "name". - # Can also be "dn" to use the full DN as rolename. - rolename: cn - # Resolve nested roles transitive (roles which are members of other roles and so on ...) - resolve_nested_roles: true - userbase: 'ou=people,dc=example,dc=com' - # Filter to search for users (currently in the whole subtree beneath userbase) - # {0} is substituted with the username - usersearch: '(uid={0})' - # Skip users matching a user name, a wildcard or a regex pattern - #skip_users: - # - 'cn=Michael Jackson,ou*people,o=TEST' - # - '/\S*/' - roles_from_another_ldap: - description: "Authorize via another Active Directory" - http_enabled: false - transport_enabled: false - authorization_backend: - type: ldap - #config goes here ... - # auth_failure_listeners: - # ip_rate_limiting: - # type: ip - # allowed_tries: 10 - # time_window_seconds: 3600 - # block_expiry_seconds: 600 - # max_blocked_clients: 100000 - # max_tracked_clients: 100000 - # internal_authentication_backend_limiting: - # type: username - # authentication_backend: intern - # allowed_tries: 10 - # time_window_seconds: 3600 - # block_expiry_seconds: 600 - # max_blocked_clients: 100000 - # max_tracked_clients: 100000 diff --git a/deploy/charts/opensearch/files/opensearch-security/config.yml b/deploy/charts/opensearch/files/opensearch-security/config.yml new file mode 120000 index 00000000..f2259046 --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch-security/config.yml @@ -0,0 +1 @@ +../../../../../security/es_roles/opensearch/config.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch-security/internal_users.yml b/deploy/charts/opensearch/files/opensearch-security/internal_users.yml deleted file mode 100644 index 268ae30b..00000000 --- a/deploy/charts/opensearch/files/opensearch-security/internal_users.yml +++ /dev/null @@ -1,80 +0,0 @@ -# This is the internal user database -# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh - -_meta: - type: "internalusers" - config_version: 2 - -#password is: admin -admin: - hash: $2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG - reserved: true - backend_roles: - - "admin" - - "all_access" - description: "admin user" - -anomalyadmin: - hash: "$2y$12$TRwAAJgnNo67w3rVUz4FIeLx9Dy/llB79zf9I15CKJ9vkM4ZzAd3." - reserved: false - opendistro_security_roles: - - "anomaly_full_access" - description: "Demo anomaly admin user, using internal role" - -#password is: logstash -logstash: - hash: $2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2 - backend_roles: - - "logstash" - -#password is: kibanaserver -kibanaserver: - reserved: true - hash: $2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H. - description: "OpenSearch Dashboards user" - backend_roles: - - "readall" - - "admin" - - "kibanaserver" - - "kibanauser" - -#password is: kibanaro -kibanaro: - hash: $2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC - reserved: false - backend_roles: - - "kibanauser" - - "readall" - attributes: - attribute1: "value1" - attribute2: "value2" - attribute3: "value3" - description: "OpenSearch Dashboards read only user, using external role mapping" - -#password is: readall -readall: - hash: $2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2 - reserved: false - backend_roles: - - "readall" - description: "readall user" - -#password is: snapshotrestore -snapshotrestore: - hash: $2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W - reserved: false - backend_roles: - - "snapshotrestore" - description: "snapshotrestore user" - - -# Define your internal users here -new-user: - hash: "$2y$12$88IFVl6IfIwCFh5aQYfOmuXVL9j2hz/GusQb35o.4sdTDAEMTOD.K" - reserved: false - hidden: false - backend_roles: - - "specify-some-backend-role-here" - attributes: - attribute1: "value1" - static: false \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch-security/internal_users.yml b/deploy/charts/opensearch/files/opensearch-security/internal_users.yml new file mode 120000 index 00000000..2a13b3bc --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch-security/internal_users.yml @@ -0,0 +1 @@ +../../../../../security/es_roles/opensearch/internal_users.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch-security/roles.yml b/deploy/charts/opensearch/files/opensearch-security/roles.yml deleted file mode 100644 index 77693806..00000000 --- a/deploy/charts/opensearch/files/opensearch-security/roles.yml +++ /dev/null @@ -1,530 +0,0 @@ -_meta: - type: "roles" - config_version: 2 - -# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards -kibana_read_only: - reserved: true - -# The security REST API access role is used to assign specific users access to change the security settings through the REST API. -security_rest_api_access: - reserved: true - -security_rest_api_full_access: - reserved: true - cluster_permissions: - - 'restapi:admin/actiongroups' - - 'restapi:admin/allowlist' - - 'restapi:admin/config/update' - - 'restapi:admin/internalusers' - - 'restapi:admin/nodesdn' - - 'restapi:admin/roles' - - 'restapi:admin/rolesmapping' - - 'restapi:admin/ssl/certs/info' - - 'restapi:admin/ssl/certs/reload' - - 'restapi:admin/tenants' - -# Allows users to view monitors, destinations and alerts -alerting_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/alerting/alerts/get' - - 'cluster:admin/opendistro/alerting/destination/get' - - 'cluster:admin/opendistro/alerting/monitor/get' - - 'cluster:admin/opendistro/alerting/monitor/search' - - 'cluster:admin/opensearch/alerting/comments/search' - - 'cluster:admin/opensearch/alerting/findings/get' - - 'cluster:admin/opensearch/alerting/remote/indexes/get' - - 'cluster:admin/opensearch/alerting/workflow/get' - - 'cluster:admin/opensearch/alerting/workflow_alerts/get' - -# Allows users to view and acknowledge alerts -alerting_ack_alerts: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/alerting/alerts/*' - - 'cluster:admin/opendistro/alerting/chained_alerts/*' - - 'cluster:admin/opendistro/alerting/workflow_alerts/*' - - 'cluster:admin/opensearch/alerting/comments/*' - -# Allows users to use all alerting functionality -alerting_full_access: - reserved: true - cluster_permissions: - - 'cluster_monitor' - - 'cluster:admin/opendistro/alerting/*' - - 'cluster:admin/opensearch/alerting/*' - - 'cluster:admin/opensearch/notifications/feature/publish' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices_monitor' - - 'indices:admin/aliases/get' - - 'indices:admin/mappings/get' - -# Allow users to read Anomaly Detection detectors and results -anomaly_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/ad/detector/info' - - 'cluster:admin/opendistro/ad/detector/search' - - 'cluster:admin/opendistro/ad/detectors/get' - - 'cluster:admin/opendistro/ad/result/search' - - 'cluster:admin/opendistro/ad/tasks/search' - - 'cluster:admin/opendistro/ad/detector/validate' - - 'cluster:admin/opendistro/ad/result/topAnomalies' - -# Allows users to use all Anomaly Detection functionality -anomaly_full_access: - reserved: true - cluster_permissions: - - "cluster:admin/ingest/pipeline/delete" - - "cluster:admin/ingest/pipeline/put" - - 'cluster_monitor' - - 'cluster:admin/opendistro/ad/*' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/aliases/get' - - 'indices:admin/mappings/fields/get' - - 'indices:admin/mappings/fields/get*' - - 'indices:admin/mappings/get' - - 'indices:admin/resolve/index' - - 'indices:admin/setting/put' - - 'indices:data/read/field_caps*' - - 'indices:data/read/search' - - 'indices_monitor' - -# Allow users to execute read only k-NN actions -knn_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/knn_search_model_action' - - 'cluster:admin/knn_get_model_action' - - 'cluster:admin/knn_stats_action' - -# Allow users to use all k-NN functionality -knn_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/knn_training_model_action' - - 'cluster:admin/knn_training_job_router_action' - - 'cluster:admin/knn_training_job_route_decision_info_action' - - 'cluster:admin/knn_warmup_action' - - 'cluster:admin/knn_delete_model_action' - - 'cluster:admin/knn_remove_model_from_cache_action' - - 'cluster:admin/knn_update_model_graveyard_action' - - 'cluster:admin/knn_search_model_action' - - 'cluster:admin/knn_get_model_action' - - 'cluster:admin/knn_stats_action' - -# Allow users to execute read only ip2geo datasource action -ip2geo_datasource_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/geospatial/datasource/get' - -# Allow users to use all ip2geo datasource action -ip2geo_datasource_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/geospatial/datasource/*' - -# Allows users to read Notebooks -notebooks_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/notebooks/list' - - 'cluster:admin/opendistro/notebooks/get' - -# Allows users to all Notebooks functionality -notebooks_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/notebooks/create' - - 'cluster:admin/opendistro/notebooks/update' - - 'cluster:admin/opendistro/notebooks/delete' - - 'cluster:admin/opendistro/notebooks/get' - - 'cluster:admin/opendistro/notebooks/list' - -# Allows users to read observability objects -observability_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/observability/get' - -# Allows users to all Observability functionality -observability_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/observability/create' - - 'cluster:admin/opensearch/observability/update' - - 'cluster:admin/opensearch/observability/delete' - - 'cluster:admin/opensearch/observability/get' - -# Allows users to all PPL functionality -ppl_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/ppl' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/mappings/get' - - 'indices:data/read/search*' - - 'indices:monitor/settings/get' - -# Allows users to read and download Reports -reports_instances_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/reports/instance/list' - - 'cluster:admin/opendistro/reports/instance/get' - - 'cluster:admin/opendistro/reports/menu/download' - -# Allows users to read and download Reports and Report-definitions -reports_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/reports/definition/get' - - 'cluster:admin/opendistro/reports/definition/list' - - 'cluster:admin/opendistro/reports/instance/list' - - 'cluster:admin/opendistro/reports/instance/get' - - 'cluster:admin/opendistro/reports/menu/download' - -# Allows users to all Reports functionality -reports_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/reports/definition/create' - - 'cluster:admin/opendistro/reports/definition/update' - - 'cluster:admin/opendistro/reports/definition/on_demand' - - 'cluster:admin/opendistro/reports/definition/delete' - - 'cluster:admin/opendistro/reports/definition/get' - - 'cluster:admin/opendistro/reports/definition/list' - - 'cluster:admin/opendistro/reports/instance/list' - - 'cluster:admin/opendistro/reports/instance/get' - - 'cluster:admin/opendistro/reports/menu/download' - -# Allows users to use all asynchronous-search functionality -asynchronous_search_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/asynchronous_search/*' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:data/read/search*' - -# Allows users to read stored asynchronous-search results -asynchronous_search_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opendistro/asynchronous_search/get' - -# Allows user to use all index_management actions - ism policies, rollups, transforms -index_management_full_access: - reserved: true - cluster_permissions: - - "cluster:admin/opendistro/ism/*" - - "cluster:admin/opendistro/rollup/*" - - "cluster:admin/opendistro/transform/*" - - "cluster:admin/opensearch/controlcenter/lron/*" - - "cluster:admin/opensearch/notifications/channels/get" - - "cluster:admin/opensearch/notifications/feature/publish" - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/opensearch/ism/*' - -# Allows users to use all cross cluster replication functionality at leader cluster -cross_cluster_replication_leader_full_access: - reserved: true - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - "indices:admin/plugins/replication/index/setup/validate" - - "indices:data/read/plugins/replication/changes" - - "indices:data/read/plugins/replication/file_chunk" - -# Allows users to use all cross cluster replication functionality at follower cluster -cross_cluster_replication_follower_full_access: - reserved: true - cluster_permissions: - - "cluster:admin/plugins/replication/autofollow/update" - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - "indices:admin/plugins/replication/index/setup/validate" - - "indices:data/write/plugins/replication/changes" - - "indices:admin/plugins/replication/index/start" - - "indices:admin/plugins/replication/index/pause" - - "indices:admin/plugins/replication/index/resume" - - "indices:admin/plugins/replication/index/stop" - - "indices:admin/plugins/replication/index/update" - - "indices:admin/plugins/replication/index/status_check" - -# Allows users to use all cross cluster search functionality at remote cluster -cross_cluster_search_remote_full_access: - reserved: true - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/shards/search_shards' - - 'indices:data/read/search' - -# Allow users to operate query assistant -query_assistant_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/ml/config/get' - - 'cluster:admin/opensearch/ml/execute' - - 'cluster:admin/opensearch/ml/predict' - - 'cluster:admin/opensearch/ppl' - -# Allow users to read ML stats/models/tasks -ml_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/ml/config/get' - - 'cluster:admin/opensearch/ml/connectors/get' - - 'cluster:admin/opensearch/ml/connectors/search' - - 'cluster:admin/opensearch/ml/controllers/get' - - 'cluster:admin/opensearch/ml/memory/conversation/get' - - 'cluster:admin/opensearch/ml/memory/conversation/interaction/search' - - 'cluster:admin/opensearch/ml/memory/conversation/list' - - 'cluster:admin/opensearch/ml/memory/conversation/search' - - 'cluster:admin/opensearch/ml/memory/interaction/get' - - 'cluster:admin/opensearch/ml/memory/interaction/list' - - 'cluster:admin/opensearch/ml/memory/trace/get' - - 'cluster:admin/opensearch/ml/model_groups/get' - - 'cluster:admin/opensearch/ml/model_groups/search' - - 'cluster:admin/opensearch/ml/models/get' - - 'cluster:admin/opensearch/ml/models/search' - - 'cluster:admin/opensearch/ml/profile/nodes' - - 'cluster:admin/opensearch/ml/stats/nodes' - - 'cluster:admin/opensearch/ml/tasks/get' - - 'cluster:admin/opensearch/ml/tasks/search' - - 'cluster:admin/opensearch/ml/tools/get' - - 'cluster:admin/opensearch/ml/tools/list' - -# Allows users to use all ML functionality -ml_full_access: - reserved: true - cluster_permissions: - - 'cluster_monitor' - - 'cluster:admin/opensearch/ml/*' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices_monitor' - -# Allows users to use all Notifications functionality -notifications_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/notifications/*' - -# Allows users to read Notifications config/channels -notifications_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/notifications/configs/get' - - 'cluster:admin/opensearch/notifications/features' - - 'cluster:admin/opensearch/notifications/channels/get' - -# Allows users to use all snapshot management functionality -snapshot_management_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/snapshot_management/*' - - 'cluster:admin/opensearch/notifications/feature/publish' - - 'cluster:admin/repository/*' - - 'cluster:admin/snapshot/*' - -# Allows users to see snapshots, repositories, and snapshot management policies -snapshot_management_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/snapshot_management/policy/get' - - 'cluster:admin/opensearch/snapshot_management/policy/search' - - 'cluster:admin/opensearch/snapshot_management/policy/explain' - - 'cluster:admin/repository/get' - - 'cluster:admin/snapshot/get' - -# Allows user to use point in time functionality -point_in_time_full_access: - reserved: true - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'manage_point_in_time' - -# Allows users to see security analytics detectors and others -security_analytics_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/securityanalytics/alerts/get' - - 'cluster:admin/opensearch/securityanalytics/correlationAlerts/get' - - 'cluster:admin/opensearch/securityanalytics/correlations/findings' - - 'cluster:admin/opensearch/securityanalytics/correlations/list' - - 'cluster:admin/opensearch/securityanalytics/detector/get' - - 'cluster:admin/opensearch/securityanalytics/detector/search' - - 'cluster:admin/opensearch/securityanalytics/findings/get' - - 'cluster:admin/opensearch/securityanalytics/logtype/search' - - 'cluster:admin/opensearch/securityanalytics/mapping/get' - - 'cluster:admin/opensearch/securityanalytics/mapping/view/get' - - 'cluster:admin/opensearch/securityanalytics/rule/get' - - 'cluster:admin/opensearch/securityanalytics/rule/search' - - 'cluster:admin/opensearch/securityanalytics/threatintel/alerts/get' - - 'cluster:admin/opensearch/securityanalytics/threatintel/iocs/findings/get' - - 'cluster:admin/opensearch/securityanalytics/threatintel/iocs/list' - - 'cluster:admin/opensearch/securityanalytics/threatintel/monitors/search' - - 'cluster:admin/opensearch/securityanalytics/threatintel/sources/get' - - 'cluster:admin/opensearch/securityanalytics/threatintel/sources/search' - -# Allows users to use all security analytics functionality -security_analytics_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/securityanalytics/alerts/*' - - 'cluster:admin/opensearch/securityanalytics/connections/*' - - 'cluster:admin/opensearch/securityanalytics/correlationAlerts/*' - - 'cluster:admin/opensearch/securityanalytics/correlations/*' - - 'cluster:admin/opensearch/securityanalytics/detector/*' - - 'cluster:admin/opensearch/securityanalytics/findings/*' - - 'cluster:admin/opensearch/securityanalytics/logtype/*' - - 'cluster:admin/opensearch/securityanalytics/mapping/*' - - 'cluster:admin/opensearch/securityanalytics/rule/*' - - 'cluster:admin/opensearch/securityanalytics/threatintel/*' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/mapping/put' - - 'indices:admin/mappings/get' - -# Allows users to view and acknowledge alerts -security_analytics_ack_alerts: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/securityanalytics/alerts/*' - - 'cluster:admin/opensearch/securityanalytics/correlationAlerts/*' - - 'cluster:admin/opensearch/securityanalytics/threatintel/alerts/*' - -# Allows users to use all Flow Framework functionality -flow_framework_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/flow_framework/*' - - 'cluster_monitor' - index_permissions: - - index_patterns: - - '*' - allowed_actions: - - 'indices:admin/aliases/get' - - 'indices:admin/mappings/get' - - 'indices_monitor' - -# Allow users to read flow framework's workflows and their state -flow_framework_read_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/flow_framework/workflow/get' - - 'cluster:admin/opensearch/flow_framework/workflow/search' - - 'cluster:admin/opensearch/flow_framework/workflow_state/get' - - 'cluster:admin/opensearch/flow_framework/workflow_state/search' - - 'cluster:admin/opensearch/flow_framework/workflow_step/get' - -# Allows users to use all query insights APIs -query_insights_full_access: - reserved: true - cluster_permissions: - - 'cluster:admin/opensearch/insights/top_queries/*' - index_permissions: - - index_patterns: - - 'top_queries_by_*' - allowed_actions: - - "indices_all" - -# Allow users to execute read only LTR actions -ltr_read_access: - reserved: true - cluster_permissions: - - cluster:admin/ltr/caches/stats - - cluster:admin/ltr/featurestore/list - - cluster:admin/ltr/stats - -# Allow users to execute all LTR actions -ltr_full_access: - reserved: true - cluster_permissions: - - cluster:admin/ltr/* - -# Custom dashboard role to make the all_access role work -dashboards_system_access: - cluster_permissions: - - "cluster:monitor/*" - index_permissions: - - index_patterns: - - ".opensearch_dashboards*" - allowed_actions: - - "system:admin/system_index" - - "indices:admin/*" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/create" - - index_patterns: - - ".opensearch-notifications*" - allowed_actions: - - "system:admin/system_index" - - "indices:admin/*" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/create" - - tenant_permissions: - - tenant_patterns: - - "*" - allowed_actions: - - "kibana_all_read" - - "kibana_all_write" - -######################################################################################################################################################################## -######################################################################################################################################################################## -######################################################################################################################################################################## -######################################################################################################################################################################## -## Sample role -complex-role: - reserved: false - hidden: false - cluster_permissions: - - "read" - - "cluster:monitor/nodes/stats" - - "cluster:monitor/task/get" - index_permissions: - - index_patterns: - - "opensearch_dashboards_sample_data_*" - dls: "{\"match\": {\"FlightDelay\": true}}" - fls: - - "~FlightNum" - masked_fields: - - "Carrier" - allowed_actions: - - "read" - tenant_permissions: - - tenant_patterns: - - "analyst_*" - allowed_actions: - - "kibana_all_write" - static: false diff --git a/deploy/charts/opensearch/files/opensearch-security/roles.yml b/deploy/charts/opensearch/files/opensearch-security/roles.yml new file mode 120000 index 00000000..3f883a84 --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch-security/roles.yml @@ -0,0 +1 @@ +../../../../../security/es_roles/opensearch/roles.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml b/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml deleted file mode 100644 index ba604379..00000000 --- a/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml +++ /dev/null @@ -1,90 +0,0 @@ -# In this file users, backendroles and hosts can be mapped to Security roles. -# Permissions for OpenSearch roles are configured in roles.yml - -_meta: - type: "rolesmapping" - config_version: 2 - -admin: - reserved: true - backend_roles: - - admin - hosts: [] - users: - - admin - and_backend_roles: [] - -dashboards_system_access: - users: - - "admin" - backend_roles: - - "admin" - - "all_access" - -all_access: - reserved: true - hidden: false - backend_roles: - - "admin" - hosts: [] - users: [ "admin" ] - and_backend_roles: [""] - description: "Maps admin to all_access" - -own_index: - reserved: false - hidden: false - backend_roles: [] - hosts: [] - users: - - "*" - and_backend_roles: [] - description: "Allow full access to an index named like the username" - -logstash: - reserved: false - backend_roles: - - "logstash" - -kibana_user: - reserved: false - backend_roles: - - "kibanauser" - description: "Maps kibanauser to kibana_user" - -readall: - reserved: true - hidden: false - backend_roles: - - "readall" - hosts: [] - users: [] - and_backend_roles: [] - -manage_snapshots: - reserved: true - hidden: false - backend_roles: - - "snapshotrestore" - hosts: [] - users: [] - and_backend_roles: [] - -kibana_server: - reserved: true - hidden: false - backend_roles: [] - hosts: [] - users: ["kibanaserver"] - and_backend_roles: [] - -# sample role -complex-role: - reserved: false - hidden: false - backend_roles: - - "ldap-analyst" - hosts: [] - users: - - "new-user" - and_backend_roles: [] diff --git a/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml b/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml new file mode 120000 index 00000000..3fc4568e --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch-security/roles_mapping.yml @@ -0,0 +1 @@ +../../../../../security/es_roles/opensearch/roles_mapping.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch.yml b/deploy/charts/opensearch/files/opensearch.yml deleted file mode 100644 index bbc9fd3c..00000000 --- a/deploy/charts/opensearch/files/opensearch.yml +++ /dev/null @@ -1,218 +0,0 @@ -network.host: ${ELASTICSEARCH_NETWORK_HOST} -network.publish_host: ${ELASTICSEARCH_NETWORK_PUBLISH_HOST} - -###### CLUSTER CONFIGURATION ###### - -bootstrap.system_call_filter: false - -# To allow multiple nodes add the node"s IP addresses to the discovery pool. -# ["", "", ""] -discovery.seed_hosts: ${ELASTICSEARCH_SEED_HOSTS} -cluster.initial_cluster_manager_nodes: ${ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES} -discovery.type: zen - -# WARNING: revise all the lines below before you go into production -plugins.security.ssl.transport.pemcert_filepath: esnode.crt -plugins.security.ssl.transport.pemkey_filepath: esnode.key -plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.crt -plugins.security.ssl.transport.enforce_hostname_verification: false -#plugins.security.ssl.transport.keystore_filepath: esnode-pcks12.key -#plugins.security.ssl.transport.truststore_filepath: esnode-pcks12.key -plugins.security.ssl.http.enabled: true -plugins.security.ssl.transport.enabled: true -#plugins.security.ssl.http.keystore_type: PKCS12 -plugins.security.ssl.http.pemcert_filepath: esnode.crt -plugins.security.ssl.http.pemkey_filepath: esnode.key -plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.crt -plugins.security.allow_unsafe_democertificates: false -plugins.security.allow_default_init_securityindex: true - -plugins.security.authcz.admin_dn: - - "CN=admin,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=cogstack,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-1,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-2,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-3,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - CN=admin - - CN=localhost - - CN=nifi - - CN=cogstack - -# IMPORTANT: Make sure you define the CN (Company Name) settings correctly, below you can see they match the server addresses of the containers i.e: elasticsearch-1, elasticsearch-2, -# and also extra patterns defined, these could also be domain names etc. -plugins.security.nodes_dn: - - "CN=cogstack,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-1,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-2,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=elasticsearch-3,OU=cogstack,O=cogstack,L=UK,ST=London,C=UK" - - "CN=nifi" - - "CN=esnode*" - - "CN=localhost" - - "CN=cogstack" - - "CN=elasticsearch" - -plugins.security.enable_snapshot_restore_privilege: true -plugins.security.check_snapshot_restore_write_privileges: true -cluster.routing.allocation.disk.threshold_enabled: true - -plugins.security.ssl.http.clientauth_mode : OPTIONAL - -plugins.security.roles_mapping_resolution: MAPPING_ONLY - -############## REST Management API configuration settings ############## -# Enable or disable role based access to the REST management API -# Default is that no role is allowed to access the REST management API. -plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] - -# Disable particular endpoints and their HTTP methods for roles. -# By default all endpoints/methods are allowed. -#plugins.security.restapi.endpoints_disabled..: -# Example: -#plugins.security.restapi.endpoints_disabled.all_access.ACTIONGROUPS: ["PUT","POST","DELETE"] -#plugins.security.restapi.endpoints_disabled.xyz_role.LICENSE: ["DELETE"] - -# The following endpoints exist: -# ACTIONGROUPS -# CACHE -# CONFIG -# ROLES -# ROLESMAPPING -# INTERNALUSERS -# SYSTEMINFO -# PERMISSIONSINFO - -############## Auditlog configuration settings ############## - -# Destination of the auditlog events -plugins.security.audit.type: internal_opensearch -plugins.security.audit.config.http_endpoints: ${ELASTICSEARCH_HOSTS} -plugins.security.audit.config.enable_ssl_client_auth: true - -#plugins.security.audit.enable_rest: false -#plugins.security.audit.enable_transport: false -#plugins.security.audit.config.disabled_rest_categories: NONE -#plugins.security.audit.config.disabled_transport_categories: NONE -#plugins.security.audit.config.index: auditlog6 # make sure you secure this index properly -#plugins.security.audit.config.index: "'auditlog6-'YYYY.MM.dd" #rotates index daily - make sure you secure this index properly -#plugins.security.audit.config.type: auditlog -#plugins.security.audit.config.username: auditloguser -#plugins.security.audit.config.password: auditlogpassword -#plugins.security.audit.config.enable_ssl: false -#plugins.security.audit.config.verify_hostnames: false -#plugins.security.audit.config.cert_alias: mycert -#plugins.security.audit.config.pemkey_filepath: key.pem -#plugins.security.audit.config.pemkey_content: <...pem base 64 content> -#plugins.security.audit.config.pemkey_password: secret -#plugins.security.audit.config.pemcert_filepath: cert.pem -#plugins.security.audit.config.pemcert_content: <...pem base 64 content> -#plugins.security.audit.config.pemtrustedcas_filepath: ca.pem -#plugins.security.audit.config.pemtrustedcas_content: <...pem base 64 content> - -# webhook settings -#plugins.security.audit.config.webhook.url: "http://mywebhook/endpoint" -# One of URL_PARAMETER_GET,URL_PARAMETER_POST,TEXT,JSON,SLACK -#plugins.security.audit.config.webhook.format: JSON -#plugins.security.audit.config.webhook.ssl.verify: false -#plugins.security.audit.config.webhook.ssl.pemtrustedcas_filepath: ca.pem -#plugins.security.audit.config.webhook.ssl.pemtrustedcas_content: <...pem base 64 content> - -# log4j settings -#plugins.security.audit.config.log4j.logger_name: auditlogger -#plugins.security.audit.config.log4j.level: INFO - -############## Kerberos configuration settings ############## -# If Kerberos authentication should be used you have to configure: - -# The Path to the krb5.conf file -# Can be absolute or relative to the OpenSearch config directory -#plugins.security.kerberos.krb5_filepath: '/etc/krb5.conf' - -# The Path to the keytab where the acceptor_principal credentials are stored. -# Must be relative to the OpenSearch config directory -#plugins.security.kerberos.acceptor_keytab_filepath: 'eskeytab.tab' - -# Acceptor (Server) Principal name, must be present in acceptor_keytab_path file -#plugins.security.kerberos.acceptor_principal: 'HTTP/localhost' - - -############## Expert settings ############## -# WARNING: Expert settings, do only use if you know what you are doing -# If you set wrong values here this this could be a security risk -# or make OpenSearch Security stop working - -# Name of the index where .opendistro_security stores its configuration. - -#plugins.security.config_index_name: .opendistro_security - -# This defines the OID of server node certificates -#plugins.security.cert.oid: '1.2.3.4.5.5' - -# This specifies the implementation of org.opensearch.security.transport.InterClusterRequestEvaluator -# that is used to determine inter-cluster request. -# Instances of org.opensearch.security.transport.InterClusterRequestEvaluator must implement a single argument -# constructor that takes an org.opensearch.common.settings.Settings -#plugins.security.cert.intercluster_request_evaluator_class: org.opensearch.security.transport.DefaultInterClusterRequestEvaluator - -# By default, normal users can restore snapshots if they have the priviliges 'cluster:admin/snapshot/restore', -# 'indices:admin/create', and 'indices:data/write/index' for the indices to be restored. -# To disable snapshot restore for normal users set 'plugins.security.enable_snapshot_restore_privilege: false'. -# This makes it so that only snapshot restore requests signed by an admin TLS certificate are accepted. -# A snapshot can only be restored when it does not contain global state and does not restore the '.opendistro_security' index -# If 'plugins.security.check_snapshot_restore_write_privileges: false' is set then the additional indices checks are omitted. -#plugins.security.enable_snapshot_restore_privilege: true -#plugins.security.check_snapshot_restore_write_privileges: true - -# Authentication cache timeout in minutes (A value of 0 disables caching, default is 60) -#plugins.security.cache.ttl_minutes: 60 - -# Disable OpenSearch Security -# WARNING: This can expose your configuration (including passwords) to the public. -#plugins.security.disabled: false - -# Protected indices are even more secure than normal indices. These indices require a role to access like any other index, but they require an additional role -# to be visible, listed in the plugins.security.protected_indices.roles setting. -# Enable protected indices -# plugins.security.protected_indices.enabled: true -# Specify a list of roles a user must be member of to touch any protected index. -# plugins.security.protected_indices.roles: ['all_access'] - -# System indices are similar to security index, except the contents are not encrypted. -# Indices configured as system indices can be accessed by only super-admin and no role will provide access to these indices. -# Enable system indices -plugins.security.system_indices.enabled: true -plugins.security.system_indices.permission.enabled: true - - -# Enable SQL-like queries, allows you to join across indices etc. -plugins.sql.enabled: true -plugins.ppl.enabled: true - -# Calcite (SQL v3 engine) -plugins.calcite.enabled: true -# WARNING: exists only in OpenSearch > 3.4, please DISABLE if you are not on this version as this setting doesn't exist. -plugins.calcite.all_join_types.allowed: true - -# Specify a list of indices to mark as system. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. -plugins.security.system_indices.indices: [ - ".opensearch_dashboards*", - ".opensearch-alerting-config", - ".opensearch-notifications-*", - ".opendistro-ism-*", - ".opendistro-reports-*", - ".opendistro-asynchronous-search-response*", - ".replication-metadata-store", - ".opendistro-notebooks", - ".opendistro-anomaly-results*", - ".opendistro-anomaly-detector*", - ".opendistro-anomaly-checkpoints", - ".opendistro-anomaly-detection-state" -] - -# coordinator-level kill switch for searches -search.cancel_after_time_interval: 5m - -# shard-level default timeout (can be overridden per request via "timeout") -search.default_search_timeout: 5m - -# relevant if SQL is paging via PIT -search.default_keep_alive: 10m diff --git a/deploy/charts/opensearch/files/opensearch.yml b/deploy/charts/opensearch/files/opensearch.yml new file mode 120000 index 00000000..c7b0afca --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch.yml @@ -0,0 +1 @@ +../../../../services/elasticsearch/config/opensearch.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/opensearch_dashboards.yml b/deploy/charts/opensearch/files/opensearch_dashboards.yml deleted file mode 100644 index 72a3103d..00000000 --- a/deploy/charts/opensearch/files/opensearch_dashboards.yml +++ /dev/null @@ -1,147 +0,0 @@ -# OpenSearch Dashboards is served by a back end server. This setting specifies the port to use. -server.port: 5601 - -# Specifies the address to which the OpenSearch Dashboards server will bind. IP addresses and host names are both valid values. -# The default is 'localhost', which usually means remote machines will not be able to connect. -# To allow connections from remote users, set this parameter to a non-loopback address. -server.host: "0.0.0.0" - -# Enables you to specify a path to mount OpenSearch Dashboards at if you are running behind a proxy. -# Use the `server.rewriteBasePath` setting to tell OpenSearch Dashboards if it should remove the basePath -# from requests it receives, and to prevent a deprecation warning at startup. -# This setting cannot end in a slash. -#server.basePath: "" - -# Specifies whether OpenSearch Dashboards should rewrite requests that are prefixed with -# `server.basePath` or require that they are rewritten by your reverse proxy. -#server.rewriteBasePath: false - -# The maximum payload size in bytes for incoming server requests. -#server.maxPayloadBytes: 1048576 - -# The OpenSearch Dashboards server's name. This is used for display purposes. -server.name: ${KIBANA_SERVER_NAME} - -# The URLs of the OpenSearch instances to use for all your queries. -opensearch.hosts: ${ELASTICSEARCH_HOSTS} - -opensearch_security.multitenancy.enabled: true -opensearch_security.multitenancy.tenants.enable_global: true -opensearch_security.multitenancy.tenants.enable_private: true -opensearch_security.multitenancy.tenants.preferred: ["Global"] -opensearch_security.readonly_mode.roles: ["kibana_read_only"] -opensearch_security.multitenancy.enable_filter: true -opensearch_security.cookie.secure: true - -# OpenSearch Dashboards uses an index in OpenSearch to store saved searches, visualizations and -# dashboards. OpenSearch Dashboards creates a new index if the index doesn't already exist. - -# We changed this from .opensearch_dashboards to .kibana because of a permission bug in OpenSearch Dashboards 2.19.x and 3.0.0 -# report: https://github.com/opensearch-project/security/issues/5360 -opensearchDashboards.index: ".opensearch_dashboards" - -# The default application to load. -#opensearchDashboards.defaultAppId: "home" - -# Setting for an optimized healthcheck that only uses the local OpenSearch node to do Dashboards healthcheck. -# This settings should be used for large clusters or for clusters with ingest heavy nodes. -# It allows Dashboards to only healthcheck using the local OpenSearch node rather than fan out requests across all nodes. -# -# It requires the user to create an OpenSearch node attribute with the same name as the value used in the setting -# This node attribute should assign all nodes of the same cluster an integer value that increments with each new cluster that is spun up -# e.g. in opensearch.yml file you would set the value to a setting using node.attr.cluster_id: -# Should only be enabled if there is a corresponding node attribute created in your OpenSearch config that matches the value here -#opensearch.optimizedHealthcheckId: "cluster_id" - -# If your OpenSearch is protected with basic authentication, these settings provide -# the username and password that the OpenSearch Dashboards server uses to perform maintenance on the OpenSearch Dashboards -# index at startup. Your OpenSearch Dashboards users still need to authenticate with OpenSearch, which -# is proxied through the OpenSearch Dashboards server. -opensearch.username: ${KIBANA_USER:-"kibanaserver"} -opensearch.password: ${KIBANA_PASSWORD:-"kibanaserver"} - -# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. -# These settings enable SSL for outgoing requests from the OpenSearch Dashboards server to the browser. -server.ssl.enabled: true -server.ssl.certificate: "/certificates/elastic/opensearch/es_kibana_client.pem" -server.ssl.key: "/certificates/elastic/opensearch/es_kibana_client.key" - -# Optional settings that provide the paths to the PEM-format SSL certificate and key files. -# These files are used to verify the identity of OpenSearch Dashboards to OpenSearch and are required when -# xpack.security.http.ssl.client_authentication in OpenSearch is set to required. -opensearch.ssl.certificate: "/certificates/elastic/opensearch/admin.crt" -opensearch.ssl.key: "/certificates/elastic/opensearch/admin.key.pem" - -# Optional setting that enables you to specify a path to the PEM file for the certificate -# authority for your OpenSearch instance. -opensearch.ssl.certificateAuthorities: ["/certificates/elastic/opensearch/elastic-stack-ca.crt.pem"] - -# To disregard the validity of SSL certificates, change this setting's value to 'none'. -opensearch.ssl.verificationMode: full - -# Time in milliseconds to wait for OpenSearch to respond to pings. Defaults to the value of -# the opensearch.requestTimeout setting. -opensearch.pingTimeout: 10000 - -# Time in milliseconds to wait for responses from the back end or OpenSearch. This value -# must be a positive integer. -opensearch.requestTimeout: 200000 - -# List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side -# headers, set this value to [] (an empty list). -opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] - -# Header names and values that are sent to OpenSearch. Any custom headers cannot be overwritten -# by client-side headers, regardless of the opensearch.requestHeadersAllowlist configuration. -# opensearch.customHeaders: {} - -# Time in milliseconds for OpenSearch to wait for responses from shards. Set to 0 to disable. -# opensearch.shardTimeout: 30000 - -# Logs queries sent to OpenSearch. Requires logging.verbose set to true. -# opensearch.logQueries: false - -# Specifies the path where OpenSearch Dashboards creates the process ID file. -# pid.file: /var/run/opensearchDashboards.pid - -# Enables you to specify a file where OpenSearch Dashboards stores log output. -# logging.dest: stdout - -# Set the value of this setting to true to suppress all logging output. -# logging.silent: false - -# Set the value of this setting to true to suppress all logging output other than error messages. -# logging.quiet: false - -# Set the value of this setting to true to log all events, including system usage information -# and all requests. -# logging.verbose: false - -# Set the interval in milliseconds to sample system and process performance -# metrics. Minimum is 100ms. Defaults to 5000. -# ops.interval: 5000 - -# Specifies locale to be used for all localizable strings, dates and number formats. -# Supported languages are the following: English - en , by default , Chinese - zh-CN . -# i18n.locale: "en" - -# Set the allowlist to check input graphite Url. Allowlist is the default check list. -#vis_type_timeline.graphiteAllowedUrls: ['https://www.hostedgraphite.com/UID/ACCESS_KEY/graphite'] - -# Set the value of this setting to true to capture region blocked warnings and errors -# for your map rendering services. -# map.showRegionBlockedWarning: false - -ml_commons_dashboards.enabled: true - -savedObjects.permission.enabled: true - -# Set the value to true to enable multiple data source feature -data_source.enabled: true -# Set the value to true to enable workspace feature -workspace.enabled: true -# Set the value to true to enable explore feature -explore.enabled: true - -opensearchDashboards.dashboardAdmin.users: ["admin"] -opensearchDashboards.dashboardAdmin.groups: ["admin","all_access"] diff --git a/deploy/charts/opensearch/files/opensearch_dashboards.yml b/deploy/charts/opensearch/files/opensearch_dashboards.yml new file mode 120000 index 00000000..b0681d40 --- /dev/null +++ b/deploy/charts/opensearch/files/opensearch_dashboards.yml @@ -0,0 +1 @@ +../../../../services/kibana/config/opensearch.yml \ No newline at end of file diff --git a/deploy/charts/opensearch/files/users-elasticsearch.envfile b/deploy/charts/opensearch/files/users-elasticsearch.envfile new file mode 120000 index 00000000..305682ac --- /dev/null +++ b/deploy/charts/opensearch/files/users-elasticsearch.envfile @@ -0,0 +1 @@ +../../../../security/env/users_elasticsearch.env \ No newline at end of file diff --git a/deploy/charts/opensearch/templates/NOTES.txt b/deploy/charts/opensearch/templates/NOTES.txt index a824b0c2..1b9c1928 100644 --- a/deploy/charts/opensearch/templates/NOTES.txt +++ b/deploy/charts/opensearch/templates/NOTES.txt @@ -1,13 +1,21 @@ +{{- if .Values.opensearch.enabled }} 1. OpenSearch client service: {{ include "cogstack-opensearch.clientServiceName" . }}.{{ .Release.Namespace }}.svc +{{- end }} +{{- if .Values.dashboards.enabled }} 2. OpenSearch Dashboards service (if enabled): {{ include "cogstack-opensearch.dashboardsServiceName" . }}.{{ .Release.Namespace }}.svc +{{- end }} 3. This chart expects TLS and security cert files in Kubernetes Secrets. Configure: +{{- if .Values.opensearch.enabled }} - .Values.certificates.opensearchSecretName +{{- end }} +{{- if .Values.dashboards.enabled }} - .Values.certificates.dashboardsSecretName +{{- end }} 4. Credentials are read from Secret: {{ include "cogstack-opensearch.credentialsSecretName" . }} diff --git a/deploy/charts/opensearch/templates/_helpers.tpl b/deploy/charts/opensearch/templates/_helpers.tpl index 811d3af5..b0ff02bd 100644 --- a/deploy/charts/opensearch/templates/_helpers.tpl +++ b/deploy/charts/opensearch/templates/_helpers.tpl @@ -64,9 +64,103 @@ app.kubernetes.io/component: dashboards {{- end -}} {{- end -}} -{{/* Env ConfigMap name */}} -{{- define "cogstack-opensearch.envConfigMapName" -}} -{{- printf "%s-elasticsearch-env" (include "cogstack-opensearch.fullname" .) -}} +{{/* Snapshot backup PVC names */}} +{{- define "cogstack-opensearch.snapshotBackupDataPvcName" -}} +{{- printf "%s-snapshot-backup-data" (include "cogstack-opensearch.fullname" .) -}} +{{- end -}} + +{{- define "cogstack-opensearch.snapshotBackupConfigPvcName" -}} +{{- printf "%s-snapshot-backup-config" (include "cogstack-opensearch.fullname" .) -}} +{{- end -}} + +{{/* Parse deploy/elasticsearch.env into a filtered YAML map */}} +{{- define "cogstack-opensearch.parsedEnvFile" -}} +{{- $root := . -}} +{{- $envData := dict -}} +{{- $rawEnv := $root.Values.envFile.raw | default ($root.Files.Get "files/deploy-elasticsearch.envfile") -}} +{{- if $rawEnv -}} +{{- $renderedEnv := tpl $rawEnv $root -}} +{{- range $line := splitList "\n" $renderedEnv }} + {{- $clean := trim (replace "\r" "" $line) -}} + {{- if and $clean (not (hasPrefix "#" $clean)) -}} + {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} + {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} + {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} + {{- if has $key $root.Values.envFile.includeKeys -}} + {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} + {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} + {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} + {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} + {{- end -}} + {{- if regexMatch "^\\$[A-Za-z_][A-Za-z0-9_]*$" $val -}} + {{- $refKey := trimPrefix "$" $val -}} + {{- if hasKey $envData $refKey -}} + {{- $val = index $envData $refKey -}} + {{- end -}} + {{- end -}} + {{- $_ := set $envData $key $val -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{ toYaml $envData }} +{{- end -}} + +{{/* Parse security/env/users_elasticsearch.env into a filtered YAML map */}} +{{- define "cogstack-opensearch.parsedUsersEnvFile" -}} +{{- $root := . -}} +{{- $usersData := dict -}} +{{- $rawUsers := $root.Values.usersEnvFile.raw | default ($root.Files.Get "files/users-elasticsearch.envfile") -}} +{{- if $rawUsers -}} +{{- $renderedUsers := tpl $rawUsers $root -}} +{{- range $line := splitList "\n" $renderedUsers }} + {{- $clean := trim (replace "\r" "" $line) -}} + {{- if and $clean (not (hasPrefix "#" $clean)) -}} + {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} + {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} + {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} + {{- if has $key $root.Values.usersEnvFile.includeKeys -}} + {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} + {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} + {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} + {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} + {{- end -}} + {{- $_ := set $usersData $key $val -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{ toYaml $usersData }} +{{- end -}} + +{{/* Parse security/env/certificates_elasticsearch.env into a filtered YAML map */}} +{{- define "cogstack-opensearch.parsedCertificatesEnvFile" -}} +{{- $root := . -}} +{{- $certData := dict -}} +{{- $rawCerts := $root.Values.certificatesEnvFile.raw | default ($root.Files.Get "files/certificates-elasticsearch.envfile") -}} +{{- if $rawCerts -}} +{{- $renderedCerts := tpl $rawCerts $root -}} +{{- range $line := splitList "\n" $renderedCerts }} + {{- $clean := trim (replace "\r" "" $line) -}} + {{- if and $clean (not (hasPrefix "#" $clean)) -}} + {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} + {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} + {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} + {{- if has $key $root.Values.certificatesEnvFile.includeKeys -}} + {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} + {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} + {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} + {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} + {{- end -}} + {{- $_ := set $certData $key $val -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{ toYaml $certData }} {{- end -}} {{/* CSV of StatefulSet pod DNS names used for discovery.seed_hosts */}} @@ -91,7 +185,15 @@ app.kubernetes.io/component: dashboards {{/* JSON array for ELASTICSEARCH_HOSTS */}} {{- define "cogstack-opensearch.elasticsearchHostsJson" -}} +{{- if .Values.opensearch.enabled -}} {{- $svc := include "cogstack-opensearch.clientServiceName" . -}} {{- $ns := .Release.Namespace -}} {{- printf "[\"https://%s.%s.svc:%d\"]" $svc $ns (int .Values.opensearch.service.httpPort) -}} +{{- else -}} +{{- if .Values.dashboards.opensearchHosts -}} +{{ toJson .Values.dashboards.opensearchHosts }} +{{- else -}} +{{- fail "dashboards.opensearchHosts must be set when opensearch.enabled=false" -}} +{{- end -}} +{{- end -}} {{- end -}} diff --git a/deploy/charts/opensearch/templates/configmap-dashboards-config.yaml b/deploy/charts/opensearch/templates/configmap-dashboards-config.yaml index 693bc120..a8615a2c 100644 --- a/deploy/charts/opensearch/templates/configmap-dashboards-config.yaml +++ b/deploy/charts/opensearch/templates/configmap-dashboards-config.yaml @@ -1,4 +1,5 @@ {{- if .Values.dashboards.enabled }} +{{- $dashboardsConfigRaw := .Values.configFiles.dashboardsRaw | default (.Files.Get "files/opensearch_dashboards.yml") -}} apiVersion: v1 kind: ConfigMap metadata: @@ -7,5 +8,5 @@ metadata: {{- include "cogstack-opensearch.labels" . | nindent 4 }} data: opensearch_dashboards.yml: | -{{ .Files.Get "files/opensearch_dashboards.yml" | indent 4 }} +{{ $dashboardsConfigRaw | indent 4 }} {{- end }} diff --git a/deploy/charts/opensearch/templates/configmap-elasticsearch-env.yaml b/deploy/charts/opensearch/templates/configmap-elasticsearch-env.yaml deleted file mode 100644 index 69b047c2..00000000 --- a/deploy/charts/opensearch/templates/configmap-elasticsearch-env.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.envFile.raw }} -{{- $envData := dict -}} -{{- $renderedEnv := tpl .Values.envFile.raw . -}} -{{- range $line := splitList "\n" $renderedEnv }} - {{- $clean := trim (replace "\r" "" $line) -}} - {{- if and $clean (not (hasPrefix "#" $clean)) -}} - {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} - {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} - {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} - {{- if has $key $.Values.envFile.includeKeys -}} - {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} - {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} - {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} - {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} - {{- end -}} - {{- $_ := set $envData $key $val -}} - {{- end -}} - {{- end -}} - {{- end -}} -{{- end }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "cogstack-opensearch.envConfigMapName" . }} - labels: - {{- include "cogstack-opensearch.labels" . | nindent 4 }} -data: - {{- toYaml $envData | nindent 2 }} -{{- end }} diff --git a/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml b/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml index 76cb9a35..b05d2843 100644 --- a/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml +++ b/deploy/charts/opensearch/templates/configmap-opensearch-config.yaml @@ -1,3 +1,6 @@ +{{- if .Values.opensearch.enabled }} +{{- $opensearchConfigRaw := .Values.configFiles.opensearchRaw | default (.Files.Get "files/opensearch.yml") -}} +{{- $log4jRaw := .Values.configFiles.log4jRaw | default (.Files.Get "files/log4j2.properties") -}} apiVersion: v1 kind: ConfigMap metadata: @@ -6,6 +9,7 @@ metadata: {{- include "cogstack-opensearch.labels" . | nindent 4 }} data: opensearch.yml: | -{{ .Files.Get "files/opensearch.yml" | indent 4 }} +{{ $opensearchConfigRaw | indent 4 }} log4j2.properties: | -{{ .Files.Get "files/log4j2.properties" | indent 4 }} +{{ $log4jRaw | indent 4 }} +{{- end }} diff --git a/deploy/charts/opensearch/templates/configmap-opensearch-security.yaml b/deploy/charts/opensearch/templates/configmap-opensearch-security.yaml index 7c8bc09a..acf1e95e 100644 --- a/deploy/charts/opensearch/templates/configmap-opensearch-security.yaml +++ b/deploy/charts/opensearch/templates/configmap-opensearch-security.yaml @@ -1,3 +1,8 @@ +{{- if .Values.opensearch.enabled }} +{{- $configRaw := .Values.securityFiles.configRaw | default (.Files.Get "files/opensearch-security/config.yml") -}} +{{- $internalUsersRaw := .Values.securityFiles.internalUsersRaw | default (.Files.Get "files/opensearch-security/internal_users.yml") -}} +{{- $rolesRaw := .Values.securityFiles.rolesRaw | default (.Files.Get "files/opensearch-security/roles.yml") -}} +{{- $rolesMappingRaw := .Values.securityFiles.rolesMappingRaw | default (.Files.Get "files/opensearch-security/roles_mapping.yml") -}} apiVersion: v1 kind: ConfigMap metadata: @@ -6,10 +11,11 @@ metadata: {{- include "cogstack-opensearch.labels" . | nindent 4 }} data: config.yml: | -{{ .Files.Get "files/opensearch-security/config.yml" | indent 4 }} +{{ $configRaw | indent 4 }} internal_users.yml: | -{{ .Files.Get "files/opensearch-security/internal_users.yml" | indent 4 }} +{{ $internalUsersRaw | indent 4 }} roles.yml: | -{{ .Files.Get "files/opensearch-security/roles.yml" | indent 4 }} +{{ $rolesRaw | indent 4 }} roles_mapping.yml: | -{{ .Files.Get "files/opensearch-security/roles_mapping.yml" | indent 4 }} +{{ $rolesMappingRaw | indent 4 }} +{{- end }} diff --git a/deploy/charts/opensearch/templates/deployment-dashboards.yaml b/deploy/charts/opensearch/templates/deployment-dashboards.yaml index 5d52f095..df46ebea 100644 --- a/deploy/charts/opensearch/templates/deployment-dashboards.yaml +++ b/deploy/charts/opensearch/templates/deployment-dashboards.yaml @@ -1,4 +1,17 @@ {{- if .Values.dashboards.enabled }} +{{- $envData := include "cogstack-opensearch.parsedEnvFile" . | fromYaml | default (dict) -}} +{{- $certData := include "cogstack-opensearch.parsedCertificatesEnvFile" . | fromYaml | default (dict) -}} +{{- $serverName := .Values.dashboards.serverName -}} +{{- if hasKey $envData "KIBANA_SERVER_NAME" -}} +{{- $serverName = index $envData "KIBANA_SERVER_NAME" -}} +{{- end -}} +{{- $dashboardsServerCert := .Values.certificates.dashboardsFiles.serverCert -}} +{{- $dashboardsServerKey := .Values.certificates.dashboardsFiles.serverKey -}} +{{- if hasKey $certData "ES_CLIENT_CERT_NAME" -}} +{{- $clientCertName := index $certData "ES_CLIENT_CERT_NAME" -}} +{{- $dashboardsServerCert = printf "%s.pem" $clientCertName -}} +{{- $dashboardsServerKey = printf "%s.key" $clientCertName -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -31,14 +44,9 @@ spec: - name: http containerPort: {{ .Values.dashboards.service.port }} protocol: TCP - {{- if .Values.envFile.raw }} - envFrom: - - configMapRef: - name: {{ include "cogstack-opensearch.envConfigMapName" . }} - {{- end }} env: - name: KIBANA_SERVER_NAME - value: {{ .Values.dashboards.serverName | quote }} + value: {{ $serverName | quote }} - name: ELASTICSEARCH_HOSTS value: {{ include "cogstack-opensearch.elasticsearchHostsJson" . | quote }} - name: KIBANA_USER @@ -88,11 +96,11 @@ spec: readOnly: true - name: dashboards-certs mountPath: /certificates/elastic/opensearch/es_kibana_client.pem - subPath: {{ .Values.certificates.dashboardsFiles.serverCert }} + subPath: {{ $dashboardsServerCert }} readOnly: true - name: dashboards-certs mountPath: /certificates/elastic/opensearch/es_kibana_client.key - subPath: {{ .Values.certificates.dashboardsFiles.serverKey }} + subPath: {{ $dashboardsServerKey }} readOnly: true volumes: - name: dashboards-config diff --git a/deploy/charts/opensearch/templates/pvc-snapshot-backups.yaml b/deploy/charts/opensearch/templates/pvc-snapshot-backups.yaml new file mode 100644 index 00000000..87339b5c --- /dev/null +++ b/deploy/charts/opensearch/templates/pvc-snapshot-backups.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.opensearch.enabled .Values.opensearch.snapshotBackups.enabled }} +{{- if not .Values.opensearch.snapshotBackups.data.existingClaim }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cogstack-opensearch.snapshotBackupDataPvcName" . }} + labels: + {{- include "cogstack-opensearch.labels" . | nindent 4 }} + app.kubernetes.io/component: opensearch +spec: + accessModes: + {{- toYaml .Values.opensearch.snapshotBackups.data.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.opensearch.snapshotBackups.data.size }} + {{- if .Values.opensearch.snapshotBackups.data.storageClassName }} + storageClassName: {{ .Values.opensearch.snapshotBackups.data.storageClassName | quote }} + {{- end }} +{{- end }} +{{- if and (not .Values.opensearch.snapshotBackups.data.existingClaim) (not .Values.opensearch.snapshotBackups.config.existingClaim) }} +--- +{{- end }} +{{- if not .Values.opensearch.snapshotBackups.config.existingClaim }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "cogstack-opensearch.snapshotBackupConfigPvcName" . }} + labels: + {{- include "cogstack-opensearch.labels" . | nindent 4 }} + app.kubernetes.io/component: opensearch +spec: + accessModes: + {{- toYaml .Values.opensearch.snapshotBackups.config.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.opensearch.snapshotBackups.config.size }} + {{- if .Values.opensearch.snapshotBackups.config.storageClassName }} + storageClassName: {{ .Values.opensearch.snapshotBackups.config.storageClassName | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/charts/opensearch/templates/secret-credentials.yaml b/deploy/charts/opensearch/templates/secret-credentials.yaml index da14822e..e4c320a4 100644 --- a/deploy/charts/opensearch/templates/secret-credentials.yaml +++ b/deploy/charts/opensearch/templates/secret-credentials.yaml @@ -1,4 +1,17 @@ -{{- if and .Values.credentials.create (not .Values.credentials.existingSecret) }} +{{- if and .Values.credentials.create (not .Values.credentials.existingSecret) (or .Values.opensearch.enabled .Values.dashboards.enabled) }} +{{- $usersData := include "cogstack-opensearch.parsedUsersEnvFile" . | fromYaml | default (dict) -}} +{{- $adminPassword := .Values.credentials.adminPassword -}} +{{- if hasKey $usersData "OPENSEARCH_INITIAL_ADMIN_PASSWORD" -}} +{{- $adminPassword = index $usersData "OPENSEARCH_INITIAL_ADMIN_PASSWORD" -}} +{{- end -}} +{{- $kibanaUser := .Values.credentials.kibanaUser -}} +{{- if hasKey $usersData "KIBANA_USER" -}} +{{- $kibanaUser = index $usersData "KIBANA_USER" -}} +{{- end -}} +{{- $kibanaPassword := .Values.credentials.kibanaPassword -}} +{{- if hasKey $usersData "KIBANA_PASSWORD" -}} +{{- $kibanaPassword = index $usersData "KIBANA_PASSWORD" -}} +{{- end -}} apiVersion: v1 kind: Secret metadata: @@ -7,7 +20,11 @@ metadata: {{- include "cogstack-opensearch.labels" . | nindent 4 }} type: Opaque stringData: - OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ .Values.credentials.adminPassword | quote }} - KIBANA_USER: {{ .Values.credentials.kibanaUser | quote }} - KIBANA_PASSWORD: {{ .Values.credentials.kibanaPassword | quote }} + {{- if .Values.opensearch.enabled }} + OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ $adminPassword | quote }} + {{- end }} + {{- if .Values.dashboards.enabled }} + KIBANA_USER: {{ $kibanaUser | quote }} + KIBANA_PASSWORD: {{ $kibanaPassword | quote }} + {{- end }} {{- end }} diff --git a/deploy/charts/opensearch/templates/service-opensearch-client.yaml b/deploy/charts/opensearch/templates/service-opensearch-client.yaml index 4d4a3071..0c1a07a4 100644 --- a/deploy/charts/opensearch/templates/service-opensearch-client.yaml +++ b/deploy/charts/opensearch/templates/service-opensearch-client.yaml @@ -1,3 +1,4 @@ +{{- if .Values.opensearch.enabled }} apiVersion: v1 kind: Service metadata: @@ -26,3 +27,4 @@ spec: protocol: TCP selector: {{- include "cogstack-opensearch.opensearchSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/deploy/charts/opensearch/templates/service-opensearch-headless.yaml b/deploy/charts/opensearch/templates/service-opensearch-headless.yaml index c36cf61e..317effe8 100644 --- a/deploy/charts/opensearch/templates/service-opensearch-headless.yaml +++ b/deploy/charts/opensearch/templates/service-opensearch-headless.yaml @@ -1,3 +1,4 @@ +{{- if .Values.opensearch.enabled }} apiVersion: v1 kind: Service metadata: @@ -20,3 +21,4 @@ spec: targetPort: analyzer selector: {{- include "cogstack-opensearch.opensearchSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml index 75b39f46..e7317786 100644 --- a/deploy/charts/opensearch/templates/statefulset-opensearch.yaml +++ b/deploy/charts/opensearch/templates/statefulset-opensearch.yaml @@ -1,3 +1,16 @@ +{{- if .Values.opensearch.enabled }} +{{- $envData := include "cogstack-opensearch.parsedEnvFile" . | fromYaml | default (dict) -}} +{{- $clusterName := .Values.opensearch.clusterName -}} +{{- if hasKey $envData "ELASTICSEARCH_CLUSTER_NAME" -}} +{{- $clusterName = index $envData "ELASTICSEARCH_CLUSTER_NAME" -}} +{{- end -}} +{{- $javaOpts := .Values.opensearch.javaOpts -}} +{{- if hasKey $envData "ELASTICSEARCH_JAVA_OPTS" -}} +{{- $javaOpts = index $envData "ELASTICSEARCH_JAVA_OPTS" -}} +{{- else if hasKey $envData "OPENSEARCH_JAVA_OPTS" -}} +{{- $javaOpts = index $envData "OPENSEARCH_JAVA_OPTS" -}} +{{- end -}} +{{- $nodeCertFiles := .Values.certificates.opensearchNodeFiles | default (list) -}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -26,6 +39,43 @@ spec: spec: securityContext: {{- toYaml .Values.opensearch.podSecurityContext | nindent 8 }} + {{- if gt (len $nodeCertFiles) 0 }} + initContainers: + - name: prepare-opensearch-certs + image: "{{ .Values.opensearch.image.repository }}:{{ .Values.opensearch.image.tag }}" + imagePullPolicy: {{ .Values.opensearch.image.pullPolicy }} + securityContext: + {{- toYaml .Values.opensearch.securityContext | nindent 12 }} + command: + - /bin/sh + - -ec + args: + - | + ordinal="${HOSTNAME##*-}" + case "$ordinal" in + {{- range $i, $nodeCert := $nodeCertFiles }} + {{ $i }}) + cert_file="{{ $nodeCert.cert }}" + key_file="{{ $nodeCert.key }}" + ;; + {{- end }} + *) + echo "No node certificate mapping configured for pod ordinal $ordinal" >&2 + exit 1 + ;; + esac + cp "/cert-source/{{ .Values.certificates.opensearchFiles.rootCA }}" "/cert-work/root-ca.crt" + cp "/cert-source/{{ .Values.certificates.opensearchFiles.adminCert }}" "/cert-work/admin.crt" + cp "/cert-source/{{ .Values.certificates.opensearchFiles.adminKey }}" "/cert-work/admin.key.pem" + cp "/cert-source/${cert_file}" "/cert-work/esnode.crt" + cp "/cert-source/${key_file}" "/cert-work/esnode.key" + volumeMounts: + - name: opensearch-certs-source + mountPath: /cert-source + readOnly: true + - name: opensearch-certs-runtime + mountPath: /cert-work + {{- end }} containers: - name: opensearch image: "{{ .Values.opensearch.image.repository }}:{{ .Values.opensearch.image.tag }}" @@ -42,11 +92,6 @@ spec: - name: analyzer containerPort: {{ .Values.opensearch.service.analyzerPort }} protocol: TCP - {{- if .Values.envFile.raw }} - envFrom: - - configMapRef: - name: {{ include "cogstack-opensearch.envConfigMapName" . }} - {{- end }} env: - name: POD_IP valueFrom: @@ -61,7 +106,7 @@ spec: - name: ELASTICSEARCH_NETWORK_PUBLISH_HOST value: "$(POD_IP)" - name: ELASTICSEARCH_CLUSTER_NAME - value: {{ .Values.opensearch.clusterName | quote }} + value: {{ $clusterName | quote }} - name: ELASTICSEARCH_SEED_HOSTS value: {{ include "cogstack-opensearch.seedHosts" . | quote }} - name: ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES @@ -69,7 +114,7 @@ spec: - name: ELASTICSEARCH_HOSTS value: {{ include "cogstack-opensearch.elasticsearchHostsJson" . | quote }} - name: OPENSEARCH_JAVA_OPTS - value: {{ .Values.opensearch.javaOpts | quote }} + value: {{ $javaOpts | quote }} - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD valueFrom: secretKeyRef: @@ -113,6 +158,28 @@ spec: - name: opensearch-security mountPath: /usr/share/opensearch/config/opensearch-security/roles_mapping.yml subPath: roles_mapping.yml + {{- if gt (len $nodeCertFiles) 0 }} + - name: opensearch-certs-runtime + mountPath: /usr/share/opensearch/config/root-ca.crt + subPath: root-ca.crt + readOnly: true + - name: opensearch-certs-runtime + mountPath: /usr/share/opensearch/config/esnode.crt + subPath: esnode.crt + readOnly: true + - name: opensearch-certs-runtime + mountPath: /usr/share/opensearch/config/esnode.key + subPath: esnode.key + readOnly: true + - name: opensearch-certs-runtime + mountPath: /usr/share/opensearch/config/admin.crt + subPath: admin.crt + readOnly: true + - name: opensearch-certs-runtime + mountPath: /usr/share/opensearch/config/admin.key.pem + subPath: admin.key.pem + readOnly: true + {{- else }} - name: opensearch-certs mountPath: /usr/share/opensearch/config/root-ca.crt subPath: {{ .Values.certificates.opensearchFiles.rootCA }} @@ -133,12 +200,19 @@ spec: mountPath: /usr/share/opensearch/config/admin.key.pem subPath: {{ .Values.certificates.opensearchFiles.adminKey }} readOnly: true + {{- end }} - name: data mountPath: /usr/share/opensearch/data - name: logs mountPath: /usr/share/opensearch/logs - name: performance-analyzer mountPath: /usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata + {{- if .Values.opensearch.snapshotBackups.enabled }} + - name: snapshot-backup-data + mountPath: {{ .Values.opensearch.snapshotBackups.data.mountPath }} + - name: snapshot-backup-config + mountPath: {{ .Values.opensearch.snapshotBackups.config.mountPath }} + {{- end }} volumes: - name: opensearch-config configMap: @@ -146,17 +220,37 @@ spec: - name: opensearch-security configMap: name: {{ include "cogstack-opensearch.fullname" . }}-opensearch-security + {{- if gt (len $nodeCertFiles) 0 }} + - name: opensearch-certs-source + secret: + secretName: {{ .Values.certificates.opensearchSecretName }} + - name: opensearch-certs-runtime + emptyDir: {} + {{- else }} - name: opensearch-certs secret: secretName: {{ .Values.certificates.opensearchSecretName }} + {{- end }} + {{- if .Values.opensearch.snapshotBackups.enabled }} + - name: snapshot-backup-data + persistentVolumeClaim: + claimName: {{ default (include "cogstack-opensearch.snapshotBackupDataPvcName" .) .Values.opensearch.snapshotBackups.data.existingClaim }} + - name: snapshot-backup-config + persistentVolumeClaim: + claimName: {{ default (include "cogstack-opensearch.snapshotBackupConfigPvcName" .) .Values.opensearch.snapshotBackups.config.existingClaim }} + {{- end }} {{- if not .Values.opensearch.persistence.enabled }} - name: data emptyDir: {} {{- end }} + {{- if not .Values.opensearch.logPersistence.enabled }} - name: logs emptyDir: {} + {{- end }} + {{- if not .Values.opensearch.performanceAnalyzerPersistence.enabled }} - name: performance-analyzer emptyDir: {} + {{- end }} {{- with .Values.opensearch.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -169,8 +263,9 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.opensearch.persistence.enabled }} + {{- if or .Values.opensearch.persistence.enabled .Values.opensearch.logPersistence.enabled .Values.opensearch.performanceAnalyzerPersistence.enabled }} volumeClaimTemplates: + {{- if .Values.opensearch.persistence.enabled }} - metadata: name: data spec: @@ -182,4 +277,32 @@ spec: {{- if .Values.opensearch.persistence.storageClassName }} storageClassName: {{ .Values.opensearch.persistence.storageClassName | quote }} {{- end }} + {{- end }} + {{- if .Values.opensearch.logPersistence.enabled }} + - metadata: + name: logs + spec: + accessModes: + {{- toYaml .Values.opensearch.logPersistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ .Values.opensearch.logPersistence.size }} + {{- if .Values.opensearch.logPersistence.storageClassName }} + storageClassName: {{ .Values.opensearch.logPersistence.storageClassName | quote }} + {{- end }} + {{- end }} + {{- if .Values.opensearch.performanceAnalyzerPersistence.enabled }} + - metadata: + name: performance-analyzer + spec: + accessModes: + {{- toYaml .Values.opensearch.performanceAnalyzerPersistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ .Values.opensearch.performanceAnalyzerPersistence.size }} + {{- if .Values.opensearch.performanceAnalyzerPersistence.storageClassName }} + storageClassName: {{ .Values.opensearch.performanceAnalyzerPersistence.storageClassName | quote }} + {{- end }} + {{- end }} {{- end }} +{{- end }} diff --git a/deploy/charts/opensearch/values.yaml b/deploy/charts/opensearch/values.yaml index 09c04024..53dd879d 100644 --- a/deploy/charts/opensearch/values.yaml +++ b/deploy/charts/opensearch/values.yaml @@ -1,23 +1,51 @@ nameOverride: "" fullnameOverride: "" +configFiles: + # Optional overrides. If empty, chart falls back to files/*.yml and files/log4j2.properties. + # Use: + # --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml + # --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties + # --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml + opensearchRaw: "" + log4jRaw: "" + dashboardsRaw: "" + envFile: - # Pass deploy/elasticsearch.env via: + # Optional override. If empty, chart falls back to files/deploy-elasticsearch.envfile. + # Use: # --set-file envFile.raw=./deploy/elasticsearch.env raw: "" - # Only these keys are imported into a ConfigMap (avoid leaking secrets from env files). + # Only these shared values are read from deploy/elasticsearch.env. + # Kubernetes-specific discovery and publish-host values are generated by the chart. includeKeys: - - ELASTICSEARCH_NETWORK_HOST - - ELASTICSEARCH_NETWORK_PUBLISH_HOST - ELASTICSEARCH_CLUSTER_NAME - - ELASTICSEARCH_SEED_HOSTS - - ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES - - ELASTICSEARCH_HOSTS - ELASTICSEARCH_JAVA_OPTS - OPENSEARCH_JAVA_OPTS - KIBANA_SERVER_NAME +usersEnvFile: + # Optional override. If empty, chart falls back to files/users-elasticsearch.envfile. + # Use: + # --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env + raw: "" + # Only these keys are imported into the credentials Secret. + includeKeys: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD + - KIBANA_USER + - KIBANA_PASSWORD + +certificatesEnvFile: + # Optional override. If empty, chart falls back to files/certificates-elasticsearch.envfile. + # Use: + # --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env + raw: "" + # Only keys listed here are imported from certificates env. + includeKeys: + - ES_CLIENT_CERT_NAME + opensearch: + enabled: true replicas: 3 image: repository: opensearchproject/opensearch @@ -38,6 +66,37 @@ opensearch: - ReadWriteOnce size: 20Gi storageClassName: "" + logPersistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 10Gi + storageClassName: "" + performanceAnalyzerPersistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 1Gi + storageClassName: "" + snapshotBackups: + # Optional parity mounts for Docker's shared backup paths. + # These should use shared storage accessible from all OpenSearch pods. + # Note: OpenSearch will only use these if path.repo is also configured in opensearch.yml. + enabled: false + data: + mountPath: /mnt/es_data_backups + existingClaim: "" + accessModes: + - ReadWriteMany + size: 20Gi + storageClassName: "" + config: + mountPath: /mnt/es_config_backups + existingClaim: "" + accessModes: + - ReadWriteMany + size: 5Gi + storageClassName: "" service: type: ClusterIP httpPort: 9200 @@ -59,6 +118,11 @@ opensearch: dashboards: enabled: true replicas: 1 + # When opensearch.enabled=false, set external OpenSearch URLs here. + # Example: + # opensearchHosts: + # - https://opensearch-client.cogstack.svc:9200 + opensearchHosts: [] image: repository: opensearchproject/opensearch-dashboards tag: "3.5.0" @@ -89,16 +153,41 @@ credentials: kibanaUser: kibanaserver kibanaPassword: kibanaserver +securityFiles: + # Optional overrides. If empty, chart falls back to files/opensearch-security/*. + # Use: + # --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml + # --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml + # --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml + # --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml + configRaw: "" + internalUsersRaw: "" + rolesRaw: "" + rolesMappingRaw: "" + certificates: opensearchSecretName: opensearch-certs dashboardsSecretName: opensearch-certs opensearchFiles: - rootCA: root-ca.crt + # Secret key names for shared OpenSearch cert material. + # These align with files under security/certificates/elastic/opensearch/. + rootCA: elastic-stack-ca.crt.pem nodeCert: esnode.crt nodeKey: esnode.key adminCert: admin.crt adminKey: admin.key.pem + opensearchNodeFiles: + # Optional per-pod node cert/key mapping using repo-style file names. + # Pod ordinal 0 uses the first entry, 1 uses the second, etc. + # If you set this to an empty list, the chart falls back to opensearchFiles.nodeCert/nodeKey. + - cert: elasticsearch-1.crt + key: elasticsearch-1.key + - cert: elasticsearch-2.crt + key: elasticsearch-2.key + - cert: elasticsearch-3.crt + key: elasticsearch-3.key dashboardsFiles: + # Secret key names for Dashboards cert material. rootCA: elastic-stack-ca.crt.pem adminCert: admin.crt adminKey: admin.key.pem diff --git a/deploy/charts/postgresql/Chart.yaml b/deploy/charts/postgresql/Chart.yaml new file mode 100644 index 00000000..5d5b96e8 --- /dev/null +++ b/deploy/charts/postgresql/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cogstack-postgresql +description: CloudNativePG-backed PostgreSQL cluster for CogStack deployments +type: application +version: 0.1.0 +appVersion: "18.1" diff --git a/deploy/charts/postgresql/README.md b/deploy/charts/postgresql/README.md new file mode 100644 index 00000000..8cd91e07 --- /dev/null +++ b/deploy/charts/postgresql/README.md @@ -0,0 +1,49 @@ +# cogstack-postgresql Helm Chart + +Helm chart for deploying the repo's production PostgreSQL database on Kubernetes using the CloudNativePG operator. + +## What this chart deploys + +- A CloudNativePG `Cluster` resource for PostgreSQL +- An application credentials `Secret` by default +- A bootstrap SQL `ConfigMap` sourced from `services/cogstack-db/pgsql/schemas/` + +## Prerequisites + +1. Install the CloudNativePG operator and CRDs first. +2. Have a working default `StorageClass`, or set `cluster.storage.storageClass` in your values file. +3. Use at least 3 worker nodes for HA production deployments. + +Pinned operator install example: + +```bash +kubectl apply --server-side -f \ + https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.28/releases/cnpg-1.28.1.yaml +``` + +## Install + +```bash +helm upgrade --install cogstack-postgresql ./deploy/charts/postgresql \ + -f ./deploy/helm/postgresql.values.yaml \ + --namespace cogstack --create-namespace +``` + +## Render templates + +```bash +helm template cogstack-postgresql ./deploy/charts/postgresql \ + -f ./deploy/helm/postgresql.values.yaml \ + --namespace cogstack +``` + +## Notes + +- The chart consumes shared repo sources through bundled files under `deploy/charts/postgresql/files/`. +- `deploy/database.env` is used as the default source for `DATABASE_DB_NAME`, `POSTGRES_DB_MAX_CONNECTIONS`, and the default CPU/memory sizing hints. +- `security/env/users_database.env` is used as the default source for `DATABASE_USER` and `DATABASE_PASSWORD`. +- The bootstrap SQL comes from the shared PostgreSQL schema files under `services/cogstack-db/pgsql/schemas/`. +- `POSTGRES_DB_SCHEMA_PREFIX` is respected when selecting bundled custom schema files after `annotations_nlp_create_schema.sql`. +- Docker-specific settings that do not map cleanly to Kubernetes, such as `DATABASE_DOCKER_SHM_SIZE`, are intentionally not consumed. +- By default, the application owner is not granted PostgreSQL superuser. If you need compose-like behavior, set `cluster.ownerSuperuser=true`. +- Backups are intentionally not enabled by default in this chart. Wire object storage and WAL archiving before using this for production data. diff --git a/deploy/charts/postgresql/files/deploy-database.envfile b/deploy/charts/postgresql/files/deploy-database.envfile new file mode 120000 index 00000000..14d5dc9c --- /dev/null +++ b/deploy/charts/postgresql/files/deploy-database.envfile @@ -0,0 +1 @@ +../../../../deploy/database.env \ No newline at end of file diff --git a/deploy/charts/postgresql/files/initdb/annotations_nlp_create_schema.sql b/deploy/charts/postgresql/files/initdb/annotations_nlp_create_schema.sql new file mode 120000 index 00000000..572f896d --- /dev/null +++ b/deploy/charts/postgresql/files/initdb/annotations_nlp_create_schema.sql @@ -0,0 +1 @@ +../../../../../services/cogstack-db/pgsql/schemas/annotations_nlp_create_schema.sql \ No newline at end of file diff --git a/deploy/charts/postgresql/files/initdb/cogstack_db_databank.sql b/deploy/charts/postgresql/files/initdb/cogstack_db_databank.sql new file mode 120000 index 00000000..083bfa1f --- /dev/null +++ b/deploy/charts/postgresql/files/initdb/cogstack_db_databank.sql @@ -0,0 +1 @@ +../../../../../services/cogstack-db/pgsql/schemas/cogstack_db_databank.sql \ No newline at end of file diff --git a/deploy/charts/postgresql/files/users-database.envfile b/deploy/charts/postgresql/files/users-database.envfile new file mode 120000 index 00000000..e5d415f5 --- /dev/null +++ b/deploy/charts/postgresql/files/users-database.envfile @@ -0,0 +1 @@ +../../../../security/env/users_database.env \ No newline at end of file diff --git a/deploy/charts/postgresql/templates/NOTES.txt b/deploy/charts/postgresql/templates/NOTES.txt new file mode 100644 index 00000000..d3bb08c5 --- /dev/null +++ b/deploy/charts/postgresql/templates/NOTES.txt @@ -0,0 +1,9 @@ +CloudNativePG operator resources must already be installed before this chart can be applied. + +PostgreSQL services created by the operator: +- read/write primary service: {{ include "cogstack-postgresql.fullname" . }}-rw +- read-only replicas service: {{ include "cogstack-postgresql.fullname" . }}-ro +- any-instance service: {{ include "cogstack-postgresql.fullname" . }}-r + +Application credentials secret: +- {{ include "cogstack-postgresql.appSecretName" . }} diff --git a/deploy/charts/postgresql/templates/_helpers.tpl b/deploy/charts/postgresql/templates/_helpers.tpl new file mode 100644 index 00000000..67b0ef33 --- /dev/null +++ b/deploy/charts/postgresql/templates/_helpers.tpl @@ -0,0 +1,117 @@ +{{/* Chart name */}} +{{- define "cogstack-postgresql.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Fully qualified name */}} +{{- define "cogstack-postgresql.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Common labels */}} +{{- define "cogstack-postgresql.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "cogstack-postgresql.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* Application credentials secret name */}} +{{- define "cogstack-postgresql.appSecretName" -}} +{{- if .Values.credentials.existingAppSecret -}} +{{- .Values.credentials.existingAppSecret -}} +{{- else -}} +{{- printf "%s-app" (include "cogstack-postgresql.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* Bootstrap SQL configmap name */}} +{{- define "cogstack-postgresql.initdbConfigMapName" -}} +{{- printf "%s-initdb" (include "cogstack-postgresql.fullname" .) -}} +{{- end -}} + +{{/* Normalize docker-style memory strings to Kubernetes quantities */}} +{{- define "cogstack-postgresql.normalizeMemory" -}} +{{- $value := toString . | trim -}} +{{- if regexMatch "^[0-9]+g$" $value -}} +{{- printf "%sGi" (trimSuffix "g" $value) -}} +{{- else if regexMatch "^[0-9]+m$" $value -}} +{{- printf "%sMi" (trimSuffix "m" $value) -}} +{{- else if regexMatch "^[0-9]+k$" $value -}} +{{- printf "%sKi" (trimSuffix "k" $value) -}} +{{- else -}} +{{- $value -}} +{{- end -}} +{{- end -}} + +{{/* Parse deploy/database.env into a filtered YAML map */}} +{{- define "cogstack-postgresql.parsedEnvFile" -}} +{{- $root := . -}} +{{- $envData := dict -}} +{{- $rawEnv := $root.Values.envFile.raw | default ($root.Files.Get "files/deploy-database.envfile") -}} +{{- if $rawEnv -}} +{{- $renderedEnv := tpl $rawEnv $root -}} +{{- range $line := splitList "\n" $renderedEnv }} + {{- $clean := trim (replace "\r" "" $line) -}} + {{- if and $clean (not (hasPrefix "#" $clean)) -}} + {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} + {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} + {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} + {{- if has $key $root.Values.envFile.includeKeys -}} + {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} + {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} + {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} + {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} + {{- end -}} + {{- if regexMatch "^\\$[A-Za-z_][A-Za-z0-9_]*$" $val -}} + {{- $refKey := trimPrefix "$" $val -}} + {{- if hasKey $envData $refKey -}} + {{- $val = index $envData $refKey -}} + {{- end -}} + {{- end -}} + {{- $_ := set $envData $key $val -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{ toYaml $envData }} +{{- end -}} + +{{/* Parse security/env/users_database.env into a filtered YAML map */}} +{{- define "cogstack-postgresql.parsedUsersEnvFile" -}} +{{- $root := . -}} +{{- $usersData := dict -}} +{{- $rawUsers := $root.Values.usersEnvFile.raw | default ($root.Files.Get "files/users-database.envfile") -}} +{{- if $rawUsers -}} +{{- $renderedUsers := tpl $rawUsers $root -}} +{{- range $line := splitList "\n" $renderedUsers }} + {{- $clean := trim (replace "\r" "" $line) -}} + {{- if and $clean (not (hasPrefix "#" $clean)) -}} + {{- if regexMatch "^[A-Za-z_][A-Za-z0-9_]*=" $clean -}} + {{- $key := regexFind "^[A-Za-z_][A-Za-z0-9_]*" $clean -}} + {{- $val := trim (regexReplaceAll "^[A-Za-z_][A-Za-z0-9_]*=" $clean "") -}} + {{- if has $key $root.Values.usersEnvFile.includeKeys -}} + {{- if and (hasPrefix "\"" $val) (hasSuffix "\"" $val) -}} + {{- $val = trimSuffix "\"" (trimPrefix "\"" $val) -}} + {{- else if and (hasPrefix "'" $val) (hasSuffix "'" $val) -}} + {{- $val = trimSuffix "'" (trimPrefix "'" $val) -}} + {{- end -}} + {{- $_ := set $usersData $key $val -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{ toYaml $usersData }} +{{- end -}} diff --git a/deploy/charts/postgresql/templates/cluster.yaml b/deploy/charts/postgresql/templates/cluster.yaml new file mode 100644 index 00000000..3a7bb9a7 --- /dev/null +++ b/deploy/charts/postgresql/templates/cluster.yaml @@ -0,0 +1,126 @@ +{{- if .Values.cluster.enabled }} +{{- $envData := include "cogstack-postgresql.parsedEnvFile" . | fromYaml | default (dict) -}} +{{- $usersData := include "cogstack-postgresql.parsedUsersEnvFile" . | fromYaml | default (dict) -}} +{{- $databaseName := "cogstack" -}} +{{- if .Values.cluster.databaseName -}} +{{- $databaseName = .Values.cluster.databaseName -}} +{{- else if hasKey $envData "DATABASE_DB_NAME" -}} +{{- $databaseName = index $envData "DATABASE_DB_NAME" -}} +{{- end -}} +{{- $ownerName := "admin" -}} +{{- if .Values.cluster.ownerName -}} +{{- $ownerName = .Values.cluster.ownerName -}} +{{- else if hasKey $usersData "DATABASE_USER" -}} +{{- $ownerName = index $usersData "DATABASE_USER" -}} +{{- end -}} +{{- $maxConnections := "100" -}} +{{- if .Values.cluster.postgresql.maxConnections -}} +{{- $maxConnections = printf "%v" .Values.cluster.postgresql.maxConnections -}} +{{- else if hasKey $envData "POSTGRES_DB_MAX_CONNECTIONS" -}} +{{- $maxConnections = printf "%v" (index $envData "POSTGRES_DB_MAX_CONNECTIONS") -}} +{{- end -}} +{{- $cpuRequest := "1" -}} +{{- if .Values.cluster.resources.requests.cpu -}} +{{- $cpuRequest = .Values.cluster.resources.requests.cpu -}} +{{- else if hasKey $envData "DATABASE_DOCKER_CPU_MIN" -}} +{{- $cpuRequest = printf "%v" (index $envData "DATABASE_DOCKER_CPU_MIN") -}} +{{- end -}} +{{- $cpuLimit := $cpuRequest -}} +{{- if .Values.cluster.resources.limits.cpu -}} +{{- $cpuLimit = .Values.cluster.resources.limits.cpu -}} +{{- else if hasKey $envData "DATABASE_DOCKER_CPU_MAX" -}} +{{- $cpuLimit = printf "%v" (index $envData "DATABASE_DOCKER_CPU_MAX") -}} +{{- end -}} +{{- $memoryRequest := "1Gi" -}} +{{- if .Values.cluster.resources.requests.memory -}} +{{- $memoryRequest = .Values.cluster.resources.requests.memory -}} +{{- else if hasKey $envData "DATABASE_DOCKER_RAM" -}} +{{- $memoryRequest = include "cogstack-postgresql.normalizeMemory" (index $envData "DATABASE_DOCKER_RAM") -}} +{{- end -}} +{{- $memoryLimit := $memoryRequest -}} +{{- if .Values.cluster.resources.limits.memory -}} +{{- $memoryLimit = .Values.cluster.resources.limits.memory -}} +{{- else if hasKey $envData "DATABASE_DOCKER_RAM" -}} +{{- $memoryLimit = include "cogstack-postgresql.normalizeMemory" (index $envData "DATABASE_DOCKER_RAM") -}} +{{- end -}} +{{- $schemaPrefix := "cogstack_db" -}} +{{- if hasKey $envData "POSTGRES_DB_SCHEMA_PREFIX" -}} +{{- $schemaPrefix = index $envData "POSTGRES_DB_SCHEMA_PREFIX" -}} +{{- end -}} +{{- $annotationsSchemaKey := .Values.initdb.annotationsSchemaKey -}} +{{- $sqlFiles := .Files.Glob "files/initdb/*.sql" -}} +{{- $selectedSchemaKeys := list -}} +{{- $annotationsSchemaPresent := false -}} +{{- range $path, $_ := $sqlFiles }} + {{- $key := base $path -}} + {{- if eq $key $annotationsSchemaKey -}} + {{- $annotationsSchemaPresent = true -}} + {{- else if hasPrefix $schemaPrefix $key -}} + {{- $selectedSchemaKeys = append $selectedSchemaKeys $key -}} + {{- end -}} +{{- end -}} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ include "cogstack-postgresql.fullname" . }} + labels: + {{- include "cogstack-postgresql.labels" . | nindent 4 }} +spec: + description: "CogStack production PostgreSQL cluster managed by CloudNativePG" + instances: {{ .Values.cluster.instances }} + imageName: {{ .Values.cluster.imageName | quote }} + imagePullPolicy: {{ .Values.cluster.imagePullPolicy }} + enableSuperuserAccess: {{ .Values.cluster.enableSuperuserAccess }} + bootstrap: + initdb: + database: {{ $databaseName | quote }} + owner: {{ $ownerName | quote }} + secret: + name: {{ include "cogstack-postgresql.appSecretName" . }} + {{- if or $annotationsSchemaPresent (gt (len $selectedSchemaKeys) 0) (gt (len .Values.initdb.extraConfigMapRefs) 0) (gt (len .Values.initdb.extraSecretRefs) 0) }} + postInitApplicationSQLRefs: + {{- if or $annotationsSchemaPresent (gt (len $selectedSchemaKeys) 0) (gt (len .Values.initdb.extraConfigMapRefs) 0) }} + configMapRefs: + {{- if $annotationsSchemaPresent }} + - name: {{ include "cogstack-postgresql.initdbConfigMapName" . }} + key: {{ $annotationsSchemaKey }} + {{- end }} + {{- range $key := $selectedSchemaKeys }} + - name: {{ include "cogstack-postgresql.initdbConfigMapName" $ }} + key: {{ $key }} + {{- end }} + {{- with .Values.initdb.extraConfigMapRefs }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- with .Values.initdb.extraSecretRefs }} + secretRefs: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.cluster.ownerSuperuser }} + managed: + roles: + - name: {{ $ownerName | quote }} + ensure: present + login: true + superuser: true + passwordSecret: + name: {{ include "cogstack-postgresql.appSecretName" . }} + {{- end }} + postgresql: + parameters: + max_connections: {{ $maxConnections | quote }} + resources: + requests: + cpu: {{ $cpuRequest | quote }} + memory: {{ $memoryRequest | quote }} + limits: + cpu: {{ $cpuLimit | quote }} + memory: {{ $memoryLimit | quote }} + storage: + size: {{ .Values.cluster.storage.size | quote }} + {{- if .Values.cluster.storage.storageClass }} + storageClass: {{ .Values.cluster.storage.storageClass | quote }} + {{- end }} +{{- end }} diff --git a/deploy/charts/postgresql/templates/configmap-initdb.yaml b/deploy/charts/postgresql/templates/configmap-initdb.yaml new file mode 100644 index 00000000..ef2134a7 --- /dev/null +++ b/deploy/charts/postgresql/templates/configmap-initdb.yaml @@ -0,0 +1,16 @@ +{{- if .Values.cluster.enabled }} +{{- $sqlFiles := .Files.Glob "files/initdb/*.sql" -}} +{{- if $sqlFiles }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cogstack-postgresql.initdbConfigMapName" . }} + labels: + {{- include "cogstack-postgresql.labels" . | nindent 4 }} +data: + {{- range $path, $_ := $sqlFiles }} + {{ base $path }}: | +{{ $.Files.Get $path | indent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/charts/postgresql/templates/secret-app.yaml b/deploy/charts/postgresql/templates/secret-app.yaml new file mode 100644 index 00000000..5c79c8e3 --- /dev/null +++ b/deploy/charts/postgresql/templates/secret-app.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.cluster.enabled (not .Values.credentials.existingAppSecret) }} +{{- $usersData := include "cogstack-postgresql.parsedUsersEnvFile" . | fromYaml | default (dict) -}} +{{- $ownerName := "admin" -}} +{{- if .Values.cluster.ownerName -}} +{{- $ownerName = .Values.cluster.ownerName -}} +{{- else if hasKey $usersData "DATABASE_USER" -}} +{{- $ownerName = index $usersData "DATABASE_USER" -}} +{{- end -}} +{{- $ownerPassword := "admin" -}} +{{- if .Values.credentials.password -}} +{{- $ownerPassword = .Values.credentials.password -}} +{{- else if hasKey $usersData "DATABASE_PASSWORD" -}} +{{- $ownerPassword = index $usersData "DATABASE_PASSWORD" -}} +{{- end -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cogstack-postgresql.appSecretName" . }} + labels: + {{- include "cogstack-postgresql.labels" . | nindent 4 }} +type: kubernetes.io/basic-auth +stringData: + username: {{ $ownerName | quote }} + password: {{ $ownerPassword | quote }} +{{- end }} diff --git a/deploy/charts/postgresql/values.yaml b/deploy/charts/postgresql/values.yaml new file mode 100644 index 00000000..ea40088f --- /dev/null +++ b/deploy/charts/postgresql/values.yaml @@ -0,0 +1,55 @@ +nameOverride: "" +fullnameOverride: "" + +envFile: + # Optional override. If empty, chart falls back to files/deploy-database.envfile. + # Use: + # --set-file envFile.raw=./deploy/database.env + raw: "" + includeKeys: + - POSTGRES_DB_MAX_CONNECTIONS + - POSTGRES_DB_SCHEMA_PREFIX + - DATABASE_DB_NAME + - DATABASE_DOCKER_CPU_MIN + - DATABASE_DOCKER_CPU_MAX + - DATABASE_DOCKER_RAM + +usersEnvFile: + # Optional override. If empty, chart falls back to files/users-database.envfile. + # Use: + # --set-file usersEnvFile.raw=./security/env/users_database.env + raw: "" + includeKeys: + - DATABASE_USER + - DATABASE_PASSWORD + +cluster: + enabled: true + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:18.1-system-trixie + imagePullPolicy: IfNotPresent + databaseName: "" + ownerName: "" + ownerSuperuser: false + enableSuperuserAccess: false + storage: + size: 20Gi + storageClass: "" + resources: + requests: + cpu: "" + memory: "" + limits: + cpu: "" + memory: "" + postgresql: + maxConnections: "" + +credentials: + existingAppSecret: "" + password: "" + +initdb: + annotationsSchemaKey: annotations_nlp_create_schema.sql + extraConfigMapRefs: [] + extraSecretRefs: [] diff --git a/deploy/helm/opensearch.values.yaml b/deploy/helm/opensearch.values.yaml new file mode 100644 index 00000000..a75904fa --- /dev/null +++ b/deploy/helm/opensearch.values.yaml @@ -0,0 +1,29 @@ +# Cluster-specific overrides for the OpenSearch Helm chart. +# +# Shared OpenSearch, Dashboards, security YAML, and selected env files are not +# listed here: the chart already consumes the repo's canonical files from +# deploy/, services/, and security/ via its bundled defaults. + +certificates: + opensearchSecretName: opensearch-certs + dashboardsSecretName: opensearch-certs + +# Uncomment to use a pre-created credentials Secret instead of chart defaults. +# credentials: +# create: false +# existingSecret: opensearch-credentials + +# Example storage overrides: +# opensearch: +# persistence: +# storageClassName: standard +# logPersistence: +# storageClassName: standard +# performanceAnalyzerPersistence: +# storageClassName: standard +# snapshotBackups: +# enabled: true +# data: +# existingClaim: shared-es-data-backups +# config: +# existingClaim: shared-es-config-backups diff --git a/deploy/observability/data-prepper/config/data-prepper-config.yaml b/deploy/observability/data-prepper/config/data-prepper-config.yaml new file mode 100644 index 00000000..85a86064 --- /dev/null +++ b/deploy/observability/data-prepper/config/data-prepper-config.yaml @@ -0,0 +1,6 @@ +# Data Prepper control API (dev profile, no TLS) +ssl: false +serverPort: 4900 +peer_forwarder: + ssl: false + discovery_mode: local_node diff --git a/deploy/observability/data-prepper/pipelines/pipelines.yaml b/deploy/observability/data-prepper/pipelines/pipelines.yaml new file mode 100644 index 00000000..90e36fdc --- /dev/null +++ b/deploy/observability/data-prepper/pipelines/pipelines.yaml @@ -0,0 +1,44 @@ +entry-pipeline: + source: + otel_trace_source: + port: 21890 + ssl: false + authentication: + unauthenticated: + buffer: + bounded_blocking: + buffer_size: 500000 + batch_size: 10000 + sink: + - pipeline: + name: raw-trace-pipeline + - pipeline: + name: service-map-pipeline + +raw-trace-pipeline: + source: + pipeline: + name: entry-pipeline + processor: + - otel_traces: + sink: + - opensearch: + hosts: ["https://elasticsearch-1:9200"] + insecure: true + username: "admin" + password: "Kibanaserver!1" + index_type: trace-analytics-raw + +service-map-pipeline: + source: + pipeline: + name: entry-pipeline + processor: + - service_map: + sink: + - opensearch: + hosts: ["https://elasticsearch-1:9200"] + insecure: true + username: "admin" + password: "Kibanaserver!1" + index_type: trace-analytics-service-map diff --git a/deploy/observability/otel-collector.yaml b/deploy/observability/otel-collector.yaml new file mode 100644 index 00000000..d45e8e73 --- /dev/null +++ b/deploy/observability/otel-collector.yaml @@ -0,0 +1,33 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + otlp: + endpoint: data-prepper:21890 + tls: + # Data Prepper source below uses plaintext in this dev profile. + insecure: true + debug: + verbosity: basic + +processors: + batch: {} + +extensions: + health_check: {} + +service: + extensions: [health_check] + telemetry: + logs: + level: info + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp, debug] diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index b22ee7d0..da768f03 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -70,17 +70,17 @@ Quick usage: ```bash # render manifests helm template cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env + -f ./deploy/helm/opensearch.values.yaml # install or upgrade helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \ - --set-file envFile.raw=./deploy/elasticsearch.env \ + -f ./deploy/helm/opensearch.values.yaml \ --namespace cogstack --create-namespace ``` > The chart expects pre-created Kubernetes Secrets for TLS materials (see the chart README). -> The `--set-file envFile.raw=...` flag injects values from `deploy/elasticsearch.env` into pod environment variables. -> Only keys in `envFile.includeKeys` are imported into the chart ConfigMap. +> The chart already consumes the shared OpenSearch, Dashboards, and security YAML files automatically from this repo. +> The values file is only for cluster-specific overrides such as secret names, storage classes, replicas, and snapshot PVC claims. ## 🧰 Makefile Command Overview diff --git a/docs/deploy/services.md b/docs/deploy/services.md index 288290cf..3874707a 100644 --- a/docs/deploy/services.md +++ b/docs/deploy/services.md @@ -410,11 +410,23 @@ Web UI for exploring indexed data, visualising documents, managing index templat - URL: **https://localhost:5601** -#### credentials +#### Credentials - **OpenSearch Dashboards:** `admin` / `admin` - **Elasticsearch Native:** `elastic` / `kibanaserver` +#### First login setup (OpenSearch Dashboards) + +After signing in to OpenSearch Dashboards: + +1. Open the user menu and choose **Switch tenant**. +2. Select `Global` so saved objects are shared across users/roles. +3. Go to **Stack Management** → **Workspaces** and click **Create workspace**. +4. Create a workspace (for example `cogstack-main`) in the `Global` tenant. +5. Create data views/dashboards in that workspace. + +`Private` tenant is isolated per user and should be avoided for shared environments. + #### Containers - `cogstack-kibana` (OpenSearch Dashboards or Kibana depending on configuration) diff --git a/docs/security/elasticsearch_opensearch.md b/docs/security/elasticsearch_opensearch.md index 90f96466..6c3806a9 100644 --- a/docs/security/elasticsearch_opensearch.md +++ b/docs/security/elasticsearch_opensearch.md @@ -26,7 +26,7 @@ All scripts reference the following environment configuration files: |------|----------| | `certificates_elasticsearch.env` | Hostnames, instance names, and certificate parameters for ES / OpenSearch nodes | | `certificates_general.env` | Root CA configuration | -| `elasticsearch_users.env` | Internal user credentials | +| `users_elasticsearch.env` | Internal user credentials | Reload them before running any security-related script: @@ -73,7 +73,7 @@ security/certificates/elastic/ │ ├── sample-kibana.yml │ └── README.txt └── opensearch/ - ├── admin.*, es_kibana_client.*, root-ca.* + ├── admin.*, es_kibana_client.*, elastic-stack-ca.*, root-ca.* └── elasticsearch/{1,2,3}/... ``` @@ -126,7 +126,7 @@ This produces: | File | Purpose | |------|----------| -| `admin.pem`, `admin-key.pem` | Admin dashboard certificate | +| `admin.crt`, `admin.key.pem` | Admin dashboard certificate | | `es_kibana_client.pem`, `es_kibana_client.key` | Client certificate for Kibana/OpenDashboard | | `*.jks` | Node keystores/truststores for HTTPS and inter-node encryption | @@ -143,9 +143,9 @@ security/certificates/elastic/opensearch/ | Platform | Required Certificates | Source Folder | |-----------|----------------------|----------------| | **Kibana** | `elasticsearch-{1,2,3}.crt`, `elasticsearch-{1,2,3}.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/elasticsearch/` | -| **OpenDashboard (OpenSearch)** | `admin.pem`, `admin-key.pem`, `es_kibana_client.pem`, `es_kibana_client.key` | `security/certificates/elastic/opensearch/` | +| **OpenDashboard (OpenSearch)** | `admin.crt`, `admin.key.pem`, `es_kibana_client.pem`, `es_kibana_client.key`, `elastic-stack-ca.crt.pem` | `security/certificates/elastic/opensearch/` | -All certificate references in `services/kibana/config/kibana_opensearch.yml` or `services.yml` must point to these locations. +All certificate references in `services/kibana/config/opensearch.yml` or `services.yml` must point to these locations. --- @@ -218,6 +218,28 @@ Troubleshooting: OpenSearch includes default roles (`admin`, `kibanaserver`, `readall`, `snapshotrestore`, etc.) — always change their passwords after first run. +##### OpenSearch Dashboards post-login setup (Global tenant + workspace) + +After your first login to `https://localhost:5601` (default: `admin` / `admin`): + +1. Open the user menu (top-right) and select **Switch tenant**. +2. Select `Global`, then apply the change. +3. Open **Stack Management** → **Workspaces** (or the workspace switcher in the header) and click **Create workspace**. +4. Enter a workspace name (for example `cogstack-main`) and keep it in the `Global` tenant. +5. Save, then create data views and dashboards inside this workspace. + +Use `Global` for shared dashboards and saved objects. +`Private` tenant content is user-specific and not visible to other users. + +If tenant switching or workspace creation is not available in the UI, verify these settings in `services/kibana/config/opensearch.yml`: + +```yaml +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.enable_global: true +opensearch_security.multitenancy.tenants.preferred: ["Global"] +workspace.enabled: true +``` + --- #### Elasticsearch (native) @@ -230,7 +252,7 @@ bash ./create_es_native_credentials.sh This script creates system users, roles, and a service account token for Kibana. -You can modify credentials in `security/env/elasticsearch_users.env`. +You can modify credentials in `security/env/users_elasticsearch.env`. **New roles** created: diff --git a/nifi/user_python_extensions/parse_service_response.py b/nifi/user_python_extensions/parse_service_response.py index b75070db..5254c4b9 100644 --- a/nifi/user_python_extensions/parse_service_response.py +++ b/nifi/user_python_extensions/parse_service_response.py @@ -24,7 +24,7 @@ class Java: implements = ['org.apache.nifi.python.processor.FlowFileTransform'] class ProcessorDetails: - version = '0.0.1' + version = '0.0.2' def __init__(self, jvm: JVMView): super().__init__(jvm) @@ -115,6 +115,8 @@ def process(self, context: ProcessContext, flowFile: JavaObject) -> FlowFileTran records: dict | list[dict] = json.loads(input_raw_bytes.decode("utf-8")) + additional_attributes: dict = {} + if isinstance(records, dict): records = [records] @@ -131,6 +133,9 @@ def process(self, context: ProcessContext, flowFile: JavaObject) -> FlowFileTran if "footer" in result: for k, v in result["footer"].items(): _record[k] = v + + if self.document_id_field_name in _record: + additional_attributes[self.document_id_field_name] = str(_record[self.document_id_field_name]) output_contents.append(_record) @@ -178,12 +183,15 @@ def process(self, context: ProcessContext, flowFile: JavaObject) -> FlowFileTran str(footer[self.document_id_field_name]) + "_" + str(annotation_id) ) + additional_attributes["annotation_id"] = _output_annotated_record["annotation_id"] + output_contents.append(_output_annotated_record) # add properties to flowfile attributes attributes: dict = {k: str(v) for k, v in flowFile.getAttributes().items()} attributes["output_text_field_name"] = str(self.output_text_field_name) attributes["mime.type"] = "application/json" + attributes.update(additional_attributes) return FlowFileTransformResult( relationship=self.REL_SUCCESS.name, diff --git a/services/ocr-service b/services/ocr-service index e78a1495..fb8f4937 160000 --- a/services/ocr-service +++ b/services/ocr-service @@ -1 +1 @@ -Subproject commit e78a1495ad5e24d318cb15bcd55314ef3b9e33f5 +Subproject commit fb8f4937a589f4df62134d90d4f55f3f0363f1b8