Charts

deploy/Makefile

@@ -11,6 +11,16 @@ REMOTE_SSH_KEY_ARG = $(if $(strip $(REMOTE_SSH_KEY)),-i $(REMOTE_SSH_KEY),)
 HELM_OPENSEARCH_RELEASE ?= cogstack-opensearch
 HELM_OPENSEARCH_NAMESPACE ?= cogstack
 HELM_OPENSEARCH_CHART ?= ./charts/opensearch
+HELM_OPENSEARCH_VALUES_FILE ?= ./helm/opensearch.values.yaml
+HELM_OPENSEARCH_VALUES_ARG = -f $(HELM_OPENSEARCH_VALUES_FILE)
+HELM_POSTGRESQL_RELEASE ?= cogstack-postgresql
+HELM_POSTGRESQL_NAMESPACE ?= cogstack
+HELM_POSTGRESQL_CHART ?= ./charts/postgresql
+HELM_POSTGRESQL_VALUES_FILE ?= ./helm/postgresql.values.yaml
+HELM_POSTGRESQL_VALUES_ARG = -f $(HELM_POSTGRESQL_VALUES_FILE)
+CNPG_OPERATOR_MINOR ?= 1.28
+CNPG_OPERATOR_VERSION ?= 1.28.1
+CNPG_OPERATOR_MANIFEST ?= https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-$(CNPG_OPERATOR_MINOR)/releases/cnpg-$(CNPG_OPERATOR_VERSION).yaml
 
 define WITH_ENV
 set -a && source ./export_env_vars.sh;
@@ -72,11 +82,20 @@ load-env: ## Load variables from export_env_vars.sh in a subshell
 show-env: ## Print sorted environment variables after loading export_env_vars.sh
 	${WITH_ENV} >/dev/null 2>&1; printenv | sort
 
-helm-template-opensearch: ## Render OpenSearch chart using deploy/elasticsearch.env
-	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) --set-file envFile.raw=./elasticsearch.env
+helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml
+	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG)
 
-helm-install-opensearch: ## Install/upgrade OpenSearch chart using deploy/elasticsearch.env
-	helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) --set-file envFile.raw=./elasticsearch.env --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace
+helm-install-opensearch: ## Install/upgrade OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml
+	helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace
+
+kube-install-cnpg-operator: ## Install CloudNativePG operator/CRDs from the pinned upstream manifest
+	kubectl apply --server-side -f $(CNPG_OPERATOR_MANIFEST)
+
+helm-template-postgresql: ## Render CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml
+	helm template $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE)
+
+helm-install-postgresql: ## Install/upgrade CloudNativePG PostgreSQL chart using chart defaults plus ./helm/postgresql.values.yaml
+	helm upgrade --install $(HELM_POSTGRESQL_RELEASE) $(HELM_POSTGRESQL_CHART) $(HELM_POSTGRESQL_VALUES_ARG) --namespace $(HELM_POSTGRESQL_NAMESPACE) --create-namespace
 
 
 remote-deploy-service: ## Deploy one or more services to a remote machine via SSH + docker compose
@@ -105,7 +124,7 @@ _check-remote-params:
 		exit 1; \
 	fi
 
-.PHONY: remote-deploy-service remote-stop-service remote-delete-service _check-remote-params helm-template-opensearch helm-install-opensearch
+.PHONY: remote-deploy-service remote-stop-service remote-delete-service _check-remote-params helm-template-opensearch helm-install-opensearch kube-install-cnpg-operator helm-template-postgresql helm-install-postgresql
 
 # start services
 

deploy/charts/README.md

@@ -9,19 +9,22 @@ This directory contains Helm charts owned by this repository's deployment layer.
 
 ## Current charts
 
-- `opensearch/` - OpenSearch + OpenSearch Dashboards chart used by this repo.
+- `opensearch/` - OpenSearch and/or OpenSearch Dashboards chart used by this repo.
+- `postgresql/` - CloudNativePG-backed PostgreSQL cluster chart used by this repo.
 
 ## Quick usage
 
 ```bash
 # Render manifests
 helm template cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file envFile.raw=./deploy/elasticsearch.env
+  -f ./deploy/helm/opensearch.values.yaml
 
 # Install/upgrade
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
+  -f ./deploy/helm/opensearch.values.yaml \
   --namespace cogstack --create-namespace
 ```
 
-Only keys in `envFile.includeKeys` are imported from the env file.
+The OpenSearch and Dashboards config files should come from `services/`, and the security and env files from `security/` and `deploy/`, so Docker and Kubernetes use the same source files.
+The values file is for cluster-specific overrides only; it does not need to repeat the shared YAML or env file paths.
+Only keys in `envFile.includeKeys`, `usersEnvFile.includeKeys`, and `certificatesEnvFile.includeKeys` are imported.

deploy/charts/opensearch/README.md

@@ -1,57 +1,116 @@
 # cogstack-opensearch Helm Chart
 
-Helm chart for deploying OpenSearch plus OpenSearch Dashboards using the CogStack configuration baseline.
+Helm chart for deploying OpenSearch and/or OpenSearch Dashboards using the CogStack configuration baseline.
 
 ## What this chart deploys
 
-- OpenSearch `StatefulSet` (default: 3 replicas)
-- OpenSearch client + headless Services
+- OpenSearch `StatefulSet` (default: 3 replicas, controlled by `opensearch.enabled`)
+- OpenSearch client + headless Services (when `opensearch.enabled=true`)
 - OpenSearch Dashboards `Deployment` + Service (enabled by default)
 - ConfigMaps for:
-  - `opensearch.yml`
-  - `log4j2.properties`
-  - OpenSearch Security files (`config.yml`, `internal_users.yml`, `roles.yml`, `roles_mapping.yml`)
+  - `opensearch.yml` (when `opensearch.enabled=true`)
+  - `log4j2.properties` (when `opensearch.enabled=true`)
+  - OpenSearch Security files (`config.yml`, `internal_users.yml`, `roles.yml`, `roles_mapping.yml`) (when `opensearch.enabled=true`)
   - `opensearch_dashboards.yml`
+- PVC-backed `data`, `logs`, and performance-analyzer storage for OpenSearch by default
 
 ## Prerequisites
 
-1. Kubernetes cluster with dynamic PV provisioning (or set `persistence.enabled=false`).
-2. Kubernetes Secrets containing TLS materials.
+1. Kubernetes cluster with dynamic PV provisioning (if `opensearch.enabled=true` and `persistence.enabled=true`).
+2. Kubernetes Secrets containing TLS materials for enabled components.
 3. If `credentials.create=false`, an existing Secret with:
-   - `OPENSEARCH_INITIAL_ADMIN_PASSWORD`
-   - `KIBANA_USER`
-   - `KIBANA_PASSWORD`
+   - `OPENSEARCH_INITIAL_ADMIN_PASSWORD` when `opensearch.enabled=true`
+   - `KIBANA_USER` and `KIBANA_PASSWORD` when `dashboards.enabled=true`
 
 ## Required certificate secrets
 
-Set these in `values.yaml`:
+Set these in `values.yaml` for enabled components:
 
-- `certificates.opensearchSecretName`
-- `certificates.dashboardsSecretName`
+- `certificates.opensearchSecretName` (if `opensearch.enabled=true`)
+- `certificates.dashboardsSecretName` (if `dashboards.enabled=true`)
 
 Secret keys are mapped via:
 
-- `certificates.opensearchFiles.*`
-- `certificates.dashboardsFiles.*`
+- `certificates.opensearchFiles.*` (if `opensearch.enabled=true`)
+- `certificates.opensearchNodeFiles[*]` for per-pod node cert/key selection (if `opensearch.enabled=true`)
+- `certificates.dashboardsFiles.*` (if `dashboards.enabled=true`)
+
+Repo-aligned certificate source paths:
+
+- OpenSearch shared certs:
+  - `security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem`
+  - `security/certificates/elastic/opensearch/admin.crt`
+  - `security/certificates/elastic/opensearch/admin.key.pem`
+- OpenSearch node certs:
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt`
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.key`
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.crt`
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.key`
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.crt`
+  - `security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.key`
+- Dashboards certs:
+  - `security/certificates/elastic/opensearch/es_kibana_client.pem`
+  - `security/certificates/elastic/opensearch/es_kibana_client.key`
+
+Example secret creation from the repo layout:
+
+```bash
+kubectl create secret generic opensearch-certs \
+  --from-file=elastic-stack-ca.crt.pem=./security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  --from-file=admin.crt=./security/certificates/elastic/opensearch/admin.crt \
+  --from-file=admin.key.pem=./security/certificates/elastic/opensearch/admin.key.pem \
+  --from-file=elasticsearch-1.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt \
+  --from-file=elasticsearch-1.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-1/elasticsearch-1.key \
+  --from-file=elasticsearch-2.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.crt \
+  --from-file=elasticsearch-2.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-2/elasticsearch-2.key \
+  --from-file=elasticsearch-3.crt=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.crt \
+  --from-file=elasticsearch-3.key=./security/certificates/elastic/opensearch/elasticsearch/elasticsearch-3/elasticsearch-3.key \
+  --from-file=es_kibana_client.pem=./security/certificates/elastic/opensearch/es_kibana_client.pem \
+  --from-file=es_kibana_client.key=./security/certificates/elastic/opensearch/es_kibana_client.key
+```
 
 ## Install
 
 ```bash
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
+  -f ./deploy/helm/opensearch.values.yaml \
+  --namespace cogstack --create-namespace
+```
+
+## Dashboards-only install
+
+Use this mode when OpenSearch is managed externally:
+
+```bash
+helm upgrade --install cogstack-dashboards ./deploy/charts/opensearch \
+  --set opensearch.enabled=false \
+  --set dashboards.enabled=true \
+  --set 'dashboards.opensearchHosts[0]=https://opensearch-client.cogstack.svc:9200' \
   --namespace cogstack --create-namespace
 ```
 
 ## Render templates
 
 ```bash
 helm template cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file envFile.raw=./deploy/elasticsearch.env
+  -f ./deploy/helm/opensearch.values.yaml
 ```
 
 ## Notes
 
-- The chart packages current repository config files under `files/`.
-- `envFile.raw` can be set from `deploy/elasticsearch.env` and is loaded via `envFrom` into OpenSearch and Dashboards.
-- Only keys listed in `envFile.includeKeys` are imported (to avoid leaking secrets from env files into ConfigMaps).
+- Helm templates cannot read arbitrary `../../...` paths directly; `.Files.Get` only sees files packaged inside the chart.
+- In this repo, the chart `files/` entries are symlinked to the shared `deploy/`, `services/`, and `security/` sources so Docker and Kubernetes stay aligned.
+- The standard install/render commands now use `-f ./deploy/helm/opensearch.values.yaml`; that file is for cluster-specific overrides only.
+- The shared `services/`, `security/`, and selected `deploy/` env files are consumed automatically by the chart defaults; you do not need to repeat those paths in the values file.
+- `envFile.raw` defaults to `deploy/elasticsearch.env` and can still be overridden; the chart reads only `ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, and `KIBANA_SERVER_NAME`, while pod IP and discovery hosts remain Kubernetes-specific.
+- `usersEnvFile.raw` defaults to `security/env/users_elasticsearch.env` and can still be overridden; only the credential keys required by the enabled components are imported.
+- `certificatesEnvFile.raw` defaults to `security/env/certificates_elasticsearch.env` and can still be overridden; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`<name>.pem` / `<name>.key`).
+- `deploy/elasticsearch.env` shared values are used where they make sense on Kubernetes (`ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, `KIBANA_SERVER_NAME`), while pod IP and discovery hosts remain Kubernetes-specific.
+- By default, `certificates.opensearchNodeFiles[*]` maps pod ordinals `0/1/2` to repo-style node cert keys `elasticsearch-1/2/3`.
+- `opensearch.logPersistence` and `opensearch.performanceAnalyzerPersistence` default to PVC-backed storage to stay closer to the Docker Compose deployment.
+- `opensearch.snapshotBackups` adds shared PVC-backed mounts for `/mnt/es_data_backups` and `/mnt/es_config_backups`; use RWX storage or set `existingClaim` values, and still set `path.repo` in the shared OpenSearch config if you want the cluster to use them.
+- `configFiles.opensearchRaw` can be set from `services/elasticsearch/config/opensearch.yml`.
+- `configFiles.log4jRaw` can be set from `services/elasticsearch/config/log4j2_opensearch.properties`.
+- `configFiles.dashboardsRaw` can be set from `services/kibana/config/opensearch.yml`.
+- `securityFiles.*Raw` can be set from `security/es_roles/opensearch/*.yml` and overrides the chart-bundled OpenSearch security files.
 - Review security and certificate settings before production use.

deploy/charts/opensearch/files/certificates-elasticsearch.envfile

@@ -0,0 +1 @@
+../../../../security/env/certificates_elasticsearch.env

deploy/charts/opensearch/files/deploy-elasticsearch.envfile

@@ -0,0 +1 @@
+../../../../deploy/elasticsearch.env

deploy/charts/opensearch/files/log4j2.properties

@@ -0,0 +1 @@
+../../../../services/elasticsearch/config/log4j2_opensearch.properties