diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 931b262..5f6319e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,8 +344,9 @@ jobs: case "$version" in *"\$Format:"*) echo 'Chart version was not materialized'; exit 1;; esac helm lint helm-chart helm package helm-chart --destination release-artifacts + mv "release-artifacts/manticoresearch-${version}.tgz" "release-artifacts/streams-${version}.tgz" echo "version=${version}" >> "$GITHUB_OUTPUT" - echo "asset=release-artifacts/manticoresearch-${version}.tgz" >> "$GITHUB_OUTPUT" + echo "asset=release-artifacts/streams-${version}.tgz" >> "$GITHUB_OUTPUT" - name: Create GitHub Release uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2.3.2 with: diff --git a/README.md b/README.md index 2c9a935..8735c7a 100644 --- a/README.md +++ b/README.md @@ -1,155 +1,56 @@ -# Kafka UI +# Manticore Streams -**Attention!!!** Min kubernetes version: 1.20 -### Kafka Installation +Manticore Streams is a Kubernetes application for filtering Apache Kafka topics with Manticore Search. Define filtering rules in the UI, process incoming documents at scale, and write matched documents to another Kafka topic. -If you don't have it, you install and run Kafka Server +## Requirements -```bash -helm repo add bitnami https://charts.bitnami.com/bitnami -helm --namespace kafka install --set listeners.client.protocol=PLAINTEXT --set listeners.controller.protocol=PLAINTEXT my-kafka bitnami/kafka -``` - -Don't worry about topics, Stream handler app create it automatically. - -In that case Kafka host will `my-kafka.kafka.svc.cluster.local` (inner Kubernetes). It will need us on goals creation stage +- A Kubernetes cluster +- Helm 3 +- An Apache Kafka cluster reachable from the Kubernetes workloads -If you see that either `pod/my-kafka-0` or `pod/my-kafka-zookeeper-0` is not running in `kubectl get all -n kafka` try: -``` -kubectl edit statefulset.apps/my-kafka-zookeeper -n kafka -kubectl edit statefulset.apps/my-kafka -n kafka -``` -To make it run under root: -``` - securityContext: - fsGroup: 0 - runAsUser: 0 -``` +The release chart uses public images from `ghcr.io/manticoresoftware/streams`; no image-pull secret is required. -**Exposing** +## Install -For exposing Kafka outside cluster we've modify Kafka helm chart config: +1. Download the `streams-.tgz` chart asset for the desired version from the [GitHub Releases page](https://github.com/manticoresoftware/streams/releases). +2. Install it with Helm. Choose a release name, namespace, administrator credentials, and the hostname served by your ingress controller: +```sh +helm install manticore-streams ./streams-.tgz \ + --namespace manticore-streams \ + --create-namespace \ + --set ui.admin.email='admin@example.com' \ + --set ui.admin.pass='change-this-password' \ + --set ingress.hosts[0].host='streams.example.com' \ + --set ingress.hosts[0].paths[0]='/' ``` -externalAccess.enabled=true -externalAccess.service.type=NodePort -externalAccess.serivce.nodePorts[0]='30001' -``` -It's work only if we will edit `values.yaml`. -Don't try to specify parameters by adding `--set` arguments to `helm install` command. This will not work!! - -### MKC Installation - -When Kafka server are runned, we can start deploying `MKC` chart: - -The chart defaults to private images hosted at `ghcr.io/manticoresoftware/streams`. Before installing, create an image-pull secret in the release namespace with a GitHub token that has `read:packages` access: - -`kubectl create secret docker-registry registry-manticore-streams --namespace={namespace} --docker-server=ghcr.io --docker-username={github-user} --docker-password={github-token}` - -To use a differently named secret, set `imagePullSecrets` in your values file. - -For installing you must specify auth l/p and ingress host of you project - -**Helm 2** - -`helm install --name {name} --namespace {namesace} ---set ui.admin.email="user@manticoresearch.com" ---set ui.admin.pass="myNewPassword" ---set ingress.hosts[0].host="kafka.manticoresearch.com" ---set ingress.hosts[0].paths[0]="/" ./manticoresearch-{version}.tgz` - -**Helm 3** - -`helm install {name} --namespace {namesace} ---set ui.admin.email="user@manticoresearch.com" ---set ui.admin.pass="myNewPassword" ---set ingress.hosts[0].host="kafka.manticoresearch.com" ---set ingress.hosts[0].paths[0]="/" ./manticoresearch-{version}.tgz` - +To deploy without an ingress controller, add `--set ingress.enabled=false` and expose the UI service by the mechanism appropriate for your cluster. -The release package uses the `values.yaml` embedded when it was built. To use your own edited values file, pass it explicitly with `-f values.yaml`: +Verify that the workloads are running: -For **Helm 2** - -`helm install --name {name} --namespace {namespace} -f values.yaml ./manticoresearch-{version}.tgz` - -For **Helm 3** - -`helm install {name} --namespace {namespace} -f values.yaml ./manticoresearch-{version}.tgz` - -Important parameters there: - -* ui.admin.email - email for super admin auth -* ui.admin.pass - password for super admin auth -* ingress.hosts - set host for UI access - -That's all - - -### Removing - -For **Helm 3** removing just run `helm uninstall {name} -n {namespace}` - -**Helm 2** can't handle dynamically created pods, do after removing we must clean up k8s: - -**Warning!!!**: don't use these commands in "default" namespace. -In case if you installed the chart to that namespace, you must remove it remove "by hand"!! -``` -kubectl -n {namespace} delete all --all -kubectl -n {namespace} delete cm --all -kubectl -n {namespace} delete pvc --all +```sh +kubectl get pods --namespace manticore-streams ``` -In some cases helm can not correctly remove chart and on reinstalling you can get error: +After signing in to the UI, configure sources, destinations, and streams. The Kafka brokers and topics you configure must be reachable from the Manticore Streams namespace. -``` -Error: rendered manifests contain a resource that already exists. Unable to continue with install: -existing resource conflict: kind: SomeResouce, namespace: , name: ResourceName -``` +## Upgrade and uninstall -You must remove it yourself: +Download the desired release asset, then upgrade the existing release with the same values: -``` -kubectl delete all -n {namespace} --all -kubectl delete pvc -n {namespace} --all -kubectl delete sa ui-admin-sa-{namespace} -n {namespace} -kubectl delete ClusterRole ui-admin-role-{namespace} -n {namespace} -kubectl delete ClusterRoleBinding ui-admin-{namespace} -n {namespace} -kubectl delete ingress kafka-pq -n {namespace} +```sh +helm upgrade manticore-streams ./streams-.tgz \ + --namespace manticore-streams \ + --reuse-values ``` +To remove the release: -### Our local cluster tests: -1) Must make vpn to kuber01 -2) Run +```sh +helm uninstall manticore-streams --namespace manticore-streams ``` -kafkacat -b kafka01.example.com:9092,kafka02.example.com:9092,kafka03.example.com:9092 \ --G mkc20200116 sina.firehose.out | kafkacat -b 78.47.167.154:30001 -t my-docs -``` - -### Api tokens -For using API first generate token from `/tokens` section. +## Documentation -After, add headers to your request: -``` -Authorization: Bearer $TOKEN -Accept: application/json -``` - -Endpoints: -``` -/api/admin/source/getList -/api/admin/source/add -/api/admin/source/delete -/api/admin/destination/getList -/api/admin/destination/add -/api/admin/destination/delete -/api/manager/getRulesList -/api/manager/addRule -/api/manager/deleteRule -/api/admin/process/add -/api/admin/process/assignUser -/api/admin/process/unassignUser -/api/admin/process/remove/{id} +See [docs/README.md](docs/README.md) for the full documentation index, including chart values, Kafka setup guidance, and operating Manticore Streams. \ No newline at end of file diff --git a/dev-environment/k8s_tests/dev-server-k3d.sh b/dev-environment/k8s_tests/dev-server-k3d.sh index 5d2f304..e8a2d4c 100755 --- a/dev-environment/k8s_tests/dev-server-k3d.sh +++ b/dev-environment/k8s_tests/dev-server-k3d.sh @@ -204,7 +204,6 @@ chart_overrides() { printf '%s\n' \ '--set' 'ingress.enabled=false' \ '--set' 'podSecurityPolicy.enabled=false' \ - '--set-json' 'imagePullSecrets=[]' \ '--set' 'worker.resources.requests.cpu=250m' \ '--set' 'worker.resources.requests.memory=512Mi' \ '--set' 'worker.resources.limits.cpu=1' \ diff --git a/docs/HelmVariables.md b/docs/HelmVariables.md index 46f5fe7..29d462d 100644 --- a/docs/HelmVariables.md +++ b/docs/HelmVariables.md @@ -1,10 +1,5 @@ # Helm Variables -### Container registry section -Parameter | Description | Default ---------- | ----------- | ------- -`imagePullSecrets` | Kubernetes image-pull secret names for private GHCR images | `[{name: registry-manticore-streams}]` - ### UI section Parameter | Description | Default --------- | ----------- | ------- diff --git a/docs/Installation.md b/docs/Installation.md index 3298292..7890c5c 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -1,98 +1,67 @@ # Installation -Manticore Streams for now supports only Apache Kafka as a source of an input stream. Read -this [article](InstallFromScratch/Kafka.md) about installation of Apache Kafka +Manticore Streams consumes Apache Kafka topics. Before installing it, make sure that Kafka is reachable from the Kubernetes namespace where Manticore Streams will run. See [Kafka setup](InstallFromScratch/Kafka.md) for a basic Kubernetes example. ---- +## Requirements -# Cloning +- Kubernetes +- Helm 3 +- An ingress controller when `ingress.enabled` is left at its default of `true` +- An Apache Kafka cluster reachable from the Manticore Streams workloads -### ⚠️ Attention! ⚠️ +## Download the chart -You can't use **GIT clone**. Download the `.tgz` chart asset from the GitHub Release for the desired version instead. The release package has matching chart and image versions already materialized. +Download the `streams-.tgz` asset for the desired version from [GitHub Releases](https://github.com/manticoresoftware/streams/releases). The package contains matching chart and image versions. Images are public at `ghcr.io/manticoresoftware/streams`, so no image-pull secret is required. -## Container images +## Install with Helm -The chart defaults to private images hosted at `ghcr.io/manticoresoftware/streams`. Before installing, create an image-pull secret in the release namespace with a GitHub token that has `read:packages` access: - -`kubectl create secret docker-registry registry-manticore-streams --namespace={namespace} --docker-server=ghcr.io --docker-username={github-user} --docker-password={github-token}` - -To use a differently named secret, set `imagePullSecrets` in your values file. - -## Helm - - -For installing, you must specify login and password for the admin, and ingress host of you project - -Important parameters there: - -* ui.admin.email - email for super admin auth -* ui.admin.pass - password for super admin auth -* ingress.hosts - set host for UI access - - - -### Helm 3 - - +Choose a release name, namespace, administrator credentials, and ingress hostname. The following command creates the namespace when it does not already exist: +```sh +helm install manticore-streams ./streams-.tgz \ + --namespace manticore-streams \ + --create-namespace \ + --set ui.admin.email='admin@example.com' \ + --set ui.admin.pass='change-this-password' \ + --set ingress.hosts[0].host='streams.example.com' \ + --set ingress.hosts[0].paths[0]='/' ``` -helm install {name} --namespace {namespace} \ ---set ui.admin.email="user@manticoresearch.com" \ ---set ui.admin.pass="myNewPassword" \ ---set ingress.hosts[0].host="kafka.manticoresearch.com" \ ---set ingress.hosts[0].paths[0]="/" ./manticoresearch-{version}.tgz -``` - - -### Helm 2 +To deploy without an ingress controller, add `--set ingress.enabled=false` and expose the UI service using the mechanism appropriate for the cluster. - +Check deployment status: +```sh +kubectl get pods --namespace manticore-streams ``` -helm install --name {name} --namespace {namespace} \ ---set ui.admin.email="user@manticoresearch.com" \ ---set ui.admin.pass="myNewPassword" \ ---set ingress.hosts[0].host="kafka.manticoresearch.com" \ ---set ingress.hosts[0].paths[0]="/" ./manticoresearch-{version}.tgz -``` - - - -The release package uses the `values.yaml` embedded when it was built. To use your own edited values file, pass it explicitly with `-f values.yaml`: - - - -### Helm 3 - - -```helm install {name} --namespace {namespace} -f values.yaml ./manticoresearch-{version}.tgz``` +Once the UI is available, sign in using the administrator credentials and configure Kafka sources, destinations, and streams. - +## Custom values -### Helm 2 +To keep installation settings in a file, put them in `values.yaml` and pass the file to Helm: - - -```helm install --name {name} --namespace {namespace} -f values.yaml ./manticoresearch-{version}.tgz``` - - +```sh +helm install manticore-streams ./streams-.tgz \ + --namespace manticore-streams \ + --create-namespace \ + --values values.yaml +``` -**That's all** +See [Helm chart variables](HelmVariables.md) for the available settings. -___ +## Upgrade and uninstall -Check for deploy: +Upgrade an existing deployment with a downloaded release asset: +```sh +helm upgrade manticore-streams ./streams-.tgz \ + --namespace manticore-streams \ + --reuse-values ``` -$ kubectl get po -n {namespace} -NAME READY STATUS RESTARTS AGE -mkc-columnar-0 1/1 Running 0 1m -mkc-scaler-69554f869b-g7b5f 1/1 Running 0 1m -mkc-ui-0 1/1 Running 0 1m -mkc-ui-mysql-0 1/1 Running 0 1m -``` -After our project has been deployed we can start to create streams +Remove the release with: + +```sh +helm uninstall manticore-streams --namespace manticore-streams +``` \ No newline at end of file diff --git a/docs/Restore.md b/docs/Restore.md index 86ac73d..3dc785e 100644 --- a/docs/Restore.md +++ b/docs/Restore.md @@ -9,7 +9,7 @@ Backups are stored in the `backup / data` folder * database.sql - backup of UI pod -* indexes-mkc-m{N}-manticore.tar.gz - stream's backup where stream id = `N` +* indexes-ms-m{N}-manticore.tar.gz - stream's backup where stream id = `N` Recovery takes place gradually, stream after stream. diff --git a/helm-chart/templates/deployment-scaler.yaml b/helm-chart/templates/deployment-scaler.yaml index 0395b97..82ae11c 100644 --- a/helm-chart/templates/deployment-scaler.yaml +++ b/helm-chart/templates/deployment-scaler.yaml @@ -121,10 +121,6 @@ spec: mountPath: /var/www/html resources: {{- toYaml .Values.scaler.nginx.resources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumes: - name: config-volume configMap: diff --git a/helm-chart/templates/deployment-ui.yaml b/helm-chart/templates/deployment-ui.yaml index 5cdd1ee..22194a0 100644 --- a/helm-chart/templates/deployment-ui.yaml +++ b/helm-chart/templates/deployment-ui.yaml @@ -189,10 +189,6 @@ spec: subPath: nginx.conf resources: {{- toYaml .Values.ui.nginx.resources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumes: - name: process-config configMap: diff --git a/helm-chart/templates/ingress.yaml b/helm-chart/templates/ingress.yaml index 5545514..adc97cc 100644 --- a/helm-chart/templates/ingress.yaml +++ b/helm-chart/templates/ingress.yaml @@ -10,7 +10,7 @@ apiVersion: extensions/v1beta1 {{- end }} kind: Ingress metadata: - name: mkc + name: {{ $fullName }} labels: {{- include "manticore-streams.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} diff --git a/helm-chart/templates/migration-job.yaml b/helm-chart/templates/migration-job.yaml index 0d89e96..a867726 100644 --- a/helm-chart/templates/migration-job.yaml +++ b/helm-chart/templates/migration-job.yaml @@ -78,10 +78,6 @@ spec: value: "1024" resources: {{- toYaml .Values.ui.initResources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumes: - name: process-config configMap: diff --git a/helm-chart/templates/post-upgrade-hook.yaml b/helm-chart/templates/post-upgrade-hook.yaml index 5385929..5a6982a 100644 --- a/helm-chart/templates/post-upgrade-hook.yaml +++ b/helm-chart/templates/post-upgrade-hook.yaml @@ -136,10 +136,6 @@ spec: {{- end }} resources: {{- toYaml .Values.ui.resources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumes: - name: process-config configMap: diff --git a/helm-chart/templates/pre-delete-hook.yaml b/helm-chart/templates/pre-delete-hook.yaml index cbe80bb..10ac071 100644 --- a/helm-chart/templates/pre-delete-hook.yaml +++ b/helm-chart/templates/pre-delete-hook.yaml @@ -89,10 +89,6 @@ spec: {{- end }} resources: {{- toYaml .Values.ui.resources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumes: - name: process-config configMap: diff --git a/helm-chart/templates/stateful-set-columnar.yaml b/helm-chart/templates/stateful-set-columnar.yaml index 3100dac..53b7a53 100644 --- a/helm-chart/templates/stateful-set-columnar.yaml +++ b/helm-chart/templates/stateful-set-columnar.yaml @@ -73,10 +73,6 @@ spec: {{- end }} resources: {{- toYaml .Values.columnar.resources | nindent 10 }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} volumeClaimTemplates: - metadata: name: storage diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index bae49d6..1fd4558 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -1,6 +1,3 @@ -imagePullSecrets: - - name: registry-manticore-streams - ui: replicas: 1 userId: 33 diff --git a/ui/app/Http/Controllers/GoalsController.php b/ui/app/Http/Controllers/GoalsController.php index dcc672d..52a39ae 100644 --- a/ui/app/Http/Controllers/GoalsController.php +++ b/ui/app/Http/Controllers/GoalsController.php @@ -88,7 +88,7 @@ public function add(Request $request): JsonResponse if ( ! $request->get('group')) { $group = md5(microtime()); $group = substr($group, 0, 8); - $request->request->add(['group' => 'MKC_' . $group]); + $request->request->add(['group' => 'MS_' . $group]); } $errors = []; diff --git a/ui/config/app.php b/ui/config/app.php index 94950b8..0b71e8c 100644 --- a/ui/config/app.php +++ b/ui/config/app.php @@ -13,7 +13,7 @@ | */ - 'name' => env('APP_NAME', 'Manticore Kafka Connector'), + 'name' => env('APP_NAME', 'Manticore Streams'), /* |-------------------------------------------------------------------------- diff --git a/ui/database/factories/DestinationFactory.php b/ui/database/factories/DestinationFactory.php index 237b5c3..a83bb0c 100644 --- a/ui/database/factories/DestinationFactory.php +++ b/ui/database/factories/DestinationFactory.php @@ -17,7 +17,7 @@ public function definition() 'name' => $this->faker->unique()->word, 'host' => 'dev.manticoresearch.com:22', 'topic' => 'out.{username}', - 'group' => 'MKC_' . $this->faker->word, + 'group' => 'MS_' . $this->faker->word, ]; } } diff --git a/ui/database/factories/SourceFactory.php b/ui/database/factories/SourceFactory.php index 28512ab..96b2fb6 100644 --- a/ui/database/factories/SourceFactory.php +++ b/ui/database/factories/SourceFactory.php @@ -17,7 +17,7 @@ public function definition(): array 'name' => $this->faker->unique()->word, 'host' => 'dev.manticoresearch.com:22', 'topic' => 'my-docs', - 'group' => 'MKC_' . $this->faker->word, + 'group' => 'MS_' . $this->faker->word, ]; } } diff --git a/ui/storage/yamlSource/stateful-set-pipeline.yaml b/ui/storage/yamlSource/stateful-set-pipeline.yaml index 9b8023b..333c521 100644 --- a/ui/storage/yamlSource/stateful-set-pipeline.yaml +++ b/ui/storage/yamlSource/stateful-set-pipeline.yaml @@ -280,8 +280,6 @@ spec: {{- end }} resources: {{ Values.rulesChecker.resources | toYaml 10 }} {{- end }} - imagePullSecrets: - - name: registry-manticore-streams volumeClaimTemplates: - metadata: name: data diff --git a/ui/tests/Unit/PodsCreatorTest.php b/ui/tests/Unit/PodsCreatorTest.php index 01721b9..73e6971 100644 --- a/ui/tests/Unit/PodsCreatorTest.php +++ b/ui/tests/Unit/PodsCreatorTest.php @@ -27,7 +27,7 @@ class PodsCreatorTest extends TestCase 'outputHost' => 'dev.manticoresearch.com:21', 'inputTopic' => 'my-docs', 'outputTopic' => 'out.{username}', - 'groupName' => 'MKC_dolores', + 'groupName' => 'MS_dolores', ], 'worker' => [