From 08b881f9a17f27064c63e5b2a92a938cc3365362 Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 18:03:53 +0800 Subject: [PATCH 1/6] Change the service name of OCS rgw After deploying, Ceph RGW should be connected by using 'rook-ceph-rgw-s3a' --- demos/ach/bucket_notifications.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/ach/bucket_notifications.ipynb b/demos/ach/bucket_notifications.ipynb index 1a80ca2..0aed0aa 100644 --- a/demos/ach/bucket_notifications.ipynb +++ b/demos/ach/bucket_notifications.ipynb @@ -18,7 +18,7 @@ "metadata": {}, "outputs": [], "source": [ - "endpoint_url = 'http://rook-ceph-rgw-ocs-storagecluster-cephobjectstore.openshift-storage'\n", + "endpoint_url = 'http://rook-ceph-rgw-s3a.openshift-storage'\n", "aws_access_key_id = 'replace_me'\n", "aws_secret_access_key = 'replace_me'" ] @@ -230,4 +230,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} From 5aeb6c5b945f9dad89bd268290e7217b5e3a23d8 Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 18:06:32 +0800 Subject: [PATCH 2/6] Correct the service name of MySQL Service name of MySQL should be corrected with 'achdbservice' based on information inserted when deploying MariaDB --- demos/ach/grafana-mysql-datasource.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/ach/grafana-mysql-datasource.yaml b/demos/ach/grafana-mysql-datasource.yaml index 24c60ba..8e16fbb 100644 --- a/demos/ach/grafana-mysql-datasource.yaml +++ b/demos/ach/grafana-mysql-datasource.yaml @@ -8,8 +8,8 @@ spec: - type: mysql name: MySQL access: proxy - url: ach-db.ach + url: achdbservice.ach database: achdb user: achuser password: achpassword - name: mysql-grafanadatasource.yaml \ No newline at end of file + name: mysql-grafanadatasource.yaml From c5abc5fb458f005883458fdea0d70d9bac5ec179 Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 18:12:05 +0800 Subject: [PATCH 3/6] Correct the namespace of kafka source Kafka cluster is deployed on 'ach' namespace based on steps of deploying kafka. Therfore, url of bootstrapServer of kafka should be corresponded with the right namespace --- demos/ach/kafkasource-odfi-split.yaml | 2 +- demos/ach/kafkasource-rdfi-process.yaml | 2 +- demos/ach/kafkasource-rdfi-split.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/ach/kafkasource-odfi-split.yaml b/demos/ach/kafkasource-odfi-split.yaml index d5a8836..aed7004 100644 --- a/demos/ach/kafkasource-odfi-split.yaml +++ b/demos/ach/kafkasource-odfi-split.yaml @@ -6,7 +6,7 @@ metadata: spec: consumerGroup: odfi-split bootstrapServers: - - my-cluster-kafka-bootstrap.kafka:9092 + - my-cluster-kafka-bootstrap.ach:9092 topics: - merchant-upload sink: diff --git a/demos/ach/kafkasource-rdfi-process.yaml b/demos/ach/kafkasource-rdfi-process.yaml index 6a4cd16..553595c 100644 --- a/demos/ach/kafkasource-rdfi-process.yaml +++ b/demos/ach/kafkasource-rdfi-process.yaml @@ -6,7 +6,7 @@ metadata: spec: consumerGroup: rdfi-process bootstrapServers: - - my-cluster-kafka-bootstrap.kafka:9092 + - my-cluster-kafka-bootstrap.ach:9092 topics: - rdfi sink: diff --git a/demos/ach/kafkasource-rdfi-split.yaml b/demos/ach/kafkasource-rdfi-split.yaml index 170a8da..6cade29 100644 --- a/demos/ach/kafkasource-rdfi-split.yaml +++ b/demos/ach/kafkasource-rdfi-split.yaml @@ -5,7 +5,7 @@ metadata: namespace: ach spec: bootstrapServers: - - 'my-cluster-kafka-bootstrap.kafka:9092' + - 'my-cluster-kafka-bootstrap.ach:9092' consumerGroup: rdfi-split sink: ref: From 12b338aa95c398628149e1d3604ab1e72fe83488 Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 18:21:26 +0800 Subject: [PATCH 4/6] Add kfdef v0.9 definition This file contains necessary information for deploying ODH 0.9 --- demos/ach/odh/kfdef-v0.9.yaml | 123 ++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 demos/ach/odh/kfdef-v0.9.yaml diff --git a/demos/ach/odh/kfdef-v0.9.yaml b/demos/ach/odh/kfdef-v0.9.yaml new file mode 100644 index 0000000..626262b --- /dev/null +++ b/demos/ach/odh/kfdef-v0.9.yaml @@ -0,0 +1,123 @@ +apiVersion: kfdef.apps.kubeflow.org/v1 +kind: KfDef +metadata: + name: opendatahub + namespace: ach +spec: + applications: + - kustomizeConfig: + repoRef: + name: manifests + path: odh-common + name: odh-common + #Note: AI Library components requires seldon do be deployed first, make sure to place seldon before AIlibrary + - kustomizeConfig: + repoRef: + name: manifests + path: odhseldon/cluster + name: odhseldon + - kustomizeConfig: + repoRef: + name: manifests + path: ai-library/cluster + name: ai-library-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: ai-library/operator + # Note: In order to utilize ai-library, you also need to have Seldon installed + name: ai-library-operator + - kustomizeConfig: + parameters: + # Note: The admin username is admin + - name: SUPERSET_ADMIN_PASSWORD + value: admin + repoRef: + name: manifests + path: superset + name: superset + - kustomizeConfig: + parameters: + - name: namespace + value: openshift-operators + repoRef: + name: manifests + path: kafka/cluster + name: strimzi-operator + - kustomizeConfig: + repoRef: + name: manifests + path: kafka/kafka + name: kafka-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: grafana/cluster + name: grafana-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: grafana/grafana + name: grafana-instance + - kustomizeConfig: + repoRef: + name: manifests + path: radanalyticsio/spark/cluster + name: radanalyticsio-spark-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: prometheus/cluster + name: prometheus-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: prometheus/operator + name: prometheus-operator + - kustomizeConfig: + parameters: + - name: s3_endpoint_url + value: "s3.odh.com" + repoRef: + name: manifests + path: jupyterhub/jupyterhub + name: jupyterhub + - kustomizeConfig: + overlays: + #- cuda + - additional + repoRef: + name: manifests + path: jupyterhub/notebook-images + name: notebook-images + - kustomizeConfig: + repoRef: + name: manifests + path: airflow/operator + name: airflow-operator + - kustomizeConfig: + repoRef: + name: manifests + path: airflow/cluster + name: airflow-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: odhargo/cluster + name: odhargo-cluster + - kustomizeConfig: + repoRef: + name: manifests + path: odhargo/odhargo + name: odhargo + - kustomizeConfig: + repoRef: + name: manifests + path: odh-dashboard + name: odh-dashboard + repos: + - name: kf-manifests + uri: https://github.com/opendatahub-io/manifests/tarball/v1.0-branch-openshift + - name: manifests + uri: https://github.com/opendatahub-io/odh-manifests/tarball/v0.9.0 + version: v0.9.0 From cacbac79be0668d095b5b7f9b2ba4a3a7ab6fc7d Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 19:03:48 +0800 Subject: [PATCH 5/6] Update README.adoc --- demos/ach/README.adoc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/demos/ach/README.adoc b/demos/ach/README.adoc index d6db4c3..143b6dd 100644 --- a/demos/ach/README.adoc +++ b/demos/ach/README.adoc @@ -72,6 +72,13 @@ oc create project ach From now on, all the commands must be run into this `ach` project. +=== Open Data Hub +Open Data Hub(ODH) will be deployed using ODH operator. Create an instance of ODH by using the Kfdef definition in /odh. This is the version 0.9 of ODH. +[bash] +---- +oc apply -f odh/kfdef-v0.9.yaml +---- + === Kafka Usign the AMQ Stream or Strimzi operator, we can create a basic Kafka cluster, along with the needed topics. @@ -268,4 +275,4 @@ In the **tools** folder you will also find: * ach-bd.txt: various SQL commands to create and (re)initialize tables used in the auxiliary database (used to store the number of processed files) * amq-streams-ocs4.yaml: definition od the KafkaCluster created with the AMQStreams operator, using OCS4 storage for persistency * grafana-prometheus-datasource.yaml: datasource for the Grafana Operator to connect to OpenShift Prometheus. You will have to replace the secret (basicAuth) by the one used by the built-in OpenShift Grafana. -* kafdrop.yaml: installation of Kafdrop to monitor your Kafka/AMQStreams cluster \ No newline at end of file +* kafdrop.yaml: installation of Kafdrop to monitor your Kafka/AMQStreams cluster From 812a5bf6fdf9ad4bc799ebb79e787e58772aa422 Mon Sep 17 00:00:00 2001 From: Tutj Date: Thu, 24 Dec 2020 19:06:45 +0800 Subject: [PATCH 6/6] Update README.adoc --- demos/ach/README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/ach/README.adoc b/demos/ach/README.adoc index 143b6dd..ec0055d 100644 --- a/demos/ach/README.adoc +++ b/demos/ach/README.adoc @@ -101,7 +101,7 @@ Those are the secrets that will be used by the various pods we will create. First, edit the file `secrets.yaml` and replace the values for `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` that you got when creating the RGW user. -If you have followed the previous instructions for creating the Object Store, the service should have been created with the right name, `http://rook-ceph-rgw-ocs-storagecluster-cephobjectstore.openshift-storage` so you should not need to change it. +If you have followed the previous instructions for creating the Object Store, the service should have been created with the right name, `http://rook-ceph-rgw-s3a.openshift-storage` so you should not need to change it. You can leave the database parameters as they are, but you must use those when creating the database...