From ba20c3ae6d840fd1fe4d0c57e84e398f5b5f0f11 Mon Sep 17 00:00:00 2001 From: Alex Gavrisco Date: Sun, 5 Apr 2020 22:18:41 +0300 Subject: [PATCH 01/22] Add helm chart for Vector installation Signed-off-by: Alex Gavrisco Signed-off-by: MOZGIII --- distribution/helm/vector/.helmignore | 23 +++ distribution/helm/vector/Chart.yaml | 13 ++ distribution/helm/vector/README.md | 158 ++++++++++++++++++ .../helm/vector/templates/_helpers.tpl | 98 +++++++++++ .../helm/vector/templates/configmap.yaml | 60 +++++++ .../helm/vector/templates/daemonset.yaml | 84 ++++++++++ distribution/helm/vector/templates/rbac.yaml | 93 +++++++++++ .../helm/vector/templates/service.yaml | 23 +++ .../helm/vector/templates/servicemonitor.yaml | 30 ++++ distribution/helm/vector/values.yaml | 62 +++++++ 10 files changed, 644 insertions(+) create mode 100644 distribution/helm/vector/.helmignore create mode 100644 distribution/helm/vector/Chart.yaml create mode 100644 distribution/helm/vector/README.md create mode 100644 distribution/helm/vector/templates/_helpers.tpl create mode 100644 distribution/helm/vector/templates/configmap.yaml create mode 100644 distribution/helm/vector/templates/daemonset.yaml create mode 100644 distribution/helm/vector/templates/rbac.yaml create mode 100644 distribution/helm/vector/templates/service.yaml create mode 100644 distribution/helm/vector/templates/servicemonitor.yaml create mode 100644 distribution/helm/vector/values.yaml diff --git a/distribution/helm/vector/.helmignore b/distribution/helm/vector/.helmignore new file mode 100644 index 0000000000000..0e8a0eb36f4ca --- /dev/null +++ b/distribution/helm/vector/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml new file mode 100644 index 0000000000000..351e356350ac1 --- /dev/null +++ b/distribution/helm/vector/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +name: vector +type: application +icon: https://vector.dev/press/vector-icon.svg +description: A Helm chart to collect Kubernetes logs with vector +version: 0.0.1 +appVersion: 0.8 +home: https://vector.dev/ +sources: + - https://github.com/timberio/vector/ +maintainers: + - name: Vector Contributors + email: vector@timber.io diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md new file mode 100644 index 0000000000000..3c8732b822a18 --- /dev/null +++ b/distribution/helm/vector/README.md @@ -0,0 +1,158 @@ +# Vector Chart + +[Vector](https://vector.dev/) - a lightweight and ultra-fast tool for building observability pipelines. + +- [Chart Details](#chartdetails) +- [Installing the cart](#installingthechart) +- [Configuration](#configuration) +- [Examples](#examples) + +## Chart Details + +This chart will do the following: + +* Create a ConfigMap with Vector configuration; +* Install a Daemonset that provisions Vector; +* Deploy service and service monitor for integration with prometheus-operator (if enabled); + +## Installing the Chart + +To install the chart with the release name `vector` and namespace `logging`: + +### Helm 2 + +```bash +$ cd distribution/helm/vector +$ helm install --namespace logging --name vector . +``` + +### Helm 3 + +```bash +$ cd distribution/helm/vector +$ helm install --namespace logging vector . +``` + +## Configuration + +The following table lists the configurable parameters of the Vector chart and the default values. + +| Parameter | Description | Default | +| ----------------------------- | ---------------------------------- | ----------------------- | +| **General** | +| `image.repository` | Image | `timberio/vector` | +| `image.tag` | Image tag | `0.8.X-debian` | +| `image.pullPolicy` | Image pull policy | `Always` | +| `image.pullSecrets` | Image pull secrets | `nil` | +| `updateStrategy` | DaemonSet update strategy | `RollingUpdate` | +| `metrics.enabled` | Specifies whether a service for metrics should be exposed | `true` | +| `metrics.service.type` | Metrics service type | `ClusterIP` | +| `metrics.service.port` | Metrics service port | `2020` | +| `metrics.service.annotations` | Optional metrics service annotations | `nil` | +| `metrics.service.labels` | Additional labels for the metrics service definition, specified as a map. | `nil` | +| `metrics.serviceMonitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator. | `true` | +| `metrics.serviceMonitor.endpoint` | Endpoint to be used by Prometheus to scrape metrics. | `true` | +| `metrics.serviceMonitor.additionalLabels` | Additional labels that can be used for ServiceMonitor discovery by Prometheus. | `nil` | +| `metrics.serviceMonitor.namespace` | Optional namespace in which to create ServiceMonito (default: current namespace). | `nil` | +| `metrics.serviceMonitor.interval` | Scrape interval. If not set, the Prometheus default scrape interval is used. | `60s` | +| `metrics.serviceMonitor.scrapeTimeout` | Scrape timeout. If not set, the Prometheus default scrape timeout is used. | `10s` | +| `rbac.enabled` | Specifies whether RBAC should be enabled. | `true` | +| `rbac.apiVersion` | Overrides K8S API version for RBAC (by default it's determined using `Capabilities`) | `nil` | +| `rbac.serviceAccount.name` | Overrides service acount name, if not provided and RBAC is disabled, `default` will be used. | `nil` | +| `rbac.serviceAccount.annotations` | Additional annotation for the created service account. | `{}` | +| `rbac.psp.enabled` | Specifies whether a PodSecurityPolicy should be created. | `false` | +| `rbac.apiVersion` | Overrides K8S API version for PSP (by default it's determined using `Capabilities`). | `nil` | +| `env` | A list of environment variables to be used for the DaemonSet. | `[]` | +| `resources` | Pod resource requests & limits. | `{}` | +| `tolerations` | Optional daemonset tolerations. | `[]` | +| `nodeSelector` | Node labels for pod assignment. | `{}` | +| `affinity` | Expressions for affinity. | `{}` | +| `extraVolumes` | Extra volumes to be assignet to pods. | `{}` | +| `extraVolumeMounts` | Extra volume mounts for the vector container. | `{}` | +| **ConfigMap** | +| `existingConfigMap` | Name of the exising configmap to be used for Vector configuration. | `nil` | +| `globalOptions.dataDir` | Vector's data directory. | `/var/tmp/vector` | +| `logSchema.hostKey` | Log host field. | `host` | +| `logSchema.messageKey` | Log message field. | `log` | +| `logSchema.timestampKey` | Log generation timestamp field. | `time` | +| `sources.kubernetes.enabled` | Enables Kubernetes source. | `true` | +| `sources.kubernetes.sourceId` | Kubernetes source ID. | `kubernetes` | +| `sources.kubernetes.includeNamespaces` | A list of namespaces to collect logs from. | `[]` | +| `sources.kubernetes.includeContainerNames` | A list of container names to collect logs from. | `[]` | +| `sources.kubernetes.rawConfig` | Raw config to be used for the source. | `nil` | +| `sources.additionalSources` | An object of additional sources. Key will be used as source ID. | `{}` | +| `sources.additionalSources.type` | Source type. | | +| `sources.additionalSources.rawConfig` | Raw config to be used for the additional source. | `nil` | +| `transforms` | An object of transforms. Key will be used as transform ID. | `{}` | +| `transforms.type` | Tranform type. | | +| `transforms.inputs` | A list of transfor data sources. | | +| `transforms.rawConfig` | Raw config to be used for the transform. | `nil` | +| `sinks` | An object of sinks. Entry key will be used as sink ID. | `{}` | +| `sinks.type` | Sink type. | | +| `sinks.inputs` | A list of sink data sources. | | +| `sinks.rawConfig` | Raw config to be used for the sink. | `nil` | + + +## Examples + +Here are some snippets for Vector configuration. + +### Overriding Kubernetes source namespaces + +Whitelisted namespaces (or other options) can be set for the `kubernetes` source, +just by overriding `sources.kubernetes` options. + +```yaml +sources: + kubernetes: + includeNamespaces: + - default +``` + +### Using transforms + +```yaml +transforms: + envTagging: + type: add_fields + inputs: + - kubernetes + rawConfig: | + [transforms.envTagging.fields] + env = "dev" + + jsonParser: + type: json_parser + inputs: + - envTagging + rawConfig: | + field = "message" + drop_field = true + drop_invalid = false +``` + +### Using Splunk sink + +In this example the Splunk HEC sink uses token from env variable (referenced from K8S secret). + +```yaml +env: + - name: SPLUNK_TOKEN + valueFrom: + secretKeyRef: + name: vector-credentials + key: SPLUNK_TOKEN + +sinks: + splunk: + type: splunk_hec + inputs: + - jsonParser + rawConfig: | + host = "https://splunk-endpoint.net" + token = "${SPLUNK_TOKEN}" + encoding = "json" + healthcheck = true + indexed_fields = ["env", "pod_name", "pod_namespace", "container_name"] + +``` diff --git a/distribution/helm/vector/templates/_helpers.tpl b/distribution/helm/vector/templates/_helpers.tpl new file mode 100644 index 0000000000000..166428f6ad765 --- /dev/null +++ b/distribution/helm/vector/templates/_helpers.tpl @@ -0,0 +1,98 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "vector.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "vector.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "vector.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "vector.labels" -}} +helm.sh/chart: {{ include "vector.chart" . }} +{{ include "vector.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "vector.selectorLabels" -}} +app.kubernetes.io/name: {{ include "vector.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "vector.serviceAccountName" -}} +{{- if .Values.rbac.enabled -}} + {{ default (include "vector.fullname" .) .Values.rbac.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.rbac.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Define apiVersion for RBAC APIs, depending on availability. +*/}} +{{- define "rbac.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}} +rbac.authorization.k8s.io/v1 +{{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}} +rbac.authorization.k8s.io/v1beta1 +{{- else -}} +rbac.authorization.k8s.io/v1alpha1 +{{- end -}} +{{- end -}} + +{{/* +Define apiVersion for PSP APIs, depending on availability. +*/}} +{{- define "psp.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "policy/v1" -}} +policy/v1 +{{- else if .Capabilities.APIVersions.Has "policy/v1beta1" -}} +policy/v1beta1 +{{- end -}} +{{- end -}} + +{{/* +Define the name of the configmap +*/}} +{{- define "vector.configMapName" -}} +{{- if (empty .Values.existingConfigMap) -}} +{{- include "vector.fullname" . -}} +{{- else -}} +{{- .Values.existingConfigMap -}} +{{- end -}} +{{- end -}} diff --git a/distribution/helm/vector/templates/configmap.yaml b/distribution/helm/vector/templates/configmap.yaml new file mode 100644 index 0000000000000..41dcf9ef86f97 --- /dev/null +++ b/distribution/helm/vector/templates/configmap.yaml @@ -0,0 +1,60 @@ +{{- if (empty .Values.existingConfigMap) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "vector.configMapName" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +data: + vector.toml: | + data_dir = "{{ .Values.globalOptions.dataDir }}" + + {{- if .Values.logSchema }} + [log_schema] + host_key = "{{ .Values.logSchema.hostKey }}" + message_key = "{{ .Values.logSchema.messageKey }}" + timestamp_key = "{{ .Values.logSchema.timestampKey }}" + {{- end }} + + {{- if .Values.sources.kubernetes.enabled }} + [sources.{{ .Values.sources.kubernetes.sourceId }}] + type = "kubernetes" + include_namespaces = {{ .Values.sources.kubernetes.includeNamespaces | toJson }} + include_container_names = {{ .Values.sources.kubernetes.includeContainerNames | toJson }} + + {{- if not (empty .Values.sources.kubernetes.rawConfig) }} + {{- .Values.sources.kubernetes.rawConfig | nindent 6 }} + {{- end }} + {{- end }} + + {{- range $sourceId, $source := .Values.sources.additionalSources }} + [sources.{{ $sourceId }}] + type = "{{ $source.type }}" + + {{- if not (empty $source.rawConfig )}} + {{- $source.rawConfig | nindent 6}} + {{- end }} + {{- end }} + + {{- range $transformId, $transform := .Values.transforms }} + [transforms.{{ $transformId }}] + type = "{{ $transform.type }}" + inputs = {{ $transform.inputs | toJson }} + + {{- if not (empty $transform.rawConfig )}} + {{- $transform.rawConfig | nindent 6}} + {{- end }} + {{- end }} + + {{- range $sinkId, $sink := .Values.sinks }} + [sinks.{{ $sinkId }}] + type = "{{ $sink.type }}" + inputs = {{ $sink.inputs | toJson }} + + {{- if not (empty $sink.rawConfig )}} + {{- $sink.rawConfig | nindent 6}} + {{- end }} + {{- end }} + + +{{- end }} diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml new file mode 100644 index 0000000000000..29a0d661b8500 --- /dev/null +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "vector.fullname" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "vector.selectorLabels" . | nindent 6 }} + minReadySeconds: 1 + updateStrategy: + type: {{ .Values.updateStrategy }} + template: + metadata: + labels: + {{- include "vector.selectorLabels" . | nindent 8 }} + spec: + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + containers: + - name: vector + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + args: + - "-w" + {{- if .Values.metrics.enabled }} + - "-m" + - "0.0.0.0:{{ .Values.metrics.service.port }}" + {{- end }} + env: + {{- toYaml .Values.env | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if or .Values.metrics.enabled }} + ports: + - name: metrics + containerPort: {{ .Values.metrics.service.port }} + protocol: TCP + {{- end }} + volumeMounts: + - name: var-log + mountPath: /var/log/ + readOnly: true + - name: var-lib + mountPath: /var/lib + readOnly: true + - name: data-dir + mountPath: "{{ .Values.globalOptions.dataDir }}" + - name: config-dir + mountPath: /etc/vector + readOnly: true + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: + {{ toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: + {{ toYaml .Values.affinity | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{ toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: var-log + hostPath: + path: /var/log/ + - name: var-lib + hostPath: + path: /var/lib/ + - name: data-dir + emptyDir: {} + - name: config-dir + configMap: + name: {{ include "vector.configMapName" . }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} diff --git a/distribution/helm/vector/templates/rbac.yaml b/distribution/helm/vector/templates/rbac.yaml new file mode 100644 index 0000000000000..1ec86fe569add --- /dev/null +++ b/distribution/helm/vector/templates/rbac.yaml @@ -0,0 +1,93 @@ +{{- if .Values.rbac.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "vector.serviceAccountName" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} + {{- with .Values.rbac.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: {{ default (include "rbac.apiVersion" .) .Values.rbac.apiVersion }} +kind: ClusterRole +metadata: + name: {{ template "vector.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get +{{- if .Values.rbac.psp.enabled }} + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - {{ template "vector.fullname" . }} + verbs: + - use +{{- end }} +--- +apiVersion: {{ default (include "rbac.apiVersion" .) .Values.rbac.apiVersion }} +kind: ClusterRoleBinding +metadata: + name: {{ template "vector.fullname" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "vector.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "vector.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +{{- if .Values.rbac.psp.enabled }} +apiVersion: {{ default (include "psp.apiVersion" .) .Values.rbac.psp.apiVersion }} +kind: PodSecurityPolicy +metadata: + name: {{ template "vector.fullname" . }} +spec: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + requiredDropCapabilities: + - ALL + volumes: + - 'hostPath' + - 'configMap' + - 'secret' + allowedHostPaths: + - pathPrefix: "/var/log" + readOnly: true + - pathPrefix: "/var/lib/docker/containers" + readOnly: true + {{- range .Values.extraVolumes }} + {{- if .hostPath }} + - pathPrefix: {{ .hostPath.path }} + {{- end }} + {{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 +{{- end }} +{{- end }} diff --git a/distribution/helm/vector/templates/service.yaml b/distribution/helm/vector/templates/service.yaml new file mode 100644 index 0000000000000..65fadd47f3d8f --- /dev/null +++ b/distribution/helm/vector/templates/service.yaml @@ -0,0 +1,23 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + {{- if .Values.metrics.service.annotations }} + annotations: + {{- toYaml .Values.metrics.service.annotations | nindent 4 }} + {{- end }} + name: {{ template "vector.fullname" . }}-metrics + labels: + {{- include "vector.labels" . | nindent 4 }} + {{- if .Values.metrics.service.additionalLabels }} + {{ toYaml .Values.metrics.service.additionalLabels }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type}} + ports: + - port: {{ .Values.metrics.service.port }} + targetPort: metrics + name: metrics + selector: + {{- include "vector.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/distribution/helm/vector/templates/servicemonitor.yaml b/distribution/helm/vector/templates/servicemonitor.yaml new file mode 100644 index 0000000000000..e0c837bf8fb2b --- /dev/null +++ b/distribution/helm/vector/templates/servicemonitor.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "vector.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "vector.labels" . | nindent 4 }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{ toYaml .Values.metrics.serviceMonitor.additionalLabels }} + {{- end }} +spec: + endpoints: + - port: metrics + path: {{ .Values.metrics.serviceMonitor.endpoint }} + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "vector.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml new file mode 100644 index 0000000000000..d574915f00ca6 --- /dev/null +++ b/distribution/helm/vector/values.yaml @@ -0,0 +1,62 @@ +image: + repository: timberio/vector + tag: 0.8.X-debian + pullPolicy: Always + +updateStrategy: RollingUpdate + +metrics: + enabled: false + service: + port: 2020 + type: ClusterIP + annotations: {} + additionalLabels: {} + serviceMonitor: + enabled: false + endpoint: / + additionalLabels: {} + interval: 60s + scrapeTimeout: 10s + +rbac: + enabled: true + serviceAccount: + annotation: {} + name: + psp: + enabled: false + +env: [] + +tolerations: [] +nodeSelector: {} +affinity: {} +resources: {} + +extraVolumes: [] + +extraVolumeMounts: [] + +existingConfigMap: "" + +globalOptions: + dataDir: "/var/tmp/vector" + +logSchema: + hostKey: "host" + messageKey: "message" + timestampKey: "timestamp" + +sources: + kubernetes: + enabled: true + sourceId: kubernetes + includeNamespaces: [] + includeContainerNames: [] + + additionalSources: {} + +transforms: {} + +sinks: {} From e7ccdbfe022e4f3ea0ff5f35b4bf7d0a19dd02e9 Mon Sep 17 00:00:00 2001 From: Alex Gavrisco Date: Sun, 5 Apr 2020 22:39:21 +0300 Subject: [PATCH 02/22] Fix Markdown checkstyle Signed-off-by: Alex Gavrisco Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 3c8732b822a18..4942b4e1b6738 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -11,9 +11,9 @@ This chart will do the following: -* Create a ConfigMap with Vector configuration; -* Install a Daemonset that provisions Vector; -* Deploy service and service monitor for integration with prometheus-operator (if enabled); +- Create a ConfigMap with Vector configuration; +- Install a Daemonset that provisions Vector; +- Deploy service and service monitor for integration with prometheus-operator (if enabled); ## Installing the Chart From 86fbdc8fa1ff10bd9252f6554ca248372ca9e29b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:07:15 +0300 Subject: [PATCH 03/22] Correct formatting and typos at distribution/helm/vector/README.md Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 4942b4e1b6738..38dd981d0b5fa 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -2,8 +2,8 @@ [Vector](https://vector.dev/) - a lightweight and ultra-fast tool for building observability pipelines. -- [Chart Details](#chartdetails) -- [Installing the cart](#installingthechart) +- [Chart Details](#chart-details) +- [Installing the chart](#installing-the-chart) - [Configuration](#configuration) - [Examples](#examples) @@ -11,8 +11,8 @@ This chart will do the following: -- Create a ConfigMap with Vector configuration; -- Install a Daemonset that provisions Vector; +- Create a `ConfigMap` with Vector configuration; +- Install a `DaemonSet` that provisions Vector; - Deploy service and service monitor for integration with prometheus-operator (if enabled); ## Installing the Chart From 04b7c925305fb74b253ff04b5d9d0b5d7018b367 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:07:30 +0300 Subject: [PATCH 04/22] Use vector namespace Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 38dd981d0b5fa..9c58bbc1465d4 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -17,20 +17,20 @@ This chart will do the following: ## Installing the Chart -To install the chart with the release name `vector` and namespace `logging`: +To install the chart with the release name `vector` and namespace `vector`: ### Helm 2 ```bash $ cd distribution/helm/vector -$ helm install --namespace logging --name vector . +$ helm install --namespace vector --name vector . ``` ### Helm 3 ```bash $ cd distribution/helm/vector -$ helm install --namespace logging vector . +$ helm install --namespace vector vector . ``` ## Configuration From baf759641adcf1e233cb8b9d367586a3319b1d25 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:15:02 +0300 Subject: [PATCH 05/22] Adjust and update the configmap parameters Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 17 ++++++++--------- .../helm/vector/templates/configmap.yaml | 13 ++++++------- distribution/helm/vector/values.yaml | 7 +++---- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 9c58bbc1465d4..4da795e2326ac 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -71,15 +71,14 @@ The following table lists the configurable parameters of the Vector chart and th | `extraVolumeMounts` | Extra volume mounts for the vector container. | `{}` | | **ConfigMap** | | `existingConfigMap` | Name of the exising configmap to be used for Vector configuration. | `nil` | -| `globalOptions.dataDir` | Vector's data directory. | `/var/tmp/vector` | -| `logSchema.hostKey` | Log host field. | `host` | -| `logSchema.messageKey` | Log message field. | `log` | -| `logSchema.timestampKey` | Log generation timestamp field. | `time` | -| `sources.kubernetes.enabled` | Enables Kubernetes source. | `true` | -| `sources.kubernetes.sourceId` | Kubernetes source ID. | `kubernetes` | -| `sources.kubernetes.includeNamespaces` | A list of namespaces to collect logs from. | `[]` | -| `sources.kubernetes.includeContainerNames` | A list of container names to collect logs from. | `[]` | -| `sources.kubernetes.rawConfig` | Raw config to be used for the source. | `nil` | +| `globalOptions.dataDir` | Vector's data directory. | `/vector-data-dir` | +| `logSchema.hostKey` | The key used to hold the log host. | `host` | +| `logSchema.messageKey` | The key used to hold the log message. | `message` | +| `logSchema.sourceTypeKey` | The key used to hold the log source type. | `source_type` | +| `logSchema.timestampKey` | The key used to represent when the log was generated. | `timestamp` | +| `sources.kubernetesLogs.enabled` | Enables Kubernetes Logs source. | `true` | +| `sources.kubernetesLogs.sourceId` | Kubernetes source ID. | `kubernetes` | +| `sources.kubernetesLogs.rawConfig` | Raw config to be used for the source. | `nil` | | `sources.additionalSources` | An object of additional sources. Key will be used as source ID. | `{}` | | `sources.additionalSources.type` | Source type. | | | `sources.additionalSources.rawConfig` | Raw config to be used for the additional source. | `nil` | diff --git a/distribution/helm/vector/templates/configmap.yaml b/distribution/helm/vector/templates/configmap.yaml index 41dcf9ef86f97..438cb6df38744 100644 --- a/distribution/helm/vector/templates/configmap.yaml +++ b/distribution/helm/vector/templates/configmap.yaml @@ -13,17 +13,16 @@ data: [log_schema] host_key = "{{ .Values.logSchema.hostKey }}" message_key = "{{ .Values.logSchema.messageKey }}" + source_type_key = "{{ .Values.logSchema.sourceTypeKey }}" timestamp_key = "{{ .Values.logSchema.timestampKey }}" {{- end }} - {{- if .Values.sources.kubernetes.enabled }} - [sources.{{ .Values.sources.kubernetes.sourceId }}] - type = "kubernetes" - include_namespaces = {{ .Values.sources.kubernetes.includeNamespaces | toJson }} - include_container_names = {{ .Values.sources.kubernetes.includeContainerNames | toJson }} + {{- if .Values.sources.kubernetesLogs.enabled }} + [sources.{{ .Values.sources.kubernetesLogs.sourceId }}] + type = "kubernetes_logs" - {{- if not (empty .Values.sources.kubernetes.rawConfig) }} - {{- .Values.sources.kubernetes.rawConfig | nindent 6 }} + {{- if not (empty .Values.sources.kubernetesLogs.rawConfig) }} + {{- .Values.sources.kubernetesLogs.rawConfig | nindent 6 }} {{- end }} {{- end }} diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index d574915f00ca6..305a97e978a93 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -41,19 +41,18 @@ extraVolumeMounts: [] existingConfigMap: "" globalOptions: - dataDir: "/var/tmp/vector" + dataDir: "/vector-data-dir" logSchema: hostKey: "host" messageKey: "message" + sourceTypeKey: "source_type" timestampKey: "timestamp" sources: - kubernetes: + kubernetesLogs: enabled: true sourceId: kubernetes - includeNamespaces: [] - includeContainerNames: [] additionalSources: {} From 11d93b6fa287e29510554be52d68ff450a6ed222 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:16:31 +0300 Subject: [PATCH 06/22] Fix more typos and formatting issues Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 4da795e2326ac..0abf805b63bb2 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -64,13 +64,13 @@ The following table lists the configurable parameters of the Vector chart and th | `rbac.apiVersion` | Overrides K8S API version for PSP (by default it's determined using `Capabilities`). | `nil` | | `env` | A list of environment variables to be used for the DaemonSet. | `[]` | | `resources` | Pod resource requests & limits. | `{}` | -| `tolerations` | Optional daemonset tolerations. | `[]` | +| `tolerations` | Optional DaemonSet tolerations. | `[]` | | `nodeSelector` | Node labels for pod assignment. | `{}` | | `affinity` | Expressions for affinity. | `{}` | -| `extraVolumes` | Extra volumes to be assignet to pods. | `{}` | +| `extraVolumes` | Extra volumes to be assigned to pods. | `{}` | | `extraVolumeMounts` | Extra volume mounts for the vector container. | `{}` | | **ConfigMap** | -| `existingConfigMap` | Name of the exising configmap to be used for Vector configuration. | `nil` | +| `existingConfigMap` | Name of the existing ConfigMap to be used for Vector configuration. | `nil` | | `globalOptions.dataDir` | Vector's data directory. | `/vector-data-dir` | | `logSchema.hostKey` | The key used to hold the log host. | `host` | | `logSchema.messageKey` | The key used to hold the log message. | `message` | @@ -83,8 +83,8 @@ The following table lists the configurable parameters of the Vector chart and th | `sources.additionalSources.type` | Source type. | | | `sources.additionalSources.rawConfig` | Raw config to be used for the additional source. | `nil` | | `transforms` | An object of transforms. Key will be used as transform ID. | `{}` | -| `transforms.type` | Tranform type. | | -| `transforms.inputs` | A list of transfor data sources. | | +| `transforms.type` | Transform type. | | +| `transforms.inputs` | A list of transform data sources. | | | `transforms.rawConfig` | Raw config to be used for the transform. | `nil` | | `sinks` | An object of sinks. Entry key will be used as sink ID. | `{}` | | `sinks.type` | Sink type. | | From 3ff6108789f311353daea01fa1c3876e3e7416b6 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:21:57 +0300 Subject: [PATCH 07/22] Update the description and chart version at distribution/helm/vector/Chart.yaml Signed-off-by: MOZGIII --- distribution/helm/vector/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml index 351e356350ac1..fe3e6af784bda 100644 --- a/distribution/helm/vector/Chart.yaml +++ b/distribution/helm/vector/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v1 name: vector type: application icon: https://vector.dev/press/vector-icon.svg -description: A Helm chart to collect Kubernetes logs with vector -version: 0.0.1 +description: A Helm chart to collect Kubernetes logs with Vector +version: 0.1.0 appVersion: 0.8 home: https://vector.dev/ sources: From 233c7bb56247e698a939191e0779d4f63e502418 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:26:27 +0300 Subject: [PATCH 08/22] Remove metrics service and service monitor Apparently it's no longer available, or never was available in the first place. :( Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 11 ------- .../helm/vector/templates/daemonset.yaml | 4 --- .../helm/vector/templates/service.yaml | 23 -------------- .../helm/vector/templates/servicemonitor.yaml | 30 ------------------- distribution/helm/vector/values.yaml | 14 --------- 5 files changed, 82 deletions(-) delete mode 100644 distribution/helm/vector/templates/service.yaml delete mode 100644 distribution/helm/vector/templates/servicemonitor.yaml diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index 0abf805b63bb2..e7864979ded53 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -45,17 +45,6 @@ The following table lists the configurable parameters of the Vector chart and th | `image.pullPolicy` | Image pull policy | `Always` | | `image.pullSecrets` | Image pull secrets | `nil` | | `updateStrategy` | DaemonSet update strategy | `RollingUpdate` | -| `metrics.enabled` | Specifies whether a service for metrics should be exposed | `true` | -| `metrics.service.type` | Metrics service type | `ClusterIP` | -| `metrics.service.port` | Metrics service port | `2020` | -| `metrics.service.annotations` | Optional metrics service annotations | `nil` | -| `metrics.service.labels` | Additional labels for the metrics service definition, specified as a map. | `nil` | -| `metrics.serviceMonitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator. | `true` | -| `metrics.serviceMonitor.endpoint` | Endpoint to be used by Prometheus to scrape metrics. | `true` | -| `metrics.serviceMonitor.additionalLabels` | Additional labels that can be used for ServiceMonitor discovery by Prometheus. | `nil` | -| `metrics.serviceMonitor.namespace` | Optional namespace in which to create ServiceMonito (default: current namespace). | `nil` | -| `metrics.serviceMonitor.interval` | Scrape interval. If not set, the Prometheus default scrape interval is used. | `60s` | -| `metrics.serviceMonitor.scrapeTimeout` | Scrape timeout. If not set, the Prometheus default scrape timeout is used. | `10s` | | `rbac.enabled` | Specifies whether RBAC should be enabled. | `true` | | `rbac.apiVersion` | Overrides K8S API version for RBAC (by default it's determined using `Capabilities`) | `nil` | | `rbac.serviceAccount.name` | Overrides service acount name, if not provided and RBAC is disabled, `default` will be used. | `nil` | diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml index 29a0d661b8500..0088fc2f038d2 100644 --- a/distribution/helm/vector/templates/daemonset.yaml +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -26,10 +26,6 @@ spec: imagePullPolicy: "{{ .Values.image.pullPolicy }}" args: - "-w" - {{- if .Values.metrics.enabled }} - - "-m" - - "0.0.0.0:{{ .Values.metrics.service.port }}" - {{- end }} env: {{- toYaml .Values.env | nindent 12 }} resources: diff --git a/distribution/helm/vector/templates/service.yaml b/distribution/helm/vector/templates/service.yaml deleted file mode 100644 index 65fadd47f3d8f..0000000000000 --- a/distribution/helm/vector/templates/service.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.metrics.enabled }} -apiVersion: v1 -kind: Service -metadata: - {{- if .Values.metrics.service.annotations }} - annotations: - {{- toYaml .Values.metrics.service.annotations | nindent 4 }} - {{- end }} - name: {{ template "vector.fullname" . }}-metrics - labels: - {{- include "vector.labels" . | nindent 4 }} - {{- if .Values.metrics.service.additionalLabels }} - {{ toYaml .Values.metrics.service.additionalLabels }} - {{- end }} -spec: - type: {{ .Values.metrics.service.type}} - ports: - - port: {{ .Values.metrics.service.port }} - targetPort: metrics - name: metrics - selector: - {{- include "vector.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/distribution/helm/vector/templates/servicemonitor.yaml b/distribution/helm/vector/templates/servicemonitor.yaml deleted file mode 100644 index e0c837bf8fb2b..0000000000000 --- a/distribution/helm/vector/templates/servicemonitor.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "vector.fullname" . }} - {{- if .Values.metrics.serviceMonitor.namespace }} - namespace: {{ .Values.metrics.serviceMonitor.namespace }} - {{- end }} - labels: - {{- include "vector.labels" . | nindent 4 }} - {{- if .Values.metrics.serviceMonitor.additionalLabels }} - {{ toYaml .Values.metrics.serviceMonitor.additionalLabels }} - {{- end }} -spec: - endpoints: - - port: metrics - path: {{ .Values.metrics.serviceMonitor.endpoint }} - {{- if .Values.metrics.serviceMonitor.interval }} - interval: {{ .Values.metrics.serviceMonitor.interval }} - {{- end }} - {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} - {{- end }} - namespaceSelector: - matchNames: - - {{ .Release.Namespace }} - selector: - matchLabels: - {{- include "vector.selectorLabels" . | nindent 6 }} -{{- end }} diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index 305a97e978a93..99e016ae1c10c 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -5,20 +5,6 @@ image: updateStrategy: RollingUpdate -metrics: - enabled: false - service: - port: 2020 - type: ClusterIP - annotations: {} - additionalLabels: {} - serviceMonitor: - enabled: false - endpoint: / - additionalLabels: {} - interval: 60s - scrapeTimeout: 10s - rbac: enabled: true serviceAccount: From 1742ebaf6b8dc25e3243192b4090c7f94ec12037 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 03:26:52 +0300 Subject: [PATCH 09/22] Remove `-w` arg from vector Signed-off-by: MOZGIII --- distribution/helm/vector/templates/daemonset.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml index 0088fc2f038d2..813b5d964f431 100644 --- a/distribution/helm/vector/templates/daemonset.yaml +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -24,8 +24,6 @@ spec: - name: vector image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" - args: - - "-w" env: {{- toYaml .Values.env | nindent 12 }} resources: From 7be4df758814891211d340379b34e561337a077f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 04:20:47 +0300 Subject: [PATCH 10/22] Move PSP to a separate file Signed-off-by: MOZGIII --- distribution/helm/vector/templates/psp.yaml | 44 +++++++++++++++++++ distribution/helm/vector/templates/rbac.yaml | 45 -------------------- 2 files changed, 44 insertions(+), 45 deletions(-) create mode 100644 distribution/helm/vector/templates/psp.yaml diff --git a/distribution/helm/vector/templates/psp.yaml b/distribution/helm/vector/templates/psp.yaml new file mode 100644 index 0000000000000..cb521ff6ee9d2 --- /dev/null +++ b/distribution/helm/vector/templates/psp.yaml @@ -0,0 +1,44 @@ +{{- if .Values.rbac.psp.enabled }} +apiVersion: {{ default (include "psp.apiVersion" .) .Values.rbac.psp.apiVersion }} +kind: PodSecurityPolicy +metadata: + name: {{ template "vector.fullname" . }} +spec: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + requiredDropCapabilities: + - ALL + volumes: + - 'hostPath' + - 'configMap' + - 'secret' + allowedHostPaths: + - pathPrefix: "/var/log" + readOnly: true + - pathPrefix: "/var/lib/docker/containers" + readOnly: true + {{- range .Values.extraVolumes }} + {{- if .hostPath }} + - pathPrefix: {{ .hostPath.path }} + {{- end }} + {{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 +{{- end }} +{{- end }} diff --git a/distribution/helm/vector/templates/rbac.yaml b/distribution/helm/vector/templates/rbac.yaml index 1ec86fe569add..1dae7419c3785 100644 --- a/distribution/helm/vector/templates/rbac.yaml +++ b/distribution/helm/vector/templates/rbac.yaml @@ -46,48 +46,3 @@ subjects: - kind: ServiceAccount name: {{ include "vector.serviceAccountName" . }} namespace: {{ .Release.Namespace }} ---- -{{- if .Values.rbac.psp.enabled }} -apiVersion: {{ default (include "psp.apiVersion" .) .Values.rbac.psp.apiVersion }} -kind: PodSecurityPolicy -metadata: - name: {{ template "vector.fullname" . }} -spec: - privileged: false - allowPrivilegeEscalation: false - readOnlyRootFilesystem: false - requiredDropCapabilities: - - ALL - volumes: - - 'hostPath' - - 'configMap' - - 'secret' - allowedHostPaths: - - pathPrefix: "/var/log" - readOnly: true - - pathPrefix: "/var/lib/docker/containers" - readOnly: true - {{- range .Values.extraVolumes }} - {{- if .hostPath }} - - pathPrefix: {{ .hostPath.path }} - {{- end }} - {{- end }} - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'MustRunAsNonRoot' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 - fsGroup: - rule: 'MustRunAs' - ranges: - - min: 1 - max: 65535 -{{- end }} -{{- end }} From da5e3212d7cd4f05168d198590cd83b59dc342cd Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 05:57:53 +0300 Subject: [PATCH 11/22] Major audit and rewrite of the Helm chart Signed-off-by: MOZGIII --- distribution/helm/vector/Chart.yaml | 4 +- distribution/helm/vector/README.md | 2 +- .../helm/vector/templates/_helpers.tpl | 84 +++++++------------ .../helm/vector/templates/configmap.yaml | 33 ++++---- .../helm/vector/templates/daemonset.yaml | 74 +++++++++++----- .../{psp.yaml => podsecuritypolicy.yaml} | 7 +- distribution/helm/vector/templates/rbac.yaml | 30 +++---- .../helm/vector/templates/serviceaccount.yaml | 12 +++ distribution/helm/vector/values.yaml | 73 ++++++++++++---- 9 files changed, 183 insertions(+), 136 deletions(-) rename distribution/helm/vector/templates/{psp.yaml => podsecuritypolicy.yaml} (82%) create mode 100644 distribution/helm/vector/templates/serviceaccount.yaml diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml index fe3e6af784bda..f9a87ef95e8ce 100644 --- a/distribution/helm/vector/Chart.yaml +++ b/distribution/helm/vector/Chart.yaml @@ -3,8 +3,8 @@ name: vector type: application icon: https://vector.dev/press/vector-icon.svg description: A Helm chart to collect Kubernetes logs with Vector -version: 0.1.0 -appVersion: 0.8 +version: "0.1.0" +appVersion: "0.10" home: https://vector.dev/ sources: - https://github.com/timberio/vector/ diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md index e7864979ded53..d9fdc613c7389 100644 --- a/distribution/helm/vector/README.md +++ b/distribution/helm/vector/README.md @@ -30,7 +30,7 @@ $ helm install --namespace vector --name vector . ```bash $ cd distribution/helm/vector -$ helm install --namespace vector vector . +$ helm install --create-namespace --namespace vector vector . ``` ## Configuration diff --git a/distribution/helm/vector/templates/_helpers.tpl b/distribution/helm/vector/templates/_helpers.tpl index 166428f6ad765..7f0b50af8b879 100644 --- a/distribution/helm/vector/templates/_helpers.tpl +++ b/distribution/helm/vector/templates/_helpers.tpl @@ -3,8 +3,8 @@ Expand the name of the chart. */}} {{- define "vector.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* Create a default fully qualified app name. @@ -12,24 +12,24 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "vector.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} {{/* Create chart name and version as used by the chart label. */}} {{- define "vector.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* Common labels @@ -41,7 +41,7 @@ helm.sh/chart: {{ include "vector.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} +{{- end }} {{/* Selector labels @@ -49,50 +49,26 @@ Selector labels {{- define "vector.selectorLabels" -}} app.kubernetes.io/name: {{ include "vector.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} +{{- end }} {{/* Create the name of the service account to use */}} {{- define "vector.serviceAccountName" -}} -{{- if .Values.rbac.enabled -}} - {{ default (include "vector.fullname" .) .Values.rbac.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.rbac.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Define apiVersion for RBAC APIs, depending on availability. -*/}} -{{- define "rbac.apiVersion" -}} -{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" -}} -rbac.authorization.k8s.io/v1 -{{- else if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1beta1" -}} -rbac.authorization.k8s.io/v1beta1 -{{- else -}} -rbac.authorization.k8s.io/v1alpha1 -{{- end -}} -{{- end -}} - -{{/* -Define apiVersion for PSP APIs, depending on availability. -*/}} -{{- define "psp.apiVersion" -}} -{{- if .Capabilities.APIVersions.Has "policy/v1" -}} -policy/v1 -{{- else if .Capabilities.APIVersions.Has "policy/v1beta1" -}} -policy/v1beta1 -{{- end -}} -{{- end -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "vector.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} {{/* -Define the name of the configmap +Define the name of the ConfigMap */}} {{- define "vector.configMapName" -}} -{{- if (empty .Values.existingConfigMap) -}} -{{- include "vector.fullname" . -}} -{{- else -}} -{{- .Values.existingConfigMap -}} -{{- end -}} -{{- end -}} +{{- if (empty .Values.existingConfigMap) }} +{{- include "vector.fullname" . }} +{{- else }} +{{- .Values.existingConfigMap }} +{{- end }} +{{- end }} diff --git a/distribution/helm/vector/templates/configmap.yaml b/distribution/helm/vector/templates/configmap.yaml index 438cb6df38744..ecc145ee24e0f 100644 --- a/distribution/helm/vector/templates/configmap.yaml +++ b/distribution/helm/vector/templates/configmap.yaml @@ -9,29 +9,29 @@ data: vector.toml: | data_dir = "{{ .Values.globalOptions.dataDir }}" - {{- if .Values.logSchema }} + {{- with .Values.logSchema }} [log_schema] - host_key = "{{ .Values.logSchema.hostKey }}" - message_key = "{{ .Values.logSchema.messageKey }}" - source_type_key = "{{ .Values.logSchema.sourceTypeKey }}" - timestamp_key = "{{ .Values.logSchema.timestampKey }}" + host_key = "{{ .hostKey }}" + message_key = "{{ .messageKey }}" + source_type_key = "{{ .sourceTypeKey }}" + timestamp_key = "{{ .timestampKey }}" {{- end }} - {{- if .Values.sources.kubernetesLogs.enabled }} - [sources.{{ .Values.sources.kubernetesLogs.sourceId }}] + {{- if .Values.kubernetesLogsSource.enabled }} + [sources.{{ .Values.kubernetesLogsSource.sourceId }}] type = "kubernetes_logs" - {{- if not (empty .Values.sources.kubernetesLogs.rawConfig) }} - {{- .Values.sources.kubernetesLogs.rawConfig | nindent 6 }} + {{- with .Values.kubernetesLogsSource.rawConfig }} + {{- . | nindent 6 }} {{- end }} {{- end }} - {{- range $sourceId, $source := .Values.sources.additionalSources }} + {{- range $sourceId, $source := .Values.sources }} [sources.{{ $sourceId }}] type = "{{ $source.type }}" - {{- if not (empty $source.rawConfig )}} - {{- $source.rawConfig | nindent 6}} + {{- with $source.rawConfig }} + {{- . | nindent 6 }} {{- end }} {{- end }} @@ -40,8 +40,8 @@ data: type = "{{ $transform.type }}" inputs = {{ $transform.inputs | toJson }} - {{- if not (empty $transform.rawConfig )}} - {{- $transform.rawConfig | nindent 6}} + {{- with $transform.rawConfig }} + {{- . | nindent 6 }} {{- end }} {{- end }} @@ -50,10 +50,9 @@ data: type = "{{ $sink.type }}" inputs = {{ $sink.inputs | toJson }} - {{- if not (empty $sink.rawConfig )}} - {{- $sink.rawConfig | nindent 6}} + {{- with $sink.rawConfig }} + {{- . | nindent 6 }} {{- end }} {{- end }} - {{- end }} diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml index 813b5d964f431..e4f01d0f92d19 100644 --- a/distribution/helm/vector/templates/daemonset.yaml +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ template "vector.fullname" . }} + name: {{ include "vector.fullname" . }} labels: {{- include "vector.labels" . | nindent 4 }} spec: @@ -13,66 +13,96 @@ spec: type: {{ .Values.updateStrategy }} template: metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} labels: {{- include "vector.selectorLabels" . | nindent 8 }} + vector.dev/exclude: "true" spec: - {{- if .Values.image.pullSecrets }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ include "vector.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: vector - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" env: - {{- toYaml .Values.env | nindent 12 }} + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- if or .Values.metrics.enabled }} - ports: - - name: metrics - containerPort: {{ .Values.metrics.service.port }} - protocol: TCP - {{- end }} volumeMounts: + # Host log directory mount. - name: var-log mountPath: /var/log/ readOnly: true + # Host mount for docker and containerd log file symlinks. - name: var-lib mountPath: /var/lib readOnly: true + # Vector data dir mount. - name: data-dir mountPath: "{{ .Values.globalOptions.dataDir }}" + # Vector config dir mount. - name: config-dir mountPath: /etc/vector readOnly: true - {{- if .Values.extraVolumeMounts }} - {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + # Extra volumes. + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.nodeSelector }} + terminationGracePeriodSeconds: 60 + {{- with .Values.nodeSelector }} nodeSelector: - {{ toYaml .Values.nodeSelector | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.affinity }} + {{- with .Values.affinity }} affinity: - {{ toYaml .Values.affinity | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.tolerations }} + {{- with .Values.tolerations }} tolerations: - {{ toYaml .Values.tolerations | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} volumes: + # Log directory. - name: var-log hostPath: path: /var/log/ + # Docker and containerd log files in Kubernetes are symlinks to this folder. - name: var-lib hostPath: path: /var/lib/ + # Vector will store it's data here. - name: data-dir - emptyDir: {} + hostPath: + path: /var/lib/vector/ + # Vector config dir with a managed config map. - name: config-dir configMap: name: {{ include "vector.configMapName" . }} - {{- if .Values.extraVolumes }} - {{- toYaml .Values.extraVolumes | nindent 8 }} + # Extra volumes. + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} {{- end }} diff --git a/distribution/helm/vector/templates/psp.yaml b/distribution/helm/vector/templates/podsecuritypolicy.yaml similarity index 82% rename from distribution/helm/vector/templates/psp.yaml rename to distribution/helm/vector/templates/podsecuritypolicy.yaml index cb521ff6ee9d2..a865d4a643ff6 100644 --- a/distribution/helm/vector/templates/psp.yaml +++ b/distribution/helm/vector/templates/podsecuritypolicy.yaml @@ -1,8 +1,8 @@ -{{- if .Values.rbac.psp.enabled }} -apiVersion: {{ default (include "psp.apiVersion" .) .Values.rbac.psp.apiVersion }} +{{- if .Values.psp.enabled }} +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: - name: {{ template "vector.fullname" . }} + name: {{ include "vector.fullname" . }} spec: privileged: false allowPrivilegeEscalation: false @@ -41,4 +41,3 @@ spec: - min: 1 max: 65535 {{- end }} -{{- end }} diff --git a/distribution/helm/vector/templates/rbac.yaml b/distribution/helm/vector/templates/rbac.yaml index 1dae7419c3785..6524384767bc3 100644 --- a/distribution/helm/vector/templates/rbac.yaml +++ b/distribution/helm/vector/templates/rbac.yaml @@ -1,48 +1,38 @@ {{- if .Values.rbac.enabled -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "vector.serviceAccountName" . }} - labels: - {{- include "vector.labels" . | nindent 4 }} - {{- with .Values.rbac.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} ---- -apiVersion: {{ default (include "rbac.apiVersion" .) .Values.rbac.apiVersion }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ template "vector.fullname" . }} + name: {{ include "vector.fullname" . }} rules: - apiGroups: - "" resources: - pods verbs: - - get -{{- if .Values.rbac.psp.enabled }} + - watch +{{- if .Values.psp.enabled }} - apiGroups: - policy resources: - podsecuritypolicies - resourceNames: - - {{ template "vector.fullname" . }} verbs: - use + resourceNames: + - {{ include "vector.fullname" . }} {{- end }} --- -apiVersion: {{ default (include "rbac.apiVersion" .) .Values.rbac.apiVersion }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ template "vector.fullname" . }} + name: {{ include "vector.fullname" . }} labels: {{- include "vector.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ template "vector.fullname" . }} + name: {{ include "vector.fullname" . }} subjects: - kind: ServiceAccount name: {{ include "vector.serviceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/distribution/helm/vector/templates/serviceaccount.yaml b/distribution/helm/vector/templates/serviceaccount.yaml new file mode 100644 index 0000000000000..f81e6011bd111 --- /dev/null +++ b/distribution/helm/vector/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "vector.serviceAccountName" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index 99e016ae1c10c..f30f6f40f95e4 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -1,29 +1,58 @@ +# Default values for vector. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + image: repository: timberio/vector - tag: 0.8.X-debian - pullPolicy: Always + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" updateStrategy: RollingUpdate -rbac: - enabled: true - serviceAccount: - annotation: {} - name: - psp: - enabled: false +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 env: [] -tolerations: [] +resources: {} nodeSelector: {} +tolerations: [] affinity: {} -resources: {} extraVolumes: [] extraVolumeMounts: [] +rbac: + enabled: true + +psp: + enabled: false + existingConfigMap: "" globalOptions: @@ -35,13 +64,25 @@ logSchema: sourceTypeKey: "source_type" timestampKey: "timestamp" -sources: - kubernetesLogs: - enabled: true - sourceId: kubernetes +# The built-in kubernetes logs source. +kubernetesLogsSource: + enabled: true + sourceId: kubernetes_logs + rawConfig: null - additionalSources: {} +sources: {} + # type: "source_type" + # rawConfig: | + # option = "value" transforms: {} + # type: "transform_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" sinks: {} + # type: "sink_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" From d7e12bc7128dc20fbe104ccc5ed3007b49e72688 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 17:55:12 +0300 Subject: [PATCH 12/22] Reset version and appVersion at the chart manifest and hinted to overriding Signed-off-by: MOZGIII --- distribution/helm/vector/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml index f9a87ef95e8ce..f49c4624fe912 100644 --- a/distribution/helm/vector/Chart.yaml +++ b/distribution/helm/vector/Chart.yaml @@ -3,8 +3,8 @@ name: vector type: application icon: https://vector.dev/press/vector-icon.svg description: A Helm chart to collect Kubernetes logs with Vector -version: "0.1.0" -appVersion: "0.10" +version: "0.0.0" # overwritten via --version on publishing +appVersion: "0.0.0" # overwritten via --app-version on publishing home: https://vector.dev/ sources: - https://github.com/timberio/vector/ From d6c8e279f00b80a306a2a20e0b0ffc77df28a2b3 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 20:31:36 +0300 Subject: [PATCH 13/22] Add a script for publishing Helm charts Signed-off-by: MOZGIII --- Makefile | 3 +++ scripts/release-helm.sh | 50 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100755 scripts/release-helm.sh diff --git a/Makefile b/Makefile index 91755402515f0..277963b8f1a77 100644 --- a/Makefile +++ b/Makefile @@ -456,6 +456,9 @@ release-rollback: ## Rollback pending release changes release-s3: ## Release artifacts to S3 @scripts/release-s3.sh +release-helm: ## Package and release Helm Chart + @scripts/release-helm.sh + sync-install: ## Sync the install.sh script for access via sh.vector.dev @aws s3 cp distribution/install.sh s3://sh.vector.dev --sse --acl public-read diff --git a/scripts/release-helm.sh b/scripts/release-helm.sh new file mode 100755 index 0000000000000..8f8cfb3ef1c5e --- /dev/null +++ b/scripts/release-helm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +# release-helm.sh +# +# SUMMARY +# +# Package Helm Chart and update the Helm repo. + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -x + +CHANNEL="${CHANNEL:-"$(scripts/util/release-channel.sh)"}" +VERSION="${VERSION:-"$(scripts/version.sh)"}" + +if [[ "${USE_TEST_REPO:-"false"}" == "true" ]]; then + PUBLIC_URL="https://vector-helm-repo-tests.s3.amazonaws.com/helm/$CHANNEL" + AWS_REPO_URL="s3://vector-helm-repo-tests/helm/$CHANNEL" +else + PUBLIC_URL="https://packages.timber.io/helm/$CHANNEL" + AWS_REPO_URL="s3://packages.timber.io/helm/$CHANNEL" +fi + +WORKDIR="target/helm" + +REPO_DIR="$WORKDIR/repo" +PREVIOUS_MANIFEST="$WORKDIR/previous-manifest.yaml" + +# Prepare work directory. +rm -rf "$REPO_DIR" +mkdir -p "$REPO_DIR" + +# Package our chart. +helm package \ + distribution/helm/vector \ + --version "$VERSION" \ + --app-version "$VERSION" \ + --destination "$REPO_DIR" + +# Download previous manifest. +aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST" + +# Update the repo index file. +helm repo index \ + "$REPO_DIR" \ + --merge "$PREVIOUS_MANIFEST" \ + --url "$PUBLIC_URL" + +# Upload new files to the repo. +aws s3 cp "$REPO_DIR" "$AWS_REPO_URL" --recursive --sse --acl public-read From ead7883647a0472e444d85b99192797274a3f510 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 20:53:20 +0300 Subject: [PATCH 14/22] Allow initializing empty repo Signed-off-by: MOZGIII --- scripts/release-helm.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/release-helm.sh b/scripts/release-helm.sh index 8f8cfb3ef1c5e..cfb64e728dc9c 100755 --- a/scripts/release-helm.sh +++ b/scripts/release-helm.sh @@ -26,6 +26,10 @@ WORKDIR="target/helm" REPO_DIR="$WORKDIR/repo" PREVIOUS_MANIFEST="$WORKDIR/previous-manifest.yaml" +cature_stderr() { + { OUTPUT=$("$@" 2>&1 1>&3-); } 3>&1 +} + # Prepare work directory. rm -rf "$REPO_DIR" mkdir -p "$REPO_DIR" @@ -38,7 +42,19 @@ helm package \ --destination "$REPO_DIR" # Download previous manifest. -aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST" +# If it doesn't exist - ignore the error and continue. +if ! cature_stderr aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST"; then + EXPECTED="^fatal error:" + EXPECTED="$EXPECTED An error occurred \(404\) when calling the HeadObject operation:" + EXPECTED="$EXPECTED Key \".*/index\.yaml\" does not exist$" + if ! grep -Eq "$EXPECTED" <<<"$OUTPUT"; then + echo "$OUTPUT" >&2 + exit 1 + else + echo "Warning: repo index file doesn't exist, but we ignore the error" \ + "because we will initialize it" + fi +fi # Update the repo index file. helm repo index \ From 4335d9ce5421276dbabca8d3ec1e1be68d0965ac Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 21:15:51 +0300 Subject: [PATCH 15/22] Add release-helm CI job for nightly releases Signed-off-by: MOZGIII --- .github/workflows/nightly.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c7b50e798d5c5..54e0f7519f240 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -276,3 +276,17 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" USE_CONTAINER: none run: make release-s3 + + release-helm: + runs-on: ubuntu-latest + needs: + # This is not strictly required, but ensures that Helm Chart doesn't + # appear before the image it refers to. + - release-docker + steps: + - uses: actions/checkout@v1 + - env: + AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" + USE_CONTAINER: none + run: make release-helm From 2c18da7323dde23286f3ff4b350a87b00f97c9a7 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 21:39:05 +0300 Subject: [PATCH 16/22] Proper version resolution Signed-off-by: MOZGIII --- distribution/helm/vector/templates/_helpers.tpl | 12 ++++++++++++ distribution/helm/vector/templates/daemonset.yaml | 2 +- distribution/helm/vector/values.yaml | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/distribution/helm/vector/templates/_helpers.tpl b/distribution/helm/vector/templates/_helpers.tpl index 7f0b50af8b879..171e6f39384c9 100644 --- a/distribution/helm/vector/templates/_helpers.tpl +++ b/distribution/helm/vector/templates/_helpers.tpl @@ -72,3 +72,15 @@ Define the name of the ConfigMap {{- .Values.existingConfigMap }} {{- end }} {{- end }} + +{{/* +Resolve the actual image tag to use. +*/}} +{{- define "vector.imageTag" -}} +{{- if .Values.image.tag }} +{{- .Values.image.tag }} +{{- else }} +{{- $version := default .Chart.AppVersion .Values.image.version }} +{{- printf "%s-%s" $version .Values.image.base }} +{{- end }} +{{- end }} diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml index e4f01d0f92d19..96dc50d7a212b 100644 --- a/distribution/helm/vector/templates/daemonset.yaml +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -32,7 +32,7 @@ spec: - name: vector securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}:{{ include "vector.imageTag" . }}" imagePullPolicy: "{{ .Values.image.pullPolicy }}" env: - name: VECTOR_SELF_NODE_NAME diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index f30f6f40f95e4..f01cd7fae19ca 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -7,6 +7,8 @@ image: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" + version: "" + base: "debian" imagePullSecrets: [] nameOverride: "" From c8fd7b0c45146f7db3ccdbae9004713fc907e845 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 21:46:35 +0300 Subject: [PATCH 17/22] Proper support for nightly versions Signed-off-by: MOZGIII --- scripts/release-helm.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/release-helm.sh b/scripts/release-helm.sh index cfb64e728dc9c..5bc4a1f9a03a5 100755 --- a/scripts/release-helm.sh +++ b/scripts/release-helm.sh @@ -13,6 +13,15 @@ set -x CHANNEL="${CHANNEL:-"$(scripts/util/release-channel.sh)"}" VERSION="${VERSION:-"$(scripts/version.sh)"}" +if [[ "$CHANNEL" == "nightly" ]]; then + DATE="${DATE:-"$(date -u +%Y-%m-%d)"}" + APP_VERSION="nightly-$DATE" # matches the version part of the image tag + CHART_VERSION="$VERSION-$DATE" +else + APP_VERSION="$VERSION" # matches the version part of the image tag + CHART_VERSION="$VERSION" +fi + if [[ "${USE_TEST_REPO:-"false"}" == "true" ]]; then PUBLIC_URL="https://vector-helm-repo-tests.s3.amazonaws.com/helm/$CHANNEL" AWS_REPO_URL="s3://vector-helm-repo-tests/helm/$CHANNEL" @@ -37,8 +46,8 @@ mkdir -p "$REPO_DIR" # Package our chart. helm package \ distribution/helm/vector \ - --version "$VERSION" \ - --app-version "$VERSION" \ + --version "$CHART_VERSION" \ + --app-version "$APP_VERSION" \ --destination "$REPO_DIR" # Download previous manifest. From bdb9155fc81c897ac1630a20951c4183d5e6eec4 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 9 Jul 2020 16:19:58 +0300 Subject: [PATCH 18/22] Fix comments at distribution/helm/vector/values.yaml Signed-off-by: MOZGIII --- distribution/helm/vector/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index f01cd7fae19ca..2d201a04a6faa 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -5,8 +5,9 @@ image: repository: timberio/vector pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. + # Overrides the image tag, the default is `{image.version}-{image.base}`. tag: "" + # Overrides the image version, the default is the Chart appVersion. version: "" base: "debian" From 9151f60c900c7be11c6fcb2bb501b3867fe575e0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 9 Jul 2020 16:20:36 +0300 Subject: [PATCH 19/22] Remove outdated README.md altogether Signed-off-by: MOZGIII --- distribution/helm/vector/README.md | 146 ----------------------------- 1 file changed, 146 deletions(-) delete mode 100644 distribution/helm/vector/README.md diff --git a/distribution/helm/vector/README.md b/distribution/helm/vector/README.md deleted file mode 100644 index d9fdc613c7389..0000000000000 --- a/distribution/helm/vector/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# Vector Chart - -[Vector](https://vector.dev/) - a lightweight and ultra-fast tool for building observability pipelines. - -- [Chart Details](#chart-details) -- [Installing the chart](#installing-the-chart) -- [Configuration](#configuration) -- [Examples](#examples) - -## Chart Details - -This chart will do the following: - -- Create a `ConfigMap` with Vector configuration; -- Install a `DaemonSet` that provisions Vector; -- Deploy service and service monitor for integration with prometheus-operator (if enabled); - -## Installing the Chart - -To install the chart with the release name `vector` and namespace `vector`: - -### Helm 2 - -```bash -$ cd distribution/helm/vector -$ helm install --namespace vector --name vector . -``` - -### Helm 3 - -```bash -$ cd distribution/helm/vector -$ helm install --create-namespace --namespace vector vector . -``` - -## Configuration - -The following table lists the configurable parameters of the Vector chart and the default values. - -| Parameter | Description | Default | -| ----------------------------- | ---------------------------------- | ----------------------- | -| **General** | -| `image.repository` | Image | `timberio/vector` | -| `image.tag` | Image tag | `0.8.X-debian` | -| `image.pullPolicy` | Image pull policy | `Always` | -| `image.pullSecrets` | Image pull secrets | `nil` | -| `updateStrategy` | DaemonSet update strategy | `RollingUpdate` | -| `rbac.enabled` | Specifies whether RBAC should be enabled. | `true` | -| `rbac.apiVersion` | Overrides K8S API version for RBAC (by default it's determined using `Capabilities`) | `nil` | -| `rbac.serviceAccount.name` | Overrides service acount name, if not provided and RBAC is disabled, `default` will be used. | `nil` | -| `rbac.serviceAccount.annotations` | Additional annotation for the created service account. | `{}` | -| `rbac.psp.enabled` | Specifies whether a PodSecurityPolicy should be created. | `false` | -| `rbac.apiVersion` | Overrides K8S API version for PSP (by default it's determined using `Capabilities`). | `nil` | -| `env` | A list of environment variables to be used for the DaemonSet. | `[]` | -| `resources` | Pod resource requests & limits. | `{}` | -| `tolerations` | Optional DaemonSet tolerations. | `[]` | -| `nodeSelector` | Node labels for pod assignment. | `{}` | -| `affinity` | Expressions for affinity. | `{}` | -| `extraVolumes` | Extra volumes to be assigned to pods. | `{}` | -| `extraVolumeMounts` | Extra volume mounts for the vector container. | `{}` | -| **ConfigMap** | -| `existingConfigMap` | Name of the existing ConfigMap to be used for Vector configuration. | `nil` | -| `globalOptions.dataDir` | Vector's data directory. | `/vector-data-dir` | -| `logSchema.hostKey` | The key used to hold the log host. | `host` | -| `logSchema.messageKey` | The key used to hold the log message. | `message` | -| `logSchema.sourceTypeKey` | The key used to hold the log source type. | `source_type` | -| `logSchema.timestampKey` | The key used to represent when the log was generated. | `timestamp` | -| `sources.kubernetesLogs.enabled` | Enables Kubernetes Logs source. | `true` | -| `sources.kubernetesLogs.sourceId` | Kubernetes source ID. | `kubernetes` | -| `sources.kubernetesLogs.rawConfig` | Raw config to be used for the source. | `nil` | -| `sources.additionalSources` | An object of additional sources. Key will be used as source ID. | `{}` | -| `sources.additionalSources.type` | Source type. | | -| `sources.additionalSources.rawConfig` | Raw config to be used for the additional source. | `nil` | -| `transforms` | An object of transforms. Key will be used as transform ID. | `{}` | -| `transforms.type` | Transform type. | | -| `transforms.inputs` | A list of transform data sources. | | -| `transforms.rawConfig` | Raw config to be used for the transform. | `nil` | -| `sinks` | An object of sinks. Entry key will be used as sink ID. | `{}` | -| `sinks.type` | Sink type. | | -| `sinks.inputs` | A list of sink data sources. | | -| `sinks.rawConfig` | Raw config to be used for the sink. | `nil` | - - -## Examples - -Here are some snippets for Vector configuration. - -### Overriding Kubernetes source namespaces - -Whitelisted namespaces (or other options) can be set for the `kubernetes` source, -just by overriding `sources.kubernetes` options. - -```yaml -sources: - kubernetes: - includeNamespaces: - - default -``` - -### Using transforms - -```yaml -transforms: - envTagging: - type: add_fields - inputs: - - kubernetes - rawConfig: | - [transforms.envTagging.fields] - env = "dev" - - jsonParser: - type: json_parser - inputs: - - envTagging - rawConfig: | - field = "message" - drop_field = true - drop_invalid = false -``` - -### Using Splunk sink - -In this example the Splunk HEC sink uses token from env variable (referenced from K8S secret). - -```yaml -env: - - name: SPLUNK_TOKEN - valueFrom: - secretKeyRef: - name: vector-credentials - key: SPLUNK_TOKEN - -sinks: - splunk: - type: splunk_hec - inputs: - - jsonParser - rawConfig: | - host = "https://splunk-endpoint.net" - token = "${SPLUNK_TOKEN}" - encoding = "json" - healthcheck = true - indexed_fields = ["env", "pod_name", "pod_namespace", "container_name"] - -``` From a07bd8b369f1ac1420492742de547e2a1e136ce5 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 22 Jul 2020 20:04:15 +0300 Subject: [PATCH 20/22] Add more documentation to the values.yaml Signed-off-by: MOZGIII --- distribution/helm/vector/values.yaml | 63 +++++++++++++++++++++------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml index 2d201a04a6faa..ed17910df0733 100644 --- a/distribution/helm/vector/values.yaml +++ b/distribution/helm/vector/values.yaml @@ -11,26 +11,36 @@ image: version: "" base: "debian" +# Image pull secrets to use at the `Pod`s managed by `DaemonSet`. imagePullSecrets: [] + +# Override the chart name used in templates. nameOverride: "" +# Override the full chart name (name prefixed with release name) used in +# templates. fullnameOverride: "" updateStrategy: RollingUpdate serviceAccount: - # Specifies whether a service account should be created + # Specifies whether a service account should be created. create: true - # Annotations to add to the service account + # Annotations to add to the service account. annotations: {} # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and `create` is true, a name is generated using the `fullname` + # template. name: "" +# Annotations to add to the `Pod`s managed by `DaemonSet`. podAnnotations: {} +# PodSecurityContext to set at the `Pod`s managed by `DaemonSet`. podSecurityContext: {} # fsGroup: 2000 +# Security context to set at the `vector` container at the `Pod`s managed by +# `DaemonSet`. securityContext: {} # capabilities: # drop: @@ -39,53 +49,76 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 +# Extra env vars to pass to the `vector` container. env: [] +# Various tweakables for the `Pod`s managed by `DaemonSet`. resources: {} nodeSelector: {} tolerations: [] affinity: {} +# Additional volmes to pass to the `Pod`s managed by `DaemonSet`. extraVolumes: [] +# Additional volme mounts to pass to the `vector` container of the `Pod`s +# managed by `DaemonSet`. extraVolumeMounts: [] rbac: + # Whether to create rbac resources or not. Disable for non-rbac clusters. enabled: true psp: + # Whether to create `PodSecurityPolicy` or not. enabled: false +# Set this to non-empty value to use existing `ConfigMap` for `vector`, instead +# of using a generated one. existingConfigMap: "" +# Global parts of the generated `vector` config. globalOptions: + # Specifies the (in-container) data dir used by `vector`. dataDir: "/vector-data-dir" +# Schema part of the generated `vector` config. logSchema: hostKey: "host" messageKey: "message" sourceTypeKey: "source_type" timestampKey: "timestamp" -# The built-in kubernetes logs source. +# The "built-in" kubernetes logs source. Will be added by default, unless +# explicitly disabled. kubernetesLogsSource: + # Disable to omit the kubernetes logs source from being added. enabled: true + # The name to use for the "built-in" kubernetes logs source. sourceId: kubernetes_logs + # Raw TOML config to embed at the kubernetes logs source. rawConfig: null +# Sources to add to the generated `vector` config (besides "built-in" kubernetes +# logs source). sources: {} - # type: "source_type" - # rawConfig: | - # option = "value" + # source_name: + # type: "source_type" + # rawConfig: | + # option = "value" +# Transforms to add to the generated `vector` config. transforms: {} - # type: "transform_type" - # inputs: ["input1", "input2"] - # rawConfig: | - # option = "value" + # transform_name: + # type: "transform_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" +# Sinks to add to the generated `vector` config. sinks: {} - # type: "sink_type" - # inputs: ["input1", "input2"] - # rawConfig: | - # option = "value" + # sink_name: + # type: "sink_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" From c056440109c0b8dab8ced36881bdea5bfcae74bf Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 30 Jul 2020 04:02:56 +0300 Subject: [PATCH 21/22] Fix a typo at helm release script Signed-off-by: MOZGIII --- scripts/release-helm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release-helm.sh b/scripts/release-helm.sh index 5bc4a1f9a03a5..7a8f37bde624a 100755 --- a/scripts/release-helm.sh +++ b/scripts/release-helm.sh @@ -35,7 +35,7 @@ WORKDIR="target/helm" REPO_DIR="$WORKDIR/repo" PREVIOUS_MANIFEST="$WORKDIR/previous-manifest.yaml" -cature_stderr() { +capture_stderr() { { OUTPUT=$("$@" 2>&1 1>&3-); } 3>&1 } @@ -52,7 +52,7 @@ helm package \ # Download previous manifest. # If it doesn't exist - ignore the error and continue. -if ! cature_stderr aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST"; then +if ! capture_stderr aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST"; then EXPECTED="^fatal error:" EXPECTED="$EXPECTED An error occurred \(404\) when calling the HeadObject operation:" EXPECTED="$EXPECTED Key \".*/index\.yaml\" does not exist$" From 63120aa15a786926e0efb88b00cd35b84fbfe786 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 4 Aug 2020 07:13:23 +0300 Subject: [PATCH 22/22] Install helm and kubectl at CI Signed-off-by: MOZGIII --- .github/workflows/nightly.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 54e0f7519f240..161094c95522a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -285,7 +285,22 @@ jobs: - release-docker steps: - uses: actions/checkout@v1 - - env: + - name: Install Helm + run: | + set -xeuo pipefail + + KUBERNETES_VERSION="v1.18.6" + HELM_VERSION="v3.2.4" + + curl -Lo kubectl \ + "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + sudo install kubectl /usr/local/bin/ && rm kubectl + + curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + | tar -xzv --strip-components=1 --occurrence linux-amd64/helm + sudo install helm /usr/local/bin/ && rm helm + - name: Release Helm Chart + env: AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" USE_CONTAINER: none