chore: initial Vector Helm Chart and Helm Chart release process

.github/workflows/nightly.yml

@@ -276,3 +276,32 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}"
           USE_CONTAINER: none
         run: make release-s3
+
+  release-helm:
+    runs-on: ubuntu-latest
+    needs:
+      # This is not strictly required, but ensures that Helm Chart doesn't
+      # appear before the image it refers to.
+      - release-docker
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Helm
+        run: |
+          set -xeuo pipefail
+
+          KUBERNETES_VERSION="v1.18.6"
+          HELM_VERSION="v3.2.4"
+
+          curl -Lo kubectl \
+            "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+          sudo install kubectl /usr/local/bin/ && rm kubectl
+
+          curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \
+            | tar -xzv --strip-components=1 --occurrence linux-amd64/helm
+          sudo install helm /usr/local/bin/ && rm helm
+      - name: Release Helm Chart
+        env:
+          AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}"
+          USE_CONTAINER: none
+        run: make release-helm

Makefile

@@ -456,6 +456,9 @@ release-rollback: ## Rollback pending release changes
 release-s3: ## Release artifacts to S3
 	@scripts/release-s3.sh
 
+release-helm: ## Package and release Helm Chart
+	@scripts/release-helm.sh
+
 sync-install: ## Sync the install.sh script for access via sh.vector.dev
 	@aws s3 cp distribution/install.sh s3://sh.vector.dev --sse --acl public-read
 

distribution/helm/vector/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

distribution/helm/vector/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+name: vector
+type: application
+icon: https://vector.dev/press/vector-icon.svg
+description: A Helm chart to collect Kubernetes logs with Vector
+version: "0.0.0" # overwritten via --version on publishing
+appVersion: "0.0.0" # overwritten via --app-version on publishing
+home: https://vector.dev/
+sources:
+  - https://github.com/timberio/vector/
+maintainers:
+  - name: Vector Contributors
+    email: vector@timber.io

distribution/helm/vector/templates/_helpers.tpl

@@ -0,0 +1,86 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "vector.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "vector.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "vector.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "vector.labels" -}}
+helm.sh/chart: {{ include "vector.chart" . }}
+{{ include "vector.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "vector.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "vector.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "vector.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "vector.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Define the name of the ConfigMap
+*/}}
+{{- define "vector.configMapName" -}}
+{{- if (empty .Values.existingConfigMap) }}
+{{- include "vector.fullname" . }}
+{{- else }}
+{{- .Values.existingConfigMap }}
+{{- end }}
+{{- end }}
+
+{{/*
+Resolve the actual image tag to use.
+*/}}
+{{- define "vector.imageTag" -}}
+{{- if .Values.image.tag }}
+{{- .Values.image.tag }}
+{{- else }}
+{{- $version := default .Chart.AppVersion .Values.image.version }}
+{{- printf "%s-%s" $version .Values.image.base }}
+{{- end }}
+{{- end }}

distribution/helm/vector/templates/configmap.yaml

@@ -0,0 +1,58 @@
+{{- if (empty .Values.existingConfigMap) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "vector.configMapName" . }}
+  labels:
+    {{- include "vector.labels" . | nindent 4 }}
+data:
+  vector.toml: |
+    data_dir = "{{ .Values.globalOptions.dataDir }}"
+
+    {{- with .Values.logSchema }}
+    [log_schema]
+      host_key = "{{ .hostKey }}"
+      message_key = "{{ .messageKey }}"
+      source_type_key = "{{ .sourceTypeKey }}"
+      timestamp_key = "{{ .timestampKey }}"
+    {{- end }}
+
+    {{- if .Values.kubernetesLogsSource.enabled }}
+    [sources.{{ .Values.kubernetesLogsSource.sourceId }}]
+      type = "kubernetes_logs"
+
+      {{- with .Values.kubernetesLogsSource.rawConfig }}
+      {{- . | nindent 6 }}
+      {{- end }}
+    {{- end }}
+
+    {{- range $sourceId, $source := .Values.sources }}
+    [sources.{{ $sourceId }}]
+      type = "{{ $source.type }}"
+
+      {{- with $source.rawConfig }}
+      {{- . | nindent 6 }}
+      {{- end }}
+    {{- end }}
+
+    {{- range $transformId, $transform := .Values.transforms }}
+    [transforms.{{ $transformId }}]
+      type = "{{ $transform.type }}"
+      inputs = {{ $transform.inputs | toJson }}
+
+      {{- with $transform.rawConfig }}
+      {{- . | nindent 6 }}
+      {{- end }}
+    {{- end }}
+
+    {{- range $sinkId, $sink := .Values.sinks }}
+    [sinks.{{ $sinkId }}]
+      type = "{{ $sink.type }}"
+      inputs = {{ $sink.inputs | toJson }}
+
+      {{- with $sink.rawConfig }}
+      {{- . | nindent 6 }}
+      {{- end }}
+    {{- end }}
+
+{{- end }}

distribution/helm/vector/templates/daemonset.yaml

@@ -0,0 +1,108 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "vector.fullname" . }}
+  labels:
+    {{- include "vector.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "vector.selectorLabels" . | nindent 6 }}
+  minReadySeconds: 1
+  updateStrategy:
+    type: {{ .Values.updateStrategy }}
+  template:
+    metadata:
+    {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      labels:
+        {{- include "vector.selectorLabels" . | nindent 8 }}
+        vector.dev/exclude: "true"
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "vector.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: vector
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ include "vector.imageTag" . }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          env:
+            - name: VECTOR_SELF_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: VECTOR_SELF_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: VECTOR_SELF_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- with .Values.env }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            # Host log directory mount.
+            - name: var-log
+              mountPath: /var/log/
+              readOnly: true
+            # Host mount for docker and containerd log file symlinks.
+            - name: var-lib
+              mountPath: /var/lib
+              readOnly: true
+            # Vector data dir mount.
+            - name: data-dir
+              mountPath: "{{ .Values.globalOptions.dataDir }}"
+            # Vector config dir mount.
+            - name: config-dir
+              mountPath: /etc/vector
+              readOnly: true
+            # Extra volumes.
+            {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      terminationGracePeriodSeconds: 60
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        # Log directory.
+        - name: var-log
+          hostPath:
+            path: /var/log/
+        # Docker and containerd log files in Kubernetes are symlinks to this folder.
+        - name: var-lib
+          hostPath:
+            path: /var/lib/
+        # Vector will store it's data here.
+        - name: data-dir
+          hostPath:
+            path: /var/lib/vector/
+        # Vector config dir with a managed config map.
+        - name: config-dir
+          configMap:
+            name: {{ include "vector.configMapName" . }}
+        # Extra volumes.
+        {{- with .Values.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}

distribution/helm/vector/templates/podsecuritypolicy.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.psp.enabled }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "vector.fullname" . }}
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  readOnlyRootFilesystem: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'hostPath'
+    - 'configMap'
+    - 'secret'
+  allowedHostPaths:
+    - pathPrefix: "/var/log"
+      readOnly: true
+    - pathPrefix: "/var/lib/docker/containers"
+      readOnly: true
+    {{- range .Values.extraVolumes }}
+    {{- if .hostPath }}
+    - pathPrefix: {{ .hostPath.path }}
+    {{- end }}
+    {{- end }}
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+{{- end }}