diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c7b50e798d5c5..161094c95522a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -276,3 +276,32 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" USE_CONTAINER: none run: make release-s3 + + release-helm: + runs-on: ubuntu-latest + needs: + # This is not strictly required, but ensures that Helm Chart doesn't + # appear before the image it refers to. + - release-docker + steps: + - uses: actions/checkout@v1 + - name: Install Helm + run: | + set -xeuo pipefail + + KUBERNETES_VERSION="v1.18.6" + HELM_VERSION="v3.2.4" + + curl -Lo kubectl \ + "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + sudo install kubectl /usr/local/bin/ && rm kubectl + + curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + | tar -xzv --strip-components=1 --occurrence linux-amd64/helm + sudo install helm /usr/local/bin/ && rm helm + - name: Release Helm Chart + env: + AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" + USE_CONTAINER: none + run: make release-helm diff --git a/Makefile b/Makefile index 91755402515f0..277963b8f1a77 100644 --- a/Makefile +++ b/Makefile @@ -456,6 +456,9 @@ release-rollback: ## Rollback pending release changes release-s3: ## Release artifacts to S3 @scripts/release-s3.sh +release-helm: ## Package and release Helm Chart + @scripts/release-helm.sh + sync-install: ## Sync the install.sh script for access via sh.vector.dev @aws s3 cp distribution/install.sh s3://sh.vector.dev --sse --acl public-read diff --git a/distribution/helm/vector/.helmignore b/distribution/helm/vector/.helmignore new file mode 100644 index 0000000000000..0e8a0eb36f4ca --- /dev/null +++ b/distribution/helm/vector/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml new file mode 100644 index 0000000000000..f49c4624fe912 --- /dev/null +++ b/distribution/helm/vector/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +name: vector +type: application +icon: https://vector.dev/press/vector-icon.svg +description: A Helm chart to collect Kubernetes logs with Vector +version: "0.0.0" # overwritten via --version on publishing +appVersion: "0.0.0" # overwritten via --app-version on publishing +home: https://vector.dev/ +sources: + - https://github.com/timberio/vector/ +maintainers: + - name: Vector Contributors + email: vector@timber.io diff --git a/distribution/helm/vector/templates/_helpers.tpl b/distribution/helm/vector/templates/_helpers.tpl new file mode 100644 index 0000000000000..171e6f39384c9 --- /dev/null +++ b/distribution/helm/vector/templates/_helpers.tpl @@ -0,0 +1,86 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "vector.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "vector.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "vector.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "vector.labels" -}} +helm.sh/chart: {{ include "vector.chart" . }} +{{ include "vector.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "vector.selectorLabels" -}} +app.kubernetes.io/name: {{ include "vector.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "vector.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "vector.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Define the name of the ConfigMap +*/}} +{{- define "vector.configMapName" -}} +{{- if (empty .Values.existingConfigMap) }} +{{- include "vector.fullname" . }} +{{- else }} +{{- .Values.existingConfigMap }} +{{- end }} +{{- end }} + +{{/* +Resolve the actual image tag to use. +*/}} +{{- define "vector.imageTag" -}} +{{- if .Values.image.tag }} +{{- .Values.image.tag }} +{{- else }} +{{- $version := default .Chart.AppVersion .Values.image.version }} +{{- printf "%s-%s" $version .Values.image.base }} +{{- end }} +{{- end }} diff --git a/distribution/helm/vector/templates/configmap.yaml b/distribution/helm/vector/templates/configmap.yaml new file mode 100644 index 0000000000000..ecc145ee24e0f --- /dev/null +++ b/distribution/helm/vector/templates/configmap.yaml @@ -0,0 +1,58 @@ +{{- if (empty .Values.existingConfigMap) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "vector.configMapName" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +data: + vector.toml: | + data_dir = "{{ .Values.globalOptions.dataDir }}" + + {{- with .Values.logSchema }} + [log_schema] + host_key = "{{ .hostKey }}" + message_key = "{{ .messageKey }}" + source_type_key = "{{ .sourceTypeKey }}" + timestamp_key = "{{ .timestampKey }}" + {{- end }} + + {{- if .Values.kubernetesLogsSource.enabled }} + [sources.{{ .Values.kubernetesLogsSource.sourceId }}] + type = "kubernetes_logs" + + {{- with .Values.kubernetesLogsSource.rawConfig }} + {{- . | nindent 6 }} + {{- end }} + {{- end }} + + {{- range $sourceId, $source := .Values.sources }} + [sources.{{ $sourceId }}] + type = "{{ $source.type }}" + + {{- with $source.rawConfig }} + {{- . | nindent 6 }} + {{- end }} + {{- end }} + + {{- range $transformId, $transform := .Values.transforms }} + [transforms.{{ $transformId }}] + type = "{{ $transform.type }}" + inputs = {{ $transform.inputs | toJson }} + + {{- with $transform.rawConfig }} + {{- . | nindent 6 }} + {{- end }} + {{- end }} + + {{- range $sinkId, $sink := .Values.sinks }} + [sinks.{{ $sinkId }}] + type = "{{ $sink.type }}" + inputs = {{ $sink.inputs | toJson }} + + {{- with $sink.rawConfig }} + {{- . | nindent 6 }} + {{- end }} + {{- end }} + +{{- end }} diff --git a/distribution/helm/vector/templates/daemonset.yaml b/distribution/helm/vector/templates/daemonset.yaml new file mode 100644 index 0000000000000..96dc50d7a212b --- /dev/null +++ b/distribution/helm/vector/templates/daemonset.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "vector.fullname" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "vector.selectorLabels" . | nindent 6 }} + minReadySeconds: 1 + updateStrategy: + type: {{ .Values.updateStrategy }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "vector.selectorLabels" . | nindent 8 }} + vector.dev/exclude: "true" + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "vector.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: vector + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ include "vector.imageTag" . }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + # Host log directory mount. + - name: var-log + mountPath: /var/log/ + readOnly: true + # Host mount for docker and containerd log file symlinks. + - name: var-lib + mountPath: /var/lib + readOnly: true + # Vector data dir mount. + - name: data-dir + mountPath: "{{ .Values.globalOptions.dataDir }}" + # Vector config dir mount. + - name: config-dir + mountPath: /etc/vector + readOnly: true + # Extra volumes. + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + terminationGracePeriodSeconds: 60 + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + # Log directory. + - name: var-log + hostPath: + path: /var/log/ + # Docker and containerd log files in Kubernetes are symlinks to this folder. + - name: var-lib + hostPath: + path: /var/lib/ + # Vector will store it's data here. + - name: data-dir + hostPath: + path: /var/lib/vector/ + # Vector config dir with a managed config map. + - name: config-dir + configMap: + name: {{ include "vector.configMapName" . }} + # Extra volumes. + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/distribution/helm/vector/templates/podsecuritypolicy.yaml b/distribution/helm/vector/templates/podsecuritypolicy.yaml new file mode 100644 index 0000000000000..a865d4a643ff6 --- /dev/null +++ b/distribution/helm/vector/templates/podsecuritypolicy.yaml @@ -0,0 +1,43 @@ +{{- if .Values.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "vector.fullname" . }} +spec: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + requiredDropCapabilities: + - ALL + volumes: + - 'hostPath' + - 'configMap' + - 'secret' + allowedHostPaths: + - pathPrefix: "/var/log" + readOnly: true + - pathPrefix: "/var/lib/docker/containers" + readOnly: true + {{- range .Values.extraVolumes }} + {{- if .hostPath }} + - pathPrefix: {{ .hostPath.path }} + {{- end }} + {{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 +{{- end }} diff --git a/distribution/helm/vector/templates/rbac.yaml b/distribution/helm/vector/templates/rbac.yaml new file mode 100644 index 0000000000000..6524384767bc3 --- /dev/null +++ b/distribution/helm/vector/templates/rbac.yaml @@ -0,0 +1,38 @@ +{{- if .Values.rbac.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "vector.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - watch +{{- if .Values.psp.enabled }} + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ include "vector.fullname" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "vector.fullname" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "vector.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "vector.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/distribution/helm/vector/templates/serviceaccount.yaml b/distribution/helm/vector/templates/serviceaccount.yaml new file mode 100644 index 0000000000000..f81e6011bd111 --- /dev/null +++ b/distribution/helm/vector/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "vector.serviceAccountName" . }} + labels: + {{- include "vector.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/distribution/helm/vector/values.yaml b/distribution/helm/vector/values.yaml new file mode 100644 index 0000000000000..ed17910df0733 --- /dev/null +++ b/distribution/helm/vector/values.yaml @@ -0,0 +1,124 @@ +# Default values for vector. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: timberio/vector + pullPolicy: IfNotPresent + # Overrides the image tag, the default is `{image.version}-{image.base}`. + tag: "" + # Overrides the image version, the default is the Chart appVersion. + version: "" + base: "debian" + +# Image pull secrets to use at the `Pod`s managed by `DaemonSet`. +imagePullSecrets: [] + +# Override the chart name used in templates. +nameOverride: "" +# Override the full chart name (name prefixed with release name) used in +# templates. +fullnameOverride: "" + +updateStrategy: RollingUpdate + +serviceAccount: + # Specifies whether a service account should be created. + create: true + # Annotations to add to the service account. + annotations: {} + # The name of the service account to use. + # If not set and `create` is true, a name is generated using the `fullname` + # template. + name: "" + +# Annotations to add to the `Pod`s managed by `DaemonSet`. +podAnnotations: {} + +# PodSecurityContext to set at the `Pod`s managed by `DaemonSet`. +podSecurityContext: {} + # fsGroup: 2000 + +# Security context to set at the `vector` container at the `Pod`s managed by +# `DaemonSet`. +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# Extra env vars to pass to the `vector` container. +env: [] + +# Various tweakables for the `Pod`s managed by `DaemonSet`. +resources: {} +nodeSelector: {} +tolerations: [] +affinity: {} + +# Additional volmes to pass to the `Pod`s managed by `DaemonSet`. +extraVolumes: [] + +# Additional volme mounts to pass to the `vector` container of the `Pod`s +# managed by `DaemonSet`. +extraVolumeMounts: [] + +rbac: + # Whether to create rbac resources or not. Disable for non-rbac clusters. + enabled: true + +psp: + # Whether to create `PodSecurityPolicy` or not. + enabled: false + +# Set this to non-empty value to use existing `ConfigMap` for `vector`, instead +# of using a generated one. +existingConfigMap: "" + +# Global parts of the generated `vector` config. +globalOptions: + # Specifies the (in-container) data dir used by `vector`. + dataDir: "/vector-data-dir" + +# Schema part of the generated `vector` config. +logSchema: + hostKey: "host" + messageKey: "message" + sourceTypeKey: "source_type" + timestampKey: "timestamp" + +# The "built-in" kubernetes logs source. Will be added by default, unless +# explicitly disabled. +kubernetesLogsSource: + # Disable to omit the kubernetes logs source from being added. + enabled: true + # The name to use for the "built-in" kubernetes logs source. + sourceId: kubernetes_logs + # Raw TOML config to embed at the kubernetes logs source. + rawConfig: null + +# Sources to add to the generated `vector` config (besides "built-in" kubernetes +# logs source). +sources: {} + # source_name: + # type: "source_type" + # rawConfig: | + # option = "value" + +# Transforms to add to the generated `vector` config. +transforms: {} + # transform_name: + # type: "transform_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" + +# Sinks to add to the generated `vector` config. +sinks: {} + # sink_name: + # type: "sink_type" + # inputs: ["input1", "input2"] + # rawConfig: | + # option = "value" diff --git a/scripts/release-helm.sh b/scripts/release-helm.sh new file mode 100755 index 0000000000000..7a8f37bde624a --- /dev/null +++ b/scripts/release-helm.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# release-helm.sh +# +# SUMMARY +# +# Package Helm Chart and update the Helm repo. + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +set -x + +CHANNEL="${CHANNEL:-"$(scripts/util/release-channel.sh)"}" +VERSION="${VERSION:-"$(scripts/version.sh)"}" + +if [[ "$CHANNEL" == "nightly" ]]; then + DATE="${DATE:-"$(date -u +%Y-%m-%d)"}" + APP_VERSION="nightly-$DATE" # matches the version part of the image tag + CHART_VERSION="$VERSION-$DATE" +else + APP_VERSION="$VERSION" # matches the version part of the image tag + CHART_VERSION="$VERSION" +fi + +if [[ "${USE_TEST_REPO:-"false"}" == "true" ]]; then + PUBLIC_URL="https://vector-helm-repo-tests.s3.amazonaws.com/helm/$CHANNEL" + AWS_REPO_URL="s3://vector-helm-repo-tests/helm/$CHANNEL" +else + PUBLIC_URL="https://packages.timber.io/helm/$CHANNEL" + AWS_REPO_URL="s3://packages.timber.io/helm/$CHANNEL" +fi + +WORKDIR="target/helm" + +REPO_DIR="$WORKDIR/repo" +PREVIOUS_MANIFEST="$WORKDIR/previous-manifest.yaml" + +capture_stderr() { + { OUTPUT=$("$@" 2>&1 1>&3-); } 3>&1 +} + +# Prepare work directory. +rm -rf "$REPO_DIR" +mkdir -p "$REPO_DIR" + +# Package our chart. +helm package \ + distribution/helm/vector \ + --version "$CHART_VERSION" \ + --app-version "$APP_VERSION" \ + --destination "$REPO_DIR" + +# Download previous manifest. +# If it doesn't exist - ignore the error and continue. +if ! capture_stderr aws s3 cp "$AWS_REPO_URL/index.yaml" "$PREVIOUS_MANIFEST"; then + EXPECTED="^fatal error:" + EXPECTED="$EXPECTED An error occurred \(404\) when calling the HeadObject operation:" + EXPECTED="$EXPECTED Key \".*/index\.yaml\" does not exist$" + if ! grep -Eq "$EXPECTED" <<<"$OUTPUT"; then + echo "$OUTPUT" >&2 + exit 1 + else + echo "Warning: repo index file doesn't exist, but we ignore the error" \ + "because we will initialize it" + fi +fi + +# Update the repo index file. +helm repo index \ + "$REPO_DIR" \ + --merge "$PREVIOUS_MANIFEST" \ + --url "$PUBLIC_URL" + +# Upload new files to the repo. +aws s3 cp "$REPO_DIR" "$AWS_REPO_URL" --recursive --sse --acl public-read