feat(new source): Initial kubernetes_logs implementation

CONTRIBUTING.md

@@ -36,6 +36,7 @@ expanding into more specifics.
       1. [Tips and Tricks](#tips-and-tricks)
    1. [Benchmarking](#benchmarking)
    1. [Profiling](#profiling)
+   1. [Kubernetes](#kubernetes)
 1. [Humans](#humans)
    1. [Documentation](#documentation)
    1. [Changelog](#changelog)
@@ -547,6 +548,85 @@ cat stacks.folded | inferno-flamegraph > flamegraph.svg
 And that's it! You now have a flamegraph SVG file that can be opened and
 navigated in your favorite web browser.
 
+### Kubernetes
+
+There is a special flow for when you develop portions of Vector that are
+designed to work with Kubernetes, like `kubernetes_logs` source or the
+`deployment/kubernetes/*.yaml` configs.
+
+This flow facilitates building Vector and deploying it into a cluster.
+
+#### Requirements
+
+There are some extra requirements besides what you'd normally need to work on
+Vector:
+
+* `linux` system (create an issue if you want to work with another OS and we'll
+  help);
+* [`skaffold`](https://skaffold.dev/)
+* [`docker`](https://www.docker.com/)
+* [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+* [`kustomize`](https://kustomize.io/)
+* [`minikube`](https://minikube.sigs.k8s.io/)-powered or other k8s cluster
+* [`cargo watch`](https://github.com/passcod/cargo-watch)
+
+#### The dev flow
+
+Once you have the requirements, use the `scripts/skaffold.sh dev` command.
+
+That's it, just one command should take care of everything!
+
+It will
+
+1. build the `vector` binary in development mode,
+2. build a docker image from this binary via `skaffold/docker/Dockerfile`,
+3. deploy `vector` into the Kubernetes cluster at your current kubectl context
+   using the built docker image and a mix of our production deployment
+   configuration from the `distribution/kubernetes/*.yaml` and the special
+   dev-flow configuration at `skaffold/manifests/*.yaml`; see
+   `kustomization.yaml` for the exact specification.
+
+As the result of invoking the `scripts/skaffold.sh dev`, you should see
+a `skaffold` process running on your local machine, printing the logs from the
+deployed `vector` instance.
+
+To stop the process, press `Ctrl+C`, and wait for `skaffold` to clean up
+the cluster state and exit.
+
+`scripts/skaffold.sh` wraps `skaffold`, you can use other `skaffold` subcommands
+if it fits you better.
+
+#### Troubleshooting
+
+You might need to tweak `skaffold`, here are some hints:
+
+* `skaffold` will try to detect whether a local cluster is used; if a local
+  cluster is used, `skaffold` won't push the docker images it builds to a
+  registry.
+  See [this page](https://skaffold.dev/docs/environment/local-cluster/)
+  for how you can troubleshoot and tweak this behavior.
+
+* `skaffold` can rewrite the image name so that you don't try to push a docker
+  image to a repo that you don't have access to.
+  See [this page](https://skaffold.dev/docs/environment/image-registries/)
+  for more info.
+
+* For the rest of the `skaffold` tweaks you might want to apply check out
+  [this page](https://skaffold.dev/docs/environment/).
+
+#### Going through the dev flow manually
+
+Is some cases `skaffold` may not work. It's possible to go through the dev flow
+manually, without `skaffold`.
+
+One of the important thing `skaffold` does is it patches the configuration to
+tie things together. If you want to go without it, you'll have to take care of
+that yourself, thus some additional knowledge of Kubernetes inner workings is
+required.
+
+Essentially, the steps you have to take to deploy manually are the same that
+`skaffold` will perform, and they're outlined at the previous section.
+
 ## Humans
 
 After making your change, you'll want to prepare it for Vector's users

Cargo.toml

@@ -142,6 +142,7 @@ strip-ansi-escapes = { version = "0.1.0"}
 colored = "1.9"
 warp = { package = "warp", version = "0.2", default-features = false, optional = true }
 evmap = { version = "7", features = ["bytes"], optional = true }
+evmap10 = { package = "evmap", version = "10", features = ["bytes"], optional = true }
 logfmt = { version = "0.0.2", optional = true }
 notify = "4.0.14"
 once_cell = "1.3"
@@ -152,6 +153,7 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti
 task-compat = "0.1"
 cidr-utils = "0.4.2"
 pin-project = "0.4.22"
+k8s-openapi = { version = "0.9", features = ["v1_15"], optional = true }
 
 # For WASM
 vector-wasm = { path = "lib/vector-wasm", optional = true }
@@ -228,6 +230,10 @@ leveldb-cmake = ["leveldb", "leveldb/leveldb-sys-3"]
 wasm = ["lucetc", "lucet-runtime", "lucet-wasi", "vector-wasm", "anyhow"]
 wasm-timings = ["wasm"]
 
+# Enables kubernetes dependencies and shared code. Kubernetes-related sources,
+# transforms and sinks should depend on this feature.
+kubernetes = ["k8s-openapi", "evmap10"]
+
 # Sources
 sources = [
   "sources-docker",
@@ -246,6 +252,7 @@ sources = [
   "sources-syslog",
   "sources-tls",
   "sources-vector",
+  "sources-kubernetes-logs",
 ]
 sources-docker = ["bollard"]
 sources-file = ["bytesize"]
@@ -263,6 +270,7 @@ sources-stdin = ["bytesize"]
 sources-syslog = ["sources-socket", "syslog_loose"]
 sources-tls = ["sources-http", "sources-logplex", "sources-socket", "sources-splunk_hec"]
 sources-vector = ["sources-socket"]
+sources-kubernetes-logs = ["kubernetes", "transforms-merge", "transforms-json_parser", "transforms-regex_parser"]
 
 # Transforms
 transforms = [
@@ -425,6 +433,7 @@ kafka-integration-tests = ["sources-kafka", "sinks-kafka"]
 loki-integration-tests = ["sinks-loki"]
 pulsar-integration-tests = ["sinks-pulsar"]
 splunk-integration-tests = ["sinks-splunk_hec", "warp"]
+kubernetes-integration-tests = ["sources-kubernetes-logs"]
 
 shutdown-tests = ["sources","sinks-console","sinks-prometheus","sinks-blackhole","unix","rdkafka","transforms-log_to_metric","transforms-lua"]
 disable-resolv-conf = []

distribution/kubernetes/vector-namespaced.yaml

@@ -7,12 +7,12 @@ data:
     # Configuration for vector.
     # Docs: https://vector.dev/docs/
 
-    # Configure the controlled by the deployment.
+    # Data dir is location controlled at the `DaemonSet`.
     data_dir = "/vector-data-dir"
 
     # Ingest logs from Kubernetes.
-    [sources.kubernetes]
-        type = "kubernetes"
+    [sources.kubernetes_logs]
+        type = "kubernetes_logs"
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -28,11 +28,11 @@ spec:
     metadata:
       labels:
         name: vector
+        vector.dev/exclude: "true"
     spec:
       containers:
         - name: vector
           image: timberio/vector:latest-alpine
-          imagePullPolicy: Always
           args:
             - --config
             - /etc/vector/*.toml
@@ -61,6 +61,7 @@ spec:
             - name: config-dir
               mountPath: /etc/vector/
               readOnly: true
+      terminationGracePeriodSeconds: 60
       tolerations:
         # This toleration is to have the daemonset runnable on master nodes.
         # Remove it if your masters can't run pods.

kustomization.yaml

@@ -0,0 +1,10 @@
+# This is a part of our skaffold setup for development.
+# Do not use in production.
+
+namespace: vector
+
+resources:
+  - distribution/kubernetes/vector-global.yaml
+  - skaffold/manifests/namespace.yaml
+  - skaffold/manifests/config.yaml
+  - distribution/kubernetes/vector-namespaced.yaml

scripts/copy-docker-image-to-minikube.sh

@@ -27,7 +27,8 @@ docker save "${IMAGES[@]}" | gzip >"$IMAGES_ARCHIVE"
 # Start a subshell to preserve the env state.
 (
   # Switch to minikube docker.
-  eval "$(minikube --shell bash docker-env)"
+  # shellcheck source=minikube-docker-env.sh disable=SC1091
+  . scripts/minikube-docker-env.sh
 
   # Load images.
   docker load -i "$IMAGES_ARCHIVE"

scripts/minikube-docker-env.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if ! COMMANDS="$(minikube --shell bash docker-env)"; then
+  echo "Unable to obtain docker env from minikube; is minikube started?" >&2
+  exit 7
+fi
+
+eval "$COMMANDS"