From 1356830e6dc89ccd2aee7a9b6616a00931c86bbd Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 7 May 2020 16:15:02 +0300 Subject: [PATCH 001/118] Add kubernetes-integration-tests feature Signed-off-by: MOZGIII --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 5ab90b3347de6..4345ac4eb7bd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -425,6 +425,7 @@ kafka-integration-tests = ["sources-kafka", "sinks-kafka"] loki-integration-tests = ["sinks-loki"] pulsar-integration-tests = ["sinks-pulsar"] splunk-integration-tests = ["sinks-splunk_hec", "warp"] +kubernetes-integration-tests = [] shutdown-tests = ["sources","sinks-console","sinks-prometheus","sinks-blackhole","unix","rdkafka","transforms-log_to_metric","transforms-lua"] disable-resolv-conf = [] From 264128ef39ece8768c9602748e024559af50a160 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 7 May 2020 16:15:59 +0300 Subject: [PATCH 002/118] Enable kubernetes tests Signed-off-by: MOZGIII --- scripts/test-integration-kubernetes.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/test-integration-kubernetes.sh b/scripts/test-integration-kubernetes.sh index d62e957b4d583..329154d95fdec 100755 --- a/scripts/test-integration-kubernetes.sh +++ b/scripts/test-integration-kubernetes.sh @@ -71,8 +71,5 @@ fi export KUBE_TEST_DEPLOY_COMMAND="scripts/deploy-kubernetes-test.sh" -# TODO: enable kubernetes tests when they're implemented -exit 0 # disable the test and make them pass - # Run the tests. cargo test --no-default-features --features kubernetes-integration-tests From 18a0e550853e7d3854212a9588cbeccb003b9f75 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 22:18:48 +0300 Subject: [PATCH 003/118] Add skaffold for quick development iterations Signed-off-by: MOZGIII --- kustomization.yaml | 10 ++++++++++ scripts/minikube-docker-env.sh | 9 +++++++++ scripts/skaffold.sh | 26 ++++++++++++++++++++++++++ skaffold.yaml | 15 +++++++++++++++ skaffold/docker/Dockerfile | 10 ++++++++++ skaffold/manifests/config.yaml | 11 +++++++++++ skaffold/manifests/namespace.yaml | 4 ++++ 7 files changed, 85 insertions(+) create mode 100644 kustomization.yaml create mode 100644 scripts/minikube-docker-env.sh create mode 100755 scripts/skaffold.sh create mode 100644 skaffold.yaml create mode 100644 skaffold/docker/Dockerfile create mode 100644 skaffold/manifests/config.yaml create mode 100644 skaffold/manifests/namespace.yaml diff --git a/kustomization.yaml b/kustomization.yaml new file mode 100644 index 0000000000000..efd9cf96a1697 --- /dev/null +++ b/kustomization.yaml @@ -0,0 +1,10 @@ +# This is a part of out skaffold setup for development. +# Do not use in production. + +namespace: vector + +resources: + - distribution/kubernetes/vector-global.yaml + - skaffold/manifests/namespace.yaml + - skaffold/manifests/config.yaml + - distribution/kubernetes/vector-namespaced.yaml diff --git a/scripts/minikube-docker-env.sh b/scripts/minikube-docker-env.sh new file mode 100644 index 0000000000000..64b45ae0038ca --- /dev/null +++ b/scripts/minikube-docker-env.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +if ! COMMANDS="$(minikube docker-env)"; then + echo "Unable to obtain docker env from minikube; is minikube started?" >&2 + exit 7 +fi + +eval "$COMMANDS" diff --git a/scripts/skaffold.sh b/scripts/skaffold.sh new file mode 100755 index 0000000000000..6d356c1e29ecd --- /dev/null +++ b/scripts/skaffold.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# Inital vector build to ensure we start at a valid state. +cargo build + +# Prepare .dockerignore so we don't send the whole dir to the docker as the +# context. +cat <target/debug/.dockerignore +**/* +!vector +EOF + +# Configure docker (and skaffold) to use minikube docker host. +# shellcheck source=minikube-docker-env.sh disable=SC1091 +. scripts/minikube-docker-env.sh + +# Watch for changes in he background and rebuild the vector binary. +cargo watch -x build & + +# Kill all child processes of this bash instance. +trap 'kill -- "-$$"; exit 0' EXIT + +export SKAFFOLD_CACHE_ARTIFACTS=false +skaffold "$@" diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000000000..21fba4f685709 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,15 @@ +apiVersion: skaffold/v2beta3 +kind: Config +metadata: + name: vector +build: + local: + useDockerCLI: true + push: false + artifacts: + - image: timberio/vector + context: target/debug + docker: + dockerfile: ../../skaffold/docker/Dockerfile +deploy: + kustomize: {} diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile new file mode 100644 index 0000000000000..3c825ba040498 --- /dev/null +++ b/skaffold/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:buster-slim + +RUN apt-get update && apt-get install -y ca-certificates tzdata && rm -rf /var/lib/apt/lists/* + +ENTRYPOINT ["/usr/bin/vector"] + +ARG LOG=debug,vector=trace +ENV LOG="$LOG" + +COPY vector /usr/bin/vector diff --git a/skaffold/manifests/config.yaml b/skaffold/manifests/config.yaml new file mode 100644 index 0000000000000..890a06f705e1c --- /dev/null +++ b/skaffold/manifests/config.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-config +data: + vector.toml: | + [sinks.stdout] + type = "console" + inputs = ["kubernetes_logs"] + target = "stdout" + encoding = "json" diff --git a/skaffold/manifests/namespace.yaml b/skaffold/manifests/namespace.yaml new file mode 100644 index 0000000000000..2b7563f84a801 --- /dev/null +++ b/skaffold/manifests/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: vector From 4bc782ac76c59b94a36f77fac150a52d3f8ed2e8 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 17:58:47 +0300 Subject: [PATCH 004/118] Add kubernetes mod and cargo feature Signed-off-by: MOZGIII --- Cargo.toml | 4 ++++ src/kubernetes/mod.rs | 1 + src/lib.rs | 1 + 3 files changed, 6 insertions(+) create mode 100644 src/kubernetes/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 4345ac4eb7bd5..15febb5c7f522 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,6 +228,10 @@ leveldb-cmake = ["leveldb", "leveldb/leveldb-sys-3"] wasm = ["lucetc", "lucet-runtime", "lucet-wasi", "vector-wasm", "anyhow"] wasm-timings = ["wasm"] +# Enables kubernetes dependencies and shared code. Kubernetes-related sources, +# transforms and sinks should depend on this feature. +kubernetes = [] + # Sources sources = [ "sources-docker", diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs new file mode 100644 index 0000000000000..f77ea148fd47f --- /dev/null +++ b/src/kubernetes/mod.rs @@ -0,0 +1 @@ +#![cfg(feature = "kubernetes")] diff --git a/src/lib.rs b/src/lib.rs index cfc80f4adfe54..6ddd08047d832 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub mod async_read; pub mod hyper; #[cfg(feature = "rdkafka")] pub mod kafka; +pub mod kubernetes; pub mod list; pub mod metrics; pub mod region; From ebc5663b777d286fcb6780f5bf0018f22cf8323c Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 18:11:43 +0300 Subject: [PATCH 005/118] Add an HTTP client tailored for k8s API in an in-cluster environment Signed-off-by: MOZGIII --- src/kubernetes/client/config/in_cluster.rs | 68 +++++++++++++ src/kubernetes/client/config/mod.rs | 27 +++++ src/kubernetes/client/mod.rs | 113 +++++++++++++++++++++ src/kubernetes/mod.rs | 22 ++++ 4 files changed, 230 insertions(+) create mode 100644 src/kubernetes/client/config/in_cluster.rs create mode 100644 src/kubernetes/client/config/mod.rs create mode 100644 src/kubernetes/client/mod.rs diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs new file mode 100644 index 0000000000000..f0d7131055f9a --- /dev/null +++ b/src/kubernetes/client/config/in_cluster.rs @@ -0,0 +1,68 @@ +//! Everything related to building in-cluster configuration. + +use super::Config; +use crate::tls::TlsOptions; +use http::Uri; +use snafu::{ResultExt, Snafu}; + +impl Config { + /// Prepares a config sutable for use when running in k8s cluster. + pub fn in_cluster() -> Result { + let host = std::env::var("KUBERNETES_SERVICE_HOST").context(NotInCluster)?; + let port = std::env::var("KUBERNETES_SERVICE_PORT").context(NotInCluster)?; + + let base = Uri::builder() + .scheme("https") + .authority(join_host_port(host.as_str(), port.as_str()).as_str()) + .path_and_query("/") + .build() + .context(InvalidUrl)?; + + let token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + let root_ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + + let token = std::fs::read_to_string(token_file).context(Token)?; + + let mut tls_options = TlsOptions::default(); + tls_options.ca_file = Some(root_ca_file.into()); + + Ok(Self { + base, + token, + tls_options, + }) + } +} + +/// An error returned when building an in-cluster configuration. +#[derive(Debug, Snafu)] +pub enum Error { + /// The in-cluster configuration requested while executing not in a cluster + /// environment. + #[snafu(display("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined"))] + NotInCluster { + /// The underlying error. + source: std::env::VarError, + }, + + /// The token file could not be read successfully. + #[snafu(display("unable to read the token file"))] + Token { + /// The underlying error. + source: std::io::Error, + }, + + /// The configuration resulted in an invalid URL. + #[snafu(display("unable to construct a proper API server URL"))] + InvalidUrl { + /// The underlying error. + source: http::Error, + }, +} + +fn join_host_port(host: &str, port: &str) -> String { + if host.contains(":") { + return format!("[{}]:{}", host, port); + } + format!("{}:{}", host, port) +} diff --git a/src/kubernetes/client/config/mod.rs b/src/kubernetes/client/config/mod.rs new file mode 100644 index 0000000000000..5ceafcb26ef83 --- /dev/null +++ b/src/kubernetes/client/config/mod.rs @@ -0,0 +1,27 @@ +//! Client configuration. + +use crate::tls::TlsOptions; +use http::Uri; + +/// A k8s client configuration. +/// +/// This type is designed to hold all possible variants of the configuration. +/// It also abstracts the client from the various ways to obtain the +/// configuration. +/// +/// The implementation is fairly limited, and only covers the use cases we +/// support. +#[derive(Debug, Clone)] +pub struct Config { + /// The base URL to use when constructing HTTP requests to the k8s API + /// server. + pub base: Uri, + + /// The bearer token to use at the `Authorization` header. + pub token: String, + + /// The TLS configuration parameters to use at the HTTP client. + pub tls_options: TlsOptions, +} + +pub mod in_cluster; diff --git a/src/kubernetes/client/mod.rs b/src/kubernetes/client/mod.rs new file mode 100644 index 0000000000000..3650e64e580d4 --- /dev/null +++ b/src/kubernetes/client/mod.rs @@ -0,0 +1,113 @@ +//! A Kubernetes API client built using Vector interfaces to the system +//! resources as building blocks. +//! +//! Here are a few pointers to the resources that were used as an inpiration for +//! this mod: +//! +//! - https://github.com/kubernetes/client-go/blob/master/tools/clientcmd/api/types.go +//! +//! A part of the official Kubernetes client library (in Go) that contains +//! the structure for KUBECONFIG files. Used for reference on naming things. +//! +//! - https://github.com/kubernetes/apimachinery/blob/master/pkg/watch/watch.go +//! +//! The reference design of the watchers composition and interfaces that's +//! known to work. +//! +//! - https://github.com/kubernetes/client-go/blob/master/rest/config.go +//! +//! The reference implementation on preparing the in-cluster config. +//! + +use crate::{dns::Resolver, sinks::util::http::HttpClient, tls::TlsSettings}; +use http::{ + header::{self, HeaderValue}, + uri, Request, Response, Uri, +}; +use hyper::body::Body; + +pub mod config; + +use config::Config; + +/// A client to the k8s API. +/// +/// Wraps our in-house [`HttpClient`]. +#[derive(Debug, Clone)] +pub struct Client { + inner: HttpClient, + uri_scheme: uri::Scheme, + uri_authority: uri::Authority, + auth_header: HeaderValue, +} + +impl Client { + /// Create a new [`Client`]. + /// + /// Takes the common kubernetes API cluster configuration [`Config`] and + /// a [`Resolver`] that is generally not the part of the config, but is + /// specific to our [`HttpClient`] implementation. + /// + /// Consumes the configuration to populate the internal state. + /// Retunrs an error if the configuratiton is not valid. + // TODO: add a proper error type. + pub fn new(config: Config, resolver: Resolver) -> crate::Result { + let Config { + base, + tls_options, + token, + } = config; + + let tls_settings = TlsSettings::from_options(&Some(tls_options))?; + let inner = HttpClient::new(resolver, tls_settings)?; + + let uri::Parts { + scheme, authority, .. + } = base.into_parts(); + + let uri_scheme = scheme.ok_or_else(|| "no scheme")?; + let uri_authority = authority.ok_or_else(|| "no authority")?; + + let auth_header = format!("Bearer {}", token); + let auth_header = HeaderValue::from_str(auth_header.as_str())?; + + Ok(Self { + inner, + uri_scheme, + uri_authority, + auth_header, + }) + } + + /// Alters a request according to the client configuraion and sends it. + pub async fn send>(&mut self, req: Request) -> crate::Result> { + let req = convert_request_body(req); + let req = self.prepare_request(req); + self.inner.send(req).await + } + + fn prepare_request(&self, mut req: Request) -> Request { + self.adjust_uri(req.uri_mut()); + + req.headers_mut() + .insert(header::AUTHORIZATION, self.auth_header.clone()); + + req + } + + // TODO: figure if we can do this more efficiently. + fn adjust_uri(&self, uri: &mut Uri) { + *uri = Uri::builder() + .scheme(self.uri_scheme.clone()) + .authority(self.uri_authority.clone()) + .path_and_query(uri.path_and_query().unwrap().clone()) + .build() + .unwrap(); + } +} + +fn convert_request_body>(req: Request) -> Request { + let (parts, body) = req.into_parts(); + let body = body.into(); + Request::from_parts(parts, body) +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index f77ea148fd47f..b4de6dbf6b2d3 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -1 +1,23 @@ +//! This mod contains shared portions of the kubernetes implementations. +//! +//! Here are a few pointers to the resources that were used as an inspiration: +//! +//! - https://github.com/kubernetes/client-go/blob/master/tools/clientcmd/api/types.go +//! +//! A part of the official Kubernetes client library (in Go) that contains +//! the structure for KUBECONFIG files. Used for reference on naming things. +//! +//! - https://github.com/kubernetes/apimachinery/blob/master/pkg/watch/watch.go +//! +//! The reference design of the watchers composition and interfaces that's +//! known to work. +//! +//! - https://github.com/kubernetes/client-go/blob/master/rest/config.go +//! +//! The reference implementation on preparing the in-cluster config. +//! + #![cfg(feature = "kubernetes")] +#![warn(missing_docs)] + +pub mod client; From 6709988b9fe3a7d466c6b66561784e6de1d09e12 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 18:47:36 +0300 Subject: [PATCH 006/118] Add a decoder for chained k8s responses Signed-off-by: MOZGIII --- Cargo.lock | 37 ++ Cargo.toml | 3 +- src/kubernetes/mod.rs | 1 + src/kubernetes/multi_response_decoder.rs | 410 +++++++++++++++++++++++ 4 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 src/kubernetes/multi_response_decoder.rs diff --git a/Cargo.lock b/Cargo.lock index 8b3d960dfdf50..4265ff3fb3f25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1862,6 +1862,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "k8s-openapi" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96affb18356fc998baa692c2185d198095ffdf6e6fff3bc9efb4927952f86851" +dependencies = [ + "base64 0.11.0", + "bytes 0.5.4", + "chrono", + "http 0.2.1", + "percent-encoding 2.1.0", + "serde", + "serde-value", + "serde_json", + "url 2.1.1", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -2798,6 +2815,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" +dependencies = [ + "num-traits", +] + [[package]] name = "os_pipe" version = "0.9.2" @@ -3960,6 +3986,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.114" @@ -5467,6 +5503,7 @@ dependencies = [ "indexmap", "inventory", "jemallocator", + "k8s-openapi", "lazy_static 1.4.0", "leveldb", "libc", diff --git a/Cargo.toml b/Cargo.toml index 15febb5c7f522..8dc0e3087795e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,7 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti task-compat = "0.1" cidr-utils = "0.4.2" pin-project = "0.4.22" +k8s-openapi = { version = "0.7", features = ["v1_15"], optional = true } # For WASM vector-wasm = { path = "lib/vector-wasm", optional = true } @@ -230,7 +231,7 @@ wasm-timings = ["wasm"] # Enables kubernetes dependencies and shared code. Kubernetes-related sources, # transforms and sinks should depend on this feature. -kubernetes = [] +kubernetes = ["k8s-openapi"] # Sources sources = [ diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index b4de6dbf6b2d3..7809f64886788 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -21,3 +21,4 @@ #![warn(missing_docs)] pub mod client; +pub mod multi_response_decoder; diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs new file mode 100644 index 0000000000000..9c6dcc54198f9 --- /dev/null +++ b/src/kubernetes/multi_response_decoder.rs @@ -0,0 +1,410 @@ +//! Decode multiple [`Response`]s. + +use k8s_openapi::{http::StatusCode, Response, ResponseError}; + +/// Provides an algorithm to parse multiple [`Response`]s from multiple chunks +/// of data represented as `&[u8]`. +#[derive(Debug)] +pub struct MultiResponseDecoder { + pending_data: Vec, + responses_buffer: Vec>, +} + +impl MultiResponseDecoder +where + T: Response, +{ + /// Create a new [`MultiResponseDecoder`]. + pub fn new() -> Self { + Self { + pending_data: Vec::new(), + responses_buffer: Vec::new(), + } + } + + /// Take the next chunk of data and spit out parsed `T`s. + pub fn process_next_chunk( + &mut self, + chunk: &[u8], + ) -> std::vec::Drain<'_, Result> { + self.pending_data.extend_from_slice(chunk); + loop { + match T::try_from_parts(StatusCode::OK, &self.pending_data) { + Ok((response, consumed_bytes)) => { + debug_assert!(consumed_bytes > 0, "parser must've consumed some data"); + self.pending_data.drain(..consumed_bytes); + self.responses_buffer.push(Ok(response)); + } + Err(ResponseError::NeedMoreData) => break, + Err(error) => { + self.responses_buffer.push(Err(error)); + break; + } + }; + } + self.responses_buffer.drain(..) + } + + /// Complete the parsing. + /// + /// Call this when you're not expecting any more data chunks. + /// Produces an error if there's unparsed data remaining. + pub fn finish(self) -> Result<(), Vec> { + let Self { pending_data, .. } = self; + if pending_data.is_empty() { + return Ok(()); + } + Err(pending_data) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::{ + api::core::v1::Pod, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, + WatchResponse, + }; + + /// Test object. + type TO = WatchResponse; + + // A helper function to make a test object. + fn make_to(uid: &str) -> TO { + WatchResponse::Ok(WatchEvent::Added(Pod { + metadata: Some(ObjectMeta { + uid: Some(uid.to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + })) + } + + fn assert_test_object( + tested_test_object: Option>, + expected_uid: &str, + ) { + let actual_to = tested_test_object + .expect("expected an yielded entry, but none found") + .expect("parsing failed"); + let expected_to = make_to(expected_uid); + match (actual_to, expected_to) { + (WatchResponse::Ok(actual_event), WatchResponse::Ok(expected_event)) => { + assert_eq!(actual_event, expected_event) + } + _ => panic!("expected an event, got something else"), + } + } + + #[test] + fn test_empty() { + let dec = MultiResponseDecoder::::new(); + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_incomplete() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk(b"{"); + assert!(stream.next().is_none()); + } + + assert_eq!(dec.finish().unwrap_err(), b"{"); + } + + #[test] + fn test_rubblish() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk(b"qwerty"); + assert!(stream.next().unwrap().is_err()); + assert!(stream.next().is_none()); + } + + assert_eq!(dec.finish().unwrap_err(), b"qwerty"); + } + + #[test] + fn test_one() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_chunked() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "ob"#, + ); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"ject": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_two() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid1" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid0"); + assert_test_object(stream.next(), "uid1"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_many_chunked_1() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "ob"#, + ); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"ject": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVe"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"rsion": "v1", + "metadata": { + "uid": "uid1" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid1"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_many_chunked_2() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }{ + "type": "ADDED", + "ob"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"ject": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid1" + } + } + }{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid2" + } + } + }{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVe"#, + ); + assert_test_object(stream.next(), "uid1"); + assert_test_object(stream.next(), "uid2"); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"rsion": "v1", + "metadata": { + "uid": "uid3" + } + } + }{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid4" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid3"); + assert_test_object(stream.next(), "uid4"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_two_one_by_one() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid1" + } + } + }"#, + ); + assert_test_object(stream.next(), "uid1"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + + #[test] + fn test_incomplete_after_valid_data() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk( + br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }{"#, + ); + assert_test_object(stream.next(), "uid0"); + assert!(stream.next().is_none()); + } + + assert_eq!(dec.finish().unwrap_err(), b"{"); + } +} From 15a7297693c837c4fbb4cec870a077213cf95da7 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 22 May 2020 05:37:55 +0300 Subject: [PATCH 007/118] Add block_on_std to test_util Signed-off-by: MOZGIII --- src/test_util.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test_util.rs b/src/test_util.rs index ed1bda3466376..9eed3a486559b 100644 --- a/src/test_util.rs +++ b/src/test_util.rs @@ -239,6 +239,16 @@ where rt.block_on(future) } +pub fn block_on_std(future: F) -> F::Output +where + F: std::future::Future + Send + 'static, + F::Output: Send + 'static, +{ + let mut rt = runtime(); + + rt.block_on_std(future) +} + pub fn runtime() -> Runtime { Runtime::single_threaded().unwrap() } From bc7a2751234641a428d7f8a892929b211c919220 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 18:49:11 +0300 Subject: [PATCH 008/118] Add tools for processing HTTP bodies as streams of k8s responses Signed-off-by: MOZGIII --- src/kubernetes/mod.rs | 1 + src/kubernetes/stream.rs | 167 +++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 169 insertions(+) create mode 100644 src/kubernetes/stream.rs diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 7809f64886788..bb2c8c7bb3baf 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,3 +22,4 @@ pub mod client; pub mod multi_response_decoder; +pub mod stream; diff --git a/src/kubernetes/stream.rs b/src/kubernetes/stream.rs new file mode 100644 index 0000000000000..394b6344fc992 --- /dev/null +++ b/src/kubernetes/stream.rs @@ -0,0 +1,167 @@ +//! Work with HTTP bodies as streams of Kubernetes resources. + +use super::multi_response_decoder::MultiResponseDecoder; +use async_stream::try_stream; +use bytes05::Buf; +use futures::pin_mut; +use futures::stream::Stream; +use hyper::body::HttpBody as Body; +use k8s_openapi::{Response, ResponseError}; +use snafu::{ResultExt, Snafu}; + +/// Converts the HTTP response [`Body`] to a stream of parsed Kubernetes +/// [`Response`]s. +pub fn body(body: B) -> impl Stream::Error>>> +where + T: Response + Unpin + 'static, + B: Body, + ::Error: std::error::Error + 'static + Unpin, +{ + try_stream! { + let mut decoder: MultiResponseDecoder = MultiResponseDecoder::new(); + + debug!(message = "streaming the HTTP body"); + + pin_mut!(body); + while let Some(buf) = body.data().await { + let mut buf = buf.context(Reading)?; + let chunk = buf.to_bytes(); + let responses = decoder.process_next_chunk(chunk.as_ref()); + for response in responses { + let response = response.context(Parsing)?; + yield response; + } + } + decoder.finish().map_err(|data| Error::UnparsedDataUponCompletion { data })?; + } +} + +/// Errors that can occur in the stream. +#[derive(Debug, Snafu)] +pub enum Error +where + ReadError: std::error::Error + 'static, +{ + /// An error occured while reading the response body. + #[snafu(display("reading the data chunk failed"))] + Reading { + /// The error we got while reading. + source: ReadError, + }, + + /// An error occured while parsing the response body. + #[snafu(display("data parsing failed"))] + Parsing { + /// Response parsing error. + source: ResponseError, + }, + + /// An incomplete response remains in the buffer, but we don't expect + /// any more data. + #[snafu(display("unparsed data remaining upon completion"))] + UnparsedDataUponCompletion { + /// The unparsed data. + data: Vec, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_util; + use futures::StreamExt; + use k8s_openapi::{api::core::v1::Pod, WatchResponse}; + + fn hyper_body_from_chunks( + chunks: Vec>, + ) -> hyper::body::Body { + let in_stream = futures::stream::iter(chunks); + hyper::body::Body::wrap_stream(in_stream) + } + + #[test] + fn test_body() { + test_util::trace_init(); + test_util::block_on_std(async move { + let data = r#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }"#; + let chunks: Vec> = vec![Ok(data)]; + let sample_body = hyper_body_from_chunks(chunks); + + let out_stream = body::<_, WatchResponse>(sample_body); + pin_mut!(out_stream); + + assert!(out_stream.next().await.unwrap().is_ok()); + assert!(out_stream.next().await.is_none()); + }) + } + + #[test] + fn test_body_passes_reading_error() { + test_util::trace_init(); + test_util::block_on_std(async move { + let err = std::io::Error::new(std::io::ErrorKind::Other, "test error"); + let chunks: Vec> = vec![Err(err)]; + let sample_body = hyper_body_from_chunks(chunks); + + let out_stream = body::<_, WatchResponse>(sample_body); + pin_mut!(out_stream); + + { + let err = out_stream.next().await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Reading { source: hyper::Error { .. } })); + } + + assert!(out_stream.next().await.is_none()); + }) + } + + #[test] + fn test_body_passes_parsing_error() { + test_util::trace_init(); + test_util::block_on_std(async move { + let chunks: Vec> = vec![Ok("qwerty")]; + let sample_body = hyper_body_from_chunks(chunks); + + let out_stream = body::<_, WatchResponse>(sample_body); + pin_mut!(out_stream); + + { + let err = out_stream.next().await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Parsing { source: ResponseError::Json(_) })); + } + + assert!(out_stream.next().await.is_none()); + }) + } + + #[test] + fn test_body_uses_finish() { + test_util::trace_init(); + test_util::block_on_std(async move { + let chunks: Vec> = vec![Ok("{")]; + let sample_body = hyper_body_from_chunks(chunks); + + let out_stream = body::<_, WatchResponse>(sample_body); + pin_mut!(out_stream); + + { + let err = out_stream.next().await.unwrap().unwrap_err(); + assert!(matches!( + err, + Error::UnparsedDataUponCompletion { data } if data == vec![b'{'] + )); + } + + assert!(out_stream.next().await.is_none()); + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 6ddd08047d832..870bc6a995c1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "256"] // for async-stream #![allow(clippy::approx_constant)] #![allow(clippy::float_cmp)] #![allow(clippy::match_wild_err_arm)] From e8127b22166035a0423c86212758426909267969 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 18:56:11 +0300 Subject: [PATCH 009/118] Add Watcher trait Signed-off-by: MOZGIII --- src/kubernetes/mod.rs | 1 + src/kubernetes/watcher.rs | 71 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/kubernetes/watcher.rs diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index bb2c8c7bb3baf..0e6d046c242ae 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -23,3 +23,4 @@ pub mod client; pub mod multi_response_decoder; pub mod stream; +pub mod watcher; diff --git a/src/kubernetes/watcher.rs b/src/kubernetes/watcher.rs new file mode 100644 index 0000000000000..f0afa4d3a35a8 --- /dev/null +++ b/src/kubernetes/watcher.rs @@ -0,0 +1,71 @@ +//! Watcher abstraction. + +use futures::{future::BoxFuture, stream::Stream}; +use k8s_openapi::{WatchOptional, WatchResponse}; +use serde::de::DeserializeOwned; +use snafu::Snafu; + +/// Watch over the changes for a k8s resource. +pub trait Watcher { + /// The type of the watched object. + type Object: DeserializeOwned; + + /// The error type watcher invocation implementation uses internally. + type InvocationError: std::error::Error + Send + 'static; + + /// The error type watcher stream implementation uses internally. + type StreamError: std::error::Error + Send + 'static; + + /// The stream type produced by the watch request. + type Stream: Stream, Self::StreamError>>; + + /// Issues a single watch request and returns a stream results. + fn watch<'a>( + &'a mut self, + watch_optional: WatchOptional<'a>, + ) -> BoxFuture<'a, Result>>; +} + +pub mod invocation { + //! Invocation errors. + use super::*; + + /// Error wrapper providing a semantic wrapper around invocation errors to + /// bind meaningful and actionably common error semantics to the arbitrary + /// underlying errors. + #[derive(Debug, Snafu)] + #[snafu(visibility(pub))] + pub enum Error + where + T: std::error::Error + Send + 'static, + { + /// Desync error signals that the server went out of sync and the resource + /// version specified in the call can no longer be used. + Desync { + /// The underlying error. + source: T, + }, + + /// Any other error that may have maening for downstream but doesn't have + /// a semantics attached to it at the [`Watcher`] trait level. + Other { + /// The underlying error. + source: T, + }, + } + + impl Error + where + T: std::error::Error + Send + 'static, + { + /// Create an `Error::Desync`. + pub fn desync(source: T) -> Self { + Self::Desync { source } + } + + /// Create an `Error::Other`. + pub fn other(source: T) -> Self { + Self::Other { source } + } + } +} From c45dbd5862d70f185fd8a5a40ba5c00f0ea6cb40 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 18:58:17 +0300 Subject: [PATCH 010/118] Add ApiWatcher implementation Signed-off-by: MOZGIII --- src/kubernetes/api_watcher.rs | 136 ++++++++++++++++++++++++++++++++++ src/kubernetes/mod.rs | 1 + 2 files changed, 137 insertions(+) create mode 100644 src/kubernetes/api_watcher.rs diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs new file mode 100644 index 0000000000000..37b927e87e0ff --- /dev/null +++ b/src/kubernetes/api_watcher.rs @@ -0,0 +1,136 @@ +//! A watcher based on the k8s API. + +use super::{ + client::Client, + stream as k8s_stream, + watcher::{self, Watcher}, +}; +use futures::{ + future::BoxFuture, + stream::{BoxStream, Stream}, +}; +use http::StatusCode; +use hyper::Error as BodyError; +use k8s_openapi::{api::core::v1::Pod, WatchOptional, WatchResponse}; +use snafu::{ResultExt, Snafu}; + +// TODO: abstract around the request builder. +// The `Pod::watch_pod_for_all_namespaces` is currently hardcoded - but we +// can use this with other resource types too, and we'll probably have to +// add support for watching namespaces later on. + +/// A simple watcher atop of the Kubernetes API [`Client`]. +pub struct ApiWatcher { + client: Client, +} + +impl ApiWatcher { + /// Create a new [`ApiWatcher`]. + pub fn new(client: Client) -> Self { + Self { client } + } +} + +impl ApiWatcher { + async fn invoke( + &mut self, + watch_optional: WatchOptional<'_>, + ) -> Result< + impl Stream, k8s_stream::Error>> + 'static, + watcher::invocation::Error, + > { + // Prepare request. + let (request, _) = Pod::watch_pod_for_all_namespaces(watch_optional) + .context(invocation::RequestPreparation)?; + trace!(message = "Request prepared", ?request); + + // Send request, get response. + let response = self + .client + .send(request) + .await + .context(invocation::Request)?; + trace!(message = "Got response", ?response); + + // Handle response status code. + let status = response.status(); + if status != StatusCode::OK { + let source = invocation::Error::BadStatus { status }; + let err = if status == StatusCode::GONE { + watcher::invocation::Error::Desync { source } + } else { + watcher::invocation::Error::Other { source } + }; + Err(err)?; + } + + // Stream response body. + let body = response.into_body(); + Ok(k8s_stream::body(body)) + } + + async fn invoke_boxed_stream( + &mut self, + watch_optional: WatchOptional<'_>, + ) -> Result< + BoxStream<'static, Result, k8s_stream::Error>>, + watcher::invocation::Error, + > { + let stream = self.invoke(watch_optional).await?; + Ok(Box::pin(stream)) + } +} + +impl Watcher for ApiWatcher { + type Object = Pod; + + type InvocationError = invocation::Error; + + type StreamError = k8s_stream::Error; + type Stream = BoxStream<'static, Result, Self::StreamError>>; + + fn watch<'a>( + &'a mut self, + watch_optional: WatchOptional<'a>, + ) -> BoxFuture<'a, Result>> + { + Box::pin(self.invoke_boxed_stream(watch_optional)) + } +} + +pub mod invocation { + //! Invocation error. + use super::*; + + /// Errors that can occur while watching. + #[derive(Debug, Snafu)] + #[snafu(visibility(pub))] + pub enum Error { + /// Returned when the call-specific request builder fails. + #[snafu(display("failed to prepare an HTTP request"))] + RequestPreparation { + /// The underlying error. + source: k8s_openapi::RequestError, + }, + + /// Returned when the HTTP client fails to perform an HTTP request. + #[snafu(display("error during the HTTP request"))] + Request { + /// The error that API client retunred. + source: crate::Error, + }, + + /// Returned when the HTTP response has a bad status. + #[snafu(display("HTTP response has a bad status: {}", status))] + BadStatus { + /// The status from the HTTP response. + status: StatusCode, + }, + } + + impl From for watcher::invocation::Error { + fn from(source: Error) -> Self { + watcher::invocation::Error::Other { source } + } + } +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 0e6d046c242ae..a88b2068f2a3a 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -20,6 +20,7 @@ #![cfg(feature = "kubernetes")] #![warn(missing_docs)] +pub mod api_watcher; pub mod client; pub mod multi_response_decoder; pub mod stream; From f02bcb9c8419b48f4941fed45bebf3c7e7be43cf Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 19:00:36 +0300 Subject: [PATCH 011/118] Add MockWatcher implementation Signed-off-by: MOZGIII --- src/kubernetes/mock_watcher.rs | 110 +++++++++++++++++++++++++++++++++ src/kubernetes/mod.rs | 1 + 2 files changed, 111 insertions(+) create mode 100644 src/kubernetes/mock_watcher.rs diff --git a/src/kubernetes/mock_watcher.rs b/src/kubernetes/mock_watcher.rs new file mode 100644 index 0000000000000..2c1ba7e063bf5 --- /dev/null +++ b/src/kubernetes/mock_watcher.rs @@ -0,0 +1,110 @@ +//! A mock watcher. + +#![cfg(test)] + +use super::watcher::{self, Watcher}; +use futures::{ + future::{self, BoxFuture}, + stream::BoxStream, + Stream, +}; +use k8s_openapi::{WatchOptional, WatchResponse}; +use pin_project::pin_project; +use serde::de::DeserializeOwned; +use std::marker::PhantomData; +use std::{ + fmt, + pin::Pin, + task::{Context, Poll}, +}; + +/// A mock watcher, useful for tests. +#[derive(Debug)] +pub struct MockWatcher { + data_type: PhantomData, + invoke_fn: FI, +} + +impl MockWatcher { + /// Create a new [`MockWatcher`]. + pub fn new(invoke_fn: FI) -> Self { + let data_type = PhantomData; + Self { + data_type, + invoke_fn, + } + } +} + +impl Watcher for MockWatcher +where + T: DeserializeOwned + Send + Sync + 'static, + FI: for<'a> FnMut(WatchOptional<'a>) -> Result> + + Send + + 'static, + FS: FnMut() -> Option, StreamError>> + Send + Sync + Unpin + 'static, +{ + type Object = T; + + type StreamError = StreamError; + type Stream = BoxStream<'static, Result, Self::StreamError>>; + + type InvocationError = InvocationError; + + fn watch<'a>( + &'a mut self, + watch_optional: WatchOptional<'a>, + ) -> BoxFuture<'a, Result>> + { + let result = (self.invoke_fn)(watch_optional); + let result = result.map(|stream_fn| { + Box::pin(MockWatcherStream { + stream_fn, + data_type: PhantomData, + }) as Self::Stream + }); + Box::pin(future::ready(result)) + } +} + +#[pin_project] +struct MockWatcherStream { + data_type: PhantomData, + stream_fn: FS, +} + +impl Stream for MockWatcherStream +where + T: DeserializeOwned + Send + Sync, + FS: FnMut() -> Option, StreamError>> + Send + Sync + Unpin + 'static, +{ + type Item = Result, StreamError>; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + let this = self.project(); + Poll::Ready((this.stream_fn)()) + } +} + +/// An error kind for the mock watcher invocation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvocationError; + +/// An error kind for the mock watcher stream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamError; + +impl fmt::Display for InvocationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self, f) + } +} + +impl fmt::Display for StreamError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self, f) + } +} + +impl std::error::Error for InvocationError {} +impl std::error::Error for StreamError {} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index a88b2068f2a3a..9004fb6121a1d 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,6 +22,7 @@ pub mod api_watcher; pub mod client; +pub mod mock_watcher; pub mod multi_response_decoder; pub mod stream; pub mod watcher; From 0fd88b740eac04d024fd96ba8a056d4a8ae3e4c2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 19:10:01 +0300 Subject: [PATCH 012/118] Add a reflector implementation Signed-off-by: MOZGIII --- Cargo.lock | 21 +- Cargo.toml | 3 +- src/kubernetes/hash_value.rs | 72 ++++ src/kubernetes/mod.rs | 3 + src/kubernetes/reflector.rs | 537 +++++++++++++++++++++++++++++ src/kubernetes/resource_version.rs | 92 +++++ 6 files changed, 726 insertions(+), 2 deletions(-) create mode 100644 src/kubernetes/hash_value.rs create mode 100644 src/kubernetes/reflector.rs create mode 100644 src/kubernetes/resource_version.rs diff --git a/Cargo.lock b/Cargo.lock index 4265ff3fb3f25..200ce22afd0e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1108,6 +1108,18 @@ dependencies = [ "smallvec 1.2.0", ] +[[package]] +name = "evmap" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3ea06a83f97d3dc2eb06e51e7a729b418f0717a5558a5c870e3d5156dc558d" +dependencies = [ + "bytes 0.5.4", + "hashbag", + "slab", + "smallvec 1.2.0", +] + [[package]] name = "exitcode" version = "1.1.2" @@ -1494,6 +1506,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "hashbag" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452b31b3ffe7cf13de531eefae493301c85995a953c808d7c79a8f45abae0706" + [[package]] name = "hashbrown" version = "0.6.3" @@ -5485,7 +5503,8 @@ dependencies = [ "derive_is_enum_variant", "dirs", "elastic_responses", - "evmap", + "evmap 10.0.2", + "evmap 7.1.3", "exitcode", "file-source", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 8dc0e3087795e..630752706d4df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ strip-ansi-escapes = { version = "0.1.0"} colored = "1.9" warp = { package = "warp", version = "0.2", default-features = false, optional = true } evmap = { version = "7", features = ["bytes"], optional = true } +evmap10 = { package = "evmap", version = "10", features = ["bytes"], optional = true } logfmt = { version = "0.0.2", optional = true } notify = "4.0.14" once_cell = "1.3" @@ -231,7 +232,7 @@ wasm-timings = ["wasm"] # Enables kubernetes dependencies and shared code. Kubernetes-related sources, # transforms and sinks should depend on this feature. -kubernetes = ["k8s-openapi"] +kubernetes = ["k8s-openapi", "evmap10"] # Sources sources = [ diff --git a/src/kubernetes/hash_value.rs b/src/kubernetes/hash_value.rs new file mode 100644 index 0000000000000..aa3a1955f8e38 --- /dev/null +++ b/src/kubernetes/hash_value.rs @@ -0,0 +1,72 @@ +//! A wrapper to implement hash for k8s resource objects. + +use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; +use std::hash::{Hash, Hasher}; +use std::ops::Deref; + +/// A wrapper that provdies a [`Hash`] implementation for any k8s resource +/// object. +/// Delegates to object uid for hashing and equality. +#[derive(Debug)] +pub struct HashValue>(T); + +impl HashValue +where + T: Metadata, +{ + /// Create a new [`HashValue`] by wrapping a value of `T`. + pub fn new(value: T) -> Self { + Self(value) + } + + /// Get the `uid` from the `T`'s [`Metadata`] (if any). + pub fn uid(&self) -> Option<&str> { + let ObjectMeta { ref uid, .. } = self.0.metadata()?; + let uid = uid.as_ref()?; + Some(uid.as_str()) + } +} + +impl PartialEq for HashValue +where + T: Metadata, +{ + fn eq(&self, other: &Self) -> bool { + match (self.uid(), other.uid()) { + (Some(a), Some(b)) => a.eq(b), + (None, None) => true, + _ => false, + } + } +} + +impl Eq for HashValue where T: Metadata {} + +impl Hash for HashValue +where + T: Metadata, +{ + fn hash(&self, state: &mut H) { + self.uid().hash(state) + } +} + +impl Deref for HashValue +where + T: Metadata, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for HashValue +where + T: Metadata, +{ + fn as_ref(&self) -> &T { + &self.0 + } +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 9004fb6121a1d..e6070b0d8459c 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,7 +22,10 @@ pub mod api_watcher; pub mod client; +pub mod hash_value; pub mod mock_watcher; pub mod multi_response_decoder; +pub mod reflector; +pub mod resource_version; pub mod stream; pub mod watcher; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs new file mode 100644 index 0000000000000..96d7301f857b4 --- /dev/null +++ b/src/kubernetes/reflector.rs @@ -0,0 +1,537 @@ +//! Watch and cache the remote Kubernetes API resources. + +use super::{ + hash_value::HashValue, + resource_version, + watcher::{self, Watcher}, +}; +use evmap10::WriteHandle; +use futures::{pin_mut, stream::StreamExt}; +use k8s_openapi::{ + apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, + Metadata, WatchOptional, WatchResponse, +}; +use snafu::Snafu; +use std::convert::Infallible; +use std::time::Duration; +use tokio::time::delay_for; + +/// Watches remote Kubernetes resources and maintains a local representation of +/// the remote state. "Reflects" the remote state locally. +/// +/// Does not expose evented API, but keeps track of the resource versions and +/// will automatically resume on desync. +pub struct Reflector +where + W: Watcher, + ::Object: Metadata, +{ + watcher: W, + state_writer: WriteHandle::Object>>, + field_selector: Option, + label_selector: Option, + resource_version: resource_version::State, + pause_between_requests: Duration, +} + +impl Reflector +where + W: Watcher, + ::Object: Metadata, +{ + /// Create a new [`Cache`]. + pub fn new( + watcher: W, + state_writer: WriteHandle::Object>>, + field_selector: Option, + label_selector: Option, + pause_between_requests: Duration, + ) -> Self { + let resource_version = resource_version::State::new(); + Self { + watcher, + state_writer, + label_selector, + field_selector, + resource_version, + pause_between_requests, + } + } +} + +impl Reflector +where + W: Watcher, + ::Object: Metadata + Unpin + std::fmt::Debug, + ::InvocationError: Unpin, + ::StreamError: Unpin, +{ + /// Run the watch loop and drive the state updates via `state_writer`. + pub async fn run(&mut self) -> Result> { + // We're taking over the `self.state_writer`, clear it. + self.state_writer.purge(); + // Propagate the purge and ensure we initialize readers for the first + // time. + self.state_writer.refresh(); + + // Start the watch loop. + loop { + let invocation_result = self.issue_request().await; + let stream = match invocation_result { + Ok(val) => val, + Err(watcher::invocation::Error::Desync { source }) => { + warn!(message = "handling desync", error = ?source); + // We got desynced, reset the state and retry fetching. + // By omiting the flush here, we cache the results from the + // previous run until flush is issued when the new events + // begin arriving, reducing the time durig which the state + // has no data. + self.resource_version.reset(); + self.state_writer.purge(); + continue; + } + Err(watcher::invocation::Error::Other { source }) => { + // Not a desync, fail everything. + error!(message = "watcher error", error = ?source); + return Err(Error::Invocation { source }); + } + }; + + pin_mut!(stream); + while let Some(response) = stream.next().await { + // Any streaming error means the protocol is in an unxpected + // state. This is considered a fatal error, do not attempt + // to retry and just quit. + let response = response.map_err(|source| Error::Streaming { source })?; + + // Unpack the event. + let event = match response { + WatchResponse::Ok(event) => event, + WatchResponse::Other(_) => { + // Even though we could parse the response, we didn't + // get the data we expected on the wire. + // According to the rules, we just ignore the unknown + // responses. This may be a newly added piece of data + // our code doesn't know of. + // TODO: add more details on the data here if we + // encounter these messages in practice. + warn!(message = "got unexpected data in the watch response"); + continue; + } + }; + + // Prepare a resource version cvandidate so we can update (aka + // commit) it later. + let resource_version_candidate = + match resource_version::Candidate::from_watch_event(&event) { + Some(val) => val, + None => { + // This event doesn't have a resource version, this means + // it's not something we care about. + continue; + } + }; + + // Process the event. + self.process_event(event); + + // Record the resourse version for this event, so when we resume + // it won't be redelivered. + self.resource_version.update(resource_version_candidate); + + // Flush the changes to the state. + self.state_writer.flush(); + } + + // For the next pause duration we won't get any updates. + // This is better than flooding k8s api server with requests. + delay_for(self.pause_between_requests).await; + } + } + + /// Prepare and execute a watch request. + async fn issue_request( + &mut self, + ) -> Result<::Stream, watcher::invocation::Error<::InvocationError>> + { + let watch_optional = WatchOptional { + field_selector: self.field_selector.as_ref().map(|s| s.as_str()), + label_selector: self.label_selector.as_ref().map(|s| s.as_str()), + pretty: None, + resource_version: self.resource_version.get(), + timeout_seconds: None, + allow_watch_bookmarks: Some(true), + }; + let stream = self.watcher.watch(watch_optional).await?; + Ok(stream) + } + + /// Translate received watch event to the state update. + fn process_event(&mut self, event: WatchEvent<::Object>) { + match event { + WatchEvent::Added(object) => { + if let Some((key, value)) = kv(object) { + trace!(message = "got an object event", uid = ?key, event = "added"); + self.state_writer.insert(key, value); + } else { + warn!( + message = "got an object event but unable to obtain a key from object", + event = "added", + ); + } + } + WatchEvent::Deleted(object) => { + if let Some((key, _value)) = kv(object) { + trace!(message = "got an object event", uid = ?key, event = "deleted"); + self.state_writer.empty(key); + } else { + warn!( + message = "got an object event but unable to obtain a key from object", + event = "deleted", + ); + } + } + WatchEvent::Modified(object) => { + if let Some((key, value)) = kv(object) { + trace!(message = "got an object event", uid = ?key, event = "modified"); + self.state_writer.update(key, value); + } else { + warn!( + message = "got an object event but unable to obtain a key from object", + event = "modified", + ); + } + } + WatchEvent::Bookmark(_object) => { + // noop + trace!(message = "got an object event", event = "bookmark"); + } + _ => unreachable!("other event types should never reach this code"), + } + } +} + +/// Errors that can occur while watching. +#[derive(Debug, Snafu)] +pub enum Error +where + W: Watcher, +{ + /// Returned when the watch invocation (HTTP request) failed. + #[snafu(display("watch invocation failed"))] + Invocation { + /// The underlying invocation error. + source: ::InvocationError, + }, + + /// Returned when the stream failed with an error. + #[snafu(display("streaming error"))] + Streaming { + /// The underlying stream error. + source: ::StreamError, + }, +} + +/// An alias to the value used at [`evmap`]. +pub type Value = Box>; + +/// Build a key value pair for using in [`evmap`]. +fn kv>(object: T) -> Option<(String, Value)> { + let value = Box::new(HashValue::new(object)); + let key = value.uid()?.to_owned(); + Some((key, value)) +} + +#[cfg(test)] +mod tests { + use super::{Reflector, Value}; + use crate::{ + kubernetes::mock_watcher::{InvocationError, MockWatcher}, + kubernetes::watcher, + test_util, + }; + use k8s_openapi::{ + api::core::v1::Pod, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, + Metadata, WatchOptional, WatchResponse, + }; + use std::time::Duration; + + /// A helper function to simplify assertion on the `evmap` state. + fn gather_state(handle: &evmap10::ReadHandle>) -> Vec + where + T: Metadata + Clone, + { + let mut vec: Vec<(String, T)> = handle + .read() + .expect("expected read to be ready") + .iter() + .map(|(key, values)| { + assert_eq!(values.len(), 1); + let value = values.get_one().unwrap(); + (key.clone(), value.as_ref().as_ref().to_owned()) + }) + .collect(); + + // Sort the results by key for consistent assertions. + vec.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + + // Discard keys. + vec.into_iter().map(|(_, value)| value).collect() + } + + // A helper to build a pod object for test purposes. + fn make_pod(uid: &str, resource_version: &str) -> Pod { + Pod { + metadata: Some(ObjectMeta { + uid: Some(uid.to_owned()), + resource_version: Some(resource_version.to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + } + } + + // A helper to build a bookmark pod object. + // See https://github.com/kubernetes/enhancements/blob/e565a82680bb8e05836530ebd1abac723aab40e2/keps/sig-api-machinery/20190206-watch-bookmark.md + fn make_pod_bookmark(resource_version: &str) -> Pod { + Pod { + metadata: Some(ObjectMeta { + resource_version: Some(resource_version.to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + } + } + + // A helper enum to encode expected mock watcher invocation. + enum ExpInvRes { + Stream(Vec>), + Desync, + } + + // A simple test, to serve as a bare-bones example for adding further tests. + #[test] + fn simple_test() { + test_util::trace_init(); + + // Prepare the test flow. + let (_state_reader, state_writer) = evmap10::new(); + let mock_logic = move |_watch_optional: WatchOptional<'_>| { + if false { + return Ok(|| None); // for type inferrence + } + Err(watcher::invocation::Error::other(InvocationError)) + }; + + // Prepare watcher and reflector. + let watcher: MockWatcher = MockWatcher::new(mock_logic); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + + // Acquire an async context to run the relector. + test_util::block_on_std(async move { + // Run the test and wait for an error. + let result = reflector.run().await; + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }); + } + + // A helper function to run a flow test. + // Use this to test various flows without an actual test repetition. + // TODO: add intermediate state assertions to validate caching properties. + fn run_flow_test( + invocations: Vec<(Option, ExpInvRes)>, + expected_resulting_state: Vec, + ) { + // Prepare the test flow. + let (state_reader, state_writer) = evmap10::new(); + + let assertion_state_reader = state_reader.clone(); + let flow_expected_resulting_state = expected_resulting_state.clone(); + + let mut flow_invocations = invocations.into_iter(); + + let mock_logic = move |watch_optional: WatchOptional<'_>| { + if let Some((expected_resource_version, expected_invocation_response)) = + flow_invocations.next() + { + assert_eq!( + expected_resource_version, + watch_optional.resource_version.map(ToOwned::to_owned) // work around the borrow checker issues + ); + + let responses = match expected_invocation_response { + ExpInvRes::Stream(responses) => responses, + ExpInvRes::Desync => { + return Err(watcher::invocation::Error::desync(InvocationError)) + } + }; + + let mut responses_iter = responses.into_iter(); + return Ok(move || responses_iter.next().map(|val| Ok(WatchResponse::Ok(val)))); + } + + let resulting_state: Vec = gather_state(&assertion_state_reader); + assert_eq!(resulting_state, flow_expected_resulting_state); + + Err(watcher::invocation::Error::other(InvocationError)) + }; + + // Prepare watcher and reflector. + let watcher: MockWatcher = MockWatcher::new(mock_logic); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + + // Acquire an async context to run the relector. + test_util::block_on_std(async move { + // Run the test and wait for an error. + let result = reflector.run().await; + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Assert the state after the reflector exit. + let resulting_state: Vec = gather_state(&state_reader); + assert_eq!(resulting_state, expected_resulting_state); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }); + } + + // Test the properties of the normal execution flow. + #[test] + fn flow_test() { + test_util::trace_init(); + + let invocations = vec![ + ( + None, + ExpInvRes::Stream(vec![ + WatchEvent::Added(make_pod("uid0", "10")), + WatchEvent::Added(make_pod("uid1", "15")), + ]), + ), + ( + Some("15".to_owned()), + ExpInvRes::Stream(vec![ + WatchEvent::Modified(make_pod("uid0", "20")), + WatchEvent::Added(make_pod("uid2", "25")), + ]), + ), + ( + Some("25".to_owned()), + ExpInvRes::Stream(vec![WatchEvent::Bookmark(make_pod_bookmark("50"))]), + ), + ( + Some("50".to_owned()), + ExpInvRes::Stream(vec![ + WatchEvent::Deleted(make_pod("uid2", "55")), + WatchEvent::Modified(make_pod("uid0", "60")), + ]), + ), + ]; + let expected_resulting_state = vec![make_pod("uid0", "60"), make_pod("uid1", "15")]; + + // Use standard flow test logic. + run_flow_test(invocations, expected_resulting_state); + } + + // Test the properies of the flow with desync. + #[test] + fn desync_test() { + test_util::trace_init(); + + let invocations = vec![ + ( + None, + ExpInvRes::Stream(vec![ + WatchEvent::Added(make_pod("uid0", "10")), + WatchEvent::Added(make_pod("uid1", "15")), + ]), + ), + (Some("15".to_owned()), ExpInvRes::Desync), + ( + None, + ExpInvRes::Stream(vec![ + WatchEvent::Added(make_pod("uid20", "1000")), + WatchEvent::Added(make_pod("uid21", "1005")), + ]), + ), + ( + Some("1005".to_owned()), + ExpInvRes::Stream(vec![WatchEvent::Modified(make_pod("uid21", "1010"))]), + ), + ]; + let expected_resulting_state = vec![make_pod("uid20", "1000"), make_pod("uid21", "1010")]; + + // Use standard flow test logic. + run_flow_test(invocations, expected_resulting_state); + } + + /// Test that the state is properly initialized even if no events arrived. + #[test] + fn no_updates_state_test() { + test_util::trace_init(); + + let invocations = vec![]; + let expected_resulting_state = vec![]; + + // Use standard flow test logic. + run_flow_test(invocations, expected_resulting_state); + } + + // Test that [`WatchOptional`] is populated properly. + #[test] + fn arguments_test() { + test_util::trace_init(); + + let (_state_reader, state_writer) = evmap10::new(); + let mock_logic = move |watch_optional: WatchOptional<'_>| { + assert_eq!(watch_optional.field_selector, Some("fields")); + assert_eq!(watch_optional.label_selector, Some("labels")); + assert_eq!(watch_optional.allow_watch_bookmarks, Some(true)); + assert_eq!(watch_optional.pretty, None); + assert_eq!(watch_optional.timeout_seconds, None); + + if false { + return Ok(|| None); // for type inferrence + } + Err(watcher::invocation::Error::other(InvocationError)) + }; + + let watcher: MockWatcher = MockWatcher::new(mock_logic); + + let mut reflector = Reflector::new( + watcher, + state_writer, + Some("fields".to_owned()), + Some("labels".to_owned()), + Duration::from_secs(1), + ); + + // Acquire an async context to run the relector. + test_util::block_on_std(async move { + // Run the test and wait for an error. + let result = reflector.run().await; + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }); + } +} diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs new file mode 100644 index 0000000000000..8af302f4295c8 --- /dev/null +++ b/src/kubernetes/resource_version.rs @@ -0,0 +1,92 @@ +//! Shared state bits for watch implementations. + +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}; +use k8s_openapi::Metadata; +use std::ops::Deref; + +/// Resource version state in the context of a chain of watch requests. +#[derive(Debug, Clone)] +pub struct State(Option); + +impl State { + /// Create a new resource version [`State`]. + pub fn new() -> Self { + Self(None) + } + + /// Update the resource version from a candidate obtained earlier. + pub fn update(&mut self, candidate: Candidate) { + self.0 = Some(candidate.0); + } + + /// Reset the resource version. Use in case of a desync. + pub fn reset(&mut self) { + self.0 = None; + } + + /// Get the current resource version value. + pub fn get(&self) -> Option<&str> { + self.into() + } +} + +impl Deref for State { + type Target = Option; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> From<&'a State> for Option<&'a str> { + fn from(val: &'a State) -> Self { + match val.0 { + Some(ref val) => Some(val.as_str()), + None => None, + } + } +} + +/// A resource version candidate, can be used to update the resource version. +pub struct Candidate(String); + +impl Candidate { + /// Obtain a resource version [`Candidate`] from a [`WatchEvent`]. + pub fn from_watch_event(event: &WatchEvent) -> Option + where + T: Metadata, + { + let object = match event { + WatchEvent::Added(object) + | WatchEvent::Modified(object) + | WatchEvent::Deleted(object) + | WatchEvent::Bookmark(object) => object, + WatchEvent::ErrorStatus(_) | WatchEvent::ErrorOther(_) => return None, + }; + Self::from_object(object) + } + + /// Obtain a resource version [`Candidate`] from a object of type `T`. + pub fn from_object(object: &T) -> Option + where + T: Metadata, + { + let metadata = match object.metadata() { + Some(val) => val, + None => { + warn!(message = "Got k8s object without metadata"); + return None; + } + }; + + let new_resource_version = match metadata.resource_version { + Some(ref val) => val, + None => { + warn!(message = "Got empty resource version at object metadata"); + return None; + } + }; + + Some(Self(new_resource_version.clone())) + } +} From 1b472f9908325a5edadf68d0cc93c8309c84fc4e Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 19:18:35 +0300 Subject: [PATCH 013/118] Add a placeholder for kubernetes logs source Signed-off-by: MOZGIII --- Cargo.toml | 4 +++- src/sources/kubernetes_logs/mod.rs | 5 +++++ src/sources/mod.rs | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 src/sources/kubernetes_logs/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 630752706d4df..24a67dd8abff0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,7 @@ sources = [ "sources-syslog", "sources-tls", "sources-vector", + "sources-kubernetes-logs", ] sources-docker = ["bollard"] sources-file = ["bytesize"] @@ -269,6 +270,7 @@ sources-stdin = ["bytesize"] sources-syslog = ["sources-socket", "syslog_loose"] sources-tls = ["sources-http", "sources-logplex", "sources-socket", "sources-splunk_hec"] sources-vector = ["sources-socket"] +sources-kubernetes-logs = ["kubernetes"] # Transforms transforms = [ @@ -431,7 +433,7 @@ kafka-integration-tests = ["sources-kafka", "sinks-kafka"] loki-integration-tests = ["sinks-loki"] pulsar-integration-tests = ["sinks-pulsar"] splunk-integration-tests = ["sinks-splunk_hec", "warp"] -kubernetes-integration-tests = [] +kubernetes-integration-tests = ["sources-kubernetes-logs"] shutdown-tests = ["sources","sinks-console","sinks-prometheus","sinks-blackhole","unix","rdkafka","transforms-log_to_metric","transforms-lua"] disable-resolv-conf = [] diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs new file mode 100644 index 0000000000000..4f3d04a7f512e --- /dev/null +++ b/src/sources/kubernetes_logs/mod.rs @@ -0,0 +1,5 @@ +//! This mod implements `kubernetes_logs` source. +//! The scope of this source is to consume the log files that `kubelet` keeps +//! at `/var/log/pods` at the host of the k8s node. + +#![deny(missing_docs)] diff --git a/src/sources/mod.rs b/src/sources/mod.rs index f6f597caf44aa..a2d5ee4e2dac9 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -15,6 +15,8 @@ pub mod internal_metrics; pub mod journald; #[cfg(all(feature = "sources-kafka", feature = "rdkafka"))] pub mod kafka; +#[cfg(feature = "sources-kubernetes-logs")] +pub mod kubernetes_logs; #[cfg(feature = "sources-logplex")] pub mod logplex; #[cfg(feature = "sources-prometheus")] From f4f23f9d0b8e4af69a241832a5bb3fb411c4901d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 21:49:56 +0300 Subject: [PATCH 014/118] Add paths provider implementation based on pods state driven by reflector Signed-off-by: MOZGIII --- .../kubernetes_logs/k8s_paths_provider.rs | 230 ++++++++++++++++++ src/sources/kubernetes_logs/mod.rs | 3 + src/sources/kubernetes_logs/path_helpers.rs | 113 +++++++++ 3 files changed, 346 insertions(+) create mode 100644 src/sources/kubernetes_logs/k8s_paths_provider.rs create mode 100644 src/sources/kubernetes_logs/path_helpers.rs diff --git a/src/sources/kubernetes_logs/k8s_paths_provider.rs b/src/sources/kubernetes_logs/k8s_paths_provider.rs new file mode 100644 index 0000000000000..ee3d48d644f45 --- /dev/null +++ b/src/sources/kubernetes_logs/k8s_paths_provider.rs @@ -0,0 +1,230 @@ +//! A paths provider for k8s logs. + +#![deny(missing_docs)] + +use super::path_helpers::build_pod_logs_directory; +use crate::kubernetes as k8s; +use evmap10::ReadHandle; +use file_source::paths_provider::PathsProvider; +use k8s_openapi::api::core::v1::Pod; +use std::path::PathBuf; + +/// A paths provider implementation that uses the state obtained from the +/// the k8s API. +pub struct K8sPathsProvider { + pods_state_reader: ReadHandle>, +} + +impl K8sPathsProvider { + /// Create a new [`K8sPathsProvider`]. + pub fn new(pods_state_reader: ReadHandle>) -> Self { + Self { pods_state_reader } + } +} + +impl PathsProvider for K8sPathsProvider { + type IntoIter = Vec; + + fn paths(&self) -> Vec { + let read_ref = match self.pods_state_reader.read() { + Some(v) => v, + None => { + // The state is not initialized or gone, fallback to using an + // empty array. + // TODO: consider `panic`ing here instead - fail-fast appoach + // is always better if possible, but it's not clear if it's + // a sane strategy here. + warn!(message = "unable to read the state of the pods"); + return Vec::new(); + } + }; + + read_ref + .into_iter() + .flat_map(|(uid, values)| { + let pod = values + .get_one() + .expect("we are supposed to be working with single-item values only"); + trace!(message = "providing log paths for pod", ?uid); + list_pod_log_paths(real_glob, pod) + }) + .collect() + } +} + +fn extract_pod_logs_directory(pod: &Pod) -> Option { + let metadata = pod.metadata.as_ref()?; + let namespace = metadata.namespace.as_ref()?; + let name = metadata.name.as_ref()?; + let uid = metadata.uid.as_ref()?; + Some(build_pod_logs_directory(&namespace, &name, &uid)) +} + +fn list_pod_log_paths<'a, G, GI>(mut glob_impl: G, pod: &Pod) -> impl Iterator + 'a +where + G: FnMut(&str) -> GI + 'a, + GI: Iterator + 'a, +{ + extract_pod_logs_directory(pod) + .into_iter() + .flat_map(move |dir| { + glob_impl( + &[ + dir.to_str() + .expect("non-utf8 path to pod logs dir is not supported"), + "**/*.log", + ] + .join("/"), + ) + }) +} + +fn real_glob(pattern: &str) -> impl Iterator { + glob::glob(pattern) + .expect("the pattern is supposed to always be correct") + .flat_map(|paths| paths.into_iter()) +} + +#[cfg(test)] +mod tests { + use super::{extract_pod_logs_directory, list_pod_log_paths}; + use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; + use std::path::PathBuf; + + #[test] + fn test_extract_pod_logs_directory() { + let cases = vec![ + (Pod::default(), None), + ( + Pod { + metadata: Some(ObjectMeta::default()), + ..Pod::default() + }, + None, + ), + ( + Pod { + metadata: Some(ObjectMeta { + namespace: Some("sandbox0-ns".to_owned()), + name: Some("sandbox0-name".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + Some("/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid"), + ), + ( + Pod { + metadata: Some(ObjectMeta { + namespace: Some("sandbox0-ns".to_owned()), + name: Some("sandbox0-name".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + None, + ), + ( + Pod { + metadata: Some(ObjectMeta { + namespace: Some("sandbox0-ns".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + None, + ), + ( + Pod { + metadata: Some(ObjectMeta { + name: Some("sandbox0-name".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + None, + ), + ]; + + for (pod, expected) in cases { + assert_eq!( + extract_pod_logs_directory(&pod), + expected.map(PathBuf::from) + ); + } + } + + #[test] + fn test_list_pod_log_paths() { + let cases = vec![ + // Pod exists and has some containers that write logs. + ( + Pod { + metadata: Some(ObjectMeta { + namespace: Some("sandbox0-ns".to_owned()), + name: Some("sandbox0-name".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + // Calls to the glob mock. + vec![( + // The pattern to expect at the mock. + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/**/*.log", + // The paths to return from the mock. + vec![ + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container1/qwe.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container2/qwe.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container3/qwe.log", + ], + )], + // Expected result. + vec![ + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container1/qwe.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container2/qwe.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container3/qwe.log", + ], + ), + // Pod doesn't have the metadata set. + (Pod::default(), vec![], vec![]), + // Pod has proper metadata, but doesn't have log files. + ( + Pod { + metadata: Some(ObjectMeta { + namespace: Some("sandbox0-ns".to_owned()), + name: Some("sandbox0-name".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + }, + vec![( + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/**/*.log", + vec![], + )], + vec![], + ), + ]; + + for (pod, expected_calls, expected_paths) in cases { + // Prepare the mock fn. + let mut expected_calls = expected_calls.into_iter(); + let mock_glob = move |pattern: &str| { + let (expected_pattern, paths_to_return) = expected_calls + .next() + .expect("implementation did a call that wasn't expected"); + + assert_eq!(pattern, expected_pattern); + paths_to_return.into_iter().map(PathBuf::from) + }; + + let actual_paths: Vec<_> = list_pod_log_paths(mock_glob, &pod).collect(); + let expeced_paths: Vec<_> = expected_paths.into_iter().map(PathBuf::from).collect(); + assert_eq!(actual_paths, expeced_paths) + } + } +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 4f3d04a7f512e..b174a59cc22a5 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -3,3 +3,6 @@ //! at `/var/log/pods` at the host of the k8s node. #![deny(missing_docs)] + +mod k8s_paths_provider; +mod path_helpers; diff --git a/src/sources/kubernetes_logs/path_helpers.rs b/src/sources/kubernetes_logs/path_helpers.rs new file mode 100644 index 0000000000000..3458776b7f0ba --- /dev/null +++ b/src/sources/kubernetes_logs/path_helpers.rs @@ -0,0 +1,113 @@ +//! Simple helpers for building and parsing k8s paths. +//! +//! Loosely based on https://github.com/kubernetes/kubernetes/blob/31305966789525fca49ec26c289e565467d1f1c4/pkg/kubelet/kuberuntime/helpers.go + +#![deny(missing_docs)] + +use std::path::PathBuf; + +/// The root directory for pod logs. +const K8S_LOGS_DIR: &str = "/var/log/pods"; + +/// The delimiter used in the log path. +const LOG_PATH_DELIMITER: &str = "_"; + +/// Builds absolute log directory path for a pod sandbox. +/// +/// Based on https://github.com/kubernetes/kubernetes/blob/31305966789525fca49ec26c289e565467d1f1c4/pkg/kubelet/kuberuntime/helpers.go#L178 +pub(super) fn build_pod_logs_directory( + pod_namespace: &str, + pod_name: &str, + pod_uid: &str, +) -> PathBuf { + [ + K8S_LOGS_DIR, + &[pod_namespace, pod_name, pod_uid].join(LOG_PATH_DELIMITER), + ] + .iter() + .collect() +} + +/// Parses pod log file path and returns the log file info. +/// +/// Assumes the input is a valid pod log file name. +/// +/// Inspired by https://github.com/kubernetes/kubernetes/blob/31305966789525fca49ec26c289e565467d1f1c4/pkg/kubelet/kuberuntime/helpers.go#L186 +pub(super) fn parse_log_file_path(path: &str) -> Option> { + let mut componenets = path.rsplit("/"); + + let _log_file_name = componenets.next()?; + let container_name = componenets.next()?; + let pod_dir = componenets.next()?; + + let mut pod_dir_componenets = pod_dir.rsplit(LOG_PATH_DELIMITER); + + let pod_uid = pod_dir_componenets.next()?; + let pod_name = pod_dir_componenets.next()?; + let pod_namespace = pod_dir_componenets.next()?; + + Some(LogFileInfo { + pod_namespace, + pod_name, + pod_uid, + container_name, + }) +} + +/// Contains the information extracted from the pod log file path. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LogFileInfo<'a> { + pub pod_namespace: &'a str, + pub pod_name: &'a str, + pub pod_uid: &'a str, + pub container_name: &'a str, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_pod_logs_directory() { + let cases = vec![ + // Valid inputs. + ( + ("sandbox0-ns", "sandbox0-name", "sandbox0-uid"), + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid", + ), + // Invalid inputs. + (("", "", ""), "/var/log/pods/__"), + ]; + + for ((in_namespace, in_name, in_uid), expected) in cases.into_iter() { + assert_eq!( + build_pod_logs_directory(in_namespace, in_name, in_uid), + PathBuf::from(expected) + ); + } + } + + #[test] + fn test_parse_log_file_path() { + let cases = vec![ + // Valid inputs. + ( + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/sandbox0-container0-name/1.log", + Some(LogFileInfo { + pod_namespace: "sandbox0-ns", + pod_name: "sandbox0-name", + pod_uid: "sandbox0-uid", + container_name: "sandbox0-container0-name", + }), + ), + // Invalid inputs. + ("/var/log/pods/other", None), + ("qwe", None), + ("", None), + ]; + + for (input, expected) in cases.into_iter() { + assert_eq!(parse_log_file_path(input), expected); + } + } +} From d0fbebb6d6857e0199b429e10895328e9aed3695 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 22:00:51 +0300 Subject: [PATCH 015/118] Add parser for k8s log formats Signed-off-by: MOZGIII --- Cargo.toml | 2 +- src/event/mod.rs | 4 +- src/sources/kubernetes_logs/mod.rs | 1 + src/sources/kubernetes_logs/parser/cri.rs | 138 ++++++++++++++++++ src/sources/kubernetes_logs/parser/docker.rs | 133 +++++++++++++++++ src/sources/kubernetes_logs/parser/mod.rs | 17 +++ .../kubernetes_logs/parser/test_util.rs | 46 ++++++ src/transforms/mod.rs | 2 +- src/transforms/util/chain.rs | 23 +++ src/transforms/util/mod.rs | 7 + src/transforms/util/optional.rs | 20 +++ src/transforms/util/pick.rs | 121 +++++++++++++++ 12 files changed, 511 insertions(+), 3 deletions(-) create mode 100644 src/sources/kubernetes_logs/parser/cri.rs create mode 100644 src/sources/kubernetes_logs/parser/docker.rs create mode 100644 src/sources/kubernetes_logs/parser/mod.rs create mode 100644 src/sources/kubernetes_logs/parser/test_util.rs create mode 100644 src/transforms/util/chain.rs create mode 100644 src/transforms/util/optional.rs create mode 100644 src/transforms/util/pick.rs diff --git a/Cargo.toml b/Cargo.toml index 24a67dd8abff0..5e76b173aa415 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -270,7 +270,7 @@ sources-stdin = ["bytesize"] sources-syslog = ["sources-socket", "syslog_loose"] sources-tls = ["sources-http", "sources-logplex", "sources-socket", "sources-splunk_hec"] sources-vector = ["sources-socket"] -sources-kubernetes-logs = ["kubernetes"] +sources-kubernetes-logs = ["kubernetes", "transforms-merge", "transforms-json_parser", "transforms-regex_parser"] # Transforms transforms = [ diff --git a/src/event/mod.rs b/src/event/mod.rs index 5d00940ca1d8a..1fbcbed860292 100644 --- a/src/event/mod.rs +++ b/src/event/mod.rs @@ -26,8 +26,10 @@ pub mod proto { pub static LOG_SCHEMA: OnceCell = OnceCell::new(); +pub const PARTIAL_STR: &str = "_partial"; // TODO: clean up the _STR suffix after we get rid of atoms + lazy_static! { - pub static ref PARTIAL: Atom = Atom::from("_partial"); + pub static ref PARTIAL: Atom = Atom::from(PARTIAL_STR); static ref LOG_SCHEMA_DEFAULT: LogSchema = LogSchema { message_key: Atom::from("message"), timestamp_key: Atom::from("timestamp"), diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index b174a59cc22a5..2b5bab17d49b7 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -5,4 +5,5 @@ #![deny(missing_docs)] mod k8s_paths_provider; +mod parser; mod path_helpers; diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs new file mode 100644 index 0000000000000..ad05c999615ed --- /dev/null +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -0,0 +1,138 @@ +use crate::{ + event::{self, Event, LogEvent, Value}, + transforms::{ + regex_parser::{RegexParser, RegexParserConfig}, + Transform, + }, +}; +use lazy_static::lazy_static; +use snafu::{OptionExt, Snafu}; +use string_cache::DefaultAtom as Atom; + +pub fn build() -> Box { + Box::new(Cri::new()) +} + +lazy_static! { + pub static ref MULTILINE_TAG: Atom = Atom::from("multiline_tag"); +} + +/// Parser for the CRI log format. +/// +/// Expects logs to arrive in a CRI log format. +/// +/// CRI log format ([documentation][cri_log_format]) is a simple +/// newline-separated text format. We rely on regular expressions to parse it. +/// +/// Normalizes parsed data for consistency. +/// +/// [cri_log_format]: https://github.com/kubernetes/community/blob/ee2abbf9dbfa4523b414f99a04ddc97bd38c74b2/contributors/design-proposals/node/kubelet-cri-logging.md +pub struct Cri { + // TODO: patch `RegexParser` to expose the concerete type on build. + regex_parser: Box, +} + +impl Cri { + /// Create a new [`Cri`] parser. + pub fn new() -> Self { + let regex_parser = { + let mut rp_config = RegexParserConfig::default(); + + let pattern = r"^(?P.*) (?P(stdout|stderr)) (?P(P|F)) (?P.*)$"; + rp_config.patterns = vec![pattern.to_owned()]; + + rp_config.types.insert( + event::log_schema().timestamp_key().clone(), + "timestamp|%+".to_owned(), + ); + + RegexParser::build(&rp_config).expect("regexp patterns are static, should never fail") + }; + + Self { regex_parser } + } +} + +impl Transform for Cri { + fn transform(&mut self, event: Event) -> Option { + let mut event = self.regex_parser.transform(event)?; + normalize_event(event.as_mut_log()).ok()?; + Some(event) + } +} + +fn normalize_event(log: &mut LogEvent) -> Result<(), NormalizationError> { + // Detect if this is a partial event. + let multiline_tag = log + .remove(&MULTILINE_TAG) + .context(MultilineTagFieldMissing)?; + let multiline_tag = match multiline_tag { + Value::Bytes(val) => val, + _ => return Err(NormalizationError::MultilineTagValueUnexpectedType), + }; + + let is_partial = multiline_tag[0] == b'P'; + + // For partial messages add a partial event indicator. + if is_partial { + log.insert(event::PARTIAL_STR, true); + } + + Ok(()) +} + +#[derive(Debug, Snafu)] +enum NormalizationError { + MultilineTagFieldMissing, + MultilineTagValueUnexpectedType, +} + +#[cfg(test)] +mod tests { + use super::super::test_util; + use super::*; + + #[test] + fn test_parsing() { + let cases = vec![ + ( + "2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1", + test_util::make_log_event( + "The content of the log entry 1", + "2016-10-06T00:17:09.669794202Z", + "stdout", + false, + ), + ), + ( + "2016-10-06T00:17:09.669794202Z stdout P First line of log entry 2", + test_util::make_log_event( + "First line of log entry 2", + "2016-10-06T00:17:09.669794202Z", + "stdout", + true, + ), + ), + ( + "2016-10-06T00:17:09.669794202Z stdout P Second line of the log entry 2", + test_util::make_log_event( + "Second line of the log entry 2", + "2016-10-06T00:17:09.669794202Z", + "stdout", + true, + ), + ), + ( + "2016-10-06T00:17:10.113242941Z stderr F Last line of the log entry 2", + test_util::make_log_event( + "Last line of the log entry 2", + "2016-10-06T00:17:10.113242941Z", + "stderr", + false, + ), + ), + ]; + + test_util::test_parser(build, cases); + } +} diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs new file mode 100644 index 0000000000000..d8f1bf2bae450 --- /dev/null +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -0,0 +1,133 @@ +use crate::{ + event::{self, Event, LogEvent, Value}, + transforms::{ + json_parser::{JsonParser, JsonParserConfig}, + Transform, + }, +}; +use chrono::{DateTime, Utc}; +use lazy_static::lazy_static; +use snafu::{OptionExt, ResultExt, Snafu}; +use string_cache::DefaultAtom as Atom; + +pub fn build() -> Box { + Box::new(Docker::new()) +} + +lazy_static! { + pub static ref TIME: Atom = Atom::from("time"); + pub static ref LOG: Atom = Atom::from("log"); +} + +/// Parser for the docker log format. +/// +/// Expects logs to arrive in a JSONLines format with the fields names and +/// contents specific to the implementation of the Docker `json` log driver. +/// +/// Normalizes parsed data for consistency. +#[derive(Debug)] +pub struct Docker { + json_parser: JsonParser, +} + +impl Docker { + /// Create a new [`Docker`] parser. + pub fn new() -> Self { + let json_parser = { + let mut config = JsonParserConfig::default(); + config.drop_field = true; + + // Drop so that it's possible to detect if message is in json format. + config.drop_invalid = true; + + config.into() + }; + + Self { json_parser } + } +} + +impl Transform for Docker { + fn transform(&mut self, event: Event) -> Option { + let mut event = self.json_parser.transform(event)?; + normalize_event(event.as_mut_log()).ok()?; + Some(event) + } +} + +fn normalize_event(log: &mut LogEvent) -> Result<(), NormalizationError> { + // Parse and rename timestamp. + let time = log.remove(&TIME).context(TimeFieldMissing)?; + let time = match time { + Value::Bytes(val) => val, + _ => return Err(NormalizationError::TimeValueUnexpectedType), + }; + let time = DateTime::parse_from_rfc3339(String::from_utf8_lossy(time.as_ref()).as_ref()) + .context(TimeParsing)?; + log.insert( + event::log_schema().timestamp_key(), + time.with_timezone(&Utc), + ); + + // Parse message, remove trailing newline and detect if it's partial. + let message = log.remove(&LOG).context(LogFieldMissing)?; + let mut message = match message { + Value::Bytes(val) => val, + _ => return Err(NormalizationError::LogValueUnexpectedType), + }; + let is_partial = if message.last().map(|&b| b as char == '\n').unwrap_or(false) { + message.truncate(message.len() - 1); + false + } else { + true + }; + log.insert(event::log_schema().message_key(), message); + + // For partial messages add a partial event indicator. + if is_partial { + log.insert(event::PARTIAL_STR, true); + } + + Ok(()) +} + +#[derive(Debug, Snafu)] +enum NormalizationError { + TimeFieldMissing, + TimeValueUnexpectedType, + TimeParsing { source: chrono::ParseError }, + LogFieldMissing, + LogValueUnexpectedType, +} + +#[cfg(test)] +mod tests { + use super::super::test_util; + use super::*; + + #[test] + fn test_parsing() { + let cases = vec![ + ( + r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#, + test_util::make_log_event( + "The actual log line", + "2016-10-05T00:00:30.082640485Z", + "stderr", + false, + ), + ), + ( + r#"{"log": "The partial log line", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + test_util::make_log_event( + "The partial log line", + "2016-10-05T00:00:30.082640485Z", + "stdout", + true, + ), + ), + ]; + + test_util::test_parser(build, cases); + } +} diff --git a/src/sources/kubernetes_logs/parser/mod.rs b/src/sources/kubernetes_logs/parser/mod.rs new file mode 100644 index 0000000000000..480e7c631a68c --- /dev/null +++ b/src/sources/kubernetes_logs/parser/mod.rs @@ -0,0 +1,17 @@ +use crate::transforms::{ + util::pick::{IterPicker, Passthrough, PickOnce}, + Transform, +}; + +mod cri; +mod docker; +mod test_util; + +/// Parser for any log format supported by `kubelet`. +pub type Parser = PickOnce>>>; + +/// Build a parser for any log format supported by `kubelet`. +pub fn build() -> Parser { + let pickers = vec![docker::build(), cri::build(), Box::new(Passthrough)]; + PickOnce::new(IterPicker::new(pickers)) +} diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs new file mode 100644 index 0000000000000..b475a6437d706 --- /dev/null +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -0,0 +1,46 @@ +#![cfg(test)] + +use crate::{ + event::{Event, LogEvent}, + transforms::Transform, +}; +use chrono::{DateTime, Utc}; + +/// Build a log event for test purposes. +/// +/// The implementation is shared, and therefore consistent consistent across all +/// the parsers. +pub fn make_log_event(message: &str, timestamp: &str, stream: &str, is_partial: bool) -> LogEvent { + let mut log = LogEvent::default(); + + log.insert("message", message); + + let timestamp = DateTime::parse_from_rfc3339(timestamp) + .expect("invalid test case") + .with_timezone(&Utc); + log.insert("timestamp", timestamp); + + log.insert("stream", stream); + + if is_partial { + log.insert("_partial", true); + } + log +} + +/// Shared logic for testing parsers. +/// +/// Takes a parser builder and a list of test cases. +pub fn test_parser(builder: B, cases: Vec<(&'static str, LogEvent)>) +where + B: Fn() -> Box, +{ + for (message, expected) in cases { + let input = Event::from(message); + let mut parser = (builder)(); + let output = parser + .transform(input) + .expect("parser failed to parse the event"); + assert_eq!(Event::Log(expected), output); + } +} diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs index dd8af5e4b5da5..6e1498754cd95 100644 --- a/src/transforms/mod.rs +++ b/src/transforms/mod.rs @@ -1,7 +1,7 @@ use crate::Event; use snafu::Snafu; -mod util; +pub mod util; #[cfg(feature = "transforms-add_fields")] pub mod add_fields; diff --git a/src/transforms/util/chain.rs b/src/transforms/util/chain.rs new file mode 100644 index 0000000000000..67257b1c3bb86 --- /dev/null +++ b/src/transforms/util/chain.rs @@ -0,0 +1,23 @@ +//! Chain transforms. + +#![deny(missing_docs)] + +use crate::{event::Event, transforms::Transform}; + +/// Chains two transforms, passing the event through the first one (`.0`), and +/// then, if there's still an event to pass, through the second one (`.1`). +pub struct Two(pub A, pub B); + +impl Two { + /// Creates a new chain of two transforms. + pub fn new(first: A, second: B) -> Self { + Self(first, second) + } +} + +impl Transform for Two { + fn transform(&mut self, event: Event) -> Option { + let event = self.0.transform(event)?; + self.1.transform(event) + } +} diff --git a/src/transforms/util/mod.rs b/src/transforms/util/mod.rs index 17381a44d34c7..b8ed1cd1c020c 100644 --- a/src/transforms/util/mod.rs +++ b/src/transforms/util/mod.rs @@ -1,2 +1,9 @@ #[cfg(any(feature = "transforms-lua"))] pub mod runtime_transform; + +#[cfg(any(feature = "sources-kubernetes-logs"))] +pub mod chain; +#[cfg(any(feature = "sources-kubernetes-logs"))] +pub mod optional; +#[cfg(any(feature = "sources-kubernetes-logs"))] +pub mod pick; diff --git a/src/transforms/util/optional.rs b/src/transforms/util/optional.rs new file mode 100644 index 0000000000000..6660a58379068 --- /dev/null +++ b/src/transforms/util/optional.rs @@ -0,0 +1,20 @@ +//! Chain transforms. + +#![deny(missing_docs)] + +use crate::{event::Event, transforms::Transform}; + +/// Optional transform. +/// Passes events through the specified transform is any, otherwise passes them, +/// as-is. +/// Useful to avoid boxing the transforms. +pub struct Optional(pub Option); + +impl Transform for Optional { + fn transform(&mut self, event: Event) -> Option { + match self.0 { + Some(ref mut val) => val.transform(event), + None => Some(event), + } + } +} diff --git a/src/transforms/util/pick.rs b/src/transforms/util/pick.rs new file mode 100644 index 0000000000000..57a8ea944935a --- /dev/null +++ b/src/transforms/util/pick.rs @@ -0,0 +1,121 @@ +//! Dynamically pick a tranform to use. + +#![deny(missing_docs)] + +use crate::{event::Event, transforms::Transform}; + +/// Dynamically pick a tranform to use. +/// +/// The implementation would have multiple candidates and a way to determine +/// which transform to use. +pub trait Picker: Sized { + /// Consumes an event and the picker itself, and returns a transform to + /// use and a transformed version of the passed event. + /// + /// Note that the interface doesn't allow failures (i.e. it's not a + /// [`Result`], nor [`Option`]). + /// To handle failure modes, use provided [`Passthrough`], [`Discard`] or + /// [`Panic`] as the returned transform, and pass the event accordingly. + fn probe_event(self, event: Event) -> (Box, Option); +} + +/// Dynamically pick a transform to use once (on first event), and then use the +/// picked transform for the rest of the events. +/// +/// The first message obtained will be probed with picker, and then the system +/// will switch to using. +pub enum PickOnce +where + T: Picker + Send, +{ + /// The initial state. + Init(Option), + /// The picked transform. + Picked(Box), +} + +impl PickOnce +where + T: Picker + Send, +{ + /// Create a new [`PickOnce`] using the passed `T` as the initial state. + pub fn new(init: T) -> Self { + Self::Init(Some(init)) + } +} + +impl Transform for PickOnce +where + T: Picker + Send, +{ + fn transform(&mut self, event: Event) -> Option { + match self { + Self::Init(picker) => { + let picker = picker.take().expect("impossible state"); + let (transform, event) = picker.probe_event(event); + *self = Self::Picked(transform); + event + } + Self::Picked(transform) => transform.transform(event), + } + } +} + +/// A passthrough transform. Passes events as-is. +pub struct Passthrough; + +impl Transform for Passthrough { + fn transform(&mut self, event: Event) -> Option { + Some(event) + } +} + +/// A discard transform. Returns `None` for every event. +pub struct Discard; + +impl Transform for Discard { + fn transform(&mut self, _event: Event) -> Option { + None + } +} + +/// A panic transform. Just panics with the predefined message. +pub struct Panic(pub &'static str); + +impl Transform for Panic { + fn transform(&mut self, _event: Event) -> Option { + panic!(self.0) + } +} + +/// Pick transform from an iterator by passing an event to each of the iterated +/// transport, and choosing the first that returns non-`None`. +pub struct IterPicker(T) +where + T: IntoIterator>; + +impl IterPicker +where + T: IntoIterator>, +{ + /// Create a new `IterPicker` from an iterator. + pub fn new(iter: T) -> Self { + Self(iter) + } +} + +impl Picker for IterPicker +where + T: IntoIterator>, +{ + fn probe_event(self, event: Event) -> (Box, Option) { + self.0 + .into_iter() + .find_map(|mut candidate| { + candidate + .transform(event.clone()) + .map(|event| (candidate, Some(event))) + }) + .expect("transform candidates exausted, and no suitable transform found") + } +} From 0eabab22ad381b3194c0e8885080620e6bb529e0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 22 May 2020 07:10:04 +0300 Subject: [PATCH 016/118] Add partial events merger Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 3 +++ .../kubernetes_logs/partial_events_merger.rs | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/sources/kubernetes_logs/partial_events_merger.rs diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 2b5bab17d49b7..b9899b8245d83 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -6,4 +6,7 @@ mod k8s_paths_provider; mod parser; +mod partial_events_merger; mod path_helpers; + +const FILE_KEY: &str = "file"; diff --git a/src/sources/kubernetes_logs/partial_events_merger.rs b/src/sources/kubernetes_logs/partial_events_merger.rs new file mode 100644 index 0000000000000..652763d582dd1 --- /dev/null +++ b/src/sources/kubernetes_logs/partial_events_merger.rs @@ -0,0 +1,24 @@ +#![deny(missing_docs)] + +use super::FILE_KEY; +use crate::transforms::merge::{Merge, MergeConfig}; +use crate::{event, transforms::util::optional::Optional}; +use string_cache::Atom; + +/// Partial event merger. +pub type PartialEventsMerger = Optional; + +pub fn build(enabled: bool) -> PartialEventsMerger { + Optional(if enabled { + Some( + MergeConfig { + partial_event_marker_field: event::PARTIAL.clone(), + merge_fields: vec![event::log_schema().message_key().clone()], + stream_discriminant_fields: vec![Atom::from(FILE_KEY)], + } + .into(), + ) + } else { + None + }) +} From 66bcd794b7253e2c40cb742d6881d4c4b73b6b0b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 22:17:14 +0300 Subject: [PATCH 017/118] Add pod metadata annotator Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 1 + .../kubernetes_logs/pod_metadata_annotator.rs | 110 ++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 src/sources/kubernetes_logs/pod_metadata_annotator.rs diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index b9899b8245d83..e7501532e6d96 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -8,5 +8,6 @@ mod k8s_paths_provider; mod parser; mod partial_events_merger; mod path_helpers; +mod pod_metadata_annotator; const FILE_KEY: &str = "file"; diff --git a/src/sources/kubernetes_logs/pod_metadata_annotator.rs b/src/sources/kubernetes_logs/pod_metadata_annotator.rs new file mode 100644 index 0000000000000..d61b2eed69576 --- /dev/null +++ b/src/sources/kubernetes_logs/pod_metadata_annotator.rs @@ -0,0 +1,110 @@ +//! Annotates events with pod metadata. + +#![deny(missing_docs)] + +use super::{path_helpers::parse_log_file_path, FILE_KEY}; +use crate::{ + event::{LogEvent, Value}, + kubernetes as k8s, Event, +}; +use evmap10::ReadHandle; +use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; +use string_cache::Atom; + +/// Annotate the event with pod metadata. +pub struct PodMetadataAnnotator { + pods_state_reader: ReadHandle>, +} + +impl PodMetadataAnnotator { + /// Create a new [`PodMetadataAnnotator`]. + pub fn new(pods_state_reader: ReadHandle>) -> Self { + Self { pods_state_reader } + } +} + +impl PodMetadataAnnotator { + /// Annotates an event with the information from the [`Pod::metadata`]. + /// The event has to be obtained from kubernetes log file, and have a + /// [`FILE_KEY`] field set with a file that the line came from. + pub fn annotate(&self, event: &mut Event) -> Option<()> { + let log = event.as_mut_log(); + let uid = pod_uid_from_log_event(log)?; + let guard = self.pods_state_reader.get(&uid)?; + let entry = guard.get_one()?; + let pod: &Pod = entry.as_ref(); + let metadata = pod.metadata.as_ref()?; + annotate_from_metadata(log, &metadata); + Some(()) + } +} + +fn pod_uid_from_log_event(log: &LogEvent) -> Option { + let value = log.get(&Atom::from(FILE_KEY))?; + let file = match value { + Value::Bytes(bytes) => String::from_utf8_lossy(bytes).into_owned(), + _ => return None, + }; + let info = parse_log_file_path(file.as_str())?; + Some(info.pod_uid.to_owned()) +} + +fn annotate_from_metadata(log: &mut LogEvent, metadata: &ObjectMeta) { + for (ref key, ref val) in [ + ("kubernetes.pod_name", &metadata.name), + ("kubernetes.pod_namespace", &metadata.namespace), + ("kubernetes.pod_uid", &metadata.uid), + ] + .iter() + { + if let Some(val) = val { + log.insert(key, val); + } + } + + if let Some(labels) = &metadata.labels { + for (key, val) in labels.iter() { + log.insert(format!("{}.{}", "kubernetes.pod_labels", key), val); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pod_uid_from_log_event() { + let cases = vec![ + // Valid inputs. + ( + { + let mut log = LogEvent::default(); + log.insert( + FILE_KEY, + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/sandbox0-container0-name/1.log", + ); + log + }, + Some("sandbox0-uid"), + ), + // Invalid inputs. + (LogEvent::default(), None), + ( + { + let mut log = LogEvent::default(); + log.insert(FILE_KEY, "qwerty"); + log + }, + None, + ), + ]; + + for (log, expected) in cases.into_iter() { + assert_eq!( + pod_uid_from_log_event(&log), + expected.map(ToOwned::to_owned) + ); + } + } +} From e46acdfdcb28bfb0878c283062df51c3d7f27a1a Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 21 May 2020 22:18:09 +0300 Subject: [PATCH 018/118] Add kubernetes logs source implementation Signed-off-by: MOZGIII --- .../kubernetes/vector-namespaced.yaml | 6 +- src/internal_events/kubernetes_logs.rs | 31 +++ src/internal_events/mod.rs | 4 + src/sources/kubernetes_logs/mod.rs | 227 ++++++++++++++++++ 4 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 src/internal_events/kubernetes_logs.rs diff --git a/distribution/kubernetes/vector-namespaced.yaml b/distribution/kubernetes/vector-namespaced.yaml index dec678f2bb6a9..4fd6581be9f3c 100644 --- a/distribution/kubernetes/vector-namespaced.yaml +++ b/distribution/kubernetes/vector-namespaced.yaml @@ -11,8 +11,8 @@ data: data_dir = "/vector-data-dir" # Ingest logs from Kubernetes. - [sources.kubernetes] - type = "kubernetes" + [sources.kubernetes_logs] + type = "kubernetes_logs" --- apiVersion: apps/v1 kind: DaemonSet @@ -28,11 +28,11 @@ spec: metadata: labels: name: vector + vector.dev/exclude: "true" spec: containers: - name: vector image: timberio/vector:latest-alpine - imagePullPolicy: Always args: - --config - /etc/vector/*.toml diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs new file mode 100644 index 0000000000000..0103b6e843501 --- /dev/null +++ b/src/internal_events/kubernetes_logs.rs @@ -0,0 +1,31 @@ +use super::InternalEvent; +use metrics::counter; + +#[derive(Debug)] +pub struct KubernetesLogsEventReceived<'a> { + pub file: &'a str, + pub byte_size: usize, +} + +impl InternalEvent for KubernetesLogsEventReceived<'_> { + fn emit_logs(&self) { + trace!( + message = "received one event.", + %self.file, + rate_limit_secs = 10 + ); + } + + fn emit_metrics(&self) { + counter!( + "events_processed", 1, + "component_kind" => "source", + "component_type" => "kubernetes_logs", + ); + counter!( + "bytes_processed", self.byte_size as u64, + "component_kind" => "source", + "component_type" => "kubernetes_logs", + ); + } +} diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 0bf12382ce537..796ac17f214f7 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -4,6 +4,8 @@ mod blackhole; mod elasticsearch; mod file; mod json; +#[cfg(feature = "sources-kubernetes-logs")] +mod kubernetes_logs; #[cfg(feature = "transforms-lua")] mod lua; #[cfg(feature = "sources-prometheus")] @@ -24,6 +26,8 @@ pub use self::blackhole::*; pub use self::elasticsearch::*; pub use self::file::*; pub use self::json::*; +#[cfg(feature = "sources-kubernetes-logs")] +pub use self::kubernetes_logs::*; #[cfg(feature = "transforms-lua")] pub use self::lua::*; #[cfg(feature = "sources-prometheus")] diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index e7501532e6d96..759d5070df479 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -10,4 +10,231 @@ mod partial_events_merger; mod path_helpers; mod pod_metadata_annotator; +use crate::event::{self, Event}; +use crate::internal_events::KubernetesLogsEventReceived; +use crate::kubernetes as k8s; +use crate::{ + dns::Resolver, + shutdown::ShutdownSignal, + sources, + topology::config::{DataType, GlobalOptions, SourceConfig, SourceDescription}, + transforms::Transform, +}; +use evmap10::{self as evmap}; +use file_source::{FileServer, FileServerShutdown, Fingerprinter}; +use futures::{future::FutureExt, sink::Sink, stream::StreamExt}; +use futures01::sync::mpsc; +use k8s_paths_provider::K8sPathsProvider; +use pod_metadata_annotator::PodMetadataAnnotator; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::task::spawn_blocking; + +/// Configuration for the `kubernetes_logs` source. +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields, default)] +pub struct Config { + self_node_name: String, +} + +inventory::submit! { + SourceDescription::new_without_default::(COMPONENT_NAME) +} + +const COMPONENT_NAME: &str = "kubernetes_logs"; + +#[typetag::serde(name = "kubernetes_logs")] +impl SourceConfig for Config { + fn build( + &self, + name: &str, + globals: &GlobalOptions, + shutdown: ShutdownSignal, + out: mpsc::Sender, + ) -> crate::Result { + let source = Source::init(self, Resolver, globals, name)?; + + // TODO: this is a workaround for the legacy futures 0.1. + // When the core is updated to futures 0.3 this should be simplied + // significantly. + let out = futures::compat::Compat01As03Sink::new(out); + let fut = source.run(out, shutdown); + let fut = fut.map(|result| { + result.map_err(|error| { + error!(message = "source future failed", ?error); + }) + }); + let fut = Box::pin(fut); + let fut = futures::compat::Compat::new(fut); + let fut: sources::Source = Box::new(fut); + Ok(fut) + } + + fn output_type(&self) -> DataType { + DataType::Log + } + + fn source_type(&self) -> &'static str { + COMPONENT_NAME + } +} + +#[derive(Clone)] +struct Source { + client: k8s::client::Client, + self_node_name: String, + data_dir: PathBuf, +} + +impl Source { + fn init( + config: &Config, + resolver: Resolver, + globals: &GlobalOptions, + name: &str, + ) -> crate::Result { + let self_node_name = if config.self_node_name.is_empty() { + std::env::var("VECTOR_SELF_NODE_NAME") + .map_err(|_| "VECTOR_SELF_NODE_NAME is not set")? + } else { + config.self_node_name.clone() + }; + info!( + message = "obtained Kubernetes Node name to collect logs for (self)", + ?self_node_name + ); + + let k8s_config = k8s::client::config::Config::in_cluster()?; + let client = k8s::client::Client::new(k8s_config, resolver)?; + + let data_dir = globals.resolve_and_make_data_subdir(None, name)?; + + Ok(Self { + client, + self_node_name, + data_dir, + }) + } + + async fn run(self, out: O, shutdown: ShutdownSignal) -> crate::Result<()> + where + O: Sink + Send, + { + let Self { + client, + self_node_name, + data_dir, + } = self; + + let field_selector = format!("spec.nodeName={}", self_node_name); + let label_selector = "vector.dev/exclude!=true".to_owned(); + + let watcher = k8s::api_watcher::ApiWatcher::new(client); + let (state_reader, state_writer) = evmap::new(); + + let mut reflector = k8s::reflector::Reflector::new( + watcher, + state_writer, + Some(field_selector), + Some(label_selector), + std::time::Duration::from_secs(1), + ); + let reflector_process = reflector.run(); + + let paths_provider = K8sPathsProvider::new(state_reader.clone()); + let annotator = PodMetadataAnnotator::new(state_reader); + + // TODO: maybe some of the parameters have to be configurable. + let file_server = FileServer { + paths_provider, + max_read_bytes: 2048, + start_at_beginning: true, + ignore_before: None, + max_line_bytes: 32 * 1024, // 32 KiB + data_dir, + glob_minimum_cooldown: Duration::from_secs(10), + fingerprinter: Fingerprinter::Checksum { + fingerprint_bytes: 256, + ignored_header_bytes: 0, + }, + oldest_first: false, + remove_after: None, + }; + + let (file_source_tx, file_source_rx) = futures::channel::mpsc::channel(100); + + let span = info_span!("file_server"); + let file_server_join_handle = spawn_blocking(move || { + let _enter = span.enter(); + let result = + file_server.run(file_source_tx, futures::compat::Compat01As03::new(shutdown)); + result.unwrap() + }); + + let mut parser = parser::build(); + let mut partial_events_merger = partial_events_merger::build(true); + + let events = file_source_rx.map(|(bytes, file)| { + emit!(KubernetesLogsEventReceived { + file: &file, + byte_size: bytes.len(), + }); + create_event(bytes, file) + }); + let events = events + .filter_map(move |event| futures::future::ready(parser.transform(event))) + .filter_map(move |event| futures::future::ready(partial_events_merger.transform(event))) + .map(move |mut event| { + if annotator.annotate(&mut event).is_none() { + warn!( + message = "failed to annotate event with pod metadata", + ?event + ); + } + event + }); + + let event_processing_loop = events.map(Ok).forward(out); + + use std::future::Future; + use std::pin::Pin; + let list: Vec + Send>>> = vec![ + Box::pin(reflector_process.map(|_| todo!())), + Box::pin(file_server_join_handle.map(|result| match result { + Ok(FileServerShutdown) => info!(message = "file server completed gracefully"), + Err(error) => error!(message = "file server exited with an error", ?error), + })), + Box::pin(event_processing_loop.map(|result| { + match result { + Ok(_) => info!(message = "event_processing_loop ok"), + Err(_) => error!(message = "event_processing_loop err"), + }; + })), + ]; + let mut futs: futures::stream::FuturesUnordered<_> = list.into_iter().collect(); + + while let Some(()) = futs.next().await { + trace!("another future complete"); + } + + info!("Done"); + Ok(()) + } +} + const FILE_KEY: &str = "file"; + +fn create_event(line: bytes05::Bytes, file: String) -> Event { + let mut event = Event::from(line); + + // Add source type. + event + .as_mut_log() + .insert(event::log_schema().source_type_key(), COMPONENT_NAME); + + // Add file. + event.as_mut_log().insert(FILE_KEY, file); + + event +} From 9c49908829010c5d9e9bec21f7ee15cbd9e5b44e Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 11:37:12 +0300 Subject: [PATCH 019/118] Change reflector errors to be less restrictive Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 96d7301f857b4..0fc65d1d50069 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -67,7 +67,10 @@ where ::StreamError: Unpin, { /// Run the watch loop and drive the state updates via `state_writer`. - pub async fn run(&mut self) -> Result> { + pub async fn run( + &mut self, + ) -> Result::InvocationError, ::StreamError>> + { // We're taking over the `self.state_writer`, clear it. self.state_writer.purge(); // Propagate the purge and ensure we initialize readers for the first @@ -213,22 +216,23 @@ where /// Errors that can occur while watching. #[derive(Debug, Snafu)] -pub enum Error +pub enum Error where - W: Watcher, + I: std::error::Error + 'static, + S: std::error::Error + 'static, { /// Returned when the watch invocation (HTTP request) failed. #[snafu(display("watch invocation failed"))] Invocation { /// The underlying invocation error. - source: ::InvocationError, + source: I, }, /// Returned when the stream failed with an error. #[snafu(display("streaming error"))] Streaming { /// The underlying stream error. - source: ::StreamError, + source: S, }, } From cccb735f40415fe79809f81eb9834a4814af038a Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 11:46:34 +0300 Subject: [PATCH 020/118] Better error handling Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 759d5070df479..fff09d8936f81 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -200,15 +200,18 @@ impl Source { use std::future::Future; use std::pin::Pin; let list: Vec + Send>>> = vec![ - Box::pin(reflector_process.map(|_| todo!())), + Box::pin(reflector_process.map(|result| match result { + Ok(_val) => info!(message = "reflector process completed gracefully"), + Err(error) => error!(message = "reflector process exited with an error", ?error), + })), Box::pin(file_server_join_handle.map(|result| match result { Ok(FileServerShutdown) => info!(message = "file server completed gracefully"), Err(error) => error!(message = "file server exited with an error", ?error), })), Box::pin(event_processing_loop.map(|result| { match result { - Ok(_) => info!(message = "event_processing_loop ok"), - Err(_) => error!(message = "event_processing_loop err"), + Ok(()) => info!(message = "event processing loop completed gracefully"), + Err(_error) => error!(message = "event processing loop exited with an error"), }; })), ]; From bb8158093520a937baca405f45ff298b67c4ad6c Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 11:50:46 +0300 Subject: [PATCH 021/118] Improve error handling Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index fff09d8936f81..66059a2487dcf 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -169,7 +169,7 @@ impl Source { let _enter = span.enter(); let result = file_server.run(file_source_tx, futures::compat::Compat01As03::new(shutdown)); - result.unwrap() + result.expect("file server exited with an error") }); let mut parser = parser::build(); @@ -218,10 +218,10 @@ impl Source { let mut futs: futures::stream::FuturesUnordered<_> = list.into_iter().collect(); while let Some(()) = futs.next().await { - trace!("another future complete"); + trace!(message = "another future complete"); } - info!("Done"); + info!(message = "done"); Ok(()) } } From ad8f2f80489d133f35277e534765acbebc78a77a Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 11:53:21 +0300 Subject: [PATCH 022/118] Better error handling for event pipeline Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 66059a2487dcf..1a3152d701e21 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -120,6 +120,7 @@ impl Source { async fn run(self, out: O, shutdown: ShutdownSignal) -> crate::Result<()> where O: Sink + Send, + >::Error: std::error::Error, { let Self { client, @@ -211,7 +212,10 @@ impl Source { Box::pin(event_processing_loop.map(|result| { match result { Ok(()) => info!(message = "event processing loop completed gracefully"), - Err(_error) => error!(message = "event processing loop exited with an error"), + Err(error) => error!( + message = "event processing loop exited with an error", + ?error + ), }; })), ]; From 34f7731ff3fee69c5203ffa7a9e9a12eacc054be Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 13:30:43 +0300 Subject: [PATCH 023/118] Abstract API watcher around watch request builder Signed-off-by: MOZGIII --- src/kubernetes/api_watcher.rs | 60 +++++++++----- src/kubernetes/mod.rs | 1 + src/kubernetes/watch_request_builder.rs | 102 ++++++++++++++++++++++++ src/sources/kubernetes_logs/mod.rs | 3 +- 4 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 src/kubernetes/watch_request_builder.rs diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs index 37b927e87e0ff..427897d27de11 100644 --- a/src/kubernetes/api_watcher.rs +++ b/src/kubernetes/api_watcher.rs @@ -3,6 +3,7 @@ use super::{ client::Client, stream as k8s_stream, + watch_request_builder::WatchRequestBuilder, watcher::{self, Watcher}, }; use futures::{ @@ -11,38 +12,54 @@ use futures::{ }; use http::StatusCode; use hyper::Error as BodyError; -use k8s_openapi::{api::core::v1::Pod, WatchOptional, WatchResponse}; +use k8s_openapi::{WatchOptional, WatchResponse}; use snafu::{ResultExt, Snafu}; -// TODO: abstract around the request builder. -// The `Pod::watch_pod_for_all_namespaces` is currently hardcoded - but we -// can use this with other resource types too, and we'll probably have to -// add support for watching namespaces later on. - /// A simple watcher atop of the Kubernetes API [`Client`]. -pub struct ApiWatcher { +pub struct ApiWatcher +where + B: 'static, +{ client: Client, + request_builder: B, } -impl ApiWatcher { +impl ApiWatcher +where + B: 'static, +{ /// Create a new [`ApiWatcher`]. - pub fn new(client: Client) -> Self { - Self { client } + pub fn new(client: Client, request_builder: B) -> Self { + Self { + client, + request_builder, + } } } -impl ApiWatcher { +impl ApiWatcher +where + B: 'static + WatchRequestBuilder, + ::Object: Send + Unpin, +{ async fn invoke( &mut self, watch_optional: WatchOptional<'_>, ) -> Result< - impl Stream, k8s_stream::Error>> + 'static, + impl Stream< + Item = Result< + WatchResponse<::Object>, + k8s_stream::Error, + >, + > + 'static, watcher::invocation::Error, > { // Prepare request. - let (request, _) = Pod::watch_pod_for_all_namespaces(watch_optional) + let request = self + .request_builder + .build(watch_optional) .context(invocation::RequestPreparation)?; - trace!(message = "Request prepared", ?request); + trace!(message = "request prepared", ?request); // Send request, get response. let response = self @@ -50,7 +67,7 @@ impl ApiWatcher { .send(request) .await .context(invocation::Request)?; - trace!(message = "Got response", ?response); + trace!(message = "got response", ?response); // Handle response status code. let status = response.status(); @@ -73,7 +90,10 @@ impl ApiWatcher { &mut self, watch_optional: WatchOptional<'_>, ) -> Result< - BoxStream<'static, Result, k8s_stream::Error>>, + BoxStream< + 'static, + Result::Object>, k8s_stream::Error>, + >, watcher::invocation::Error, > { let stream = self.invoke(watch_optional).await?; @@ -81,8 +101,12 @@ impl ApiWatcher { } } -impl Watcher for ApiWatcher { - type Object = Pod; +impl Watcher for ApiWatcher +where + B: 'static + WatchRequestBuilder + Send, + ::Object: Send + Unpin, +{ + type Object = ::Object; type InvocationError = invocation::Error; diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index e6070b0d8459c..4eb0dec9a2436 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -28,4 +28,5 @@ pub mod multi_response_decoder; pub mod reflector; pub mod resource_version; pub mod stream; +pub mod watch_request_builder; pub mod watcher; diff --git a/src/kubernetes/watch_request_builder.rs b/src/kubernetes/watch_request_builder.rs new file mode 100644 index 0000000000000..6ba31f439ea48 --- /dev/null +++ b/src/kubernetes/watch_request_builder.rs @@ -0,0 +1,102 @@ +//! Build watch request for k8s API and adapters for library types. +//! +//! # Examples +//! +//! ## Non-namespaced and cluster-global +//! +//! ```rust +//! # use vector::kubernetes::watch_request_builder::WatchRequestBuilder; +//! use k8s_openapi::api::core::v1::Pod; +//! +//! let _val: Box> = +//! Box::new(Pod::watch_pod_for_all_namespaces); +//! ``` +//! +//! ## Namespaced +//! +//! ```rust +//! # use vector::kubernetes::watch_request_builder::{WatchRequestBuilder, Namespaced}; +//! use k8s_openapi::api::core::v1::Pod; +//! +//! let _val: Box> = +//! Box::new(Namespaced("default", Pod::watch_namespaced_pod)); +//! ``` +//! + +use k8s_openapi::{ + apimachinery::pkg::apis::meta::v1::ObjectMeta, + http::{Request, StatusCode}, + Metadata, RequestError, ResponseBody, WatchOptional, WatchResponse, +}; +use serde::de::DeserializeOwned; + +/// Build a watch request for the k8s API. +/// +/// See module documentation. +pub trait WatchRequestBuilder { + /// The object type that's being watched. + type Object: Metadata + DeserializeOwned; + + /// Build a watch request. + fn build<'a>( + &self, + watch_optional: WatchOptional<'a>, + ) -> Result>, RequestError>; +} + +impl WatchRequestBuilder for F +where + T: Metadata + DeserializeOwned, + F: for<'w> Fn( + WatchOptional<'w>, + ) -> Result< + ( + Request>, + fn(StatusCode) -> ResponseBody>, + ), + RequestError, + >, +{ + type Object = T; + + fn build<'a>( + &self, + watch_optional: WatchOptional<'a>, + ) -> Result>, RequestError> { + let (request, _) = (self)(watch_optional)?; + Ok(request) + } +} + +/// Wrapper for a namespaced API. +/// +/// Specify the namespace and an API request building function. +/// +/// See module documentation for an example. +pub struct Namespaced(pub N, pub F); + +impl WatchRequestBuilder for Namespaced +where + N: AsRef, + T: Metadata + DeserializeOwned, + F: for<'w> Fn( + &'w str, + WatchOptional<'w>, + ) -> Result< + ( + Request>, + fn(StatusCode) -> ResponseBody>, + ), + RequestError, + >, +{ + type Object = T; + + fn build<'a>( + &self, + watch_optional: WatchOptional<'a>, + ) -> Result>, RequestError> { + let (request, _) = (self.1)(self.0.as_ref(), watch_optional)?; + Ok(request) + } +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 1a3152d701e21..4cdd45be9d117 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -24,6 +24,7 @@ use evmap10::{self as evmap}; use file_source::{FileServer, FileServerShutdown, Fingerprinter}; use futures::{future::FutureExt, sink::Sink, stream::StreamExt}; use futures01::sync::mpsc; +use k8s_openapi::api::core::v1::Pod; use k8s_paths_provider::K8sPathsProvider; use pod_metadata_annotator::PodMetadataAnnotator; use serde::{Deserialize, Serialize}; @@ -131,7 +132,7 @@ impl Source { let field_selector = format!("spec.nodeName={}", self_node_name); let label_selector = "vector.dev/exclude!=true".to_owned(); - let watcher = k8s::api_watcher::ApiWatcher::new(client); + let watcher = k8s::api_watcher::ApiWatcher::new(client, Pod::watch_pod_for_all_namespaces); let (state_reader, state_writer) = evmap::new(); let mut reflector = k8s::reflector::Reflector::new( From d584a124f7ceb3d978c3fa4a930098a573b16f98 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 25 May 2020 14:22:01 +0300 Subject: [PATCH 024/118] Add assertions to the reflector internal state in-between the invocations Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 44 ++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 0fc65d1d50069..0e32f90b5b6f0 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -308,6 +308,9 @@ mod tests { } } + // A type alias to add expressiveness. + type StateSnapshot = Vec; + // A helper enum to encode expected mock watcher invocation. enum ExpInvRes { Stream(Vec>), @@ -350,10 +353,9 @@ mod tests { // A helper function to run a flow test. // Use this to test various flows without an actual test repetition. - // TODO: add intermediate state assertions to validate caching properties. fn run_flow_test( - invocations: Vec<(Option, ExpInvRes)>, - expected_resulting_state: Vec, + invocations: Vec<(StateSnapshot, Option, ExpInvRes)>, + expected_resulting_state: StateSnapshot, ) { // Prepare the test flow. let (state_reader, state_writer) = evmap10::new(); @@ -364,9 +366,16 @@ mod tests { let mut flow_invocations = invocations.into_iter(); let mock_logic = move |watch_optional: WatchOptional<'_>| { - if let Some((expected_resource_version, expected_invocation_response)) = - flow_invocations.next() + if let Some(( + expected_state_before_op, + expected_resource_version, + expected_invocation_response, + )) = flow_invocations.next() { + // Assert the state prior to the operation. + let state: StateSnapshot = gather_state(&assertion_state_reader); + assert_eq!(state, expected_state_before_op); + assert_eq!( expected_resource_version, watch_optional.resource_version.map(ToOwned::to_owned) // work around the borrow checker issues @@ -383,7 +392,7 @@ mod tests { return Ok(move || responses_iter.next().map(|val| Ok(WatchResponse::Ok(val)))); } - let resulting_state: Vec = gather_state(&assertion_state_reader); + let resulting_state: StateSnapshot = gather_state(&assertion_state_reader); assert_eq!(resulting_state, flow_expected_resulting_state); Err(watcher::invocation::Error::other(InvocationError)) @@ -403,7 +412,7 @@ mod tests { result.unwrap_err(); // Assert the state after the reflector exit. - let resulting_state: Vec = gather_state(&state_reader); + let resulting_state: StateSnapshot = gather_state(&state_reader); assert_eq!(resulting_state, expected_resulting_state); // Explicitly drop the reflector at the very end. @@ -420,6 +429,7 @@ mod tests { let invocations = vec![ ( + vec![], None, ExpInvRes::Stream(vec![ WatchEvent::Added(make_pod("uid0", "10")), @@ -427,6 +437,7 @@ mod tests { ]), ), ( + vec![make_pod("uid0", "10"), make_pod("uid1", "15")], Some("15".to_owned()), ExpInvRes::Stream(vec![ WatchEvent::Modified(make_pod("uid0", "20")), @@ -434,10 +445,20 @@ mod tests { ]), ), ( + vec![ + make_pod("uid0", "20"), + make_pod("uid1", "15"), + make_pod("uid2", "25"), + ], Some("25".to_owned()), ExpInvRes::Stream(vec![WatchEvent::Bookmark(make_pod_bookmark("50"))]), ), ( + vec![ + make_pod("uid0", "20"), + make_pod("uid1", "15"), + make_pod("uid2", "25"), + ], Some("50".to_owned()), ExpInvRes::Stream(vec![ WatchEvent::Deleted(make_pod("uid2", "55")), @@ -458,14 +479,20 @@ mod tests { let invocations = vec![ ( + vec![], None, ExpInvRes::Stream(vec![ WatchEvent::Added(make_pod("uid0", "10")), WatchEvent::Added(make_pod("uid1", "15")), ]), ), - (Some("15".to_owned()), ExpInvRes::Desync), ( + vec![make_pod("uid0", "10"), make_pod("uid1", "15")], + Some("15".to_owned()), + ExpInvRes::Desync, + ), + ( + vec![make_pod("uid0", "10"), make_pod("uid1", "15")], None, ExpInvRes::Stream(vec![ WatchEvent::Added(make_pod("uid20", "1000")), @@ -473,6 +500,7 @@ mod tests { ]), ), ( + vec![make_pod("uid20", "1000"), make_pod("uid21", "1005")], Some("1005".to_owned()), ExpInvRes::Stream(vec![WatchEvent::Modified(make_pod("uid21", "1010"))]), ), From 36919fc19897bd60203185a0ccb8debec29e6208 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 02:43:24 +0300 Subject: [PATCH 025/118] Fix the comment at optional tranform mod Signed-off-by: MOZGIII --- src/transforms/util/optional.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transforms/util/optional.rs b/src/transforms/util/optional.rs index 6660a58379068..df470c71b2e9e 100644 --- a/src/transforms/util/optional.rs +++ b/src/transforms/util/optional.rs @@ -1,4 +1,4 @@ -//! Chain transforms. +//! Optional transform. #![deny(missing_docs)] From 9636f3f1565c9a98e20421d83cfff6c4090392d9 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 02:45:27 +0300 Subject: [PATCH 026/118] Add a to do comment to make transform utils part of core Signed-off-by: MOZGIII --- src/transforms/util/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transforms/util/mod.rs b/src/transforms/util/mod.rs index b8ed1cd1c020c..62ea8bb08a83b 100644 --- a/src/transforms/util/mod.rs +++ b/src/transforms/util/mod.rs @@ -1,6 +1,7 @@ #[cfg(any(feature = "transforms-lua"))] pub mod runtime_transform; +// TODO: make these part of the vector core - enabled unconditionally. #[cfg(any(feature = "sources-kubernetes-logs"))] pub mod chain; #[cfg(any(feature = "sources-kubernetes-logs"))] From 366b8ca6c6ae69fdf4f9ac06fe8969c3752ef8a2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 02:46:20 +0300 Subject: [PATCH 027/118] Fix typo at kustomization.yaml Signed-off-by: MOZGIII --- kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomization.yaml b/kustomization.yaml index efd9cf96a1697..f8145e36474c6 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -1,4 +1,4 @@ -# This is a part of out skaffold setup for development. +# This is a part of our skaffold setup for development. # Do not use in production. namespace: vector From cc4a1a83db7ac0aba5c26ea6da2eb87eb889236f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 03:08:43 +0300 Subject: [PATCH 028/118] More flexible interface at resource version Signed-off-by: MOZGIII --- src/kubernetes/resource_version.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index 8af302f4295c8..982d2d3064ebf 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -15,13 +15,17 @@ impl State { } /// Update the resource version from a candidate obtained earlier. - pub fn update(&mut self, candidate: Candidate) { - self.0 = Some(candidate.0); + /// + /// Returns the previous state. + pub fn update(&mut self, candidate: Candidate) -> Option { + self.0.replace(candidate.0) } /// Reset the resource version. Use in case of a desync. - pub fn reset(&mut self) { - self.0 = None; + /// + /// Returns the previous state. + pub fn reset(&mut self) -> Option { + self.0.take() } /// Get the current resource version value. From 234b1e3663ddecc4165b124a44b144c431109fd0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 03:12:14 +0300 Subject: [PATCH 029/118] Fix the mod comment at resource version Signed-off-by: MOZGIII --- src/kubernetes/resource_version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index 982d2d3064ebf..d108fe0e87164 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -1,4 +1,4 @@ -//! Shared state bits for watch implementations. +//! A resource version types to ensure proper usage protocol. use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}; use k8s_openapi::Metadata; From 7c8358d89b670ade603596c82c677e931983c989 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 05:17:06 +0300 Subject: [PATCH 030/118] Remove manual minikube init from scripts/skaffold.sh Skaffold is capable of detecting minikube itself. It will also work with Docker Desktop on Windows and macOS out of the box. Signed-off-by: MOZGIII --- scripts/skaffold.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/skaffold.sh b/scripts/skaffold.sh index 6d356c1e29ecd..dd16c70603a7d 100755 --- a/scripts/skaffold.sh +++ b/scripts/skaffold.sh @@ -12,10 +12,6 @@ cat <target/debug/.dockerignore !vector EOF -# Configure docker (and skaffold) to use minikube docker host. -# shellcheck source=minikube-docker-env.sh disable=SC1091 -. scripts/minikube-docker-env.sh - # Watch for changes in he background and rebuild the vector binary. cargo watch -x build & From ca0d97934511a4a3470b90aa18c5bf00e4a146b8 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 05:20:40 +0300 Subject: [PATCH 031/118] Specify shell at scripts/minikube-docker-env.sh Signed-off-by: MOZGIII --- scripts/minikube-docker-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/minikube-docker-env.sh b/scripts/minikube-docker-env.sh index 64b45ae0038ca..fc2ffdd8ab9f0 100644 --- a/scripts/minikube-docker-env.sh +++ b/scripts/minikube-docker-env.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -if ! COMMANDS="$(minikube docker-env)"; then +if ! COMMANDS="$(minikube --shell bash docker-env)"; then echo "Unable to obtain docker env from minikube; is minikube started?" >&2 exit 7 fi From 8d41c330c6ac5605392389ded00d79753b92b575 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 26 May 2020 05:21:00 +0300 Subject: [PATCH 032/118] Switch scripts/copy-docker-image-to-minikube.sh to use scripts/minikube-docker-env.sh Signed-off-by: MOZGIII --- scripts/copy-docker-image-to-minikube.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/copy-docker-image-to-minikube.sh b/scripts/copy-docker-image-to-minikube.sh index 4d86f5e9d790e..0efe834b69e24 100755 --- a/scripts/copy-docker-image-to-minikube.sh +++ b/scripts/copy-docker-image-to-minikube.sh @@ -27,7 +27,8 @@ docker save "${IMAGES[@]}" | gzip >"$IMAGES_ARCHIVE" # Start a subshell to preserve the env state. ( # Switch to minikube docker. - eval "$(minikube --shell bash docker-env)" + # shellcheck source=minikube-docker-env.sh disable=SC1091 + . scripts/minikube-docker-env.sh # Load images. docker load -i "$IMAGES_ARCHIVE" From d21279af0ed6bcfac37f268a90a830fb356a346d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 28 May 2020 16:04:24 +0300 Subject: [PATCH 033/118] Switch to using device inodes for file fingerprinting Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 4cdd45be9d117..46dcf1a83bc05 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -156,10 +156,11 @@ impl Source { max_line_bytes: 32 * 1024, // 32 KiB data_dir, glob_minimum_cooldown: Duration::from_secs(10), - fingerprinter: Fingerprinter::Checksum { - fingerprint_bytes: 256, - ignored_header_bytes: 0, - }, + // Use device inodes for fingerprinting. + // - Docker recreates files on rotation: https://github.com/moby/moby/blob/75d655320e2a443185f8fa4992dc89bd2da0ea68/daemon/logger/loggerutils/logfile.go#L182-L222 + // - CRI-O recreates files on rotation: https://github.com/cri-o/cri-o/blob/ad83d2a35a30b8a336b16a0ea5f7afc6aebfb9b7/internal/oci/runtime_oci.go#L988-L1042 + // The rest should do the same. + fingerprinter: Fingerprinter::DevInode, oldest_first: false, remove_after: None, }; From 24d56045eece99d7ed6fa292610ee2d663033580 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:00:47 +0300 Subject: [PATCH 034/118] Fix grammar at in-cluster config Signed-off-by: MOZGIII --- src/kubernetes/client/config/in_cluster.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs index f0d7131055f9a..7a21a8f08bfff 100644 --- a/src/kubernetes/client/config/in_cluster.rs +++ b/src/kubernetes/client/config/in_cluster.rs @@ -6,7 +6,7 @@ use http::Uri; use snafu::{ResultExt, Snafu}; impl Config { - /// Prepares a config sutable for use when running in k8s cluster. + /// Prepares a config suitable for use when running in k8s cluster. pub fn in_cluster() -> Result { let host = std::env::var("KUBERNETES_SERVICE_HOST").context(NotInCluster)?; let port = std::env::var("KUBERNETES_SERVICE_PORT").context(NotInCluster)?; From 5ed884ca094aa78717e16cf988583ef2b5f32a93 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:04:22 +0300 Subject: [PATCH 035/118] Add the variable that's missing to the error at in-cluster config Signed-off-by: MOZGIII --- src/kubernetes/client/config/in_cluster.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs index 7a21a8f08bfff..caf9ec20e4a3b 100644 --- a/src/kubernetes/client/config/in_cluster.rs +++ b/src/kubernetes/client/config/in_cluster.rs @@ -8,8 +8,12 @@ use snafu::{ResultExt, Snafu}; impl Config { /// Prepares a config suitable for use when running in k8s cluster. pub fn in_cluster() -> Result { - let host = std::env::var("KUBERNETES_SERVICE_HOST").context(NotInCluster)?; - let port = std::env::var("KUBERNETES_SERVICE_PORT").context(NotInCluster)?; + let host = std::env::var("KUBERNETES_SERVICE_HOST").context(NotInCluster { + missing: "KUBERNETES_SERVICE_HOST", + })?; + let port = std::env::var("KUBERNETES_SERVICE_PORT").context(NotInCluster { + missing: "KUBERNETES_SERVICE_PORT", + })?; let base = Uri::builder() .scheme("https") @@ -43,6 +47,9 @@ pub enum Error { NotInCluster { /// The underlying error. source: std::env::VarError, + + /// The field that's missing. + missing: &'static str, }, /// The token file could not be read successfully. From 1c95b3035cf614c38a166ce279929d8fbd291307 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:04:53 +0300 Subject: [PATCH 036/118] Move in_cluster mod declaration to the top of the file Signed-off-by: MOZGIII --- src/kubernetes/client/config/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kubernetes/client/config/mod.rs b/src/kubernetes/client/config/mod.rs index 5ceafcb26ef83..96018d2b65ed9 100644 --- a/src/kubernetes/client/config/mod.rs +++ b/src/kubernetes/client/config/mod.rs @@ -3,6 +3,8 @@ use crate::tls::TlsOptions; use http::Uri; +pub mod in_cluster; + /// A k8s client configuration. /// /// This type is designed to hold all possible variants of the configuration. @@ -23,5 +25,3 @@ pub struct Config { /// The TLS configuration parameters to use at the HTTP client. pub tls_options: TlsOptions, } - -pub mod in_cluster; From 363ef5384c04a2ee1e948fadaf1d10553ac703b2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:09:43 +0300 Subject: [PATCH 037/118] Cut some ununsed code from src/kubernetes/resource_version.rs Signed-off-by: MOZGIII --- src/kubernetes/resource_version.rs | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index d108fe0e87164..387e2fd08d67b 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -2,7 +2,6 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}; use k8s_openapi::Metadata; -use std::ops::Deref; /// Resource version state in the context of a chain of watch requests. #[derive(Debug, Clone)] @@ -30,24 +29,7 @@ impl State { /// Get the current resource version value. pub fn get(&self) -> Option<&str> { - self.into() - } -} - -impl Deref for State { - type Target = Option; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl<'a> From<&'a State> for Option<&'a str> { - fn from(val: &'a State) -> Self { - match val.0 { - Some(ref val) => Some(val.as_str()), - None => None, - } + Some(self.0.as_ref()?.as_str()) } } From 3035739c41b1fbb8c62c3f584993eebe339d62aa Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:10:20 +0300 Subject: [PATCH 038/118] Fix typo at src/kubernetes/reflector.rs Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 0e32f90b5b6f0..63fe536f649a2 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -123,7 +123,7 @@ where } }; - // Prepare a resource version cvandidate so we can update (aka + // Prepare a resource version candidate so we can update (aka // commit) it later. let resource_version_candidate = match resource_version::Candidate::from_watch_event(&event) { From 9ae903443885da1462e1d072ddc1b5f901324dec Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:13:15 +0300 Subject: [PATCH 039/118] Move the file key to the top of the file Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 46dcf1a83bc05..9896b3af90a2a 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -32,6 +32,9 @@ use std::path::PathBuf; use std::time::Duration; use tokio::task::spawn_blocking; +/// The key we use for `file` field. +const FILE_KEY: &str = "file"; + /// Configuration for the `kubernetes_logs` source. #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(deny_unknown_fields, default)] @@ -232,8 +235,6 @@ impl Source { } } -const FILE_KEY: &str = "file"; - fn create_event(line: bytes05::Bytes, file: String) -> Event { let mut event = Event::from(line); From 19e31e165f64a24286aefabf886e414cabf5fdcb Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:16:40 +0300 Subject: [PATCH 040/118] Fix comment at parsers test Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/test_util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index b475a6437d706..5db6558c8b692 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Utc}; /// Build a log event for test purposes. /// -/// The implementation is shared, and therefore consistent consistent across all +/// The implementation is shared, and therefore consistent across all /// the parsers. pub fn make_log_event(message: &str, timestamp: &str, stream: &str, is_partial: bool) -> LogEvent { let mut log = LogEvent::default(); From f0a9bfe6f61be96c7f47f800c9b95568e7e02eb4 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:26:07 +0300 Subject: [PATCH 041/118] Add a comment for Config::self_node_name Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 9896b3af90a2a..93fc27d75a49c 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -39,6 +39,9 @@ const FILE_KEY: &str = "file"; #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(deny_unknown_fields, default)] pub struct Config { + /// The `name` of the Kubernetes `Node` that Vector runs at. + /// Required to filter the `Pod`s to only include the ones with the log + /// files accessible locally. self_node_name: String, } From dc68cf2fa6bc23fcc24957581e2087abba8bcee4 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 14:30:17 +0300 Subject: [PATCH 042/118] Allow disabling automatic partial merge Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 93fc27d75a49c..f3a1e50d68812 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -43,6 +43,10 @@ pub struct Config { /// Required to filter the `Pod`s to only include the ones with the log /// files accessible locally. self_node_name: String, + + /// Automatically merge partial events. + #[serde(default = "crate::serde::default_true")] + auto_partial_merge: bool, } inventory::submit! { @@ -92,6 +96,7 @@ struct Source { client: k8s::client::Client, self_node_name: String, data_dir: PathBuf, + auto_partial_merge: bool, } impl Source { @@ -121,6 +126,7 @@ impl Source { client, self_node_name, data_dir, + auto_partial_merge: config.auto_partial_merge, }) } @@ -133,6 +139,7 @@ impl Source { client, self_node_name, data_dir, + auto_partial_merge, } = self; let field_selector = format!("spec.nodeName={}", self_node_name); @@ -182,7 +189,7 @@ impl Source { }); let mut parser = parser::build(); - let mut partial_events_merger = partial_events_merger::build(true); + let mut partial_events_merger = partial_events_merger::build(auto_partial_merge); let events = file_source_rx.map(|(bytes, file)| { emit!(KubernetesLogsEventReceived { From 6dd579997d85723d0d238e57e3669fa7008c6b5b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 1 Jun 2020 15:13:21 +0300 Subject: [PATCH 043/118] Allow customizing the fields names used by pod metadata annotator Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 8 +- .../kubernetes_logs/pod_metadata_annotator.rs | 118 ++++++++++++++++-- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index f3a1e50d68812..23f68992d9151 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -47,6 +47,9 @@ pub struct Config { /// Automatically merge partial events. #[serde(default = "crate::serde::default_true")] auto_partial_merge: bool, + + /// Specifies the field names for metadata annotation. + annotation_fields: pod_metadata_annotator::FieldsSpec, } inventory::submit! { @@ -97,6 +100,7 @@ struct Source { self_node_name: String, data_dir: PathBuf, auto_partial_merge: bool, + fields_spec: pod_metadata_annotator::FieldsSpec, } impl Source { @@ -127,6 +131,7 @@ impl Source { self_node_name, data_dir, auto_partial_merge: config.auto_partial_merge, + fields_spec: config.annotation_fields.clone(), }) } @@ -140,6 +145,7 @@ impl Source { self_node_name, data_dir, auto_partial_merge, + fields_spec, } = self; let field_selector = format!("spec.nodeName={}", self_node_name); @@ -158,7 +164,7 @@ impl Source { let reflector_process = reflector.run(); let paths_provider = K8sPathsProvider::new(state_reader.clone()); - let annotator = PodMetadataAnnotator::new(state_reader); + let annotator = PodMetadataAnnotator::new(state_reader, fields_spec); // TODO: maybe some of the parameters have to be configurable. let file_server = FileServer { diff --git a/src/sources/kubernetes_logs/pod_metadata_annotator.rs b/src/sources/kubernetes_logs/pod_metadata_annotator.rs index d61b2eed69576..60262146dc734 100644 --- a/src/sources/kubernetes_logs/pod_metadata_annotator.rs +++ b/src/sources/kubernetes_logs/pod_metadata_annotator.rs @@ -9,17 +9,45 @@ use crate::{ }; use evmap10::ReadHandle; use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; +use serde::{Deserialize, Serialize}; use string_cache::Atom; +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(deny_unknown_fields, default)] +pub struct FieldsSpec { + pub pod_name: String, + pub pod_namespace: String, + pub pod_uid: String, + pub pod_labels: String, +} + +impl Default for FieldsSpec { + fn default() -> Self { + Self { + pod_name: "kubernetes.pod_name".to_owned(), + pod_namespace: "kubernetes.pod_namespace".to_owned(), + pod_uid: "kubernetes.pod_uid".to_owned(), + pod_labels: "kubernetes.pod_labels".to_owned(), + } + } +} + /// Annotate the event with pod metadata. pub struct PodMetadataAnnotator { pods_state_reader: ReadHandle>, + fields_spec: FieldsSpec, } impl PodMetadataAnnotator { /// Create a new [`PodMetadataAnnotator`]. - pub fn new(pods_state_reader: ReadHandle>) -> Self { - Self { pods_state_reader } + pub fn new( + pods_state_reader: ReadHandle>, + fields_spec: FieldsSpec, + ) -> Self { + Self { + pods_state_reader, + fields_spec, + } } } @@ -34,7 +62,7 @@ impl PodMetadataAnnotator { let entry = guard.get_one()?; let pod: &Pod = entry.as_ref(); let metadata = pod.metadata.as_ref()?; - annotate_from_metadata(log, &metadata); + annotate_from_metadata(log, &self.fields_spec, &metadata); Some(()) } } @@ -49,11 +77,11 @@ fn pod_uid_from_log_event(log: &LogEvent) -> Option { Some(info.pod_uid.to_owned()) } -fn annotate_from_metadata(log: &mut LogEvent, metadata: &ObjectMeta) { +fn annotate_from_metadata(log: &mut LogEvent, fields_spec: &FieldsSpec, metadata: &ObjectMeta) { for (ref key, ref val) in [ - ("kubernetes.pod_name", &metadata.name), - ("kubernetes.pod_namespace", &metadata.namespace), - ("kubernetes.pod_uid", &metadata.uid), + (&fields_spec.pod_name, &metadata.name), + (&fields_spec.pod_namespace, &metadata.namespace), + (&fields_spec.pod_uid, &metadata.uid), ] .iter() { @@ -64,7 +92,7 @@ fn annotate_from_metadata(log: &mut LogEvent, metadata: &ObjectMeta) { if let Some(labels) = &metadata.labels { for (key, val) in labels.iter() { - log.insert(format!("{}.{}", "kubernetes.pod_labels", key), val); + log.insert(format!("{}.{}", fields_spec.pod_labels, key), val); } } } @@ -107,4 +135,78 @@ mod tests { ); } } + + #[test] + fn test_annotate_from_metadata() { + let cases = vec![ + ( + FieldsSpec::default(), + ObjectMeta::default(), + LogEvent::default(), + ), + ( + FieldsSpec::default(), + ObjectMeta { + name: Some("sandbox0-name".to_owned()), + namespace: Some("sandbox0-ns".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + labels: Some( + vec![ + ("sandbox0-label0".to_owned(), "val0".to_owned()), + ("sandbox0-label1".to_owned(), "val1".to_owned()), + ] + .into_iter() + .collect(), + ), + ..ObjectMeta::default() + }, + { + let mut log = LogEvent::default(); + log.insert("kubernetes.pod_name", "sandbox0-name"); + log.insert("kubernetes.pod_namespace", "sandbox0-ns"); + log.insert("kubernetes.pod_uid", "sandbox0-uid"); + log.insert("kubernetes.pod_labels.sandbox0-label0", "val0"); + log.insert("kubernetes.pod_labels.sandbox0-label1", "val1"); + log + }, + ), + ( + FieldsSpec { + pod_name: "name".to_owned(), + pod_namespace: "ns".to_owned(), + pod_uid: "uid".to_owned(), + pod_labels: "labels".to_owned(), + }, + ObjectMeta { + name: Some("sandbox0-name".to_owned()), + namespace: Some("sandbox0-ns".to_owned()), + uid: Some("sandbox0-uid".to_owned()), + labels: Some( + vec![ + ("sandbox0-label0".to_owned(), "val0".to_owned()), + ("sandbox0-label1".to_owned(), "val1".to_owned()), + ] + .into_iter() + .collect(), + ), + ..ObjectMeta::default() + }, + { + let mut log = LogEvent::default(); + log.insert("name", "sandbox0-name"); + log.insert("ns", "sandbox0-ns"); + log.insert("uid", "sandbox0-uid"); + log.insert("labels.sandbox0-label0", "val0"); + log.insert("labels.sandbox0-label1", "val1"); + log + }, + ), + ]; + + for (fields_spec, metadata, expected) in cases.into_iter() { + let mut log = LogEvent::default(); + annotate_from_metadata(&mut log, &fields_spec, &metadata); + assert_eq!(log, expected); + } + } } From be7adacf8986be491bec18f8e3ef997f00c79a67 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 2 Jun 2020 16:54:00 +0300 Subject: [PATCH 044/118] Abstract reflector around state Adds an indirection layer while maintaining the same logic. This unlocks: - adding debounce to the evmap for advanced state management; - adding removal delay to mitigate the race condition with pod being removed while having a huge log backlog; Those were doable before, but at the cost of code clarity, reliability and maintainability. Signed-off-by: MOZGIII --- src/kubernetes/mod.rs | 1 + src/kubernetes/reflector.rs | 81 ++++++------------- src/kubernetes/state/evmap.rs | 78 ++++++++++++++++++ src/kubernetes/state/mod.rs | 27 +++++++ .../kubernetes_logs/k8s_paths_provider.rs | 4 +- src/sources/kubernetes_logs/mod.rs | 1 + .../kubernetes_logs/pod_metadata_annotator.rs | 4 +- 7 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 src/kubernetes/state/evmap.rs create mode 100644 src/kubernetes/state/mod.rs diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 4eb0dec9a2436..93860063452b1 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -27,6 +27,7 @@ pub mod mock_watcher; pub mod multi_response_decoder; pub mod reflector; pub mod resource_version; +pub mod state; pub mod stream; pub mod watch_request_builder; pub mod watcher; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 63fe536f649a2..54139b02a29a3 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -1,11 +1,9 @@ //! Watch and cache the remote Kubernetes API resources. use super::{ - hash_value::HashValue, resource_version, watcher::{self, Watcher}, }; -use evmap10::WriteHandle; use futures::{pin_mut, stream::StreamExt}; use k8s_openapi::{ apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, @@ -16,33 +14,37 @@ use std::convert::Infallible; use std::time::Duration; use tokio::time::delay_for; +use super::state; + /// Watches remote Kubernetes resources and maintains a local representation of /// the remote state. "Reflects" the remote state locally. /// /// Does not expose evented API, but keeps track of the resource versions and /// will automatically resume on desync. -pub struct Reflector +pub struct Reflector where W: Watcher, ::Object: Metadata, + S: state::Write::Object>, { watcher: W, - state_writer: WriteHandle::Object>>, + state_writer: S, field_selector: Option, label_selector: Option, resource_version: resource_version::State, pause_between_requests: Duration, } -impl Reflector +impl Reflector where W: Watcher, ::Object: Metadata, + S: state::Write::Object>, { /// Create a new [`Cache`]. pub fn new( watcher: W, - state_writer: WriteHandle::Object>>, + state_writer: S, field_selector: Option, label_selector: Option, pause_between_requests: Duration, @@ -59,24 +61,19 @@ where } } -impl Reflector +impl Reflector where W: Watcher, ::Object: Metadata + Unpin + std::fmt::Debug, ::InvocationError: Unpin, ::StreamError: Unpin, + S: state::Write::Object>, { /// Run the watch loop and drive the state updates via `state_writer`. pub async fn run( &mut self, ) -> Result::InvocationError, ::StreamError>> { - // We're taking over the `self.state_writer`, clear it. - self.state_writer.purge(); - // Propagate the purge and ensure we initialize readers for the first - // time. - self.state_writer.refresh(); - // Start the watch loop. loop { let invocation_result = self.issue_request().await; @@ -90,7 +87,7 @@ where // begin arriving, reducing the time durig which the state // has no data. self.resource_version.reset(); - self.state_writer.purge(); + self.state_writer.resync(); continue; } Err(watcher::invocation::Error::Other { source }) => { @@ -141,9 +138,6 @@ where // Record the resourse version for this event, so when we resume // it won't be redelivered. self.resource_version.update(resource_version_candidate); - - // Flush the changes to the state. - self.state_writer.flush(); } // For the next pause duration we won't get any updates. @@ -173,41 +167,20 @@ where fn process_event(&mut self, event: WatchEvent<::Object>) { match event { WatchEvent::Added(object) => { - if let Some((key, value)) = kv(object) { - trace!(message = "got an object event", uid = ?key, event = "added"); - self.state_writer.insert(key, value); - } else { - warn!( - message = "got an object event but unable to obtain a key from object", - event = "added", - ); - } + trace!(message = "got an object event", event = "added"); + self.state_writer.add(object); } WatchEvent::Deleted(object) => { - if let Some((key, _value)) = kv(object) { - trace!(message = "got an object event", uid = ?key, event = "deleted"); - self.state_writer.empty(key); - } else { - warn!( - message = "got an object event but unable to obtain a key from object", - event = "deleted", - ); - } + trace!(message = "got an object event", event = "deleted"); + self.state_writer.delete(object); } WatchEvent::Modified(object) => { - if let Some((key, value)) = kv(object) { - trace!(message = "got an object event", uid = ?key, event = "modified"); - self.state_writer.update(key, value); - } else { - warn!( - message = "got an object event but unable to obtain a key from object", - event = "modified", - ); - } + trace!(message = "got an object event", event = "modified"); + self.state_writer.update(object); } WatchEvent::Bookmark(_object) => { - // noop trace!(message = "got an object event", event = "bookmark"); + // noop } _ => unreachable!("other event types should never reach this code"), } @@ -236,19 +209,12 @@ where }, } -/// An alias to the value used at [`evmap`]. -pub type Value = Box>; - -/// Build a key value pair for using in [`evmap`]. -fn kv>(object: T) -> Option<(String, Value)> { - let value = Box::new(HashValue::new(object)); - let key = value.uid()?.to_owned(); - Some((key, value)) -} - #[cfg(test)] mod tests { - use super::{Reflector, Value}; + use super::{ + state::evmap::{Value, Writer}, + Reflector, + }; use crate::{ kubernetes::mock_watcher::{InvocationError, MockWatcher}, kubernetes::watcher, @@ -324,6 +290,7 @@ mod tests { // Prepare the test flow. let (_state_reader, state_writer) = evmap10::new(); + let state_writer = Writer::new(state_writer); let mock_logic = move |_watch_optional: WatchOptional<'_>| { if false { return Ok(|| None); // for type inferrence @@ -359,6 +326,7 @@ mod tests { ) { // Prepare the test flow. let (state_reader, state_writer) = evmap10::new(); + let state_writer = Writer::new(state_writer); let assertion_state_reader = state_reader.clone(); let flow_expected_resulting_state = expected_resulting_state.clone(); @@ -529,6 +497,7 @@ mod tests { test_util::trace_init(); let (_state_reader, state_writer) = evmap10::new(); + let state_writer = Writer::new(state_writer); let mock_logic = move |watch_optional: WatchOptional<'_>| { assert_eq!(watch_optional.field_selector, Some("fields")); assert_eq!(watch_optional.label_selector, Some("labels")); diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs new file mode 100644 index 0000000000000..f8f91dbd3dd02 --- /dev/null +++ b/src/kubernetes/state/evmap.rs @@ -0,0 +1,78 @@ +//! A state implementation backed by [`evmap10`]. + +use crate::kubernetes::hash_value::HashValue; +use evmap10::WriteHandle; +use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; + +/// A [`WriteHandle`] wrapper that implements [`super::Write`]. +/// For use as a state writer implementation for +/// [`crate::kubernetes::Reflector`]. +pub struct Writer +where + T: Metadata, +{ + inner: WriteHandle>, +} + +impl Writer +where + T: Metadata, +{ + /// Take a [`WriteHandle`], initialize it and return it wrapped with + /// [`Self`]. + pub fn new(mut inner: WriteHandle>) -> Self { + inner.purge(); + inner.refresh(); + Self { inner } + } +} + +impl super::Write for Writer +where + T: Metadata, +{ + type Item = T; + + // TODO: debounce `flush` so that when a bunch of events arrive in a row + // within a certain small time window we commit all of them at once. + // This will improve the state behaivor at resync. + + fn add(&mut self, item: Self::Item) { + if let Some((key, value)) = kv(item) { + self.inner.insert(key, value); + self.inner.flush(); + } + } + + fn update(&mut self, item: Self::Item) { + if let Some((key, value)) = kv(item) { + self.inner.update(key, value); + self.inner.flush(); + } + } + + fn delete(&mut self, item: Self::Item) { + if let Some((key, _value)) = kv(item) { + self.inner.empty(key); + self.inner.flush(); + } + } + + fn resync(&mut self) { + self.inner.purge(); + // We do not flush on resync until the next per-item operation + // arrives. + // This way we preserve the old state while waiting for the data to + // populate the new state. + } +} + +/// An alias to the value used at [`evmap`]. +pub type Value = Box>; + +/// Build a key value pair for using in [`evmap`]. +fn kv>(object: T) -> Option<(String, Value)> { + let value = Box::new(HashValue::new(object)); + let key = value.uid()?.to_owned(); + Some((key, value)) +} diff --git a/src/kubernetes/state/mod.rs b/src/kubernetes/state/mod.rs new file mode 100644 index 0000000000000..a6c8421c128e4 --- /dev/null +++ b/src/kubernetes/state/mod.rs @@ -0,0 +1,27 @@ +//! Local representation of the Kubernetes API resources state. + +use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; + +pub mod evmap; + +/// Provides the interface for write access to the cached state. +/// Used by [`super::reflector::Reflector`]. +/// +/// This abstraction allows easily stacking storage behaviour logic, without +/// exploding the complexity at the [`super::reflector::Reflector`] level. +pub trait Write { + /// A type of the k8s resource the state operates on. + type Item: Metadata; + + /// Add an object to the state. + fn add(&mut self, item: Self::Item); + + /// Update an object at the state. + fn update(&mut self, item: Self::Item); + + /// Delete on object from the state. + fn delete(&mut self, item: Self::Item); + + /// Notify the state that resync is in progress. + fn resync(&mut self); +} diff --git a/src/sources/kubernetes_logs/k8s_paths_provider.rs b/src/sources/kubernetes_logs/k8s_paths_provider.rs index ee3d48d644f45..efcfec3db9a82 100644 --- a/src/sources/kubernetes_logs/k8s_paths_provider.rs +++ b/src/sources/kubernetes_logs/k8s_paths_provider.rs @@ -12,12 +12,12 @@ use std::path::PathBuf; /// A paths provider implementation that uses the state obtained from the /// the k8s API. pub struct K8sPathsProvider { - pods_state_reader: ReadHandle>, + pods_state_reader: ReadHandle>, } impl K8sPathsProvider { /// Create a new [`K8sPathsProvider`]. - pub fn new(pods_state_reader: ReadHandle>) -> Self { + pub fn new(pods_state_reader: ReadHandle>) -> Self { Self { pods_state_reader } } } diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 23f68992d9151..c1327bc4d91d5 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -153,6 +153,7 @@ impl Source { let watcher = k8s::api_watcher::ApiWatcher::new(client, Pod::watch_pod_for_all_namespaces); let (state_reader, state_writer) = evmap::new(); + let state_writer = k8s::state::evmap::Writer::new(state_writer); let mut reflector = k8s::reflector::Reflector::new( watcher, diff --git a/src/sources/kubernetes_logs/pod_metadata_annotator.rs b/src/sources/kubernetes_logs/pod_metadata_annotator.rs index 60262146dc734..e093069d84da6 100644 --- a/src/sources/kubernetes_logs/pod_metadata_annotator.rs +++ b/src/sources/kubernetes_logs/pod_metadata_annotator.rs @@ -34,14 +34,14 @@ impl Default for FieldsSpec { /// Annotate the event with pod metadata. pub struct PodMetadataAnnotator { - pods_state_reader: ReadHandle>, + pods_state_reader: ReadHandle>, fields_spec: FieldsSpec, } impl PodMetadataAnnotator { /// Create a new [`PodMetadataAnnotator`]. pub fn new( - pods_state_reader: ReadHandle>, + pods_state_reader: ReadHandle>, fields_spec: FieldsSpec, ) -> Self { Self { From 685f0f6d49c7cd74f9ca929fd67c62ed897da72f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 2 Jun 2020 22:05:30 +0300 Subject: [PATCH 045/118] Add support for delayed delete at reflector Signed-off-by: MOZGIII --- src/kubernetes/delayed_delete.rs | 53 ++++++++++ src/kubernetes/mod.rs | 1 + src/kubernetes/reflector.rs | 162 +++++++++++++++++++++-------- src/sources/kubernetes_logs/mod.rs | 1 + 4 files changed, 172 insertions(+), 45 deletions(-) create mode 100644 src/kubernetes/delayed_delete.rs diff --git a/src/kubernetes/delayed_delete.rs b/src/kubernetes/delayed_delete.rs new file mode 100644 index 0000000000000..47bd529c68825 --- /dev/null +++ b/src/kubernetes/delayed_delete.rs @@ -0,0 +1,53 @@ +//! A delayed delete logic. + +use super::state; +use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; +use std::{ + collections::VecDeque, + time::{Duration, Instant}, +}; + +/// Implements the logic for delaying the deletion of items from the storage. +pub struct DelayedDelete { + queue: VecDeque<(T, Instant)>, + delay_for: Duration, +} + +impl DelayedDelete { + /// Create a new [`DelayedDelete`] state. + pub fn new(delay_for: Duration) -> Self { + let queue = VecDeque::new(); + Self { queue, delay_for } + } + + /// Schedules the delayed deletion of the item at the future. + pub fn schedule_delete(&mut self, item: T) { + let deadline = Instant::now() + self.delay_for; + self.queue.push_back((item, deadline)); + } + + /// Clear the delayed deletion requests. + pub fn clear(&mut self) { + self.queue.clear(); + } + + /// Perform the queued deletions. + pub fn perform(&mut self, state_writer: &mut impl state::Write) + where + T: Metadata, + { + let now = Instant::now(); + while let Some(deadline) = self.next_deadline() { + if deadline > now { + break; + } + let (item, _) = self.queue.pop_front().unwrap(); + state_writer.delete(item); + } + } + + /// Obtain the next deadline. + pub fn next_deadline(&self) -> Option { + self.queue.front().map(|(_, instant)| *instant) + } +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 93860063452b1..886536ae7946f 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,6 +22,7 @@ pub mod api_watcher; pub mod client; +pub mod delayed_delete; pub mod hash_value; pub mod mock_watcher; pub mod multi_response_decoder; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 54139b02a29a3..c1d470e1c9d1b 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -4,7 +4,10 @@ use super::{ resource_version, watcher::{self, Watcher}, }; -use futures::{pin_mut, stream::StreamExt}; +use futures::{ + pin_mut, + stream::{Stream, StreamExt}, +}; use k8s_openapi::{ apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, Metadata, WatchOptional, WatchResponse, @@ -12,9 +15,12 @@ use k8s_openapi::{ use snafu::Snafu; use std::convert::Infallible; use std::time::Duration; -use tokio::time::delay_for; +use tokio::{ + select, + time::{delay_for, delay_until}, +}; -use super::state; +use super::{delayed_delete::DelayedDelete, state}; /// Watches remote Kubernetes resources and maintains a local representation of /// the remote state. "Reflects" the remote state locally. @@ -33,6 +39,7 @@ where label_selector: Option, resource_version: resource_version::State, pause_between_requests: Duration, + delayed_delete: Option::Object>>, } impl Reflector @@ -48,8 +55,10 @@ where field_selector: Option, label_selector: Option, pause_between_requests: Duration, + delay_deletes_for: Option, ) -> Self { let resource_version = resource_version::State::new(); + let delayed_delete = delay_deletes_for.map(DelayedDelete::new); Self { watcher, state_writer, @@ -57,6 +66,7 @@ where field_selector, resource_version, pause_between_requests, + delayed_delete, } } } @@ -87,6 +97,9 @@ where // begin arriving, reducing the time durig which the state // has no data. self.resource_version.reset(); + if let Some(ref mut delayed_delete) = self.delayed_delete { + delayed_delete.clear(); + } self.state_writer.resync(); continue; } @@ -98,46 +111,40 @@ where }; pin_mut!(stream); - while let Some(response) = stream.next().await { - // Any streaming error means the protocol is in an unxpected - // state. This is considered a fatal error, do not attempt - // to retry and just quit. - let response = response.map_err(|source| Error::Streaming { source })?; - - // Unpack the event. - let event = match response { - WatchResponse::Ok(event) => event, - WatchResponse::Other(_) => { - // Even though we could parse the response, we didn't - // get the data we expected on the wire. - // According to the rules, we just ignore the unknown - // responses. This may be a newly added piece of data - // our code doesn't know of. - // TODO: add more details on the data here if we - // encounter these messages in practice. - warn!(message = "got unexpected data in the watch response"); - continue; - } - }; - - // Prepare a resource version candidate so we can update (aka - // commit) it later. - let resource_version_candidate = - match resource_version::Candidate::from_watch_event(&event) { - Some(val) => val, - None => { - // This event doesn't have a resource version, this means - // it's not something we care about. + loop { + // Obtain an value from the watch stream. + let val = if let Some(ref mut delayed_delete) = self.delayed_delete { + // If delayed delete is requested, we perform the delayed + // deletions concurrently with reading items from the watch + // responses stream. + let delayed_delete_deadline = delayed_delete.next_deadline(); + select! { + // If we get a delayted delete deadline - process the + // delayed deletes and restart the loop. + _ = async { delay_until(delayed_delete_deadline.unwrap().into()).await }, if delayed_delete_deadline.is_some() => { + delayed_delete.perform(&mut self.state_writer); continue; } - }; - - // Process the event. - self.process_event(event); + // If we got a value from the watch responses stream - + // just pass it outside. + val = stream.next() => val, + } + } else { + // Delayed deletes aren't requested, so just wait for the + // next value and pass it outside. + stream.next().await + }; - // Record the resourse version for this event, so when we resume - // it won't be redelivered. - self.resource_version.update(resource_version_candidate); + if let Some(item) = val { + // A new item arrived from the watch response stream + // first - process it. + self.process_stream_item(item)?; + } else { + // Response stream has ended. + // Break the watch reading loop so the flow can + // continue an issue a new watch request. + break; + } } // For the next pause duration we won't get any updates. @@ -163,6 +170,54 @@ where Ok(stream) } + /// Process an item from the watch response stream. + fn process_stream_item( + &mut self, + item: <::Stream as Stream>::Item, + ) -> Result<(), Error<::InvocationError, ::StreamError>> { + // Any streaming error means the protocol is in an unxpected + // state. This is considered a fatal error, do not attempt + // to retry and just quit. + let response = item.map_err(|source| Error::Streaming { source })?; + + // Unpack the event. + let event = match response { + WatchResponse::Ok(event) => event, + WatchResponse::Other(_) => { + // Even though we could parse the response, we didn't + // get the data we expected on the wire. + // According to the rules, we just ignore the unknown + // responses. This may be a newly added piece of data + // our code doesn't know of. + // TODO: add more details on the data here if we + // encounter these messages in practice. + warn!(message = "got unexpected data in the watch response"); + return Ok(()); + } + }; + + // Prepare a resource version candidate so we can update (aka commit) it + // later. + let resource_version_candidate = match resource_version::Candidate::from_watch_event(&event) + { + Some(val) => val, + None => { + // This event doesn't have a resource version, this means + // it's not something we care about. + return Ok(()); + } + }; + + // Process the event. + self.process_event(event); + + // Record the resourse version for this event, so when we resume + // it won't be redelivered. + self.resource_version.update(resource_version_candidate); + + Ok(()) + } + /// Translate received watch event to the state update. fn process_event(&mut self, event: WatchEvent<::Object>) { match event { @@ -172,7 +227,11 @@ where } WatchEvent::Deleted(object) => { trace!(message = "got an object event", event = "deleted"); - self.state_writer.delete(object); + if let Some(ref mut delayed_delete) = self.delayed_delete { + delayed_delete.schedule_delete(object); + } else { + self.state_writer.delete(object); + } } WatchEvent::Modified(object) => { trace!(message = "got an object event", event = "modified"); @@ -300,8 +359,14 @@ mod tests { // Prepare watcher and reflector. let watcher: MockWatcher = MockWatcher::new(mock_logic); - let mut reflector = - Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + None, + ); // Acquire an async context to run the relector. test_util::block_on_std(async move { @@ -368,8 +433,14 @@ mod tests { // Prepare watcher and reflector. let watcher: MockWatcher = MockWatcher::new(mock_logic); - let mut reflector = - Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + None, + ); // Acquire an async context to run the relector. test_util::block_on_std(async move { @@ -519,6 +590,7 @@ mod tests { Some("fields".to_owned()), Some("labels".to_owned()), Duration::from_secs(1), + Some(Duration::from_secs(60)), ); // Acquire an async context to run the relector. diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index c1327bc4d91d5..822d01ee87079 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -161,6 +161,7 @@ impl Source { Some(field_selector), Some(label_selector), std::time::Duration::from_secs(1), + Some(std::time::Duration::from_secs(60)), ); let reflector_process = reflector.run(); From 2b57f5e1fdd2defed2b041728154afb0219d36a3 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 4 Jun 2020 19:11:50 +0300 Subject: [PATCH 046/118] Reimplement the tests to add delayed deletion testing capability New channel-based mocks implementation allows to properly eliminate race conditions and achieve the complete determinism and reliability of the test scenario. Signed-off-by: MOZGIII --- src/kubernetes/delayed_delete.rs | 84 ++++- src/kubernetes/mock_watcher.rs | 171 +++++++--- src/kubernetes/reflector.rs | 558 +++++++++++++++++++++++-------- src/kubernetes/state/evmap.rs | 16 +- src/kubernetes/state/mock.rs | 114 +++++++ src/kubernetes/state/mod.rs | 13 +- 6 files changed, 748 insertions(+), 208 deletions(-) create mode 100644 src/kubernetes/state/mock.rs diff --git a/src/kubernetes/delayed_delete.rs b/src/kubernetes/delayed_delete.rs index 47bd529c68825..6343a51775600 100644 --- a/src/kubernetes/delayed_delete.rs +++ b/src/kubernetes/delayed_delete.rs @@ -2,10 +2,8 @@ use super::state; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; -use std::{ - collections::VecDeque, - time::{Duration, Instant}, -}; +use std::{collections::VecDeque, time::Duration}; +use tokio::time::Instant; /// Implements the logic for delaying the deletion of items from the storage. pub struct DelayedDelete { @@ -32,17 +30,23 @@ impl DelayedDelete { } /// Perform the queued deletions. - pub fn perform(&mut self, state_writer: &mut impl state::Write) + pub async fn perform(&mut self, state_writer: &mut impl state::Write) where - T: Metadata, + T: Metadata + Send, { let now = Instant::now(); while let Some(deadline) = self.next_deadline() { + trace!(message = "got delayed deletion deadline", ?deadline, ?now); if deadline > now { break; } + trace!( + message = "processing delayed deletion for deadline", + ?deadline, + ?now + ); let (item, _) = self.queue.pop_front().unwrap(); - state_writer.delete(item); + state_writer.delete(item).await; } } @@ -51,3 +55,69 @@ impl DelayedDelete { self.queue.front().map(|(_, instant)| *instant) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_util; + use futures::{channel::mpsc::channel, SinkExt, StreamExt}; + use k8s_openapi::api::core::v1::Pod; + + #[test] + fn logic() { + test_util::trace_init(); + test_util::block_on_std(async move { + tokio::time::pause(); + + let (state_events_tx, mut state_events_rx) = channel(0); + let (mut state_actions_tx, state_actions_rx) = channel(0); + let mut writer = state::mock::Writer::::new(state_events_tx, state_actions_rx); + + let mut delayed_delete = DelayedDelete::new(Duration::from_millis(10000)); + + { + assert!(delayed_delete.next_deadline().is_none()); + delayed_delete.perform(&mut writer).await; + assert!(delayed_delete.next_deadline().is_none()); + assert!(state_events_rx.try_next().is_err()); + } + + delayed_delete.schedule_delete(Pod::default()); + + { + assert!(delayed_delete.next_deadline().is_some()); + delayed_delete.perform(&mut writer).await; + assert!(delayed_delete.next_deadline().is_some()); + assert!(state_events_rx.try_next().is_err()); + } + + tokio::time::advance(Duration::from_millis(50000)).await; + + let (mut state_events_rx, _state_actions_tx) = { + let conc_fut = tokio::spawn(async move { + assert_eq!( + state_events_rx.next().await.unwrap().unwrap_op(), + (Pod::default(), state::mock::OpKind::Delete) + ); + state_actions_tx.send(()).await.unwrap(); + (state_events_rx, state_actions_tx) + }); + + assert!(delayed_delete.next_deadline().is_some()); + delayed_delete.perform(&mut writer).await; + assert!(delayed_delete.next_deadline().is_none()); + + conc_fut.await.unwrap() + }; + + { + assert!(delayed_delete.next_deadline().is_none()); + delayed_delete.perform(&mut writer).await; + assert!(delayed_delete.next_deadline().is_none()); + assert!(state_events_rx.try_next().is_err()); + } + + tokio::time::resume(); + }); + } +} diff --git a/src/kubernetes/mock_watcher.rs b/src/kubernetes/mock_watcher.rs index 2c1ba7e063bf5..04a962f12c755 100644 --- a/src/kubernetes/mock_watcher.rs +++ b/src/kubernetes/mock_watcher.rs @@ -3,46 +3,77 @@ #![cfg(test)] use super::watcher::{self, Watcher}; -use futures::{ - future::{self, BoxFuture}, - stream::BoxStream, - Stream, -}; +use async_stream::try_stream; +use futures::channel::mpsc::{Receiver, Sender}; +use futures::{future::BoxFuture, stream::BoxStream, SinkExt, StreamExt}; use k8s_openapi::{WatchOptional, WatchResponse}; -use pin_project::pin_project; use serde::de::DeserializeOwned; -use std::marker::PhantomData; -use std::{ - fmt, - pin::Pin, - task::{Context, Poll}, -}; +use std::fmt; + +/// An event that's send to the test scenario driver. +#[derive(Debug, PartialEq)] +pub enum ScenarioEvent { + Invocation(OwnedWatchOptional), + Stream, +} + +/// An action that's send from the test scenario driver to specify the +/// invocation result. +pub enum ScenarioActionInvocation +where + T: DeserializeOwned, +{ + /// Return successfully and prepare the stream with responses from the + /// passed [`Receiver`]. + Ok(Receiver>), + /// Return a desync error. + ErrDesync, + /// Return an "other" (i.e. non-desync) error. + ErrOther, +} + +/// An action that's send from the test scenario driver to specify the +/// stream item request result. +pub enum ScenarioActionStream +where + T: DeserializeOwned, +{ + /// Return a watch response. + Ok(WatchResponse), + /// Return an error. + Err, + /// Complete the stream (return `None`). + Done, +} /// A mock watcher, useful for tests. -#[derive(Debug)] -pub struct MockWatcher { - data_type: PhantomData, - invoke_fn: FI, +pub struct MockWatcher +where + T: DeserializeOwned, +{ + events_tx: Sender, + invocation_rx: Receiver>, } -impl MockWatcher { +impl MockWatcher +where + T: DeserializeOwned, +{ /// Create a new [`MockWatcher`]. - pub fn new(invoke_fn: FI) -> Self { - let data_type = PhantomData; + pub fn new( + events_tx: Sender, + invocation_rx: Receiver>, + ) -> Self { Self { - data_type, - invoke_fn, + events_tx, + invocation_rx, } } } -impl Watcher for MockWatcher +impl Watcher for MockWatcher where - T: DeserializeOwned + Send + Sync + 'static, - FI: for<'a> FnMut(WatchOptional<'a>) -> Result> - + Send - + 'static, - FS: FnMut() -> Option, StreamError>> + Send + Sync + Unpin + 'static, + T: DeserializeOwned + Send + Sync + Unpin + 'static, { type Object = T; @@ -56,33 +87,75 @@ where watch_optional: WatchOptional<'a>, ) -> BoxFuture<'a, Result>> { - let result = (self.invoke_fn)(watch_optional); - let result = result.map(|stream_fn| { - Box::pin(MockWatcherStream { - stream_fn, - data_type: PhantomData, - }) as Self::Stream - }); - Box::pin(future::ready(result)) + let mut stream_events_tx = self.events_tx.clone(); + Box::pin(async move { + self.events_tx + .send(ScenarioEvent::Invocation(watch_optional.into())) + .await + .unwrap(); + + let action = self.invocation_rx.next().await.unwrap(); + match action { + ScenarioActionInvocation::Ok(mut stream_rx) => { + let stream = Box::pin(try_stream! { + loop { + stream_events_tx.send(ScenarioEvent::Stream) + .await + .unwrap(); + + let action = stream_rx.next().await.unwrap(); + match action { + ScenarioActionStream::Ok(val) => { + yield val + }, + ScenarioActionStream::Err => { + Err(StreamError)?; + break; + }, + ScenarioActionStream::Done => break, + } + } + }) + as BoxStream< + 'static, + Result, Self::StreamError>, + >; + Ok(stream) + } + ScenarioActionInvocation::ErrDesync => { + Err(watcher::invocation::Error::desync(InvocationError)) + } + ScenarioActionInvocation::ErrOther => { + Err(watcher::invocation::Error::other(InvocationError)) + } + } + }) } } -#[pin_project] -struct MockWatcherStream { - data_type: PhantomData, - stream_fn: FS, +/// An owned variant of [`WatchOptional`]. +/// Used to send it with [`ScenarioEvent`] to avoid the headaches with +/// lifetimes. +#[derive(Debug, PartialEq, Eq)] +pub struct OwnedWatchOptional { + pub allow_watch_bookmarks: Option, + pub field_selector: Option, + pub label_selector: Option, + pub pretty: Option, + pub resource_version: Option, + pub timeout_seconds: Option, } -impl Stream for MockWatcherStream -where - T: DeserializeOwned + Send + Sync, - FS: FnMut() -> Option, StreamError>> + Send + Sync + Unpin + 'static, -{ - type Item = Result, StreamError>; - - fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { - let this = self.project(); - Poll::Ready((this.stream_fn)()) +impl<'a> From> for OwnedWatchOptional { + fn from(val: WatchOptional<'a>) -> Self { + Self { + allow_watch_bookmarks: val.allow_watch_bookmarks, + field_selector: val.field_selector.map(ToOwned::to_owned), + label_selector: val.label_selector.map(ToOwned::to_owned), + pretty: val.pretty.map(ToOwned::to_owned), + resource_version: val.resource_version.map(ToOwned::to_owned), + timeout_seconds: val.timeout_seconds, + } } } diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index c1d470e1c9d1b..b3c7537c7d497 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -30,7 +30,7 @@ use super::{delayed_delete::DelayedDelete, state}; pub struct Reflector where W: Watcher, - ::Object: Metadata, + ::Object: Metadata + Send, S: state::Write::Object>, { watcher: W, @@ -45,7 +45,7 @@ where impl Reflector where W: Watcher, - ::Object: Metadata, + ::Object: Metadata + Send, S: state::Write::Object>, { /// Create a new [`Cache`]. @@ -74,7 +74,7 @@ where impl Reflector where W: Watcher, - ::Object: Metadata + Unpin + std::fmt::Debug, + ::Object: Metadata + Send + Unpin + std::fmt::Debug, ::InvocationError: Unpin, ::StreamError: Unpin, S: state::Write::Object>, @@ -100,7 +100,7 @@ where if let Some(ref mut delayed_delete) = self.delayed_delete { delayed_delete.clear(); } - self.state_writer.resync(); + self.state_writer.resync().await; continue; } Err(watcher::invocation::Error::Other { source }) => { @@ -122,7 +122,8 @@ where // If we get a delayted delete deadline - process the // delayed deletes and restart the loop. _ = async { delay_until(delayed_delete_deadline.unwrap().into()).await }, if delayed_delete_deadline.is_some() => { - delayed_delete.perform(&mut self.state_writer); + trace!(message = "performing delayed deletes"); + delayed_delete.perform(&mut self.state_writer).await; continue; } // If we got a value from the watch responses stream - @@ -134,11 +135,12 @@ where // next value and pass it outside. stream.next().await }; + trace!(message = "got an item from watch stream"); if let Some(item) = val { // A new item arrived from the watch response stream // first - process it. - self.process_stream_item(item)?; + self.process_stream_item(item).await?; } else { // Response stream has ended. // Break the watch reading loop so the flow can @@ -171,7 +173,7 @@ where } /// Process an item from the watch response stream. - fn process_stream_item( + async fn process_stream_item( &mut self, item: <::Stream as Stream>::Item, ) -> Result<(), Error<::InvocationError, ::StreamError>> { @@ -209,7 +211,7 @@ where }; // Process the event. - self.process_event(event); + self.process_event(event).await; // Record the resourse version for this event, so when we resume // it won't be redelivered. @@ -219,23 +221,23 @@ where } /// Translate received watch event to the state update. - fn process_event(&mut self, event: WatchEvent<::Object>) { + async fn process_event(&mut self, event: WatchEvent<::Object>) { match event { WatchEvent::Added(object) => { trace!(message = "got an object event", event = "added"); - self.state_writer.add(object); + self.state_writer.add(object).await; } WatchEvent::Deleted(object) => { trace!(message = "got an object event", event = "deleted"); if let Some(ref mut delayed_delete) = self.delayed_delete { delayed_delete.schedule_delete(object); } else { - self.state_writer.delete(object); + self.state_writer.delete(object).await; } } WatchEvent::Modified(object) => { trace!(message = "got an object event", event = "modified"); - self.state_writer.update(object); + self.state_writer.update(object).await; } WatchEvent::Bookmark(_object) => { trace!(message = "got an object event", event = "bookmark"); @@ -275,14 +277,17 @@ mod tests { Reflector, }; use crate::{ - kubernetes::mock_watcher::{InvocationError, MockWatcher}, - kubernetes::watcher, + kubernetes::{ + mock_watcher::{self, MockWatcher}, + state, + }, test_util, }; + use futures::{channel::mpsc, SinkExt, StreamExt}; use k8s_openapi::{ api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, - Metadata, WatchOptional, WatchResponse, + Metadata, WatchResponse, }; use std::time::Duration; @@ -346,114 +351,54 @@ mod tests { #[test] fn simple_test() { test_util::trace_init(); - - // Prepare the test flow. - let (_state_reader, state_writer) = evmap10::new(); - let state_writer = Writer::new(state_writer); - let mock_logic = move |_watch_optional: WatchOptional<'_>| { - if false { - return Ok(|| None); // for type inferrence - } - Err(watcher::invocation::Error::other(InvocationError)) - }; - - // Prepare watcher and reflector. - let watcher: MockWatcher = MockWatcher::new(mock_logic); - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - None, - ); - - // Acquire an async context to run the relector. test_util::block_on_std(async move { + // Prepare state. + let (state_events_tx, _state_events_rx) = mpsc::channel(0); + let (_state_action_tx, state_action_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + + // Prepare reflector. + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + None, + ); + + // Run test logic. + let logic = tokio::spawn(async move { + // Wait for watcher to request next invocation. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + + // We're done with the test, send the error to terminate the + // reflector. + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrOther) + .await + .unwrap(); + }); + // Run the test and wait for an error. let result = reflector.run().await; - // The only way reflector completes is with an error, but that's ok. - // In tests we make it exit with an error to complete the test. - result.unwrap_err(); - - // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. - drop(reflector); - }); - } - - // A helper function to run a flow test. - // Use this to test various flows without an actual test repetition. - fn run_flow_test( - invocations: Vec<(StateSnapshot, Option, ExpInvRes)>, - expected_resulting_state: StateSnapshot, - ) { - // Prepare the test flow. - let (state_reader, state_writer) = evmap10::new(); - let state_writer = Writer::new(state_writer); - - let assertion_state_reader = state_reader.clone(); - let flow_expected_resulting_state = expected_resulting_state.clone(); - - let mut flow_invocations = invocations.into_iter(); - - let mock_logic = move |watch_optional: WatchOptional<'_>| { - if let Some(( - expected_state_before_op, - expected_resource_version, - expected_invocation_response, - )) = flow_invocations.next() - { - // Assert the state prior to the operation. - let state: StateSnapshot = gather_state(&assertion_state_reader); - assert_eq!(state, expected_state_before_op); - - assert_eq!( - expected_resource_version, - watch_optional.resource_version.map(ToOwned::to_owned) // work around the borrow checker issues - ); - - let responses = match expected_invocation_response { - ExpInvRes::Stream(responses) => responses, - ExpInvRes::Desync => { - return Err(watcher::invocation::Error::desync(InvocationError)) - } - }; - let mut responses_iter = responses.into_iter(); - return Ok(move || responses_iter.next().map(|val| Ok(WatchResponse::Ok(val)))); - } - - let resulting_state: StateSnapshot = gather_state(&assertion_state_reader); - assert_eq!(resulting_state, flow_expected_resulting_state); + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); - Err(watcher::invocation::Error::other(InvocationError)) - }; - - // Prepare watcher and reflector. - let watcher: MockWatcher = MockWatcher::new(mock_logic); - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - None, - ); - - // Acquire an async context to run the relector. - test_util::block_on_std(async move { - // Run the test and wait for an error. - let result = reflector.run().await; // The only way reflector completes is with an error, but that's ok. // In tests we make it exit with an error to complete the test. result.unwrap_err(); - // Assert the state after the reflector exit. - let resulting_state: StateSnapshot = gather_state(&state_reader); - assert_eq!(resulting_state, expected_resulting_state); - // Explicitly drop the reflector at the very end. // Internal evmap is dropped with the reflector, so readers won't // work after drop. @@ -562,41 +507,224 @@ mod tests { run_flow_test(invocations, expected_resulting_state); } - // Test that [`WatchOptional`] is populated properly. + // Test that [`k8s_openapi::WatchOptional`] is populated properly. #[test] fn arguments_test() { test_util::trace_init(); + test_util::block_on_std(async move { + // Prepare state. + let (state_events_tx, _state_events_rx) = mpsc::channel(0); + let (_state_action_tx, state_action_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + + // Prepare reflector. + let mut reflector = Reflector::new( + watcher, + state_writer, + Some("fields".to_owned()), + Some("labels".to_owned()), + Duration::from_secs(1), + None, + ); + + // Run test logic. + let logic = tokio::spawn(async move { + // Wait for watcher to request next invocation. + let invocation_event = watcher_events_rx.next().await.unwrap(); + + // Assert that we obtained an invocation event and obtain + // the passed `watch_optional`. + let watch_optional = match invocation_event { + mock_watcher::ScenarioEvent::Invocation(val) => val, + _ => panic!("unexpected event from watcher mock"), + }; - let (_state_reader, state_writer) = evmap10::new(); - let state_writer = Writer::new(state_writer); - let mock_logic = move |watch_optional: WatchOptional<'_>| { - assert_eq!(watch_optional.field_selector, Some("fields")); - assert_eq!(watch_optional.label_selector, Some("labels")); - assert_eq!(watch_optional.allow_watch_bookmarks, Some(true)); - assert_eq!(watch_optional.pretty, None); - assert_eq!(watch_optional.timeout_seconds, None); - - if false { - return Ok(|| None); // for type inferrence - } - Err(watcher::invocation::Error::other(InvocationError)) - }; + // Assert that the arguments are passed properly. + assert_eq!( + watch_optional, + mock_watcher::OwnedWatchOptional { + allow_watch_bookmarks: Some(true), + field_selector: Some("fields".to_owned()), + label_selector: Some("labels".to_owned()), + pretty: None, + resource_version: None, + timeout_seconds: None, + } + ); - let watcher: MockWatcher = MockWatcher::new(mock_logic); + // We're done with the test, send the error to terminate the + // reflector. + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrOther) + .await + .unwrap(); + }); - let mut reflector = Reflector::new( - watcher, - state_writer, - Some("fields".to_owned()), - Some("labels".to_owned()), - Duration::from_secs(1), - Some(Duration::from_secs(60)), - ); + // Run the test and wait for an error. + let result = reflector.run().await; + + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); + + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }) + } - // Acquire an async context to run the relector. + /// Test that the delayed delete works accordingly. + #[test] + fn test_delayed_deletes() { + test_util::trace_init(); test_util::block_on_std(async move { + // Prepare state. + let (state_events_tx, mut state_events_rx) = mpsc::channel(0); + let (mut state_action_tx, state_action_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + + // Prepare reflector. + let deletion_delay = Duration::from_secs(600); + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + Some(deletion_delay), + ); + + // Run test logic. + let logic = tokio::spawn(async move { + // Pause the time. + tokio::time::pause(); + + // Wait for watcher to request next invocation. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + + // Provide watcher with a new stream. + let (mut watch_stream_tx, watch_stream_rx) = mpsc::channel(0); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::Ok(watch_stream_rx)) + .await + .unwrap(); + + // Wait for watcher to request next item from the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Send pod addition to a stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Ok(WatchResponse::Ok( + WatchEvent::Added(make_pod("uid0", "10")), + ))) + .await + .unwrap(); + + // Let the reflector work until the pod addition propagates to + // the state. + assert_eq!( + state_events_rx.next().await.unwrap().unwrap_op(), + (make_pod("uid0", "10"), state::mock::OpKind::Add), + ); + + // Send the confirmation of the processing at the state. + state_action_tx.send(()).await.unwrap(); + + // Let the reflector work until watcher requests next event from + // the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Send pod deletion to a stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Ok(WatchResponse::Ok( + WatchEvent::Deleted(make_pod("uid0", "15")), + ))) + .await + .unwrap(); + + // Let the reflector work until watcher requests next event from + // the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Assert that the state didn't get the deletion (yet). + // State completes before the next item is requested from the + // watch stream, and since we waited for the stream item to + // be requested - we're guaranteed to have no race condition + // here. + assert!(state_events_rx.try_next().is_err()); + + // Advance the time 10 times the deletion delay. + tokio::time::advance(deletion_delay * 10).await; + + // At this point, pod deletion should propagate to the state. + // Let the reflector work until the deletion reaches the state. + assert_eq!( + state_events_rx.next().await.unwrap().unwrap_op(), + (make_pod("uid0", "15"), state::mock::OpKind::Delete), + ); + + // Send the confirmation of the processing at the state. + state_action_tx.send(()).await.unwrap(); + + // We're done with the test! Shutdown the stream and force an + // invocation error to terminate the reflector. + + // Watcher is still waiting for the item on stream. + // Send done notification to the stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Done) + .await + .unwrap(); + + // Wait for next invocation and send an error to terminate the + // flow. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrOther) + .await + .unwrap(); + + // Resume the time. + tokio::time::resume(); + }); + // Run the test and wait for an error. let result = reflector.run().await; + + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); + // The only way reflector completes is with an error, but that's ok. // In tests we make it exit with an error to complete the test. result.unwrap_err(); @@ -605,6 +733,156 @@ mod tests { // Internal evmap is dropped with the reflector, so readers won't // work after drop. drop(reflector); - }); + }) + } + + // A helper function to run a flow test. + // Use this to test various flows without the test code repetition. + fn run_flow_test( + invocations: Vec<(StateSnapshot, Option, ExpInvRes)>, + expected_resulting_state: StateSnapshot, + ) { + test_util::block_on_std(async move { + // Prepare state. + let (state_reader, state_writer) = evmap10::new(); + let state_writer = Writer::new(state_writer); + let resulting_state_reader = state_reader.clone(); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher: MockWatcher = + MockWatcher::new(watcher_events_tx, watcher_invocations_rx); + + // Prepare reflector. + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + None, + ); + + // Run test logic. + let logic = tokio::spawn(async move { + // Pause the time. + tokio::time::pause(); + + // Process the invocations. + for ( + expected_state_before_op, + expected_resource_version, + expected_invocation_response, + ) in invocations + { + // Wait for watcher to request next invocation. + let invocation_event = watcher_events_rx.next().await.unwrap(); + + // Assert that we obtained an invocation event. + let watch_optional = match invocation_event { + mock_watcher::ScenarioEvent::Invocation(val) => val, + _ => panic!("unexpected event from watcher mock"), + }; + + // Assert the current state while within the watcher stream + // item production code. + let state = gather_state(&state_reader); + assert_eq!(state, expected_state_before_op); + + // Assert the resource version passed with watch invocation. + assert_eq!(expected_resource_version, watch_optional.resource_version); + + // Determine the requested action from the test scenario. + let responses = match expected_invocation_response { + // Stream is requested, continue with the current flow. + ExpInvRes::Stream(responses) => responses, + // Desync is requested, complete the invocation with the desync. + ExpInvRes::Desync => { + // Send the desync action to mock watcher. + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrDesync) + .await + .unwrap(); + continue; + } + }; + + // Prepare channels for use in stream of the watch mock. + let (mut watch_stream_tx, watch_stream_rx) = mpsc::channel(0); + + // Send the stream action to the watch invocation. + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::Ok(watch_stream_rx)) + .await + .unwrap(); + + for response in responses { + // Wait for watcher to request next item from the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Send the requested action to the stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Ok(WatchResponse::Ok( + response, + ))) + .await + .unwrap(); + } + + // Wait for watcher to request next item from the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Send the notification that the stream is over. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Done) + .await + .unwrap(); + } + + // We're done with the test! Shutdown the stream and force an + // invocation error to terminate the reflector. + + // Wait for next invocation and send an error to terminate the + // flow. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrOther) + .await + .unwrap(); + + // Resume the time. + tokio::time::resume(); + }); + + // Run the test and wait for an error. + let result = reflector.run().await; + + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); + + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Assert the state after the reflector exit. + let resulting_state = gather_state(&resulting_state_reader); + assert_eq!(resulting_state, expected_resulting_state); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }) } } diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index f8f91dbd3dd02..246c96aa79c19 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -1,6 +1,7 @@ //! A state implementation backed by [`evmap10`]. use crate::kubernetes::hash_value::HashValue; +use async_trait::async_trait; use evmap10::WriteHandle; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; @@ -9,14 +10,14 @@ use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; /// [`crate::kubernetes::Reflector`]. pub struct Writer where - T: Metadata, + T: Metadata + Send, { inner: WriteHandle>, } impl Writer where - T: Metadata, + T: Metadata + Send, { /// Take a [`WriteHandle`], initialize it and return it wrapped with /// [`Self`]. @@ -27,9 +28,10 @@ where } } +#[async_trait] impl super::Write for Writer where - T: Metadata, + T: Metadata + Send, { type Item = T; @@ -37,28 +39,28 @@ where // within a certain small time window we commit all of them at once. // This will improve the state behaivor at resync. - fn add(&mut self, item: Self::Item) { + async fn add(&mut self, item: Self::Item) { if let Some((key, value)) = kv(item) { self.inner.insert(key, value); self.inner.flush(); } } - fn update(&mut self, item: Self::Item) { + async fn update(&mut self, item: Self::Item) { if let Some((key, value)) = kv(item) { self.inner.update(key, value); self.inner.flush(); } } - fn delete(&mut self, item: Self::Item) { + async fn delete(&mut self, item: Self::Item) { if let Some((key, _value)) = kv(item) { self.inner.empty(key); self.inner.flush(); } } - fn resync(&mut self) { + async fn resync(&mut self) { self.inner.purge(); // We do not flush on resync until the next per-item operation // arrives. diff --git a/src/kubernetes/state/mock.rs b/src/kubernetes/state/mock.rs new file mode 100644 index 0000000000000..06c2596461fa0 --- /dev/null +++ b/src/kubernetes/state/mock.rs @@ -0,0 +1,114 @@ +//! A mock state. + +#![cfg(test)] + +use async_trait::async_trait; +use futures::{ + channel::mpsc::{Receiver, Sender}, + SinkExt, StreamExt, +}; +use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; + +/// The kind of operation. +#[derive(Debug, PartialEq, Eq)] +pub enum OpKind { + /// Item added. + Add, + /// Item updated. + Update, + /// Item deleted. + Delete, +} + +/// An event that's send to the test scenario driver. +pub enum ScenarioEvent +where + T: Metadata + Send, +{ + Op(T, OpKind), + Resync, +} + +impl ScenarioEvent +where + T: Metadata + Send, +{ + pub fn unwrap_op(self) -> (T, OpKind) { + match self { + ScenarioEvent::Op(val, op) => (val, op), + ScenarioEvent::Resync => panic!("unwrap_op on resync"), + } + } +} + +/// Mock writer. +/// +/// Uses channels to communicate with the test scenario driver. +/// +/// When the call is made on the mock - sends an event to the `events_tx` and +/// waits for at action to conduct in response to the event `actions_rx`. +/// +/// Note: the only action available in the [`super::Write`] is to just continue +/// and return. +pub struct Writer +where + T: Metadata + Send, +{ + events_tx: Sender>, + actions_rx: Receiver<()>, +} + +impl Writer +where + T: Metadata + Send, +{ + /// Create a new mock writer. + /// Takes: + /// - `events_tx`, to which it sends the events when the mock action is + /// called; + /// - `actions_rx`, that is read a message from before the mock action + /// returns. + pub fn new(events_tx: Sender>, actions_rx: Receiver<()>) -> Self { + Self { + events_tx, + actions_rx, + } + } +} + +#[async_trait] +impl super::Write for Writer +where + T: Metadata + Send, +{ + type Item = T; + + async fn add(&mut self, item: Self::Item) { + self.events_tx + .send(ScenarioEvent::Op(item, OpKind::Add)) + .await + .unwrap(); + self.actions_rx.next().await.unwrap(); + } + + async fn update(&mut self, item: Self::Item) { + self.events_tx + .send(ScenarioEvent::Op(item, OpKind::Update)) + .await + .unwrap(); + self.actions_rx.next().await.unwrap(); + } + + async fn delete(&mut self, item: Self::Item) { + self.events_tx + .send(ScenarioEvent::Op(item, OpKind::Delete)) + .await + .unwrap(); + self.actions_rx.next().await.unwrap(); + } + + async fn resync(&mut self) { + self.events_tx.send(ScenarioEvent::Resync).await.unwrap(); + self.actions_rx.next().await.unwrap(); + } +} diff --git a/src/kubernetes/state/mod.rs b/src/kubernetes/state/mod.rs index a6c8421c128e4..2c713f422d39c 100644 --- a/src/kubernetes/state/mod.rs +++ b/src/kubernetes/state/mod.rs @@ -1,27 +1,30 @@ //! Local representation of the Kubernetes API resources state. +use async_trait::async_trait; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; pub mod evmap; +pub mod mock; /// Provides the interface for write access to the cached state. /// Used by [`super::reflector::Reflector`]. /// /// This abstraction allows easily stacking storage behaviour logic, without /// exploding the complexity at the [`super::reflector::Reflector`] level. +#[async_trait] pub trait Write { /// A type of the k8s resource the state operates on. - type Item: Metadata; + type Item: Metadata + Send; /// Add an object to the state. - fn add(&mut self, item: Self::Item); + async fn add(&mut self, item: Self::Item); /// Update an object at the state. - fn update(&mut self, item: Self::Item); + async fn update(&mut self, item: Self::Item); /// Delete on object from the state. - fn delete(&mut self, item: Self::Item); + async fn delete(&mut self, item: Self::Item); /// Notify the state that resync is in progress. - fn resync(&mut self); + async fn resync(&mut self); } From def5d4aa558be834812a1c4ed2ef0545215eaf1b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 18:40:54 +0300 Subject: [PATCH 047/118] Improve the request preparation code at kubernetes::client::Client Signed-off-by: MOZGIII --- src/kubernetes/client/mod.rs | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/kubernetes/client/mod.rs b/src/kubernetes/client/mod.rs index 3650e64e580d4..dc005da4a355a 100644 --- a/src/kubernetes/client/mod.rs +++ b/src/kubernetes/client/mod.rs @@ -81,33 +81,26 @@ impl Client { /// Alters a request according to the client configuraion and sends it. pub async fn send>(&mut self, req: Request) -> crate::Result> { - let req = convert_request_body(req); let req = self.prepare_request(req); self.inner.send(req).await } - fn prepare_request(&self, mut req: Request) -> Request { - self.adjust_uri(req.uri_mut()); + fn prepare_request>(&self, req: Request) -> Request { + let (mut parts, body) = req.into_parts(); + let body = body.into(); - req.headers_mut() + parts.uri = self.adjust_uri(parts.uri); + parts + .headers .insert(header::AUTHORIZATION, self.auth_header.clone()); - req + Request::from_parts(parts, body) } - // TODO: figure if we can do this more efficiently. - fn adjust_uri(&self, uri: &mut Uri) { - *uri = Uri::builder() - .scheme(self.uri_scheme.clone()) - .authority(self.uri_authority.clone()) - .path_and_query(uri.path_and_query().unwrap().clone()) - .build() - .unwrap(); + fn adjust_uri(&self, uri: Uri) -> Uri { + let mut parts = uri.into_parts(); + parts.scheme = Some(self.uri_scheme.clone()); + parts.authority = Some(self.uri_authority.clone()); + Uri::from_parts(parts).unwrap() } } - -fn convert_request_body>(req: Request) -> Request { - let (parts, body) = req.into_parts(); - let body = body.into(); - Request::from_parts(parts, body) -} From 0d70b38e32a10c1c671a6b1f5cb475cdb6c4bfa6 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 18:49:07 +0300 Subject: [PATCH 048/118] Add reexports at src/kubernetes/mod.rs Signed-off-by: MOZGIII --- src/kubernetes/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 886536ae7946f..addb4235f55a3 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -32,3 +32,10 @@ pub mod state; pub mod stream; pub mod watch_request_builder; pub mod watcher; + +// Reexports for more elegant public API. +pub use delayed_delete::DelayedDelete; +pub use hash_value::HashValue; +pub use multi_response_decoder::MultiResponseDecoder; +pub use reflector::Reflector; +pub use watch_request_builder::WatchRequestBuilder; From 242a2435fcfba780ccea34eaf3e5c130a57ff2fa Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 18:50:39 +0300 Subject: [PATCH 049/118] Adjust and use watcher error constructors Signed-off-by: MOZGIII --- src/kubernetes/api_watcher.rs | 6 +++--- src/kubernetes/watcher.rs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs index 427897d27de11..3b34162abf190 100644 --- a/src/kubernetes/api_watcher.rs +++ b/src/kubernetes/api_watcher.rs @@ -74,9 +74,9 @@ where if status != StatusCode::OK { let source = invocation::Error::BadStatus { status }; let err = if status == StatusCode::GONE { - watcher::invocation::Error::Desync { source } + watcher::invocation::Error::desync(source) } else { - watcher::invocation::Error::Other { source } + watcher::invocation::Error::other(source) }; Err(err)?; } @@ -154,7 +154,7 @@ pub mod invocation { impl From for watcher::invocation::Error { fn from(source: Error) -> Self { - watcher::invocation::Error::Other { source } + watcher::invocation::Error::other(source) } } } diff --git a/src/kubernetes/watcher.rs b/src/kubernetes/watcher.rs index f0afa4d3a35a8..4ed7e1b61a065 100644 --- a/src/kubernetes/watcher.rs +++ b/src/kubernetes/watcher.rs @@ -59,11 +59,13 @@ pub mod invocation { T: std::error::Error + Send + 'static, { /// Create an `Error::Desync`. + #[inline] pub fn desync(source: T) -> Self { Self::Desync { source } } /// Create an `Error::Other`. + #[inline] pub fn other(source: T) -> Self { Self::Other { source } } From c1ea6287a61241d0c5b7df9553c8b9a2b53f9fd2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 18:57:46 +0300 Subject: [PATCH 050/118] Eliminate unused transform Signed-off-by: MOZGIII --- src/transforms/util/chain.rs | 23 ----------------------- src/transforms/util/mod.rs | 2 -- 2 files changed, 25 deletions(-) delete mode 100644 src/transforms/util/chain.rs diff --git a/src/transforms/util/chain.rs b/src/transforms/util/chain.rs deleted file mode 100644 index 67257b1c3bb86..0000000000000 --- a/src/transforms/util/chain.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Chain transforms. - -#![deny(missing_docs)] - -use crate::{event::Event, transforms::Transform}; - -/// Chains two transforms, passing the event through the first one (`.0`), and -/// then, if there's still an event to pass, through the second one (`.1`). -pub struct Two(pub A, pub B); - -impl Two { - /// Creates a new chain of two transforms. - pub fn new(first: A, second: B) -> Self { - Self(first, second) - } -} - -impl Transform for Two { - fn transform(&mut self, event: Event) -> Option { - let event = self.0.transform(event)?; - self.1.transform(event) - } -} diff --git a/src/transforms/util/mod.rs b/src/transforms/util/mod.rs index 62ea8bb08a83b..4fd05069e50bf 100644 --- a/src/transforms/util/mod.rs +++ b/src/transforms/util/mod.rs @@ -3,8 +3,6 @@ pub mod runtime_transform; // TODO: make these part of the vector core - enabled unconditionally. #[cfg(any(feature = "sources-kubernetes-logs"))] -pub mod chain; -#[cfg(any(feature = "sources-kubernetes-logs"))] pub mod optional; #[cfg(any(feature = "sources-kubernetes-logs"))] pub mod pick; From 1be23b5736f5121cfaf67a93fa499d99a93f4225 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 19:43:59 +0300 Subject: [PATCH 051/118] Add a test for stream error behavior Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 77 ++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index b3c7537c7d497..76b5fcdf1dfe3 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -274,7 +274,7 @@ where mod tests { use super::{ state::evmap::{Value, Writer}, - Reflector, + Error, Reflector, }; use crate::{ kubernetes::{ @@ -736,6 +736,81 @@ mod tests { }) } + /// Test that stream error terminates the reflector. + #[test] + fn test_stream_error() { + test_util::trace_init(); + test_util::block_on_std(async move { + // Prepare state. + let (state_events_tx, _state_events_rx) = mpsc::channel(0); + let (_state_action_tx, state_action_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + + // Prepare reflector. + let mut reflector = Reflector::new( + watcher, + state_writer, + None, + None, + Duration::from_secs(1), + None, + ); + + // Run test logic. + let logic = tokio::spawn(async move { + // Wait for watcher to request next invocation. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + + // Provide watcher with a new stream. + let (mut watch_stream_tx, watch_stream_rx) = mpsc::channel(0); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::Ok(watch_stream_rx)) + .await + .unwrap(); + + // Wait for watcher to request next item from the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // Send an error to the stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Err) + .await + .unwrap(); + }); + + // Run the test and wait for an error. + let result = reflector.run().await; + + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); + + // Assert that the reflector properly passed the error. + assert!(matches!( + result.unwrap_err(), + Error::Streaming { + source: mock_watcher::StreamError + } + )); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }) + } + // A helper function to run a flow test. // Use this to test various flows without the test code repetition. fn run_flow_test( From 3c16cfbe6014a88143665b220d73403ce62a251e Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 5 Jun 2020 19:52:21 +0300 Subject: [PATCH 052/118] Add tests and derives at transform::util::pick Signed-off-by: MOZGIII --- src/transforms/util/pick.rs | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/transforms/util/pick.rs b/src/transforms/util/pick.rs index 57a8ea944935a..b8d1d5440197d 100644 --- a/src/transforms/util/pick.rs +++ b/src/transforms/util/pick.rs @@ -62,6 +62,7 @@ where } /// A passthrough transform. Passes events as-is. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Passthrough; impl Transform for Passthrough { @@ -71,6 +72,7 @@ impl Transform for Passthrough { } /// A discard transform. Returns `None` for every event. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Discard; impl Transform for Discard { @@ -80,6 +82,7 @@ impl Transform for Discard { } /// A panic transform. Just panics with the predefined message. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Panic(pub &'static str); impl Transform for Panic { @@ -119,3 +122,45 @@ where .expect("transform candidates exausted, and no suitable transform found") } } + +#[cfg(test)] +mod tests { + use super::*; + + fn box_transform(transform: impl Transform + 'static) -> Box { + Box::new(transform) + } + + #[test] + #[should_panic(expected = "transform candidates exausted, and no suitable transform found")] + fn iter_picker_empty() { + let picker = IterPicker::new(vec![]); + picker.probe_event(Event::new_empty_log()); + } + + #[test] + fn iter_picker_passthrough() { + let picker = IterPicker::new(vec![box_transform(Passthrough)]); + picker.probe_event(Event::new_empty_log()); + } + + #[test] + #[should_panic(expected = "test panic")] + fn iter_picker_panic() { + let picker = IterPicker::new(vec![box_transform(Panic("test panic"))]); + picker.probe_event(Event::new_empty_log()); + } + + #[test] + #[should_panic(expected = "transform candidates exausted, and no suitable transform found")] + fn iter_picker_discard() { + let picker = IterPicker::new(vec![box_transform(Discard)]); + picker.probe_event(Event::new_empty_log()); + } + + #[test] + fn iter_picker_sequence() { + let picker = IterPicker::new(vec![box_transform(Discard), box_transform(Passthrough)]); + picker.probe_event(Event::new_empty_log()); + } +} From a8ad24415a6520e2532209e619a14a359e03c8ae Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 8 Jun 2020 20:23:41 +0300 Subject: [PATCH 053/118] Require Watcher::Stream to be Send Signed-off-by: MOZGIII --- src/kubernetes/watcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kubernetes/watcher.rs b/src/kubernetes/watcher.rs index 4ed7e1b61a065..82934b5ba5d8f 100644 --- a/src/kubernetes/watcher.rs +++ b/src/kubernetes/watcher.rs @@ -17,7 +17,7 @@ pub trait Watcher { type StreamError: std::error::Error + Send + 'static; /// The stream type produced by the watch request. - type Stream: Stream, Self::StreamError>>; + type Stream: Stream, Self::StreamError>> + Send; /// Issues a single watch request and returns a stream results. fn watch<'a>( From 145f0b0c1ce7c817cc5d9cf13e9c4c7a940a5b1b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 8 Jun 2020 20:24:00 +0300 Subject: [PATCH 054/118] Add instrumentation Signed-off-by: MOZGIII --- skaffold/manifests/config.yaml | 5 +- src/internal_events/kubernetes/api_watcher.rs | 24 ++ .../kubernetes/instrumenting_state.rs | 39 +++ .../kubernetes/instrumenting_watcher.rs | 43 ++++ src/internal_events/kubernetes/mod.rs | 9 + src/internal_events/kubernetes/reflector.rs | 19 ++ src/internal_events/kubernetes/stream.rs | 14 ++ src/internal_events/mod.rs | 2 + src/kubernetes/api_watcher.rs | 7 +- src/kubernetes/instrumenting_watcher.rs | 67 +++++ src/kubernetes/mod.rs | 1 + src/kubernetes/reflector.rs | 13 +- src/kubernetes/state/instrumenting.rs | 238 ++++++++++++++++++ src/kubernetes/state/mod.rs | 1 + src/kubernetes/stream.rs | 2 + src/sources/kubernetes_logs/mod.rs | 2 + 16 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 src/internal_events/kubernetes/api_watcher.rs create mode 100644 src/internal_events/kubernetes/instrumenting_state.rs create mode 100644 src/internal_events/kubernetes/instrumenting_watcher.rs create mode 100644 src/internal_events/kubernetes/mod.rs create mode 100644 src/internal_events/kubernetes/reflector.rs create mode 100644 src/internal_events/kubernetes/stream.rs create mode 100644 src/kubernetes/instrumenting_watcher.rs create mode 100644 src/kubernetes/state/instrumenting.rs diff --git a/skaffold/manifests/config.yaml b/skaffold/manifests/config.yaml index 890a06f705e1c..e34be3fa588e0 100644 --- a/skaffold/manifests/config.yaml +++ b/skaffold/manifests/config.yaml @@ -4,8 +4,11 @@ metadata: name: vector-config data: vector.toml: | + [sources.internal_metrics] + type = "internal_metrics" + [sinks.stdout] type = "console" - inputs = ["kubernetes_logs"] + inputs = ["kubernetes_logs", "internal_metrics"] target = "stdout" encoding = "json" diff --git a/src/internal_events/kubernetes/api_watcher.rs b/src/internal_events/kubernetes/api_watcher.rs new file mode 100644 index 0000000000000..52269a4a876e3 --- /dev/null +++ b/src/internal_events/kubernetes/api_watcher.rs @@ -0,0 +1,24 @@ +use super::InternalEvent; +use std::fmt::Debug; + +#[derive(Debug)] +pub struct RequestPrepared { + pub request: R, +} + +impl InternalEvent for RequestPrepared { + fn emit_logs(&self) { + trace!(message = "request prepared", ?self.request); + } +} + +#[derive(Debug)] +pub struct ResponseReceived { + pub response: R, +} + +impl InternalEvent for ResponseReceived { + fn emit_logs(&self) { + trace!(message = "got response", ?self.response); + } +} diff --git a/src/internal_events/kubernetes/instrumenting_state.rs b/src/internal_events/kubernetes/instrumenting_state.rs new file mode 100644 index 0000000000000..8def5ecb78524 --- /dev/null +++ b/src/internal_events/kubernetes/instrumenting_state.rs @@ -0,0 +1,39 @@ +use super::InternalEvent; +use metrics::counter; +use std::fmt::Debug; + +#[derive(Debug)] +pub struct StateItemAdded; + +#[derive(Debug)] +pub struct StateItemUpdated; + +#[derive(Debug)] +pub struct StateItemDeleted; + +#[derive(Debug)] +pub struct StateResynced; + +impl InternalEvent for StateItemAdded { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "item_added"); + } +} + +impl InternalEvent for StateItemUpdated { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "item_updated"); + } +} + +impl InternalEvent for StateItemDeleted { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "item_deleted"); + } +} + +impl InternalEvent for StateResynced { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "resynced"); + } +} diff --git a/src/internal_events/kubernetes/instrumenting_watcher.rs b/src/internal_events/kubernetes/instrumenting_watcher.rs new file mode 100644 index 0000000000000..91b69c7b4cced --- /dev/null +++ b/src/internal_events/kubernetes/instrumenting_watcher.rs @@ -0,0 +1,43 @@ +use super::InternalEvent; +use metrics::counter; +use std::fmt::Debug; + +#[derive(Debug)] +pub struct WatchRequestInvoked; + +impl InternalEvent for WatchRequestInvoked { + fn emit_metrics(&self) { + counter!("k8s_watch_request_invoked", 1); + } +} + +#[derive(Debug)] +pub struct WatchRequestInvocationFailed { + pub error: E, +} + +impl InternalEvent for WatchRequestInvocationFailed { + fn emit_logs(&self) { + error!(message = "watch invocation failed", ?self.error); + } +} + +#[derive(Debug)] +pub struct WatchStreamItemObtained; + +impl InternalEvent for WatchStreamItemObtained { + fn emit_metrics(&self) { + counter!("k8s_watch_stream_items_obtained", 1); + } +} + +#[derive(Debug)] +pub struct WatchStreamErrored { + pub error: E, +} + +impl InternalEvent for WatchStreamErrored { + fn emit_logs(&self) { + error!(message = "watch stream errored", ?self.error); + } +} diff --git a/src/internal_events/kubernetes/mod.rs b/src/internal_events/kubernetes/mod.rs new file mode 100644 index 0000000000000..f5caf3340ecb9 --- /dev/null +++ b/src/internal_events/kubernetes/mod.rs @@ -0,0 +1,9 @@ +#![cfg(feature = "kubernetes")] + +use super::InternalEvent; + +pub mod api_watcher; +pub mod instrumenting_state; +pub mod instrumenting_watcher; +pub mod reflector; +pub mod stream; diff --git a/src/internal_events/kubernetes/reflector.rs b/src/internal_events/kubernetes/reflector.rs new file mode 100644 index 0000000000000..4090aab0a475f --- /dev/null +++ b/src/internal_events/kubernetes/reflector.rs @@ -0,0 +1,19 @@ +use super::InternalEvent; +use metrics::counter; + +/// Emitted when reflector gets a desync from the watch command. +#[derive(Debug)] +pub struct DesyncReceived { + /// The underlying error. + pub error: E, +} + +impl InternalEvent for DesyncReceived { + fn emit_logs(&self) { + warn!(message = "handling desync", error = ?self.error); + } + + fn emit_metrics(&self) { + counter!("k8s_reflector_desyncs", 1); + } +} diff --git a/src/internal_events/kubernetes/stream.rs b/src/internal_events/kubernetes/stream.rs new file mode 100644 index 0000000000000..964d553eabcb3 --- /dev/null +++ b/src/internal_events/kubernetes/stream.rs @@ -0,0 +1,14 @@ +use super::InternalEvent; +use metrics::counter; + +#[derive(Debug)] +pub struct ChunkProcessed { + pub byte_size: usize, +} + +impl InternalEvent for ChunkProcessed { + fn emit_metrics(&self) { + counter!("k8s_stream_chunks_processed", 1); + counter!("k8s_stream_bytes_processed", self.byte_size as u64); + } +} diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 796ac17f214f7..f44edb4857b2b 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -20,6 +20,8 @@ mod vector; #[cfg(feature = "wasm")] mod wasm; +pub mod kubernetes; + pub use self::add_fields::*; pub use self::aws_kinesis_streams::*; pub use self::blackhole::*; diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs index 3b34162abf190..68805dce4f495 100644 --- a/src/kubernetes/api_watcher.rs +++ b/src/kubernetes/api_watcher.rs @@ -6,6 +6,7 @@ use super::{ watch_request_builder::WatchRequestBuilder, watcher::{self, Watcher}, }; +use crate::internal_events::kubernetes::api_watcher as internal_events; use futures::{ future::BoxFuture, stream::{BoxStream, Stream}, @@ -59,7 +60,7 @@ where .request_builder .build(watch_optional) .context(invocation::RequestPreparation)?; - trace!(message = "request prepared", ?request); + emit!(internal_events::RequestPrepared { request: &request }); // Send request, get response. let response = self @@ -67,7 +68,9 @@ where .send(request) .await .context(invocation::Request)?; - trace!(message = "got response", ?response); + emit!(internal_events::ResponseReceived { + response: &response + }); // Handle response status code. let status = response.status(); diff --git a/src/kubernetes/instrumenting_watcher.rs b/src/kubernetes/instrumenting_watcher.rs new file mode 100644 index 0000000000000..7cfd1cba00e29 --- /dev/null +++ b/src/kubernetes/instrumenting_watcher.rs @@ -0,0 +1,67 @@ +//! A watcher that adds instrumentation. + +use super::watcher::{self, Watcher}; +use crate::internal_events::kubernetes::instrumenting_watcher as internal_events; +use futures::{future::BoxFuture, stream::BoxStream, FutureExt, StreamExt}; +use k8s_openapi::{WatchOptional, WatchResponse}; + +/// A watcher that wraps another watcher with instrumentation calls. +pub struct InstrumentingWatcher +where + T: Watcher, +{ + inner: T, +} + +impl InstrumentingWatcher +where + T: Watcher, +{ + /// Create a new [`InstrumentingWatcher`]. + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +impl Watcher for InstrumentingWatcher +where + T: Watcher, + ::Stream: 'static, +{ + type Object = ::Object; + + type InvocationError = ::InvocationError; + + type StreamError = ::StreamError; + type Stream = BoxStream<'static, Result, Self::StreamError>>; + + fn watch<'a>( + &'a mut self, + watch_optional: WatchOptional<'a>, + ) -> BoxFuture<'a, Result>> + { + Box::pin(self.inner.watch(watch_optional).map(|result| { + result + .map(|stream| { + emit!(internal_events::WatchRequestInvoked); + Box::pin(stream.map(|item_result| { + item_result + .map(|item| { + emit!(internal_events::WatchStreamItemObtained); + item + }) + .map_err(|error| { + emit!(internal_events::WatchRequestInvocationFailed { + error: &error + }); + error + }) + })) as BoxStream<'static, _> + }) + .map_err(|error| { + emit!(internal_events::WatchRequestInvocationFailed { error: &error }); + error + }) + })) + } +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index addb4235f55a3..3d1fb1399ae1f 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -24,6 +24,7 @@ pub mod api_watcher; pub mod client; pub mod delayed_delete; pub mod hash_value; +pub mod instrumenting_watcher; pub mod mock_watcher; pub mod multi_response_decoder; pub mod reflector; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 76b5fcdf1dfe3..013d6388e91fa 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -4,6 +4,7 @@ use super::{ resource_version, watcher::{self, Watcher}, }; +use crate::internal_events::kubernetes::reflector as internal_events; use futures::{ pin_mut, stream::{Stream, StreamExt}, @@ -90,7 +91,7 @@ where let stream = match invocation_result { Ok(val) => val, Err(watcher::invocation::Error::Desync { source }) => { - warn!(message = "handling desync", error = ?source); + emit!(internal_events::DesyncReceived { error: source }); // We got desynced, reset the state and retry fetching. // By omiting the flush here, we cache the results from the // previous run until flush is issued when the new events @@ -278,6 +279,7 @@ mod tests { }; use crate::{ kubernetes::{ + instrumenting_watcher::InstrumentingWatcher, mock_watcher::{self, MockWatcher}, state, }, @@ -356,11 +358,13 @@ mod tests { let (state_events_tx, _state_events_rx) = mpsc::channel(0); let (_state_action_tx, state_action_rx) = mpsc::channel(0); let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let state_writer = state::instrumenting::Writer::new(state_writer); // Prepare watcher. let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. let mut reflector = Reflector::new( @@ -521,6 +525,7 @@ mod tests { let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. let mut reflector = Reflector::new( @@ -592,11 +597,13 @@ mod tests { let (state_events_tx, mut state_events_rx) = mpsc::channel(0); let (mut state_action_tx, state_action_rx) = mpsc::channel(0); let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let state_writer = state::instrumenting::Writer::new(state_writer); // Prepare watcher. let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. let deletion_delay = Duration::from_secs(600); @@ -745,11 +752,13 @@ mod tests { let (state_events_tx, _state_events_rx) = mpsc::channel(0); let (_state_action_tx, state_action_rx) = mpsc::channel(0); let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let state_writer = state::instrumenting::Writer::new(state_writer); // Prepare watcher. let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. let mut reflector = Reflector::new( @@ -821,6 +830,7 @@ mod tests { // Prepare state. let (state_reader, state_writer) = evmap10::new(); let state_writer = Writer::new(state_writer); + let state_writer = state::instrumenting::Writer::new(state_writer); let resulting_state_reader = state_reader.clone(); // Prepare watcher. @@ -828,6 +838,7 @@ mod tests { let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); let watcher: MockWatcher = MockWatcher::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. let mut reflector = Reflector::new( diff --git a/src/kubernetes/state/instrumenting.rs b/src/kubernetes/state/instrumenting.rs new file mode 100644 index 0000000000000..1b4273767395d --- /dev/null +++ b/src/kubernetes/state/instrumenting.rs @@ -0,0 +1,238 @@ +//! An instrumenting state wrapper. + +use crate::internal_events::kubernetes::instrumenting_state as internal_events; +use async_trait::async_trait; + +/// A [`super::Write`] implementatiom that wraps another [`super::Write`] and +/// adds instrumentation. +pub struct Writer { + inner: T, +} + +impl Writer { + /// Take a [`super::Write`] and return it wrapped with [`Self`]. + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +#[async_trait] +impl super::Write for Writer +where + T: super::Write + Send, +{ + type Item = ::Item; + + async fn add(&mut self, item: Self::Item) { + emit!(internal_events::StateItemAdded); + self.inner.add(item).await + } + + async fn update(&mut self, item: Self::Item) { + emit!(internal_events::StateItemUpdated); + self.inner.update(item).await + } + + async fn delete(&mut self, item: Self::Item) { + emit!(internal_events::StateItemDeleted); + self.inner.delete(item).await + } + + async fn resync(&mut self) { + emit!(internal_events::StateResynced); + self.inner.resync().await + } +} + +#[cfg(test)] +mod tests { + use super::super::{mock, Write}; + use super::*; + use crate::test_util; + use futures::{channel::mpsc, SinkExt, StreamExt}; + use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; + + fn prepare_test() -> ( + Writer>, + mpsc::Receiver>, + mpsc::Sender<()>, + ) { + let (events_tx, events_rx) = mpsc::channel(0); + let (actions_tx, actions_rx) = mpsc::channel(0); + let writer = mock::Writer::new(events_tx, actions_rx); + let writer = Writer::new(writer); + (writer, events_rx, actions_tx) + } + + fn make_pod() -> Pod { + Pod { + metadata: Some(ObjectMeta { + name: Some("pod_name".to_owned()), + uid: Some("pod_uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + } + } + + fn get_metric_value(op_kind: &'static str) -> Option { + let controller = crate::metrics::CONTROLLER.get().unwrap_or_else(|| { + crate::metrics::init().unwrap(); + crate::metrics::CONTROLLER + .get() + .expect("failed to init metric container") + }); + + let key = metrics_core::Key::from_name_and_labels( + "k8s_state_ops", + vec![metrics_core::Label::new("op_kind", op_kind)], + ); + controller + .snapshot() + .into_measurements() + .into_iter() + .find_map(|(candidate_key, measurement)| { + if candidate_key == key { + Some(measurement) + } else { + None + } + }) + } + + fn assert_counter_incremented( + before: Option, + after: Option, + ) { + let before = before.unwrap_or_else(|| metrics_runtime::Measurement::Counter(0)); + let after = after.expect("after value was None"); + + let (before, after) = match (before, after) { + ( + metrics_runtime::Measurement::Counter(before), + metrics_runtime::Measurement::Counter(after), + ) => (before, after), + _ => panic!("metrics kind mismatch"), + }; + + let difference = after - before; + + assert_eq!(difference, 1); + } + + #[test] + fn add() { + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + let pod = make_pod(); + + let join = { + let pod = pod.clone(); + let before = get_metric_value("item_added"); + tokio::spawn(async move { + assert_eq!( + events_rx.next().await.unwrap().unwrap_op(), + (pod, mock::OpKind::Add) + ); + + // By now metrics should've updated. + let after = get_metric_value("item_added"); + assert_counter_incremented(before, after); + + actions_tx.send(()).await.unwrap(); + }) + }; + + writer.add(pod).await; + join.await.unwrap(); + }) + } + + #[test] + fn update() { + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + let pod = make_pod(); + + let join = { + let pod = pod.clone(); + let before = get_metric_value("item_updated"); + tokio::spawn(async move { + assert_eq!( + events_rx.next().await.unwrap().unwrap_op(), + (pod, mock::OpKind::Update) + ); + + // By now metrics should've updated. + let after = get_metric_value("item_updated"); + assert_counter_incremented(before, after); + + actions_tx.send(()).await.unwrap(); + }) + }; + + writer.update(pod).await; + join.await.unwrap(); + }) + } + + #[test] + fn delete() { + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + let pod = make_pod(); + + let join = { + let pod = pod.clone(); + let before = get_metric_value("item_deleted"); + tokio::spawn(async move { + assert_eq!( + events_rx.next().await.unwrap().unwrap_op(), + (pod, mock::OpKind::Delete) + ); + + // By now metrics should've updated. + let after = get_metric_value("item_deleted"); + assert_counter_incremented(before, after); + + actions_tx.send(()).await.unwrap(); + }) + }; + + writer.delete(pod).await; + join.await.unwrap(); + }) + } + + #[test] + fn resync() { + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + let join = { + let before = get_metric_value("resynced"); + tokio::spawn(async move { + assert!(matches!( + events_rx.next().await.unwrap(), + mock::ScenarioEvent::Resync + )); + + let after = get_metric_value("resynced"); + assert_counter_incremented(before, after); + + actions_tx.send(()).await.unwrap(); + }) + }; + + writer.resync().await; + join.await.unwrap(); + }) + } +} diff --git a/src/kubernetes/state/mod.rs b/src/kubernetes/state/mod.rs index 2c713f422d39c..05dc96ff1094f 100644 --- a/src/kubernetes/state/mod.rs +++ b/src/kubernetes/state/mod.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; pub mod evmap; +pub mod instrumenting; pub mod mock; /// Provides the interface for write access to the cached state. diff --git a/src/kubernetes/stream.rs b/src/kubernetes/stream.rs index 394b6344fc992..8b28007370526 100644 --- a/src/kubernetes/stream.rs +++ b/src/kubernetes/stream.rs @@ -1,6 +1,7 @@ //! Work with HTTP bodies as streams of Kubernetes resources. use super::multi_response_decoder::MultiResponseDecoder; +use crate::internal_events::kubernetes::stream as internal_events; use async_stream::try_stream; use bytes05::Buf; use futures::pin_mut; @@ -27,6 +28,7 @@ where let mut buf = buf.context(Reading)?; let chunk = buf.to_bytes(); let responses = decoder.process_next_chunk(chunk.as_ref()); + emit!(internal_events::ChunkProcessed{ byte_size: chunk.len() }); for response in responses { let response = response.context(Parsing)?; yield response; diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 822d01ee87079..83af1bb71ccee 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -152,8 +152,10 @@ impl Source { let label_selector = "vector.dev/exclude!=true".to_owned(); let watcher = k8s::api_watcher::ApiWatcher::new(client, Pod::watch_pod_for_all_namespaces); + let watcher = k8s::instrumenting_watcher::InstrumentingWatcher::new(watcher); let (state_reader, state_writer) = evmap::new(); let state_writer = k8s::state::evmap::Writer::new(state_writer); + let state_writer = k8s::state::instrumenting::Writer::new(state_writer); let mut reflector = k8s::reflector::Reflector::new( watcher, From d868ac5fdd1f46f951bc2a412f62cf55896b60a2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 10 Jun 2020 00:00:31 +0300 Subject: [PATCH 055/118] Add state maintenance and move delayed delete logic into to a state wrapper This unlocks further complicating the logic around state without making reflector overcomplicated. This better aligns with the goal of building composable and testable loosely coupled components. Signed-off-by: MOZGIII --- .../kubernetes/instrumenting_state.rs | 18 ++ src/kubernetes/delayed_delete.rs | 123 -------- src/kubernetes/mod.rs | 2 - src/kubernetes/reflector.rs | 255 +++++++++------ src/kubernetes/state/delayed_delete.rs | 298 ++++++++++++++++++ src/kubernetes/state/evmap.rs | 23 +- src/kubernetes/state/instrumenting.rs | 114 ++++++- src/kubernetes/state/mock.rs | 80 ++++- src/kubernetes/state/mod.rs | 33 ++ src/sources/kubernetes_logs/mod.rs | 5 +- 10 files changed, 709 insertions(+), 242 deletions(-) delete mode 100644 src/kubernetes/delayed_delete.rs create mode 100644 src/kubernetes/state/delayed_delete.rs diff --git a/src/internal_events/kubernetes/instrumenting_state.rs b/src/internal_events/kubernetes/instrumenting_state.rs index 8def5ecb78524..bb743a836ad54 100644 --- a/src/internal_events/kubernetes/instrumenting_state.rs +++ b/src/internal_events/kubernetes/instrumenting_state.rs @@ -14,6 +14,12 @@ pub struct StateItemDeleted; #[derive(Debug)] pub struct StateResynced; +#[derive(Debug)] +pub struct StateMaintenanceRequested; + +#[derive(Debug)] +pub struct StateMaintenancePerformed; + impl InternalEvent for StateItemAdded { fn emit_metrics(&self) { counter!("k8s_state_ops", 1, "op_kind" => "item_added"); @@ -37,3 +43,15 @@ impl InternalEvent for StateResynced { counter!("k8s_state_ops", 1, "op_kind" => "resynced"); } } + +impl InternalEvent for StateMaintenanceRequested { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "maintenace_requested"); + } +} + +impl InternalEvent for StateMaintenancePerformed { + fn emit_metrics(&self) { + counter!("k8s_state_ops", 1, "op_kind" => "maintenace_performed"); + } +} diff --git a/src/kubernetes/delayed_delete.rs b/src/kubernetes/delayed_delete.rs deleted file mode 100644 index 6343a51775600..0000000000000 --- a/src/kubernetes/delayed_delete.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! A delayed delete logic. - -use super::state; -use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; -use std::{collections::VecDeque, time::Duration}; -use tokio::time::Instant; - -/// Implements the logic for delaying the deletion of items from the storage. -pub struct DelayedDelete { - queue: VecDeque<(T, Instant)>, - delay_for: Duration, -} - -impl DelayedDelete { - /// Create a new [`DelayedDelete`] state. - pub fn new(delay_for: Duration) -> Self { - let queue = VecDeque::new(); - Self { queue, delay_for } - } - - /// Schedules the delayed deletion of the item at the future. - pub fn schedule_delete(&mut self, item: T) { - let deadline = Instant::now() + self.delay_for; - self.queue.push_back((item, deadline)); - } - - /// Clear the delayed deletion requests. - pub fn clear(&mut self) { - self.queue.clear(); - } - - /// Perform the queued deletions. - pub async fn perform(&mut self, state_writer: &mut impl state::Write) - where - T: Metadata + Send, - { - let now = Instant::now(); - while let Some(deadline) = self.next_deadline() { - trace!(message = "got delayed deletion deadline", ?deadline, ?now); - if deadline > now { - break; - } - trace!( - message = "processing delayed deletion for deadline", - ?deadline, - ?now - ); - let (item, _) = self.queue.pop_front().unwrap(); - state_writer.delete(item).await; - } - } - - /// Obtain the next deadline. - pub fn next_deadline(&self) -> Option { - self.queue.front().map(|(_, instant)| *instant) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_util; - use futures::{channel::mpsc::channel, SinkExt, StreamExt}; - use k8s_openapi::api::core::v1::Pod; - - #[test] - fn logic() { - test_util::trace_init(); - test_util::block_on_std(async move { - tokio::time::pause(); - - let (state_events_tx, mut state_events_rx) = channel(0); - let (mut state_actions_tx, state_actions_rx) = channel(0); - let mut writer = state::mock::Writer::::new(state_events_tx, state_actions_rx); - - let mut delayed_delete = DelayedDelete::new(Duration::from_millis(10000)); - - { - assert!(delayed_delete.next_deadline().is_none()); - delayed_delete.perform(&mut writer).await; - assert!(delayed_delete.next_deadline().is_none()); - assert!(state_events_rx.try_next().is_err()); - } - - delayed_delete.schedule_delete(Pod::default()); - - { - assert!(delayed_delete.next_deadline().is_some()); - delayed_delete.perform(&mut writer).await; - assert!(delayed_delete.next_deadline().is_some()); - assert!(state_events_rx.try_next().is_err()); - } - - tokio::time::advance(Duration::from_millis(50000)).await; - - let (mut state_events_rx, _state_actions_tx) = { - let conc_fut = tokio::spawn(async move { - assert_eq!( - state_events_rx.next().await.unwrap().unwrap_op(), - (Pod::default(), state::mock::OpKind::Delete) - ); - state_actions_tx.send(()).await.unwrap(); - (state_events_rx, state_actions_tx) - }); - - assert!(delayed_delete.next_deadline().is_some()); - delayed_delete.perform(&mut writer).await; - assert!(delayed_delete.next_deadline().is_none()); - - conc_fut.await.unwrap() - }; - - { - assert!(delayed_delete.next_deadline().is_none()); - delayed_delete.perform(&mut writer).await; - assert!(delayed_delete.next_deadline().is_none()); - assert!(state_events_rx.try_next().is_err()); - } - - tokio::time::resume(); - }); - } -} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index 3d1fb1399ae1f..ac84b15e05e76 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,7 +22,6 @@ pub mod api_watcher; pub mod client; -pub mod delayed_delete; pub mod hash_value; pub mod instrumenting_watcher; pub mod mock_watcher; @@ -35,7 +34,6 @@ pub mod watch_request_builder; pub mod watcher; // Reexports for more elegant public API. -pub use delayed_delete::DelayedDelete; pub use hash_value::HashValue; pub use multi_response_decoder::MultiResponseDecoder; pub use reflector::Reflector; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 013d6388e91fa..74390833ab6f7 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -1,7 +1,7 @@ //! Watch and cache the remote Kubernetes API resources. use super::{ - resource_version, + resource_version, state, watcher::{self, Watcher}, }; use crate::internal_events::kubernetes::reflector as internal_events; @@ -16,12 +16,7 @@ use k8s_openapi::{ use snafu::Snafu; use std::convert::Infallible; use std::time::Duration; -use tokio::{ - select, - time::{delay_for, delay_until}, -}; - -use super::{delayed_delete::DelayedDelete, state}; +use tokio::{select, time::delay_for}; /// Watches remote Kubernetes resources and maintains a local representation of /// the remote state. "Reflects" the remote state locally. @@ -32,7 +27,7 @@ pub struct Reflector where W: Watcher, ::Object: Metadata + Send, - S: state::Write::Object>, + S: state::MaintainedWrite::Object>, { watcher: W, state_writer: S, @@ -40,14 +35,13 @@ where label_selector: Option, resource_version: resource_version::State, pause_between_requests: Duration, - delayed_delete: Option::Object>>, } impl Reflector where W: Watcher, ::Object: Metadata + Send, - S: state::Write::Object>, + S: state::MaintainedWrite::Object>, { /// Create a new [`Cache`]. pub fn new( @@ -56,10 +50,8 @@ where field_selector: Option, label_selector: Option, pause_between_requests: Duration, - delay_deletes_for: Option, ) -> Self { let resource_version = resource_version::State::new(); - let delayed_delete = delay_deletes_for.map(DelayedDelete::new); Self { watcher, state_writer, @@ -67,7 +59,6 @@ where field_selector, resource_version, pause_between_requests, - delayed_delete, } } } @@ -78,7 +69,7 @@ where ::Object: Metadata + Send + Unpin + std::fmt::Debug, ::InvocationError: Unpin, ::StreamError: Unpin, - S: state::Write::Object>, + S: state::MaintainedWrite::Object>, { /// Run the watch loop and drive the state updates via `state_writer`. pub async fn run( @@ -93,14 +84,7 @@ where Err(watcher::invocation::Error::Desync { source }) => { emit!(internal_events::DesyncReceived { error: source }); // We got desynced, reset the state and retry fetching. - // By omiting the flush here, we cache the results from the - // previous run until flush is issued when the new events - // begin arriving, reducing the time durig which the state - // has no data. self.resource_version.reset(); - if let Some(ref mut delayed_delete) = self.delayed_delete { - delayed_delete.clear(); - } self.state_writer.resync().await; continue; } @@ -114,27 +98,19 @@ where pin_mut!(stream); loop { // Obtain an value from the watch stream. - let val = if let Some(ref mut delayed_delete) = self.delayed_delete { - // If delayed delete is requested, we perform the delayed - // deletions concurrently with reading items from the watch - // responses stream. - let delayed_delete_deadline = delayed_delete.next_deadline(); - select! { - // If we get a delayted delete deadline - process the - // delayed deletes and restart the loop. - _ = async { delay_until(delayed_delete_deadline.unwrap().into()).await }, if delayed_delete_deadline.is_some() => { - trace!(message = "performing delayed deletes"); - delayed_delete.perform(&mut self.state_writer).await; - continue; - } - // If we got a value from the watch responses stream - - // just pass it outside. - val = stream.next() => val, + // If maintenance is requested, we perform it concurrently + // to reading items from the watch stream. + let maintenance_request = self.state_writer.maintenance_request(); + let val = select! { + // If we got a maintenance request - perform the + // maintenance. + _ = async { maintenance_request.unwrap().await }, if maintenance_request.is_some() => { + self.state_writer.perform_maintenance().await; + continue; } - } else { - // Delayed deletes aren't requested, so just wait for the - // next value and pass it outside. - stream.next().await + // If we got a value from the watch stream - just pass it + // outside. + val = stream.next() => val, }; trace!(message = "got an item from watch stream"); @@ -230,11 +206,7 @@ where } WatchEvent::Deleted(object) => { trace!(message = "got an object event", event = "deleted"); - if let Some(ref mut delayed_delete) = self.delayed_delete { - delayed_delete.schedule_delete(object); - } else { - self.state_writer.delete(object).await; - } + self.state_writer.delete(object).await; } WatchEvent::Modified(object) => { trace!(message = "got an object event", event = "modified"); @@ -356,8 +328,8 @@ mod tests { test_util::block_on_std(async move { // Prepare state. let (state_events_tx, _state_events_rx) = mpsc::channel(0); - let (_state_action_tx, state_action_rx) = mpsc::channel(0); - let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let (_state_actions_tx, state_actions_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_actions_rx); let state_writer = state::instrumenting::Writer::new(state_writer); // Prepare watcher. @@ -367,14 +339,8 @@ mod tests { let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - None, - ); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); // Run test logic. let logic = tokio::spawn(async move { @@ -518,8 +484,8 @@ mod tests { test_util::block_on_std(async move { // Prepare state. let (state_events_tx, _state_events_rx) = mpsc::channel(0); - let (_state_action_tx, state_action_rx) = mpsc::channel(0); - let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let (_state_actions_tx, state_actions_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_actions_rx); // Prepare watcher. let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); @@ -534,7 +500,6 @@ mod tests { Some("fields".to_owned()), Some("labels".to_owned()), Duration::from_secs(1), - None, ); // Run test logic. @@ -595,9 +560,11 @@ mod tests { test_util::block_on_std(async move { // Prepare state. let (state_events_tx, mut state_events_rx) = mpsc::channel(0); - let (mut state_action_tx, state_action_rx) = mpsc::channel(0); - let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let (mut state_actions_tx, state_actions_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_actions_rx); let state_writer = state::instrumenting::Writer::new(state_writer); + let deletion_delay = Duration::from_secs(600); + let state_writer = state::delayed_delete::Writer::new(state_writer, deletion_delay); // Prepare watcher. let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); @@ -606,15 +573,8 @@ mod tests { let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. - let deletion_delay = Duration::from_secs(600); - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - Some(deletion_delay), - ); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); // Run test logic. let logic = tokio::spawn(async move { @@ -656,7 +616,7 @@ mod tests { ); // Send the confirmation of the processing at the state. - state_action_tx.send(()).await.unwrap(); + state_actions_tx.send(()).await.unwrap(); // Let the reflector work until watcher requests next event from // the stream. @@ -690,15 +650,27 @@ mod tests { // Advance the time 10 times the deletion delay. tokio::time::advance(deletion_delay * 10).await; - // At this point, pod deletion should propagate to the state. - // Let the reflector work until the deletion reaches the state. + // At this point, maintenance should be performed, for both + // delayed deletion state and mock state. + + // Delayed deletes are processed first. assert_eq!( state_events_rx.next().await.unwrap().unwrap_op(), (make_pod("uid0", "15"), state::mock::OpKind::Delete), ); // Send the confirmation of the processing at the state. - state_action_tx.send(()).await.unwrap(); + state_actions_tx.send(()).await.unwrap(); + + // Then, the maintenance event should be triggered. + // This completes the `perform_maintenance` call. + assert!(matches!( + state_events_rx.next().await.unwrap(), + state::mock::ScenarioEvent::Maintenance + )); + + // Send the confirmation of the processing at the state. + state_actions_tx.send(()).await.unwrap(); // We're done with the test! Shutdown the stream and force an // invocation error to terminate the reflector. @@ -750,8 +722,8 @@ mod tests { test_util::block_on_std(async move { // Prepare state. let (state_events_tx, _state_events_rx) = mpsc::channel(0); - let (_state_action_tx, state_action_rx) = mpsc::channel(0); - let state_writer = state::mock::Writer::new(state_events_tx, state_action_rx); + let (_state_actions_tx, state_actions_rx) = mpsc::channel(0); + let state_writer = state::mock::Writer::new(state_events_tx, state_actions_rx); let state_writer = state::instrumenting::Writer::new(state_writer); // Prepare watcher. @@ -761,14 +733,8 @@ mod tests { let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - None, - ); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); // Run test logic. let logic = tokio::spawn(async move { @@ -820,6 +786,117 @@ mod tests { }) } + /// Test that maintenance works accordingly. + #[test] + fn test_maintenance() { + test_util::trace_init(); + test_util::block_on_std(async move { + // Prepare state. + let (state_events_tx, mut state_events_rx) = mpsc::channel(0); + let (mut state_actions_tx, state_actions_rx) = mpsc::channel(0); + let (state_maintenance_request_events_tx, mut state_maintenance_request_events_rx) = + mpsc::channel(0); + let (mut state_maintenance_request_actions_tx, state_maintenance_request_actions_rx) = + mpsc::channel(0); + let state_writer = state::mock::Writer::new_with_maintenance( + state_events_tx, + state_actions_rx, + state_maintenance_request_events_tx, + state_maintenance_request_actions_rx, + ); + let state_writer = state::instrumenting::Writer::new(state_writer); + + // Prepare watcher. + let (watcher_events_tx, mut watcher_events_rx) = mpsc::channel(0); + let (mut watcher_invocations_tx, watcher_invocations_rx) = mpsc::channel(0); + let watcher = MockWatcher::::new(watcher_events_tx, watcher_invocations_rx); + let watcher = InstrumentingWatcher::new(watcher); + + // Prepare reflector. + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + + // Run test logic. + let logic = tokio::spawn(async move { + // Wait for watcher to request next invocation. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + + // Assert that maintenance request events didn't arrive yet. + assert!(state_maintenance_request_events_rx.try_next().is_err()); + + // Provide watcher with a new stream. + let (mut watch_stream_tx, watch_stream_rx) = mpsc::channel(0); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::Ok(watch_stream_rx)) + .await + .unwrap(); + + // Wait for reflector to request a state maintenance. + state_maintenance_request_events_rx.next().await.unwrap(); + + // Send the maintenance request action to advance to the + // maintenance. + state_maintenance_request_actions_tx.send(()).await.unwrap(); + + // Wait for a maintenance perform event arrival. + assert!(matches!( + state_events_rx.next().await.unwrap(), + state::mock::ScenarioEvent::Maintenance + )); + + // Send the confirmation of the state maintenance. + state_actions_tx.send(()).await.unwrap(); + + // Let the reflector work until watcher requests next event from + // the stream. + assert_eq!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Stream + ); + + // We're done with the test! Shutdown the stream and force an + // invocation error to terminate the reflector. + + // Watcher is still waiting for the item on stream. + // Send done notification to the stream. + watch_stream_tx + .send(mock_watcher::ScenarioActionStream::Done) + .await + .unwrap(); + + // Wait for next invocation and send an error to terminate the + // flow. + assert!(matches!( + watcher_events_rx.next().await.unwrap(), + mock_watcher::ScenarioEvent::Invocation(_) + )); + watcher_invocations_tx + .send(mock_watcher::ScenarioActionInvocation::ErrOther) + .await + .unwrap(); + }); + + // Run the test and wait for an error. + let result = reflector.run().await; + + // Join on the logic first, to report logic errors with higher + // priority. + logic.await.unwrap(); + + // The only way reflector completes is with an error, but that's ok. + // In tests we make it exit with an error to complete the test. + result.unwrap_err(); + + // Explicitly drop the reflector at the very end. + // Internal evmap is dropped with the reflector, so readers won't + // work after drop. + drop(reflector); + }) + } + // A helper function to run a flow test. // Use this to test various flows without the test code repetition. fn run_flow_test( @@ -841,14 +918,8 @@ mod tests { let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. - let mut reflector = Reflector::new( - watcher, - state_writer, - None, - None, - Duration::from_secs(1), - None, - ); + let mut reflector = + Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); // Run test logic. let logic = tokio::spawn(async move { diff --git a/src/kubernetes/state/delayed_delete.rs b/src/kubernetes/state/delayed_delete.rs new file mode 100644 index 0000000000000..028fb2495d589 --- /dev/null +++ b/src/kubernetes/state/delayed_delete.rs @@ -0,0 +1,298 @@ +//! A state wrapper that delays deletes. + +use async_trait::async_trait; +use futures::{future::BoxFuture, FutureExt}; +use std::{collections::VecDeque, time::Duration}; +use tokio::time::{delay_until, Instant}; + +/// A [`super::Write`] implementatiom that wraps another [`super::Write`] and +/// delays the delete calls. +/// Implements the logic for delaying the deletion of items from the storage. +pub struct Writer +where + T: super::Write + Send, + ::Item: Send + Sync, +{ + inner: T, + queue: VecDeque<(::Item, Instant)>, + delay_for: Duration, +} + +impl Writer +where + T: super::Write + Send, + ::Item: Send + Sync, +{ + /// Take a [`super::Write`] and return it wrapped with [`Self`]. + pub fn new(inner: T, delay_for: Duration) -> Self { + let queue = VecDeque::new(); + Self { + inner, + queue, + delay_for, + } + } +} + +impl Writer +where + T: super::Write + Send, + ::Item: Send + Sync, +{ + /// Schedules the delayed deletion of the item at the future. + pub fn schedule_delete(&mut self, item: ::Item) { + let deadline = Instant::now() + self.delay_for; + self.queue.push_back((item, deadline)); + } + + /// Clear the delayed deletion requests. + pub fn clear(&mut self) { + self.queue.clear(); + } + + /// Perform the queued deletions. + pub async fn perform(&mut self) { + let now = Instant::now(); + while let Some((_, deadline)) = self.queue.front() { + let deadline = *deadline; + trace!(message = "got delayed deletion deadline", ?deadline, ?now); + if deadline > now { + break; + } + trace!( + message = "processing delayed deletion for deadline", + ?deadline, + ?now + ); + let (item, _) = self.queue.pop_front().unwrap(); + self.inner.delete(item).await; + } + } + + /// Obtain the next deadline. + pub fn next_deadline(&self) -> Option { + self.queue.front().map(|(_, instant)| *instant) + } +} + +#[async_trait] +impl super::Write for Writer +where + T: super::Write + Send, + ::Item: Send + Sync, +{ + type Item = ::Item; + + async fn add(&mut self, item: Self::Item) { + self.inner.add(item).await + } + + async fn update(&mut self, item: Self::Item) { + self.inner.update(item).await + } + + async fn delete(&mut self, item: Self::Item) { + let deadline = Instant::now() + self.delay_for; + self.queue.push_back((item, deadline)); + } + + async fn resync(&mut self) { + self.queue.clear(); + self.inner.resync().await + } +} + +#[async_trait] +impl super::MaintainedWrite for Writer +where + T: super::MaintainedWrite + Send, + ::Item: Send + Sync, +{ + fn maintenance_request(&mut self) -> Option> { + let delayed_delete_deadline = self + .next_deadline() + .map(|delayed_delete_deadline| delay_until(delayed_delete_deadline)); + let downstream = self.inner.maintenance_request(); + + match (downstream, delayed_delete_deadline) { + (Some(downstream), Some(delayed_delete_deadline)) => { + let fut = futures::future::select(downstream, delayed_delete_deadline) + .map(|either| either.factor_first().0); + Some(Box::pin(fut)) + } + (None, Some(delayed_delete_deadline)) => Some(Box::pin(delayed_delete_deadline)), + (Some(downstream), None) => Some(downstream), + (None, None) => None, + } + } + + async fn perform_maintenance(&mut self) { + // Perform delayed deletes. + self.perform().await; + + // Do the downstream maintenance. + self.inner.perform_maintenance().await; + } +} + +#[cfg(test)] +mod tests { + use super::super::{mock, MaintainedWrite, Write}; + use super::*; + use crate::test_util; + use futures::{channel::mpsc, SinkExt, StreamExt}; + use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; + + const DELAY_FOR: Duration = Duration::from_secs(3600); + + fn prepare_test() -> ( + Writer>, + mpsc::Receiver>, + mpsc::Sender<()>, + ) { + let (events_tx, events_rx) = mpsc::channel(0); + let (actions_tx, actions_rx) = mpsc::channel(0); + let writer = mock::Writer::new(events_tx, actions_rx); + let writer = Writer::new(writer, DELAY_FOR); + (writer, events_rx, actions_tx) + } + + fn make_pod() -> Pod { + Pod { + metadata: Some(ObjectMeta { + name: Some("pod_name".to_owned()), + uid: Some("pod_uid".to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + } + } + + fn no_maintenance_test_flow(ft: FT, fa: FA) + where + FT: for<'a> FnOnce(&'a mut (dyn Write + Send)) -> BoxFuture<'a, ()> + + Send + + 'static, + FA: FnOnce(mock::ScenarioEvent) + Send + 'static, + { + test_util::trace_init(); + test_util::block_on_std(async move { + tokio::time::pause(); + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + // Ensure that right after construction maintenance is not required. + assert!(writer.maintenance_request().is_none()); + + let join = { + tokio::spawn(async move { + let event = events_rx.next().await.unwrap(); + fa(event); + actions_tx.send(()).await.unwrap(); + }) + }; + + // Ensure that before the operation maintenance is not required. + assert!(writer.maintenance_request().is_none()); + + { + let fut = ft(&mut writer); + // pin_mut!(fut); + fut.await; + } + + // Ensure that after the operation maintenance is not required. + assert!(writer.maintenance_request().is_none()); + + join.await.unwrap(); + tokio::time::resume(); + + // Ensure that finally maintenance is not required. + assert!(writer.maintenance_request().is_none()); + }) + } + + #[test] + fn add() { + let pod = make_pod(); + let assert_pod = pod.clone(); + no_maintenance_test_flow( + |writer| Box::pin(writer.add(pod)), + |event| assert_eq!(event.unwrap_op(), (assert_pod, mock::OpKind::Add)), + ) + } + + #[test] + fn update() { + let pod = make_pod(); + let assert_pod = pod.clone(); + no_maintenance_test_flow( + |writer| Box::pin(writer.update(pod)), + |event| assert_eq!(event.unwrap_op(), (assert_pod, mock::OpKind::Update)), + ) + } + + #[test] + fn delete() { + test_util::trace_init(); + test_util::block_on_std(async { + // Freeze time. + tokio::time::pause(); + + // Prepare test parameters. + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + // Ensure that right after construction maintenance is not required. + assert!(writer.maintenance_request().is_none()); + + // Prepare a mock pod. + let pod = make_pod(); + + writer.delete(pod.clone()).await; + + // Ensure the deletion event didn't trigger the actual deletion immediately. + assert!(events_rx.try_next().is_err()); + + // Ensure maintenance request is now present. + let maintenance_request = writer + .maintenance_request() + .expect("maintenance request should be present"); + + // Advance time. + tokio::time::advance(DELAY_FOR * 2).await; + + // At this point, maintenance request should be ready. + maintenance_request.await; + + // Run the assertion on the delete operation to ensure maintenance + // actually causes a delete. + let join = tokio::spawn(async move { + // Control for the deletion action. + let event = events_rx.next().await.unwrap(); + assert_eq!(event.unwrap_op(), (pod, mock::OpKind::Delete)); + actions_tx.send(()).await.unwrap(); + + // Control for the mock perform maintenance call (donwstream maintenance). + let event = events_rx.next().await.unwrap(); + assert!(matches!(event, mock::ScenarioEvent::Maintenance)); + actions_tx.send(()).await.unwrap(); + }); + + // Perform maintenance. + writer.perform_maintenance().await; + + // Join on assertion to guarantee panic propagation. + join.await.unwrap(); + + // Unfreeze time. + tokio::time::resume(); + }) + } + + #[test] + fn resync() { + no_maintenance_test_flow( + |writer| Box::pin(writer.resync()), + |event| assert!(matches!(event, mock::ScenarioEvent::Resync)), + ) + } +} diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index 246c96aa79c19..5480e9ba75b1f 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -3,6 +3,7 @@ use crate::kubernetes::hash_value::HashValue; use async_trait::async_trait; use evmap10::WriteHandle; +use futures::future::BoxFuture; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; /// A [`WriteHandle`] wrapper that implements [`super::Write`]. @@ -61,11 +62,25 @@ where } async fn resync(&mut self) { + // By omiting the flush here, we cache the results from the + // previous run until flush is issued when the new events + // begin arriving, reducing the time durig which the state + // has no data. self.inner.purge(); - // We do not flush on resync until the next per-item operation - // arrives. - // This way we preserve the old state while waiting for the data to - // populate the new state. + } +} + +#[async_trait] +impl super::MaintainedWrite for Writer +where + T: Metadata + Send, +{ + fn maintenance_request(&mut self) -> Option> { + None + } + + async fn perform_maintenance(&mut self) { + // noop } } diff --git a/src/kubernetes/state/instrumenting.rs b/src/kubernetes/state/instrumenting.rs index 1b4273767395d..a2cbbe51e5157 100644 --- a/src/kubernetes/state/instrumenting.rs +++ b/src/kubernetes/state/instrumenting.rs @@ -2,6 +2,7 @@ use crate::internal_events::kubernetes::instrumenting_state as internal_events; use async_trait::async_trait; +use futures::future::BoxFuture; /// A [`super::Write`] implementatiom that wraps another [`super::Write`] and /// adds instrumentation. @@ -44,13 +45,33 @@ where } } +#[async_trait] +impl super::MaintainedWrite for Writer +where + T: super::MaintainedWrite + Send, +{ + fn maintenance_request(&mut self) -> Option> { + self.inner.maintenance_request().map(|future| { + emit!(internal_events::StateMaintenanceRequested); + future + }) + } + + async fn perform_maintenance(&mut self) { + emit!(internal_events::StateMaintenancePerformed); + self.inner.perform_maintenance().await + } +} + #[cfg(test)] mod tests { - use super::super::{mock, Write}; + use super::super::{mock, MaintainedWrite, Write}; use super::*; use crate::test_util; use futures::{channel::mpsc, SinkExt, StreamExt}; use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; + use once_cell::sync::OnceCell; + use std::sync::{Mutex, MutexGuard}; fn prepare_test() -> ( Writer>, @@ -100,12 +121,13 @@ mod tests { }) } - fn assert_counter_incremented( + fn assert_counter_changed( before: Option, after: Option, + expected_difference: u64, ) { let before = before.unwrap_or_else(|| metrics_runtime::Measurement::Counter(0)); - let after = after.expect("after value was None"); + let after = after.unwrap_or_else(|| metrics_runtime::Measurement::Counter(0)); let (before, after) = match (before, after) { ( @@ -117,11 +139,20 @@ mod tests { let difference = after - before; - assert_eq!(difference, 1); + assert_eq!(difference, expected_difference); + } + + /// Guarantees only one test will run at a time. + /// This is required because we assert on a global state, and we don't + /// want interference. + fn tests_lock() -> MutexGuard<'static, ()> { + static INSTANCE: OnceCell> = OnceCell::new(); + INSTANCE.get_or_init(|| Mutex::new(())).lock().unwrap() } #[test] fn add() { + let _guard = tests_lock(); test_util::trace_init(); test_util::block_on_std(async { let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); @@ -139,7 +170,7 @@ mod tests { // By now metrics should've updated. let after = get_metric_value("item_added"); - assert_counter_incremented(before, after); + assert_counter_changed(before, after, 1); actions_tx.send(()).await.unwrap(); }) @@ -152,6 +183,7 @@ mod tests { #[test] fn update() { + let _guard = tests_lock(); test_util::trace_init(); test_util::block_on_std(async { let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); @@ -169,7 +201,7 @@ mod tests { // By now metrics should've updated. let after = get_metric_value("item_updated"); - assert_counter_incremented(before, after); + assert_counter_changed(before, after, 1); actions_tx.send(()).await.unwrap(); }) @@ -182,6 +214,7 @@ mod tests { #[test] fn delete() { + let _guard = tests_lock(); test_util::trace_init(); test_util::block_on_std(async { let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); @@ -199,7 +232,7 @@ mod tests { // By now metrics should've updated. let after = get_metric_value("item_deleted"); - assert_counter_incremented(before, after); + assert_counter_changed(before, after, 1); actions_tx.send(()).await.unwrap(); }) @@ -212,6 +245,7 @@ mod tests { #[test] fn resync() { + let _guard = tests_lock(); test_util::trace_init(); test_util::block_on_std(async { let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); @@ -225,7 +259,7 @@ mod tests { )); let after = get_metric_value("resynced"); - assert_counter_incremented(before, after); + assert_counter_changed(before, after, 1); actions_tx.send(()).await.unwrap(); }) @@ -235,4 +269,68 @@ mod tests { join.await.unwrap(); }) } + + #[test] + fn request_maintenance_without_maintenance() { + let _guard = tests_lock(); + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, _events_rx, _actions_tx) = prepare_test(); + let before = get_metric_value("maintenace_requested"); + let _ = writer.maintenance_request(); + let after = get_metric_value("maintenace_requested"); + assert_counter_changed(before, after, 0); + }) + } + + #[test] + fn request_maintenance_with_maintenance() { + let _guard = tests_lock(); + test_util::trace_init(); + test_util::block_on_std(async { + let (events_tx, _events_rx) = mpsc::channel(0); + let (_actions_tx, actions_rx) = mpsc::channel(0); + let (maintenance_request_events_tx, _maintenance_request_events_rx) = mpsc::channel(0); + let (_maintenance_request_actions_tx, maintenance_request_actions_rx) = + mpsc::channel(0); + let writer = mock::Writer::::new_with_maintenance( + events_tx, + actions_rx, + maintenance_request_events_tx, + maintenance_request_actions_rx, + ); + let mut writer = Writer::new(writer); + let before = get_metric_value("maintenace_requested"); + let _ = writer.maintenance_request(); + let after = get_metric_value("maintenace_requested"); + assert_counter_changed(before, after, 1); + }) + } + + #[test] + fn perform_maintenance() { + let _guard = tests_lock(); + test_util::trace_init(); + test_util::block_on_std(async { + let (mut writer, mut events_rx, mut actions_tx) = prepare_test(); + + let join = { + let before = get_metric_value("maintenace_performed"); + tokio::spawn(async move { + assert!(matches!( + events_rx.next().await.unwrap(), + mock::ScenarioEvent::Maintenance + )); + + let after = get_metric_value("maintenace_performed"); + assert_counter_changed(before, after, 1); + + actions_tx.send(()).await.unwrap(); + }) + }; + + writer.perform_maintenance().await; + join.await.unwrap(); + }) + } } diff --git a/src/kubernetes/state/mock.rs b/src/kubernetes/state/mock.rs index 06c2596461fa0..684df8a9e4ca1 100644 --- a/src/kubernetes/state/mock.rs +++ b/src/kubernetes/state/mock.rs @@ -5,11 +5,12 @@ use async_trait::async_trait; use futures::{ channel::mpsc::{Receiver, Sender}, + future::BoxFuture, SinkExt, StreamExt, }; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; -/// The kind of operation. +/// The kind of item-scoped operation. #[derive(Debug, PartialEq, Eq)] pub enum OpKind { /// Item added. @@ -20,13 +21,17 @@ pub enum OpKind { Delete, } -/// An event that's send to the test scenario driver. +/// An event that's send to the test scenario driver for operations flow. pub enum ScenarioEvent where T: Metadata + Send, { - Op(T, OpKind), + /// An item-scoped operation. + Item(T, OpKind), + /// Resync operation. Resync, + /// Maintenance is performed. + Maintenance, } impl ScenarioEvent @@ -35,8 +40,8 @@ where { pub fn unwrap_op(self) -> (T, OpKind) { match self { - ScenarioEvent::Op(val, op) => (val, op), - ScenarioEvent::Resync => panic!("unwrap_op on resync"), + ScenarioEvent::Item(val, op) => (val, op), + _ => panic!("unwrap_op on non-item op"), } } } @@ -56,6 +61,7 @@ where { events_tx: Sender>, actions_rx: Receiver<()>, + maintenance_request: Option<(Sender<()>, Receiver<()>)>, } impl Writer @@ -64,14 +70,41 @@ where { /// Create a new mock writer. /// Takes: - /// - `events_tx`, to which it sends the events when the mock action is - /// called; - /// - `actions_rx`, that is read a message from before the mock action + /// - `events_tx` - a message is sent here at the beginning of the + /// operation. + /// - `actions_rx` - a message is read from here before the operation /// returns. pub fn new(events_tx: Sender>, actions_rx: Receiver<()>) -> Self { Self { events_tx, actions_rx, + maintenance_request: None, + } + } + + /// Create a new mock writer (with maintenance flow). + /// Takes: + /// - `events_tx` - a message is sent here at the beginning of the + /// operation. + /// - `actions_rx` - a message is read from here before the operation + /// returns; + /// - `maintenance_request_events_tx` - a message is sent here at the + /// beginning of the maintenance request; + /// - `maintenance_request_events_tx` - a message is read from here before + /// the maintenance request returns. + pub fn new_with_maintenance( + events_tx: Sender>, + actions_rx: Receiver<()>, + maintenance_request_events_tx: Sender<()>, + maintenance_request_actions_rx: Receiver<()>, + ) -> Self { + Self { + events_tx, + actions_rx, + maintenance_request: Some(( + maintenance_request_events_tx, + maintenance_request_actions_rx, + )), } } } @@ -85,7 +118,7 @@ where async fn add(&mut self, item: Self::Item) { self.events_tx - .send(ScenarioEvent::Op(item, OpKind::Add)) + .send(ScenarioEvent::Item(item, OpKind::Add)) .await .unwrap(); self.actions_rx.next().await.unwrap(); @@ -93,7 +126,7 @@ where async fn update(&mut self, item: Self::Item) { self.events_tx - .send(ScenarioEvent::Op(item, OpKind::Update)) + .send(ScenarioEvent::Item(item, OpKind::Update)) .await .unwrap(); self.actions_rx.next().await.unwrap(); @@ -101,7 +134,7 @@ where async fn delete(&mut self, item: Self::Item) { self.events_tx - .send(ScenarioEvent::Op(item, OpKind::Delete)) + .send(ScenarioEvent::Item(item, OpKind::Delete)) .await .unwrap(); self.actions_rx.next().await.unwrap(); @@ -112,3 +145,28 @@ where self.actions_rx.next().await.unwrap(); } } + +#[async_trait] +impl super::MaintainedWrite for Writer +where + T: Metadata + Send, +{ + fn maintenance_request(&mut self) -> Option> { + if let Some((ref mut events_tx, ref mut actions_rx)) = self.maintenance_request { + Some(Box::pin(async move { + events_tx.send(()).await.unwrap(); + actions_rx.next().await.unwrap(); + })) + } else { + None + } + } + + async fn perform_maintenance(&mut self) { + self.events_tx + .send(ScenarioEvent::Maintenance) + .await + .unwrap(); + self.actions_rx.next().await.unwrap(); + } +} diff --git a/src/kubernetes/state/mod.rs b/src/kubernetes/state/mod.rs index 05dc96ff1094f..e5dc91096f768 100644 --- a/src/kubernetes/state/mod.rs +++ b/src/kubernetes/state/mod.rs @@ -1,8 +1,10 @@ //! Local representation of the Kubernetes API resources state. use async_trait::async_trait; +use futures::future::BoxFuture; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; +pub mod delayed_delete; pub mod evmap; pub mod instrumenting; pub mod mock; @@ -29,3 +31,34 @@ pub trait Write { /// Notify the state that resync is in progress. async fn resync(&mut self); } + +/// An extension of the [`Write`] type that adds maintenance support. +#[async_trait] +pub trait MaintainedWrite: Write { + /// A future that resolves when maintenance is required. + /// + /// Does not perform the maintenance itself, users must call + /// [`perform_maintenance`] to actually perform the maintenance. + /// + /// `None` if the state doesn't require maintenance, and + /// [`perform_maintenance`] shouldn't be called. + /// [`futures::future::FusedFuture`] should've been used here, but it's + /// not not trivially implementable with `async/await` syntax, so [`Option`] + /// wrapper is used instead for the same purpose. + /// + /// Circumstances of whether maintenance is required or not can change at + /// runtime. for instance, whether the maintenance is required can + /// depend on whether state is empty on not. Ultimately it's up to the state + /// implementation to decide whether maintenance is needed or not. + fn maintenance_request(&mut self) -> Option>; + + /// Perform the maintenance. + /// + /// If this function is called when no maintenance is required, this + /// function should just return. + /// + /// Wrapper [`MaintainedWrite`]s should always call the + /// `perform_maintenance` of the wrapped state when `perform_maintenance` is + /// called. + async fn perform_maintenance(&mut self); +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 83af1bb71ccee..624548dbd3402 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -156,14 +156,15 @@ impl Source { let (state_reader, state_writer) = evmap::new(); let state_writer = k8s::state::evmap::Writer::new(state_writer); let state_writer = k8s::state::instrumenting::Writer::new(state_writer); + let state_writer = + k8s::state::delayed_delete::Writer::new(state_writer, Duration::from_secs(60)); let mut reflector = k8s::reflector::Reflector::new( watcher, state_writer, Some(field_selector), Some(label_selector), - std::time::Duration::from_secs(1), - Some(std::time::Duration::from_secs(60)), + Duration::from_secs(1), ); let reflector_process = reflector.run(); From 5be1a819e208d64a2bfbee433bbe3b1c7a4742bd Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 10 Jun 2020 01:28:15 +0300 Subject: [PATCH 056/118] Ignore instrumenting state tests There no way to assert individual emits, and asserting metrics directly causes issues: - these tests break the internal tests at the metrics implementation itself, since we end up initializing the metrics controller twice; - testing metrics introduces unintended coupling between subsystems, ideally we only need to assert that we emit, but avoid assumptions on what the results of that emit are. Signed-off-by: MOZGIII --- src/kubernetes/state/instrumenting.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/kubernetes/state/instrumenting.rs b/src/kubernetes/state/instrumenting.rs index a2cbbe51e5157..1ba20b7cc4d51 100644 --- a/src/kubernetes/state/instrumenting.rs +++ b/src/kubernetes/state/instrumenting.rs @@ -150,6 +150,7 @@ mod tests { INSTANCE.get_or_init(|| Mutex::new(())).lock().unwrap() } + #[ignore] #[test] fn add() { let _guard = tests_lock(); @@ -181,6 +182,7 @@ mod tests { }) } + #[ignore] #[test] fn update() { let _guard = tests_lock(); @@ -212,6 +214,7 @@ mod tests { }) } + #[ignore] #[test] fn delete() { let _guard = tests_lock(); @@ -243,6 +246,7 @@ mod tests { }) } + #[ignore] #[test] fn resync() { let _guard = tests_lock(); @@ -270,6 +274,7 @@ mod tests { }) } + #[ignore] #[test] fn request_maintenance_without_maintenance() { let _guard = tests_lock(); @@ -283,6 +288,7 @@ mod tests { }) } + #[ignore] #[test] fn request_maintenance_with_maintenance() { let _guard = tests_lock(); @@ -307,6 +313,7 @@ mod tests { }) } + #[ignore] #[test] fn perform_maintenance() { let _guard = tests_lock(); From 1cc6efc93748c266f8f64d69a2aae1e4ede09b9d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 10 Jun 2020 22:44:08 +0300 Subject: [PATCH 057/118] Clean up reflector tests Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 45 +++++++++++++------------------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 74390833ab6f7..89e92f6fe5e9c 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -245,10 +245,7 @@ where #[cfg(test)] mod tests { - use super::{ - state::evmap::{Value, Writer}, - Error, Reflector, - }; + use super::{Error, Reflector}; use crate::{ kubernetes::{ instrumenting_watcher::InstrumentingWatcher, @@ -266,7 +263,7 @@ mod tests { use std::time::Duration; /// A helper function to simplify assertion on the `evmap` state. - fn gather_state(handle: &evmap10::ReadHandle>) -> Vec + fn gather_state(handle: &evmap10::ReadHandle>) -> Vec where T: Metadata + Clone, { @@ -370,8 +367,6 @@ mod tests { result.unwrap_err(); // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. drop(reflector); }); } @@ -547,8 +542,6 @@ mod tests { result.unwrap_err(); // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. drop(reflector); }) } @@ -558,6 +551,9 @@ mod tests { fn test_delayed_deletes() { test_util::trace_init(); test_util::block_on_std(async move { + // Freeze time. + tokio::time::pause(); + // Prepare state. let (state_events_tx, mut state_events_rx) = mpsc::channel(0); let (mut state_actions_tx, state_actions_rx) = mpsc::channel(0); @@ -578,9 +574,6 @@ mod tests { // Run test logic. let logic = tokio::spawn(async move { - // Pause the time. - tokio::time::pause(); - // Wait for watcher to request next invocation. assert!(matches!( watcher_events_rx.next().await.unwrap(), @@ -692,9 +685,6 @@ mod tests { .send(mock_watcher::ScenarioActionInvocation::ErrOther) .await .unwrap(); - - // Resume the time. - tokio::time::resume(); }); // Run the test and wait for an error. @@ -709,9 +699,10 @@ mod tests { result.unwrap_err(); // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. drop(reflector); + + // Unfreeze time. + tokio::time::resume(); }) } @@ -780,8 +771,6 @@ mod tests { )); // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. drop(reflector); }) } @@ -891,8 +880,6 @@ mod tests { result.unwrap_err(); // Explicitly drop the reflector at the very end. - // Internal evmap is dropped with the reflector, so readers won't - // work after drop. drop(reflector); }) } @@ -904,9 +891,12 @@ mod tests { expected_resulting_state: StateSnapshot, ) { test_util::block_on_std(async move { + // Freeze time. + tokio::time::pause(); + // Prepare state. let (state_reader, state_writer) = evmap10::new(); - let state_writer = Writer::new(state_writer); + let state_writer = state::evmap::Writer::new(state_writer); let state_writer = state::instrumenting::Writer::new(state_writer); let resulting_state_reader = state_reader.clone(); @@ -923,9 +913,6 @@ mod tests { // Run test logic. let logic = tokio::spawn(async move { - // Pause the time. - tokio::time::pause(); - // Process the invocations. for ( expected_state_before_op, @@ -948,7 +935,7 @@ mod tests { assert_eq!(state, expected_state_before_op); // Assert the resource version passed with watch invocation. - assert_eq!(expected_resource_version, watch_optional.resource_version); + assert_eq!(watch_optional.resource_version, expected_resource_version); // Determine the requested action from the test scenario. let responses = match expected_invocation_response { @@ -1016,9 +1003,6 @@ mod tests { .send(mock_watcher::ScenarioActionInvocation::ErrOther) .await .unwrap(); - - // Resume the time. - tokio::time::resume(); }); // Run the test and wait for an error. @@ -1040,6 +1024,9 @@ mod tests { // Internal evmap is dropped with the reflector, so readers won't // work after drop. drop(reflector); + + // Unfreeze time. + tokio::time::resume(); }) } } From 3b86b27597decd65fbc9c550df268baf8dcf4521 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 10 Jun 2020 21:50:15 +0300 Subject: [PATCH 058/118] Add flush debouncing to the evmap state Signed-off-by: MOZGIII --- src/kubernetes/debounce.rs | 245 +++++++++++++++++++++++++++++ src/kubernetes/mod.rs | 2 + src/kubernetes/reflector.rs | 2 +- src/kubernetes/state/evmap.rs | 55 +++++-- src/sources/kubernetes_logs/mod.rs | 3 +- 5 files changed, 293 insertions(+), 14 deletions(-) create mode 100644 src/kubernetes/debounce.rs diff --git a/src/kubernetes/debounce.rs b/src/kubernetes/debounce.rs new file mode 100644 index 0000000000000..f9d8d7b52e7dc --- /dev/null +++ b/src/kubernetes/debounce.rs @@ -0,0 +1,245 @@ +//! Arbitrary signal debouncing logic. +//! +//! Call [`Debounce::signal`] multiple times within the debounce time window, +//! and the [`Debounce::debounced`] will be resolved only once. + +use std::time::Duration; +use tokio::time::{delay_until, Instant}; + +/// Provides an arbitrary signal debouncing. +pub struct Debounce { + sequence_start: Option, + time: Duration, +} + +impl Debounce { + /// Create a new [`Debounce`]. + pub fn new(time: Duration) -> Self { + Self { + sequence_start: None, + time, + } + } + + /// Trigger a signal to debounce. + pub fn signal(&mut self) { + if self.sequence_start.is_some() { + return; + } + + self.sequence_start = Some(Instant::now() + self.time); + } + + /// Debounced signal. + /// + /// This function resolves after a debounce timeout since the first signal + /// in sequence expires. + /// If there hasn't been a signal, or the debounce timeout isn't yet + /// exausted - the future will be in a pending state. + pub async fn debounced(&mut self) { + let sequence_start = match self.sequence_start { + Some(val) => val, + None => futures::future::pending().await, + }; + + delay_until(sequence_start).await; + self.sequence_start = None; + } + + /// This function exposes the state of the debounce logic. + /// If this returns `false`, you shouldn't `poll` on [`debounced`], as it's + /// pending indefinitely. + pub fn is_debouncing(&self) -> bool { + self.sequence_start.is_some() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::{pin_mut, poll}; + + const TEST_DELAY_FRACTION: Duration = Duration::from_secs(60 * 60); // one hour + const TEST_DELAY: Duration = Duration::from_secs(24 * 60 * 60); // one day + + #[tokio::test] + async fn one_signal() { + tokio::time::pause(); + + let mut debounce = Debounce::new(TEST_DELAY); + assert!(debounce.sequence_start.is_none()); + + // Issue a signal. + debounce.signal(); + assert!(debounce.sequence_start.is_some()); + + { + // Request debounced signal. + let fut = debounce.debounced(); + pin_mut!(fut); + + // Shouldn't be available immediately. + assert!(poll!(&mut fut).is_pending()); + + // Simulate that we waited for some time, but no long enouh for the + // debounce to happen. + tokio::time::advance(TEST_DELAY_FRACTION).await; + + // Still shouldn't be available. + assert!(poll!(&mut fut).is_pending()); + + // Then wait long enough for debounce timeout to pass. + tokio::time::advance(TEST_DELAY * 2).await; + + // Should finally be available. + assert!(poll!(&mut fut).is_ready()); + } + + assert!(debounce.sequence_start.is_none()); + + tokio::time::resume(); + } + + #[tokio::test] + async fn late_request() { + tokio::time::pause(); + + let mut debounce = Debounce::new(TEST_DELAY); + assert!(debounce.sequence_start.is_none()); + + // Issue a signal. + debounce.signal(); + assert!(debounce.sequence_start.is_some()); + + // Simulate that we waited long enough. + tokio::time::advance(TEST_DELAY * 2).await; + assert!(debounce.sequence_start.is_some()); + + { + // Request a debounced signal. + let fut = debounce.debounced(); + pin_mut!(fut); + + // Should be available immediately. + assert!(poll!(&mut fut).is_ready()); + } + + assert!(debounce.sequence_start.is_none()); + + tokio::time::resume(); + } + + #[tokio::test] + async fn multiple_signals() { + tokio::time::pause(); + + let mut debounce = Debounce::new(TEST_DELAY); + assert!(debounce.sequence_start.is_none()); + + debounce.signal(); + + let first_signal_timestamp = debounce.sequence_start; + assert!(first_signal_timestamp.is_some()); + + debounce.signal(); + assert_eq!(debounce.sequence_start, first_signal_timestamp); + + tokio::time::advance(TEST_DELAY_FRACTION).await; + + debounce.signal(); + assert_eq!(debounce.sequence_start, first_signal_timestamp); + + { + let fut = debounce.debounced(); + pin_mut!(fut); + + assert!(poll!(&mut fut).is_pending()); + + tokio::time::advance(TEST_DELAY_FRACTION).await; + + assert!(poll!(&mut fut).is_pending()); + + tokio::time::advance(TEST_DELAY * 2).await; + + assert!(poll!(&mut fut).is_ready()); + } + + assert!(debounce.sequence_start.is_none()); + + tokio::time::resume(); + } + + #[tokio::test] + async fn sequence() { + tokio::time::pause(); + + let mut debounce = Debounce::new(TEST_DELAY); + assert!(debounce.sequence_start.is_none()); + + debounce.signal(); + + let first_signal_timestamp = debounce.sequence_start; + assert!(first_signal_timestamp.is_some()); + + debounce.signal(); + assert_eq!(debounce.sequence_start, first_signal_timestamp); + + tokio::time::advance(TEST_DELAY_FRACTION).await; + + debounce.signal(); + assert_eq!(debounce.sequence_start, first_signal_timestamp); + + { + let fut = debounce.debounced(); + pin_mut!(fut); + + assert!(poll!(&mut fut).is_pending()); + + tokio::time::advance(TEST_DELAY * 2).await; + + assert!(poll!(&mut fut).is_ready()); + } + + assert!(debounce.sequence_start.is_none()); + + debounce.signal(); + + let second_signal_timestamp = debounce.sequence_start; + assert!(second_signal_timestamp.is_some()); + assert_ne!(second_signal_timestamp, first_signal_timestamp); + + { + let fut = debounce.debounced(); + pin_mut!(fut); + + assert!(poll!(&mut fut).is_pending()); + + tokio::time::advance(TEST_DELAY * 2).await; + + assert!(poll!(&mut fut).is_ready()); + } + + assert!(debounce.sequence_start.is_none()); + + tokio::time::resume(); + } + + #[tokio::test] + async fn is_debouncing() { + tokio::time::pause(); + + let mut debounce = Debounce::new(TEST_DELAY); + assert_eq!(debounce.is_debouncing(), false); + + debounce.signal(); + assert_eq!(debounce.is_debouncing(), true); + + tokio::time::advance(TEST_DELAY * 2).await; + assert_eq!(debounce.is_debouncing(), true); + + debounce.debounced().await; + assert_eq!(debounce.is_debouncing(), false); + + tokio::time::resume(); + } +} diff --git a/src/kubernetes/mod.rs b/src/kubernetes/mod.rs index ac84b15e05e76..f9df1a26cf21c 100644 --- a/src/kubernetes/mod.rs +++ b/src/kubernetes/mod.rs @@ -22,6 +22,7 @@ pub mod api_watcher; pub mod client; +pub mod debounce; pub mod hash_value; pub mod instrumenting_watcher; pub mod mock_watcher; @@ -34,6 +35,7 @@ pub mod watch_request_builder; pub mod watcher; // Reexports for more elegant public API. +pub use debounce::Debounce; pub use hash_value::HashValue; pub use multi_response_decoder::MultiResponseDecoder; pub use reflector::Reflector; diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 89e92f6fe5e9c..9ff25c26d5749 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -896,7 +896,7 @@ mod tests { // Prepare state. let (state_reader, state_writer) = evmap10::new(); - let state_writer = state::evmap::Writer::new(state_writer); + let state_writer = state::evmap::Writer::new(state_writer, None); // test without debounce to avouid complexity let state_writer = state::instrumenting::Writer::new(state_writer); let resulting_state_reader = state_reader.clone(); diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index 5480e9ba75b1f..4598c692b4303 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -1,10 +1,11 @@ //! A state implementation backed by [`evmap10`]. -use crate::kubernetes::hash_value::HashValue; +use crate::kubernetes::{debounce::Debounce, hash_value::HashValue}; use async_trait::async_trait; use evmap10::WriteHandle; use futures::future::BoxFuture; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::ObjectMeta, Metadata}; +use std::time::Duration; /// A [`WriteHandle`] wrapper that implements [`super::Write`]. /// For use as a state writer implementation for @@ -14,6 +15,7 @@ where T: Metadata + Send, { inner: WriteHandle>, + debounced_flush: Option, } impl Writer @@ -22,10 +24,36 @@ where { /// Take a [`WriteHandle`], initialize it and return it wrapped with /// [`Self`]. - pub fn new(mut inner: WriteHandle>) -> Self { + pub fn new( + mut inner: WriteHandle>, + flush_debounce_timeout: Option, + ) -> Self { + // Prepare inner. inner.purge(); inner.refresh(); - Self { inner } + + // Prepare flush debounce. + let debounced_flush = flush_debounce_timeout.map(Debounce::new); + + Self { + inner, + debounced_flush, + } + } + + /// Debounced `flush`. + /// When a number of flush events arrive un a row, we buffer them such that + /// only the last one in the chain is propagated. + /// This is intended to improve the state behaivor at resync - by delaying + /// the `flush` proparagion, we maximize the time `evmap` remains populated, + /// ideally allowing a single transition from non-populated to populated + /// state. + fn debounced_flush(&mut self) { + if let Some(ref mut debounced_flush) = self.debounced_flush { + debounced_flush.signal(); + } else { + self.inner.flush(); + } } } @@ -36,35 +64,31 @@ where { type Item = T; - // TODO: debounce `flush` so that when a bunch of events arrive in a row - // within a certain small time window we commit all of them at once. - // This will improve the state behaivor at resync. - async fn add(&mut self, item: Self::Item) { if let Some((key, value)) = kv(item) { self.inner.insert(key, value); - self.inner.flush(); + self.debounced_flush(); } } async fn update(&mut self, item: Self::Item) { if let Some((key, value)) = kv(item) { self.inner.update(key, value); - self.inner.flush(); + self.debounced_flush(); } } async fn delete(&mut self, item: Self::Item) { if let Some((key, _value)) = kv(item) { self.inner.empty(key); - self.inner.flush(); + self.debounced_flush(); } } async fn resync(&mut self) { // By omiting the flush here, we cache the results from the // previous run until flush is issued when the new events - // begin arriving, reducing the time durig which the state + // begin arriving, reducing the time during which the state // has no data. self.inner.purge(); } @@ -76,11 +100,18 @@ where T: Metadata + Send, { fn maintenance_request(&mut self) -> Option> { + if let Some(ref mut debounced_flush) = self.debounced_flush { + if debounced_flush.is_debouncing() { + return Some(Box::pin(debounced_flush.debounced())); + } + } None } async fn perform_maintenance(&mut self) { - // noop + if self.debounced_flush.is_some() { + self.inner.flush(); + } } } diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 624548dbd3402..cd350cfcfba2a 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -154,7 +154,8 @@ impl Source { let watcher = k8s::api_watcher::ApiWatcher::new(client, Pod::watch_pod_for_all_namespaces); let watcher = k8s::instrumenting_watcher::InstrumentingWatcher::new(watcher); let (state_reader, state_writer) = evmap::new(); - let state_writer = k8s::state::evmap::Writer::new(state_writer); + let state_writer = + k8s::state::evmap::Writer::new(state_writer, Some(Duration::from_millis(10))); let state_writer = k8s::state::instrumenting::Writer::new(state_writer); let state_writer = k8s::state::delayed_delete::Writer::new(state_writer, Duration::from_secs(60)); From bbe3fc94a729f824677717152d2a204b439b8a30 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 10 Jun 2020 23:26:39 +0300 Subject: [PATCH 059/118] Proper delay control at main test flow Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 9ff25c26d5749..684856b9233cb 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -908,8 +908,9 @@ mod tests { let watcher = InstrumentingWatcher::new(watcher); // Prepare reflector. + let pause_between_requests = Duration::from_secs(60 * 60); // 1 hour let mut reflector = - Reflector::new(watcher, state_writer, None, None, Duration::from_secs(1)); + Reflector::new(watcher, state_writer, None, None, pause_between_requests); // Run test logic. let logic = tokio::spawn(async move { @@ -988,6 +989,10 @@ mod tests { .send(mock_watcher::ScenarioActionStream::Done) .await .unwrap(); + + // Advance the time to scroll pass the delay till next + // invocation. + tokio::time::advance(pause_between_requests * 2).await; } // We're done with the test! Shutdown the stream and force an From f61c2ec32d44f607410f230a1f68ba76b2c68e98 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 11 Jun 2020 01:09:03 +0300 Subject: [PATCH 060/118] Add evmap tests with and without debounce Signed-off-by: MOZGIII --- src/kubernetes/state/evmap.rs | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index 4598c692b4303..562d34966e0f9 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -124,3 +124,80 @@ fn kv>(object: T) -> Option<(String, Value)> { let key = value.uid()?.to_owned(); Some((key, value)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::kubernetes::state::{MaintainedWrite, Write}; + use k8s_openapi::api::core::v1::Pod; + + fn make_pod(uid: &str) -> Pod { + Pod { + metadata: Some(ObjectMeta { + uid: Some(uid.to_owned()), + ..ObjectMeta::default() + }), + ..Pod::default() + } + } + + #[test] + fn test_kv() { + let pod = make_pod("uid"); + let (key, val) = kv(pod.clone()).unwrap(); + assert_eq!(key, "uid"); + assert_eq!(val, Box::new(HashValue::new(pod))); + } + + #[tokio::test] + async fn test_without_debounce() { + let (state_reader, state_writer) = evmap10::new(); + let mut state_writer = Writer::new(state_writer, None); + + assert_eq!(state_reader.is_empty(), true); + assert!(state_writer.maintenance_request().is_none()); + + state_writer.add(make_pod("uid0")).await; + + assert_eq!(state_reader.is_empty(), false); + assert!(state_writer.maintenance_request().is_none()); + + drop(state_writer); + } + + #[tokio::test] + async fn test_with_debounce() { + // Due to https://github.com/tokio-rs/tokio/issues/2090 we're not + // pausing the time. + + let (state_reader, state_writer) = evmap10::new(); + let flush_debounce_timeout = Duration::from_millis(100); + let mut state_writer = Writer::new(state_writer, Some(flush_debounce_timeout)); + + assert_eq!(state_reader.is_empty(), true); + assert!(state_writer.maintenance_request().is_none()); + + state_writer.add(make_pod("uid0")).await; + state_writer.add(make_pod("uid1")).await; + + assert_eq!(state_reader.is_empty(), true); + assert!(state_writer.maintenance_request().is_some()); + + let join = tokio::spawn(async move { + let mut state_writer = state_writer; + state_writer.maintenance_request().unwrap().await; + state_writer.perform_maintenance().await; + state_writer + }); + + assert_eq!(state_reader.is_empty(), true); + + tokio::time::delay_for(flush_debounce_timeout * 2).await; + let mut state_writer = join.await.unwrap(); + + assert_eq!(state_reader.is_empty(), false); + assert!(state_writer.maintenance_request().is_none()); + + drop(state_writer); + } +} From bab77670ab48fe47b24ca4085d1e070037c9e888 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 00:47:53 +0300 Subject: [PATCH 061/118] Fix a typo Signed-off-by: MOZGIII --- src/kubernetes/client/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kubernetes/client/mod.rs b/src/kubernetes/client/mod.rs index dc005da4a355a..a46948dfaa35c 100644 --- a/src/kubernetes/client/mod.rs +++ b/src/kubernetes/client/mod.rs @@ -1,8 +1,8 @@ //! A Kubernetes API client built using Vector interfaces to the system //! resources as building blocks. //! -//! Here are a few pointers to the resources that were used as an inpiration for -//! this mod: +//! Here are a few pointers to the resources that were used as an inspiration +//! for this mod: //! //! - https://github.com/kubernetes/client-go/blob/master/tools/clientcmd/api/types.go //! From 38da9ce388c701727a20872bb594a080b5199fb9 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 01:10:01 +0300 Subject: [PATCH 062/118] Document the controversial join_host_port Signed-off-by: MOZGIII --- src/kubernetes/client/config/in_cluster.rs | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs index caf9ec20e4a3b..795a5da6b0853 100644 --- a/src/kubernetes/client/config/in_cluster.rs +++ b/src/kubernetes/client/config/in_cluster.rs @@ -67,9 +67,41 @@ pub enum Error { }, } +/// This function implements the exact same logic that Go's `net.JoinHostPort` +/// has. +/// Rust doesn't have anything like this out of the box, yet the reference +/// kubernetes client in-cluster config implementation uses it: +/// https://github.com/kubernetes/client-go/blob/3d5c80942cce510064da1ab62c579e190a0230fd/rest/config.go#L484 +/// +/// To avoid needlessly complicating the logic here, we simply implement the +/// `net.JoinHostPort` as it is in Go: https://golang.org/pkg/net/#JoinHostPort fn join_host_port(host: &str, port: &str) -> String { if host.contains(":") { return format!("[{}]:{}", host, port); } format!("{}:{}", host, port) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_join_host_port() { + // IPv4 + assert_eq!(join_host_port("0.0.0.0", "1234"), "0.0.0.0:1234"); + assert_eq!(join_host_port("127.0.0.1", "443"), "127.0.0.1:443"); + // IPv6 + assert_eq!(join_host_port("::", "1234"), "[::]:1234"); + assert_eq!( + join_host_port("2001:0db8:0000:0000:0000:8a2e:0370:7334", "1234"), + "[2001:0db8:0000:0000:0000:8a2e:0370:7334]:1234" + ); + assert_eq!( + join_host_port("2001:db8::8a2e:370:7334", "1234"), + "[2001:db8::8a2e:370:7334]:1234" + ); + // DNS + assert_eq!(join_host_port("example.com", "1234"), "example.com:1234"); + } +} From b4b549508f729494f5bf4b4fe060a18b493c3382 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 03:36:50 +0300 Subject: [PATCH 063/118] Improve instrumenting watcher events Signed-off-by: MOZGIII --- .../kubernetes/instrumenting_watcher.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/internal_events/kubernetes/instrumenting_watcher.rs b/src/internal_events/kubernetes/instrumenting_watcher.rs index 91b69c7b4cced..599fa6d106d86 100644 --- a/src/internal_events/kubernetes/instrumenting_watcher.rs +++ b/src/internal_events/kubernetes/instrumenting_watcher.rs @@ -7,7 +7,7 @@ pub struct WatchRequestInvoked; impl InternalEvent for WatchRequestInvoked { fn emit_metrics(&self) { - counter!("k8s_watch_request_invoked", 1); + counter!("k8s_watch_requests_invoked", 1); } } @@ -18,7 +18,11 @@ pub struct WatchRequestInvocationFailed { impl InternalEvent for WatchRequestInvocationFailed { fn emit_logs(&self) { - error!(message = "watch invocation failed", ?self.error); + error!(message = "watch invocation failed", error = ?self.error, rate_limit_secs = 5); + } + + fn emit_metrics(&self) { + counter!("k8s_watch_requests_failed", 1); } } @@ -38,6 +42,10 @@ pub struct WatchStreamErrored { impl InternalEvent for WatchStreamErrored { fn emit_logs(&self) { - error!(message = "watch stream errored", ?self.error); + error!(message = "watch stream errored", error = ?self.error, rate_limit_secs = 5); + } + + fn emit_metrics(&self) { + counter!("k8s_watch_stream_errors", 1); } } From 473bb12b53ae54cd41354c36dd6c8f7f9b67064d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 03:38:15 +0300 Subject: [PATCH 064/118] Improve api watcher events Signed-off-by: MOZGIII --- src/internal_events/kubernetes/api_watcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal_events/kubernetes/api_watcher.rs b/src/internal_events/kubernetes/api_watcher.rs index 52269a4a876e3..cabfa35d19f01 100644 --- a/src/internal_events/kubernetes/api_watcher.rs +++ b/src/internal_events/kubernetes/api_watcher.rs @@ -8,7 +8,7 @@ pub struct RequestPrepared { impl InternalEvent for RequestPrepared { fn emit_logs(&self) { - trace!(message = "request prepared", ?self.request); + trace!(message = "request prepared", request = ?self.request); } } @@ -19,6 +19,6 @@ pub struct ResponseReceived { impl InternalEvent for ResponseReceived { fn emit_logs(&self) { - trace!(message = "got response", ?self.response); + trace!(message = "got response", response = ?self.response); } } From 484e4ee6c3a59705127374997769c51a9ba879fb Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 19:41:21 +0300 Subject: [PATCH 065/118] Rename init to new Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index cd350cfcfba2a..9cbf9766ad1d1 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -67,7 +67,7 @@ impl SourceConfig for Config { shutdown: ShutdownSignal, out: mpsc::Sender, ) -> crate::Result { - let source = Source::init(self, Resolver, globals, name)?; + let source = Source::new(self, Resolver, globals, name)?; // TODO: this is a workaround for the legacy futures 0.1. // When the core is updated to futures 0.3 this should be simplied @@ -104,7 +104,7 @@ struct Source { } impl Source { - fn init( + fn new( config: &Config, resolver: Resolver, globals: &GlobalOptions, From 851dad6116825475a90b01f19f8a88e5cad85db1 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 12 Jun 2020 21:07:48 +0300 Subject: [PATCH 066/118] Use Strings at parser tests Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/cri.rs | 8 ++++---- src/sources/kubernetes_logs/parser/docker.rs | 4 ++-- src/sources/kubernetes_logs/parser/test_util.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index ad05c999615ed..fc11da152365a 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -96,7 +96,7 @@ mod tests { fn test_parsing() { let cases = vec![ ( - "2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1", + "2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1".into(), test_util::make_log_event( "The content of the log entry 1", "2016-10-06T00:17:09.669794202Z", @@ -105,7 +105,7 @@ mod tests { ), ), ( - "2016-10-06T00:17:09.669794202Z stdout P First line of log entry 2", + "2016-10-06T00:17:09.669794202Z stdout P First line of log entry 2".into(), test_util::make_log_event( "First line of log entry 2", "2016-10-06T00:17:09.669794202Z", @@ -114,7 +114,7 @@ mod tests { ), ), ( - "2016-10-06T00:17:09.669794202Z stdout P Second line of the log entry 2", + "2016-10-06T00:17:09.669794202Z stdout P Second line of the log entry 2".into(), test_util::make_log_event( "Second line of the log entry 2", "2016-10-06T00:17:09.669794202Z", @@ -123,7 +123,7 @@ mod tests { ), ), ( - "2016-10-06T00:17:10.113242941Z stderr F Last line of the log entry 2", + "2016-10-06T00:17:10.113242941Z stderr F Last line of the log entry 2".into(), test_util::make_log_event( "Last line of the log entry 2", "2016-10-06T00:17:10.113242941Z", diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index d8f1bf2bae450..f0bc2e83eb24f 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -109,7 +109,7 @@ mod tests { fn test_parsing() { let cases = vec![ ( - r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#, + r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), test_util::make_log_event( "The actual log line", "2016-10-05T00:00:30.082640485Z", @@ -118,7 +118,7 @@ mod tests { ), ), ( - r#"{"log": "The partial log line", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + r#"{"log": "The partial log line", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), test_util::make_log_event( "The partial log line", "2016-10-05T00:00:30.082640485Z", diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index 5db6558c8b692..8a27d04cbd12a 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -31,7 +31,7 @@ pub fn make_log_event(message: &str, timestamp: &str, stream: &str, is_partial: /// Shared logic for testing parsers. /// /// Takes a parser builder and a list of test cases. -pub fn test_parser(builder: B, cases: Vec<(&'static str, LogEvent)>) +pub fn test_parser(builder: B, cases: Vec<(String, LogEvent)>) where B: Fn() -> Box, { From e2d754261fd5f9a62abb1d449ee8539f58329b23 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 13 Jun 2020 00:34:23 +0300 Subject: [PATCH 067/118] Corrected partial message detection hueristics at docker parser Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/docker.rs | 93 +++++++++++++++----- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index f0bc2e83eb24f..4ce619bc849c5 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -55,6 +55,8 @@ impl Transform for Docker { } } +const DOCKER_MESSAGE_SPLIT_THRESHOLD: usize = 16 * 1024; // 16 Kib + fn normalize_event(log: &mut LogEvent) -> Result<(), NormalizationError> { // Parse and rename timestamp. let time = log.remove(&TIME).context(TimeFieldMissing)?; @@ -75,11 +77,21 @@ fn normalize_event(log: &mut LogEvent) -> Result<(), NormalizationError> { Value::Bytes(val) => val, _ => return Err(NormalizationError::LogValueUnexpectedType), }; - let is_partial = if message.last().map(|&b| b as char == '\n').unwrap_or(false) { + // Here we apply out heuristics to detect if messge is partial. + // Partial messages are only split in docker at the maximum message length + // (`DOCKER_MESSAGE_SPLIT_THRESHOLD`). + // Thus, for a message to be partial it also has to have exactly that + // length. + // Now, whether that message will or won't actually be partial if it has + // exactly the max length is unknown. We consider all messages with the + // exact length of `DOCKER_MESSAGE_SPLIT_THRESHOLD` bytes partial + // by default, and then, if they end with newline - consider that + // an exception and make them non-partial. + // This is still not ideal, and can potentially be improved. + let mut is_partial = message.len() == DOCKER_MESSAGE_SPLIT_THRESHOLD; + if message.last().map(|&b| b as char == '\n').unwrap_or(false) { message.truncate(message.len() - 1); - false - } else { - true + is_partial = false; }; log.insert(event::log_schema().message_key(), message); @@ -105,28 +117,65 @@ mod tests { use super::super::test_util; use super::*; + fn make_long_string(base: &str, len: usize) -> String { + base.chars().cycle().take(len).collect() + } + #[test] fn test_parsing() { - let cases = vec![ - ( - r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), - test_util::make_log_event( - "The actual log line", - "2016-10-05T00:00:30.082640485Z", - "stderr", - false, + let cases = + vec![ + ( + r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), + test_util::make_log_event( + "The actual log line", + "2016-10-05T00:00:30.082640485Z", + "stderr", + false, + ), + ), + ( + r#"{"log": "A line without newline chan at the end", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), + test_util::make_log_event( + "A line without newline chan at the end", + "2016-10-05T00:00:30.082640485Z", + "stdout", + false, + ), + ), + // Partial message due to message length. + ( + [ + r#"{"log": ""#, + make_long_string("partial ", 16 * 1024).as_str(), + r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + ] + .join(""), + test_util::make_log_event( + make_long_string("partial ",16 * 1024).as_str(), + "2016-10-05T00:00:30.082640485Z", + "stdout", + true, + ), ), - ), - ( - r#"{"log": "The partial log line", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), - test_util::make_log_event( - "The partial log line", - "2016-10-05T00:00:30.082640485Z", - "stdout", - true, + // Non-partial message, because message length matches but + // the message also ends with newline. + ( + [ + r#"{"log": ""#, + make_long_string("non-partial ", 16 * 1024 - 1).as_str(), + r"\n", + r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + ] + .join(""), + test_util::make_log_event( + make_long_string("non-partial ", 16 * 1024 - 1).as_str(), + "2016-10-05T00:00:30.082640485Z", + "stdout", + false, + ), ), - ), - ]; + ]; test_util::test_parser(build, cases); } From 8409d38f18c592b846945847964d9cfdfa2cf144 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 13 Jun 2020 00:35:01 +0300 Subject: [PATCH 068/118] Hint for where's what at parser test case Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/test_util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index 8a27d04cbd12a..8097088d8f217 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -41,6 +41,6 @@ where let output = parser .transform(input) .expect("parser failed to parse the event"); - assert_eq!(Event::Log(expected), output); + assert_eq!(Event::Log(expected), output, "expected left, actual right"); } } From 3111ac473fc165e4fe39a08a3b9cfa5d2855a0ae Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 11:06:32 +0300 Subject: [PATCH 069/118] Bump base image at skaffold/docker/Dockerfile to debian:bullseye-slim Without it local builds don't work host OS' with glibc 2.29 Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index 3c825ba040498..3e6842b47f5d6 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim RUN apt-get update && apt-get install -y ca-certificates tzdata && rm -rf /var/lib/apt/lists/* From 22839e4d1b573b929cdd4ff6de659c0274e76de0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 11:06:46 +0300 Subject: [PATCH 070/118] Better script layout at skaffold/docker/Dockerfile Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index 3e6842b47f5d6..cbcc434eb93f2 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -1,6 +1,10 @@ FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y ca-certificates tzdata && rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install -y \ + ca-certificates \ + tzdata \ + && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/usr/bin/vector"] From dfaed5e099ddd71714364cdbaa0d7e281e5e2f79 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 11:57:56 +0300 Subject: [PATCH 071/118] Convert an annotation failure warn log to internal event Signed-off-by: MOZGIII --- src/internal_events/kubernetes_logs.rs | 23 +++++++++++++++++++++++ src/sources/kubernetes_logs/mod.rs | 7 ++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index 0103b6e843501..a32ed44fd96e3 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,4 +1,5 @@ use super::InternalEvent; +use crate::Event; use metrics::counter; #[derive(Debug)] @@ -29,3 +30,25 @@ impl InternalEvent for KubernetesLogsEventReceived<'_> { ); } } + +#[derive(Debug)] +pub struct KubernetesLogsEventAnnotationFailed<'a> { + pub event: &'a Event, +} + +impl InternalEvent for KubernetesLogsEventAnnotationFailed<'_> { + fn emit_logs(&self) { + warn!( + message = "failed to annotate event with pod metadata", + event = ?self.event + ); + } + + fn emit_metrics(&self) { + counter!( + "k8s_event_annotation_failures", 1, + "component_kind" => "source", + "component_type" => "kubernetes_logs", + ); + } +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 9cbf9766ad1d1..3d67238319ac0 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -11,7 +11,7 @@ mod path_helpers; mod pod_metadata_annotator; use crate::event::{self, Event}; -use crate::internal_events::KubernetesLogsEventReceived; +use crate::internal_events::{KubernetesLogsEventAnnotationFailed, KubernetesLogsEventReceived}; use crate::kubernetes as k8s; use crate::{ dns::Resolver, @@ -215,10 +215,7 @@ impl Source { .filter_map(move |event| futures::future::ready(partial_events_merger.transform(event))) .map(move |mut event| { if annotator.annotate(&mut event).is_none() { - warn!( - message = "failed to annotate event with pod metadata", - ?event - ); + emit!(KubernetesLogsEventAnnotationFailed { event: &event }); } event }); From 0cdae0a341aae8618f0cfc33cfe188789ba3f299 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 12:35:44 +0300 Subject: [PATCH 072/118] Correct the shutdown logic Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 40 +++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 3d67238319ac0..4a0813397bfee 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -22,7 +22,7 @@ use crate::{ }; use evmap10::{self as evmap}; use file_source::{FileServer, FileServerShutdown, Fingerprinter}; -use futures::{future::FutureExt, sink::Sink, stream::StreamExt}; +use futures::{future::FutureExt, pin_mut, sink::Sink, stream::StreamExt}; use futures01::sync::mpsc; use k8s_openapi::api::core::v1::Pod; use k8s_paths_provider::K8sPathsProvider; @@ -222,15 +222,26 @@ impl Source { let event_processing_loop = events.map(Ok).forward(out); + let (reflector_process_shutdown_signal_tx, reflector_process_shutdown_signal_rx) = + futures::channel::oneshot::channel(); + pin_mut!(reflector_process); + let reflector_process_with_shutdown = + cancel_on_signal(reflector_process, reflector_process_shutdown_signal_rx); + use std::future::Future; use std::pin::Pin; let list: Vec + Send>>> = vec![ - Box::pin(reflector_process.map(|result| match result { - Ok(_val) => info!(message = "reflector process completed gracefully"), + Box::pin(reflector_process_with_shutdown.map(|result| match result { + Ok(()) => info!(message = "reflector process completed gracefully"), Err(error) => error!(message = "reflector process exited with an error", ?error), })), Box::pin(file_server_join_handle.map(|result| match result { - Ok(FileServerShutdown) => info!(message = "file server completed gracefully"), + Ok(FileServerShutdown) => { + info!(message = "file server completed gracefully"); + reflector_process_shutdown_signal_tx + .send(()) + .expect("unable to trigger reflector shutdown"); + } Err(error) => error!(message = "file server exited with an error", ?error), })), Box::pin(event_processing_loop.map(|result| { @@ -267,3 +278,24 @@ fn create_event(line: bytes05::Bytes, file: String) -> Event { event } + +/// Takes a `future` and a oneshot channel as a `signal`, and returns a future +/// that completes when the `future` errors or `signal` is sent or cancelled. +/// If `signal` is sent or cancelled, the `future` is dropped (and not polled +/// anymore). +async fn cancel_on_signal( + future: F, + signal: futures::channel::oneshot::Receiver<()>, +) -> Result<(), E> +where + F: std::future::Future> + Unpin, +{ + use futures::future::Either; + match futures::future::select(future, signal).await { + Either::Left((future_result, _)) => match future_result { + Ok(_infallible) => unreachable!("ok value is infallible, thus impossible to reach"), + Err(err) => Err(err), + }, + Either::Right((_signal_result, _)) => Ok(()), + } +} From 1115c0d7df34711ef7a986a42f35c00f0bf81e0d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 13:02:45 +0300 Subject: [PATCH 073/118] Specify STOPSIGNAL at skaffold/docker/Dockerfile Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index cbcc434eb93f2..6193a632653c0 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -12,3 +12,5 @@ ARG LOG=debug,vector=trace ENV LOG="$LOG" COPY vector /usr/bin/vector + +STOPSIGNAL SIGTERM From ca03bb45a363fb385872b1a29688fb25849d3ed9 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 13:07:58 +0300 Subject: [PATCH 074/118] Set terminationGracePeriodSeconds at distribution/kubernetes/vector-namespaced.yaml Signed-off-by: MOZGIII --- distribution/kubernetes/vector-namespaced.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/kubernetes/vector-namespaced.yaml b/distribution/kubernetes/vector-namespaced.yaml index 4fd6581be9f3c..4354ea78bc664 100644 --- a/distribution/kubernetes/vector-namespaced.yaml +++ b/distribution/kubernetes/vector-namespaced.yaml @@ -61,6 +61,7 @@ spec: - name: config-dir mountPath: /etc/vector/ readOnly: true + terminationGracePeriodSeconds: 60 tolerations: # This toleration is to have the daemonset runnable on master nodes. # Remove it if your masters can't run pods. From 741d0bb8ec0ad963bdca52823ef02cdb543deff1 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 13:22:05 +0300 Subject: [PATCH 075/118] Fix paths generation on Windows Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/path_helpers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sources/kubernetes_logs/path_helpers.rs b/src/sources/kubernetes_logs/path_helpers.rs index 3458776b7f0ba..8acd9e50db758 100644 --- a/src/sources/kubernetes_logs/path_helpers.rs +++ b/src/sources/kubernetes_logs/path_helpers.rs @@ -24,8 +24,8 @@ pub(super) fn build_pod_logs_directory( K8S_LOGS_DIR, &[pod_namespace, pod_name, pod_uid].join(LOG_PATH_DELIMITER), ] - .iter() - .collect() + .join("/") + .into() } /// Parses pod log file path and returns the log file info. From d11ef18c39904c44570cb3a74b4fd665ba8201f0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 17:16:01 +0300 Subject: [PATCH 076/118] Add a to do to unignore instrumenting state tests Signed-off-by: MOZGIII --- src/kubernetes/state/instrumenting.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/kubernetes/state/instrumenting.rs b/src/kubernetes/state/instrumenting.rs index 1ba20b7cc4d51..2401bd7838507 100644 --- a/src/kubernetes/state/instrumenting.rs +++ b/src/kubernetes/state/instrumenting.rs @@ -150,6 +150,18 @@ mod tests { INSTANCE.get_or_init(|| Mutex::new(())).lock().unwrap() } + // TODO: tests here are ignored because they cause interference with + // the metrics tests. + // There is no way to assert individual emits, and asserting metrics + // directly causes issues: + // - these tests break the internal tests at the metrics implementation + // itself, since we end up initializing the metrics controller twice; + // - testing metrics introduces unintended coupling between subsystems, + // ideally we only need to assert that we emit, but avoid assumptions on + // what the results of that emit are. + // Unignore them and/or properly reimplemenmt once the issues above are + // resolved. + #[ignore] #[test] fn add() { From 53d6958565ccc8933466609d5fcdf06d55f358a8 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 18:13:03 +0300 Subject: [PATCH 077/118] Add Kubernetes section to the CONTRIBUTING.md Signed-off-by: MOZGIII --- CONTRIBUTING.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6058c9d9f794..fb4f0889fff2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,7 @@ expanding into more specifics. 1. [Tips and Tricks](#tips-and-tricks) 1. [Benchmarking](#benchmarking) 1. [Profiling](#profiling) + 1. [Kubernetes](#kubernetes) 1. [Humans](#humans) 1. [Documentation](#documentation) 1. [Changelog](#changelog) @@ -547,6 +548,44 @@ cat stacks.folded | inferno-flamegraph > flamegraph.svg And that's it! You now have a flamegraph SVG file that can be opened and navigated in your favorite web browser. +### Kubernetes + +There is a special flow for when you develop portions of Vector that are +designed to work with Kubernetes, like `kubernetes_logs` source or the +`deployment/kubernetes/*.yaml` configs. + +This flow facilitates building Vector and deploying it into a cluster. + +There are some extra requirements: + +* `linux` system (create an issue if you want to work with another OS and we'll + help); +* [`skaffold`](https://skaffold.dev/) +* [`kustomize`](https://kustomize.io/) +* [`minikube`](https://minikube.sigs.k8s.io/)-powered or other k8s cluster +* [`cargo watch`](https://github.com/passcod/cargo-watch) + +Once you have the requirements, use the `scripts/skaffold.sh dev` command. + +It will + +1. build the `vector` binary in development mode, +2. build a docker image from this binary via `skaffold/docker/Dockerfile`, +3. deploy `vector` into the Kubernetes cluster at your current kubectl context + using the built docker image and a mix of our production deployment + configuration from the `distribution/kubernetes/*.yaml` and the special + dev-flow configuration at `skaffold/manifests/*.yaml`. + +As the result of invoking the `scripts/skaffold.sh dev`, you should see +a `skaffold` process running on your local machine, printing the logs from the +deployed `vector` instance. + +To stop the process, press `Ctrl+C`, and wait for `skaffold` to clean up +the cluster state and exit. + +`scripts/skaffold.sh` wraps `skaffold`, you can use other `skaffold` subcommands +if it fits you better. + ## Humans After making your change, you'll want to prepare it for Vector's users From 8170521e1432b140aa819fae805f8e042e35dc5f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 19:04:53 +0300 Subject: [PATCH 078/118] Solve the issue with the config generation Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 4a0813397bfee..ce6940cac1e5e 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -35,6 +35,9 @@ use tokio::task::spawn_blocking; /// The key we use for `file` field. const FILE_KEY: &str = "file"; +/// The `self_node_name` value env var key. +const SELF_NODE_NAME_ENV_KEY: &str = "VECTOR_SELF_NODE_NAME"; + /// Configuration for the `kubernetes_logs` source. #[derive(Deserialize, Serialize, Debug, Clone, Default)] #[serde(deny_unknown_fields, default)] @@ -42,6 +45,7 @@ pub struct Config { /// The `name` of the Kubernetes `Node` that Vector runs at. /// Required to filter the `Pod`s to only include the ones with the log /// files accessible locally. + #[serde(default = "default_self_node_name_env_template")] self_node_name: String, /// Automatically merge partial events. @@ -110,9 +114,15 @@ impl Source { globals: &GlobalOptions, name: &str, ) -> crate::Result { - let self_node_name = if config.self_node_name.is_empty() { - std::env::var("VECTOR_SELF_NODE_NAME") - .map_err(|_| "VECTOR_SELF_NODE_NAME is not set")? + let self_node_name = if config.self_node_name.is_empty() + || config.self_node_name == default_self_node_name_env_template() + { + std::env::var(SELF_NODE_NAME_ENV_KEY).map_err(|_| { + format!( + "self_node_name config value or {} env var is not set", + SELF_NODE_NAME_ENV_KEY + ) + })? } else { config.self_node_name.clone() }; @@ -299,3 +309,9 @@ where Either::Right((_signal_result, _)) => Ok(()), } } + +/// This function returns the default value for `self_node_name` variable +/// as it should be at the generated config file. +fn default_self_node_name_env_template() -> String { + format!("${{{}}}", SELF_NODE_NAME_ENV_KEY) +} From a6a515a9bffc62c1ff4e0eb0d48b96e67947548d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 15 Jun 2020 19:06:52 +0300 Subject: [PATCH 079/118] Better document the intent for the kubernetes_logs source Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index ce6940cac1e5e..ff77be182071a 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -1,6 +1,7 @@ //! This mod implements `kubernetes_logs` source. //! The scope of this source is to consume the log files that `kubelet` keeps -//! at `/var/log/pods` at the host of the k8s node. +//! at `/var/log/pods` at the host of the k8s node when `vector` itself is +//! running inside the cluster as a `DaemonSet`. #![deny(missing_docs)] From a18720cef973c869af87f4f2215cc1c1e5eeab7f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 16 Jun 2020 13:32:30 +0300 Subject: [PATCH 080/118] Print vector version upon docker build at skaffold/docker/Dockerfile This is so that we catch the error with inability to exec vector at container build time, rather than at runtime. Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index 6193a632653c0..0d266fe199e35 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -13,4 +13,6 @@ ENV LOG="$LOG" COPY vector /usr/bin/vector +RUN /usr/bin/vector --version + STOPSIGNAL SIGTERM From c0327c3620499668ea4ed34836b6a5278c5755ef Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 16 Jun 2020 13:55:17 +0300 Subject: [PATCH 081/118] Add more build-time output at skaffold/docker/Dockerfile Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index 0d266fe199e35..b32e8fc7912bd 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -13,6 +13,9 @@ ENV LOG="$LOG" COPY vector /usr/bin/vector -RUN /usr/bin/vector --version +RUN set -x \ + && ls -la /usr/bin/vector \ + && ldd /usr/bin/vector \ + && /usr/bin/vector --version STOPSIGNAL SIGTERM From 1e44894ffabb64a7863c6b999f7a85640d32221f Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 16 Jun 2020 14:22:34 +0300 Subject: [PATCH 082/118] Add patchelf at skaffold/docker/Dockerfile Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index b32e8fc7912bd..da198f724bb24 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -4,6 +4,7 @@ RUN apt-get update \ && apt-get install -y \ ca-certificates \ tzdata \ + patchelf \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/usr/bin/vector"] @@ -16,6 +17,8 @@ COPY vector /usr/bin/vector RUN set -x \ && ls -la /usr/bin/vector \ && ldd /usr/bin/vector \ + && patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 /usr/bin/vector \ + && ldd /usr/bin/vector \ && /usr/bin/vector --version STOPSIGNAL SIGTERM From bed7aa4ffc7d22092908f0846e7d29076918bba1 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 16 Jun 2020 14:23:00 +0300 Subject: [PATCH 083/118] Optimize commands order at skaffold/docker/Dockerfile for layer caching Signed-off-by: MOZGIII --- skaffold/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/skaffold/docker/Dockerfile b/skaffold/docker/Dockerfile index da198f724bb24..87947b2a8bcc6 100644 --- a/skaffold/docker/Dockerfile +++ b/skaffold/docker/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/usr/bin/vector"] +STOPSIGNAL SIGTERM ARG LOG=debug,vector=trace ENV LOG="$LOG" @@ -20,5 +21,3 @@ RUN set -x \ && patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 /usr/bin/vector \ && ldd /usr/bin/vector \ && /usr/bin/vector --version - -STOPSIGNAL SIGTERM From fe4b2c5b7359d36dfb466beeea06edba17dad294 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 17 Jun 2020 22:14:02 +0300 Subject: [PATCH 084/118] Add kubectl at CONTRIBUTING.md Signed-off-by: MOZGIII --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb4f0889fff2d..26dc891d47c9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -561,6 +561,7 @@ There are some extra requirements: * `linux` system (create an issue if you want to work with another OS and we'll help); * [`skaffold`](https://skaffold.dev/) +* [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) * [`kustomize`](https://kustomize.io/) * [`minikube`](https://minikube.sigs.k8s.io/)-powered or other k8s cluster * [`cargo watch`](https://github.com/passcod/cargo-watch) From f5dc48f5169f81e45509cb44efa204f75c5e483e Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 18 Jun 2020 17:22:25 +0300 Subject: [PATCH 085/118] Add additional details at CONTRIBUTING.md Signed-off-by: MOZGIII --- CONTRIBUTING.md | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 26dc891d47c9d..2bc9c32084475 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -556,18 +556,26 @@ designed to work with Kubernetes, like `kubernetes_logs` source or the This flow facilitates building Vector and deploying it into a cluster. -There are some extra requirements: +#### Requirements + +There are some extra requirements besides what you'd normally need to work on +Vector: * `linux` system (create an issue if you want to work with another OS and we'll help); * [`skaffold`](https://skaffold.dev/) +* [`docker`](https://www.docker.com/) * [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) * [`kustomize`](https://kustomize.io/) * [`minikube`](https://minikube.sigs.k8s.io/)-powered or other k8s cluster * [`cargo watch`](https://github.com/passcod/cargo-watch) +#### The dev flow + Once you have the requirements, use the `scripts/skaffold.sh dev` command. +That's it, just one command should take care of everything! + It will 1. build the `vector` binary in development mode, @@ -575,7 +583,8 @@ It will 3. deploy `vector` into the Kubernetes cluster at your current kubectl context using the built docker image and a mix of our production deployment configuration from the `distribution/kubernetes/*.yaml` and the special - dev-flow configuration at `skaffold/manifests/*.yaml`. + dev-flow configuration at `skaffold/manifests/*.yaml`; see + `kustomization.yaml` for the exact specification. As the result of invoking the `scripts/skaffold.sh dev`, you should see a `skaffold` process running on your local machine, printing the logs from the @@ -587,6 +596,36 @@ the cluster state and exit. `scripts/skaffold.sh` wraps `skaffold`, you can use other `skaffold` subcommands if it fits you better. +#### Troubleshooting + +You might need to tweak `skaffold`, here are some hints: + +* `skaffold` will try to detect whether a local cluster is used; in local + cluster is used it won't push the docker images it builds to a registry. + See [this page](https://skaffold.dev/docs/environment/local-cluster/) + for how you can troubleshoot and tweak this behavior. + +* `skaffold` can rewrite the image name so that you don't try to push a docker + image to a repo that you don't have access to. + See [this page](https://skaffold.dev/docs/environment/image-registries/) + for more info. + +* For the rest of the `skaffold` tweaks you might want to apply check out + [this page](https://skaffold.dev/docs/environment/). + +#### Going through the dev flow manually + +Is some cases `skaffold` may not work. It's possible to go through the dev flow +manually, without `skaffold`. + +One of the important thing `skaffold` does is it patches the configuration to +tie things together. If you want to go without it, you'll have to take care of +that yourself, thus some additional knowledge of Kubernetes inner workings is +required. + +Essentially, the steps you have to take to deploy manually are the same that +`skaffold` will perform, and they're outlined at the previous section. + ## Humans After making your change, you'll want to prepare it for Vector's users From 7c7fbbbe0c209500d47c09b8d9b10979f6812339 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Fri, 19 Jun 2020 06:07:10 +0300 Subject: [PATCH 086/118] Fix the data dir description Signed-off-by: MOZGIII --- distribution/kubernetes/vector-namespaced.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/kubernetes/vector-namespaced.yaml b/distribution/kubernetes/vector-namespaced.yaml index 4354ea78bc664..6e52aaba2ff7f 100644 --- a/distribution/kubernetes/vector-namespaced.yaml +++ b/distribution/kubernetes/vector-namespaced.yaml @@ -7,7 +7,7 @@ data: # Configuration for vector. # Docs: https://vector.dev/docs/ - # Configure the controlled by the deployment. + # Data dir is location controlled at the `DaemonSet`. data_dir = "/vector-data-dir" # Ingest logs from Kubernetes. From 2032d9911ee64ed72537fc6d3a61693b67c7d4d8 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 22 Jun 2020 05:09:56 +0300 Subject: [PATCH 087/118] Move transform utils to the source mod to avoid introducing global items Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 1 + src/sources/kubernetes_logs/parser/mod.rs | 6 ++---- src/sources/kubernetes_logs/partial_events_merger.rs | 3 ++- src/sources/kubernetes_logs/transform_utils/mod.rs | 2 ++ .../kubernetes_logs/transform_utils}/optional.rs | 0 .../kubernetes_logs/transform_utils}/pick.rs | 0 src/transforms/util/mod.rs | 6 ------ 7 files changed, 7 insertions(+), 11 deletions(-) create mode 100644 src/sources/kubernetes_logs/transform_utils/mod.rs rename src/{transforms/util => sources/kubernetes_logs/transform_utils}/optional.rs (100%) rename src/{transforms/util => sources/kubernetes_logs/transform_utils}/pick.rs (100%) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index ff77be182071a..4606b07d7de79 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -10,6 +10,7 @@ mod parser; mod partial_events_merger; mod path_helpers; mod pod_metadata_annotator; +mod transform_utils; use crate::event::{self, Event}; use crate::internal_events::{KubernetesLogsEventAnnotationFailed, KubernetesLogsEventReceived}; diff --git a/src/sources/kubernetes_logs/parser/mod.rs b/src/sources/kubernetes_logs/parser/mod.rs index 480e7c631a68c..f243ff3970d64 100644 --- a/src/sources/kubernetes_logs/parser/mod.rs +++ b/src/sources/kubernetes_logs/parser/mod.rs @@ -1,7 +1,5 @@ -use crate::transforms::{ - util::pick::{IterPicker, Passthrough, PickOnce}, - Transform, -}; +use super::transform_utils::pick::{IterPicker, Passthrough, PickOnce}; +use crate::transforms::Transform; mod cri; mod docker; diff --git a/src/sources/kubernetes_logs/partial_events_merger.rs b/src/sources/kubernetes_logs/partial_events_merger.rs index 652763d582dd1..27e90bef39e98 100644 --- a/src/sources/kubernetes_logs/partial_events_merger.rs +++ b/src/sources/kubernetes_logs/partial_events_merger.rs @@ -1,8 +1,9 @@ #![deny(missing_docs)] +use super::transform_utils::optional::Optional; use super::FILE_KEY; +use crate::event; use crate::transforms::merge::{Merge, MergeConfig}; -use crate::{event, transforms::util::optional::Optional}; use string_cache::Atom; /// Partial event merger. diff --git a/src/sources/kubernetes_logs/transform_utils/mod.rs b/src/sources/kubernetes_logs/transform_utils/mod.rs new file mode 100644 index 0000000000000..8ab9504dbdcfd --- /dev/null +++ b/src/sources/kubernetes_logs/transform_utils/mod.rs @@ -0,0 +1,2 @@ +pub mod optional; +pub mod pick; diff --git a/src/transforms/util/optional.rs b/src/sources/kubernetes_logs/transform_utils/optional.rs similarity index 100% rename from src/transforms/util/optional.rs rename to src/sources/kubernetes_logs/transform_utils/optional.rs diff --git a/src/transforms/util/pick.rs b/src/sources/kubernetes_logs/transform_utils/pick.rs similarity index 100% rename from src/transforms/util/pick.rs rename to src/sources/kubernetes_logs/transform_utils/pick.rs diff --git a/src/transforms/util/mod.rs b/src/transforms/util/mod.rs index 4fd05069e50bf..17381a44d34c7 100644 --- a/src/transforms/util/mod.rs +++ b/src/transforms/util/mod.rs @@ -1,8 +1,2 @@ #[cfg(any(feature = "transforms-lua"))] pub mod runtime_transform; - -// TODO: make these part of the vector core - enabled unconditionally. -#[cfg(any(feature = "sources-kubernetes-logs"))] -pub mod optional; -#[cfg(any(feature = "sources-kubernetes-logs"))] -pub mod pick; From 8d12cf206f2223a7476b8e7bb9d1b6c848516c64 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 22 Jun 2020 05:14:19 +0300 Subject: [PATCH 088/118] Fix the typo at CONTRIBUTING.md Signed-off-by: MOZGIII --- CONTRIBUTING.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bc9c32084475..806e324db6729 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -600,8 +600,9 @@ if it fits you better. You might need to tweak `skaffold`, here are some hints: -* `skaffold` will try to detect whether a local cluster is used; in local - cluster is used it won't push the docker images it builds to a registry. +* `skaffold` will try to detect whether a local cluster is used; if a local + cluster is used, `skaffold` won't push the docker images it builds to a + registry. See [this page](https://skaffold.dev/docs/environment/local-cluster/) for how you can troubleshoot and tweak this behavior. From a74d9766c7bc2d7f2367f7bbd16853578f7fafaa Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 22 Jun 2020 05:19:24 +0300 Subject: [PATCH 089/118] Eliminate the ApiWatcher::invoke_boxed_stream Signed-off-by: MOZGIII --- src/kubernetes/api_watcher.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs index 68805dce4f495..ed435055f4426 100644 --- a/src/kubernetes/api_watcher.rs +++ b/src/kubernetes/api_watcher.rs @@ -88,20 +88,6 @@ where let body = response.into_body(); Ok(k8s_stream::body(body)) } - - async fn invoke_boxed_stream( - &mut self, - watch_optional: WatchOptional<'_>, - ) -> Result< - BoxStream< - 'static, - Result::Object>, k8s_stream::Error>, - >, - watcher::invocation::Error, - > { - let stream = self.invoke(watch_optional).await?; - Ok(Box::pin(stream)) - } } impl Watcher for ApiWatcher @@ -121,7 +107,12 @@ where watch_optional: WatchOptional<'a>, ) -> BoxFuture<'a, Result>> { - Box::pin(self.invoke_boxed_stream(watch_optional)) + Box::pin(async move { + self.invoke(watch_optional) + .await + .map(Box::pin) + .map(|stream| stream as BoxStream<_>) + }) } } From 9b80bb97763ac81b4473d18f7a2e74e4f73416f2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 22 Jun 2020 05:25:13 +0300 Subject: [PATCH 090/118] Add code docs for join_host_port Signed-off-by: MOZGIII --- src/kubernetes/client/config/in_cluster.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs index 795a5da6b0853..6ce45de2839c0 100644 --- a/src/kubernetes/client/config/in_cluster.rs +++ b/src/kubernetes/client/config/in_cluster.rs @@ -77,8 +77,10 @@ pub enum Error { /// `net.JoinHostPort` as it is in Go: https://golang.org/pkg/net/#JoinHostPort fn join_host_port(host: &str, port: &str) -> String { if host.contains(":") { + // If IPv6 address is used, use a special notation. return format!("[{}]:{}", host, port); } + // Use traditional notation for domain names and IPv4 addresses. format!("{}:{}", host, port) } From 4052c8e5f790f23edd415f27d5c8643283ba64c8 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 10:02:20 +0300 Subject: [PATCH 091/118] Add a lifecycle system to manage futures sanely and reliably Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/lifecycle.rs | 134 +++++++++++++++++++++++ src/sources/kubernetes_logs/mod.rs | 113 ++++++++----------- src/sources/kubernetes_logs/util.rs | 70 ++++++++++++ 3 files changed, 252 insertions(+), 65 deletions(-) create mode 100644 src/sources/kubernetes_logs/lifecycle.rs create mode 100644 src/sources/kubernetes_logs/util.rs diff --git a/src/sources/kubernetes_logs/lifecycle.rs b/src/sources/kubernetes_logs/lifecycle.rs new file mode 100644 index 0000000000000..ea0c75aed392e --- /dev/null +++ b/src/sources/kubernetes_logs/lifecycle.rs @@ -0,0 +1,134 @@ +use crate::shutdown::{ShutdownSignal, ShutdownSignalToken}; +use futures::channel::oneshot; +use futures::future::{select, BoxFuture, Either}; +use futures::StreamExt; +use futures::{compat::Compat01As03, pin_mut, ready, stream::FuturesUnordered}; +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +/// Lifecycle encapsulates logic for managing a lifecycle of multiple futures +/// that are bounded together by a shared shutdown condition. +/// +/// If any of the futures completes, or global shutdown it requeted, all of the +/// managed futures are requested to shutdown. They can do so gracefully after +/// completing their work. +#[derive(Debug)] +pub struct Lifecycle<'bound> { + futs: FuturesUnordered>, + fut_shutdowns: Vec>, +} + +/// Holds a "global" shutdown signal or shutdown signal token. +/// Effectively used to hold the token or signal such that it can be dropped +/// after the shutdown is complete. +#[derive(Debug)] +pub enum GlobalShutdownToken { + /// The global shutdown singal was consumed, and we have a raw + /// [`ShutdownSignalToken`] now. + Token(ShutdownSignalToken), + /// The [`ShutdownSignal`] wasn't consumed, and still holds on to the + /// [`ShutdownSignalToken`]. Keep it around. + Ununsed(ShutdownSignal), +} + +impl<'bound> Lifecycle<'bound> { + /// Create a new [`Lifecycle`]. + pub fn new() -> Self { + Self { + futs: FuturesUnordered::new(), + fut_shutdowns: Vec::new(), + } + } + + /// Add a new future to be managed by the [`Lifecycle`]. + /// + /// Returns a [`Slot`] to be bound with the `Future`, and + /// a [`ShutdownHandle`] that is to be used by the bound future to wait for + /// shutdown. + pub fn add(&mut self) -> (Slot<'bound, '_>, ShutdownHandle) { + let (tx, rx) = oneshot::channel(); + let slot = Slot { + lifecycle: self, + shutdown_trigger: tx, + }; + let shutdown_handle = ShutdownHandle(rx); + (slot, shutdown_handle) + } + + /// Run the managed futures and keep track of the shutdown process. + pub async fn run(mut self, mut global_shutdown: ShutdownSignal) -> GlobalShutdownToken { + let first_task_fut = self.futs.next(); + pin_mut!(first_task_fut); + + let global_shutdown_fut = Compat01As03::new(&mut global_shutdown); + let token = match select(first_task_fut, global_shutdown_fut).await { + Either::Left((None, _)) => { + trace!(message = "lifecycle had no tasks upon run, we're done"); + GlobalShutdownToken::Ununsed(global_shutdown) + } + Either::Left((Some(()), _)) => { + trace!(message = "lifecycle had the first task completed"); + GlobalShutdownToken::Ununsed(global_shutdown) + } + Either::Right((shutdown_signal_token_result, _)) => { + let shutdown_signal_token = shutdown_signal_token_result.unwrap(); + trace!(message = "lifecycle got a global shutdown request"); + GlobalShutdownToken::Token(shutdown_signal_token) + } + }; + + // Send the shutdowns to all managed futures. + for fut_shutdown in self.fut_shutdowns { + if fut_shutdown.send(()).is_err() { + trace!( + message = "error while sending a future shutdown, \ + the receiver is already dropped; \ + this is not a problem" + ); + } + } + + // Wait for all the futures to complete. + while let Some(()) = self.futs.next().await { + trace!(message = "a lifecycle-managed future completed after shutdown was requested"); + } + + // Return the global shutdown token so that caller can perform it's + // cleanup. + token + } +} + +/// Represents an unbounded slot at the lifecycle. +#[derive(Debug)] +pub struct Slot<'bound, 'lc> { + lifecycle: &'lc mut Lifecycle<'bound>, + shutdown_trigger: oneshot::Sender<()>, +} + +impl<'bound, 'lc> Slot<'bound, 'lc> { + /// Bind the lifecycle slot to a concrete future. + /// The passed future MUST start it's shutdown process when requested to + /// shutdown via the signal passed from the corresponding + /// [`ShutdownHandle`]. + pub fn bind(self, future: BoxFuture<'bound, ()>) { + self.lifecycle.futs.push(future); + self.lifecycle.fut_shutdowns.push(self.shutdown_trigger); + } +} + +/// A handle that allows waiting for the lifecycle-issued shutdown. +#[derive(Debug)] +pub struct ShutdownHandle(oneshot::Receiver<()>); + +impl Future for ShutdownHandle { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let _ = ready!(Pin::new(&mut self.0).poll(cx)); + Poll::Ready(()) + } +} diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 4606b07d7de79..fa5812e24606d 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -6,11 +6,13 @@ #![deny(missing_docs)] mod k8s_paths_provider; +mod lifecycle; mod parser; mod partial_events_merger; mod path_helpers; mod pod_metadata_annotator; mod transform_utils; +mod util; use crate::event::{self, Event}; use crate::internal_events::{KubernetesLogsEventAnnotationFailed, KubernetesLogsEventReceived}; @@ -22,17 +24,18 @@ use crate::{ topology::config::{DataType, GlobalOptions, SourceConfig, SourceDescription}, transforms::Transform, }; +use bytes05::Bytes; use evmap10::{self as evmap}; use file_source::{FileServer, FileServerShutdown, Fingerprinter}; -use futures::{future::FutureExt, pin_mut, sink::Sink, stream::StreamExt}; +use futures::{future::FutureExt, sink::Sink, stream::StreamExt}; use futures01::sync::mpsc; use k8s_openapi::api::core::v1::Pod; use k8s_paths_provider::K8sPathsProvider; +use lifecycle::Lifecycle; use pod_metadata_annotator::PodMetadataAnnotator; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; -use tokio::task::spawn_blocking; /// The key we use for `file` field. const FILE_KEY: &str = "file"; @@ -147,9 +150,9 @@ impl Source { }) } - async fn run(self, out: O, shutdown: ShutdownSignal) -> crate::Result<()> + async fn run(self, out: O, global_shutdown: ShutdownSignal) -> crate::Result<()> where - O: Sink + Send, + O: Sink + Send + 'static, >::Error: std::error::Error, { let Self { @@ -202,15 +205,8 @@ impl Source { remove_after: None, }; - let (file_source_tx, file_source_rx) = futures::channel::mpsc::channel(100); - - let span = info_span!("file_server"); - let file_server_join_handle = spawn_blocking(move || { - let _enter = span.enter(); - let result = - file_server.run(file_source_tx, futures::compat::Compat01As03::new(shutdown)); - result.expect("file server exited with an error") - }); + let (file_source_tx, file_source_rx) = + futures::channel::mpsc::channel::<(Bytes, String)>(100); let mut parser = parser::build(); let mut partial_events_merger = partial_events_merger::build(auto_partial_merge); @@ -234,50 +230,58 @@ impl Source { let event_processing_loop = events.map(Ok).forward(out); - let (reflector_process_shutdown_signal_tx, reflector_process_shutdown_signal_rx) = - futures::channel::oneshot::channel(); - pin_mut!(reflector_process); - let reflector_process_with_shutdown = - cancel_on_signal(reflector_process, reflector_process_shutdown_signal_rx); - - use std::future::Future; - use std::pin::Pin; - let list: Vec + Send>>> = vec![ - Box::pin(reflector_process_with_shutdown.map(|result| match result { - Ok(()) => info!(message = "reflector process completed gracefully"), - Err(error) => error!(message = "reflector process exited with an error", ?error), - })), - Box::pin(file_server_join_handle.map(|result| match result { - Ok(FileServerShutdown) => { - info!(message = "file server completed gracefully"); - reflector_process_shutdown_signal_tx - .send(()) - .expect("unable to trigger reflector shutdown"); + let mut lifecycle = Lifecycle::new(); + { + let (slot, shutdown) = lifecycle.add(); + let fut = + util::cancel_on_signal(reflector_process, shutdown).map(|result| match result { + Ok(()) => info!(message = "reflector process completed gracefully"), + Err(error) => { + error!(message = "reflector process exited with an error", ?error) + } + }); + slot.bind(Box::pin(fut)); + } + { + let (slot, shutdown) = lifecycle.add(); + let fut = util::run_file_server(file_server, file_source_tx, shutdown).map(|result| { + match result { + Ok(FileServerShutdown) => info!(message = "file server completed gracefully"), + Err(error) => error!(message = "file server exited with an error", ?error), } - Err(error) => error!(message = "file server exited with an error", ?error), - })), - Box::pin(event_processing_loop.map(|result| { + }); + slot.bind(Box::pin(fut)); + } + { + let (slot, shutdown) = lifecycle.add(); + let fut = util::complete_with_deadline_on_signal( + event_processing_loop, + shutdown, + Duration::from_secs(30), // more than enough time to propagate + ) + .map(|result| { match result { - Ok(()) => info!(message = "event processing loop completed gracefully"), - Err(error) => error!( + Ok(Ok(())) => info!(message = "event processing loop completed gracefully"), + Ok(Err(error)) => error!( message = "event processing loop exited with an error", ?error ), + Err(error) => error!( + message = "event processing loop timed out during the shutdown", + ?error + ), }; - })), - ]; - let mut futs: futures::stream::FuturesUnordered<_> = list.into_iter().collect(); - - while let Some(()) = futs.next().await { - trace!(message = "another future complete"); + }); + slot.bind(Box::pin(fut)); } + lifecycle.run(global_shutdown).await; info!(message = "done"); Ok(()) } } -fn create_event(line: bytes05::Bytes, file: String) -> Event { +fn create_event(line: Bytes, file: String) -> Event { let mut event = Event::from(line); // Add source type. @@ -291,27 +295,6 @@ fn create_event(line: bytes05::Bytes, file: String) -> Event { event } -/// Takes a `future` and a oneshot channel as a `signal`, and returns a future -/// that completes when the `future` errors or `signal` is sent or cancelled. -/// If `signal` is sent or cancelled, the `future` is dropped (and not polled -/// anymore). -async fn cancel_on_signal( - future: F, - signal: futures::channel::oneshot::Receiver<()>, -) -> Result<(), E> -where - F: std::future::Future> + Unpin, -{ - use futures::future::Either; - match futures::future::select(future, signal).await { - Either::Left((future_result, _)) => match future_result { - Ok(_infallible) => unreachable!("ok value is infallible, thus impossible to reach"), - Err(err) => Err(err), - }, - Either::Right((_signal_result, _)) => Ok(()), - } -} - /// This function returns the default value for `self_node_name` variable /// as it should be at the generated config file. fn default_self_node_name_env_template() -> String { diff --git a/src/sources/kubernetes_logs/util.rs b/src/sources/kubernetes_logs/util.rs new file mode 100644 index 0000000000000..acf5f18ce056c --- /dev/null +++ b/src/sources/kubernetes_logs/util.rs @@ -0,0 +1,70 @@ +use bytes05::Bytes; +use file_source::{paths_provider::PathsProvider, FileServer, FileServerShutdown}; +use futures::future::{select, Either}; +use futures::{pin_mut, Sink}; +use std::convert::Infallible; +use std::error::Error; +use std::{future::Future, time::Duration}; +use tokio::task::spawn_blocking; + +/// A tiny wrapper around a [`FileServer`] that runs it as a [`spawn_blocking`] +/// task. +pub async fn run_file_server( + file_server: FileServer, + chans: C, + shutdown: S, +) -> Result +where + PP: PathsProvider + Send + 'static, + C: Sink<(Bytes, String)> + Unpin + Send + 'static, + >::Error: Error + Send, + S: Future + Unpin + Send + 'static, +{ + let span = info_span!("file_server"); + let join_handle = spawn_blocking(move || { + let _enter = span.enter(); + let result = file_server.run(chans, shutdown); + result.expect("file server exited with an error") + }); + join_handle.await +} + +/// Takes a `future` returning a result with an [`Infallible`] Ok-value and +/// a `signal`, and returns a future that completes when the `future` errors or +/// the `signal` completes. +/// If `signal` is sent or cancelled, the `future` is dropped (and not polled +/// anymore). +pub async fn cancel_on_signal(future: F, signal: S) -> Result<(), E> +where + F: Future>, + S: Future, +{ + pin_mut!(future); + pin_mut!(signal); + match select(future, signal).await { + Either::Left((future_result, _)) => match future_result { + Ok(_infallible) => unreachable!("ok value is infallible, thus impossible to reach"), + Err(err) => Err(err), + }, + Either::Right(((), _)) => Ok(()), + } +} + +pub async fn complete_with_deadline_on_signal( + future: F, + signal: S, + deadline: Duration, +) -> Result<::Output, tokio::time::Elapsed> +where + F: Future, + S: Future, +{ + pin_mut!(future); + pin_mut!(signal); + let future = match select(future, signal).await { + Either::Left((future_output, _)) => return Ok(future_output), + Either::Right(((), future)) => future, + }; + pin_mut!(future); + tokio::time::timeout(deadline, future).await +} From 6840d395a6803e9fd2e09571006600b420fe2c33 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 10:38:18 +0300 Subject: [PATCH 092/118] Reorganized the mod and use clauses at src/sources/kubernetes_logs/mod.rs Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index fa5812e24606d..5b386ecc90787 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -5,15 +5,6 @@ #![deny(missing_docs)] -mod k8s_paths_provider; -mod lifecycle; -mod parser; -mod partial_events_merger; -mod path_helpers; -mod pod_metadata_annotator; -mod transform_utils; -mod util; - use crate::event::{self, Event}; use crate::internal_events::{KubernetesLogsEventAnnotationFailed, KubernetesLogsEventReceived}; use crate::kubernetes as k8s; @@ -30,13 +21,23 @@ use file_source::{FileServer, FileServerShutdown, Fingerprinter}; use futures::{future::FutureExt, sink::Sink, stream::StreamExt}; use futures01::sync::mpsc; use k8s_openapi::api::core::v1::Pod; -use k8s_paths_provider::K8sPathsProvider; -use lifecycle::Lifecycle; -use pod_metadata_annotator::PodMetadataAnnotator; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::time::Duration; +mod k8s_paths_provider; +mod lifecycle; +mod parser; +mod partial_events_merger; +mod path_helpers; +mod pod_metadata_annotator; +mod transform_utils; +mod util; + +use k8s_paths_provider::K8sPathsProvider; +use lifecycle::Lifecycle; +use pod_metadata_annotator::PodMetadataAnnotator; + /// The key we use for `file` field. const FILE_KEY: &str = "file"; From 43a07350c4acb7a72fec5a601dc9afe4e174b27d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 10:44:01 +0300 Subject: [PATCH 093/118] Switch fingerprinter to checksum Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 5b386ecc90787..e41caf479c09a 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -197,11 +197,12 @@ impl Source { max_line_bytes: 32 * 1024, // 32 KiB data_dir, glob_minimum_cooldown: Duration::from_secs(10), - // Use device inodes for fingerprinting. - // - Docker recreates files on rotation: https://github.com/moby/moby/blob/75d655320e2a443185f8fa4992dc89bd2da0ea68/daemon/logger/loggerutils/logfile.go#L182-L222 - // - CRI-O recreates files on rotation: https://github.com/cri-o/cri-o/blob/ad83d2a35a30b8a336b16a0ea5f7afc6aebfb9b7/internal/oci/runtime_oci.go#L988-L1042 - // The rest should do the same. - fingerprinter: Fingerprinter::DevInode, + fingerprinter: Fingerprinter::Checksum { + // This value is based on the smallest meaningful log file size, + // taking into account Docker and CRI log formats. + fingerprint_bytes: 40, + ignored_header_bytes: 0, + }, oldest_first: false, remove_after: None, }; From d01dd882e1c2880c10a80e7e4628e70a55d0a1f0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 11:02:06 +0300 Subject: [PATCH 094/118] Revert "Switch fingerprinter to checksum" Apparently it breaks everything. E2E tests start failing. We need a better way. This reverts commit a4c2a7d88d0f06d70da9738518ae74b30a6fb835. Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index e41caf479c09a..5b386ecc90787 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -197,12 +197,11 @@ impl Source { max_line_bytes: 32 * 1024, // 32 KiB data_dir, glob_minimum_cooldown: Duration::from_secs(10), - fingerprinter: Fingerprinter::Checksum { - // This value is based on the smallest meaningful log file size, - // taking into account Docker and CRI log formats. - fingerprint_bytes: 40, - ignored_header_bytes: 0, - }, + // Use device inodes for fingerprinting. + // - Docker recreates files on rotation: https://github.com/moby/moby/blob/75d655320e2a443185f8fa4992dc89bd2da0ea68/daemon/logger/loggerutils/logfile.go#L182-L222 + // - CRI-O recreates files on rotation: https://github.com/cri-o/cri-o/blob/ad83d2a35a30b8a336b16a0ea5f7afc6aebfb9b7/internal/oci/runtime_oci.go#L988-L1042 + // The rest should do the same. + fingerprinter: Fingerprinter::DevInode, oldest_first: false, remove_after: None, }; From 343193ece123fbb04814e93ec8e3b04fa88f2b60 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 23 Jun 2020 13:22:36 +0300 Subject: [PATCH 095/118] Invert the condition at Debounce::signal Signed-off-by: MOZGIII --- src/kubernetes/debounce.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/kubernetes/debounce.rs b/src/kubernetes/debounce.rs index f9d8d7b52e7dc..0af7ac1cbcbf0 100644 --- a/src/kubernetes/debounce.rs +++ b/src/kubernetes/debounce.rs @@ -23,11 +23,9 @@ impl Debounce { /// Trigger a signal to debounce. pub fn signal(&mut self) { - if self.sequence_start.is_some() { - return; + if self.sequence_start.is_none() { + self.sequence_start = Some(Instant::now() + self.time); } - - self.sequence_start = Some(Instant::now() + self.time); } /// Debounced signal. From 3c6046f9ee8c02cd375435cc7bfee5350981e46a Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 25 Jun 2020 05:22:09 +0300 Subject: [PATCH 096/118] Fix typo at src/internal_events/kubernetes/instrumenting_state.rs Signed-off-by: MOZGIII --- src/internal_events/kubernetes/instrumenting_state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal_events/kubernetes/instrumenting_state.rs b/src/internal_events/kubernetes/instrumenting_state.rs index bb743a836ad54..9c256336dbcb1 100644 --- a/src/internal_events/kubernetes/instrumenting_state.rs +++ b/src/internal_events/kubernetes/instrumenting_state.rs @@ -46,12 +46,12 @@ impl InternalEvent for StateResynced { impl InternalEvent for StateMaintenanceRequested { fn emit_metrics(&self) { - counter!("k8s_state_ops", 1, "op_kind" => "maintenace_requested"); + counter!("k8s_state_ops", 1, "op_kind" => "maintenance_requested"); } } impl InternalEvent for StateMaintenancePerformed { fn emit_metrics(&self) { - counter!("k8s_state_ops", 1, "op_kind" => "maintenace_performed"); + counter!("k8s_state_ops", 1, "op_kind" => "maintenance_performed"); } } From 182469bdcc392b5460d517f57256a8b979b4b3bb Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 25 Jun 2020 05:23:59 +0300 Subject: [PATCH 097/118] Remove the rate limit from KubernetesLogsEventReceived Signed-off-by: MOZGIII --- src/internal_events/kubernetes_logs.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index a32ed44fd96e3..38f31ff2d0dd7 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -13,7 +13,6 @@ impl InternalEvent for KubernetesLogsEventReceived<'_> { trace!( message = "received one event.", %self.file, - rate_limit_secs = 10 ); } From 17d3f8fcf806f6703ef9b7b88e7e746c5ab9bb58 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 25 Jun 2020 05:24:26 +0300 Subject: [PATCH 098/118] Adjust the log style at KubernetesLogsEventReceived Signed-off-by: MOZGIII --- src/internal_events/kubernetes_logs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index 38f31ff2d0dd7..4add1ea95e218 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -11,8 +11,8 @@ pub struct KubernetesLogsEventReceived<'a> { impl InternalEvent for KubernetesLogsEventReceived<'_> { fn emit_logs(&self) { trace!( - message = "received one event.", - %self.file, + message = "received one event", + file = %self.file ); } From 36679b03ede6b8c239e5847cf7d322babeec8e75 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 25 Jun 2020 05:44:20 +0300 Subject: [PATCH 099/118] Move pod metadata annotation to access file path directly Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 18 +++--- .../kubernetes_logs/pod_metadata_annotator.rs | 59 ++----------------- 2 files changed, 14 insertions(+), 63 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 5b386ecc90787..88a200feee2aa 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -212,21 +212,21 @@ impl Source { let mut parser = parser::build(); let mut partial_events_merger = partial_events_merger::build(auto_partial_merge); - let events = file_source_rx.map(|(bytes, file)| { + let events = file_source_rx.map(move |(bytes, file)| { emit!(KubernetesLogsEventReceived { file: &file, byte_size: bytes.len(), }); - create_event(bytes, file) + let mut event = create_event(bytes, &file); + if annotator.annotate(&mut event, &file).is_none() { + emit!(KubernetesLogsEventAnnotationFailed { event: &event }); + } + event }); let events = events .filter_map(move |event| futures::future::ready(parser.transform(event))) - .filter_map(move |event| futures::future::ready(partial_events_merger.transform(event))) - .map(move |mut event| { - if annotator.annotate(&mut event).is_none() { - emit!(KubernetesLogsEventAnnotationFailed { event: &event }); - } - event + .filter_map(move |event| { + futures::future::ready(partial_events_merger.transform(event)) }); let event_processing_loop = events.map(Ok).forward(out); @@ -282,7 +282,7 @@ impl Source { } } -fn create_event(line: Bytes, file: String) -> Event { +fn create_event(line: Bytes, file: &str) -> Event { let mut event = Event::from(line); // Add source type. diff --git a/src/sources/kubernetes_logs/pod_metadata_annotator.rs b/src/sources/kubernetes_logs/pod_metadata_annotator.rs index e093069d84da6..af842bed8f59d 100644 --- a/src/sources/kubernetes_logs/pod_metadata_annotator.rs +++ b/src/sources/kubernetes_logs/pod_metadata_annotator.rs @@ -2,15 +2,11 @@ #![deny(missing_docs)] -use super::{path_helpers::parse_log_file_path, FILE_KEY}; -use crate::{ - event::{LogEvent, Value}, - kubernetes as k8s, Event, -}; +use super::path_helpers::parse_log_file_path; +use crate::{event::LogEvent, kubernetes as k8s, Event}; use evmap10::ReadHandle; use k8s_openapi::{api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::ObjectMeta}; use serde::{Deserialize, Serialize}; -use string_cache::Atom; #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(deny_unknown_fields, default)] @@ -55,10 +51,10 @@ impl PodMetadataAnnotator { /// Annotates an event with the information from the [`Pod::metadata`]. /// The event has to be obtained from kubernetes log file, and have a /// [`FILE_KEY`] field set with a file that the line came from. - pub fn annotate(&self, event: &mut Event) -> Option<()> { + pub fn annotate(&self, event: &mut Event, file: &str) -> Option<()> { let log = event.as_mut_log(); - let uid = pod_uid_from_log_event(log)?; - let guard = self.pods_state_reader.get(&uid)?; + let uid = parse_log_file_path(file)?.pod_uid; + let guard = self.pods_state_reader.get(uid)?; let entry = guard.get_one()?; let pod: &Pod = entry.as_ref(); let metadata = pod.metadata.as_ref()?; @@ -67,16 +63,6 @@ impl PodMetadataAnnotator { } } -fn pod_uid_from_log_event(log: &LogEvent) -> Option { - let value = log.get(&Atom::from(FILE_KEY))?; - let file = match value { - Value::Bytes(bytes) => String::from_utf8_lossy(bytes).into_owned(), - _ => return None, - }; - let info = parse_log_file_path(file.as_str())?; - Some(info.pod_uid.to_owned()) -} - fn annotate_from_metadata(log: &mut LogEvent, fields_spec: &FieldsSpec, metadata: &ObjectMeta) { for (ref key, ref val) in [ (&fields_spec.pod_name, &metadata.name), @@ -101,41 +87,6 @@ fn annotate_from_metadata(log: &mut LogEvent, fields_spec: &FieldsSpec, metadata mod tests { use super::*; - #[test] - fn test_pod_uid_from_log_event() { - let cases = vec![ - // Valid inputs. - ( - { - let mut log = LogEvent::default(); - log.insert( - FILE_KEY, - "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/sandbox0-container0-name/1.log", - ); - log - }, - Some("sandbox0-uid"), - ), - // Invalid inputs. - (LogEvent::default(), None), - ( - { - let mut log = LogEvent::default(); - log.insert(FILE_KEY, "qwerty"); - log - }, - None, - ), - ]; - - for (log, expected) in cases.into_iter() { - assert_eq!( - pod_uid_from_log_event(&log), - expected.map(ToOwned::to_owned) - ); - } - } - #[test] fn test_annotate_from_metadata() { let cases = vec![ From 480999445c5959f394118719556df0665fe6713d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 25 Jun 2020 08:44:57 +0300 Subject: [PATCH 100/118] Add test to ensure MultiResponseDecoder doesn't leak memory It was controversial, and so this test is added. Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 9c6dcc54198f9..9fd3bd036d6b9 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -407,4 +407,45 @@ mod tests { assert_eq!(dec.finish().unwrap_err(), b"{"); } + + #[test] + fn test_memory_usage() { + let mut dec = MultiResponseDecoder::::new(); + + let chunk = br#"{ + "type": "ADDED", + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "uid": "uid0" + } + } + }"#; + let mut chunks = chunk.iter().cycle(); + + let max_chunks_per_iter = 15; + + // Simulate processing a huge number of items. + for _ in 0..100_000 { + // Take random amout of bytes from the chunks iter and prepare the + // next chunk. + let to_take = rand::random::() % (chunk.len() * max_chunks_per_iter); + let next_chunk = (&mut chunks).take(to_take).cloned().collect::>(); + + // Process the chunk data. + let stream = dec.process_next_chunk(next_chunk.as_ref()); + drop(stream); // consume all the emitted items + } + + // Check that `pending_data` capacity didn't grow out way of hand. + // If we had issues with memory management, it would be the one + // to blow first. + assert!(dec.pending_data.capacity() <= chunk.len() * 100); + + // Ensure that response buffer never grows beyond it's capacitty limit. + // Capacity limit is set based on heuristics about `Vec` internals, and + // is adjusted to be as low as possible. + assert!(dec.responses_buffer.capacity() <= (max_chunks_per_iter + 2).next_power_of_two()); + } } From f0f2e36e3384a72fff6fe0bd5107698bfcdb832d Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 27 Jun 2020 15:27:06 +0300 Subject: [PATCH 101/118] Simplified the picker logic and added tests Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/cri.rs | 22 +-- src/sources/kubernetes_logs/parser/docker.rs | 115 ++++++------ src/sources/kubernetes_logs/parser/mod.rs | 9 +- src/sources/kubernetes_logs/parser/picker.rs | 75 ++++++++ .../kubernetes_logs/parser/test_util.rs | 5 +- .../kubernetes_logs/transform_utils/mod.rs | 1 - .../kubernetes_logs/transform_utils/pick.rs | 166 ------------------ 7 files changed, 149 insertions(+), 244 deletions(-) create mode 100644 src/sources/kubernetes_logs/parser/picker.rs delete mode 100644 src/sources/kubernetes_logs/transform_utils/pick.rs diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index fc11da152365a..dfa5946f1a1d9 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -9,10 +9,6 @@ use lazy_static::lazy_static; use snafu::{OptionExt, Snafu}; use string_cache::DefaultAtom as Atom; -pub fn build() -> Box { - Box::new(Cri::new()) -} - lazy_static! { pub static ref MULTILINE_TAG: Atom = Atom::from("multiline_tag"); } @@ -88,13 +84,14 @@ enum NormalizationError { } #[cfg(test)] -mod tests { +pub mod tests { use super::super::test_util; - use super::*; + use super::Cri; + use crate::event::LogEvent; - #[test] - fn test_parsing() { - let cases = vec![ + /// Shared test cases. + pub fn cases() -> Vec<(String, LogEvent)> { + vec![ ( "2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1".into(), test_util::make_log_event( @@ -131,8 +128,11 @@ mod tests { false, ), ), - ]; + ] + } - test_util::test_parser(build, cases); + #[test] + fn test_parsing() { + test_util::test_parser(|| Cri::new(), cases()); } } diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index 4ce619bc849c5..9bd01f4f2010d 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -10,10 +10,6 @@ use lazy_static::lazy_static; use snafu::{OptionExt, ResultExt, Snafu}; use string_cache::DefaultAtom as Atom; -pub fn build() -> Box { - Box::new(Docker::new()) -} - lazy_static! { pub static ref TIME: Atom = Atom::from("time"); pub static ref LOG: Atom = Atom::from("log"); @@ -113,70 +109,73 @@ enum NormalizationError { } #[cfg(test)] -mod tests { +pub mod tests { use super::super::test_util; - use super::*; + use super::Docker; + use crate::event::LogEvent; fn make_long_string(base: &str, len: usize) -> String { base.chars().cycle().take(len).collect() } - #[test] - fn test_parsing() { - let cases = - vec![ - ( - r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), - test_util::make_log_event( - "The actual log line", - "2016-10-05T00:00:30.082640485Z", - "stderr", - false, - ), + /// Shared test cases. + pub fn cases() -> Vec<(String, LogEvent)> { + vec![ + ( + r#"{"log": "The actual log line\n", "stream": "stderr", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), + test_util::make_log_event( + "The actual log line", + "2016-10-05T00:00:30.082640485Z", + "stderr", + false, ), - ( - r#"{"log": "A line without newline chan at the end", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), - test_util::make_log_event( - "A line without newline chan at the end", - "2016-10-05T00:00:30.082640485Z", - "stdout", - false, - ), + ), + ( + r#"{"log": "A line without newline chan at the end", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#.into(), + test_util::make_log_event( + "A line without newline chan at the end", + "2016-10-05T00:00:30.082640485Z", + "stdout", + false, ), - // Partial message due to message length. - ( - [ - r#"{"log": ""#, - make_long_string("partial ", 16 * 1024).as_str(), - r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, - ] - .join(""), - test_util::make_log_event( - make_long_string("partial ",16 * 1024).as_str(), - "2016-10-05T00:00:30.082640485Z", - "stdout", - true, - ), + ), + // Partial message due to message length. + ( + [ + r#"{"log": ""#, + make_long_string("partial ", 16 * 1024).as_str(), + r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + ] + .join(""), + test_util::make_log_event( + make_long_string("partial ",16 * 1024).as_str(), + "2016-10-05T00:00:30.082640485Z", + "stdout", + true, ), - // Non-partial message, because message length matches but - // the message also ends with newline. - ( - [ - r#"{"log": ""#, - make_long_string("non-partial ", 16 * 1024 - 1).as_str(), - r"\n", - r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, - ] - .join(""), - test_util::make_log_event( - make_long_string("non-partial ", 16 * 1024 - 1).as_str(), - "2016-10-05T00:00:30.082640485Z", - "stdout", - false, - ), + ), + // Non-partial message, because message length matches but + // the message also ends with newline. + ( + [ + r#"{"log": ""#, + make_long_string("non-partial ", 16 * 1024 - 1).as_str(), + r"\n", + r#"", "stream": "stdout", "time": "2016-10-05T00:00:30.082640485Z"}"#, + ] + .join(""), + test_util::make_log_event( + make_long_string("non-partial ", 16 * 1024 - 1).as_str(), + "2016-10-05T00:00:30.082640485Z", + "stdout", + false, ), - ]; + ), + ] + } - test_util::test_parser(build, cases); + #[test] + fn test_parsing() { + test_util::test_parser(|| Docker::new(), cases()); } } diff --git a/src/sources/kubernetes_logs/parser/mod.rs b/src/sources/kubernetes_logs/parser/mod.rs index f243ff3970d64..fbb7befdd7a24 100644 --- a/src/sources/kubernetes_logs/parser/mod.rs +++ b/src/sources/kubernetes_logs/parser/mod.rs @@ -1,15 +1,12 @@ -use super::transform_utils::pick::{IterPicker, Passthrough, PickOnce}; -use crate::transforms::Transform; - mod cri; mod docker; +mod picker; mod test_util; /// Parser for any log format supported by `kubelet`. -pub type Parser = PickOnce>>>; +pub type Parser = picker::Picker; /// Build a parser for any log format supported by `kubelet`. pub fn build() -> Parser { - let pickers = vec![docker::build(), cri::build(), Box::new(Passthrough)]; - PickOnce::new(IterPicker::new(pickers)) + picker::Picker::new() } diff --git a/src/sources/kubernetes_logs/parser/picker.rs b/src/sources/kubernetes_logs/parser/picker.rs new file mode 100644 index 0000000000000..4fb713972560a --- /dev/null +++ b/src/sources/kubernetes_logs/parser/picker.rs @@ -0,0 +1,75 @@ +use super::{cri::Cri, docker::Docker}; +use crate::{ + event::{self, Event, Value}, + transforms::Transform, +}; + +pub enum Picker { + Init, + Docker(Docker), + Cri(Cri), +} + +impl Picker { + pub fn new() -> Self { + Picker::Init + } +} + +impl Transform for Picker { + fn transform(&mut self, event: Event) -> Option { + match self { + Picker::Init => { + let message = event + .as_log() + .get(event::log_schema().message_key()) + .expect("message key must be present"); + let bytes = if let Value::Bytes(bytes) = message { + bytes + } else { + panic!("message value must be Bytes"); + }; + if bytes.len() > 1 && bytes[0] == b'{' { + *self = Picker::Docker(Docker::new()) + } else { + *self = Picker::Cri(Cri::new()) + } + self.transform(event) + } + Picker::Docker(t) => t.transform(event), + Picker::Cri(t) => t.transform(event), + } + } +} + +#[cfg(test)] +mod tests { + use super::super::{cri, docker, test_util}; + use super::{Picker, Transform}; + use crate::{event::LogEvent, Event}; + + /// Picker has to work for all test cases for underlying parsers. + fn cases() -> Vec<(String, LogEvent)> { + let mut cases = vec![]; + cases.extend(docker::tests::cases()); + cases.extend(cri::tests::cases()); + cases + } + + #[test] + fn test_parsing() { + test_util::test_parser(|| Picker::new(), cases()); + } + + #[test] + fn test_parsing_invalid() { + let cases = vec!["", "qwe", "{"]; + + for message in cases { + let input = Event::from(message); + let mut picker = Picker::new(); + let output = picker.transform(input); + assert!(output.is_none()); + } + } +} diff --git a/src/sources/kubernetes_logs/parser/test_util.rs b/src/sources/kubernetes_logs/parser/test_util.rs index 8097088d8f217..44149c0ce23b6 100644 --- a/src/sources/kubernetes_logs/parser/test_util.rs +++ b/src/sources/kubernetes_logs/parser/test_util.rs @@ -31,9 +31,10 @@ pub fn make_log_event(message: &str, timestamp: &str, stream: &str, is_partial: /// Shared logic for testing parsers. /// /// Takes a parser builder and a list of test cases. -pub fn test_parser(builder: B, cases: Vec<(String, LogEvent)>) +pub fn test_parser(builder: B, cases: Vec<(String, LogEvent)>) where - B: Fn() -> Box, + B: Fn() -> T, + T: Transform, { for (message, expected) in cases { let input = Event::from(message); diff --git a/src/sources/kubernetes_logs/transform_utils/mod.rs b/src/sources/kubernetes_logs/transform_utils/mod.rs index 8ab9504dbdcfd..4b24267e84d53 100644 --- a/src/sources/kubernetes_logs/transform_utils/mod.rs +++ b/src/sources/kubernetes_logs/transform_utils/mod.rs @@ -1,2 +1 @@ pub mod optional; -pub mod pick; diff --git a/src/sources/kubernetes_logs/transform_utils/pick.rs b/src/sources/kubernetes_logs/transform_utils/pick.rs deleted file mode 100644 index b8d1d5440197d..0000000000000 --- a/src/sources/kubernetes_logs/transform_utils/pick.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Dynamically pick a tranform to use. - -#![deny(missing_docs)] - -use crate::{event::Event, transforms::Transform}; - -/// Dynamically pick a tranform to use. -/// -/// The implementation would have multiple candidates and a way to determine -/// which transform to use. -pub trait Picker: Sized { - /// Consumes an event and the picker itself, and returns a transform to - /// use and a transformed version of the passed event. - /// - /// Note that the interface doesn't allow failures (i.e. it's not a - /// [`Result`], nor [`Option`]). - /// To handle failure modes, use provided [`Passthrough`], [`Discard`] or - /// [`Panic`] as the returned transform, and pass the event accordingly. - fn probe_event(self, event: Event) -> (Box, Option); -} - -/// Dynamically pick a transform to use once (on first event), and then use the -/// picked transform for the rest of the events. -/// -/// The first message obtained will be probed with picker, and then the system -/// will switch to using. -pub enum PickOnce -where - T: Picker + Send, -{ - /// The initial state. - Init(Option), - /// The picked transform. - Picked(Box), -} - -impl PickOnce -where - T: Picker + Send, -{ - /// Create a new [`PickOnce`] using the passed `T` as the initial state. - pub fn new(init: T) -> Self { - Self::Init(Some(init)) - } -} - -impl Transform for PickOnce -where - T: Picker + Send, -{ - fn transform(&mut self, event: Event) -> Option { - match self { - Self::Init(picker) => { - let picker = picker.take().expect("impossible state"); - let (transform, event) = picker.probe_event(event); - *self = Self::Picked(transform); - event - } - Self::Picked(transform) => transform.transform(event), - } - } -} - -/// A passthrough transform. Passes events as-is. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct Passthrough; - -impl Transform for Passthrough { - fn transform(&mut self, event: Event) -> Option { - Some(event) - } -} - -/// A discard transform. Returns `None` for every event. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct Discard; - -impl Transform for Discard { - fn transform(&mut self, _event: Event) -> Option { - None - } -} - -/// A panic transform. Just panics with the predefined message. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct Panic(pub &'static str); - -impl Transform for Panic { - fn transform(&mut self, _event: Event) -> Option { - panic!(self.0) - } -} - -/// Pick transform from an iterator by passing an event to each of the iterated -/// transport, and choosing the first that returns non-`None`. -pub struct IterPicker(T) -where - T: IntoIterator>; - -impl IterPicker -where - T: IntoIterator>, -{ - /// Create a new `IterPicker` from an iterator. - pub fn new(iter: T) -> Self { - Self(iter) - } -} - -impl Picker for IterPicker -where - T: IntoIterator>, -{ - fn probe_event(self, event: Event) -> (Box, Option) { - self.0 - .into_iter() - .find_map(|mut candidate| { - candidate - .transform(event.clone()) - .map(|event| (candidate, Some(event))) - }) - .expect("transform candidates exausted, and no suitable transform found") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn box_transform(transform: impl Transform + 'static) -> Box { - Box::new(transform) - } - - #[test] - #[should_panic(expected = "transform candidates exausted, and no suitable transform found")] - fn iter_picker_empty() { - let picker = IterPicker::new(vec![]); - picker.probe_event(Event::new_empty_log()); - } - - #[test] - fn iter_picker_passthrough() { - let picker = IterPicker::new(vec![box_transform(Passthrough)]); - picker.probe_event(Event::new_empty_log()); - } - - #[test] - #[should_panic(expected = "test panic")] - fn iter_picker_panic() { - let picker = IterPicker::new(vec![box_transform(Panic("test panic"))]); - picker.probe_event(Event::new_empty_log()); - } - - #[test] - #[should_panic(expected = "transform candidates exausted, and no suitable transform found")] - fn iter_picker_discard() { - let picker = IterPicker::new(vec![box_transform(Discard)]); - picker.probe_event(Event::new_empty_log()); - } - - #[test] - fn iter_picker_sequence() { - let picker = IterPicker::new(vec![box_transform(Discard), box_transform(Passthrough)]); - picker.probe_event(Event::new_empty_log()); - } -} From a839e3c38a01c0286324fe2c9072cda51ff84180 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 27 Jun 2020 16:11:14 +0300 Subject: [PATCH 102/118] Add a special case for `\n` at the MultiResponseDecoder::finish Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 9fd3bd036d6b9..2d84f1ac51eee 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -51,7 +51,9 @@ where /// Produces an error if there's unparsed data remaining. pub fn finish(self) -> Result<(), Vec> { let Self { pending_data, .. } = self; - if pending_data.is_empty() { + // Kubernetes sometimes adds `\n` to the response, consider this + // a valid termination case. + if pending_data.is_empty() || pending_data == b"\n" { return Ok(()); } Err(pending_data) @@ -408,6 +410,18 @@ mod tests { assert_eq!(dec.finish().unwrap_err(), b"{"); } + #[test] + fn test_allows_unparsed_newlines_at_finish() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk(b"\n"); + assert!(stream.next().is_none()); + } + + assert!(dec.finish().is_ok()); + } + #[test] fn test_memory_usage() { let mut dec = MultiResponseDecoder::::new(); From 91b77578fb20a65012d060b505c4763c039e3984 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 27 Jun 2020 17:27:06 +0300 Subject: [PATCH 103/118] Workaround for watch API failures This should've been fixed by now, but it's still causing issues with older k8s versions that we want to support. Signed-off-by: MOZGIII --- src/kubernetes/reflector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 684856b9233cb..63ff882f07898 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -142,7 +142,7 @@ where label_selector: self.label_selector.as_ref().map(|s| s.as_str()), pretty: None, resource_version: self.resource_version.get(), - timeout_seconds: None, + timeout_seconds: Some(290), // https://github.com/kubernetes/kubernetes/issues/6513 allow_watch_bookmarks: Some(true), }; let stream = self.watcher.watch(watch_optional).await?; @@ -518,7 +518,7 @@ mod tests { label_selector: Some("labels".to_owned()), pretty: None, resource_version: None, - timeout_seconds: None, + timeout_seconds: Some(290), } ); From 7d2559ea855035e029c7bba8328b3f3c07a2b6d3 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sat, 27 Jun 2020 17:51:16 +0300 Subject: [PATCH 104/118] Add a long line test case for the CRI format parser Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/parser/cri.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index dfa5946f1a1d9..7baa8f0951e7d 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -89,6 +89,10 @@ pub mod tests { use super::Cri; use crate::event::LogEvent; + fn make_long_string(base: &str, len: usize) -> String { + base.chars().cycle().take(len).collect() + } + /// Shared test cases. pub fn cases() -> Vec<(String, LogEvent)> { vec![ @@ -128,6 +132,20 @@ pub mod tests { false, ), ), + // A part of the partial message with a realistic length. + ( + [ + r#"2016-10-06T00:17:10.113242941Z stdout P "#, + make_long_string("very long message ", 16 * 1024).as_str(), + ] + .join(""), + test_util::make_log_event( + make_long_string("very long message ", 16 * 1024).as_str(), + "2016-10-06T00:17:10.113242941Z", + "stdout", + true, + ), + ), ] } From 70471dc7e30baf33111fa35324d3046f561a7bef Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sun, 28 Jun 2020 10:00:29 +0300 Subject: [PATCH 105/118] Log the buffer state at response decoding error Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 2d84f1ac51eee..4e7c2fb439fb2 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -37,6 +37,7 @@ where } Err(ResponseError::NeedMoreData) => break, Err(error) => { + trace!(message = "error while decoding response", pending_data = ?self.pending_data, ?error); self.responses_buffer.push(Err(error)); break; } From 487b66d7e8b1aa4a22945add9bf912c6a3a86652 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sun, 28 Jun 2020 10:45:50 +0300 Subject: [PATCH 106/118] Handle data parsing errors as stream ends Signed-off-by: MOZGIII --- src/kubernetes/stream.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/kubernetes/stream.rs b/src/kubernetes/stream.rs index 8b28007370526..b1f607153ef4d 100644 --- a/src/kubernetes/stream.rs +++ b/src/kubernetes/stream.rs @@ -30,6 +30,16 @@ where let responses = decoder.process_next_chunk(chunk.as_ref()); emit!(internal_events::ChunkProcessed{ byte_size: chunk.len() }); for response in responses { + // Sometimes Kubernetes API starts returning `null`s in + // the object field while streaming the response. + // Handle it as if the stream has ended. + // See https://github.com/kubernetes/client-go/issues/334 + if let Err(ResponseError::Json(error)) = &response { + if error.is_data() { + warn!(message = "handling response json parsing data error as steram end", ?error); + return; + } + } let response = response.context(Parsing)?; yield response; } @@ -166,4 +176,18 @@ mod tests { assert!(out_stream.next().await.is_none()); }) } + + #[test] + fn test_sudden_null() { + test_util::trace_init(); + test_util::block_on_std(async move { + let chunks: Vec> = vec![Ok("null")]; + let sample_body = hyper_body_from_chunks(chunks); + + let out_stream = body::<_, WatchResponse>(sample_body); + pin_mut!(out_stream); + + assert!(out_stream.next().await.is_none()); + }) + } } From 295f48818ea907440b60c683073fbdb8991f104b Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 29 Jun 2020 12:08:55 +0300 Subject: [PATCH 107/118] Ensure we only read logs under container name subdirectory Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/k8s_paths_provider.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sources/kubernetes_logs/k8s_paths_provider.rs b/src/sources/kubernetes_logs/k8s_paths_provider.rs index efcfec3db9a82..09fd3b70ca53d 100644 --- a/src/sources/kubernetes_logs/k8s_paths_provider.rs +++ b/src/sources/kubernetes_logs/k8s_paths_provider.rs @@ -69,10 +69,16 @@ where .into_iter() .flat_map(move |dir| { glob_impl( + // We seek to match the paths like + // `//.log` - paths managed by + // the `kubelet` as part of Kubernetes core logging + // architecture. + // In some setups, there will also be paths like + // `/.log` - those we want to skip. &[ dir.to_str() .expect("non-utf8 path to pod logs dir is not supported"), - "**/*.log", + "*/*.log", ] .join("/"), ) @@ -174,7 +180,7 @@ mod tests { // Calls to the glob mock. vec![( // The pattern to expect at the mock. - "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/**/*.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/*/*.log", // The paths to return from the mock. vec![ "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/container1/qwe.log", @@ -203,7 +209,7 @@ mod tests { ..Pod::default() }, vec![( - "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/**/*.log", + "/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid/*/*.log", vec![], )], vec![], From c651606d0a5088d0fca105b3841d796e0988bad7 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 29 Jun 2020 13:29:06 +0300 Subject: [PATCH 108/118] Promote trace to an error at src/kubernetes/multi_response_decoder.rs Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 4e7c2fb439fb2..c1cde2a872847 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -37,7 +37,7 @@ where } Err(ResponseError::NeedMoreData) => break, Err(error) => { - trace!(message = "error while decoding response", pending_data = ?self.pending_data, ?error); + error!(message = "error while decoding response", pending_data = ?self.pending_data, ?error); self.responses_buffer.push(Err(error)); break; } From 8b7b221d340e2d8d5c7f3dacc646a9bfc6a46826 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 29 Jun 2020 18:49:18 +0300 Subject: [PATCH 109/118] Add a test case for bookmark parsing error Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 44 +++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index c1cde2a872847..f7c6e28f79d8f 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -65,7 +65,7 @@ where mod tests { use super::*; use k8s_openapi::{ - api::core::v1::Pod, + api::core::v1::{Pod, PodSpec}, apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, WatchResponse, }; @@ -463,4 +463,46 @@ mod tests { // is adjusted to be as low as possible. assert!(dec.responses_buffer.capacity() <= (max_chunks_per_iter + 2).next_power_of_two()); } + + #[test] + fn test_practical_error_case_1() { + let mut dec = MultiResponseDecoder::::new(); + + { + let mut stream = dec.process_next_chunk(&[ + 10, 123, 34, 116, 121, 112, 101, 34, 58, 34, 66, 79, 79, 75, 77, 65, 82, 75, 34, + 44, 34, 111, 98, 106, 101, 99, 116, 34, 58, 123, 34, 107, 105, 110, 100, 34, 58, + 34, 80, 111, 100, 34, 44, 34, 97, 112, 105, 86, 101, 114, 115, 105, 111, 110, 34, + 58, 34, 118, 49, 34, 44, 34, 109, 101, 116, 97, 100, 97, 116, 97, 34, 58, 123, 34, + 114, 101, 115, 111, 117, 114, 99, 101, 86, 101, 114, 115, 105, 111, 110, 34, 58, + 34, 51, 56, 52, 53, 34, 44, 34, 99, 114, 101, 97, 116, 105, 111, 110, 84, 105, 109, + 101, 115, 116, 97, 109, 112, 34, 58, 110, 117, 108, 108, 125, 44, 34, 115, 112, + 101, 99, 34, 58, 123, 34, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 34, 58, + 110, 117, 108, 108, 125, 44, 34, 115, 116, 97, 116, 117, 115, 34, 58, 123, 125, + 125, 125, 10, + ]); + let actual_to = stream + .next() + .expect("expected an yielded entry, but none found") + .expect("parsing failed"); + let expected_event = WatchEvent::Bookmark(Pod { + metadata: Some(ObjectMeta { + resource_version: Some("3845".into()), + creation_timestamp: None, + ..ObjectMeta::default() + }), + spec: Some(PodSpec { + containers: vec![], + ..PodSpec::default() + }), + ..Pod::default() + }); + match actual_to { + WatchResponse::Ok(actual_event) => assert_eq!(actual_event, expected_event), + _ => panic!("expected an event, got something else"), + } + } + + assert!(dec.finish().is_ok()); + } } From 968e311b12893d80a1c4c2c0095367d5385e48cd Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 29 Jun 2020 19:09:00 +0300 Subject: [PATCH 110/118] Remove meaningless leading \n from the boorkmark test Signed-off-by: MOZGIII --- src/kubernetes/multi_response_decoder.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index f7c6e28f79d8f..062244ef86420 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -470,16 +470,15 @@ mod tests { { let mut stream = dec.process_next_chunk(&[ - 10, 123, 34, 116, 121, 112, 101, 34, 58, 34, 66, 79, 79, 75, 77, 65, 82, 75, 34, - 44, 34, 111, 98, 106, 101, 99, 116, 34, 58, 123, 34, 107, 105, 110, 100, 34, 58, - 34, 80, 111, 100, 34, 44, 34, 97, 112, 105, 86, 101, 114, 115, 105, 111, 110, 34, - 58, 34, 118, 49, 34, 44, 34, 109, 101, 116, 97, 100, 97, 116, 97, 34, 58, 123, 34, - 114, 101, 115, 111, 117, 114, 99, 101, 86, 101, 114, 115, 105, 111, 110, 34, 58, - 34, 51, 56, 52, 53, 34, 44, 34, 99, 114, 101, 97, 116, 105, 111, 110, 84, 105, 109, - 101, 115, 116, 97, 109, 112, 34, 58, 110, 117, 108, 108, 125, 44, 34, 115, 112, - 101, 99, 34, 58, 123, 34, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 34, 58, - 110, 117, 108, 108, 125, 44, 34, 115, 116, 97, 116, 117, 115, 34, 58, 123, 125, - 125, 125, 10, + 123, 34, 116, 121, 112, 101, 34, 58, 34, 66, 79, 79, 75, 77, 65, 82, 75, 34, 44, + 34, 111, 98, 106, 101, 99, 116, 34, 58, 123, 34, 107, 105, 110, 100, 34, 58, 34, + 80, 111, 100, 34, 44, 34, 97, 112, 105, 86, 101, 114, 115, 105, 111, 110, 34, 58, + 34, 118, 49, 34, 44, 34, 109, 101, 116, 97, 100, 97, 116, 97, 34, 58, 123, 34, 114, + 101, 115, 111, 117, 114, 99, 101, 86, 101, 114, 115, 105, 111, 110, 34, 58, 34, 51, + 56, 52, 53, 34, 44, 34, 99, 114, 101, 97, 116, 105, 111, 110, 84, 105, 109, 101, + 115, 116, 97, 109, 112, 34, 58, 110, 117, 108, 108, 125, 44, 34, 115, 112, 101, 99, + 34, 58, 123, 34, 99, 111, 110, 116, 97, 105, 110, 101, 114, 115, 34, 58, 110, 117, + 108, 108, 125, 44, 34, 115, 116, 97, 116, 117, 115, 34, 58, 123, 125, 125, 125, 10, ]); let actual_to = stream .next() From a64eae36851a4d1ba1f052f431460a492a8ea3e0 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 30 Jun 2020 02:04:34 +0300 Subject: [PATCH 111/118] Update k8s-openapi to a branch with a fix for the bookmark parsing issue Signed-off-by: MOZGIII --- Cargo.lock | 7 +++---- Cargo.toml | 2 +- src/kubernetes/mock_watcher.rs | 12 ++++++------ src/kubernetes/multi_response_decoder.rs | 17 ++++------------- src/kubernetes/reflector.rs | 18 ++++-------------- src/kubernetes/resource_version.rs | 6 ++++-- src/kubernetes/watcher.rs | 4 ++-- 7 files changed, 24 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 200ce22afd0e9..9e65f0fb81ace 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1882,11 +1882,10 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96affb18356fc998baa692c2185d198095ffdf6e6fff3bc9efb4927952f86851" +version = "0.8.0" +source = "git+https://github.com/Arnavion/k8s-openapi.git?branch=fix-70#ce5e754a4dc9873ee76a4428bee65cc7dc3115fd" dependencies = [ - "base64 0.11.0", + "base64 0.12.0", "bytes 0.5.4", "chrono", "http 0.2.1", diff --git a/Cargo.toml b/Cargo.toml index 5e76b173aa415..3799048f29986 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,7 +153,7 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti task-compat = "0.1" cidr-utils = "0.4.2" pin-project = "0.4.22" -k8s-openapi = { version = "0.7", features = ["v1_15"], optional = true } +k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true, git = "https://github.com/Arnavion/k8s-openapi.git", branch = "fix-70" } # For WASM vector-wasm = { path = "lib/vector-wasm", optional = true } diff --git a/src/kubernetes/mock_watcher.rs b/src/kubernetes/mock_watcher.rs index 04a962f12c755..7a55a47eeaa13 100644 --- a/src/kubernetes/mock_watcher.rs +++ b/src/kubernetes/mock_watcher.rs @@ -6,7 +6,7 @@ use super::watcher::{self, Watcher}; use async_stream::try_stream; use futures::channel::mpsc::{Receiver, Sender}; use futures::{future::BoxFuture, stream::BoxStream, SinkExt, StreamExt}; -use k8s_openapi::{WatchOptional, WatchResponse}; +use k8s_openapi::{Resource, WatchOptional, WatchResponse}; use serde::de::DeserializeOwned; use std::fmt; @@ -21,7 +21,7 @@ pub enum ScenarioEvent { /// invocation result. pub enum ScenarioActionInvocation where - T: DeserializeOwned, + T: DeserializeOwned + Resource, { /// Return successfully and prepare the stream with responses from the /// passed [`Receiver`]. @@ -36,7 +36,7 @@ where /// stream item request result. pub enum ScenarioActionStream where - T: DeserializeOwned, + T: DeserializeOwned + Resource, { /// Return a watch response. Ok(WatchResponse), @@ -49,7 +49,7 @@ where /// A mock watcher, useful for tests. pub struct MockWatcher where - T: DeserializeOwned, + T: DeserializeOwned + Resource, { events_tx: Sender, invocation_rx: Receiver>, @@ -57,7 +57,7 @@ where impl MockWatcher where - T: DeserializeOwned, + T: DeserializeOwned + Resource, { /// Create a new [`MockWatcher`]. pub fn new( @@ -73,7 +73,7 @@ where impl Watcher for MockWatcher where - T: DeserializeOwned + Send + Sync + Unpin + 'static, + T: DeserializeOwned + Resource + Send + Sync + Unpin + 'static, { type Object = T; diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 062244ef86420..37a1fe0643f4d 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -65,7 +65,7 @@ where mod tests { use super::*; use k8s_openapi::{ - api::core::v1::{Pod, PodSpec}, + api::core::v1::Pod, apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}, WatchResponse, }; @@ -484,18 +484,9 @@ mod tests { .next() .expect("expected an yielded entry, but none found") .expect("parsing failed"); - let expected_event = WatchEvent::Bookmark(Pod { - metadata: Some(ObjectMeta { - resource_version: Some("3845".into()), - creation_timestamp: None, - ..ObjectMeta::default() - }), - spec: Some(PodSpec { - containers: vec![], - ..PodSpec::default() - }), - ..Pod::default() - }); + let expected_event = WatchEvent::Bookmark { + resource_version: "3845".into(), + }; match actual_to { WatchResponse::Ok(actual_event) => assert_eq!(actual_event, expected_event), _ => panic!("expected an event, got something else"), diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 63ff882f07898..4b6edb840c70b 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -212,7 +212,7 @@ where trace!(message = "got an object event", event = "modified"); self.state_writer.update(object).await; } - WatchEvent::Bookmark(_object) => { + WatchEvent::Bookmark { .. } => { trace!(message = "got an object event", event = "bookmark"); // noop } @@ -297,18 +297,6 @@ mod tests { } } - // A helper to build a bookmark pod object. - // See https://github.com/kubernetes/enhancements/blob/e565a82680bb8e05836530ebd1abac723aab40e2/keps/sig-api-machinery/20190206-watch-bookmark.md - fn make_pod_bookmark(resource_version: &str) -> Pod { - Pod { - metadata: Some(ObjectMeta { - resource_version: Some(resource_version.to_owned()), - ..ObjectMeta::default() - }), - ..Pod::default() - } - } - // A type alias to add expressiveness. type StateSnapshot = Vec; @@ -400,7 +388,9 @@ mod tests { make_pod("uid2", "25"), ], Some("25".to_owned()), - ExpInvRes::Stream(vec![WatchEvent::Bookmark(make_pod_bookmark("50"))]), + ExpInvRes::Stream(vec![WatchEvent::Bookmark { + resource_version: "50".into(), + }]), ), ( vec![ diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index 387e2fd08d67b..6c4d142602a88 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -45,8 +45,10 @@ impl Candidate { let object = match event { WatchEvent::Added(object) | WatchEvent::Modified(object) - | WatchEvent::Deleted(object) - | WatchEvent::Bookmark(object) => object, + | WatchEvent::Deleted(object) => object, + WatchEvent::Bookmark { resource_version } => { + return Some(Self(resource_version.clone())) + } WatchEvent::ErrorStatus(_) | WatchEvent::ErrorOther(_) => return None, }; Self::from_object(object) diff --git a/src/kubernetes/watcher.rs b/src/kubernetes/watcher.rs index 82934b5ba5d8f..997f7d977ad31 100644 --- a/src/kubernetes/watcher.rs +++ b/src/kubernetes/watcher.rs @@ -1,14 +1,14 @@ //! Watcher abstraction. use futures::{future::BoxFuture, stream::Stream}; -use k8s_openapi::{WatchOptional, WatchResponse}; +use k8s_openapi::{Resource, WatchOptional, WatchResponse}; use serde::de::DeserializeOwned; use snafu::Snafu; /// Watch over the changes for a k8s resource. pub trait Watcher { /// The type of the watched object. - type Object: DeserializeOwned; + type Object: DeserializeOwned + Resource; /// The error type watcher invocation implementation uses internally. type InvocationError: std::error::Error + Send + 'static; From 670e46cd393e1013972b073d79a174fa833cde58 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Mon, 29 Jun 2020 19:16:09 +0300 Subject: [PATCH 112/118] Switch kubernetes_logs source to Fingerprinter::FirstLineChecksum Signed-off-by: MOZGIII --- src/sources/kubernetes_logs/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index 88a200feee2aa..729c649becc1d 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -189,19 +189,18 @@ impl Source { let annotator = PodMetadataAnnotator::new(state_reader, fields_spec); // TODO: maybe some of the parameters have to be configurable. + let max_line_bytes = 32 * 1024; // 32 KiB let file_server = FileServer { paths_provider, max_read_bytes: 2048, start_at_beginning: true, ignore_before: None, - max_line_bytes: 32 * 1024, // 32 KiB + max_line_bytes, data_dir, glob_minimum_cooldown: Duration::from_secs(10), - // Use device inodes for fingerprinting. - // - Docker recreates files on rotation: https://github.com/moby/moby/blob/75d655320e2a443185f8fa4992dc89bd2da0ea68/daemon/logger/loggerutils/logfile.go#L182-L222 - // - CRI-O recreates files on rotation: https://github.com/cri-o/cri-o/blob/ad83d2a35a30b8a336b16a0ea5f7afc6aebfb9b7/internal/oci/runtime_oci.go#L988-L1042 - // The rest should do the same. - fingerprinter: Fingerprinter::DevInode, + fingerprinter: Fingerprinter::FirstLineChecksum { + max_line_length: max_line_bytes, + }, oldest_first: false, remove_after: None, }; From 3fd9d5b5a387705103eb3be181a9284f5c69de07 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 1 Jul 2020 10:44:52 +0300 Subject: [PATCH 113/118] Revert "Handle data parsing errors as stream ends" This reverts commit c0e1695b8757a3d67cd8e0722c38be724775bd5e. Signed-off-by: MOZGIII --- src/kubernetes/stream.rs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/kubernetes/stream.rs b/src/kubernetes/stream.rs index b1f607153ef4d..8b28007370526 100644 --- a/src/kubernetes/stream.rs +++ b/src/kubernetes/stream.rs @@ -30,16 +30,6 @@ where let responses = decoder.process_next_chunk(chunk.as_ref()); emit!(internal_events::ChunkProcessed{ byte_size: chunk.len() }); for response in responses { - // Sometimes Kubernetes API starts returning `null`s in - // the object field while streaming the response. - // Handle it as if the stream has ended. - // See https://github.com/kubernetes/client-go/issues/334 - if let Err(ResponseError::Json(error)) = &response { - if error.is_data() { - warn!(message = "handling response json parsing data error as steram end", ?error); - return; - } - } let response = response.context(Parsing)?; yield response; } @@ -176,18 +166,4 @@ mod tests { assert!(out_stream.next().await.is_none()); }) } - - #[test] - fn test_sudden_null() { - test_util::trace_init(); - test_util::block_on_std(async move { - let chunks: Vec> = vec![Ok("null")]; - let sample_body = hyper_body_from_chunks(chunks); - - let out_stream = body::<_, WatchResponse>(sample_body); - pin_mut!(out_stream); - - assert!(out_stream.next().await.is_none()); - }) - } } From f5b961a5f392cc7a12ec89558a4efc4bfec74760 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 1 Jul 2020 11:07:24 +0300 Subject: [PATCH 114/118] Switch k8s-openapi git repo to our fork No actual changes there compared to upstream, just for preservation Signed-off-by: MOZGIII --- Cargo.lock | 2 +- Cargo.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e65f0fb81ace..b76ff3da094d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1883,7 +1883,7 @@ dependencies = [ [[package]] name = "k8s-openapi" version = "0.8.0" -source = "git+https://github.com/Arnavion/k8s-openapi.git?branch=fix-70#ce5e754a4dc9873ee76a4428bee65cc7dc3115fd" +source = "git+https://github.com/timberio/k8s-openapi.git?branch=fix-70#ce5e754a4dc9873ee76a4428bee65cc7dc3115fd" dependencies = [ "base64 0.12.0", "bytes 0.5.4", diff --git a/Cargo.toml b/Cargo.toml index 3799048f29986..4defd672d1d5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,7 +153,8 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti task-compat = "0.1" cidr-utils = "0.4.2" pin-project = "0.4.22" -k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true, git = "https://github.com/Arnavion/k8s-openapi.git", branch = "fix-70" } +# TODO: switch to crates.io after fix for https://github.com/Arnavion/k8s-openapi/issues/70 is released +k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true, git = "https://github.com/timberio/k8s-openapi.git", branch = "fix-70" } # For WASM vector-wasm = { path = "lib/vector-wasm", optional = true } From 8fd1fb3d2f8045df224b85a36fbae04f1d3b6b9e Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Wed, 1 Jul 2020 11:20:59 +0300 Subject: [PATCH 115/118] Use cargo patch instead of per-crate git spec for k8s-openapi Signed-off-by: MOZGIII --- Cargo.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4defd672d1d5b..0d538f3b263be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,8 +153,7 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti task-compat = "0.1" cidr-utils = "0.4.2" pin-project = "0.4.22" -# TODO: switch to crates.io after fix for https://github.com/Arnavion/k8s-openapi/issues/70 is released -k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true, git = "https://github.com/timberio/k8s-openapi.git", branch = "fix-70" } +k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true } # For WASM vector-wasm = { path = "lib/vector-wasm", optional = true } @@ -458,3 +457,7 @@ required-features = ["transforms-wasm", "transforms-lua"] [patch.'https://github.com/tower-rs/tower'] tower-layer = "0.3" + +[patch.crates-io] +# TODO: switch to crates.io after fix for https://github.com/Arnavion/k8s-openapi/issues/70 is released +k8s-openapi = { git = "https://github.com/timberio/k8s-openapi.git", branch = "fix-70" } From 2e4a7e1b3aff513023406259728a59ea22737b53 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 7 Jul 2020 16:27:27 +0300 Subject: [PATCH 116/118] Switch to the upstream of k8s-openapi Signed-off-by: MOZGIII --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/kubernetes/hash_value.rs | 2 +- src/kubernetes/multi_response_decoder.rs | 4 +-- src/kubernetes/reflector.rs | 4 +-- src/kubernetes/resource_version.rs | 8 +---- src/kubernetes/state/delayed_delete.rs | 4 +-- src/kubernetes/state/evmap.rs | 4 +-- src/kubernetes/state/instrumenting.rs | 4 +-- .../kubernetes_logs/k8s_paths_provider.rs | 33 ++++++++----------- .../kubernetes_logs/pod_metadata_annotator.rs | 3 +- 11 files changed, 28 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b76ff3da094d6..f7e58fc90a5d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1883,7 +1883,7 @@ dependencies = [ [[package]] name = "k8s-openapi" version = "0.8.0" -source = "git+https://github.com/timberio/k8s-openapi.git?branch=fix-70#ce5e754a4dc9873ee76a4428bee65cc7dc3115fd" +source = "git+https://github.com/Arnavion/k8s-openapi.git?rev=f506c01134b5280faaa1d003c7445178df7fa36e#f506c01134b5280faaa1d003c7445178df7fa36e" dependencies = [ "base64 0.12.0", "bytes 0.5.4", diff --git a/Cargo.toml b/Cargo.toml index 0d538f3b263be..ed0688ffc2058 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -460,4 +460,4 @@ tower-layer = "0.3" [patch.crates-io] # TODO: switch to crates.io after fix for https://github.com/Arnavion/k8s-openapi/issues/70 is released -k8s-openapi = { git = "https://github.com/timberio/k8s-openapi.git", branch = "fix-70" } +k8s-openapi = { git = "https://github.com/Arnavion/k8s-openapi.git", rev = "f506c01134b5280faaa1d003c7445178df7fa36e" } diff --git a/src/kubernetes/hash_value.rs b/src/kubernetes/hash_value.rs index aa3a1955f8e38..2270dbf35fa08 100644 --- a/src/kubernetes/hash_value.rs +++ b/src/kubernetes/hash_value.rs @@ -21,7 +21,7 @@ where /// Get the `uid` from the `T`'s [`Metadata`] (if any). pub fn uid(&self) -> Option<&str> { - let ObjectMeta { ref uid, .. } = self.0.metadata()?; + let ObjectMeta { ref uid, .. } = self.0.metadata(); let uid = uid.as_ref()?; Some(uid.as_str()) } diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 37a1fe0643f4d..2f3c88a660a94 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -76,10 +76,10 @@ mod tests { // A helper function to make a test object. fn make_to(uid: &str) -> TO { WatchResponse::Ok(WatchEvent::Added(Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { uid: Some(uid.to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() })) } diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 4b6edb840c70b..4261061d9c207 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -288,11 +288,11 @@ mod tests { // A helper to build a pod object for test purposes. fn make_pod(uid: &str, resource_version: &str) -> Pod { Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { uid: Some(uid.to_owned()), resource_version: Some(resource_version.to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() } } diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index 6c4d142602a88..a03d7946e75e0 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -59,13 +59,7 @@ impl Candidate { where T: Metadata, { - let metadata = match object.metadata() { - Some(val) => val, - None => { - warn!(message = "Got k8s object without metadata"); - return None; - } - }; + let metadata = object.metadata(); let new_resource_version = match metadata.resource_version { Some(ref val) => val, diff --git a/src/kubernetes/state/delayed_delete.rs b/src/kubernetes/state/delayed_delete.rs index 028fb2495d589..736a07fc3141b 100644 --- a/src/kubernetes/state/delayed_delete.rs +++ b/src/kubernetes/state/delayed_delete.rs @@ -159,11 +159,11 @@ mod tests { fn make_pod() -> Pod { Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { name: Some("pod_name".to_owned()), uid: Some("pod_uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() } } diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index 562d34966e0f9..fa881341799aa 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -133,10 +133,10 @@ mod tests { fn make_pod(uid: &str) -> Pod { Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { uid: Some(uid.to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() } } diff --git a/src/kubernetes/state/instrumenting.rs b/src/kubernetes/state/instrumenting.rs index 2401bd7838507..979d2794dd415 100644 --- a/src/kubernetes/state/instrumenting.rs +++ b/src/kubernetes/state/instrumenting.rs @@ -87,11 +87,11 @@ mod tests { fn make_pod() -> Pod { Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { name: Some("pod_name".to_owned()), uid: Some("pod_uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() } } diff --git a/src/sources/kubernetes_logs/k8s_paths_provider.rs b/src/sources/kubernetes_logs/k8s_paths_provider.rs index 09fd3b70ca53d..fecbc487f056b 100644 --- a/src/sources/kubernetes_logs/k8s_paths_provider.rs +++ b/src/sources/kubernetes_logs/k8s_paths_provider.rs @@ -53,7 +53,7 @@ impl PathsProvider for K8sPathsProvider { } fn extract_pod_logs_directory(pod: &Pod) -> Option { - let metadata = pod.metadata.as_ref()?; + let metadata = &pod.metadata; let namespace = metadata.namespace.as_ref()?; let name = metadata.name.as_ref()?; let uid = metadata.uid.as_ref()?; @@ -103,52 +103,45 @@ mod tests { (Pod::default(), None), ( Pod { - metadata: Some(ObjectMeta::default()), - ..Pod::default() - }, - None, - ), - ( - Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { namespace: Some("sandbox0-ns".to_owned()), name: Some("sandbox0-name".to_owned()), uid: Some("sandbox0-uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, Some("/var/log/pods/sandbox0-ns_sandbox0-name_sandbox0-uid"), ), ( Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { namespace: Some("sandbox0-ns".to_owned()), name: Some("sandbox0-name".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, None, ), ( Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { namespace: Some("sandbox0-ns".to_owned()), uid: Some("sandbox0-uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, None, ), ( Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { name: Some("sandbox0-name".to_owned()), uid: Some("sandbox0-uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, None, @@ -169,12 +162,12 @@ mod tests { // Pod exists and has some containers that write logs. ( Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { namespace: Some("sandbox0-ns".to_owned()), name: Some("sandbox0-name".to_owned()), uid: Some("sandbox0-uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, // Calls to the glob mock. @@ -200,12 +193,12 @@ mod tests { // Pod has proper metadata, but doesn't have log files. ( Pod { - metadata: Some(ObjectMeta { + metadata: ObjectMeta { namespace: Some("sandbox0-ns".to_owned()), name: Some("sandbox0-name".to_owned()), uid: Some("sandbox0-uid".to_owned()), ..ObjectMeta::default() - }), + }, ..Pod::default() }, vec![( diff --git a/src/sources/kubernetes_logs/pod_metadata_annotator.rs b/src/sources/kubernetes_logs/pod_metadata_annotator.rs index af842bed8f59d..b1e919e24dea6 100644 --- a/src/sources/kubernetes_logs/pod_metadata_annotator.rs +++ b/src/sources/kubernetes_logs/pod_metadata_annotator.rs @@ -57,8 +57,7 @@ impl PodMetadataAnnotator { let guard = self.pods_state_reader.get(uid)?; let entry = guard.get_one()?; let pod: &Pod = entry.as_ref(); - let metadata = pod.metadata.as_ref()?; - annotate_from_metadata(log, &self.fields_spec, &metadata); + annotate_from_metadata(log, &self.fields_spec, &pod.metadata); Some(()) } } From 4a05afc45c5a9398b25372c009ed4c48cd158bd9 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Tue, 14 Jul 2020 03:30:14 +0300 Subject: [PATCH 117/118] Fix clippy offences Signed-off-by: MOZGIII --- src/kubernetes/api_watcher.rs | 2 +- src/kubernetes/client/config/in_cluster.rs | 2 +- src/kubernetes/multi_response_decoder.rs | 2 +- src/kubernetes/reflector.rs | 4 ++-- src/kubernetes/resource_version.rs | 2 +- src/kubernetes/state/delayed_delete.rs | 4 +--- src/sources/kubernetes_logs/parser/cri.rs | 2 +- src/sources/kubernetes_logs/parser/docker.rs | 2 +- src/sources/kubernetes_logs/parser/picker.rs | 2 +- src/sources/kubernetes_logs/path_helpers.rs | 2 +- 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/kubernetes/api_watcher.rs b/src/kubernetes/api_watcher.rs index ed435055f4426..729b5d8ad2cc2 100644 --- a/src/kubernetes/api_watcher.rs +++ b/src/kubernetes/api_watcher.rs @@ -81,7 +81,7 @@ where } else { watcher::invocation::Error::other(source) }; - Err(err)?; + return Err(err); } // Stream response body. diff --git a/src/kubernetes/client/config/in_cluster.rs b/src/kubernetes/client/config/in_cluster.rs index 6ce45de2839c0..cb2b4f956eacb 100644 --- a/src/kubernetes/client/config/in_cluster.rs +++ b/src/kubernetes/client/config/in_cluster.rs @@ -76,7 +76,7 @@ pub enum Error { /// To avoid needlessly complicating the logic here, we simply implement the /// `net.JoinHostPort` as it is in Go: https://golang.org/pkg/net/#JoinHostPort fn join_host_port(host: &str, port: &str) -> String { - if host.contains(":") { + if host.contains(':') { // If IPv6 address is used, use a special notation. return format!("[{}]:{}", host, port); } diff --git a/src/kubernetes/multi_response_decoder.rs b/src/kubernetes/multi_response_decoder.rs index 2f3c88a660a94..034a9ba886491 100644 --- a/src/kubernetes/multi_response_decoder.rs +++ b/src/kubernetes/multi_response_decoder.rs @@ -4,7 +4,7 @@ use k8s_openapi::{http::StatusCode, Response, ResponseError}; /// Provides an algorithm to parse multiple [`Response`]s from multiple chunks /// of data represented as `&[u8]`. -#[derive(Debug)] +#[derive(Debug, Default)] pub struct MultiResponseDecoder { pending_data: Vec, responses_buffer: Vec>, diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 4261061d9c207..7b42596354032 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -138,8 +138,8 @@ where ) -> Result<::Stream, watcher::invocation::Error<::InvocationError>> { let watch_optional = WatchOptional { - field_selector: self.field_selector.as_ref().map(|s| s.as_str()), - label_selector: self.label_selector.as_ref().map(|s| s.as_str()), + field_selector: self.field_selector.as_deref(), + label_selector: self.label_selector.as_deref(), pretty: None, resource_version: self.resource_version.get(), timeout_seconds: Some(290), // https://github.com/kubernetes/kubernetes/issues/6513 diff --git a/src/kubernetes/resource_version.rs b/src/kubernetes/resource_version.rs index a03d7946e75e0..d611381255b51 100644 --- a/src/kubernetes/resource_version.rs +++ b/src/kubernetes/resource_version.rs @@ -4,7 +4,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, WatchEvent}; use k8s_openapi::Metadata; /// Resource version state in the context of a chain of watch requests. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct State(Option); impl State { diff --git a/src/kubernetes/state/delayed_delete.rs b/src/kubernetes/state/delayed_delete.rs index 736a07fc3141b..7dad291f6112a 100644 --- a/src/kubernetes/state/delayed_delete.rs +++ b/src/kubernetes/state/delayed_delete.rs @@ -109,9 +109,7 @@ where ::Item: Send + Sync, { fn maintenance_request(&mut self) -> Option> { - let delayed_delete_deadline = self - .next_deadline() - .map(|delayed_delete_deadline| delay_until(delayed_delete_deadline)); + let delayed_delete_deadline = self.next_deadline().map(delay_until); let downstream = self.inner.maintenance_request(); match (downstream, delayed_delete_deadline) { diff --git a/src/sources/kubernetes_logs/parser/cri.rs b/src/sources/kubernetes_logs/parser/cri.rs index 7baa8f0951e7d..3d1c402ac049b 100644 --- a/src/sources/kubernetes_logs/parser/cri.rs +++ b/src/sources/kubernetes_logs/parser/cri.rs @@ -151,6 +151,6 @@ pub mod tests { #[test] fn test_parsing() { - test_util::test_parser(|| Cri::new(), cases()); + test_util::test_parser(Cri::new, cases()); } } diff --git a/src/sources/kubernetes_logs/parser/docker.rs b/src/sources/kubernetes_logs/parser/docker.rs index 9bd01f4f2010d..0a79fb1ba02f7 100644 --- a/src/sources/kubernetes_logs/parser/docker.rs +++ b/src/sources/kubernetes_logs/parser/docker.rs @@ -176,6 +176,6 @@ pub mod tests { #[test] fn test_parsing() { - test_util::test_parser(|| Docker::new(), cases()); + test_util::test_parser(Docker::new, cases()); } } diff --git a/src/sources/kubernetes_logs/parser/picker.rs b/src/sources/kubernetes_logs/parser/picker.rs index 4fb713972560a..52ce4bdee2c4d 100644 --- a/src/sources/kubernetes_logs/parser/picker.rs +++ b/src/sources/kubernetes_logs/parser/picker.rs @@ -58,7 +58,7 @@ mod tests { #[test] fn test_parsing() { - test_util::test_parser(|| Picker::new(), cases()); + test_util::test_parser(Picker::new, cases()); } #[test] diff --git a/src/sources/kubernetes_logs/path_helpers.rs b/src/sources/kubernetes_logs/path_helpers.rs index 8acd9e50db758..2035d26a0b4f3 100644 --- a/src/sources/kubernetes_logs/path_helpers.rs +++ b/src/sources/kubernetes_logs/path_helpers.rs @@ -34,7 +34,7 @@ pub(super) fn build_pod_logs_directory( /// /// Inspired by https://github.com/kubernetes/kubernetes/blob/31305966789525fca49ec26c289e565467d1f1c4/pkg/kubelet/kuberuntime/helpers.go#L186 pub(super) fn parse_log_file_path(path: &str) -> Option> { - let mut componenets = path.rsplit("/"); + let mut componenets = path.rsplit('/'); let _log_file_name = componenets.next()?; let container_name = componenets.next()?; From 86233a7dbe46331ca967e4e9aac75a09ccfeeaa2 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Sun, 19 Jul 2020 11:37:00 +0300 Subject: [PATCH 118/118] Bump k8s-openapi and switch to crates.io version Signed-off-by: MOZGIII --- Cargo.lock | 41 +++++++++++++++++++++-------------------- Cargo.toml | 6 +----- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7e58fc90a5d1..2a4a9c3063511 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,10 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] -name = "adler" -version = "0.2.3" +name = "adler32" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" [[package]] name = "ahash" @@ -1193,9 +1193,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "flate2" -version = "1.0.16" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" +checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" dependencies = [ "cfg-if", "crc32fast", @@ -1788,9 +1788,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.1.7" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621b50c176968fd3b0bd71f821a28a0ea98db2b5aea966b2fbb8bd1b7d310328" +checksum = "2bf98296081bd2cb540acc09ef9c97f22b7e487841520350293605db1b2c7a27" dependencies = [ "ctor", "ghost", @@ -1799,9 +1799,9 @@ dependencies = [ [[package]] name = "inventory-impl" -version = "0.1.7" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f99a4111304bade76468d05beab3487c226e4fe4c4de1c4e8f006e815762db73" +checksum = "0a8e30575afe28eea36a9a39136b70b2fb6b0dd0a212a5bd1f30a498395c0274" dependencies = [ "proc-macro2 1.0.18", "quote 1.0.2", @@ -1882,8 +1882,9 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.8.0" -source = "git+https://github.com/Arnavion/k8s-openapi.git?rev=f506c01134b5280faaa1d003c7445178df7fa36e#f506c01134b5280faaa1d003c7445178df7fa36e" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f95fd36c08ce592e67400a0f1a66f432196997d5a7e9a97e8743c33d8a9312" dependencies = [ "base64 0.12.0", "bytes 0.5.4", @@ -2474,11 +2475,11 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.0" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +checksum = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625" dependencies = [ - "adler", + "adler32", ] [[package]] @@ -2834,9 +2835,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" +checksum = "9fe9037165d7023b1228bc4ae9a2fa1a2b0095eca6c2998c624723dfd01314a5" dependencies = [ "num-traits", ] @@ -4005,9 +4006,9 @@ dependencies = [ [[package]] name = "serde-value" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ "ordered-float", "serde", @@ -4926,9 +4927,9 @@ dependencies = [ [[package]] name = "tokio-uds" -version = "0.2.7" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +checksum = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" dependencies = [ "bytes 0.4.12", "futures 0.1.29", diff --git a/Cargo.toml b/Cargo.toml index ed0688ffc2058..9d12a1c9c3be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,7 +153,7 @@ pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runti task-compat = "0.1" cidr-utils = "0.4.2" pin-project = "0.4.22" -k8s-openapi = { version = "0.8", features = ["v1_15"], optional = true } +k8s-openapi = { version = "0.9", features = ["v1_15"], optional = true } # For WASM vector-wasm = { path = "lib/vector-wasm", optional = true } @@ -457,7 +457,3 @@ required-features = ["transforms-wasm", "transforms-lua"] [patch.'https://github.com/tower-rs/tower'] tower-layer = "0.3" - -[patch.crates-io] -# TODO: switch to crates.io after fix for https://github.com/Arnavion/k8s-openapi/issues/70 is released -k8s-openapi = { git = "https://github.com/Arnavion/k8s-openapi.git", rev = "f506c01134b5280faaa1d003c7445178df7fa36e" }