diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15f47a67dc7a4..21fb445124e9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - kube: [1.14.10, 1.17.2] + kube: [1.13.12, 1.14.10, 1.17.2] steps: - name: Load vector uses: actions/download-artifact@v1 diff --git a/config/kubernetes/vector-daemonset.yaml b/config/kubernetes/vector-daemonset.yaml index 1659adfbe3691..148e5f4e602a8 100644 --- a/config/kubernetes/vector-daemonset.yaml +++ b/config/kubernetes/vector-daemonset.yaml @@ -115,6 +115,10 @@ spec: mountPath: /etc/vector readOnly: true env: + - name: VECTOR_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid - name: VECTOR_NODE_NAME valueFrom: fieldRef: diff --git a/src/sources/kubernetes/file_source_builder.rs b/src/sources/kubernetes/file_source_builder.rs index 09888f7e79963..e022d14fbdafa 100644 --- a/src/sources/kubernetes/file_source_builder.rs +++ b/src/sources/kubernetes/file_source_builder.rs @@ -28,6 +28,7 @@ impl<'a> FileSourceBuilder<'a> { kube_name: &str, globals: &GlobalOptions, shutdown: ShutdownSignal, + vector_pod_uid: &str, ) -> crate::Result<(mpsc::Receiver, Source)> { self.file_config.include.extend( Self::file_source_include(self.config)? @@ -35,7 +36,7 @@ impl<'a> FileSourceBuilder<'a> { .map(Into::into), ); self.file_config.exclude.extend( - Self::file_source_exclude(self.config) + Self::file_source_exclude(self.config, vector_pod_uid) .into_iter() .map(Into::into), ); @@ -251,7 +252,7 @@ impl<'a> FileSourceBuilder<'a> { /// with include, so exclude isn't necessary. /// b) if user has included "kube-system" or "vector*", then that is a sign that user wants /// to log it so excluding it is not valid. - fn file_source_exclude(config: &KubernetesConfig) -> Vec { + fn file_source_exclude(config: &KubernetesConfig, vector_pod_uid: &str) -> Vec { // True if there is no includes let no_include = config.include_container_names.is_empty() && config.include_namespaces.is_empty() @@ -266,11 +267,11 @@ impl<'a> FileSourceBuilder<'a> { // This is correct, but on best effort basis filtering out of logs from kuberentes system components. // More specificly, it will work for all Kubernetes 1.14 and higher, and for some bellow that. exclude.push((LOG_DIRECTORY.to_owned() + r"kube-system_*").into()); - - // NOTE: for now exclude images with name vector, it's a rough solution, but necessary for now - exclude.push((LOG_DIRECTORY.to_owned() + r"*/vector*").into()); } + // Always exclude vector + exclude.push((LOG_DIRECTORY.to_owned() + &format!("*{}/*", vector_pod_uid)).into()); + exclude } } diff --git a/src/sources/kubernetes/mod.rs b/src/sources/kubernetes/mod.rs index 1493383b1486c..143782694e4c6 100644 --- a/src/sources/kubernetes/mod.rs +++ b/src/sources/kubernetes/mod.rs @@ -21,6 +21,7 @@ use futures01::{sync::mpsc, Future, Sink, Stream}; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use snafu::Snafu; +use std::env::{self, VarError}; use string_cache::DefaultAtom as Atom; // ?NOTE @@ -33,6 +34,9 @@ use string_cache::DefaultAtom as Atom; /// Location in which by Kubernetes CRI, container runtimes are to store logs. const LOG_DIRECTORY: &str = r"/var/log/pods/"; +/// Enviorment variable through which we are receiving uid of this vector's pod. +const VECTOR_POD_UID_ENV: &str = "VECTOR_POD_UID"; + lazy_static! { pub static ref POD_UID: Atom = Atom::from("object_uid"); } @@ -43,6 +47,12 @@ enum BuildError { UidToLarge { uid: String }, #[snafu(display("UID contains illegal characters: {:?}", uid))] IllegalCharacterInUid { uid: String }, + #[snafu(display( + "Enviorment variable {}, that must be defined with this Vector's Pod's UID, is {:?}", + env, + error + ))] + PodUid { env: &'static str, error: VarError }, } #[derive(Deserialize, Serialize, Debug, Clone, Default)] @@ -71,8 +81,16 @@ impl SourceConfig for KubernetesConfig { let now = TimeFilter::new(); - let (file_recv, file_source) = - file_source_builder::FileSourceBuilder::new(self).build(name, globals, shutdown)?; + let vector_pod_uid = env::var(VECTOR_POD_UID_ENV).map_err(|error| BuildError::PodUid { + env: VECTOR_POD_UID_ENV, + error, + })?; + let (file_recv, file_source) = file_source_builder::FileSourceBuilder::new(self).build( + name, + globals, + shutdown, + &vector_pod_uid, + )?; let mut transform_file = transform_file()?; let mut transform_pod_uid = transform_pod_uid()?; diff --git a/src/sources/kubernetes/test.rs b/src/sources/kubernetes/test.rs index cc627affa82b4..3cdd4d5d6889b 100644 --- a/src/sources/kubernetes/test.rs +++ b/src/sources/kubernetes/test.rs @@ -138,6 +138,10 @@ spec: - name: tmp mountPath: /tmp/vector/ env: + - name: VECTOR_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid - name: VECTOR_NODE_NAME valueFrom: fieldRef: @@ -616,7 +620,7 @@ fn kube_multi_log() { #[test] fn kube_object_uid() { - let namespace = "kube-object-uid".to_owned(); //format!("object-uid-{}", Uuid::new_v4()); + let namespace = format!("object-uid-{}", Uuid::new_v4()); let message = random_string(300); let user_namespace = user_namespace(&namespace);