diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index f671e4cd177e6..8ed91acfdb3b5 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -191,20 +191,56 @@ impl Source { let annotator = PodMetadataAnnotator::new(state_reader, fields_spec); // TODO: maybe some of the parameters have to be configurable. + + // The 16KB is the maximum size of the payload at single line for both + // docker and CRI log formats. + // We take a double of that to account for metadata and padding, and to + // have a power of two rounding. Line splitting is countered at the + // parsers, see the `partial_events_merger` logic. let max_line_bytes = 32 * 1024; // 32 KiB let file_server = FileServer { + // Use our special paths provider. paths_provider, + // This is the default value for the read buffer size. max_read_bytes: 2048, - start_at_beginning: true, + // We want to use checkpoining mechanism, and resume from where we + // left off. + start_at_beginning: false, + // We're now aware of the use cases that would require specifying + // the starting point in time since when we should collect the logs, + // so we just disable it. If users ask, we can expose it. There may + // be other, more sound ways for users considering the use of this + // option to solvce their use case, so take consideration. ignore_before: None, + // Max line length to expect during regular log reads, see the + // explanation above. max_line_bytes, + // The directory where to keep the checkpoints. data_dir, + // This value specifies not exactly the globbing, but interval + // between the polling the files to watch from the `paths_provider`. + // This is quite efficient, yet might still create some load of the + // file system, so this call is 10 times larger than the default for + // the files. glob_minimum_cooldown: Duration::from_secs(10), + // The shape of the log files is well-known in the Kubernetes + // environment, so we pick the a specially crafted fingerprinter + // for the log files. fingerprinter: Fingerprinter::FirstLineChecksum { + // Max line length to expect during fingerprinting, see the + // explanation above. max_line_length: max_line_bytes, }, + // We expect the files distribution to not be a concern because of + // the way we pick files for gathering: for each container, only the + // last log file is currently picked. Thus there's no need for + // ordering, as each logical log stream is guaranteed to start with + // just one file, makis it impossible to interleave with other + // relevant log lines in the absense of such relevant log lines. oldest_first: false, + // We do not remove the log files, `kubelet` is responsible for it. remove_after: None, + // The standard emitter. emitter: FileSourceInternalEventsEmitter, };