docs(transform-vector): trim stale text and config comments

.cargo/config.toml

@@ -1,13 +1,7 @@
 # Per-target CPU optimisations.
 #
-# x86_64 takes AVX2, BMI1/2 and FMA (Haswell and later), which is the floor the
-# rest of the DFE Rust fleet already ships. aarch64 stays on the generic model,
-# where NEON is baseline.
-#
-# These belong under [target.<triple>], never [build]. The ARC native runner
-# sets CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS, which counts as a target
-# entry, and cargo reads build.rustflags only when no target entry exists -- so
-# flags placed under [build] would be dropped in CI without a word.
+# Under [target.<triple>], never [build]: cargo drops build.rustflags when a
+# CARGO_TARGET_<triple>_RUSTFLAGS is set, as CI runners do.
 
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=x86-64-v3", "-C", "link-arg=-fuse-ld=mold"]

.gitleaks.toml

@@ -1,39 +1,14 @@
-# Project-specific gitleaks allowlist
-# Extends the CI base configuration (ci/.gitleaks.toml)
-#
-# The CI runs gitleaks to detect secrets in your code. This file lets you
-# exclude false positives specific to your project.
-#
-# Documentation: https://github.com/gitleaks/gitleaks#configuration
+# Project gitleaks allowlist: https://github.com/gitleaks/gitleaks#configuration
 
 [extend]
 useDefault = true
 
 [allowlist]
 description = "Project-specific false positive allowlist"
 
-# Paths to exclude from scanning (regex patterns)
 paths = [
-    # Git submodules (have their own scanning)
     '''(^|/)ci/''',
     '''(^|/)ai/''',
-    # The bundled filebeat pipeline is ported public Vector templates, not
-    # credentials: the Meraki event subtype '8021x_eap_failure' matches a key
-    # rule. Allowlisted the same way in dfe-transform-vrl, where it is maintained.
+    # The Meraki event subtype '8021x_eap_failure' matches a key rule.
     '''pipelines/filebeat/filebeat\.yaml''',
-    # Uncomment and customise as needed
-    # '''tests/.*\.py''',
-    # '''tests/fixtures/.*''',
-    # '''.env.example''',
 ]
-
-# Regex patterns for known false positives
-# regexes = [
-#     '''YOUR_.*_TOKEN''',
-#     '''your-.*-token''',
-#     '''test-token''',
-#     '''example\.com''',
-# ]
-
-# Stopwords - findings containing these are likely false positives
-# stopwords = ["example", "sample", "test", "mock", "fake", "dummy", "placeholder"]

.hyperi-ci.yaml

@@ -6,21 +6,13 @@
 # Copyright: (c) 2026 HYPERI PTY LIMITED
 classification: product
 
-# Project language (auto-detected if not specified)
 language: rust
 
 # Quality checks
 quality:
   enabled: true
   # Advisory ignores fed to cargo-audit and osv-scanner.
   ignore:
-    # GHSA-w9wp-h8wv-79jx / CVE-2026-48504: opentelemetry_sdk 0.31.0 reaches the
-    # lock only through scalo. The flaw is in BaggagePropagator's header parsing,
-    # and neither this repo nor scalo registers a Baggage or other text map
-    # propagator, so it is never called. Recorded as not reachable in the DFE
-    # threat model (dfe-infra docs/THREAT-MODEL.md, accepted risks). scalo holds
-    # 0.31 because metrics-exporter-opentelemetry 0.2.1 requires opentelemetry
-    # ^0.31; tracked in hyperi-io/scalo-rs#281.
     - tool: osv-scanner
       id: GHSA-w9wp-h8wv-79jx
       reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator registered in dfe-transform-vector or scalo. Not reachable per dfe-infra docs/THREAT-MODEL.md accepted risks. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281."
@@ -30,9 +22,7 @@ test:
   enabled: true
   coverage: false
   rust:
-    # The metrics tests need process-per-test isolation: a global recorder
-    # installs once per process, so under cargo test only the first test to
-    # run gets a live registry. Fail rather than silently degrade to it.
+    # The metrics tests need a process each: the global recorder installs once per process.
     nextest: true
   # A release runs the full tier: the Vector validator cases and the opt-in
   # Kafka pipelines. tests/TESTING.md says which tests are which.
@@ -52,15 +42,7 @@ build:
     targets:
       - x86_64-unknown-linux-gnu
       - aarch64-unknown-linux-gnu
-    # Tier 2 release-track optimisation. hyperi-ci applies jemalloc + fat LTO
-    # on the release channel by itself; this section opts in to PGO + BOLT on
-    # top, and both are gated on that channel (optimize.py:125,129).
-    #
-    # What it covers: the supervisor's own binary -- the direct-transport
-    # bridge, the subprocess watch and the Vector metrics merge. Vector's
-    # per-record CPU on the bus transport is inside the upstream binary the
-    # image downloads rather than compiles (scripts/fetch-vector.sh), so no
-    # PGO or BOLT we run here reaches it.
+    # PGO and BOLT reach the supervisor's own binary only, not the Vector binary the image downloads.
     optimize:
       pgo:
         enabled: true
@@ -71,7 +53,7 @@ build:
         # the compiler toward startup paths and turns PGO NEGATIVE.
         duration_secs: 300
       bolt:
-        enabled: true  # Linux only; skipped on non-Linux targets
+        enabled: true
 
 # Release (where the artefacts go). The thin chart is assembled from the
 # emitted contract and ships beside the image.
@@ -88,7 +70,4 @@ release:
     platforms:
       - linux/amd64
       - linux/arm64
-    # Vector binary install + data dirs are spliced into the Dockerfile
-    # by `src/deployment.rs::emit_dockerfile()` (consumer-side post-process
-    # over scalo's base output). When hyperi-ci's overlay framework
-    # lands, this moves to `release.container.overlays:` here.
+    # `src/deployment.rs::emit_dockerfile()` splices the Vector install into the Dockerfile.

Cargo.toml

@@ -25,10 +25,7 @@ path = "src/lib.rs"
 name = "dfe-transform-vector"
 path = "src/main.rs"
 
-# PGO workload driver -- built only with `--features pgo-driver`, by
-# scripts/pgo-workload.sh during a release-channel Tier 2 build. hyperi-ci
-# skips feature-gated bins when it detects and packages binaries, so this one
-# never reaches the publish payload.
+# PGO workload driver (scripts/pgo-workload.sh), built only with `--features pgo-driver`.
 [[bin]]
 name = "pgo-driver"
 path = "src/bin/pgo_driver.rs"
@@ -39,35 +36,9 @@ required-features = ["pgo-driver"]
 tokio = { version = ">=1.54.1, <2", features = ["full"] }
 tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] }
 
-# Shared library (CLI framework, deployment artefact generation, logging)
-#
-# `expression` compiles the CEL tiers of scalo's transport filters: without it
-# a tier 2 or tier 3 filter in config is refused at load.
-#
-# `health` is REQUIRED for a truthful `/readyz`: the metrics listener only
-# consults `HealthRegistry` under that feature, and its other readiness hook
-# is snapshotted before the app can set one.
-#
-# `transport-grpc-vector-compat` carries the direct transport: the supervisor
-# speaks scalo's `Transport/Push` to the rest of DFE and Vector's own
-# `vector.Vector/PushEvents` to the subprocess, and scalo has both halves of the
-# Vector protocol (`VectorCompatClient`, `VectorCompatService`). `lifecycle`
-# carries the idle gate a transform with no source topics reports through.
-# `worker-batch` carries the `BatchEngine` pipeline the bridge runs each leg on,
-# which holds a push until its records are delivered.
-#
-# `transport-kafka` carries `KafkaConfig`, the security floor `Config::validate`
-# holds the generated Kafka source and sink to, and `MESSAGE_MAX_BYTES`, the
-# record ceiling the generated sink takes. Only its config is used: the image
-# ships no librdkafka, so nothing here may build an rdkafka client.
-#
-# `scaling` carries the `ScalingPressure` the runtime serves at
-# `/scaling/pressure` and writes into the `scaling_pressure` gauge; the
-# supervisor drives its circuit gate directly, so it is named here as well as
-# arriving through `cli-service`.
-#
-# The floor moves with `release.helm.library` in .hyperi-ci.yaml: the library
-# renders only the contract version its scalo release writes.
+# `health` is REQUIRED for a truthful `/readyz`. `transport-kafka` is for its
+# config only: the image ships no librdkafka, so nothing may build an rdkafka
+# client. The floor moves with `release.helm.library` in .hyperi-ci.yaml.
 scalo = { version = ">=2.14.29, <3", features = [
     "cli-service",
     "config-reload",
@@ -101,8 +72,7 @@ bytes = ">=1.12.1, <2"
 serde = { version = ">=1.0.229, <2", features = ["derive"] }
 serde_json = ">=1.0.154, <2"
 
-# JSON Schema derivation for the reflectable config-schema capability
-# (scalo-rs#6). Same version range as scalo so cargo unifies to one schemars.
+# Config-schema derivation. Same range as scalo so cargo unifies to one schemars.
 schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order"] }
 
 # Error handling
@@ -116,37 +86,26 @@ tracing-subscriber = { version = ">=0.3.23, <0.4", features = ["env-filter", "js
 # Metrics
 metrics = ">=0.24.6, <0.25"
 
-# Note: Vector metrics proxy uses raw TcpStream (no HTTP client crate needed).
-# /metrics, /livez and /readyz are served by scalo's MetricsManager listener.
-
 # Unix signals (sending SIGTERM/SIGHUP to child process), and renameat2 to
 # swap a reload's staged config in whole
 nix = { version = ">=0.31.3, <0.32", features = ["signal", "process", "fs"] }
 
-# Allocator (Tier 1 release-track optimisation; CI opts in per channel)
+# Allocator
 tikv-jemallocator = { version = ">=0.7.0, <0.8", optional = true }
 prometheus-parse = ">=0.2.5, <0.3"
 
 [features]
-# CI (hyperi-ci) opts in via --features jemalloc per release channel.
-# Local cargo build uses the system allocator.
 default = []
 jemalloc = ["dep:tikv-jemallocator"]
-# The PGO workload driver bin. Needs no dependency the supervisor does not
-# already carry, so a production build is unaffected either way.
 pgo-driver = []
 
 [dev-dependencies]
 tempfile = ">=3.27.0, <4"
 tokio-test = ">=0.4.6, <0.5"
-# The WS21 filebeat corpus (tests/common/filebeat.rs) is read in memory from
-# dfe-transform-vrl's committed tar.gz; chrono compares transformed timestamps
-# against the elastic golden outputs.
+# Reads dfe-transform-vrl's filebeat corpus tar.gz in memory.
 flate2 = ">=1.1.10, <2"
 tar = ">=0.4.46, <0.5"
 chrono = ">=0.4.45, <0.5"
-# transport-kafka-test-support is the run's shared broker container the Kafka
-# tests use.
 scalo = { version = ">=2.14.29, <3", features = [
     "transport-kafka",
     "transport-kafka-test-support",
@@ -185,22 +144,18 @@ dbg_macro = "deny"
 module_name_repetitions = "allow"
 # #[must_use] goes where dropping the value is a bug, not on every pure function.
 must_use_candidate = "allow"
-# The lints below fire in tests/ (54 sites) and one in main.rs, and the rest of src is clean.
-# 6 doc comments, 5 map then unwrap_or chains, 2 let-else candidates.
+# The lints below fire in tests/.
 doc_markdown = "allow"
 map_unwrap_or = "allow"
 manual_let_else = "allow"
-# 10 raw strings, 8 closures, 8 bare #[ignore] attributes.
 needless_raw_string_hashes = "allow"
 redundant_closure_for_method_calls = "allow"
 ignore_without_reason = "allow"
-# 2 float compares, 2 file extension compares, 2 similar names, 2 Duration subtractions, 2 empty asserts.
 float_cmp = "allow"
 case_sensitive_file_extension_comparisons = "allow"
 similar_names = "allow"
 unchecked_time_subtraction = "allow"
 assert_is_empty = "allow"
-# 1 single-arm match, 1 filter_map then next, 1 push_str of a format!, and 2 functions over 150 lines (the startup sequence in main.rs, one e2e test).
 single_match_else = "allow"
 filter_map_next = "allow"
 format_push_string = "allow"

clippy.toml

@@ -6,53 +6,13 @@
 # License:   BUSL-1.1
 # Copyright: (c) 2026 HYPERI PTY LIMITED
 #
-# Clippy configuration file. Lint levels are set in Cargo.toml [lints.clippy].
-# This file configures thresholds and behaviour.
+# Lint levels are in Cargo.toml [lints.clippy].
 
-# -----------------------------------------------------------------------------
-# Complexity Thresholds
-# -----------------------------------------------------------------------------
-
-# Maximum number of function arguments before warning
 too-many-arguments-threshold = 7
-
-# Cognitive complexity threshold (cyclomatic complexity)
 cognitive-complexity-threshold = 25
-
-# Type complexity threshold (nested generics)
 type-complexity-threshold = 350
-
-# Maximum lines in a function before warning
 too-many-lines-threshold = 150
 
 # unwrap_used / expect_used stay denied in shipped code, not in #[test] or #[cfg(test)]
 allow-unwrap-in-tests = true
 allow-expect-in-tests = true
-
-# -----------------------------------------------------------------------------
-# Style Preferences
-# -----------------------------------------------------------------------------
-
-# MSRV (Minimum Supported Rust Version) - clippy uses this for suggestions
-# msrv = "1.85"
-
-# -----------------------------------------------------------------------------
-# Documentation
-# -----------------------------------------------------------------------------
-# Clippy lint levels should be configured in Cargo.toml:
-#
-# [lints.clippy]
-# pedantic = "warn"
-# nursery = "warn"
-# unwrap_used = "deny"
-# expect_used = "deny"
-# panic = "deny"
-# dbg_macro = "deny"          # Deny in CI, allow locally
-# todo = "warn"
-# unimplemented = "warn"
-#
-# # Allow common pedantic false positives
-# module_name_repetitions = "allow"
-# must_use_candidate = "allow"
-# missing_errors_doc = "allow"
-# missing_panics_doc = "allow"

deny.toml

@@ -31,8 +31,6 @@ allow = [
 multiple-versions = "warn"
 wildcards = "deny"
 skip = [
-    # prometheus-parse 0.2.5 (latest, Dec 2023) pins itertools ^0.12; everything
-    # else resolves to 0.14 through prost-derive. Nothing upstream to move to.
     { crate = "itertools@0.12.1", reason = "prometheus-parse 0.2.5 pins ^0.12" },
 ]
 

docs/internals.md

@@ -474,8 +474,7 @@ vector:
 | `disabled` | Skip check entirely. |
 
 Version check failures are logged at startup. No runtime version metrics are
-currently emitted — this is deferred until the scalo `version-check` feature
-is adopted.
+emitted.
 
 ### 7.2 Version Update Strategy
 

docs/librdkafka.md

@@ -46,8 +46,8 @@ after the whole merge, so `librdkafka_options` cannot reach them:
 | `sasl.mechanism` | `{source,sink}.sasl.mechanism` | SASL is on |
 | `enable.ssl.certificate.verification` | `{source,sink}.tls.skip_verify` | TLS is on and verification is skipped |
 
-Setting one of these in `librdkafka_options` used to be discarded in silence.
-`config-check` now refuses the config and names the big dial that owns the key.
+`config-check` refuses a config that sets one of these in `librdkafka_options`,
+and names the big dial that owns the key.
 The conditions above are part of the rule: with SASL off nothing derives
 `sasl.mechanism`, so setting it by hand there is still allowed.
 
@@ -201,7 +201,7 @@ baseline).
 
 | Setting | Side | Value | Reason |
 |---------|------|-------|--------|
-| `queue.buffering.max.kbytes` | producer | `262144` (256 MiB) | Pods typically have 2-4 GiB memory. Default 1 GiB producer queue is too large. |
+| `queue.buffering.max.kbytes` | producer | `262144` (256 MiB) | The default 1 GiB producer queue is too large for a transform-vector pod. |
 | `enable.auto.commit` | consumer | `true` | Vector's kafka source stores offsets on delivery and leaves the periodic flush to librdkafka's commit timer, which the baseline's `false` never arms. Vector still commits on rebalance and clean shutdown, so the baseline stalled reported lag between those points rather than losing offsets. Lag is what KEDA scales on here. |
 | `message.max.bytes` | producer | `16777216` (16 MiB), scalo's `MESSAGE_MAX_BYTES` | The record ceiling every other stage's producer and every topic scalo creates carry; librdkafka's own `1000000` refuses a larger record. Written only where neither layer 1 nor `sink.librdkafka_options` sets the key, so either of those still decides it. The size cap in front of the sink reads the same value. |
 

pipelines/filebeat/README.md

@@ -16,7 +16,7 @@ dfe-transform-vector: the same VRL dfe-transform-vrl runs, wrapped in a Vector
 this directory and the pipeline runs -- there is nothing to author.
 
 > **INTERIM.** Elastic compatibility is being replaced by
-> `dfe-transform-elastic` (Rust-native, currently in beta). New integrations
+> `dfe-transform-elastic` (Rust-native). New integrations
 > should NOT build on this pipeline.
 
 ## Using it

pipelines/filebeat/filebeat.yaml

@@ -58,8 +58,8 @@ transforms:
       #
       # INTERIM: this pipeline keeps DFE 2.1 filebeat feeds working on
       # dfe-transform-vrl. Elastic compatibility is being replaced by
-      # dfe-transform-elastic (Rust-native, currently in beta); new integrations
-      # should NOT build on this file.
+      # dfe-transform-elastic (Rust-native), and new integrations should NOT
+      # build on this file.
       #
       # Deliberate departures from the 2.1 templates:
       #   - geoip enrichment is NOT ported: dfe-loader owns geoip enrichment in

rustfmt.toml

@@ -6,28 +6,11 @@
 # License:   BUSL-1.1
 # Copyright: (c) 2026 HYPERI PTY LIMITED
 
-# Rust edition for parsing
 edition = "2024"
-
-# Line width (100 is common for modern displays)
 max_width = 100
-
-# Indentation
 tab_spaces = 4
 hard_tabs = false
-
-# Use default heuristics for formatting choices
 use_small_heuristics = "Default"
-
-# Reorder imports alphabetically
 reorder_imports = true
-
-# Use field init shorthand where possible: Foo { x } instead of Foo { x: x }
 use_field_init_shorthand = true
-
-# Use try! shorthand: ? instead of try!()
 use_try_shorthand = true
-
-# Nightly-only (uncomment when stable rustfmt supports these)
-# imports_granularity = "Module"
-# group_imports = "StdExternalCrate"