Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions docs/config-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -292,8 +292,6 @@
"_source"
],
"source_output": "_source",
"disable_json_tables": [],
"disable_raw_tables": [],
"remove_routing_fields": true
}
},
Expand Down Expand Up @@ -1718,7 +1716,7 @@
"default": {}
},
"capture_json": {
"description": "Store complete original Kafka message as JSON before transformation",
"description": "Not read: `capture_mode` decides whether `_json` is written.",
"type": "boolean",
"default": true
},
Expand Down Expand Up @@ -1765,22 +1763,6 @@
"type": "string",
"default": "_source"
},
"disable_json_tables": {
"description": "Tables where _json capture is disabled (e.g., [\"dfe.metrics\"])",
"type": "array",
"items": {
"type": "string"
},
"default": []
},
"disable_raw_tables": {
"description": "Tables where _raw capture is disabled (e.g., [\"dfe.metrics\"])",
"type": "array",
"items": {
"type": "string"
},
"default": []
},
"remove_routing_fields": {
"description": "Remove routing fields from output after extraction",
"type": "boolean",
Expand Down
16 changes: 1 addition & 15 deletions docs/config-schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,6 @@ properties:
source_fields:
- _source
source_output: _source
disable_json_tables: []
disable_raw_tables: []
remove_routing_fields: true
field_mapping:
description: Field mapping overrides. **Restart required.**
Expand Down Expand Up @@ -1553,7 +1551,7 @@ $defs:
$ref: "#/$defs/CaptureMode"
default: {}
capture_json:
description: Store complete original Kafka message as JSON before transformation
description: "Not read: `capture_mode` decides whether `_json` is written."
type: boolean
default: true
json_output:
Expand Down Expand Up @@ -1592,18 +1590,6 @@ $defs:
description: Output field name for _source
type: string
default: _source
disable_json_tables:
description: Tables where _json capture is disabled (e.g., ["dfe.metrics"])
type: array
items:
type: string
default: []
disable_raw_tables:
description: Tables where _raw capture is disabled (e.g., ["dfe.metrics"])
type: array
items:
type: string
default: []
remove_routing_fields:
description: Remove routing fields from output after extraction
type: boolean
Expand Down
3 changes: 1 addition & 2 deletions docs/transport/common-header.md
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,7 @@ metadata:
```

`capture_mode` controls `_json` and `_raw` population -- see the Capture Modes
section below. Legacy booleans `capture_json` / `capture_raw` are still accepted
for backwards compatibility and mapped onto `capture_mode`.
section below.

### `_tags`

Expand Down
8 changes: 1 addition & 7 deletions src/config/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1135,8 +1135,6 @@ mod tests {
assert!(config.capture_raw);
assert_eq!(config.raw_output, "_raw");
assert_eq!(config.raw_source_fields, [] as [std::string::String; 0]);
assert_eq!(config.disable_json_tables, [] as [std::string::String; 0]);
assert_eq!(config.disable_raw_tables, [] as [std::string::String; 0]);
}

#[test]
Expand All @@ -1158,9 +1156,7 @@ mod tests {
"json_output": "_custom_json",
"capture_raw": false,
"raw_output": "_custom_raw",
"raw_source_fields": ["original_log", "raw_message"],
"disable_json_tables": ["common.metrics"],
"disable_raw_tables": ["common.health"]
"raw_source_fields": ["original_log", "raw_message"]
}"#;
let config: MetadataConfig = serde_json::from_str(json_str).unwrap();
assert!(!config.capture_json);
Expand All @@ -1171,8 +1167,6 @@ mod tests {
config.raw_source_fields,
vec!["original_log", "raw_message"]
);
assert_eq!(config.disable_json_tables, vec!["common.metrics"]);
assert_eq!(config.disable_raw_tables, vec!["common.health"]);
}

// ========================================================================
Expand Down
23 changes: 5 additions & 18 deletions src/config/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1032,15 +1032,15 @@ pub struct MetadataConfig {
#[serde(default)]
pub table_capture_modes: HashMap<String, CaptureMode>,

// _json capture (Common Header v2) — DEPRECATED, use capture_mode
/// Store complete original Kafka message as JSON before transformation
// _json capture (Common Header v2)
/// Not read: `capture_mode` decides whether `_json` is written.
#[serde(alias = "capture_logjson", default = "default_true")]
pub capture_json: bool,
/// Output field name for _json
#[serde(alias = "logjson_output")]
pub json_output: String,

// _raw field injection (Common Header v2) — DEPRECATED, use capture_mode
// _raw field injection (Common Header v2)
// Implements @renamed: first(source_fields...) → raw_output
// Silent no-op if destination already present in data
/// Enable _raw field injection from source (zero-copy rename)
Expand All @@ -1061,12 +1061,6 @@ pub struct MetadataConfig {
/// Output field name for _source
pub source_output: String,

// Per-table capture overrides — DEPRECATED, use table_capture_modes
/// Tables where _json capture is disabled (e.g., ["dfe.metrics"])
pub disable_json_tables: Vec<String>,
/// Tables where _raw capture is disabled (e.g., ["dfe.metrics"])
pub disable_raw_tables: Vec<String>,

// Routing field removal (Common Header v2)
/// Remove routing fields from output after extraction
pub remove_routing_fields: bool,
Expand Down Expand Up @@ -1095,11 +1089,11 @@ impl Default for MetadataConfig {
capture_mode: CaptureMode::Full,
table_capture_modes: HashMap::new(),

// _json capture defaults (deprecated — use capture_mode)
// _json capture defaults
capture_json: true,
json_output: "_json".to_string(),

// _raw capture defaults (deprecated — use capture_mode)
// _raw capture defaults
capture_raw: true,
raw_source_fields: Vec::new(),
raw_output: "_raw".to_string(),
Expand All @@ -1109,10 +1103,6 @@ impl Default for MetadataConfig {
source_fields: vec!["_source".to_string()],
source_output: "_source".to_string(),

// Per-table overrides (deprecated — use table_capture_modes)
disable_json_tables: vec![],
disable_raw_tables: vec![],

// Routing field removal defaults
remove_routing_fields: true,
}
Expand Down Expand Up @@ -1302,9 +1292,6 @@ pub enum CaptureMode {
/// 1. DDL: `@capture_mode: raw_only` in table COMMENT
/// 2. Config: `table_capture_modes` map
/// 3. Config: global `capture_mode`
///
/// Legacy fields (`disable_json_tables`, `@no_capture_json`) are deprecated
/// but still supported for one release cycle.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct TableCaptureConfig {
pub mode: CaptureMode,
Expand Down
Loading
Loading