From 4d4e34ad40f4ed7c2d7a5d42f022dace67f27718 Mon Sep 17 00:00:00 2001 From: Will Killian Date: Wed, 22 Jul 2026 08:09:35 -0400 Subject: [PATCH 1/2] fix(pii): preserve trusted scope metadata Signed-off-by: Will Killian --- crates/pii-redaction/README.md | 7 + crates/pii-redaction/src/trajectory.rs | 45 +++++- .../tests/unit/component_tests.rs | 153 +++++++++++++++++- 3 files changed, 195 insertions(+), 10 deletions(-) diff --git a/crates/pii-redaction/README.md b/crates/pii-redaction/README.md index 911dc2ddc..b532f7df3 100644 --- a/crates/pii-redaction/README.md +++ b/crates/pii-redaction/README.md @@ -154,6 +154,13 @@ fields remain usable. This choice affects canonical event fields before subscriber fan-out; exporter-owned resource attributes are outside this boundary. +For Scope events, the preset retains direct string values for the trusted +low-cardinality classification fields `nemo_relay_scope_role`, `agent_kind`, +`hook_event_name`, `gateway_config_profile`, `gateway_mode`, `turn_source`, +`harness`, `source`, and `identity_quality`. Do not place PII or conversational +content in these fields. Arbitrary metadata and unexpected non-string values +continue through the preset's normal semantic redaction. + The preset defines its own action and therefore cannot be combined with `action`, `detector`, `pattern`, `target_paths`, or mask-specific fields. Its optional `replacement` defaults to `[REDACTED]`. diff --git a/crates/pii-redaction/src/trajectory.rs b/crates/pii-redaction/src/trajectory.rs index 05039cc47..367e8285a 100644 --- a/crates/pii-redaction/src/trajectory.rs +++ b/crates/pii-redaction/src/trajectory.rs @@ -11,6 +11,18 @@ use nemo_relay::api::event::{CategoryProfile, Event}; use nemo_relay::codec::request::AnnotatedLlmRequest; use nemo_relay::codec::response::AnnotatedLlmResponse; +const TRUSTED_SCOPE_METADATA_FIELDS: &[&str] = &[ + "nemo_relay_scope_role", + "agent_kind", + "hook_event_name", + "gateway_config_profile", + "gateway_mode", + "turn_source", + "harness", + "source", + "identity_quality", +]; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum CustomMarkPayloadPolicy { Preserve, @@ -134,9 +146,13 @@ impl TrajectorySanitizer { .data .map(|value| redact_semantic_content(value, &self.replacement, None)); } - fields.metadata = fields - .metadata - .map(|value| redact_semantic_content(value, &self.replacement, None)); + fields.metadata = fields.metadata.map(|value| { + if matches!(event, Event::Scope(_)) { + sanitize_scope_metadata(value, &self.replacement) + } else { + redact_semantic_content(value, &self.replacement, None) + } + }); fields.category_profile = fields .category_profile .and_then(|profile| sanitize_category_profile(profile, &self.replacement)); @@ -144,6 +160,29 @@ impl TrajectorySanitizer { } } +fn sanitize_scope_metadata(value: Json, replacement: &str) -> Json { + let Json::Object(values) = value else { + return redact_semantic_content(value, replacement, None); + }; + Json::Object( + values + .into_iter() + .map(|(key, value)| { + let value = if is_trusted_scope_metadata_field(&key) && value.is_string() { + value + } else { + redact_semantic_content(value, replacement, Some(&key)) + }; + (key, value) + }) + .collect(), + ) +} + +fn is_trusted_scope_metadata_field(key: &str) -> bool { + TRUSTED_SCOPE_METADATA_FIELDS.contains(&key) +} + fn is_known_content_bearing_mark(name: &str) -> bool { matches!( name, diff --git a/crates/pii-redaction/tests/unit/component_tests.rs b/crates/pii-redaction/tests/unit/component_tests.rs index 9e79a56f1..f0fd7b17b 100644 --- a/crates/pii-redaction/tests/unit/component_tests.rs +++ b/crates/pii-redaction/tests/unit/component_tests.rs @@ -537,6 +537,109 @@ fn trajectory_preset_redacts_known_marks_and_nested_scope_content() { assert_eq!(sanitized.metadata.as_ref().unwrap()["note"], "[REDACTED]"); } +#[test] +fn trajectory_preset_preserves_trusted_scope_metadata_only() { + let callback = crate::builtin::event_sanitize_callback(trajectory_backend(None, "preserve")); + let metadata = json!({ + "nemo_relay_scope_role": "turn", + "agent_kind": "codex", + "hook_event_name": "UserPromptSubmit", + "gateway_config_profile": "development", + "gateway_mode": "passthrough", + "turn_source": "user_prompt", + "harness": "codex", + "source": "hook", + "identity_quality": "native", + "private_note": "private context", + "nested": {"harness": "private nested context"} + }); + let expected_metadata = json!({ + "nemo_relay_scope_role": "turn", + "agent_kind": "codex", + "hook_event_name": "UserPromptSubmit", + "gateway_config_profile": "development", + "gateway_mode": "passthrough", + "turn_source": "user_prompt", + "harness": "codex", + "source": "hook", + "identity_quality": "native", + "private_note": "[REDACTED]", + "nested": {"harness": "[REDACTED]"} + }); + + for (scope_category, category) in [ + (ScopeCategory::Start, EventCategory::agent()), + (ScopeCategory::End, EventCategory::agent()), + (ScopeCategory::Start, EventCategory::llm()), + (ScopeCategory::End, EventCategory::llm()), + (ScopeCategory::Start, EventCategory::tool()), + (ScopeCategory::End, EventCategory::tool()), + ] { + let event = Event::Scope(ScopeEvent::new( + BaseEvent::builder().name("trusted-scope").build(), + scope_category, + Vec::new(), + category, + None, + )); + let sanitized = callback( + &event, + EventSanitizeFields { + data: None, + category_profile: None, + metadata: Some(metadata.clone()), + }, + ); + assert_eq!(sanitized.metadata, Some(expected_metadata.clone())); + } + + let malformed = Event::Scope(ScopeEvent::new( + BaseEvent::builder().name("malformed-trusted-scope").build(), + ScopeCategory::Start, + Vec::new(), + EventCategory::agent(), + None, + )); + let sanitized = callback( + &malformed, + EventSanitizeFields { + data: None, + category_profile: None, + metadata: Some(json!({ + "harness": {"private": "private context"}, + "source": 42, + "identity_quality": ["private context"] + })), + }, + ); + assert_eq!( + sanitized.metadata, + Some(json!({ + "harness": {"private": "[REDACTED]"}, + "source": 0, + "identity_quality": ["[REDACTED]"] + })) + ); + + let mark = Event::Mark(MarkEvent::new( + BaseEvent::builder().name("llm.chunk").build(), + Some(EventCategory::custom()), + Some(CategoryProfile::builder().subtype("llm.chunk").build()), + )); + let sanitized = callback( + &mark, + EventSanitizeFields { + data: None, + category_profile: mark.category_profile().cloned(), + metadata: Some(json!({"harness": "codex", "source": "hook"})), + }, + ); + assert_eq!( + sanitized.metadata, + Some(json!({"harness": "[REDACTED]", "source": "[REDACTED]"})) + ); +} + #[test] fn trajectory_custom_mark_policy_is_explicit_and_shape_preserving() { let event = Event::Mark(MarkEvent::new( @@ -1629,12 +1732,24 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { let raw_pii = "person@example.com"; let raw_context = "private trajectory context canary"; + let trusted_scope_metadata = json!({ + "nemo_relay_scope_role": "session", + "agent_kind": "hermes", + "hook_event_name": "SessionStart", + "gateway_config_profile": "development", + "gateway_mode": "passthrough", + "turn_source": "user_prompt", + "harness": "hermes", + "source": "hook", + "identity_quality": "native", + "session_owner": raw_pii + }); let agent = push_scope( PushScopeParams::builder() .name("hermes-agent") .scope_type(ScopeType::Agent) .input(json!({"prompt": raw_context, "request_id": "request-1"})) - .metadata(json!({"session_owner": raw_pii})) + .metadata(trusted_scope_metadata.clone()) .build(), ) .unwrap(); @@ -1655,6 +1770,7 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { ) .unwrap(); + crate::api::subscriber::flush_subscribers().unwrap(); atof.force_flush().unwrap(); let trajectory = atif.export().unwrap(); otel.force_flush().unwrap(); @@ -1665,12 +1781,12 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { let atif_json = serde_json::to_string(&trajectory).unwrap(); let otel_debug = format!("{:?}", otel_exporter.get_finished_spans().unwrap()); let openinference_debug = format!("{:?}", openinference_exporter.get_finished_spans().unwrap()); - for (surface, output) in [ - ("subscriber", subscriber_json), - ("ATOF", atof_json), - ("ATIF", atif_json), - ("OpenTelemetry", otel_debug), - ("OpenInference", openinference_debug), + for (surface, output, retains_scope_metadata) in [ + ("subscriber", subscriber_json, true), + ("ATOF", atof_json, true), + ("ATIF", atif_json, false), + ("OpenTelemetry", otel_debug, true), + ("OpenInference", openinference_debug, true), ] { assert!( !output.contains(raw_pii), @@ -1680,6 +1796,19 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { !output.contains(raw_context), "trajectory context leaked through {surface}: {output}" ); + if retains_scope_metadata { + for (key, value) in trusted_scope_metadata + .as_object() + .unwrap() + .iter() + .filter(|(key, _)| *key != "session_owner") + { + assert!( + output.contains(key) && output.contains(value.as_str().unwrap()), + "trusted scope metadata {key} was not retained in {surface}: {output}" + ); + } + } } let captured = captured_events_snapshot(&captured); @@ -1691,6 +1820,16 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { .unwrap(); assert_eq!(agent_start.data().unwrap()["request_id"], "request-1"); assert_eq!(agent_start.data().unwrap()["prompt"], "[REDACTED]"); + let agent_metadata = agent_start.metadata().unwrap(); + for (key, value) in trusted_scope_metadata + .as_object() + .unwrap() + .iter() + .filter(|(key, _)| *key != "session_owner") + { + assert_eq!(agent_metadata[key], *value, "{key}"); + } + assert_eq!(agent_metadata["session_owner"], "[REDACTED]"); let custom_mark = captured .iter() .find(|event| event.name() == "hermes.checkpoint") From dff9d998b1eaa04647c559b999cd5e21d1cb56f5 Mon Sep 17 00:00:00 2001 From: Will Killian Date: Wed, 22 Jul 2026 10:38:10 -0400 Subject: [PATCH 2/2] fix(pii): retain typed scope metadata Signed-off-by: Will Killian --- crates/core/src/observability/mod.rs | 9 ++++++ .../unit/observability/openinference_tests.rs | 10 +++++- .../tests/unit/observability/otel_tests.rs | 10 +++++- crates/pii-redaction/README.md | 9 ++++-- crates/pii-redaction/src/trajectory.rs | 21 +++++++++--- .../tests/unit/component_tests.rs | 32 +++++++++++++++++-- 6 files changed, 79 insertions(+), 12 deletions(-) diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index b55db2253..2edd20043 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -299,6 +299,15 @@ pub(crate) fn push_session_identity_attributes( { attributes.push(KeyValue::new("user.id", user_id.to_string())); } + if let Some(agent_kind) = metadata + .and_then(|value| value.get("agent_kind")) + .and_then(crate::json::Json::as_str) + { + attributes.push(KeyValue::new( + "nemo_relay.agent.kind", + agent_kind.to_string(), + )); + } if let Ok(stack) = crate::api::runtime::current_scope_stack().read() { attributes.push(KeyValue::new( "nemo_relay.session.instance_id", diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index 31352353c..537b43774 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -826,7 +826,11 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { .unwrap() .root_uuid() .to_string(); - let identity = json!({"session_id": "logical-session", "user_id": "alice"}); + let identity = json!({ + "session_id": "logical-session", + "user_id": "alice", + "agent_kind": "claude-code" + }); callback(&make_start_event_with_metadata( root_uuid, @@ -895,6 +899,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let root_attributes = attr_map(&root.attributes); assert_eq!(root_attributes["session.id"], "logical-session"); assert_eq!(root_attributes["user.id"], "alice"); + assert_eq!(root_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( root_attributes["nemo_relay.session.instance_id"], instance_id @@ -910,6 +915,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { assert!(!child_attributes.contains_key("session.id")); assert!(!child_attributes.contains_key("user.id")); assert!(!child_attributes.contains_key("nemo_relay.session.instance_id")); + assert!(!child_attributes.contains_key("nemo_relay.agent.kind")); assert_eq!(child_attributes["openinference.metadata.user_id"], "alice"); let second_root_attributes = attr_map(&second_root.attributes); assert_eq!(second_root_attributes["session.id"], "logical-session"); @@ -930,6 +936,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let mark_attributes = attr_map(&root.events.events[0].attributes); assert_eq!(mark_attributes["session.id"], "logical-session"); assert_eq!(mark_attributes["user.id"], "alice"); + assert_eq!(mark_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( mark_attributes["nemo_relay.session.instance_id"], root_attributes["nemo_relay.session.instance_id"] @@ -937,6 +944,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let orphan_attributes = attr_map(&orphan_mark.attributes); assert_eq!(orphan_attributes["session.id"], "logical-session"); assert_eq!(orphan_attributes["user.id"], "alice"); + assert_eq!(orphan_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( orphan_attributes["nemo_relay.session.instance_id"], root_attributes["nemo_relay.session.instance_id"] diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index cec581fca..dde05684c 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -717,7 +717,11 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { .unwrap() .root_uuid() .to_string(); - let identity = json!({"session_id": "logical-session", "user_id": "alice"}); + let identity = json!({ + "session_id": "logical-session", + "user_id": "alice", + "agent_kind": "claude-code" + }); callback(&make_start_event_with_metadata( root_uuid, @@ -786,6 +790,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let root_attributes = attr_map(&root.attributes); assert_eq!(root_attributes["session.id"], "logical-session"); assert_eq!(root_attributes["user.id"], "alice"); + assert_eq!(root_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( root_attributes["nemo_relay.session.instance_id"], instance_id @@ -801,6 +806,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { assert!(!child_attributes.contains_key("session.id")); assert!(!child_attributes.contains_key("user.id")); assert!(!child_attributes.contains_key("nemo_relay.session.instance_id")); + assert!(!child_attributes.contains_key("nemo_relay.agent.kind")); assert_eq!( child_attributes["nemo_relay.start.metadata.user_id"], "alice" @@ -824,6 +830,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let mark_attributes = attr_map(&root.events.events[0].attributes); assert_eq!(mark_attributes["session.id"], "logical-session"); assert_eq!(mark_attributes["user.id"], "alice"); + assert_eq!(mark_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( mark_attributes["nemo_relay.session.instance_id"], root_attributes["nemo_relay.session.instance_id"] @@ -831,6 +838,7 @@ fn session_identity_is_projected_on_trace_roots_and_marks_only() { let orphan_attributes = attr_map(&orphan_mark.attributes); assert_eq!(orphan_attributes["session.id"], "logical-session"); assert_eq!(orphan_attributes["user.id"], "alice"); + assert_eq!(orphan_attributes["nemo_relay.agent.kind"], "claude-code"); assert_eq!( orphan_attributes["nemo_relay.session.instance_id"], root_attributes["nemo_relay.session.instance_id"] diff --git a/crates/pii-redaction/README.md b/crates/pii-redaction/README.md index b532f7df3..1b0f820da 100644 --- a/crates/pii-redaction/README.md +++ b/crates/pii-redaction/README.md @@ -157,9 +157,12 @@ boundary. For Scope events, the preset retains direct string values for the trusted low-cardinality classification fields `nemo_relay_scope_role`, `agent_kind`, `hook_event_name`, `gateway_config_profile`, `gateway_mode`, `turn_source`, -`harness`, `source`, and `identity_quality`. Do not place PII or conversational -content in these fields. Arbitrary metadata and unexpected non-string values -continue through the preset's normal semantic redaction. +`harness`, `source`, `identity_quality`, `gateway_path`, +`llm_correlation_status`, `llm_correlation_source`, `tool_correlation_status`, +`tool_correlation_source`, `otel.status_code`, and `fidelity_source`. It also +retains the direct boolean `provider_payload_exact`. Do not place PII or +conversational content in these fields. Arbitrary metadata and unexpected value +types continue through the preset's normal semantic redaction. The preset defines its own action and therefore cannot be combined with `action`, `detector`, `pattern`, `target_paths`, or mask-specific fields. Its diff --git a/crates/pii-redaction/src/trajectory.rs b/crates/pii-redaction/src/trajectory.rs index 367e8285a..383604714 100644 --- a/crates/pii-redaction/src/trajectory.rs +++ b/crates/pii-redaction/src/trajectory.rs @@ -11,7 +11,7 @@ use nemo_relay::api::event::{CategoryProfile, Event}; use nemo_relay::codec::request::AnnotatedLlmRequest; use nemo_relay::codec::response::AnnotatedLlmResponse; -const TRUSTED_SCOPE_METADATA_FIELDS: &[&str] = &[ +const TRUSTED_STRING_SCOPE_METADATA_FIELDS: &[&str] = &[ "nemo_relay_scope_role", "agent_kind", "hook_event_name", @@ -21,8 +21,17 @@ const TRUSTED_SCOPE_METADATA_FIELDS: &[&str] = &[ "harness", "source", "identity_quality", + "gateway_path", + "llm_correlation_status", + "llm_correlation_source", + "tool_correlation_status", + "tool_correlation_source", + "otel.status_code", + "fidelity_source", ]; +const TRUSTED_BOOLEAN_SCOPE_METADATA_FIELDS: &[&str] = &["provider_payload_exact"]; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum CustomMarkPayloadPolicy { Preserve, @@ -168,7 +177,7 @@ fn sanitize_scope_metadata(value: Json, replacement: &str) -> Json { values .into_iter() .map(|(key, value)| { - let value = if is_trusted_scope_metadata_field(&key) && value.is_string() { + let value = if is_trusted_scope_metadata_value(&key, &value) { value } else { redact_semantic_content(value, replacement, Some(&key)) @@ -179,8 +188,12 @@ fn sanitize_scope_metadata(value: Json, replacement: &str) -> Json { ) } -fn is_trusted_scope_metadata_field(key: &str) -> bool { - TRUSTED_SCOPE_METADATA_FIELDS.contains(&key) +fn is_trusted_scope_metadata_value(key: &str, value: &Json) -> bool { + match value { + Json::String(_) => TRUSTED_STRING_SCOPE_METADATA_FIELDS.contains(&key), + Json::Bool(_) => TRUSTED_BOOLEAN_SCOPE_METADATA_FIELDS.contains(&key), + _ => false, + } } fn is_known_content_bearing_mark(name: &str) -> bool { diff --git a/crates/pii-redaction/tests/unit/component_tests.rs b/crates/pii-redaction/tests/unit/component_tests.rs index 554250e43..3ac54253e 100644 --- a/crates/pii-redaction/tests/unit/component_tests.rs +++ b/crates/pii-redaction/tests/unit/component_tests.rs @@ -612,6 +612,14 @@ fn trajectory_preset_preserves_trusted_scope_metadata_only() { "harness": "codex", "source": "hook", "identity_quality": "native", + "gateway_path": "responses", + "llm_correlation_status": "matched", + "llm_correlation_source": "provider", + "tool_correlation_status": "matched", + "tool_correlation_source": "provider", + "otel.status_code": "OK", + "fidelity_source": "provider", + "provider_payload_exact": true, "private_note": "private context", "nested": {"harness": "private nested context"} }); @@ -625,6 +633,14 @@ fn trajectory_preset_preserves_trusted_scope_metadata_only() { "harness": "codex", "source": "hook", "identity_quality": "native", + "gateway_path": "responses", + "llm_correlation_status": "matched", + "llm_correlation_source": "provider", + "tool_correlation_status": "matched", + "tool_correlation_source": "provider", + "otel.status_code": "OK", + "fidelity_source": "provider", + "provider_payload_exact": true, "private_note": "[REDACTED]", "nested": {"harness": "[REDACTED]"} }); @@ -670,7 +686,8 @@ fn trajectory_preset_preserves_trusted_scope_metadata_only() { metadata: Some(json!({ "harness": {"private": "private context"}, "source": 42, - "identity_quality": ["private context"] + "identity_quality": ["private context"], + "provider_payload_exact": "private context" })), }, ); @@ -679,7 +696,8 @@ fn trajectory_preset_preserves_trusted_scope_metadata_only() { Some(json!({ "harness": {"private": "[REDACTED]"}, "source": 0, - "identity_quality": ["[REDACTED]"] + "identity_quality": ["[REDACTED]"], + "provider_payload_exact": "[REDACTED]" })) ); @@ -1804,6 +1822,14 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { "harness": "hermes", "source": "hook", "identity_quality": "native", + "gateway_path": "responses", + "llm_correlation_status": "matched", + "llm_correlation_source": "provider", + "tool_correlation_status": "matched", + "tool_correlation_source": "provider", + "otel.status_code": "OK", + "fidelity_source": "provider", + "provider_payload_exact": true, "session_owner": raw_pii }); let agent = push_scope( @@ -1866,7 +1892,7 @@ fn sanitized_trajectory_content_never_reaches_subscribers_or_exporters() { .filter(|(key, _)| *key != "session_owner") { assert!( - output.contains(key) && output.contains(value.as_str().unwrap()), + output.contains(key) && output.contains(value.to_string().trim_matches('"')), "trusted scope metadata {key} was not retained in {surface}: {output}" ); }