diff --git a/CHANGELOG.md b/CHANGELOG.md index b76cd284702..16e02d7e7f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Features**: +- Transform deprecated `gen_ai.request.messages`, `gen_ai.response.text`, and `gen_ai.response.tool_calls` attributes into their normalized replacements (`gen_ai.input.messages` and `gen_ai.output.messages`). ([#6201](https://github.com/getsentry/relay/pull/6201)) - Infer span descriptions via `sentry-conventions`. ([#6093](https://github.com/getsentry/relay/pull/6093)) - Raises the size limit for the flags context to 64KiB. ([#6137](https://github.com/getsentry/relay/pull/6137)) - Add segment_names field to Replay events. ([#6134](https://github.com/getsentry/relay/pull/6134)) diff --git a/relay-conventions/build/attributes.rs b/relay-conventions/build/attributes.rs index 34f56e201d5..6f7d8fb5cb1 100644 --- a/relay-conventions/build/attributes.rs +++ b/relay-conventions/build/attributes.rs @@ -40,6 +40,12 @@ pub enum DeprecationStatus { Backfill, /// Only write the replacement name. Normalize, + /// Move the value to the replacement name and apply a value transformation. + /// + /// For write behavior purposes, this produces `CurrentName` — the generic attribute + /// renaming logic leaves these attributes alone. Dedicated transformation code handles + /// the full move-and-reshape. + Transform, } /// Information about an attribute's deprecation. @@ -126,6 +132,11 @@ fn format_write_behavior(deprecation: Option<&Deprecation>) -> String { DeprecationStatus::Normalize => { format!("WriteBehavior::NewName({name})") } + DeprecationStatus::Transform => { + // Transformations are handled by dedicated code, not by the generic + // attribute renaming logic. Leave the attribute at its current name. + "WriteBehavior::CurrentName".to_owned() + } } } diff --git a/relay-conventions/sentry-conventions b/relay-conventions/sentry-conventions index 989dc716d72..d8be4e4ebb1 160000 --- a/relay-conventions/sentry-conventions +++ b/relay-conventions/sentry-conventions @@ -1 +1 @@ -Subproject commit 989dc716d72d64e8dd30a3085414e15d41d6fcf7 +Subproject commit d8be4e4ebb1a2258f4a30b6e4c8a6aea6f585f49 diff --git a/relay-event-normalization/src/eap/ai.rs b/relay-event-normalization/src/eap/ai.rs index 2d875a2f05c..49368d28ca4 100644 --- a/relay-event-normalization/src/eap/ai.rs +++ b/relay-event-normalization/src/eap/ai.rs @@ -5,6 +5,7 @@ use relay_event_schema::protocol::Attributes; use relay_protocol::Annotated; use crate::ModelMetadata; +use crate::eap::gen_ai_transform; use crate::span::ai; use crate::statsd::{Counters, map_origin_to_integration, platform_tag}; @@ -34,6 +35,7 @@ pub fn normalize_ai( return; } + gen_ai_transform::transform_gen_ai(attributes); normalize_model(attributes); normalize_ai_type(attributes); normalize_total_tokens(attributes); diff --git a/relay-event-normalization/src/eap/gen_ai_transform.rs b/relay-event-normalization/src/eap/gen_ai_transform.rs new file mode 100644 index 00000000000..d84a715e3fb --- /dev/null +++ b/relay-event-normalization/src/eap/gen_ai_transform.rs @@ -0,0 +1,633 @@ +//! Transforms deprecated gen_ai attributes into their normalized replacements. +//! +//! These transformations go beyond simple renaming — they reshape attribute values +//! to match the new schema. For the transformation specifications, see: +//! +//! +//! Called from [`super::normalize_ai`] as part of AI span normalization. +//! Attributes with `_status: "transform"` in sentry-conventions produce +//! `WriteBehavior::CurrentName`, so [`super::normalize_attribute_names`] leaves +//! them alone — the full move-and-reshape is handled here. + +// This module intentionally reads deprecated attribute keys to transform their values. +#![allow(deprecated)] + +use relay_conventions::attributes::*; +use relay_event_schema::protocol::Attributes; + +/// Applies gen_ai attribute transformations. +/// +/// Reads from deprecated attribute keys, reshapes the values, writes to the +/// canonical keys, and removes the deprecated keys. +pub(crate) fn transform_gen_ai(attributes: &mut Attributes) { + transform_request_messages(attributes); + transform_response_to_output_messages(attributes); +} + +/// Transforms `gen_ai.request.messages` → `gen_ai.input.messages`. +/// +/// Each message's `content` field is converted into a `parts` array: +/// - String content → `[{"type": "text", "content": ""}]` +/// - Array content → kept, with `text` copied to `content` on each part if missing +/// - Other/missing content → message left unchanged +/// +/// If the value isn't a valid JSON array of message objects, it is moved as-is. +fn transform_request_messages(attributes: &mut Attributes) { + if attributes.contains_key(GEN_AI__INPUT__MESSAGES) { + attributes.remove(GEN_AI__REQUEST__MESSAGES); + return; + } + + let Some(raw) = get_str(attributes, GEN_AI__REQUEST__MESSAGES) else { + return; + }; + + let Ok(messages) = serde_json::from_str::>(&raw) else { + // Not a JSON array — move as-is (just a key rename). + if let Some(attr) = attributes.remove(GEN_AI__REQUEST__MESSAGES) { + attributes + .0 + .insert(GEN_AI__INPUT__MESSAGES.to_owned(), attr); + } + return; + }; + + let transformed: Vec = messages + .into_iter() + .map(|mut message| { + if let Some(parts) = content_to_parts(message.get("content")) + && let Some(obj) = message.as_object_mut() + { + obj.remove("content"); + obj.insert("parts".to_owned(), parts); + } + message + }) + .collect(); + + if let Ok(json) = serde_json::to_string(&transformed) { + attributes.insert(GEN_AI__INPUT__MESSAGES.to_owned(), json); + } + attributes.remove(GEN_AI__REQUEST__MESSAGES); +} + +/// Converts a message's `content` field into the new `parts` format. +fn content_to_parts(content: Option<&serde_json::Value>) -> Option { + match content? { + serde_json::Value::String(s) => Some(serde_json::json!([ + {"type": "text", "content": s} + ])), + serde_json::Value::Array(items) => { + let parts: Vec = items + .iter() + .map(|part| { + let mut part = part.clone(); + if let Some(obj) = part.as_object_mut() + && !obj.contains_key("content") + && let Some(text) = obj.get("text").cloned() + { + obj.insert("content".to_owned(), text); + } + part + }) + .collect(); + Some(serde_json::Value::Array(parts)) + } + // Other types (objects for tool messages, etc.) — leave unchanged. + _ => None, + } +} + +/// Transforms `gen_ai.response.text` + `gen_ai.response.tool_calls` → `gen_ai.output.messages`. +/// +/// Builds one assistant message with parts from both sources. +/// +/// `gen_ai.response.text` can be: +/// - A plain string: `"hello"` +/// - A JSON string: `"\"hello\""` +/// - A JSON array of strings: `["hello", "world"]` +/// - A JSON object with content: `{"content": "hello"}` +/// - A JSON array of objects with content: `[{"content": "hello"}]` +fn transform_response_to_output_messages(attributes: &mut Attributes) { + if attributes.contains_key(GEN_AI__OUTPUT__MESSAGES) { + attributes.remove(GEN_AI__RESPONSE__TEXT); + attributes.remove(GEN_AI__RESPONSE__TOOL_CALLS); + return; + } + + let response_text = get_str(attributes, GEN_AI__RESPONSE__TEXT); + let tool_calls_raw = get_str(attributes, GEN_AI__RESPONSE__TOOL_CALLS); + + if response_text.is_none() && tool_calls_raw.is_none() { + return; + } + + let mut parts = Vec::new(); + + if let Some(ref text) = response_text { + response_text_to_parts(text, &mut parts); + } + + if let Some(ref raw) = tool_calls_raw { + tool_calls_to_parts(raw, &mut parts); + } + + // Always clean up deprecated keys, even if no parts could be extracted. + attributes.remove(GEN_AI__RESPONSE__TEXT); + attributes.remove(GEN_AI__RESPONSE__TOOL_CALLS); + + if parts.is_empty() { + return; + } + + let output = serde_json::json!([{"role": "assistant", "parts": parts}]); + if let Ok(json) = serde_json::to_string(&output) { + attributes.insert(GEN_AI__OUTPUT__MESSAGES.to_owned(), json); + } +} + +/// Extracts text parts from a `gen_ai.response.text` value. +fn response_text_to_parts(raw: &str, parts: &mut Vec) { + let Ok(parsed) = serde_json::from_str::(raw) else { + // Not valid JSON — treat the entire raw string as plain text. + parts.push(serde_json::json!({"type": "text", "content": raw})); + return; + }; + + match parsed { + serde_json::Value::String(s) => { + parts.push(serde_json::json!({"type": "text", "content": s})); + } + serde_json::Value::Array(items) => { + for item in items { + match item { + serde_json::Value::String(s) => { + parts.push(serde_json::json!({"type": "text", "content": s})); + } + serde_json::Value::Object(ref obj) => { + if let Some(content) = obj.get("content") { + parts.push(serde_json::json!({"type": "text", "content": content})); + } + } + _ => {} + } + } + } + serde_json::Value::Object(ref obj) => { + if let Some(content) = obj.get("content") { + parts.push(serde_json::json!({"type": "text", "content": content})); + } + } + _ => { + // Other JSON primitives — use raw string as text. + parts.push(serde_json::json!({"type": "text", "content": raw})); + } + } +} + +/// Extracts tool_call parts from a `gen_ai.response.tool_calls` value. +fn tool_calls_to_parts(raw: &str, parts: &mut Vec) { + let Ok(tool_calls) = serde_json::from_str::>(raw) else { + return; + }; + for mut tool_call in tool_calls { + if let Some(obj) = tool_call.as_object_mut() { + obj.insert( + "type".to_owned(), + serde_json::Value::String("tool_call".to_owned()), + ); + } + parts.push(tool_call); + } +} + +/// Helper to get a string attribute value as an owned String. +fn get_str(attributes: &Attributes, key: &str) -> Option { + attributes + .get_value(key) + .and_then(|v| v.as_str()) + .map(|s| s.to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_attributes(pairs: &[(&str, &str)]) -> Attributes { + let mut attrs = Attributes::new(); + for (key, value) in pairs { + attrs.insert(key.to_string(), value.to_string()); + } + attrs + } + + fn get_string(attributes: &Attributes, key: &str) -> Option { + attributes.get_value(key)?.as_str().map(|s| s.to_owned()) + } + + fn parse_json(s: &str) -> serde_json::Value { + serde_json::from_str(s).unwrap() + } + + mod request_messages { + use super::*; + + #[test] + fn string_content_to_parts() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":"hello"}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "content": "hello"}]}]), + ); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + + #[test] + fn array_content_to_parts() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":[{"type":"text","text":"hello"}]}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "text": "hello", "content": "hello"}]}]), + ); + } + + #[test] + fn does_not_overwrite_existing() { + let existing = r#"[{"role":"user","parts":[{"type":"text","content":"existing"}]}]"#; + let mut attrs = make_attributes(&[ + (GEN_AI__INPUT__MESSAGES, existing), + ( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":"ignored"}]"#, + ), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!(parse_json(&result), parse_json(existing)); + // Deprecated key cleaned up even when canonical key exists. + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + + #[test] + fn preserves_extra_fields() { + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"system","content":"be helpful","metadata":{"key":"value"}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + let parsed = parse_json(&result); + assert_eq!(parsed[0]["role"], "system"); + assert_eq!(parsed[0]["metadata"]["key"], "value"); + assert!(parsed[0].get("content").is_none()); + } + + #[test] + fn preserves_unconvertible_content() { + // Tool messages have object content — left unchanged. + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"tool","content":{"result":"ok"}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "tool", "content": {"result": "ok"}}]), + ); + } + + #[test] + fn messages_with_metadata() { + // Spec example: "system and user messages with metadata" + let mut attrs = make_attributes(&[( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"system","content":"You are a helpful assistant.","response_metadata":{}},{"role":"user","content":"What is the capital of France?","response_metadata":{}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + let parsed = parse_json(&result); + assert_eq!(parsed[0]["role"], "system"); + assert_eq!(parsed[0]["response_metadata"], serde_json::json!({})); + assert_eq!( + parsed[0]["parts"], + serde_json::json!([{"type": "text", "content": "You are a helpful assistant."}]), + ); + assert!(parsed[0].get("content").is_none()); + assert_eq!(parsed[1]["role"], "user"); + assert_eq!(parsed[1]["response_metadata"], serde_json::json!({})); + assert_eq!( + parsed[1]["parts"], + serde_json::json!([{"type": "text", "content": "What is the capital of France?"}]), + ); + } + + #[test] + fn no_messages_is_noop() { + let mut attrs = make_attributes(&[]); + transform_gen_ai(&mut attrs); + assert!(get_string(&attrs, GEN_AI__INPUT__MESSAGES).is_none()); + } + + #[test] + fn invalid_json_moves_as_is() { + let mut attrs = make_attributes(&[(GEN_AI__REQUEST__MESSAGES, "not json")]); + + transform_gen_ai(&mut attrs); + + assert_eq!( + get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(), + "not json" + ); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + } + + mod response_to_output { + use super::*; + + #[test] + fn plain_text() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, "hello")]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"} + ]}]), + ); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + + #[test] + fn json_string() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#""hello world""#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello world"} + ]}]), + ); + } + + #[test] + fn array_of_strings() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#"["hello","world"]"#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"type": "text", "content": "world"} + ]}]), + ); + } + + #[test] + fn object_with_content() { + let mut attrs = make_attributes(&[(GEN_AI__RESPONSE__TEXT, r#"{"content":"hello"}"#)]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"} + ]}]), + ); + } + + #[test] + fn array_of_objects_with_content() { + let mut attrs = make_attributes(&[( + GEN_AI__RESPONSE__TEXT, + r#"[{"content":"hello"},{"content":"world"}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"type": "text", "content": "world"} + ]}]), + ); + } + + #[test] + fn text_and_tool_calls() { + let mut attrs = make_attributes(&[ + (GEN_AI__RESPONSE__TEXT, "hello"), + ( + GEN_AI__RESPONSE__TOOL_CALLS, + r#"[{"id":"call_1","name":"weather","arguments":{}}]"#, + ), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"id": "call_1", "name": "weather", "arguments": {}, "type": "tool_call"} + ]}]), + ); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TOOL_CALLS).is_none()); + } + + #[test] + fn tool_calls_only() { + let mut attrs = make_attributes(&[( + GEN_AI__RESPONSE__TOOL_CALLS, + r#"[{"id":"call_1","name":"weather","arguments":{}}]"#, + )]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"id": "call_1", "name": "weather", "arguments": {}, "type": "tool_call"} + ]}]), + ); + } + + #[test] + fn does_not_overwrite_existing() { + let existing = + r#"[{"role":"assistant","parts":[{"type":"text","content":"existing"}]}]"#; + let mut attrs = make_attributes(&[ + (GEN_AI__OUTPUT__MESSAGES, existing), + (GEN_AI__RESPONSE__TEXT, "ignored"), + ]); + + transform_gen_ai(&mut attrs); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!(parse_json(&result), parse_json(existing)); + // Deprecated keys cleaned up even when canonical key exists. + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + + #[test] + fn no_response_attributes_is_noop() { + let mut attrs = make_attributes(&[]); + transform_gen_ai(&mut attrs); + assert!(get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).is_none()); + } + } + + /// Integration tests: SpanV1 → SpanV2 → normalize_attribute_names → normalize_ai. + /// + /// `normalize_attribute_names` leaves `Transform` attributes alone (`WriteBehavior::CurrentName`), + /// then `normalize_ai` calls `transform_gen_ai` to reshape and move them. + mod integration { + use relay_event_schema::protocol::Span as SpanV1; + use relay_protocol::Annotated; + + use super::*; + use crate::eap; + + fn normalize_span_v1(json: &str) -> Attributes { + let span_v1 = Annotated::::from_json(json) + .unwrap() + .into_value() + .unwrap(); + let mut span_v2 = relay_spans::span_v1_to_span_v2(span_v1, false); + eap::normalize_attribute_names(&mut span_v2.attributes); + eap::normalize_ai(&mut span_v2.attributes, None, None); + span_v2.attributes.into_value().unwrap_or_default() + } + + #[test] + fn span_v1_request_messages() { + let attrs = normalize_span_v1( + r#"{ + "data": { + "gen_ai.request.messages": "[{\"role\":\"user\",\"content\":\"hello\"}]", + "gen_ai.request.model": "gpt-4o", + "sentry.op": "gen_ai.generate_text" + } + }"#, + ); + + let result = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "content": "hello"}]}]), + ); + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + } + + #[test] + fn span_v1_response_text() { + let attrs = normalize_span_v1( + r#"{ + "data": { + "gen_ai.response.text": "hello", + "gen_ai.request.model": "gpt-4o", + "sentry.op": "gen_ai.generate_text" + } + }"#, + ); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [{"type": "text", "content": "hello"}]}]), + ); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + + #[test] + fn span_v1_response_text_and_tool_calls() { + let attrs = normalize_span_v1( + r#"{ + "data": { + "gen_ai.response.text": "hello", + "gen_ai.response.tool_calls": "[{\"id\":\"c1\",\"name\":\"weather\",\"arguments\":{}}]", + "gen_ai.request.model": "gpt-4o", + "sentry.op": "gen_ai.generate_text" + } + }"#, + ); + + let result = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&result), + serde_json::json!([{"role": "assistant", "parts": [ + {"type": "text", "content": "hello"}, + {"id": "c1", "name": "weather", "arguments": {}, "type": "tool_call"} + ]}]), + ); + } + + #[test] + fn span_v2_deprecated_keys() { + let mut attrs = make_attributes(&[ + ( + GEN_AI__REQUEST__MESSAGES, + r#"[{"role":"user","content":"hi"}]"#, + ), + (GEN_AI__RESPONSE__TEXT, "bye"), + (SENTRY__OP, "gen_ai.generate_text"), + ]); + + transform_gen_ai(&mut attrs); + + let input = get_string(&attrs, GEN_AI__INPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&input), + serde_json::json!([{"role": "user", "parts": [{"type": "text", "content": "hi"}]}]), + ); + + let output = get_string(&attrs, GEN_AI__OUTPUT__MESSAGES).unwrap(); + assert_eq!( + parse_json(&output), + serde_json::json!([{"role": "assistant", "parts": [{"type": "text", "content": "bye"}]}]), + ); + + assert!(get_string(&attrs, GEN_AI__REQUEST__MESSAGES).is_none()); + assert!(get_string(&attrs, GEN_AI__RESPONSE__TEXT).is_none()); + } + } +} diff --git a/relay-event-normalization/src/eap/mod.rs b/relay-event-normalization/src/eap/mod.rs index b52f7f7f671..0d28fc429f2 100644 --- a/relay-event-normalization/src/eap/mod.rs +++ b/relay-event-normalization/src/eap/mod.rs @@ -25,6 +25,7 @@ use crate::span::tag_extraction::{ use crate::{ClientHints, FromUserAgentInfo as _, RawUserAgentInfo}; mod ai; +mod gen_ai_transform; mod mobile; mod size; pub mod time; diff --git a/relay-event-schema/src/protocol/span.rs b/relay-event-schema/src/protocol/span.rs index b5bfb337339..b2d5d01296e 100644 --- a/relay-event-schema/src/protocol/span.rs +++ b/relay-event-schema/src/protocol/span.rs @@ -546,7 +546,6 @@ pub struct SpanData { #[metastructure( field = "gen_ai.input.messages", legacy_alias = "gen_ai.prompt", - legacy_alias = "gen_ai.request.messages", legacy_alias = "ai.prompt.messages" )] pub gen_ai_input_messages: Annotated, @@ -570,10 +569,8 @@ pub struct SpanData { /// The output messages from the model call. #[metastructure( field = "gen_ai.output.messages", - legacy_alias = "gen_ai.response.tool_calls", legacy_alias = "ai.response.toolCalls", legacy_alias = "ai.tool_calls", - legacy_alias = "gen_ai.response.text", legacy_alias = "ai.response.text", legacy_alias = "ai.responses" )] diff --git a/tests/integration/test_ai.py b/tests/integration/test_ai.py index 9363c47ba2a..671f1d3df88 100644 --- a/tests/integration/test_ai.py +++ b/tests/integration/test_ai.py @@ -1,6 +1,8 @@ +import json from datetime import datetime, timezone from sentry_relay.consts import DataCategory +from sentry_sdk.envelope import Envelope, Item, PayloadRef from .asserts import matches_any, time_within_delta @@ -392,7 +394,7 @@ def test_ai_spans_example_transaction( "gen_ai.response.model": {"type": "string", "value": "gpt-4o"}, "gen_ai.output.messages": { "type": "string", - "value": "True. \n\n- London: 61°F \n- San Francisco: 13°C", + "value": matches_any(), }, "gen_ai.response.tokens_per_second": {"type": "double", "value": 130.0}, "gen_ai.usage.input_tokens": {"type": "integer", "value": 245}, @@ -509,7 +511,7 @@ def test_ai_spans_example_transaction( "value": "gpt-4o-2024-08-06", }, "gen_ai.response.tokens_per_second": {"type": "double", "value": 92.0}, - "gen_ai.output.messages": { + "gen_ai.response.tool_calls": { "type": "string", "value": "some_tool_calls", }, @@ -1037,10 +1039,7 @@ def test_ai_spans_example_transaction( }, "gen_ai.output.messages": { "type": "string", - "value": "True. \n" - "\n" - "- London: 61°F \n" - "- San Francisco: 13°C", + "value": matches_any(), }, "gen_ai.response.tokens_per_second": {"type": "double", "value": 38.0}, "gen_ai.provider.name": {"type": "string", "value": "openai.responses"}, @@ -1292,3 +1291,280 @@ def test_ai_spans_example_transaction( "quantity": 10, }, ] + + +TEST_CONFIG = { + "outcomes": { + "emit_outcomes": True, + } +} + + +def envelope_with_spans(*payloads: dict, trace_info=None, metadata=None) -> Envelope: + envelope = Envelope() + envelope.add_item( + Item( + type="span", + payload=PayloadRef(json={"items": payloads, **(metadata or {})}), + content_type="application/vnd.sentry.items.span.v2+json", + headers={"item_count": len(payloads)}, + ) + ) + envelope.headers["trace"] = trace_info + return envelope + + +def test_gen_ai_transform_request_messages_v1( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV1 (transaction) path: gen_ai.request.messages with JSON content + gets transformed to gen_ai.input.messages with parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing()) + + ts = datetime.now(timezone.utc) + messages = json.dumps([{"role": "user", "content": "What is the weather?"}]) + + relay.send_transaction( + project_id, + { + "contexts": { + "trace": { + "span_id": "aaaa000000000001", + "trace_id": "a9351cd574f092f6acad48e250981f11", + "op": "gen_ai.generate_text", + "origin": "manual", + "status": "ok", + }, + }, + "spans": [ + { + "span_id": "bbbb000000000001", + "trace_id": "a9351cd574f092f6acad48e250981f11", + "parent_span_id": "aaaa000000000001", + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "status": "ok", + "op": "gen_ai.generate_text", + "data": { + "gen_ai.request.messages": messages, + "gen_ai.request.model": "gpt-4o", + }, + }, + ], + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "transaction": "test-transform", + "type": "transaction", + "transaction_info": {"source": "custom"}, + "platform": "python", + }, + ) + + spans = spans_consumer.get_spans(n=2) + ai_span = next(s for s in spans if s["span_id"] == "bbbb000000000001") + attrs = ai_span["attributes"] + + result = json.loads(attrs["gen_ai.input.messages"]["value"]) + assert result == [ + {"role": "user", "parts": [{"type": "text", "content": "What is the weather?"}]} + ] + # Deprecated key must not appear. + assert "gen_ai.request.messages" not in attrs + + +def test_gen_ai_transform_response_to_output_v1( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV1 (transaction) path: gen_ai.response.text gets transformed into + gen_ai.output.messages with assistant parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing()) + + ts = datetime.now(timezone.utc) + relay.send_transaction( + project_id, + { + "contexts": { + "trace": { + "span_id": "aaaa000000000002", + "trace_id": "b9351cd574f092f6acad48e250981f11", + "op": "gen_ai.generate_text", + "origin": "manual", + "status": "ok", + }, + }, + "spans": [ + { + "span_id": "bbbb000000000002", + "trace_id": "b9351cd574f092f6acad48e250981f11", + "parent_span_id": "aaaa000000000002", + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "status": "ok", + "op": "gen_ai.generate_text", + "data": { + "gen_ai.response.text": "The weather is sunny.", + "gen_ai.request.model": "gpt-4o", + }, + }, + ], + "start_timestamp": ts.timestamp() - 0.5, + "timestamp": ts.timestamp(), + "transaction": "test-transform", + "type": "transaction", + "transaction_info": {"source": "custom"}, + "platform": "python", + }, + ) + + spans = spans_consumer.get_spans(n=2) + ai_span = next(s for s in spans if s["span_id"] == "bbbb000000000002") + attrs = ai_span["attributes"] + + result = json.loads(attrs["gen_ai.output.messages"]["value"]) + assert result == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "The weather is sunny."}], + } + ] + # Deprecated key must not appear. + assert "gen_ai.response.text" not in attrs + + +def test_gen_ai_transform_request_messages_v2( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV2 (direct span ingestion) path: gen_ai.request.messages with JSON content + gets transformed to gen_ai.input.messages with parts format. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + ts = datetime.now(timezone.utc) + messages = json.dumps([{"role": "user", "content": "What is the weather?"}]) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "c9351cd574f092f6acad48e250981f11", + "span_id": "cccc000000000001", + "is_segment": True, + "name": "gen_ai.generate_text", + "status": "ok", + "attributes": { + "sentry.op": {"value": "gen_ai.generate_text", "type": "string"}, + "gen_ai.request.messages": {"value": messages, "type": "string"}, + "gen_ai.request.model": {"value": "gpt-4o", "type": "string"}, + }, + }, + trace_info={ + "trace_id": "c9351cd574f092f6acad48e250981f11", + "public_key": mini_sentry.get_dsn_public_key(project_id), + }, + ) + + relay.send_envelope(project_id, envelope) + + spans = spans_consumer.get_spans(n=1) + attrs = spans[0]["attributes"] + + result = json.loads(attrs["gen_ai.input.messages"]["value"]) + assert result == [ + {"role": "user", "parts": [{"type": "text", "content": "What is the weather?"}]} + ] + # Deprecated key must not appear. + assert "gen_ai.request.messages" not in attrs + + +def test_gen_ai_transform_response_text_and_tool_calls_v2( + mini_sentry, + relay, + relay_with_processing, + spans_consumer, +): + """ + SpanV2 (direct span ingestion) path: both gen_ai.response.text and + gen_ai.response.tool_calls get combined into gen_ai.output.messages. + """ + spans_consumer = spans_consumer() + + project_id = 42 + mini_sentry.add_full_project_config(project_id) + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + ts = datetime.now(timezone.utc) + tool_calls = json.dumps( + [{"id": "call_1", "name": "get_weather", "arguments": {"city": "London"}}] + ) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp(), + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "d9351cd574f092f6acad48e250981f11", + "span_id": "dddd000000000001", + "is_segment": True, + "name": "gen_ai.generate_text", + "status": "ok", + "attributes": { + "sentry.op": {"value": "gen_ai.generate_text", "type": "string"}, + "gen_ai.response.text": {"value": "Let me check.", "type": "string"}, + "gen_ai.response.tool_calls": {"value": tool_calls, "type": "string"}, + "gen_ai.request.model": {"value": "gpt-4o", "type": "string"}, + }, + }, + trace_info={ + "trace_id": "d9351cd574f092f6acad48e250981f11", + "public_key": mini_sentry.get_dsn_public_key(project_id), + }, + ) + + relay.send_envelope(project_id, envelope) + + spans = spans_consumer.get_spans(n=1) + attrs = spans[0]["attributes"] + + result = json.loads(attrs["gen_ai.output.messages"]["value"]) + assert result == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "Let me check."}, + { + "id": "call_1", + "name": "get_weather", + "arguments": {"city": "London"}, + "type": "tool_call", + }, + ], + } + ] + # Deprecated keys must not appear. + assert "gen_ai.response.text" not in attrs + assert "gen_ai.response.tool_calls" not in attrs