Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
# SPDX-License-Identifier: Apache-2.0

# Default ownership for repository files.
* @nvidia/nat-developers
* @nvidia/nemo-relay-developers

# CODEOWNERS changes require repository administrator review.
/.github/CODEOWNERS @nvidia/nemo-relay-admins

# Attribution files require dependency approver review due to change in version(s)
ATTRIBUTIONS-*.md @nvidia/nat-dep-approvers
ATTRIBUTIONS-*.md @nvidia/nemo-relay-dep-approvers

# Documentation changes must go to technical writers and NAT developers.
docs/ @lvojtku @nvidia/NAT-developers
fern/ @lvojtku @nvidia/NAT-developers
# Documentation changes must go to technical writers and NeMo Relay developers.
docs/ @nvidia/nemo-relay-docs-reviewers @nvidia/nemo-relay-developers
fern/ @nvidia/nemo-relay-docs-reviewers @nvidia/nemo-relay-developers
9 changes: 6 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ requiring changes to the existing agent stack. It gives coding agents,
applications, framework integrations, middleware, and observability backends a
shared runtime for scopes, policy, plugins, and lifecycle events.

For how Relay complements OpenTelemetry GenAI conventions and observability or
evaluation products, see [the Ecosystem guide](https://docs.nvidia.com/nemo/relay/about-nemo-relay/ecosystem).

## Where To Start

| Goal | Start With |
Expand All @@ -30,7 +33,7 @@ shared runtime for scopes, policy, plugins, and lifecycle events.
| Use LangChain, LangGraph, Deep Agents, or OpenClaw | [Supported Integrations](https://docs.nvidia.com/nemo/relay/supported-integrations/about) |
| Build a framework or provider integration | [Integrate into Frameworks](https://docs.nvidia.com/nemo/relay/integrate-into-frameworks/about) |
| Export ATOF, ATIF, OpenTelemetry, or OpenInference | [Observability Plugin](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/about) |
| Package reusable middleware or exporters | [Build Plugins](https://docs.nvidia.com/nemo/relay/v0.5.0/build-plugins/about) |
| Package reusable middleware or exporters | [Build Plugins](https://docs.nvidia.com/nemo/relay/build-plugins/about) |
| Develop or test this repository from source | [CONTRIBUTING.md](CONTRIBUTING.md) |


Expand Down Expand Up @@ -211,7 +214,7 @@ uv add nemo-relay

# Node.js
# Requires Node.js 24 or newer.
npm install nemo-relay-node
npm install nemo-relay-node@0.6.0

# Rust
cargo add nemo-relay
Expand Down Expand Up @@ -335,8 +338,8 @@ End-user documentation lives at
Important local entry points:

- [Overview](https://docs.nvidia.com/nemo/relay/about-nemo-relay/overview)
- [Getting Started](https://docs.nvidia.com/nemo/relay/getting-started/quick-start)
- [Installation](https://docs.nvidia.com/nemo/relay/getting-started/installation)
- [Agent Runtime Primer](https://docs.nvidia.com/nemo/relay/getting-started/agent-runtime-primer)
- [Testing and Docs](https://docs.nvidia.com/nemo/relay/contribute/testing-and-docs)

For source builds, tests, and contribution workflow, refer to
Expand Down
90 changes: 84 additions & 6 deletions crates/adaptive/src/acg/ir_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ pub fn build_prompt_ir(request: &AnnotatedLlmRequest) -> Result<PromptIR> {
let mut sequence_index: u32 = 0;
let mut inserted_tool_blocks = false;

if let Some(instructions) = &request.instructions {
blocks.push(build_text_block(
&mut sequence_index,
instructions,
PromptRole::System,
ProvenanceLabel::System,
Some("instructions"),
));
}

for message in &request.messages {
if should_insert_tool_blocks_before_message(inserted_tool_blocks, request, message) {
append_tool_schema_blocks(&mut blocks, &mut sequence_index, request.tools.as_deref())?;
Expand Down Expand Up @@ -71,7 +81,9 @@ fn should_insert_tool_blocks_before_message(
request: &AnnotatedLlmRequest,
message: &Message,
) -> bool {
!inserted_tool_blocks && !matches!(message, Message::System { .. }) && request.tools.is_some()
!inserted_tool_blocks
&& !matches!(message, Message::System { .. } | Message::Developer { .. })
&& request.tools.is_some()
}

fn append_message_blocks(
Expand All @@ -87,6 +99,13 @@ fn append_message_blocks(
ProvenanceLabel::System,
None,
)),
Message::Developer { content, .. } => blocks.push(build_text_block(
sequence_index,
content,
PromptRole::System,
ProvenanceLabel::Developer,
Some("developer"),
)),
Message::User { content, .. } => blocks.push(build_text_block(
sequence_index,
content,
Expand All @@ -112,6 +131,37 @@ fn append_message_blocks(
content,
tool_call_id,
)),
Message::Function { content, name } => {
let content = MessageContent::Text(content.clone().unwrap_or_default());
blocks.push(build_text_block(
sequence_index,
&content,
PromptRole::Tool,
ProvenanceLabel::Tool,
Some(name),
));
}
Message::ToolCallItem { name, .. } => blocks.push(build_serialized_message_block(
sequence_index,
message,
PromptRole::Assistant,
ProvenanceLabel::Developer,
Some(name),
)?),
Message::ToolResultItem { call_id, .. } => blocks.push(build_serialized_message_block(
sequence_index,
message,
PromptRole::Tool,
ProvenanceLabel::Tool,
Some(call_id),
)?),
Message::ProviderNative { kind, .. } => blocks.push(build_serialized_message_block(
sequence_index,
message,
PromptRole::User,
ProvenanceLabel::Developer,
Some(kind),
)?),
}

Ok(())
Expand Down Expand Up @@ -147,15 +197,36 @@ fn extract_text(content: &MessageContent) -> String {
MessageContent::Text(text) => text.clone(),
MessageContent::Parts(parts) => parts
.iter()
.filter_map(|part| match part {
ContentPart::Text { text } => Some(text.as_str()),
ContentPart::ImageUrl { .. } => None,
.map(|part| match part {
ContentPart::Text { text, .. } => text.clone(),
ContentPart::Refusal { refusal, .. } => refusal.clone(),
ContentPart::ImageUrl { .. } | ContentPart::Image { .. } => String::new(),
ContentPart::Audio { .. }
| ContentPart::File { .. }
| ContentPart::ToolUse { .. }
| ContentPart::ToolResult { .. }
| ContentPart::ProviderNative { .. } => {
serde_json::to_string(part).unwrap_or_default()
}
})
.filter(|text| !text.is_empty())
.collect::<Vec<_>>()
.join("\n"),
}
}

fn build_serialized_message_block(
seq: &mut u32,
message: &Message,
role: PromptRole,
provenance: ProvenanceLabel,
suffix: Option<&str>,
) -> Result<PromptBlock> {
let value = serde_json::to_value(message)?;
let content = MessageContent::Text(canonicalize_value(&value)?);
Ok(build_text_block(seq, &content, role, provenance, suffix))
}

fn generate_span_id(role: PromptRole, index: u32, suffix: Option<&str>) -> SpanId {
let role_str = match role {
PromptRole::System => "system",
Expand Down Expand Up @@ -257,7 +328,7 @@ fn build_tool_schema_block(seq: &mut u32, tool: &ToolDefinition) -> Result<Promp
let canonical = canonicalize_value(&tool_value)?;
let content = normalize_whitespace(&canonical);
let index = *seq;
let span_id = generate_span_id(PromptRole::System, index, Some(&tool.function.name));
let span_id = generate_span_id(PromptRole::System, index, Some(tool_definition_name(tool)));
*seq += 1;

Ok(PromptBlock {
Expand All @@ -279,13 +350,20 @@ fn build_tool_schema_hashes(tools: &[ToolDefinition]) -> Result<Vec<ToolSchemaHa
let value = serde_json::to_value(tool_definition)?;
let canonical = canonicalize_value(&value)?;
Ok(ToolSchemaHash {
tool_name: tool_definition.function.name.clone(),
tool_name: tool_definition_name(tool_definition).to_string(),
schema_hash: sha256_hex(&canonical),
})
})
.collect()
}

fn tool_definition_name(tool: &ToolDefinition) -> &str {
match tool {
ToolDefinition::Function { function, .. } => &function.name,
ToolDefinition::ProviderNative { kind, .. } => kind,
}
}

fn compute_request_hash(request: &AnnotatedLlmRequest) -> Result<String> {
let value = serde_json::to_value(request)?;
let canonical = canonicalize_value(&value)?;
Expand Down
92 changes: 79 additions & 13 deletions crates/adaptive/src/acg_profile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,26 +76,47 @@ fn derive_key_parts(annotated_request: &AnnotatedLlmRequest) -> AcgKeyParts<'_>
fn message_role_tag(message: &Message) -> &'static str {
match message {
Message::System { .. } => "system",
Message::Developer { .. } => "developer",
Message::User { .. } => "user",
Message::Assistant { .. } => "assistant",
Message::Tool { .. } => "tool",
Message::Function { .. } => "function",
Message::ToolCallItem { .. } => "tool_call",
Message::ToolResultItem { .. } => "tool_result",
Message::ProviderNative { .. } => "provider_native",
}
}

fn system_prompt_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String {
let system_content = annotated_request
.messages
.iter()
.filter_map(|message| match message {
Message::System { content, .. } => Some(extract_text(content)),
_ => None,
})
.collect::<Vec<_>>()
.join("\n");
let mut system_content = Vec::new();
if let Some(instructions) = &annotated_request.instructions {
system_content.push(serde_json::json!({
"source": "instructions",
"content": instructions,
}));
}
system_content.extend(
annotated_request
.messages
.iter()
.filter_map(|message| match message {
Message::System { content, .. } => Some(serde_json::json!({
"source": "message",
"role": "system",
"content": content,
})),
Message::Developer { content, .. } => Some(serde_json::json!({
"source": "message",
"role": "developer",
"content": content,
})),
_ => None,
}),
);
if system_content.is_empty() {
"no-system".to_string()
} else {
sha256_hex(&system_content)
hash_canonical_json(&serde_json::Value::Array(system_content))
}
}

Expand Down Expand Up @@ -148,7 +169,7 @@ fn learning_seed_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String
.messages
.iter()
.find_map(|message| match message {
Message::System { .. } => None,
Message::System { .. } | Message::Developer { .. } => None,
Message::User { content, .. } => {
Some(format!("user:{}", sha256_hex(&extract_text(content))))
}
Expand All @@ -160,10 +181,46 @@ fn learning_seed_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String
Message::Tool { content, .. } => {
Some(format!("tool:{}", sha256_hex(&extract_text(content))))
}
Message::Function { content, name } => Some(format!(
"function:{}",
hash_canonical_json(&serde_json::json!({
"name": name,
"content": content,
}))
)),
Message::ToolCallItem {
name, arguments, ..
} => Some(format!(
"tool-call:{}",
hash_canonical_json(&serde_json::json!({
"name": name,
"arguments": arguments,
}))
)),
Message::ToolResultItem { output, .. } => {
Some(format!("tool-result:{}", sha256_hex(&output.to_string())))
}
Message::ProviderNative {
provider,
kind,
value,
} => Some(format!(
"native:{}",
hash_canonical_json(&serde_json::json!({
"provider": provider,
"kind": kind,
"value": value,
}))
)),
})
.unwrap_or_else(|| "no-seed".to_string())
}

fn hash_canonical_json(value: &serde_json::Value) -> String {
let canonical = canonicalize_value(value).unwrap_or_else(|_| value.to_string());
sha256_hex(&canonical)
}

fn tool_schema_fingerprint(tools: Option<&[ToolDefinition]>) -> String {
let Some(tools) = tools else {
return "no-tools".to_string();
Expand All @@ -189,12 +246,21 @@ fn extract_text(content: &MessageContent) -> String {
MessageContent::Parts(parts) => parts
.iter()
.map(|part| match part {
ContentPart::Text { text } => text.clone(),
ContentPart::ImageUrl { image_url } => format!(
ContentPart::Text { text, .. } => text.clone(),
ContentPart::Refusal { refusal, .. } => refusal.clone(),
ContentPart::ImageUrl { image_url, .. } => format!(
"[image:{}:{}]",
image_url.detail.as_deref().unwrap_or("none"),
sha256_hex(&image_url.url)
),
ContentPart::Image { .. }
| ContentPart::Audio { .. }
| ContentPart::File { .. }
| ContentPart::ToolUse { .. }
| ContentPart::ToolResult { .. }
| ContentPart::ProviderNative { .. } => {
serde_json::to_string(part).unwrap_or_default()
}
})
.collect::<Vec<_>>()
.join("\n"),
Expand Down
10 changes: 8 additions & 2 deletions crates/adaptive/tests/integration/acg_module_surface_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ fn acg_module_surface_policy_and_ir_builder_symbols_compile_from_canonical_names
assert!(envelope.policy.warm_first_enabled);

let request = AnnotatedLlmRequest {
instructions: None,
api_specific: None,
messages: vec![
Message::System {
content: MessageContent::Text("You are helpful.".to_string()),
Expand Down Expand Up @@ -175,6 +177,8 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
use nemo_relay_adaptive::acg::prompt_ir::{BlockContentType, PromptRole};

let request = AnnotatedLlmRequest {
instructions: None,
api_specific: None,
messages: vec![
Message::System {
content: MessageContent::Text("You are helpful.".to_string()),
Expand All @@ -187,8 +191,7 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
],
model: Some("claude-sonnet".to_string()),
params: None,
tools: Some(vec![ToolDefinition {
tool_type: "function".to_string(),
tools: Some(vec![ToolDefinition::Function {
function: FunctionDefinition {
name: "get_weather".to_string(),
description: Some("Look up the weather".to_string()),
Expand All @@ -198,7 +201,10 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
"location": {"type": "string"}
}
})),
strict: None,
extra: serde_json::Map::new(),
},
extra: serde_json::Map::new(),
}]),
tool_choice: None,
store: None,
Expand Down
2 changes: 2 additions & 0 deletions crates/adaptive/tests/integration/redis_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ fn make_test_run(agent_id: &str) -> RunRecord {

fn sample_annotated_request(model: &str) -> AnnotatedLlmRequest {
AnnotatedLlmRequest {
instructions: None,
api_specific: None,
messages: vec![
Message::System {
content: MessageContent::Text("You are a careful planner".to_string()),
Expand Down
Loading
Loading