Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 78 additions & 6 deletions crates/libsy/src/algorithms/llm_class.rs
Original file line number Diff line number Diff line change
Expand Up @@ -342,9 +342,12 @@ impl TaskClassifierConfig {
message: "max_output_tokens must be at least 1".to_string(),
});
}
if self.message_hash_fallback && self.classify_trigger != ClassifyTrigger::NewSession {
// Only `every_request` is rejected: it retains nothing, so a fallback identity has
// nothing to key. Both retaining triggers can use it.
if self.message_hash_fallback && self.classify_trigger == ClassifyTrigger::EveryRequest {
return Err(LibsyError::AlgorithmError {
message: "message_hash_fallback requires classify_trigger = new_session"
message: "message_hash_fallback requires classify_trigger = new_session or \
user_turn"
.to_string(),
});
}
Expand Down Expand Up @@ -414,9 +417,12 @@ impl CustomClassifierConfig {
message: "max_output_tokens must be at least 1".to_string(),
});
}
if self.message_hash_fallback && self.classify_trigger != ClassifyTrigger::NewSession {
// Only `every_request` is rejected: it retains nothing, so a fallback identity has
// nothing to key. Both retaining triggers can use it.
if self.message_hash_fallback && self.classify_trigger == ClassifyTrigger::EveryRequest {
return Err(LibsyError::AlgorithmError {
message: "message_hash_fallback requires classify_trigger = new_session"
message: "message_hash_fallback requires classify_trigger = new_session or \
user_turn"
.to_string(),
});
}
Expand Down Expand Up @@ -866,9 +872,13 @@ impl LlmTaskClassifier {
config: ClassifierRouteConfig,
) -> Result<Self> {
algorithm::ensure_model_is_target(&targets, &config.default_target)?;
if config.message_hash_fallback && config.classify_trigger != ClassifyTrigger::NewSession {
// Only `every_request` is rejected: it retains nothing, so a fallback identity has
// nothing to key. Both retaining triggers can use it.
if config.message_hash_fallback && config.classify_trigger == ClassifyTrigger::EveryRequest
{
return Err(LibsyError::AlgorithmError {
message: "message_hash_fallback requires classify_trigger = new_session"
message: "message_hash_fallback requires classify_trigger = new_session or \
user_turn"
.to_string(),
});
}
Expand Down Expand Up @@ -1297,6 +1307,68 @@ mod tests {
);
}

#[tokio::test]
async fn user_turn_holds_its_target_without_a_session_id() -> Result<()> {
// The reason the combination is allowed: benchmark harnesses and raw API callers
// often send no session id, and without a fallback identity `user_turn` re-judges
// every request, which is `every_request` under a different name.
let recorder = Arc::new(Recorder::default());
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: ModelId::from("judge"),
efficient_target: ModelId::from("efficient"),
capable_target: ModelId::from("capable"),
config: TaskClassifierConfig {
classify_trigger: ClassifyTrigger::UserTurn,
message_hash_fallback: true,
recent_turn_window: None,
..test_config(TEST_THRESHOLD)
},
})?);

// A tool continuation of the same conversation: the last message is not the user's.
let mut continuation = classify_request();
continuation
.llm_request
.messages
.push(Message::text(Role::Assistant, "calling a tool"));

test_drive(router.clone(), classify_request(), recorder.serve()).await?;
test_drive(router.clone(), continuation, recorder.serve()).await?;

// The judge runs once, for the opening user turn; the continuation rides the
// retained target rather than being judged again.
assert_eq!(
recorder.calls(),
vec!["judge", "efficient", "efficient"],
"the continuation should reuse the retained target"
);
Ok(())
}

#[test]
fn message_hash_fallback_is_allowed_on_every_retaining_trigger() -> Result<()> {
// Both triggers retain a target between requests, so both can key that target on
// a message hash when the caller sends no session id. `every_request` retains
// nothing and stays rejected, which `invalid_classifier_config_is_rejected` covers.
for trigger in [ClassifyTrigger::NewSession, ClassifyTrigger::UserTurn] {
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
judge_target: ModelId::from("judge"),
efficient_target: ModelId::from("e"),
capable_target: ModelId::from("c"),
config: TaskClassifierConfig {
base_threshold: 0.5,
message_hash_fallback: true,
classify_trigger: trigger,
..TaskClassifierConfig::default()
},
})
.map_err(|error| LibsyError::AlgorithmError {
message: format!("{trigger:?} with message_hash_fallback rejected: {error}"),
})?;
}
Ok(())
}

#[test]
fn invalid_classifier_config_is_rejected() -> Result<()> {
for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/toml_schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ Capability mode classifies before serving. See
| `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. |
| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
| `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. |
| `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. |
| `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
| `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |
| `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. |

Expand Down Expand Up @@ -185,7 +185,7 @@ policy selector, and routes to any configured target label.
| `response_schema` | Yes | — | Inner JSON Schema encoded as a TOML string. Switchyard adds the provider wrapper. |
| `policy` | Yes | — | Policy table. `target_selector` accepts a JSON Pointer such as `/decision/target`. |
| `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. |
| `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. |
| `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
| `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |

Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies
Expand Down