Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions src/AgentCore.Application/Runtime/CallSession.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1055,13 +1055,9 @@ private async Task<TurnResult> CompleteTurnAsync(
_events.RaiseDiagnostic(CallEventKind.EmptyReply, _time.GetUtcNow(), turn.Index);
}

// §7's ask is charged only for a turn whose own words reached the caller. The clarification
// rides an instruction injected before the run, so a run that ended in the fallback reply, or
// one moderation refused, never put the question — and a slot recorded as asked is silent for
// the rest of the call.
if (failure is null && !refused)
{
_clarifications.CommitAsks();
_clarifications.CommitAsks(reply);
}

// What this turn adds to the transcript. It is built here and written at the end of the
Expand Down
13 changes: 7 additions & 6 deletions src/AgentCore.Application/Runtime/ClarificationText.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,21 @@ internal static string Instruction(
{
var confirm = first ? "One thing is not yet confirmed" : "Another thing is not yet confirmed";
return $"{confirm}: {description} Everything found is for {candidates[0]}. "
+ "Ask the caller whether that is what they have before giving advice specific to it.";
+ "Answer what the caller just said; where that calls for advice specific to it, ask the "
+ "caller first whether that is what they have.";
}

var known = first ? "One thing is not yet known" : "Another thing is not yet known";

if (candidates.Count > maxCandidates)
{
return $"{known}: {description} Ask the caller, and do not give advice specific to one until "
+ "they answer.";
return $"{known}: {description} Answer what the caller just said; where that calls for advice "
+ "specific to one, ask the caller first.";
}

return $"{known}: {description} It is one of {JoinAlternatives(candidates)}. Ask the caller, and do "
+ "not give advice that is specific to one until they answer. Anything that applies to all of "
+ "them is still fair game.";
return $"{known}: {description} It is one of {JoinAlternatives(candidates)}. Answer what the caller "
+ "just said; where that calls for advice specific to one, ask the caller which first. Anything "
+ "that applies to all of them is still fair game.";
}

/// <summary>Renders channel 2's sentence: the probe's note, as a search result.</summary>
Expand Down
12 changes: 10 additions & 2 deletions src/AgentCore.Application/Runtime/Clarifications.cs
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,12 @@ internal void Ask(string name, LastNamed named, bool spendsReset)
}
}

/// <summary>Makes every ask this turn staged permanent (see <see cref="Ask"/>).</summary>
internal void CommitAsks()
/// <summary>Makes the asks this turn staged and put permanent (see <see cref="Ask"/>).</summary>
/// <param name="reply">The words the caller heard, which decide which staged asks were put.</param>
internal void CommitAsks(string reply)
{
ArgumentNullException.ThrowIfNull(reply);

lock (_gate)
{
foreach (var state in _slots.Values)
Expand All @@ -113,6 +116,11 @@ internal void CommitAsks()
continue;
}

if (!SpokenAsk.NamedIn(reply, ask.Named))
{
continue;
}

state.LastNamed = ask.Named;
state.NamedAsks = ask.NamedAsks;
state.ResetSpent = ask.ResetSpent;
Expand Down
41 changes: 41 additions & 0 deletions src/AgentCore.Application/Runtime/SpokenAsk.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
using AgentCore.Application.State;

namespace AgentCore.Application.Runtime;

/// <summary>
/// Whether a turn's own reply actually put the question channel 1 staged for it.
/// </summary>
internal static class SpokenAsk
{
/// <summary>Whether <paramref name="reply"/> put the question <paramref name="named"/> staged.</summary>
/// <param name="reply">The words the caller heard.</param>
/// <param name="named">What the staged ask offered.</param>
/// <returns>
/// <see langword="true"/> where the reply names one of the offered values. A record naming no
/// values — the over-cap form, whose instruction lists none — carries nothing to look for, so it
/// counts as asked rather than being dropped on every turn alike.
/// </returns>
internal static bool NamedIn(string reply, Clarifications.LastNamed named)
{
ArgumentNullException.ThrowIfNull(reply);

if (named.Kind != Clarifications.LastNamedKind.Set || named.Values is not { } values)
{
return true;
}

var folded = VocabularyFold.Fold(reply);

foreach (var value in values)
{
var candidate = VocabularyFold.Fold(value);

if (candidate.Length > 0 && folded.Contains(candidate, StringComparison.Ordinal))
{
return true;
}
}

return false;
}
}
101 changes: 85 additions & 16 deletions src/AgentCore.Application/State/ExactStateValueLinker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -81,34 +81,24 @@ public LinkResult Link(string mention, VocabularyView vocabulary, IReadOnlySet<s
return new LinkResult(LinkOutcome.Linked, [original]);
}

var prefixedBy = PrefixedSiblings(vocabulary, only);
var prefixedBy = PrefixedSiblings(vocabulary, only, Extensions(candidates, hits[only]));

if (prefixedBy.Count == 0)
{
return new LinkResult(LinkOutcome.Linked, [original]);
}

// K21's tie-break did not apply, and `only` is a strict prefix of another value in the
// collection: the caller could mean either, so this stays a near-tie rather than a guess.
// The full set is re-sorted rather than just prepending `original` ahead of a sorted
// `prefixedBy`: a mixed-case vocabulary can make `original` compare greater than one of its
// own prefixed siblings under ordinal order (uppercase sorts before lowercase), which a bare
// prepend would leave out of order.
prefixedBy.Add(original);
prefixedBy.Sort(StringComparer.Ordinal);
return new LinkResult(LinkOutcome.Ambiguous, prefixedBy);
}

/// <summary>
/// Every value whose normalised key has <paramref name="key"/> as a strict prefix, in the
/// collection's own spelling.
/// Every value whose normalised key has <paramref name="key"/> as a strict prefix and that
/// <paramref name="extensions"/> does not rule out, in the collection's own spelling.
/// </summary>
/// <remarks>
/// Under ordinal order every strict prefix extension of <paramref name="key"/> sorts after it
/// and the extensions are contiguous, so this is a binary search plus a walk over the matches
/// rather than a scan of the whole vocabulary.
/// </remarks>
private static List<string> PrefixedSiblings(VocabularyView vocabulary, string key)
private static List<string> PrefixedSiblings(
VocabularyView vocabulary, string key, IReadOnlyList<string> extensions)
{
var ordered = KeysInOrder.GetValue(
vocabulary,
Expand All @@ -120,12 +110,91 @@ private static List<string> PrefixedSiblings(VocabularyView vocabulary, string k
List<string> siblings = [];
for (; scan < ordered.Length && ordered[scan].StartsWith(key, StringComparison.Ordinal); scan++)
{
siblings.Add(vocabulary.NormalisedToOriginal[ordered[scan]]);
if (!RuledOut(ordered[scan], key, extensions))
{
siblings.Add(vocabulary.NormalisedToOriginal[ordered[scan]]);
}
}

return siblings;
}

/// <summary>
/// Every candidate spelling the mention offers that begins where the matched value begins and
/// runs past its end — what the caller spelled around the hit, rather than what the collection
/// holds.
/// </summary>
/// <param name="candidates">Every spelling the mention admits, from <see cref="BuildCandidates"/>.</param>
/// <param name="matched">The token spans the matched value was read from.</param>
/// <returns>The extended spellings, each carrying the matched key as a strict prefix.</returns>
private static List<string> Extensions(
Dictionary<string, HashSet<(int Start, int End)>> candidates,
HashSet<(int Start, int End)> matched)
{
List<string> extensions = [];

foreach (var (key, spans) in candidates)
{
if (spans.Any(span => matched.Any(hit => hit.Start == span.Start && span.End > hit.End)))
{
extensions.Add(key);
}
}

return extensions;
}

/// <summary>
/// Whether the caller's own spelling rules this sibling out: they spelled a peer of the
/// sibling's own tail — as long, and of the same character class — that is not it.
/// </summary>
/// <param name="sibling">The sibling's normalised key.</param>
/// <param name="key">The matched value's normalised key, a strict prefix of both sides.</param>
/// <param name="extensions">What the caller spelled past the match.</param>
/// <returns><see langword="true"/> to drop the sibling.</returns>
private static bool RuledOut(string sibling, string key, IReadOnlyList<string> extensions)
{
var tail = sibling[key.Length..];
var ruledOut = false;

foreach (var extension in extensions)
{
// The caller is still spelling toward this sibling, so nothing they said rules it out —
// and no other extension of the same mention can overturn that.
if (sibling.StartsWith(extension, StringComparison.Ordinal))
{
return false;
}

ruledOut = ruledOut || IsPeer(extension[key.Length..], tail);
}

return ruledOut;
}

/// <summary>Whether two tails are alternatives: the same length, and both all numbers or both all letters.</summary>
private static bool IsPeer(string left, string right)
=> left.Length == right.Length
&& ((All(left, IsNumber) && All(right, IsNumber)) || (All(left, Rune.IsLetter) && All(right, Rune.IsLetter)));

private static bool All(string text, Func<Rune, bool> predicate)
{
if (text.Length == 0)
{
return false;
}

foreach (var rune in text.EnumerateRunes())
{
if (!predicate(rune))
{
return false;
}
}

return true;
}

/// <summary>
/// Discards a hit whose every span sits inside a span of some longer hit (K10) — tested per
/// span, so different spans of one hit may be swallowed by different longer hits, and no longer
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
using AgentCore.Application.Configuration.Compilation;
using AgentCore.Application.Configuration.Parsing;
using AgentCore.Application.Configuration.Validation;
using AgentCore.Application.Runtime;
using AgentCore.Application.State;
using AgentCore.Application.Tests.Fakes;
using AgentCore.TestSupport;
using Xunit;

namespace AgentCore.Application.Tests.Runtime;

/// <summary>
/// Channel 1 across a turn the caller spent on something else. The instruction rides every turn
/// after the near-tie, whatever that turn says, so the reply — not the injection — is what charges
/// the ask: a turn answered with sympathy rather than the question must leave the slot free to ask
/// on the next turn that calls for it.
/// </summary>
public sealed class CallSessionOffTopicTurnTests
{
private const string AppliesToDescription = "The model, as printed on the machine.";

private const string Yaml =
"""
apiVersion: agentcore/v1
name: clarification-off-topic
state:
applies_to:
type: string
writer: extractor
description: "The model, as printed on the machine."
vocabulary: { from: knowledge }
extractor:
model: { ref: fill }
when: after_reply
providers:
call: { kind: telnyx-relay }
speech:
stt: { kind: telnyx-relay }
tts: { kind: telnyx-relay }
knowledge:
kind: qdrant
collection: kb
fields: { body: text }
scope:
template: "facets.{key}"
fromState: [applies_to]
wildcard: { value: "*", facets: [applies_to] }
ambiguity: { maxCandidates: 6, maxAsks: 2 }
agents:
items:
- id: only
""";

[Fact]
public async Task AnOffTopicTurn_DoesNotSpendTheQuestion()
{
using SequencedChatClient reply = new(
"I can help with that.",
"""{"applies_to":"the CT900"}""",
"I'm sorry to hear that. What is going wrong?",
"{}",
"Is it the ct900 or the ct900ent?",
"{}",
"Happy to help.",
"{}");

var session = Build(reply);
var instruction = ClarificationText.Instruction(
AppliesToDescription, ["ct900", "ct900ent"], 6, first: true);

await session.RunTurnAsync("I have a CT900 and a CT900ENT", TestContext.Current.CancellationToken);
Assert.Equal(string.Empty, reply.SystemText(0));

await session.RunTurnAsync("i hate you", TestContext.Current.CancellationToken);
Assert.Equal(instruction, reply.SystemText(2));

// The turn answered the caller instead of putting the question, so the ask went uncharged and
// the same instruction rides the next turn rather than the slot falling silent under K37.
await session.RunTurnAsync("which belt does mine take?", TestContext.Current.CancellationToken);
Assert.Equal(instruction, reply.SystemText(4));

// That reply named both candidates, so the ask is charged and K37 suppresses the repeat.
await session.RunTurnAsync("thanks", TestContext.Current.CancellationToken);
Assert.Equal(string.Empty, reply.SystemText(6));
}

private static CallSession Build(SequencedChatClient reply)
{
var chatClients = new FakeChatClientFactory(reply);
var compiled = ConfigurationCompiler.Compile(
ConfigurationLoader.LoadYaml(Yaml), new AgentCompilationContext(chatClients));

var extractor = CallSessionFactory.CreateExtractor(compiled, chatClients);
VocabularyCache vocabulary = new();
vocabulary.Replace("applies_to", ["ct900", "ct900ent"], maxValues: 2000);

return new CallSessionFactory(
compiled, new GuardEvaluator(compiled.Configuration.Guards), extractor, vocabulary: vocabulary)
.Create("call-1");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ public async Task TheUnchangedSet_IsNotAskedAgain_UntilItChanges()
var turn1 = await provider.InvokingAsync(Invoking(session), TestContext.Current.CancellationToken);
Assert.NotNull(turn1.Messages);
Assert.Equal(1, clarifications.Read("applies_to").NamedAsks);
clarifications.CommitAsks();
clarifications.CommitAsks("Is it a ct900 or a ct900ent?");

// Turn 2: the same set. namedAsks is at the cap (1), but K37 suppresses before the cap is
// even reached, because nothing changed.
Expand All @@ -298,15 +298,15 @@ public async Task TheUnchangedSet_IsNotAskedAgain_UntilItChanges()
Assert.Null(turn2.Messages);
Assert.Equal(1, clarifications.Read("applies_to").NamedAsks);
Assert.False(clarifications.Read("applies_to").ResetSpent);
clarifications.CommitAsks();
clarifications.CommitAsks("okay.");

// Turn 3: the set changes. namedAsks is at the cap and differs from lastNamed, so the one
// reset fires: namedAsks returns to 1 (not 0), and the reset is spent.
clarifications.BeginTurn();
clarifications.Update("applies_to", s => s.Pending = ["ct800", "ct800ent"]);
var turn3 = await provider.InvokingAsync(Invoking(session), TestContext.Current.CancellationToken);
Assert.NotNull(turn3.Messages);
clarifications.CommitAsks();
clarifications.CommitAsks("Is it a ct800 or a ct800ent?");
var afterReset = clarifications.Read("applies_to");
Assert.Equal(1, afterReset.NamedAsks);
Assert.True(afterReset.ResetSpent);
Expand Down Expand Up @@ -340,7 +340,7 @@ public async Task TwoDifferentOverCapSets_AreNotBothSpoken()
var turn1 = await provider.InvokingAsync(Invoking(session), TestContext.Current.CancellationToken);
Assert.NotNull(turn1.Messages);
Assert.Equal(1, clarifications.Read("applies_to").NamedAsks);
clarifications.CommitAsks();
clarifications.CommitAsks("Which model is it?");

clarifications.BeginTurn();
clarifications.Update("applies_to", s => s.Pending = ["h", "i", "j", "k", "l", "m", "n"]);
Expand Down
Loading
Loading