From 7e95aa24ec53ac67e61f352990177f3544c7e70b Mon Sep 17 00:00:00 2001 From: MatthewHsu1 Date: Thu, 8 Oct 2026 01:16:18 -0600 Subject: [PATCH 1/2] feat: voice.plan builtin and SetVoicePlanCommand for GPT-Live - Add SetVoicePlanCommand: the plan a GPT-Live voice runs between the backend's answers. Each plan replaces the one before it. Limit is 1,000 characters. - Add the opt-in voice.plan builtin. It sends the plan through the turn's channel. On a channel with no separate voice, it answers with a tool error. - LiveChannel numbers each plan ("Plan N. This replaces every earlier plan.") and sends context and plans one at a time, in order. - Builtins bind a required text input the model left out as null, so the tool answers with its own tool error instead of a throw. --- config/example.yaml | 4 + .../Commands/SetVoicePlanCommand.cs | 34 ++++ .../PublicAPI.Unshipped.txt | 13 ++ .../Tools/Builtin/BuiltinToolNames.cs | 3 + .../Tools/Builtin/BuiltinToolOptions.cs | 35 +++- .../Tools/Builtin/BuiltinToolSource.cs | 1 + .../Tools/Builtin/VoicePlanTool.cs | 60 +++++++ .../Tools/Builtin/VoicePlanToolDefinition.cs | 27 +++ .../Vendors/OpenAiLive/Call/LiveChannel.cs | 74 ++++++-- .../Configuration/ConfigurationLoaderTests.cs | 5 +- .../Configuration/ExampleDocument.cs | 1 + .../Configuration/ExampleDocumentStateJson.cs | 5 + .../Conversation/SetVoicePlanCommandTests.cs | 35 ++++ .../Tools/FilePublishToolDefinitionTests.cs | 21 +++ .../Tools/VoicePlanToolDefinitionTests.cs | 154 +++++++++++++++++ .../Fakes/GatedToolCallingChatClient.cs | 8 +- .../OpenAiLive/OpenAiLiveVoicePlanTests.cs | 159 ++++++++++++++++++ 17 files changed, 623 insertions(+), 16 deletions(-) create mode 100644 src/AgentCore.Application/Conversation/Commands/SetVoicePlanCommand.cs create mode 100644 src/AgentCore.Application/Tools/Builtin/VoicePlanTool.cs create mode 100644 src/AgentCore.Application/Tools/Builtin/VoicePlanToolDefinition.cs create mode 100644 tests/AgentCore.Application.Tests/Conversation/SetVoicePlanCommandTests.cs create mode 100644 tests/AgentCore.Application.Tests/Tools/VoicePlanToolDefinitionTests.cs create mode 100644 tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs diff --git a/config/example.yaml b/config/example.yaml index 6ed935dd..fccd59cc 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -197,6 +197,10 @@ tools: # deliverable: the file is copied into providers.blobs. - { id: publish, kind: builtin, uses: file.publish } + # kind: builtin, a plain function for a GPT-Live call. The model sets the plan the voice runs between its + # answers; a new plan replaces the last. On any other channel it answers that there is no separate voice. + - { id: plan, kind: builtin, uses: voice.plan } + # kind: binding, typed form. Not used in this example. The host registered # options.Bind("CloseCase", (string caseId, string reason) => ...) # The method parameters ARE the schema, so write no parameters: here. Writing one is a boot diff --git a/src/AgentCore.Application/Conversation/Commands/SetVoicePlanCommand.cs b/src/AgentCore.Application/Conversation/Commands/SetVoicePlanCommand.cs new file mode 100644 index 00000000..dd1f5ddf --- /dev/null +++ b/src/AgentCore.Application/Conversation/Commands/SetVoicePlanCommand.cs @@ -0,0 +1,34 @@ +namespace AgentCore.Application.Conversation.Commands +{ + /// + /// The plan a voice that speaks for itself runs between the backend's answers: the next step, what to ask or watch + /// for, and when to delegate again. Each plan replaces the one before it. + /// + public sealed record SetVoicePlanCommand : ChannelCommand + { + /// + /// The longest plan, in characters. One GPT-Live append holds 500 tokens; 1,000 characters leaves room for the + /// channel's own line before the plan, even at about 3 characters per token. + /// + public const int MaxLength = 1000; + + /// Keeps the plan. + /// The plan, as the voice should read it. + /// + /// is , empty, white space, or longer than . + /// + public SetVoicePlanCommand(string text) + { + ArgumentException.ThrowIfNullOrWhiteSpace(text); + if (text.Length > MaxLength) + { + throw new ArgumentException($"The plan is {text.Length} characters. The limit is {MaxLength}.", nameof(text)); + } + + Text = text; + } + + /// Gets the plan. + public string Text { get; } + } +} diff --git a/src/AgentCore.Application/PublicAPI.Unshipped.txt b/src/AgentCore.Application/PublicAPI.Unshipped.txt index 7fe2746d..ce742257 100644 --- a/src/AgentCore.Application/PublicAPI.Unshipped.txt +++ b/src/AgentCore.Application/PublicAPI.Unshipped.txt @@ -3181,3 +3181,16 @@ AgentCore.Application.Conversation.StoredConversation.StoredConversation(AgentCo AgentCore.Application.Ports.IConversations.LinkFileAsync(string! conversationId, string! name, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask AgentCore.Application.Blobs.BlobOwner static AgentCore.Application.Blobs.BlobOwner.IsSafe(string? ownerId) -> bool +AgentCore.Application.Conversation.Commands.SetVoicePlanCommand +AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.SetVoicePlanCommand(string! text) -> void +AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? other) -> bool +AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Text.get -> string! +override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.$() -> AgentCore.Application.Conversation.Commands.SetVoicePlanCommand! +override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(object? obj) -> bool +override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.GetHashCode() -> int +override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.ToString() -> string! +override sealed AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(AgentCore.Application.Conversation.Commands.ChannelCommand? other) -> bool +static AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.operator ==(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? left, AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? right) -> bool +static AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.operator !=(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? left, AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? right) -> bool +const AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.MaxLength = 1000 -> int +const AgentCore.Application.Tools.Builtin.BuiltinToolNames.VoicePlan = "voice.plan" -> string! diff --git a/src/AgentCore.Application/Tools/Builtin/BuiltinToolNames.cs b/src/AgentCore.Application/Tools/Builtin/BuiltinToolNames.cs index 6b190218..5d466d90 100644 --- a/src/AgentCore.Application/Tools/Builtin/BuiltinToolNames.cs +++ b/src/AgentCore.Application/Tools/Builtin/BuiltinToolNames.cs @@ -8,5 +8,8 @@ public static class BuiltinToolNames /// Hands one file from the conversation's workspace to the person, through the blob store. public const string FilePublish = "file.publish"; + + /// Sets the plan a voice that speaks for itself runs between answers, such as GPT-Live's. + public const string VoicePlan = "voice.plan"; } } diff --git a/src/AgentCore.Application/Tools/Builtin/BuiltinToolOptions.cs b/src/AgentCore.Application/Tools/Builtin/BuiltinToolOptions.cs index f8475519..e899ba82 100644 --- a/src/AgentCore.Application/Tools/Builtin/BuiltinToolOptions.cs +++ b/src/AgentCore.Application/Tools/Builtin/BuiltinToolOptions.cs @@ -1,3 +1,5 @@ +using System.Reflection; +using System.Text.Json; using AgentCore.Application.Configuration.Schema; using AgentCore.Application.Tools.Binding; using Microsoft.Extensions.AI; @@ -12,7 +14,8 @@ internal static class BuiltinToolOptions /// Builds the options every built-in is created with. /// The declaration the document holds. /// The options: the declared id as the name, the resolved description, no result schema, - /// and the turn bound from the conversation rather than filled by the model. + /// the turn bound from the conversation rather than filled by the model, and a required text input the model + /// left out bound as . internal static AIFunctionFactoryOptions Options(ToolConfiguration tool) { return new() @@ -20,8 +23,36 @@ internal static AIFunctionFactoryOptions Options(ToolConfiguration tool) Name = tool.Id, Description = tool.Description ?? string.Empty, ExcludeResultSchema = true, - ConfigureParameterBinding = ToolParameterBindings.For, + ConfigureParameterBinding = Bind, }; } + + private static AIFunctionFactoryOptions.ParameterBindingOptions Bind(ParameterInfo parameter) + { + AIFunctionFactoryOptions.ParameterBindingOptions runtime = ToolParameterBindings.For(parameter); + if (runtime.BindParameter is not null) + { + return runtime; + } + + return parameter.ParameterType == typeof(string) && !parameter.HasDefaultValue + ? new() { BindParameter = (bound, arguments) => TextOf(bound.Name!, arguments) } + : default; + } + + private static string? TextOf(string name, AIFunctionArguments arguments) + { + return arguments.TryGetValue(name, out object? value) + ? value switch + { + null => null, + string text => text, + JsonElement { ValueKind: JsonValueKind.String } element => element.GetString(), + JsonElement { ValueKind: JsonValueKind.Null } => null, + JsonElement element => element.GetRawText(), + _ => value.ToString(), + } + : null; + } } } diff --git a/src/AgentCore.Application/Tools/Builtin/BuiltinToolSource.cs b/src/AgentCore.Application/Tools/Builtin/BuiltinToolSource.cs index 0fb72e14..cc1498ce 100644 --- a/src/AgentCore.Application/Tools/Builtin/BuiltinToolSource.cs +++ b/src/AgentCore.Application/Tools/Builtin/BuiltinToolSource.cs @@ -16,6 +16,7 @@ public sealed class BuiltinToolSource : IToolSource { new WebSearchToolDefinition(), new FilePublishToolDefinition(), + new VoicePlanToolDefinition(), }.ToDictionary(definition => definition.Name, StringComparer.Ordinal); private readonly BuiltinToolPorts _ports; diff --git a/src/AgentCore.Application/Tools/Builtin/VoicePlanTool.cs b/src/AgentCore.Application/Tools/Builtin/VoicePlanTool.cs new file mode 100644 index 00000000..acb1c12c --- /dev/null +++ b/src/AgentCore.Application/Tools/Builtin/VoicePlanTool.cs @@ -0,0 +1,60 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Text.Json.Nodes; +using AgentCore.Application.Configuration.Schema; +using AgentCore.Application.Conversation.Commands; +using AgentCore.Application.Runtime.Session; +using AgentCore.Application.Runtime.Turn; +using Microsoft.Extensions.AI; + +namespace AgentCore.Application.Tools.Builtin +{ + /// The uses: voice.plan function: one through the turn's channel. + internal sealed class VoicePlanTool(ToolConfiguration tool) + { + /// Builds the function the model calls, named after the declaration. + /// The function. + public AIFunction AsAIFunction() + { + return AIFunctionFactory.Create(Plan, BuiltinToolOptions.Options(tool)); + } + + private JsonObject Plan( + [Description("The plan for the voice, at most 1,000 characters: the next step, what to ask the caller, and when to delegate again.")] string plan, + TurnInvocation? turn = null) + { + if (string.IsNullOrWhiteSpace(plan)) + { + return Failed("plan is required: the next step for the voice."); + } + + if (plan.Length > SetVoicePlanCommand.MaxLength) + { + return Failed($"The plan is {plan.Length} characters. The limit is {SetVoicePlanCommand.MaxLength}. Shorten it and call {tool.Id} again."); + } + + if (turn is null) + { + return Failed("No conversation is running, so there is no voice to send the plan to."); + } + + IChannelControl channel = turn.Channel ?? UnattachedChannel.Instance; + return channel.Send(new SetVoicePlanCommand(plan)) switch + { + ChannelCommandResult.Scheduled => new JsonObject + { + ["sent"] = true, + ["message"] = "Sent. The voice follows this plan until you send a new one.", + }, + ChannelCommandResult.NotSupported => Failed("This conversation has no separate voice. The plan was not sent."), + ChannelCommandResult.Ending => Failed("The conversation is ending. The plan was not sent."), + _ => throw new UnreachableException($"A channel answered {nameof(SetVoicePlanCommand)} with an unknown result."), + }; + } + + private JsonObject Failed(string message) + { + return ToolErrorResult.Create(tool.Id, message); + } + } +} diff --git a/src/AgentCore.Application/Tools/Builtin/VoicePlanToolDefinition.cs b/src/AgentCore.Application/Tools/Builtin/VoicePlanToolDefinition.cs new file mode 100644 index 00000000..3935f25b --- /dev/null +++ b/src/AgentCore.Application/Tools/Builtin/VoicePlanToolDefinition.cs @@ -0,0 +1,27 @@ +using AgentCore.Application.Configuration.Schema; +using Microsoft.Extensions.AI; + +namespace AgentCore.Application.Tools.Builtin +{ + /// The uses: voice.plan builtin: one . It needs no port. + internal sealed class VoicePlanToolDefinition : IBuiltinToolDefinition + { + /// + public string Name => BuiltinToolNames.VoicePlan; + + /// + public string DefaultDescription => + "Sets the plan the voice runs between your answers: the next step, what to ask the caller or watch for, " + + "and when to delegate to you again. A new plan replaces the last one. " + + "Words for the caller go in your reply, never in the plan."; + + /// + public AITool Build(ToolConfiguration tool, BuiltinToolPorts ports) + { + ArgumentNullException.ThrowIfNull(tool); + ArgumentNullException.ThrowIfNull(ports); + + return new VoicePlanTool(tool).AsAIFunction(); + } + } +} diff --git a/src/AgentCore.AspNetCore/Vendors/OpenAiLive/Call/LiveChannel.cs b/src/AgentCore.AspNetCore/Vendors/OpenAiLive/Call/LiveChannel.cs index 77781d28..c72c6fc7 100644 --- a/src/AgentCore.AspNetCore/Vendors/OpenAiLive/Call/LiveChannel.cs +++ b/src/AgentCore.AspNetCore/Vendors/OpenAiLive/Call/LiveChannel.cs @@ -9,7 +9,7 @@ namespace AgentCore.AspNetCore.Vendors.OpenAiLive.Call /// can send a command from that notice on. A transfer is the 's, once the call runs. Voice /// context goes out as silent context at once, which in a probe never made the voice speak or cut its speech /// (docs/probes/live-late-brief); context sent before session.started waits for it, as OpenAI's guide sends - /// nothing before it. + /// nothing before it. A plan goes out the same way, numbered, so a newer plan replaces the older ones. /// internal sealed class LiveChannel(string callId, ILogger logger) : IConversationChannel { @@ -17,6 +17,12 @@ internal sealed class LiveChannel(string callId, ILogger logger) : IConversation private readonly List _waiting = []; + private int _plans; + + // The last send in line. Each send waits for the one before it, so context and plans reach GPT-Live in the + // order they were numbered and queued, even when one is sent while earlier ones are still going out. + private Task _lastSend = Task.CompletedTask; + private LiveTransfer? _transfer; private Func? _tell; @@ -29,7 +35,8 @@ public ChannelCommandResult Send(ChannelCommand command) return command switch { TransferCommand transfer => Volatile.Read(ref _transfer)?.Send(transfer) ?? ChannelCommandResult.NotSupported, - AddVoiceContextCommand context => Add(context.Text), + AddVoiceContextCommand context => Add(context.Text, numbered: false), + SetVoicePlanCommand plan => Add(plan.Text, numbered: true), _ => ChannelCommandResult.NotSupported, }; } @@ -45,38 +52,85 @@ internal void Transfers(LiveTransfer transfer) /// The call's token. Later context goes out under it too. internal async ValueTask OpenAsync(Func tell, CancellationToken cancellationToken) { - string[] waiting; + List<(string Text, Task Previous, TaskCompletionSource Done)> waiting = []; + lock (_gate) { _stopping = cancellationToken; + _tell = tell; - waiting = [.. _waiting]; + + foreach (string text in _waiting) + { + (Task previous, TaskCompletionSource done) = TakePlaceInLine(); + waiting.Add((text, previous, done)); + } + _waiting.Clear(); } - foreach (string text in waiting) - { - await tell(text, cancellationToken).ConfigureAwait(false); - } + await Task.WhenAll(waiting.Select(item => SendInTurnAsync(item.Previous, item.Done, () => tell(item.Text, cancellationToken).AsTask()))) + .ConfigureAwait(false); } - private ChannelCommandResult Add(string text) + private ChannelCommandResult Add(string text, bool numbered) { Func? tell; + + Task previous; + + TaskCompletionSource done; + lock (_gate) { + if (numbered) + { + // GPT-Live cannot delete an append; this line is what makes a newer plan replace the older ones. + text = $"Plan {++_plans}. This replaces every earlier plan.\n{text}"; + } + tell = _tell; + if (tell is null) { _waiting.Add(text); return ChannelCommandResult.Scheduled; } + + (previous, done) = TakePlaceInLine(); } - _ = TellAsync(tell, text); + _ = SendInTurnAsync(previous, done, () => TellAsync(tell, text)); + return ChannelCommandResult.Scheduled; } + // Call under _gate. The send itself runs outside the lock. + private (Task Previous, TaskCompletionSource Done) TakePlaceInLine() + { + TaskCompletionSource done = new(TaskCreationOptions.RunContinuationsAsynchronously); + + Task previous = _lastSend; + + _lastSend = done.Task; + + return (previous, done); + } + + // A place in line only ever completes, never faults, so a failed send never stops the ones after it. + private static async Task SendInTurnAsync(Task previous, TaskCompletionSource done, Func send) + { + try + { + await previous.ConfigureAwait(false); + await send().ConfigureAwait(false); + } + finally + { + done.SetResult(); + } + } + private async Task TellAsync(Func tell, string text) { try diff --git a/tests/AgentCore.Application.Tests/Configuration/ConfigurationLoaderTests.cs b/tests/AgentCore.Application.Tests/Configuration/ConfigurationLoaderTests.cs index ee498585..c9d32e5b 100644 --- a/tests/AgentCore.Application.Tests/Configuration/ConfigurationLoaderTests.cs +++ b/tests/AgentCore.Application.Tests/Configuration/ConfigurationLoaderTests.cs @@ -70,7 +70,7 @@ public void Example_KeepsEveryGuardAsRawJsonLogic() [Fact] public void Example_BindsEveryToolKind() { - Assert.Equal(4, Example.Tools.Count); + Assert.Equal(5, Example.Tools.Count); ToolConfiguration binding = Example.Tools[1]; Assert.Equal(ToolKind.Binding, binding.Kind); @@ -83,6 +83,9 @@ public void Example_BindsEveryToolKind() Assert.Equal(ToolKind.Builtin, Example.Tools[3].Kind); Assert.Equal("file.publish", Example.Tools[3].Uses); + + Assert.Equal(ToolKind.Builtin, Example.Tools[4].Kind); + Assert.Equal("voice.plan", Example.Tools[4].Uses); } [Fact] diff --git a/tests/AgentCore.Application.Tests/Configuration/ExampleDocument.cs b/tests/AgentCore.Application.Tests/Configuration/ExampleDocument.cs index ddb56c54..00c9184c 100644 --- a/tests/AgentCore.Application.Tests/Configuration/ExampleDocument.cs +++ b/tests/AgentCore.Application.Tests/Configuration/ExampleDocument.cs @@ -94,6 +94,7 @@ internal static class ExampleDocument required: [ summary ] - { id: search, kind: builtin, uses: web.search } - { id: publish, kind: builtin, uses: file.publish } + - { id: plan, kind: builtin, uses: voice.plan } agents: defaults: model: { ref: reply, temperature: 0.3 } diff --git a/tests/AgentCore.Application.Tests/Configuration/ExampleDocumentStateJson.cs b/tests/AgentCore.Application.Tests/Configuration/ExampleDocumentStateJson.cs index 288a21f1..6afb58b2 100644 --- a/tests/AgentCore.Application.Tests/Configuration/ExampleDocumentStateJson.cs +++ b/tests/AgentCore.Application.Tests/Configuration/ExampleDocumentStateJson.cs @@ -222,6 +222,11 @@ internal static class ExampleDocumentStateJson "id": "publish", "kind": "builtin", "uses": "file.publish" + }, + { + "id": "plan", + "kind": "builtin", + "uses": "voice.plan" } ], """; diff --git a/tests/AgentCore.Application.Tests/Conversation/SetVoicePlanCommandTests.cs b/tests/AgentCore.Application.Tests/Conversation/SetVoicePlanCommandTests.cs new file mode 100644 index 00000000..ca5923f1 --- /dev/null +++ b/tests/AgentCore.Application.Tests/Conversation/SetVoicePlanCommandTests.cs @@ -0,0 +1,35 @@ +using AgentCore.Application.Conversation.Commands; +using Xunit; + +namespace AgentCore.Application.Tests.Conversation +{ + /// + /// A plan replaces the plan before it, so it is refused whole rather than cut: a cut plan loses its end, which + /// usually says when to delegate again. One GPT-Live append holds 500 tokens. + /// + public sealed class SetVoicePlanCommandTests + { + [Fact] + public void APlanAtTheLimitIsKeptWhole() + { + string plan = new('a', 1000); + + Assert.Equal(plan, new SetVoicePlanCommand(plan).Text); + } + + [Fact] + public void APlanOverTheLimitIsRefusedWithItsLengthAndTheLimit() + { + ArgumentException refused = Assert.Throws(() => new SetVoicePlanCommand(new string('a', 1001))); + + Assert.Contains("1001", refused.Message, StringComparison.Ordinal); + Assert.Contains("1000", refused.Message, StringComparison.Ordinal); + } + + [Fact] + public void AWhiteSpacePlanIsRefused() + { + _ = Assert.Throws(() => new SetVoicePlanCommand(" ")); + } + } +} diff --git a/tests/AgentCore.Application.Tests/Tools/FilePublishToolDefinitionTests.cs b/tests/AgentCore.Application.Tests/Tools/FilePublishToolDefinitionTests.cs index cc53bc4b..bacba9bd 100644 --- a/tests/AgentCore.Application.Tests/Tools/FilePublishToolDefinitionTests.cs +++ b/tests/AgentCore.Application.Tests/Tools/FilePublishToolDefinitionTests.cs @@ -179,6 +179,27 @@ public async Task Publish_AFileInASubfolder_IsStoredUnderItsOwnName() Assert.True(_blobs.Blobs.ContainsKey(("conversation-1", "chart.png"))); } + [Fact] + public async Task Publish_NoPathArgument_IsRefusedWithItsOwnToolErrorNotAThrow() + { + AIFunction function = (AIFunction)Build(); + AIFunctionArguments arguments = new(); + _ = new TurnInvocation + { + ConversationId = "conversation-1", + TurnIndex = 0, + Stage = string.Empty, + Workspace = Path.Combine(_root, "conversation-1"), + OuterCallId = "tc_1", + }.FileIn(arguments); + + JsonObject result = Assert.IsType( + ToolResultJson.ToNode(await function.InvokeAsync(arguments, TestContext.Current.CancellationToken))); + + Assert.True(ToolErrorResult.IsError(result)); + Assert.Equal("path is required: the file's path, relative to the workspace.", result["message"]!.GetValue()); + } + [Fact] public async Task Publish_OutsideAnyTurn_IsRefusedWithAToolError() { diff --git a/tests/AgentCore.Application.Tests/Tools/VoicePlanToolDefinitionTests.cs b/tests/AgentCore.Application.Tests/Tools/VoicePlanToolDefinitionTests.cs new file mode 100644 index 00000000..75c2b30b --- /dev/null +++ b/tests/AgentCore.Application.Tests/Tools/VoicePlanToolDefinitionTests.cs @@ -0,0 +1,154 @@ +using System.Text.Json.Nodes; +using AgentCore.Application.Configuration.Schema; +using AgentCore.Application.Conversation.Commands; +using AgentCore.Application.Runtime.Turn; +using AgentCore.Application.Tools; +using AgentCore.Application.Tools.Builtin; +using Microsoft.Extensions.AI; +using Xunit; + +namespace AgentCore.Application.Tests.Tools +{ + /// + /// The uses: voice.plan builtin: it checks the plan once, sends it through the turn's channel, and tells the + /// model whether the voice got it. It returns an error result and never throws. + /// + public sealed class VoicePlanToolDefinitionTests + { + private static readonly ToolConfiguration Declared = + new() { Id = "plan", Kind = ToolKind.Builtin, Uses = BuiltinToolNames.VoicePlan, Description = "Set the voice's plan." }; + + [Fact] + public void TheModelSeesPlanAsItsOneRequiredInput() + { + AIFunction function = (AIFunction)new VoicePlanToolDefinition().Build(Declared, new BuiltinToolPorts(ChatClients: null)); + + Assert.Equal(["plan"], function.JsonSchema.GetProperty("properties").EnumerateObject().Select(property => property.Name)); + Assert.Equal(["plan"], function.JsonSchema.GetProperty("required").EnumerateArray().Select(name => name.GetString())); + } + + [Fact] + public async Task AMissingPlanIsItsOwnErrorResultNotAThrow() + { + RecordingChannel channel = new(ChannelCommandResult.Scheduled); + + JsonObject result = await PlanAsync(plan: null, channel); + + Assert.Equal("plan is required: the next step for the voice.", Message(result)); + Assert.Empty(channel.Seen); + } + + [Fact] + public async Task APlanTheChannelTakesIsSentWhole() + { + RecordingChannel channel = new(ChannelCommandResult.Scheduled); + + JsonObject result = await PlanAsync("Next: ask for the serial number.", channel); + + Assert.True((bool?)result["sent"]); + SetVoicePlanCommand sent = Assert.IsType(Assert.Single(channel.Seen)); + Assert.Equal("Next: ask for the serial number.", sent.Text); + } + + [Fact] + public async Task APlanAtTheLimitIsSent() + { + RecordingChannel channel = new(ChannelCommandResult.Scheduled); + + JsonObject result = await PlanAsync(new string('a', SetVoicePlanCommand.MaxLength), channel); + + Assert.True((bool?)result["sent"]); + } + + [Fact] + public async Task APlanOverTheLimitTellsTheModelToShortenItAndSendsNothing() + { + RecordingChannel channel = new(ChannelCommandResult.Scheduled); + + JsonObject result = await PlanAsync(new string('a', 1820), channel); + + Assert.Equal("The plan is 1820 characters. The limit is 1000. Shorten it and call plan again.", Message(result)); + Assert.Empty(channel.Seen); + } + + [Fact] + public async Task AWhiteSpacePlanIsAnErrorResultNotAThrow() + { + RecordingChannel channel = new(ChannelCommandResult.Scheduled); + + JsonObject result = await PlanAsync(" ", channel); + + Assert.Equal("plan is required: the next step for the voice.", Message(result)); + Assert.Empty(channel.Seen); + } + + [Fact] + public async Task AChannelWithNoVoiceSaysThePlanWasNotSent() + { + JsonObject result = await PlanAsync("Next: ask for the model.", new RecordingChannel(ChannelCommandResult.NotSupported)); + + Assert.Equal("This conversation has no separate voice. The plan was not sent.", Message(result)); + } + + [Fact] + public async Task AnEndingConversationSaysThePlanWasNotSent() + { + JsonObject result = await PlanAsync("Next: ask for the model.", new RecordingChannel(ChannelCommandResult.Ending)); + + Assert.Equal("The conversation is ending. The plan was not sent.", Message(result)); + } + + [Fact] + public async Task ATurnWithNoChannelSaysThereIsNoSeparateVoice() + { + JsonObject result = await PlanAsync("Next: ask for the model.", channel: null); + + Assert.Equal("This conversation has no separate voice. The plan was not sent.", Message(result)); + } + + [Fact] + public async Task NoConversationRunningSaysThereIsNoVoice() + { + JsonObject result = await PlanAsync("Next: ask for the model.", channel: null, turn: false); + + Assert.Equal("No conversation is running, so there is no voice to send the plan to.", Message(result)); + } + + private static string? Message(JsonObject result) + { + Assert.True((bool?)result[ToolErrorResult.ErrorProperty]); + return (string?)result[ToolErrorResult.MessageProperty]; + } + + private static async Task PlanAsync(string? plan, IChannelControl? channel, bool turn = true) + { + AIFunction function = (AIFunction)new VoicePlanToolDefinition().Build(Declared, new BuiltinToolPorts(ChatClients: null)); + AIFunctionArguments arguments = []; + if (plan is not null) + { + arguments["plan"] = plan; + } + + if (turn) + { + _ = new TurnInvocation { ConversationId = "conversation-1", TurnIndex = 0, Stage = string.Empty, Channel = channel } + .FileIn(arguments); + } + + object? result = await function.InvokeAsync(arguments, TestContext.Current.CancellationToken); + + return Assert.IsType(ToolResultJson.ToNode(result)); + } + + private sealed class RecordingChannel(ChannelCommandResult answer) : IChannelControl + { + public List Seen { get; } = []; + + public ChannelCommandResult Send(ChannelCommand command) + { + Seen.Add(command); + return answer; + } + } + } +} diff --git a/tests/AgentCore.AspNetCore.Tests/Fakes/GatedToolCallingChatClient.cs b/tests/AgentCore.AspNetCore.Tests/Fakes/GatedToolCallingChatClient.cs index 394bcba1..56960ff0 100644 --- a/tests/AgentCore.AspNetCore.Tests/Fakes/GatedToolCallingChatClient.cs +++ b/tests/AgentCore.AspNetCore.Tests/Fakes/GatedToolCallingChatClient.cs @@ -2,9 +2,11 @@ namespace AgentCore.AspNetCore.Tests.Fakes { - /// Calls the first tool it is offered, once, then answers in words. - internal sealed class GatedToolCallingChatClient : IChatClient + /// Calls the first tool it is offered, once, with , then answers in words. + internal sealed class GatedToolCallingChatClient(IReadOnlyDictionary? arguments = null) : IChatClient { + private static readonly Dictionary DefaultArguments = new(StringComparer.Ordinal) { ["to"] = "a@b.com" }; + private int _requests; /// Gets a task that completes once a second request reached the model. @@ -31,7 +33,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( [new FunctionCallContent( "conversation_1", tool.Name, - new Dictionary(StringComparer.Ordinal) { ["to"] = "a@b.com" })]); + new Dictionary(arguments ?? DefaultArguments, StringComparer.Ordinal))]); yield break; } diff --git a/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs b/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs new file mode 100644 index 00000000..35d58ff2 --- /dev/null +++ b/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs @@ -0,0 +1,159 @@ +using System.Text.Json.Nodes; +using AgentCore.Application.Conversation.Commands; +using AgentCore.Application.Tools.Builtin; +using AgentCore.AspNetCore.Tests.Fakes; +using AgentCore.AspNetCore.Vendors.OpenAiLive.Wire; +using Xunit; + +namespace AgentCore.AspNetCore.Tests.Vendors.OpenAiLive +{ + /// + /// The backend steers GPT-Live with plans: silent context that names the next step. GPT-Live cannot delete an + /// append, so each plan is numbered and says it replaces the earlier ones (docs/probes/live-roadmap: no stale step + /// in 20 calls with that line). + /// + public sealed class OpenAiLiveVoicePlanTests + { + private const string First = "Next: ask which treadmill model they have.\n\"Then\" ask for the serial number."; + + private const string Second = "Next: ask whether they ever lubricated the belt."; + + private const string Third = "Next: ask which day this week suits them for a technician visit."; + + private const string PlanYaml = """ + apiVersion: agentcore/v1 + tools: + - { id: plan, kind: builtin, uses: voice.plan, description: "Set the voice's plan." } + agents: + defaults: { clock: false } + items: + - { id: only, instructions: "answer the caller", tools: [ plan ] } + entries: + main: + agent: only + """; + + [Fact(Timeout = 30_000)] + public async Task APlanGoesOutAsSilentSessionContextUnderItsNumber() + { + await using RunningLiveCall running = await RunningLiveCall.StartAsync(new StallOnCueChatClient(), []); + _ = await running.Sideband.WaitForSentAsync(IsGreeting); + + ChannelCommandResult result = running.Call.Session.Send(new SetVoicePlanCommand(First)); + JsonObject plan = await running.Sideband.WaitForSentAsync(IsPlan); + + Assert.Equal(ChannelCommandResult.Scheduled, result); + Assert.Null(plan["delegation_id"]); + Assert.Equal("Plan 1. This replaces every earlier plan.\n" + First, (string?)plan["content"]); + } + + [Fact(Timeout = 30_000)] + public async Task EachPlanTakesTheNextNumber() + { + await using RunningLiveCall running = await RunningLiveCall.StartAsync(new StallOnCueChatClient(), []); + _ = await running.Sideband.WaitForSentAsync(IsGreeting); + + _ = running.Call.Session.Send(new SetVoicePlanCommand(First)); + _ = running.Call.Session.Send(new SetVoicePlanCommand(Second)); + await running.Sideband.WaitForDrainAsync(); + + Assert.Equal( + ["Plan 1. This replaces every earlier plan.\n" + First, "Plan 2. This replaces every earlier plan.\n" + Second], + PlansSent(running)); + } + + // The guide sends nothing before session.started; plans sent earlier keep their order and numbers. + [Fact(Timeout = 30_000)] + public async Task PlansSentBeforeTheSessionStartedWaitThenGoOutInOrder() + { + await using RunningLiveCall running = await RunningLiveCall.StartAsync(new StallOnCueChatClient(), [], started: false); + _ = running.Call.Session.Send(new SetVoicePlanCommand(First)); + _ = running.Call.Session.Send(new SetVoicePlanCommand(Second)); + await running.Sideband.WaitForDrainAsync(); + bool sentBeforeTheStart = running.Sideband.Sent.Count > 0; + + running.Sideband.Push([RunningLiveCall.Started]); + _ = await running.Sideband.WaitForSentAsync(IsGreeting); + + Assert.False(sentBeforeTheStart); + Assert.Equal( + ["Plan 1. This replaces every earlier plan.\n" + First, "Plan 2. This replaces every earlier plan.\n" + Second], + PlansSent(running)); + } + + // A plan sent while the waiting plans are still going out must queue behind them: if Plan 3 went out before + // Plan 2, the voice would read "Plan 2. This replaces every earlier plan." last and follow a stale plan. + [Fact(Timeout = 30_000)] + public async Task APlanSentWhileWaitingPlansGoOutQueuesBehindThem() + { + TaskCompletionSource socket = new(TaskCreationOptions.RunContinuationsAsynchronously); + FakeSideband sideband = new() { SendsHeldUntil = socket.Task }; + await using RunningLiveCall running = await RunningLiveCall.StartAsync(new StallOnCueChatClient(), [], sideband: sideband, started: false); + _ = running.Call.Session.Send(new SetVoicePlanCommand(First)); + _ = running.Call.Session.Send(new SetVoicePlanCommand(Second)); + + running.Sideband.Push([RunningLiveCall.Started]); + _ = await running.Sideband.WaitForSentAsync(IsPlan); + _ = running.Call.Session.Send(new SetVoicePlanCommand(Third)); + socket.SetResult(); + _ = await running.Sideband.WaitForSentAsync(IsGreeting); + await running.Sideband.WaitForDrainAsync(); + + Assert.Equal( + [ + "Plan 1. This replaces every earlier plan.\n" + First, + "Plan 2. This replaces every earlier plan.\n" + Second, + "Plan 3. This replaces every earlier plan.\n" + Third, + ], + PlansSent(running)); + } + + // The plan goes out while the backend runs and the answer after it, so the voice reads the plan first + // (spec "Flow"). The probe sent the plan 0.2 s before the commentary. + [Fact(Timeout = 30_000)] + public async Task ABackendThatCallsVoicePlanReachesTheVoiceBeforeItsAnswer() + { + GatedToolCallingChatClient model = new(new Dictionary(StringComparer.Ordinal) { ["plan"] = First }); + await using RunningLiveCall running = await RunningLiveCall.StartAsync( + model, [], yaml: PlanYaml, tools: tool => new VoicePlanToolDefinition().Build(tool, new BuiltinToolPorts(ChatClients: null))); + _ = await running.Sideband.WaitForSentAsync(IsGreeting); + + running.Sideband.Push(Ask("item_1", "My treadmill belt keeps slipping.", 1000)); + _ = await running.Sideband.WaitForSentAsync(IsCommentary); + + List sent = [.. running.Sideband.Sent]; + int plan = sent.FindIndex(IsPlan); + Assert.Equal("Plan 1. This replaces every earlier plan.\n" + First, (string?)sent[plan]["content"]); + Assert.InRange(plan, 0, sent.FindIndex(IsCommentary) - 1); + } + + private static List PlansSent(RunningLiveCall running) + { + return [.. running.Sideband.Sent.Where(IsPlan).Select(sent => (string?)sent["content"])]; + } + + private static bool IsGreeting(JsonObject sent) + { + return (string?)sent["content"] == RunningLiveCall.Greeting; + } + + private static bool IsCommentary(JsonObject sent) + { + return (string?)sent["type"] == "session.commentary.append" && (string?)sent["delegation_id"] == "item_1"; + } + + private static string[] Ask(string delegationId, string words, int startMs) + { + return [ + new JsonObject { ["type"] = OpenAiLiveEvents.InputTranscriptDelta, ["delta"] = " " + words, ["start_ms"] = startMs, ["end_ms"] = startMs + 1000 }.ToJsonString(), + new JsonObject { ["type"] = OpenAiLiveEvents.DelegationCreated, ["delegation"] = new JsonObject { ["id"] = delegationId } }.ToJsonString(), + ]; + } + + private static bool IsPlan(JsonObject sent) + { + return (string?)sent["type"] == "session.thinking.append" + && ((string?)sent["content"])?.StartsWith("Plan ", StringComparison.Ordinal) == true; + } + } +} From ecebc32d6e93c4af3793e5e429875e67ac4aac02 Mon Sep 17 00:00:00 2001 From: MatthewHsu1 Date: Thu, 8 Oct 2026 01:22:54 -0600 Subject: [PATCH 2/2] test: wait for the third plan before checking the plan order The test waited for the greeting, which does not wait for the plan line. Plan 3 could still be in line when the test read the sent list. --- .../Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs b/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs index 35d58ff2..73d0ad56 100644 --- a/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs +++ b/tests/AgentCore.AspNetCore.Tests/Vendors/OpenAiLive/OpenAiLiveVoicePlanTests.cs @@ -96,8 +96,7 @@ public async Task APlanSentWhileWaitingPlansGoOutQueuesBehindThem() _ = await running.Sideband.WaitForSentAsync(IsPlan); _ = running.Call.Session.Send(new SetVoicePlanCommand(Third)); socket.SetResult(); - _ = await running.Sideband.WaitForSentAsync(IsGreeting); - await running.Sideband.WaitForDrainAsync(); + _ = await running.Sideband.WaitForSentAsync(sent => IsPlan(sent) && ((string?)sent["content"])!.StartsWith("Plan 3.", StringComparison.Ordinal)); Assert.Equal( [