feat: voice.plan builtin and SetVoicePlanCommand for GPT-Live

config/example.yaml

@@ -197,6 +197,10 @@ tools:
   # deliverable: the file is copied into providers.blobs.
   - { id: publish, kind: builtin, uses: file.publish }
 
+  # kind: builtin, a plain function for a GPT-Live call. The model sets the plan the voice runs between its
+  # answers; a new plan replaces the last. On any other channel it answers that there is no separate voice.
+  - { id: plan, kind: builtin, uses: voice.plan }
+
   # kind: binding, typed form. Not used in this example. The host registered
   #   options.Bind("CloseCase", (string caseId, string reason) => ...)
   # The method parameters ARE the schema, so write no parameters: here. Writing one is a boot

src/AgentCore.Application/Conversation/Commands/SetVoicePlanCommand.cs

@@ -0,0 +1,34 @@
+namespace AgentCore.Application.Conversation.Commands
+{
+    /// <summary>
+    /// The plan a voice that speaks for itself runs between the backend's answers: the next step, what to ask or watch
+    /// for, and when to delegate again. Each plan replaces the one before it.
+    /// </summary>
+    public sealed record SetVoicePlanCommand : ChannelCommand
+    {
+        /// <summary>
+        /// The longest plan, in characters. One GPT-Live append holds 500 tokens; 1,000 characters leaves room for the
+        /// channel's own line before the plan, even at about 3 characters per token.
+        /// </summary>
+        public const int MaxLength = 1000;
+
+        /// <summary>Keeps the plan.</summary>
+        /// <param name="text">The plan, as the voice should read it.</param>
+        /// <exception cref="ArgumentException">
+        /// <paramref name="text"/> is <see langword="null"/>, empty, white space, or longer than <see cref="MaxLength"/>.
+        /// </exception>
+        public SetVoicePlanCommand(string text)
+        {
+            ArgumentException.ThrowIfNullOrWhiteSpace(text);
+            if (text.Length > MaxLength)
+            {
+                throw new ArgumentException($"The plan is {text.Length} characters. The limit is {MaxLength}.", nameof(text));
+            }
+
+            Text = text;
+        }
+
+        /// <summary>Gets the plan.</summary>
+        public string Text { get; }
+    }
+}

src/AgentCore.Application/PublicAPI.Unshipped.txt

@@ -3181,3 +3181,16 @@ AgentCore.Application.Conversation.StoredConversation.StoredConversation(AgentCo
 AgentCore.Application.Ports.IConversations.LinkFileAsync(string! conversationId, string! name, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask<System.Uri?>
 AgentCore.Application.Blobs.BlobOwner
 static AgentCore.Application.Blobs.BlobOwner.IsSafe(string? ownerId) -> bool
+AgentCore.Application.Conversation.Commands.SetVoicePlanCommand
+AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.SetVoicePlanCommand(string! text) -> void
+AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? other) -> bool
+AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Text.get -> string!
+override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.<Clone>$() -> AgentCore.Application.Conversation.Commands.SetVoicePlanCommand!
+override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(object? obj) -> bool
+override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.GetHashCode() -> int
+override AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.ToString() -> string!
+override sealed AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.Equals(AgentCore.Application.Conversation.Commands.ChannelCommand? other) -> bool
+static AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.operator ==(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? left, AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? right) -> bool
+static AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.operator !=(AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? left, AgentCore.Application.Conversation.Commands.SetVoicePlanCommand? right) -> bool
+const AgentCore.Application.Conversation.Commands.SetVoicePlanCommand.MaxLength = 1000 -> int
+const AgentCore.Application.Tools.Builtin.BuiltinToolNames.VoicePlan = "voice.plan" -> string!

src/AgentCore.Application/Tools/Builtin/BuiltinToolNames.cs

@@ -8,5 +8,8 @@ public static class BuiltinToolNames
 
         /// <summary>Hands one file from the conversation's workspace to the person, through the blob store.</summary>
         public const string FilePublish = "file.publish";
+
+        /// <summary>Sets the plan a voice that speaks for itself runs between answers, such as GPT-Live's.</summary>
+        public const string VoicePlan = "voice.plan";
     }
 }

src/AgentCore.Application/Tools/Builtin/BuiltinToolOptions.cs

@@ -1,3 +1,5 @@
+using System.Reflection;
+using System.Text.Json;
 using AgentCore.Application.Configuration.Schema;
 using AgentCore.Application.Tools.Binding;
 using Microsoft.Extensions.AI;
@@ -12,16 +14,45 @@ internal static class BuiltinToolOptions
         /// <summary>Builds the options every built-in is created with.</summary>
         /// <param name="tool">The declaration the document holds.</param>
         /// <returns>The options: the declared id as the name, the resolved description, no result schema,
-        /// and the turn bound from the conversation rather than filled by the model.</returns>
+        /// the turn bound from the conversation rather than filled by the model, and a required text input the model
+        /// left out bound as <see langword="null"/>.</returns>
         internal static AIFunctionFactoryOptions Options(ToolConfiguration tool)
         {
             return new()
             {
                 Name = tool.Id,
                 Description = tool.Description ?? string.Empty,
                 ExcludeResultSchema = true,
-                ConfigureParameterBinding = ToolParameterBindings.For,
+                ConfigureParameterBinding = Bind,
             };
         }
+
+        private static AIFunctionFactoryOptions.ParameterBindingOptions Bind(ParameterInfo parameter)
+        {
+            AIFunctionFactoryOptions.ParameterBindingOptions runtime = ToolParameterBindings.For(parameter);
+            if (runtime.BindParameter is not null)
+            {
+                return runtime;
+            }
+
+            return parameter.ParameterType == typeof(string) && !parameter.HasDefaultValue
+                ? new() { BindParameter = (bound, arguments) => TextOf(bound.Name!, arguments) }
+                : default;
+        }
+
+        private static string? TextOf(string name, AIFunctionArguments arguments)
+        {
+            return arguments.TryGetValue(name, out object? value)
+                ? value switch
+                {
+                    null => null,
+                    string text => text,
+                    JsonElement { ValueKind: JsonValueKind.String } element => element.GetString(),
+                    JsonElement { ValueKind: JsonValueKind.Null } => null,
+                    JsonElement element => element.GetRawText(),
+                    _ => value.ToString(),
+                }
+                : null;
+        }
     }
 }

src/AgentCore.Application/Tools/Builtin/BuiltinToolSource.cs

@@ -16,6 +16,7 @@ public sealed class BuiltinToolSource : IToolSource
             {
                 new WebSearchToolDefinition(),
                 new FilePublishToolDefinition(),
+                new VoicePlanToolDefinition(),
             }.ToDictionary(definition => definition.Name, StringComparer.Ordinal);
 
         private readonly BuiltinToolPorts _ports;

src/AgentCore.Application/Tools/Builtin/VoicePlanTool.cs

@@ -0,0 +1,60 @@
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Text.Json.Nodes;
+using AgentCore.Application.Configuration.Schema;
+using AgentCore.Application.Conversation.Commands;
+using AgentCore.Application.Runtime.Session;
+using AgentCore.Application.Runtime.Turn;
+using Microsoft.Extensions.AI;
+
+namespace AgentCore.Application.Tools.Builtin
+{
+    /// <summary>The <c>uses: voice.plan</c> function: one <see cref="SetVoicePlanCommand"/> through the turn's channel.</summary>
+    internal sealed class VoicePlanTool(ToolConfiguration tool)
+    {
+        /// <summary>Builds the function the model calls, named after the declaration.</summary>
+        /// <returns>The function.</returns>
+        public AIFunction AsAIFunction()
+        {
+            return AIFunctionFactory.Create(Plan, BuiltinToolOptions.Options(tool));
+        }
+
+        private JsonObject Plan(
+            [Description("The plan for the voice, at most 1,000 characters: the next step, what to ask the caller, and when to delegate again.")] string plan,
+            TurnInvocation? turn = null)
+        {
+            if (string.IsNullOrWhiteSpace(plan))
+            {
+                return Failed("plan is required: the next step for the voice.");
+            }
+
+            if (plan.Length > SetVoicePlanCommand.MaxLength)
+            {
+                return Failed($"The plan is {plan.Length} characters. The limit is {SetVoicePlanCommand.MaxLength}. Shorten it and call {tool.Id} again.");
+            }
+
+            if (turn is null)
+            {
+                return Failed("No conversation is running, so there is no voice to send the plan to.");
+            }
+
+            IChannelControl channel = turn.Channel ?? UnattachedChannel.Instance;
+            return channel.Send(new SetVoicePlanCommand(plan)) switch
+            {
+                ChannelCommandResult.Scheduled => new JsonObject
+                {
+                    ["sent"] = true,
+                    ["message"] = "Sent. The voice follows this plan until you send a new one.",
+                },
+                ChannelCommandResult.NotSupported => Failed("This conversation has no separate voice. The plan was not sent."),
+                ChannelCommandResult.Ending => Failed("The conversation is ending. The plan was not sent."),
+                _ => throw new UnreachableException($"A channel answered {nameof(SetVoicePlanCommand)} with an unknown result."),
+            };
+        }
+
+        private JsonObject Failed(string message)
+        {
+            return ToolErrorResult.Create(tool.Id, message);
+        }
+    }
+}

src/AgentCore.Application/Tools/Builtin/VoicePlanToolDefinition.cs

@@ -0,0 +1,27 @@
+using AgentCore.Application.Configuration.Schema;
+using Microsoft.Extensions.AI;
+
+namespace AgentCore.Application.Tools.Builtin
+{
+    /// <summary>The <c>uses: voice.plan</c> builtin: one <see cref="VoicePlanTool"/>. It needs no port.</summary>
+    internal sealed class VoicePlanToolDefinition : IBuiltinToolDefinition
+    {
+        /// <inheritdoc />
+        public string Name => BuiltinToolNames.VoicePlan;
+
+        /// <inheritdoc />
+        public string DefaultDescription =>
+            "Sets the plan the voice runs between your answers: the next step, what to ask the caller or watch for, "
+            + "and when to delegate to you again. A new plan replaces the last one. "
+            + "Words for the caller go in your reply, never in the plan.";
+
+        /// <inheritdoc />
+        public AITool Build(ToolConfiguration tool, BuiltinToolPorts ports)
+        {
+            ArgumentNullException.ThrowIfNull(tool);
+            ArgumentNullException.ThrowIfNull(ports);
+
+            return new VoicePlanTool(tool).AsAIFunction();
+        }
+    }
+}

src/AgentCore.AspNetCore/Vendors/OpenAiLive/Call/LiveChannel.cs

@@ -9,14 +9,20 @@ namespace AgentCore.AspNetCore.Vendors.OpenAiLive.Call
     /// can send a command from that notice on. A transfer is the <see cref="LiveTransfer"/>'s, once the call runs. Voice
     /// context goes out as silent context at once, which in a probe never made the voice speak or cut its speech
     /// (docs/probes/live-late-brief); context sent before <c>session.started</c> waits for it, as OpenAI's guide sends
-    /// nothing before it.
+    /// nothing before it. A plan goes out the same way, numbered, so a newer plan replaces the older ones.
     /// </summary>
     internal sealed class LiveChannel(string callId, ILogger logger) : IConversationChannel
     {
         private readonly Lock _gate = new();
 
         private readonly List<string> _waiting = [];
 
+        private int _plans;
+
+        // The last send in line. Each send waits for the one before it, so context and plans reach GPT-Live in the
+        // order they were numbered and queued, even when one is sent while earlier ones are still going out.
+        private Task _lastSend = Task.CompletedTask;
+
         private LiveTransfer? _transfer;
 
         private Func<string, CancellationToken, ValueTask>? _tell;
@@ -29,7 +35,8 @@ public ChannelCommandResult Send(ChannelCommand command)
             return command switch
             {
                 TransferCommand transfer => Volatile.Read(ref _transfer)?.Send(transfer) ?? ChannelCommandResult.NotSupported,
-                AddVoiceContextCommand context => Add(context.Text),
+                AddVoiceContextCommand context => Add(context.Text, numbered: false),
+                SetVoicePlanCommand plan => Add(plan.Text, numbered: true),
                 _ => ChannelCommandResult.NotSupported,
             };
         }
@@ -45,38 +52,85 @@ internal void Transfers(LiveTransfer transfer)
         /// <param name="cancellationToken">The call's token. Later context goes out under it too.</param>
         internal async ValueTask OpenAsync(Func<string, CancellationToken, ValueTask> tell, CancellationToken cancellationToken)
         {
-            string[] waiting;
+            List<(string Text, Task Previous, TaskCompletionSource Done)> waiting = [];
+
             lock (_gate)
             {
                 _stopping = cancellationToken;
+
                 _tell = tell;
-                waiting = [.. _waiting];
+
+                foreach (string text in _waiting)
+                {
+                    (Task previous, TaskCompletionSource done) = TakePlaceInLine();
+                    waiting.Add((text, previous, done));
+                }
+
                 _waiting.Clear();
             }
 
-            foreach (string text in waiting)
-            {
-                await tell(text, cancellationToken).ConfigureAwait(false);
-            }
+            await Task.WhenAll(waiting.Select(item => SendInTurnAsync(item.Previous, item.Done, () => tell(item.Text, cancellationToken).AsTask())))
+                .ConfigureAwait(false);
         }
 
-        private ChannelCommandResult Add(string text)
+        private ChannelCommandResult Add(string text, bool numbered)
         {
             Func<string, CancellationToken, ValueTask>? tell;
+
+            Task previous;
+
+            TaskCompletionSource done;
+
             lock (_gate)
             {
+                if (numbered)
+                {
+                    // GPT-Live cannot delete an append; this line is what makes a newer plan replace the older ones.
+                    text = $"Plan {++_plans}. This replaces every earlier plan.\n{text}";
+                }
+
                 tell = _tell;
+
                 if (tell is null)
                 {
                     _waiting.Add(text);
                     return ChannelCommandResult.Scheduled;
                 }
+
+                (previous, done) = TakePlaceInLine();
             }
 
-            _ = TellAsync(tell, text);
+            _ = SendInTurnAsync(previous, done, () => TellAsync(tell, text));
+
             return ChannelCommandResult.Scheduled;
         }
 
+        // Call under _gate. The send itself runs outside the lock.
+        private (Task Previous, TaskCompletionSource Done) TakePlaceInLine()
+        {
+            TaskCompletionSource done = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+            Task previous = _lastSend;
+
+            _lastSend = done.Task;
+
+            return (previous, done);
+        }
+
+        // A place in line only ever completes, never faults, so a failed send never stops the ones after it.
+        private static async Task SendInTurnAsync(Task previous, TaskCompletionSource done, Func<Task> send)
+        {
+            try
+            {
+                await previous.ConfigureAwait(false);
+                await send().ConfigureAwait(false);
+            }
+            finally
+            {
+                done.SetResult();
+            }
+        }
+
         private async Task TellAsync(Func<string, CancellationToken, ValueTask> tell, string text)
         {
             try

tests/AgentCore.Application.Tests/Configuration/ConfigurationLoaderTests.cs

@@ -70,7 +70,7 @@ public void Example_KeepsEveryGuardAsRawJsonLogic()
         [Fact]
         public void Example_BindsEveryToolKind()
         {
-            Assert.Equal(4, Example.Tools.Count);
+            Assert.Equal(5, Example.Tools.Count);
 
             ToolConfiguration binding = Example.Tools[1];
             Assert.Equal(ToolKind.Binding, binding.Kind);
@@ -83,6 +83,9 @@ public void Example_BindsEveryToolKind()
 
             Assert.Equal(ToolKind.Builtin, Example.Tools[3].Kind);
             Assert.Equal("file.publish", Example.Tools[3].Uses);
+
+            Assert.Equal(ToolKind.Builtin, Example.Tools[4].Kind);
+            Assert.Equal("voice.plan", Example.Tools[4].Uses);
         }
 
         [Fact]

tests/AgentCore.Application.Tests/Configuration/ExampleDocument.cs

@@ -94,6 +94,7 @@ internal static class ExampleDocument
               required: [ summary ]
           - { id: search, kind: builtin, uses: web.search }
           - { id: publish, kind: builtin, uses: file.publish }
+          - { id: plan, kind: builtin, uses: voice.plan }
         agents:
           defaults:
             model: { ref: reply, temperature: 0.3 }

tests/AgentCore.Application.Tests/Configuration/ExampleDocumentStateJson.cs

@@ -222,6 +222,11 @@ internal static class ExampleDocumentStateJson
               "id": "publish",
               "kind": "builtin",
               "uses": "file.publish"
+            },
+            {
+              "id": "plan",
+              "kind": "builtin",
+              "uses": "voice.plan"
             }
           ],
         """;