fix(agent): draft screening answers in chunks under the worker's own deadline (#410) — v1.68.7

.env.production.example

@@ -1,7 +1,7 @@
 # Production deployments intentionally have no working secret defaults.
 # Copy this file to .env.production, replace every CHANGE-ME value, keep the
 # result out of version control, and pin a released OSApplyTrack version.
-OSAPPLYTRACK_VERSION=1.68.5
+OSAPPLYTRACK_VERSION=1.68.7
 POSTGRES_USER=applytrack
 POSTGRES_PASSWORD=CHANGE-ME-use-a-long-random-password
 POSTGRES_DB=applytrack
@@ -53,3 +53,6 @@ APPLYTRACK_SECRETS_KEY=CHANGE-ME-use-an-independent-long-random-key
 # enabled in Settings · Agent). The browser + proxy containers are on by default
 # in this stack; the agent reaches them at ws://browser:3000/ and http://proxy:3128.
 # AGENT_INTERVAL=300
+# The agent's own deadline per model completion (the api keeps Llm__TimeoutSeconds=120
+# for requests). Screening questions are drafted 8 per completion, each under this.
+# AGENT_LLM_TIMEOUT=300

BACKLOG.md

@@ -164,7 +164,7 @@ review items and unsupported forms still need human attention.
 - [x] [#401 — Mobile: Errors and Due tables overflow or crush on a phone — Retry cut off](https://github.com/CryptoJones/OSApplyTrack/issues/401) (fixed in 1.66.1)
 - [ ] [#398 — Follow-ups: opt-in to let the agent send a drafted follow-up without review](https://github.com/CryptoJones/OSApplyTrack/issues/398)
 - [x] [#408 — Agent: LinkedIn SDUI Easy Apply — the contact step's City typeahead is still left unmapped (SMX)](https://github.com/CryptoJones/OSApplyTrack/issues/408) (fixed in 1.67.0: the "City" was a Work-experience role editor LinkedIn opens on Next, not the contact City; the driver now answers what Next brings up, names each role's fields, and fills Description from the résumé. Found re-running the Ready lane after #406: `CityLabel()` matches and the résumé location is typed, but the typeahead is never committed on the SDUI dialog; the run stops at the contact step with `City` the only unmapped field)
-- [ ] [#410 — Agent: a packet's whole answer draft rides one LLM completion under the API's 120 s timeout — Orbis's 29-question form timed out twice and every answer was voided](https://github.com/CryptoJones/OSApplyTrack/issues/410) (not the network or the endpoint — a 36 KB request from the agent's own netns answers in 1.5 s; the real 22-question prompt just takes poolside longer than two minutes. Mitigated on pluto with `Llm__TimeoutSeconds=600` on the agent; the fix is a worker-side ceiling, chunked drafting, or both)
+- [x] [#410 — Agent: a packet's whole answer draft rides one LLM completion under the API's 120 s timeout — Orbis's 29-question form timed out twice and every answer was voided](https://github.com/CryptoJones/OSApplyTrack/issues/410) (fixed in 1.68.7: screening questions go to the model eight per completion, so a completion that stalls or errors voids only its own chunk and the chunks already drafted are kept; two failed completions in a row stop the draft and hand the rest to the person instead of waiting out another deadline per chunk. The worker's model calls run under their own `Agent__LlmTimeoutSeconds` (default 300; 0 inherits) rather than the api's request-sized `Llm__TimeoutSeconds`, so pluto's `Llm__TimeoutSeconds=600` override on the agent can come off. No `max_tokens` on the drafting call: a thinking model that writes its scratchpad into `content` would be cut off before the JSON, and the completer already tolerates that scratchpad)
 - [x] [#409 — Agent: LinkedIn SDUI Easy Apply — Trility's dialog never reaches Submit within 12 steps](https://github.com/CryptoJones/OSApplyTrack/issues/409) (fixed in 1.67.0: the dialog really is 14 pages; the limit is 40 and a Next that changes nothing twice fails naming the page. Was: `mapped: 0, unmapped: []` — the driver presses Next twelve times on a step it cannot leave and reports nothing about it; needs the screenshot and the live dialog)
 - [x] [#431 — Agent: a checkbox group is discovered as one Yes/No question per option (Fieldwire, Stripe) — fixed in 1.67.0: one multiselect titled by its legend; old split packets still fill](https://github.com/CryptoJones/OSApplyTrack/issues/431)
 - [x] [#432 — Agent: a react-select question is discovered with no options, so drafted prose matches no choice — fixed in 1.67.0: menus are opened and read; prose is fitted to the option it means](https://github.com/CryptoJones/OSApplyTrack/issues/432)

README.md

@@ -258,6 +258,7 @@ All configuration is environment variables (see [`.env.example`](./.env.example)
 | `APPLYTRACK_SECRETS_KEY_FILE` | `/var/lib/applytrack/secrets.key` in a container, else `~/.local/share/applytrack/secrets.key` | Where the generated key is kept when `APPLYTRACK_SECRETS_KEY` is unset. The compose files and quadlets mount a `secrets` volume there, shared by the api and agent containers and mounted read-only into the poller, which reads the key but never generates one. |
 | `APPLYTRACK_SECRETS_KEY_PREVIOUS` | _(empty)_ | The old key(s), `;`-separated, during a [key rotation](#encryption-at-rest): decrypt-only; the startup sweep re-keys every row to the current key. Remove once the sweep has run. |
 | `Agent__Enabled` / `Agent__IntervalSeconds` | `false` / `300` | Turns a container from this image into the **agent worker** (see [The agent](#the-agent)). The compose files run one as the `agent` service; the API container itself leaves it off. Per tenant the agent is still off until enabled in **Settings · Agent**. |
+| `Agent__LlmTimeoutSeconds` | `300` | The worker's own deadline for one model completion. `Llm__TimeoutSeconds` (120) guards a request a person is waiting on; the worker is a background job, and a packet's drafting call used to die at exactly that deadline. Screening questions are drafted in chunks of 8, each under this budget, so one stalled completion voids one chunk, not the packet. `0` inherits `Llm__TimeoutSeconds`. Set on the `agent` service. |
 | `Browser__Endpoint` / `Browser__Proxy` | _(empty)_ | The Playwright server the agent drives (`ws://browser:3000/`) and the forward proxy it must use (`http://proxy:3128`). Unset = no browser: packets are still prepared and you apply via copy-and-open. Set on the `agent` service only: the worker stamps a heartbeat row (`agent_workers`) saying it has a browser, and the api lets Submit and packet/prepare queue a run while that heartbeat is fresh (3 min). See [The agent](#the-agent), step 4. |
 | `AGENT_DB_PASSWORD` | _(required in production)_ | Set on the **api** (the migrating container): it creates the least-privilege `applytrack_agent` Postgres role on boot, or resets its password to this, and re-applies its grants. The `agent` container connects as it with `Migrations__Mode=wait` (it cannot migrate, so it waits for the api to). |
 | `POLLER_DB_PASSWORD` | _(required in production)_ | The same for the poller's `applytrack_poller` role, which the `poller` container's `DATABASE_URL` connects as: it reads profiles, blacklists and the dedup ledger, stages leads, and keeps board sessions, with no access to sessions, sign-in tokens, résumés, board passwords or DDL. Unset, the role is not created and the poller keeps whatever `DATABASE_URL` says (the quickstart uses the owner). |
@@ -991,6 +992,12 @@ published port — the compose files start it as the `agent` service, and
 database connection across multi-minute model calls, which is why it is a
 separate container with its own small pool rather than a thread in the API. Two
 containers now migrate on boot; the migrator serializes on an advisory lock.
+Its model calls run under their own deadline, `Agent__LlmTimeoutSeconds` (300),
+not the api's request-sized `Llm__TimeoutSeconds`; a form's screening questions
+are drafted eight per completion, so a completion that stalls costs that chunk
+and the rest of the packet is still drafted. Two stalls in a row stop the draft
+and hand the remaining questions to you rather than wait out another deadline
+per chunk.
 
 ## Security & hardening
 

api/ApplyTrack.Api.Tests/AgentLlmTimeoutTests.cs

@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 Aaron K. Clark
+
+using ApplyTrack.Api.Agent;
+using ApplyTrack.Api.Llm;
+
+namespace ApplyTrack.Api.Tests;
+
+/// <summary>
+/// The worker's model calls run under its own deadline (#410): the api's 120 s guards a
+/// request someone is waiting on; the worker is a background job whose drafting call was
+/// dying at exactly that deadline.
+/// </summary>
+public class AgentLlmTimeoutTests
+{
+    private static readonly LlmOptions Api = new()
+    {
+        BaseUrl = "http://llm.local/v1", Model = "m", ApiKey = "k", TimeoutSeconds = 120, DailyCapPerTenant = 50,
+    };
+
+    [Fact]
+    public void The_worker_keeps_the_endpoint_and_takes_its_own_deadline()
+    {
+        var worker = AgentWorker.WorkerLlmOptions(Api, new AgentOptions { LlmTimeoutSeconds = 300 });
+
+        Assert.Equal(300, worker.TimeoutSeconds);
+        Assert.Equal(Api.BaseUrl, worker.BaseUrl);
+        Assert.Equal(Api.Model, worker.Model);
+        Assert.Equal(Api.ApiKey, worker.ApiKey);
+        Assert.Equal(Api.DailyCapPerTenant, worker.DailyCapPerTenant);
+        Assert.Equal(120, Api.TimeoutSeconds);                      // the api's singleton is untouched
+        Assert.Equal(300, EffectiveLlmConfig.Resolve(worker, null).TimeoutSeconds);
+    }
+
+    [Fact]
+    public void Zero_inherits_the_apis_deadline()
+    {
+        Assert.Same(Api, AgentWorker.WorkerLlmOptions(Api, new AgentOptions { LlmTimeoutSeconds = 0 }));
+        Assert.Same(Api, AgentWorker.WorkerLlmOptions(Api, new AgentOptions { LlmTimeoutSeconds = 120 }));
+    }
+
+    [Fact]
+    public void The_default_is_longer_than_a_request_deadline()
+    {
+        Assert.True(new AgentOptions().LlmTimeoutSeconds > new LlmOptions().TimeoutSeconds);
+    }
+}

api/ApplyTrack.Api.Tests/AnswerDrafterTests.cs

@@ -325,6 +325,82 @@ public void A_social_profile_box_takes_that_sites_link_or_stays_empty()
         Assert.Contains("link in your résumé", why);
     }
 
+    private static List<PacketQuestion> ScreeningQuestions(int count) =>
+        Enumerable.Range(1, count)
+            .Select(i => new PacketQuestion($"q{i}", $"Screening question {i}: describe your experience with topic {i}",
+                true, PacketQuestion.Textarea, [], PacketQuestion.Custom))
+            .ToList();
+
+    /// <summary>The ids the stub was asked for, in the order the prompt listed them.</summary>
+    private static List<string> AskedIds(string userPrompt) =>
+        System.Text.RegularExpressions.Regex.Matches(userPrompt, @"^- id: (\S+) \|", System.Text.RegularExpressions.RegexOptions.Multiline)
+            .Select(m => m.Groups[1].Value).ToList();
+
+    private static string AnswerAll(string userPrompt) =>
+        "{\"answers\":[" + string.Join(",", AskedIds(userPrompt).Select(id => $"{{\"id\":\"{id}\",\"answer\":\"Did {id}.\"}}")) + "]}";
+
+    [Fact]
+    public async Task Screening_questions_go_to_the_model_in_chunks_and_every_chunk_is_kept()
+    {
+        // #410: Orbis's 22 questions rode one completion, which stalled at the endpoint for
+        // the full timeout, twice. Chunked, each prompt carries a few questions and all of
+        // them still get drafted.
+        var questions = ScreeningQuestions(20);
+        var prompts = new List<List<string>>();
+        var stub = new StubLlmClient((_, user, _) => { prompts.Add(AskedIds(user)); return AnswerAll(user); });
+
+        var (answers, review) = await new AnswerDrafter(new StructuredCompleter(stub), questionsPerCall: 8)
+            .DraftAsync(questions, Ctx(), Cfg);
+
+        Assert.Equal(3, stub.Calls);
+        Assert.Equal([8, 8, 4], prompts.Select(p => p.Count));
+        Assert.Equal(questions.Select(q => q.Id), prompts.SelectMany(p => p));   // each question asked once, in order
+        Assert.Equal(20, answers.Count);
+        Assert.Equal("Did q13.", answers["q13"]);
+        Assert.Empty(review);
+    }
+
+    [Fact]
+    public async Task One_stalled_chunk_voids_only_its_own_answers_and_the_rest_of_the_packet_is_drafted()
+    {
+        var questions = ScreeningQuestions(12);
+        var stub = new StubLlmClient((_, user, _) =>
+            AskedIds(user).Contains("q5")
+                ? throw new LlmUnavailableException("could not reach the LLM endpoint (The operation was canceled.)")
+                : AnswerAll(user));
+
+        var (answers, review) = await new AnswerDrafter(new StructuredCompleter(stub), questionsPerCall: 4)
+            .DraftAsync(questions, Ctx(), Cfg);
+
+        Assert.Equal(3, stub.Calls);                                   // the third chunk was still attempted
+        Assert.Equal(8, answers.Count);
+        Assert.All(new[] { "q1", "q4", "q9", "q12" }, id => Assert.Equal($"Did {id}.", answers[id]));
+        Assert.Equal(["q5", "q6", "q7", "q8"], review.Select(r => r.Id).Order());
+        Assert.All(review, r => Assert.StartsWith("no draft: could not reach the LLM endpoint", r.Reason));
+    }
+
+    [Fact]
+    public async Task Two_failed_chunks_in_a_row_stop_the_draft_without_spending_another_timeout_per_chunk()
+    {
+        // One stall is intermittent (the same Orbis prompt answered in 7 s on the next run);
+        // two in a row is the endpoint being down, and each further chunk would only wait out
+        // another full deadline. The rest is flagged for the person, not attempted.
+        var questions = ScreeningQuestions(16);
+        var stub = new StubLlmClient((_, user, _) =>
+            AskedIds(user).Contains("q1")
+                ? AnswerAll(user)
+                : throw new LlmUnavailableException("the LLM endpoint returned HTTP 503"));
+
+        var (answers, review) = await new AnswerDrafter(new StructuredCompleter(stub), questionsPerCall: 4)
+            .DraftAsync(questions, Ctx(), Cfg);
+
+        Assert.Equal(3, stub.Calls);                                   // chunks 1 (ok), 2 and 3 (failed); 4 never sent
+        Assert.Equal(4, answers.Count);                                // the first chunk's work is kept
+        Assert.Equal(12, review.Count);
+        Assert.All(review.Where(r => int.Parse(r.Id[1..]) <= 12), r => Assert.Equal("no draft: the LLM endpoint returned HTTP 503", r.Reason));
+        Assert.All(review.Where(r => int.Parse(r.Id[1..]) > 12), r => Assert.StartsWith("no draft: not attempted after 2 failed completions in a row", r.Reason));
+    }
+
     [Fact]
     public async Task A_dead_model_still_yields_a_packet_with_everything_owed_flagged()
     {

api/ApplyTrack.Api/Agent/AgentOptions.cs

@@ -35,4 +35,13 @@ public sealed class AgentOptions
     /// at peak a pass, a submit run parked up to eight minutes on a security code, the code
     /// poll and the heartbeat held all four at once.</summary>
     public int MaxPoolSize { get; set; } = 8;
+
+    /// <summary>
+    /// The worker's own deadline for one model completion, in seconds (#410). The api's
+    /// <c>Llm__TimeoutSeconds</c> (120) guards an HTTP request a person is waiting on; the
+    /// worker is a background job, and a packet's drafting call was dying at exactly that
+    /// deadline. 0 inherits <c>Llm__TimeoutSeconds</c>. Each chunk of a draft (see
+    /// <see cref="AnswerDrafter.DefaultQuestionsPerCall"/>) gets this budget on its own.
+    /// </summary>
+    public int LlmTimeoutSeconds { get; set; } = 300;
 }

api/ApplyTrack.Api/Agent/AgentWorker.cs

@@ -146,6 +146,26 @@ private async Task<bool> AnyRenewalRequestedAsync(CancellationToken ct)
     // This container's name: the quadlet/compose service name, or the pod's hostname.
     private readonly string _workerId = Environment.MachineName;
 
+    /// <summary>
+    /// The instance LLM settings as the worker uses them: the same endpoint, model, key and
+    /// cap, under the worker's own deadline (<see cref="AgentOptions.LlmTimeoutSeconds"/>)
+    /// instead of the api's request-sized one (#410). The api's singleton is left as it is —
+    /// a request still gets its 120 s. Public for tests.
+    /// </summary>
+    public static LlmOptions WorkerLlmOptions(LlmOptions api, AgentOptions options)
+    {
+        if (options.LlmTimeoutSeconds <= 0 || options.LlmTimeoutSeconds == api.TimeoutSeconds)
+            return api;
+        return new LlmOptions
+        {
+            BaseUrl = api.BaseUrl,
+            Model = api.Model,
+            ApiKey = api.ApiKey,
+            TimeoutSeconds = options.LlmTimeoutSeconds,
+            DailyCapPerTenant = api.DailyCapPerTenant,
+        };
+    }
+
     public AgentWorker(
         string connectionString, AgentOptions options, LlmOptions llmOptions,
         SecretProtector protector, LeadEvaluator evaluator, PacketBuilder packets,
@@ -167,7 +187,7 @@ public AgentWorker(
         };
         _db = NpgsqlDataSource.Create(csb.ConnectionString);
         _options = options;
-        _llmOptions = llmOptions;
+        _llmOptions = WorkerLlmOptions(llmOptions, options);
         _protector = protector;
         _evaluator = evaluator;
         _packets = packets;