From 0f91f21ce4197cc649e73a390247fa6bd88f4a92 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:36:39 +0000 Subject: [PATCH] fix(automations): Say that send_message is how an automation run posts A scheduled reminder whose instruction said "Post a reminder in this channel" ended as misconfigured. The run searched for a Slack posting tool, found none, and blocked the automation, although finishAutomationRun offered send_message. Say in the finishAutomationRun description that send_message is the only way that a run posts, so a missing Slack posting tool does not make the run misconfigured. Add a delivery eval for this instruction. Co-Authored-By: Daniel Griesser --- .../junior-evals/evals/scheduler/README.md | 3 +++ .../evals/scheduler/delivery.eval.ts | 19 +++++++++++++++++++ .../tools/runtime/finish-automation-run.ts | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/junior-evals/evals/scheduler/README.md b/packages/junior-evals/evals/scheduler/README.md index 0e64ed928b..b5bb3d665c 100644 --- a/packages/junior-evals/evals/scheduler/README.md +++ b/packages/junior-evals/evals/scheduler/README.md @@ -38,6 +38,9 @@ Each failure seen in production has one realistic case: `evals/integration/scheduler/delivery.eval.ts`. - "me" does not reach the creator (#2014): the creator mention case in `evals/integration/scheduler/delivery.eval.ts`. +- A reminder that says "post in this channel" ends as `misconfigured` + because no Slack posting tool exists: the post-in-this-channel case in + `delivery.eval.ts`. - Unattended runs ask questions, or post when a condition is not met, and the silence marker leaks into Slack (#2014, #1741): the condition cases in `evals/integration/coding/event-automations/delivery.eval.ts`. diff --git a/packages/junior-evals/evals/scheduler/delivery.eval.ts b/packages/junior-evals/evals/scheduler/delivery.eval.ts index 24383cf7d6..c63329bb7f 100644 --- a/packages/junior-evals/evals/scheduler/delivery.eval.ts +++ b/packages/junior-evals/evals/scheduler/delivery.eval.ts @@ -74,6 +74,25 @@ describe("Scheduled Delivery", () => { expect(delivery.replies[0]!.text).not.toMatch(/<@[UW]/); }); + test("when a due reminder says to post in this channel, deliver it instead of looking for a posting tool", async ({ + run, + }) => { + await insertScheduledAutomation({ + credentialMode: "system", + destination: slackChannel(), + due: true, + task: "Post a reminder in this channel asking everyone to reply in this message's thread with their weekly update. A couple of sentences is enough.", + }); + + const delivery = await run(heartbeat()); + + expect( + completedToolCalls("finishAutomationRun", delivery).map(toolOutput), + ).toEqual([expect.objectContaining({ result: "send_message" })]); + expect(delivery.replies).toHaveLength(1); + expect(delivery.replies[0]!.text).toMatch(/weekly update/i); + }); + test("when a due reminder goes to its creator's DM, deliver the reminder itself", async ({ run, }) => { diff --git a/packages/junior/src/chat/tools/runtime/finish-automation-run.ts b/packages/junior/src/chat/tools/runtime/finish-automation-run.ts index c795958e02..78013dc9f7 100644 --- a/packages/junior/src/chat/tools/runtime/finish-automation-run.ts +++ b/packages/junior/src/chat/tools/runtime/finish-automation-run.ts @@ -29,7 +29,7 @@ export function createFinishAutomationRunTool(options: { readOnlyHint: false, }, description: options.sendsMessage - ? `End this automation run. Use \`send_message\` with the finished reminder, digest, alert, or answer; the runtime posts it to the stored destinations. ${RESULT_GUIDANCE}` + ? `End this automation run. Use \`send_message\` with the finished reminder, digest, alert, or answer; the runtime posts it to the stored destinations. This is the only way that the run posts, so a missing Slack posting tool does not make the run \`misconfigured\`. ${RESULT_GUIDANCE}` : `End this automation run. This automation posts nothing. ${RESULT_GUIDANCE}`, inputSchema: z.object({ result: z.enum(results),