diff --git a/examples/alchemy-cloudflare/README.md b/examples/alchemy-cloudflare/README.md index b91cb74..6e84ea7 100644 --- a/examples/alchemy-cloudflare/README.md +++ b/examples/alchemy-cloudflare/README.md @@ -155,16 +155,42 @@ The AgentSession runs a [Fold](https://github.com/humanlayer/fold) agent on Open **Its tools.** -| Kind | Tools | -| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Files and shell | `read`, `write`, `edit`, `apply_patch`, `bash` | -| Git, with the App's token | `git_fetch`, `git_pull`, `git_merge`, `git_push` (its own branch only, never forced) | -| GitHub | `github_discussion`, `github_comments`, `github_post_comment` (with `reply_to` for review threads), and, on pull requests, the diff, checks, reviews, and line comments | -| Progress | `update_plan`, a checklist comment the agent keeps up to date | -| Web | `web_search`, `web_fetch`, and the repository's skills | +| Kind | Tools | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Files and shell | `read`, `write`, `edit`, `apply_patch`, `bash` | +| Git, with the App's token | `git_fetch`, `git_pull`, `git_merge`, `git_push` (its own branch only, never forced) | +| GitHub | `github_discussion`, `github_comments`, `github_post_comment` (with `reply_to` for review threads), and, on pull requests, the diff, reviews, and line comments | +| Checks | `github_pull_request_checks`, and `github_check_failure` for a failed check's output, annotations, failed steps, and log (the whole log is saved in the workspace) | +| Progress | `update_plan`, a checklist comment the agent keeps up to date | +| Web | `web_search`, `web_fetch`, and the repository's skills | Its final answer is posted for it. It uses `github_post_comment` for anything else, such as answering each line comment in its own thread. +## When checks fail + +Once the agent has been mentioned on a pull request, the app follows it. When a check fails on the pull request's latest commit, the agent looks into it without being asked. + +```mermaid +sequenceDiagram + autonumber + participant GH as GitHub + participant MB as DeliveryMailbox + participant AS as AgentSession + + GH->>MB: check_run completed: failure + MB->>GH: is it on the latest commit? + Note over MB: older commits, passing,
and skipped checks are ignored + MB->>AS: "these checks failed" + delivery + AS->>GH: github_check_failure: output,
annotations, failed steps, log + alt the fix is clear + AS->>GH: commit and push to the branch + end + AS->>MB: complete(cause and fix) + MB->>GH: post it on the pull request +``` + +Checks that fail together arrive as one request. A check that fails after the agent pushes a fix starts another turn, so a fix the agent can't get right keeps coming back to it. + ## Labels New issues and pull requests are labeled by [Workers AI's Clef](https://developers.cloudflare.com/workers-ai/) model, which scores each of GitHub's default labels. A label scoring 0.7 or more is added. This needs no mention and no agent. @@ -174,7 +200,7 @@ New issues and pull requests are labeled by [Workers AI's Clef](https://develope ```text src/ ├── Worker.ts Ingress Worker: webhook routes and the delivery API -├── GithubBot.ts GitHub callbacks: labels, mention access, the request, handoff +├── GithubBot.ts GitHub callbacks: labels, mentions, failed checks, handoff ├── DeliveryMailboxDO.ts the mailbox Durable Object ├── AgentSessionDO.ts the agent: model, tools, system prompt, session start and resume ├── DeliveryTurn.ts one delivery's turn: send the prompt, report back, recover after restarts @@ -182,6 +208,7 @@ src/ ├── GitHubTools.ts, GitTools.ts the agent's GitHub and git tools ├── DeliveryPlanTool.ts update_plan ├── BashTool.ts bash on the shell or the container +├── CheckTools.ts a pull request's checks, and why one failed ├── Workspace.ts the AgentSession's view of its Computer ├── AutoLabel.ts Workers AI labels └── computer/ the Computer Durable Object, its Worker, and the container image diff --git a/examples/alchemy-cloudflare/src/AgentSessionDO.ts b/examples/alchemy-cloudflare/src/AgentSessionDO.ts index 6d6c79d..f7af821 100644 --- a/examples/alchemy-cloudflare/src/AgentSessionDO.ts +++ b/examples/alchemy-cloudflare/src/AgentSessionDO.ts @@ -17,6 +17,7 @@ import { Config, Context, Effect, FileSystem, Layer, Match, Option, Path, Predic import { HttpClient } from 'effect/http' import { bashTool } from './BashTool' +import { checkTools } from './CheckTools' import { WORKSPACE_ROOT } from './computer/Contract' import { DeliveryApi } from './DeliveryApi' import { updatePlanTool } from './DeliveryPlanTool' @@ -85,6 +86,10 @@ export const workBranchFor = (discussion: GitHubDiscussion) => }), ) +/** For a pull request, the tools for its checks. */ +const pullRequestCheckTools = ({ githubDiscussion }: AgentSessionMetadata) => + Predicate.isTagged(githubDiscussion, 'GitHubPullRequest') ? checkTools(githubDiscussion) : [] + /** For an issue with a branch, the tool that opens its pull request. */ const issuePullRequestTools = ({ githubDiscussion, workBranch, defaultBranch }: AgentSessionMetadata) => Predicate.isTagged(githubDiscussion, 'GitHubIssue') && Predicate.isNotNull(workBranch) @@ -107,7 +112,7 @@ const branchPolicy = ({ githubDiscussion, branch, defaultBranch, workBranch }: A GitHubIssue: () => `When the work is ready for review, push it, then open a pull request with github_create_pull_request; if one is already open from ${workBranch}, it returns that one. Put the pull request's link in your answer.`, GitHubPullRequest: () => - `${workBranch} is this pull request's branch, so pushing updates the pull request.`, + `${workBranch} is this pull request's branch, so pushing updates the pull request. Use github_pull_request_checks and github_check_failure to see why a check failed.`, }), ), `To bring in the latest ${defaultBranch}, git_fetch it, then git_merge origin/${defaultBranch}.`, @@ -218,6 +223,7 @@ const AgentConversationLive = Layer.effect( ...githubTools(metadata.githubDiscussion), ...gitTools({ repository: metadata.githubDiscussion.ref, workBranch: metadata.workBranch }), ...issuePullRequestTools(metadata), + ...pullRequestCheckTools(metadata), ], }) diff --git a/examples/alchemy-cloudflare/src/BashTool.ts b/examples/alchemy-cloudflare/src/BashTool.ts index 4523729..b275100 100644 --- a/examples/alchemy-cloudflare/src/BashTool.ts +++ b/examples/alchemy-cloudflare/src/BashTool.ts @@ -8,19 +8,18 @@ */ import { OutputStore } from '@humanlayer/fold-agent' import { - CurrentToolCall, defaultMaxBytes, defaultMaxLines, defineTool, formatSize, ToolResultFailure, ToolResultText, - truncateTail, type FoldTool, } from '@humanlayer/fold-core' import { Effect, Schema } from 'effect' import { type CommandOutput, WORKSPACE_ROOT } from './computer/Contract' +import { visibleOutput } from './ToolOutput' import { Workspace } from './Workspace' const DEFAULT_TIMEOUT_MS = 120_000 @@ -71,33 +70,6 @@ const DESCRIPTION = const combinedOutput = ({ stdout, stderr }: CommandOutput) => stdout.length > 0 && stderr.length > 0 ? `${stdout.replace(/\n?$/, '\n')}${stderr}` : stdout + stderr -/** - * The output as the model sees it: its tail, and when that is not all of it, where the whole output is - * saved. Saving is best-effort; the notice says when it failed. - */ -const visibleOutput = (text: string) => - Effect.gen(function* () { - const truncation = truncateTail(text) - if (!truncation.truncated) return text - const outputStore = yield* OutputStore - const { toolCallId } = yield* CurrentToolCall - const saved = yield* outputStore.append(toolCallId, text).pipe( - Effect.tap((ref) => - Effect.logInfo('bash.output_saved').pipe( - Effect.annotateLogs({ path: ref.path, lines: truncation.totalLines, bytes: text.length }), - ), - ), - Effect.map((ref) => `Full output: ${ref.path}`), - Effect.catch((error) => - Effect.logWarning('bash.output_save failed', error).pipe( - Effect.as('The full output could not be saved'), - ), - ), - ) - const start = truncation.totalLines - truncation.outputLines + 1 - return `${truncation.content}\n\n[Showing lines ${start}-${truncation.totalLines} of ${truncation.totalLines}. ${saved}]` - }) - /** The bash tool, over the session's {@link Workspace}, saving long output to the {@link OutputStore}. */ export const bashTool: FoldTool = defineTool({ name: 'bash', @@ -127,7 +99,7 @@ export const bashTool: FoldTool = defineTool({ timeoutMs, }) .pipe(Effect.mapError((error) => `The shell could not run the command: ${error.message}`)) - const text = (yield* visibleOutput(combinedOutput(output))).replace(/\n+$/, '') + const text = (yield* visibleOutput(combinedOutput(output), 'bash')).replace(/\n+$/, '') if (output.status === 'cancelled') { return yield* Effect.fail(`${text}\n\nCommand timed out after ${timeoutMs} milliseconds`) diff --git a/examples/alchemy-cloudflare/src/CheckTools.ts b/examples/alchemy-cloudflare/src/CheckTools.ts new file mode 100644 index 0000000..c50754e --- /dev/null +++ b/examples/alchemy-cloudflare/src/CheckTools.ts @@ -0,0 +1,129 @@ +/** + * The agent's tools for a pull request's checks: what ran on its latest commit, and why one failed. A failed + * check's full job log is saved in the workspace, and the agent sees its end. + */ +import { + GitHubCheckRun, + GitHubCheckRunRef, + GitHubId, + type GitHubApi, + type GitHubCheckAnnotation, + type GitHubCheckRunInfo, + type GitHubPullRequest, +} from '@humanlayer/channels-github' +import type { OutputStore } from '@humanlayer/fold-agent' +import { defineTool, ToolResultFailure, ToolResultText, type FoldTool } from '@humanlayer/fold-core' +import { Array as Arr, Effect, Predicate, Schema } from 'effect' + +import { visibleOutput } from './ToolOutput' +import { NoParameters } from './ToolParameters' + +/** Longest check summary or output text shown in full. */ +const MAX_TEXT_LENGTH = 2_000 +/** Most annotations listed for one check. */ +const MAX_ANNOTATIONS = 50 + +const clip = (text: string | null) => + Predicate.isNull(text) || text.length <= MAX_TEXT_LENGTH ? text : `${text.slice(0, MAX_TEXT_LENGTH)}…` + +const failure = (error: { readonly message: string }) => + ToolResultFailure.make({ text: `GitHub request failed: ${error.message}` }) + +/** A check as the agent sees it in a list. */ +const checkSummary = (check: GitHubCheckRunInfo) => ({ + id: check.ref.id, + name: check.name, + status: check.status, + conclusion: check.conclusion, + title: check.outputTitle, + summary: clip(check.outputSummary), + annotations: check.annotationCount, + url: check.detailsUrl ?? check.url, +}) + +/** The checks that ran on the pull request's latest commit, each with its ID for `github_check_failure`. */ +export const listPullRequestChecks = Effect.fn('agent_session.list_checks')(function* (pullRequest: GitHubPullRequest) { + const { headSha } = yield* pullRequest.fetchInfo() + const runs = yield* pullRequest.listCheckRunsForRef(headSha) + const checks = yield* Effect.forEach(runs, (run) => run.fetchInfo(), { concurrency: 8 }) + const current = checks.filter((check) => check.headSha === headSha).map(checkSummary) + return ToolResultText.make({ text: JSON.stringify({ headSha, checks: current }, null, 2) }) +}) + +const annotationLine = (annotation: GitHubCheckAnnotation) => { + const lines = + annotation.startLine === annotation.endLine + ? `${annotation.startLine}` + : `${annotation.startLine}-${annotation.endLine}` + const text = [annotation.title, annotation.message].filter(Predicate.isNotNull).join(': ') + return `- ${annotation.level ?? 'note'} at ${annotation.path}:${lines}: ${text}` +} + +/** + * Why a check failed: its result and output, its annotations, and, for a GitHub Actions job, its failed + * steps and the end of its log, with the whole log saved in the workspace. + */ +export const describeCheckFailure = Effect.fn('agent_session.describe_check_failure')(function* ( + pullRequest: GitHubPullRequest, + checkRunId: number, +) { + const { installationId, repositoryId, owner, repository } = pullRequest.ref + const checkRun = GitHubCheckRun.make({ + ref: GitHubCheckRunRef.make({ installationId, repositoryId, owner, repository, id: GitHubId.make(checkRunId) }), + }) + const info = yield* checkRun.fetchInfo() + const annotations = info.annotationCount > 0 ? yield* checkRun.listAnnotations() : [] + const job = yield* checkRun.resolveActionsJob() + + const sections: Array = [ + `Check "${info.name}" (check run ${info.ref.id}) on ${info.headSha.slice(0, 7)}: ${info.conclusion ?? info.status}. ${info.detailsUrl ?? info.url ?? ''}`.trim(), + ] + const output = [info.outputTitle, clip(info.outputSummary), clip(info.outputText)].filter(Predicate.isNotNull) + if (Arr.isArrayNonEmpty(output)) sections.push(`Output:\n${output.join('\n\n')}`) + if (Arr.isReadonlyArrayNonEmpty(annotations)) { + const listed = annotations.slice(0, MAX_ANNOTATIONS).map(annotationLine) + const more = annotations.length > MAX_ANNOTATIONS ? [`- …and ${annotations.length - MAX_ANNOTATIONS} more`] : [] + sections.push(`Annotations:\n${[...listed, ...more].join('\n')}`) + } + if (Predicate.isNull(job)) { + sections.push('This check is not a GitHub Actions job, so there is no job log.') + return ToolResultText.make({ text: sections.join('\n\n') }) + } + const jobInfo = yield* job.fetchInfo() + const failedSteps = jobInfo.steps.filter( + (step) => + Predicate.isNotNull(step.conclusion) && step.conclusion !== 'success' && step.conclusion !== 'skipped', + ) + sections.push( + Arr.isReadonlyArrayNonEmpty(failedSteps) + ? `Failed steps in job "${jobInfo.name}":\n${failedSteps.map((step) => `- ${step.number}. ${step.name}: ${step.conclusion}`).join('\n')}` + : `Job "${jobInfo.name}" has no failed steps.`, + ) + const log = yield* job.downloadLog() + sections.push(`Job log:\n${yield* visibleOutput(log, 'github_check_failure')}`) + return ToolResultText.make({ text: sections.join('\n\n') }) +}) + +/** The check tools for a pull request session. */ +export const checkTools = (pullRequest: GitHubPullRequest): ReadonlyArray> => [ + defineTool({ + name: 'github_pull_request_checks', + description: + "List the checks on the pull request's latest commit: each one's ID, name, status, result, summary, annotation count, and link.", + parameters: NoParameters, + success: ToolResultText, + failure: ToolResultFailure, + handler: () => listPullRequestChecks(pullRequest).pipe(Effect.mapError(failure)), + }), + defineTool({ + name: 'github_check_failure', + description: + "Find out why a check failed: its output, its annotations, and, for a GitHub Actions job, the failed steps and the end of the job's log. The whole log is saved in the workspace.", + parameters: Schema.Struct({ + check_run_id: GitHubId.annotate({ description: 'The check run ID, from github_pull_request_checks' }), + }), + success: ToolResultText, + failure: ToolResultFailure, + handler: ({ check_run_id }) => describeCheckFailure(pullRequest, check_run_id).pipe(Effect.mapError(failure)), + }), +] diff --git a/examples/alchemy-cloudflare/src/GitHubTools.ts b/examples/alchemy-cloudflare/src/GitHubTools.ts index 98ee900..ebcde52 100644 --- a/examples/alchemy-cloudflare/src/GitHubTools.ts +++ b/examples/alchemy-cloudflare/src/GitHubTools.ts @@ -1,5 +1,4 @@ import { - GitHubCheckRuns, GitHubContent, GitHubId, GitHubIssueInfo, @@ -169,17 +168,6 @@ export const githubTools = (discussion: GitHubDiscussion): ReadonlyArray - pullRequest - .listCheckRuns() - .pipe(Effect.flatMap(jsonResult(GitHubCheckRuns)), Effect.mapError(failure)), - }), defineTool({ name: 'github_pull_request_reviews', description: diff --git a/examples/alchemy-cloudflare/src/GithubBot.ts b/examples/alchemy-cloudflare/src/GithubBot.ts index e17eae6..c9378ef 100644 --- a/examples/alchemy-cloudflare/src/GithubBot.ts +++ b/examples/alchemy-cloudflare/src/GithubBot.ts @@ -21,11 +21,14 @@ import { hasGitHubAccess, type GitHubApiError, type GitHubCallbackHandlers, + type GitHubCheckConclusion, type GitHubIssueComment, type GitHubIssueCreated, type GitHubMentioned, type GitHubParticipant, + type GitHubPrCheckCompleted, type GitHubPrCreated, + type GitHubPrEvent, type GitHubRepositoryRef, type GitHubReviewComment, } from '@humanlayer/channels-github' @@ -175,6 +178,37 @@ export const gitHubRequest = (event: GitHubMentioned) => { ]) } +/** Check results that need the agent: the check failed, or could not run. */ +const FAILED_CONCLUSIONS: ReadonlySet = new Set([ + 'failure', + 'timed_out', + 'action_required', + 'startup_failure', +]) + +const isFailedCheck = (event: GitHubPrEvent): event is GitHubPrCheckCompleted => + Predicate.isTagged(event, 'GitHubPrCheckCompleted') && FAILED_CONCLUSIONS.has(event.conclusion) + +/** The failed checks among a batch's events that ran on `headSha`, the pull request's latest commit, each once. */ +export const failedChecksOnHead = (events: ReadonlyArray, headSha: string) => + Arr.dedupeWith( + events.filter(isFailedCheck).filter((check) => check.headSha === headSha), + (a, b) => a.checkRunId === b.checkRunId, + ) + +/** The agent's request when checks fail on the latest commit of a pull request it follows. */ +export const checkFailurePrompt = (checks: ReadonlyArray, headSha: string) => + [ + `Checks failed on this pull request's latest commit, ${headSha.slice(0, 7)}. Nobody mentioned you; these failures are the request.`, + '', + ...checks.map( + (check) => + `- \`${check.name}\`: ${check.conclusion.replace('_', ' ')} (check run ${check.checkRunId})${Predicate.isNull(check.detailsUrl) ? '' : `, ${check.detailsUrl}`}`, + ), + '', + 'Find out why with github_check_failure. If the cause is clear and the fix belongs in this pull request, fix it, check it as far as you can, and push. Otherwise explain the cause and what to change. Your answer is posted on the pull request.', + ].join('\n') + /** * An authorized mention subscribes the discussion, so later comments and checks reach this mailbox, and * hands the delivery to the discussion's AgentSession, which finishes it when its turn ends. Until then the @@ -261,7 +295,28 @@ export const githubHandlers = { }, onSubscribedPrEvents: (event, context) => Effect.gen(function* () { - /** TODO if the issue(s) are failing CI checks then we shoudl like address them */ + if (!event.events.some(isFailedCheck)) return + const { headSha } = yield* event.pullRequest.fetchInfo() + const failed = failedChecksOnHead(event.events, headSha) + if (Arr.isReadonlyArrayEmpty(failed)) { + yield* Effect.logInfo('Failed checks ignored: not on the latest commit') + return + } + yield* Effect.logInfo('Checks failed on the latest commit').pipe( + Effect.annotateLogs({ checks: failed.map(({ name }) => name).join(', '), head_sha: headSha }), + ) + const message = yield* Schema.encodeEffect(AgentSessionMessage)( + AgentSessionMessage.make({ + prompt: checkFailurePrompt(failed, headSha), + githubDiscussion: event.pullRequest, + requestComments: [], + deliveryId: context.deliveryId, + accessToken: context.accessToken, + }), + ).pipe(Effect.orDie) + const agentSessions = yield* AgentSessions + yield* agentSessions.getByName(event.pullRequest.mailboxKey).send(message) + return yield* context.handoff() }).pipe( Effect.annotateLogs({ ...githubRepositoryLogAnnotations(event.pullRequest.ref), diff --git a/examples/alchemy-cloudflare/src/ToolOutput.ts b/examples/alchemy-cloudflare/src/ToolOutput.ts new file mode 100644 index 0000000..326051d --- /dev/null +++ b/examples/alchemy-cloudflare/src/ToolOutput.ts @@ -0,0 +1,31 @@ +import { OutputStore } from '@humanlayer/fold-agent' +import { CurrentToolCall, truncateTail } from '@humanlayer/fold-core' +import { Effect } from 'effect' + +/** + * Long tool output as the model sees it: its tail, and when that is not all of it, where the whole output + * is saved in the workspace. Saving is best-effort; the notice says when it failed. `tool` names the tool + * in the logs. + */ +export const visibleOutput = (text: string, tool: string) => + Effect.gen(function* () { + const truncation = truncateTail(text) + if (!truncation.truncated) return text + const outputStore = yield* OutputStore + const { toolCallId } = yield* CurrentToolCall + const saved = yield* outputStore.append(toolCallId, text).pipe( + Effect.tap((ref) => + Effect.logInfo(`${tool}.output_saved`).pipe( + Effect.annotateLogs({ path: ref.path, lines: truncation.totalLines, bytes: text.length }), + ), + ), + Effect.map((ref) => `Full output: ${ref.path}`), + Effect.catch((error) => + Effect.logWarning(`${tool}.output_save failed`, error).pipe( + Effect.as('The full output could not be saved'), + ), + ), + ) + const start = truncation.totalLines - truncation.outputLines + 1 + return `${truncation.content}\n\n[Showing lines ${start}-${truncation.totalLines} of ${truncation.totalLines}. ${saved}]` + }) diff --git a/examples/alchemy-cloudflare/test/check-tools.vi.test.ts b/examples/alchemy-cloudflare/test/check-tools.vi.test.ts new file mode 100644 index 0000000..f0148a3 --- /dev/null +++ b/examples/alchemy-cloudflare/test/check-tools.vi.test.ts @@ -0,0 +1,300 @@ +import { describe, it } from '@effect/vitest' +import { + GitHubActionsJob, + GitHubApi, + GitHubCheckRun, + GitHubCheckRunInfo, + GitHubEventId, + GitHubId, + GitHubParticipant, + GitHubPrCheckCompleted, + GitHubPrCommentCreated, + GitHubPullRequest, + GitHubPullRequestInfo, + GitHubPullRequestRef, + GitHubSubscribedPrEvents, + GitHubIssueComment, + type GitHubCheckConclusion, +} from '@humanlayer/channels-github' +import { OutputStore, OutputStoreRef } from '@humanlayer/fold-agent' +import { CurrentToolCall, ToolCallId } from '@humanlayer/fold-core' +import { RuntimeContext } from 'alchemy/RuntimeContext' +import { Effect, Layer, Ref } from 'effect' + +import { makeTestDeliveryExecution } from '../../../packages/delivery/test/delivery-execution' +import { AgentSessions } from '../src/AgentSessionDO' +import { AutoLabel } from '../src/AutoLabel' +import { describeCheckFailure, listPullRequestChecks } from '../src/CheckTools' +import type { AgentSessionMessage } from '../src/DeliveryTurn' +import { checkFailurePrompt, failedChecksOnHead, githubHandlers } from '../src/GithubBot' + +const repository = { + installationId: GitHubId.make(100), + repositoryId: GitHubId.make(200), + owner: 'humanlayer', + repository: 'effect-channels', +} +const pullRequest = GitHubPullRequest.make({ + ref: GitHubPullRequestRef.make({ ...repository, number: GitHubId.make(43) }), + mailboxKey: 'github:pr:43', +}) +const HEAD = 'aaaaaaa1111' +const alice = GitHubParticipant.make({ id: GitHubId.make(1), login: 'alice', type: 'User' }) + +const checkInfo = (id: number, name: string, headSha: string, conclusion: GitHubCheckConclusion | null) => + GitHubCheckRunInfo.make({ + ref: { ...repository, id: GitHubId.make(id) }, + name, + headSha, + status: 'completed', + conclusion, + startedAt: null, + completedAt: null, + apiUrl: `https://api.github.com/repos/humanlayer/effect-channels/check-runs/${id}`, + url: null, + detailsUrl: `https://github.com/humanlayer/effect-channels/actions/runs/1/job/${id}`, + checkSuiteId: null, + outputTitle: conclusion === 'failure' ? '2 tests failed' : null, + outputSummary: null, + outputText: null, + annotationCount: conclusion === 'failure' ? 1 : 0, + }) + +const infos = [ + checkInfo(1, 'build', HEAD, 'success'), + checkInfo(2, 'test', HEAD, 'failure'), + checkInfo(3, 'old', 'bbb', 'failure'), +] + +const apiWithLog = (log: string) => + Layer.mock(GitHubApi, { + fetchPullRequest: () => + Effect.succeed( + GitHubPullRequestInfo.make({ + ref: pullRequest.ref, + title: 'Fix the crash', + body: null, + state: 'open', + url: 'https://github.com/humanlayer/effect-channels/pull/43', + author: alice, + draft: false, + merged: false, + headRef: 'fix', + headSha: HEAD, + headRepository: null, + baseRef: 'main', + baseSha: 'def', + }), + ), + listCheckRunsForRef: () => Effect.succeed(infos.map((info) => GitHubCheckRun.make({ ref: info.ref }))), + fetchCheckRun: ({ checkRun }) => Effect.succeed(infos.find((info) => info.ref.id === checkRun.id) ?? infos[0]!), + listCheckRunAnnotations: () => + Effect.succeed([ + { + path: 'src/app.test.ts', + startLine: 12, + endLine: 12, + startColumn: null, + endColumn: null, + level: 'failure' as const, + title: 'app handles null config', + message: 'expected null to be "default"', + rawDetails: null, + blobUrl: 'https://github.com/humanlayer/effect-channels/blob/aaa/src/app.test.ts', + }, + ]), + resolveActionsJob: () => + Effect.succeed(GitHubActionsJob.make({ ref: { ...repository, id: GitHubId.make(900) } })), + fetchActionsJob: () => + Effect.succeed({ + ref: { ...repository, id: GitHubId.make(900) }, + runId: GitHubId.make(1), + name: 'test', + status: 'completed' as const, + conclusion: 'failure' as const, + headSha: HEAD, + apiUrl: 'https://api.github.com/x', + url: null, + startedAt: null, + completedAt: null, + checkRunUrl: 'https://api.github.com/y', + steps: [ + { + name: 'Install', + status: 'completed' as const, + conclusion: 'success' as const, + number: 1, + startedAt: null, + completedAt: null, + }, + { + name: 'Run tests', + status: 'completed' as const, + conclusion: 'failure' as const, + number: 2, + startedAt: null, + completedAt: null, + }, + ], + }), + downloadActionsJobLog: () => Effect.succeed(log), + }) + +const api = apiWithLog('FAIL src/app.test.ts\n expected null to be "default"') + +const toolCall = Layer.mergeAll( + Layer.mock(CurrentToolCall, { toolCallId: ToolCallId.make('tool_call_a00000030000000000000000') }), + Layer.mock(OutputStore, { + directory: '/workspace/.fold/tool-output', + refFor: (toolCallId) => + OutputStoreRef.make({ toolCallId, path: `/workspace/.fold/tool-output/${toolCallId}.log` }), + append: (toolCallId) => + Effect.succeed(OutputStoreRef.make({ toolCallId, path: `/workspace/.fold/tool-output/${toolCallId}.log` })), + }), +) + +describe('github_pull_request_checks', () => { + it.effect("lists the latest commit's checks, with IDs, and leaves out older commits'", ({ expect }) => + Effect.gen(function* () { + const result = yield* listPullRequestChecks(pullRequest).pipe(Effect.provide(api)) + const check = (info: GitHubCheckRunInfo) => ({ + id: info.ref.id, + name: info.name, + status: info.status, + conclusion: info.conclusion, + title: info.outputTitle, + summary: info.outputSummary, + annotations: info.annotationCount, + url: info.detailsUrl, + }) + + expect(result.text).toBe( + JSON.stringify({ headSha: HEAD, checks: [check(infos[0]!), check(infos[1]!)] }, null, 2), + ) + }), + ) +}) + +describe('github_check_failure', () => { + it.effect('shows the failure, its annotations, the failed steps, and the job log', ({ expect }) => + Effect.gen(function* () { + const result = yield* describeCheckFailure(pullRequest, 2).pipe(Effect.provide(Layer.merge(api, toolCall))) + + expect(result.text).toBe( + [ + 'Check "test" (check run 2) on aaaaaaa: failure. https://github.com/humanlayer/effect-channels/actions/runs/1/job/2', + 'Output:\n2 tests failed', + 'Annotations:\n- failure at src/app.test.ts:12: app handles null config: expected null to be "default"', + 'Failed steps in job "test":\n- 2. Run tests: failure', + 'Job log:\nFAIL src/app.test.ts\n expected null to be "default"', + ].join('\n\n'), + ) + }), + ) +}) + +describe('github_check_failure with a long log', () => { + it.effect('shows the end of the log and saves all of it in the workspace', ({ expect }) => + Effect.gen(function* () { + const log = Array.from({ length: 3_000 }, (_, line) => `line ${line + 1}`).join('\n') + const result = yield* describeCheckFailure(pullRequest, 2).pipe( + Effect.provide(Layer.merge(apiWithLog(log), toolCall)), + ) + + expect(result.text).toContain('line 3000') + expect(result.text).not.toContain('line 1\n') + expect(result.text).toContain( + 'Full output: /workspace/.fold/tool-output/tool_call_a00000030000000000000000.log', + ) + }), + ) +}) + +describe('failed checks on a followed pull request', () => { + const completed = (id: number, name: string, conclusion: GitHubCheckConclusion, headSha = HEAD) => + GitHubPrCheckCompleted.make({ + pullRequest, + actor: alice, + eventId: GitHubEventId.make(`check-${id}-${headSha}`), + checkRunId: GitHubId.make(id), + name, + status: 'completed', + conclusion, + detailsUrl: `https://github.com/humanlayer/effect-channels/actions/runs/1/job/${id}`, + headSha, + checkSuiteId: null, + startedAt: null, + completedAt: null, + }) + + it('keeps failures on the latest commit, each once, and ignores the rest', ({ expect }) => { + const comment = GitHubPrCommentCreated.make({ + pullRequest, + actor: alice, + eventId: GitHubEventId.make('comment-1'), + comment: GitHubIssueComment.make({ + ref: { discussion: { _tag: 'PullRequest', ref: pullRequest.ref }, id: GitHubId.make(5) }, + body: 'looks good', + url: 'https://github.com/humanlayer/effect-channels/pull/43#issuecomment-5', + author: alice, + }), + }) + const events = [ + completed(1, 'build', 'success'), + completed(2, 'test', 'failure'), + completed(2, 'test', 'failure'), + completed(3, 'lint', 'timed_out'), + completed(4, 'deploy', 'skipped'), + completed(5, 'test', 'failure', 'bbbbbbb'), + comment, + ] + + expect(failedChecksOnHead(events, HEAD).map(({ checkRunId }) => checkRunId)).toEqual([2, 3]) + }) + + it('asks the agent to find the cause and fix it if it can', ({ expect }) => { + expect(checkFailurePrompt([completed(2, 'test', 'failure'), completed(3, 'lint', 'timed_out')], HEAD)).toBe( + [ + "Checks failed on this pull request's latest commit, aaaaaaa. Nobody mentioned you; these failures are the request.", + '', + '- `test`: failure (check run 2), https://github.com/humanlayer/effect-channels/actions/runs/1/job/2', + '- `lint`: timed out (check run 3), https://github.com/humanlayer/effect-channels/actions/runs/1/job/3', + '', + 'Find out why with github_check_failure. If the cause is clear and the fix belongs in this pull request, fix it, check it as far as you can, and push. Otherwise explain the cause and what to change. Your answer is posted on the pull request.', + ].join('\n'), + ) + }) + + it.effect('start a turn on the latest commit, and hand the delivery off', ({ expect }) => + Effect.gen(function* () { + const delivery = yield* makeTestDeliveryExecution(pullRequest.mailboxKey) + const sent = yield* Ref.make>([]) + const agentSessions = AgentSessions.of({ + getByName: () => ({ send: (message) => Ref.update(sent, (all) => [...all, message]) }), + }) + const services = Layer.mergeAll( + api, + Layer.succeed(AgentSessions, agentSessions), + Layer.mock(AutoLabel, {}), + Layer.mock(RuntimeContext, { Type: 'Worker', id: 'check-failure-test', env: {} }), + ) + const run = (events: readonly [GitHubPrCheckCompleted, ...Array]) => + githubHandlers + .onSubscribedPrEvents( + GitHubSubscribedPrEvents.make({ pullRequest, events }), + delivery.execution.context, + ) + .pipe(Effect.provide(services)) + + yield* run([completed(4, 'deploy', 'success'), completed(5, 'test', 'failure', 'bbbbbbb')]) + expect(yield* Ref.get(sent)).toEqual([]) + expect(yield* Ref.get(delivery.handoffs)).toEqual([]) + + yield* run([completed(2, 'test', 'failure')]) + const [message] = yield* Ref.get(sent) + expect(message?.prompt).toBe(checkFailurePrompt([completed(2, 'test', 'failure')], HEAD)) + expect(message?.requestComments).toEqual([]) + expect(yield* Ref.get(delivery.handoffs)).toHaveLength(1) + }), + ) +})