diff --git a/.agents/agents.md b/.agents/agents.md index 73a3c3a8fe..5de1bb6c40 100644 --- a/.agents/agents.md +++ b/.agents/agents.md @@ -30,6 +30,7 @@ specifications. | `devcontainer-spec` | `.devcontainer/` | devcontainer spec compliance | | `codefactor` | Codefactor issues | Go source flagged by Codefactor | | `ui-polish` | desktop UI | `desktop/src/renderer/src/` (Svelte/TS) | +| `agent-analytics` | agent-fleet analytics | `hack/analytics/`, `.agents/`, `pkg/`/`cmd/` (data-driven only), OpenHands Cloud API (read-only) | The `pkg/*` agents split `pkg/` into logical categories so multiple agents review in parallel without overlapping scope. diff --git a/.agents/agents/agent-analytics/agent.md b/.agents/agents/agent-analytics/agent.md new file mode 100644 index 0000000000..706adf444e --- /dev/null +++ b/.agents/agents/agent-analytics/agent.md @@ -0,0 +1,152 @@ +--- +id: agent-analytics +name: agent-analytics +description: Reviews the previous day's agent-fleet runs with Python data-science tooling, clusters failure modes, and ships one deterministic intervention (script, prompt, or code fix) that lowers agent cognitive load and reduces recurring errors. +enabled: true +--- + +You are the analytics engineer for the devsy-org/devsy agent fleet. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Review the previous day's agent runs, reduce the errors and failures agents encounter by applying deterministic methods that lower cognitive load, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. + +## Scope + +- `hack/analytics/` — the deterministic run-analysis pipeline (analyze_runs.py + outputs) +- `.agents/agents/`, `.agents/tasks/`, `hack/automations/agents.yaml` — agent prompts/schedules +- `pkg/`, `cmd/` — small code fixes only when the data pinpoints a recurring failure mode +- OpenHands Cloud API (`/api/v1/app-conversations/search`, `/api/v1/conversation/{id}/events/search`) read-only, via OPENHANDS_API_KEY + +Runtime secrets: DEVSY_GITHUB_APP_PRIVATE_KEY, DEVSY_GITHUB_APP_COMMIT_USER. App id: use ${DEVSY_GITHUB_APP_ID:-}. + +STEP 0 — Check for and install the Go toolchain + uv if missing: + set -e + # Check for Go toolchain — install only what is missing + export PATH="/usr/local/go/bin:$(go env GOPATH 2>/dev/null)/bin:$PATH" + if ! command -v go >/dev/null 2>&1; then + curl -sSL https://go.dev/dl/go1.26.3.linux-amd64.tar.gz -o /tmp/go.tgz + sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf /tmp/go.tgz && rm /tmp/go.tgz + export PATH="/usr/local/go/bin:$(go env GOPATH)/bin:$PATH" + fi + if ! command -v task >/dev/null 2>&1; then + go install github.com/go-task/task/v3/cmd/task@latest + fi + if ! command -v golangci-lint >/dev/null 2>&1; then + curl -sSL https://github.com/golangci/golangci-lint/releases/download/v2.12.2/golangci-lint-2.12.2-linux-amd64.tar.gz -o /tmp/gcl.tgz + tar -C /tmp -xzf /tmp/gcl.tgz && sudo mv /tmp/golangci-lint-2.12.2-linux-amd64/golangci-lint /usr/local/bin/ && rm -rf /tmp/gcl.tgz /tmp/golangci-lint-2.12.2-linux-amd64 + fi + # Check for uv — install only if missing (scripts declare deps inline via PEP 723) + if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + go version && task --version && golangci-lint --version && uv --version && uv run hack/analytics/analyze_runs.py --sample --out-dir /tmp/analytics-verify >/dev/null +cd into the cloned devsy repo workspace and run: task cli:tidy + +Before selecting your task, check for open PRs to avoid creating a duplicate. Run: + TOKEN=$(task github:app:sign-commit -- -token) + gh api repos/devsy-org/devsy/pulls --jq '.[] | "\(.number) \(.title) \(.head.ref)"' +Review the titles and branch names. If a PR already addresses the issue/area you plan to +work on, skip it and pick a different one. Do not open a PR for a fix that is already in +flight. If every candidate is already covered, do nothing and report "no actionable task +found — all candidates already have open PRs". + +STEP 1 — Gather yesterday's run data deterministically. The repo ships hack/analytics/analyze_runs.py, a self-contained uv script (PEP 723 inline deps: typer, numpy, scikit-learn, pandas, seaborn, matplotlib, prettytable). Run it ONLY with uv run (never pip/venv) and yesterday's date window: + uv run hack/analytics/analyze_runs.py --since yesterday --until today \ + --out-dir dist/analytics +It authenticates with OPENHANDS_API_KEY (read-only), lists app-conversations created in the window, pulls each conversation's events, and writes: + - dist/analytics/runs.csv (one row per run: agent, status, duration, event/tool counts, error count) + - dist/analytics/failure_clusters.csv (sklearn KMeans clusters of failure signatures) + - dist/analytics/top_errors.csv (ranked recurring error signatures) + - dist/analytics/daily_report.md (human-readable summary with recommendations) + - dist/analytics/failure_heatmap.png (seaborn visualization of failure mode x agent) +If OPENHANDS_API_KEY is unavailable OR no runs exist for the window, fall back to the bundled synthetic sample so the pipeline still produces outputs: add `--sample`. Treat the API as the source of truth when available; the sample is only for offline development. +The analyzer also zips the five outputs and uploads them to https://temp.sh/upload (3-day expiry, returns a plain-text share URL), then writes a `## pr-gate` verdict at the end of daily_report.md: ACTIONABLE (top failure signature recurs across >=2 runs at >=10% share of failed runs) or NOT-ACTIONABLE. Set ANALYTICS_NO_UPLOAD=1 to skip the upload (e.g. in an offline sandbox). If temp.sh is unreachable or rejects the upload the upload is recorded as skipped with the reason; the pipeline still completes. + +STEP 2 — Analyze the daily_report.md and top_errors.csv. Identify ONE high-impact, deterministic intervention that reduces the cognitive load agents spend recovering from errors, so reasoning budget stays on the primary task. Prefer, in order of reviewability: + - A script/automation fix in hack/analytics/ or .agents/ that codifies a recurring recovery (e.g. a missing-tool check, a PR-duplicate guard, a deterministic pre-flight). + - A prompt (agent.md) clarification that removes an ambiguity the data shows agents repeatedly stumble on (stale assertion, wrong resolution order, missing masking rule). Prompt edits must be regenerated through hack/automations/agents.yaml — never hand-edit a generated agent.md. + - A small code fix in pkg/ or cmd/ that eliminates a recurring failure mode the data pinpoints. +Pick exactly ONE. The intervention must be justified by a number from the analytics output (cluster size, error count, recurrence rate). Quote the metric in the PR body. + +STEP 3 — Implement the ONE intervention. Keep it minimal and reviewable in ~20 minutes. If it touches a generated agent prompt, edit hack/automations/agents.yaml and regenerate: `go run ./hack/automations` then confirm `go run ./hack/automations -check` is clean. + +STEP 4 — Verify (CRITICAL — use CI-equivalent lint): + - mkdir -p dist + - git fetch --quiet origin main + - task cli:lint:ci # Must be 0 new issues. + - task cli:test + - uv run hack/analytics/analyze_runs.py --sample --out-dir /tmp/analytics-verify # smoke test the pipeline (sets ANALYTICS_NO_UPLOAD=1 to skip the live upload in CI) + - uvx --from mypy mypy --strict --ignore-missing-imports --disable-error-code untyped-decorator hack/analytics/analyze_runs.py # static typing (must be 0 issues) + - uvx --from radon radon cc -s hack/analytics/analyze_runs.py | grep -oE '\([0-9]+\)' | tr -d '()' | sort -n -r | head -1 # cyclomatic complexity; must be < 10 + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different intervention. + +STEP 5 — PR-GATE DECISION (read the `## pr-gate` verdict in daily_report.md): + - If the verdict is NOT-ACTIONABLE, DO NOT create a PR. Report "no actionable failure mode found — pr-gate NOT-ACTIONABLE" and stop. A run that opens a PR against a NOT-ACTIONABLE verdict is a FAILED run. + - If the verdict is ACTIONABLE, write the PR body to /tmp/pr_body.md. It MUST contain: the analytics window (since/until dates), the pr-gate verdict, the headline metric(s) from daily_report.md that motivated the change (e.g. "cluster_0: 7 runs, signature: missing-tool 'golangci-lint'"), the ONE intervention, the verification performed (lint:ci, cli:test, uv smoke run, mypy, radon complexity), the temp.sh upload link (or the skip reason if temp.sh was unreachable/rejected), and attach/reference the visualization (failure_heatmap.png path or PR-attached image). State the expected reduction in failure recurrence. + +FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). + - git fetch --quiet origin main + - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) + - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. + - task cli:test # Must pass (known pre-existing failures excepted per STEP above). + If format or lint reports ANY issue in your changed files, fix the root cause and re-run + until both are clean. Do NOT proceed to the commit step with formatting or lint errors. + Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), + golines (line length > 120). Running task cli:format first auto-fixes most of these. + +STEP 6 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. + - git fetch --quiet origin main + - git checkout -b agent-analytics/ origin/main + - task github:app:sign-commit -- -m "" "" + - Confirm output: verified=true. + +STEP 7 — Open the PR as the app (no GITHUB_TOKEN): + - Write the PR body to /tmp/pr_body.md. It MUST include: the analytics window, the pr-gate verdict, the headline metric(s) from daily_report.md, the ONE intervention, the verification performed (lint:ci, cli:test, uv smoke run, mypy, radon complexity), the temp.sh upload link or skip reason, the visualization reference, a line "This PR was created by an AI agent as part of an automated daily agent analytics job." + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. + +STEP 8 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): + GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop + changes, the `Desktop CI` `lint-and-test` job. `task cli:lint:ci` mirrors the Go lint job + locally, but the agent must still verify the PR's actual checks and fix any that fail. + - Wait for checks to start, then poll until they complete (timeout ~15 min): + gh pr checks "$PR_NUMBER" --repo devsy-org/devsy --interval 30 --watch --fail-fast >/dev/null 2>&1 || true + gh pr checks "$PR_NUMBER" --repo devsy-org/devsy --json name,state,conclusion + (`gh pr checks --watch` blocks until checks finish; if it returns early or is unavailable in + this environment, poll the REST API instead: + `gh api repos/devsy-org/devsy/commits/$(git rev-parse HEAD)/check-runs --jq '.check_runs[] | {name,state,conclusion}'` + in a loop until every `state` is `completed`.) + - If all checks are `SUCCESS`/`NEUTRAL`/`SKIPPED`, the run is complete; nothing more to do. + - If a check FAILED (most often `Lint`), do NOT abandon the PR: + 1. Fetch the failing job's log: `gh pr view "$PR_NUMBER" --repo devsy-org/devsy --json statusCheckRollup --jq '.statusCheckRollup[] | select(.conclusion=="FAILURE") | .name'`, + then `gh run view --repo devsy-org/devsy --log-failed | tail -200` to read the linter output. + 2. Diagnose the failure against your diff. Common lint failures that slip past `cli:lint:ci` + (shallow clone merge-base, format-after-lint, or env differences): unused import, shadow, + gocritic, errcheck, gci/gofumpt formatting. Fix the root cause in the source — do NOT add + `//nolint` or disable linters. + 3. Re-apply the fix, then re-verify locally exactly as CI does: + - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. + - Desktop files: `cd desktop && npx biome check --write && npm run check`. + 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): + - task github:app:sign-commit -- -m "" "" + - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch + 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. + - Cap the loop at ONE follow-up fix commit. If a second round is needed, stop, leave the PR + in its current state, and report the remaining failing check name + log tail in the run + summary so a human can finish it. Do not pile up many fixup commits. + +Constraints: ONE intervention per run, justified by a metric from the analytics output. Do not invent metrics. Run the script only via `uv run` (never pip/venv). If OPENHANDS_API_KEY is unavailable, use --sample and say so. ONLY create a PR when the pr-gate verdict in daily_report.md is ACTIONABLE; if NOT-ACTIONABLE, do nothing and report "no actionable failure mode found — pr-gate NOT-ACTIONABLE". Never use GITHUB_TOKEN for the commit or PR. + +## Self-improvement + +At the end of each run, persist findings (recurring failure signatures, cluster centroids +that map to actionable fixes, metric baselines) to the running automation via the +automation service — not the git repo — using the service's agentic memory if available. +Propose a `description` amendment to this `agent.md` for human review when the analysis +method or intervention priorities change. diff --git a/.agents/agents/ci-optimizer/agent.md b/.agents/agents/ci-optimizer/agent.md index 5161a7549f..820749bea6 100644 --- a/.agents/agents/ci-optimizer/agent.md +++ b/.agents/agents/ci-optimizer/agent.md @@ -65,6 +65,9 @@ STEP 5 — Validate (REQUIRED before committing): - If validation reveals your edit is wrong, fix it or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -74,39 +77,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 6 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 6 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b ci-optimizer/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 7 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the run(s) you inspected (id + conclusion/duration), the problem found, the minimal change, the validation performed (YAML parse result; act result or "act/docker unavailable"), a line "This PR was created by an AI agent as part of an automated daily CI optimization job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"ci-optimizer/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 8 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -131,7 +117,6 @@ STEP 8 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/cmd-reviewer/agent.md b/.agents/agents/cmd-reviewer/agent.md index 0742269028..92794ba0fa 100644 --- a/.agents/agents/cmd-reviewer/agent.md +++ b/.agents/agents/cmd-reviewer/agent.md @@ -44,14 +44,17 @@ STEP 1 — Pick ONE cmd/ subdirectory to review. Enumerate with: ls cmd/. Pick o STEP 2 — Make ONE small, focused improvement reviewable in ~20 minutes. Prefer correctness/help-text fixes over behavior changes. -STEP 3 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): +STEP 3 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - - task cli:lint:ci # checks ONLY new issues introduced by your change (mirrors CI). Must be 0 issues. - - task cli:test # unit tests. - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, that is the baseline; proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. + - task cli:lint:ci # Must be 0 new issues. + - task cli:test + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -61,39 +64,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 4 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 4 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b cmd-reviewer/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 5 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the cmd/ subdirectory reviewed, the issue found, the minimal change, the verification performed (task cli:lint:ci result, task cli:test result), a line "This PR was created by an AI agent as part of an automated daily CLI review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"cmd-reviewer/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 6 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -118,7 +104,6 @@ STEP 6 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/codefactor/agent.md b/.agents/agents/codefactor/agent.md index 00b07bcb62..a0fea84878 100644 --- a/.agents/agents/codefactor/agent.md +++ b/.agents/agents/codefactor/agent.md @@ -66,16 +66,19 @@ STEP 1 — Fetch ALL Codefactor issues (deterministic, paginated, no auth): STEP 2 — Pick ONE issue (any category: Maintainability, Duplication, Complexity). Prefer the most contained change reviewable in ~20 minutes. Pick exactly ONE. (Open PRs were already listed in STEP 0 — do not pick an issue addressed by one of them.) -STEP 3 — Fix it: read FilePath around StartLine..EndLine, make the minimal correct change. Do NOT disable linters or add workaround comments. +STEP 3 — Fix it: read FilePath around StartLine..EndLine, make the minimal correct change. Do NOT disable linters or add workaround comments. If your change touched pkg/git, ensure you didn't introduce NEW failures. -STEP 4 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): +STEP 4 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - - task cli:lint:ci # checks ONLY new issues introduced by your change (mirrors CI). Must be 0 issues. - - task cli:test # unit tests. - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, that is the baseline; proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different issue. + - task cli:lint:ci # Must be 0 new issues. + - task cli:test + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -85,39 +88,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 5 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 5 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b fix/codefactor/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "fix: " "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "fix: " "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 6 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the Codefactor issue Key, file:line, the Codefactor message, the fix summary, and that task cli:lint:ci passed (0 new issues) + task cli:test status. Add label "codefactor" if it exists., a line "This PR was created by an AI agent as part of an automated daily Codefactor issue fix job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"fix: \",\"head\":\"fix/codefactor/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title fix: -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 7 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -142,7 +128,6 @@ STEP 7 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/devcontainer-spec/agent.md b/.agents/agents/devcontainer-spec/agent.md index d40cdfeaae..d11d1f239e 100644 --- a/.agents/agents/devcontainer-spec/agent.md +++ b/.agents/agents/devcontainer-spec/agent.md @@ -51,7 +51,7 @@ STEP 2 — Identify ONE spec requirement that is declared but not implemented, O STEP 3 — Implement or correct it. If features changed, regenerate the lock via: devcontainer build --workspace-folder . --image-name /tmp/dc-test 2>/dev/null || true (best-effort; lock regeneration may require docker). -STEP 4 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): +STEP 4 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - task cli:lint:ci # Must be 0 new issues. @@ -60,6 +60,9 @@ STEP 4 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -69,39 +72,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 5 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 5 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b devcontainer-spec/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 6 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the spec requirement, the gap found, the change, the verification performed (lint:ci, cli:test, e2e result), a line "This PR was created by an AI agent as part of an automated daily devcontainer spec job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"devcontainer-spec/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 7 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -126,7 +112,6 @@ STEP 7 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/docs-keeper/agent.md b/.agents/agents/docs-keeper/agent.md index 3841ddefa0..57b6a2c368 100644 --- a/.agents/agents/docs-keeper/agent.md +++ b/.agents/agents/docs-keeper/agent.md @@ -36,6 +36,9 @@ STEP 3 — Verify: - If the doc references a CLI flag, confirm it exists in cmd/ source. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -45,39 +48,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 4 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 4 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b docs-keeper/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "docs: " "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "docs: " "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 5 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the doc area, the drift found, the correction, the code verification performed, a line "This PR was created by an AI agent as part of an automated daily docs job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"docs: \",\"head\":\"docs-keeper/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title docs: -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 6 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -102,7 +88,6 @@ STEP 6 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/integration-test/agent.md b/.agents/agents/integration-test/agent.md index 76fff24030..b9510135e9 100644 --- a/.agents/agents/integration-test/agent.md +++ b/.agents/agents/integration-test/agent.md @@ -47,15 +47,18 @@ STEP 1 — Authenticate gh read-only with the Devsy GitHub App token to inspect STEP 2 — Fix the flaky test OR add an edge-case test the logs reveal as uncovered. Use real code paths, not mocks. Use the e2e framework helpers (e2e/framework/). Keep the change reviewable in 15 minutes. -STEP 3 — Verify (CRITICAL): +STEP 3 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - task cli:lint:ci # Must be 0 new issues. - - task cli:test # unit tests (exclude e2e). + - task cli:test - Run the focused suite: task cli:test:e2e:suite -- "" (or task cli:test:e2e:focus -- "") - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different test. + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -65,39 +68,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 4 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 4 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b integration-test/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "test(e2e): " "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "test(e2e): " "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 5 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the suite/test, the logs inspected (run id), the fix/edge-case added, the verification performed (lint:ci, cli:test, focused e2e suite result), a line "This PR was created by an AI agent as part of an automated daily e2e review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"test(e2e): \",\"head\":\"integration-test/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title test(e2e): -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 6 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -122,7 +108,6 @@ STEP 6 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/lint-fixer/agent.md b/.agents/agents/lint-fixer/agent.md index 3fefb8ba58..4d927327bd 100644 --- a/.agents/agents/lint-fixer/agent.md +++ b/.agents/agents/lint-fixer/agent.md @@ -57,12 +57,15 @@ STEP 2 — Pick ONE category of findings. Apply mechanical fixes via task cli:li STEP 3 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues introduced. + - task cli:lint:ci # Must be 0 new issues. - task cli:test - (cd desktop && npm run check) # svelte-check / type check, if web files touched - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different category. + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -72,39 +75,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 4 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 4 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b lint-fixer/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "style: " "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "style: " "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 5 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the linter + category, the findings fixed (count), the fix method (mechanical/manual), the verification performed (lint:ci, cli:test, desktop check if web touched), a line "This PR was created by an AI agent as part of an automated daily lint fix job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"style: \",\"head\":\"lint-fixer/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title style: -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 6 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -129,7 +115,6 @@ STEP 6 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/pkg-container/agent.md b/.agents/agents/pkg-container/agent.md index 2a74cdf2ab..a730c6fdb1 100644 --- a/.agents/agents/pkg-container/agent.md +++ b/.agents/agents/pkg-container/agent.md @@ -51,6 +51,9 @@ STEP 2 — Verify (CRITICAL — use CI-equivalent lint): KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -60,39 +63,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 3 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 3 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b pkg-container/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 4 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test), a line "This PR was created by an AI agent as part of an automated daily package review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"pkg-container/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 5 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -117,7 +103,6 @@ STEP 5 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/pkg-core-agent/agent.md b/.agents/agents/pkg-core-agent/agent.md index 4f0b1792db..e969258fcf 100644 --- a/.agents/agents/pkg-core-agent/agent.md +++ b/.agents/agents/pkg-core-agent/agent.md @@ -51,6 +51,9 @@ STEP 2 — Verify (CRITICAL — use CI-equivalent lint): KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -60,39 +63,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 3 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 3 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b pkg-core-agent/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 4 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test), a line "This PR was created by an AI agent as part of an automated daily package review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"pkg-core-agent/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 5 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -117,7 +103,6 @@ STEP 5 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/pkg-ssh-git/agent.md b/.agents/agents/pkg-ssh-git/agent.md index c645449cae..51a1756afc 100644 --- a/.agents/agents/pkg-ssh-git/agent.md +++ b/.agents/agents/pkg-ssh-git/agent.md @@ -41,16 +41,19 @@ work on, skip it and pick a different one. Do not open a PR for a fix that is al flight. If every candidate is already covered, do nothing and report "no actionable task found — all candidates already have open PRs". -STEP 1 — Pick ONE package from the scope. Review for: error handling, resource leaks (file/goroutine), lowercase logging, test coverage of edge cases (auth failure, key rotation, permission errors). Never log or print secret/key material; mask credentials in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. +STEP 1 — Pick ONE package from the scope. Review for: error handling, resource leaks (file/goroutine), lowercase logging, test coverage of edge cases (auth failure, key rotation, permission errors). Never log or print secret/key material; mask credentials in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. If your change touched pkg/git, ensure you didn't introduce NEW failures. STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - task cli:lint:ci # Must be 0 new issues. - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -60,39 +63,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 3 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 3 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b pkg-ssh-git/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 4 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test), a line "This PR was created by an AI agent as part of an automated daily package review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"pkg-ssh-git/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 5 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -117,7 +103,6 @@ STEP 5 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/pkg-system-platform/agent.md b/.agents/agents/pkg-system-platform/agent.md index 346c3633a9..eca22c8556 100644 --- a/.agents/agents/pkg-system-platform/agent.md +++ b/.agents/agents/pkg-system-platform/agent.md @@ -47,16 +47,19 @@ work on, skip it and pick a different one. Do not open a PR for a fix that is al flight. If every candidate is already covered, do nothing and report "no actionable task found — all candidates already have open PRs". -STEP 1 — Pick ONE package from the scope (rotate across runs). Review for: cross-platform correctness (Linux/macOS/Windows), path handling, lowercase logging, secret masking in pkg/secrets/pkg/token, and edge-case tests. Never print secret/token values; mask in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. +STEP 1 — Pick ONE package from the scope (rotate across runs). Review for: cross-platform correctness (Linux/macOS/Windows), path handling, lowercase logging, secret masking in pkg/secrets/pkg/token, and edge-case tests. Never print secret/token values; mask in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. If your change touched pkg/git, ensure you didn't introduce NEW failures. STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - mkdir -p dist - git fetch --quiet origin main - task cli:lint:ci # Must be 0 new issues. - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -66,39 +69,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 3 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 3 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b pkg-system-platform/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 4 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test), a line "This PR was created by an AI agent as part of an automated daily package review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"pkg-system-platform/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 5 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -123,7 +109,6 @@ STEP 5 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/pkg-tunnel-network/agent.md b/.agents/agents/pkg-tunnel-network/agent.md index e829e72438..8108f9d05b 100644 --- a/.agents/agents/pkg-tunnel-network/agent.md +++ b/.agents/agents/pkg-tunnel-network/agent.md @@ -56,6 +56,9 @@ STEP 2 — Verify (CRITICAL — use CI-equivalent lint): KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -65,39 +68,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 3 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 3 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b pkg-tunnel-network/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "" "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 4 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test, cli:build:grpc if proto touched), a line "This PR was created by an AI agent as part of an automated daily package review job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"\",\"head\":\"pkg-tunnel-network/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 5 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -122,7 +108,6 @@ STEP 5 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/agents/ui-polish/agent.md b/.agents/agents/ui-polish/agent.md index 9b86e9d9c4..97474f80f6 100644 --- a/.agents/agents/ui-polish/agent.md +++ b/.agents/agents/ui-polish/agent.md @@ -66,6 +66,9 @@ STEP 4 — Verify (REQUIRED before committing): STEP 5 — Write the PR body to /tmp/pr_body.md. It MUST contain: Problem (1-3 sentences, with the file/component), Change (what you changed, minimal), Why it's safe / reviewable in <15 min, Verification (the exact checks you ran and their results). FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -75,39 +78,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP 6 — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP 6 — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b ui-polish/ origin/main - - git add - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m "fix(ui): " "" - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add - task github:app:sign-commit -- -m "fix(ui): " "" - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP 7 — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: Problem, Change, Why it's safe/reviewable in <15 min, Verification (the exact checks run and their results), a line "This PR was created by an AI agent as part of an automated daily UI polish job." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"fix(ui): \",\"head\":\"ui-polish/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title fix(ui): -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP 8 — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -132,7 +118,6 @@ STEP 8 — Ensure status checks pass (lint failures are the most common reason a - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/.agents/tasks/agent-analytics/task.md b/.agents/tasks/agent-analytics/task.md new file mode 100644 index 0000000000..05189b2dd9 --- /dev/null +++ b/.agents/tasks/agent-analytics/task.md @@ -0,0 +1,13 @@ +--- +kind: task +id: agent-analytics +agent: agent-analytics +--- + +Run the `agent-analytics` agent daily to review the previous day's agent-fleet runs. + +Gathers run telemetry from the OpenHands Cloud API (read-only, via `OPENHANDS_API_KEY`), +clusters failure modes with the Python data-science stack in `hack/analytics/analyze_runs.py`, +and ships one deterministic intervention (script, prompt, or small code fix) that lowers the +cognitive load agents spend recovering from errors — keeping reasoning budget on the primary +task. Persists findings to the running automation via the automation service. diff --git a/.gitignore b/.gitignore index 88bfab2a8c..3f53c33788 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /dist # Unit test targets /main +/sign_commit /profile.out /tagsvendor/ node_modules/ @@ -16,3 +17,6 @@ desktop/dist/ desktop/release/ desktop/playwright-report/ desktop/test-results/ +__pycache__/ +*.pyc +dist/analytics/ diff --git a/AGENTS.md b/AGENTS.md index f835f3002b..b72ad5d3f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,7 +123,19 @@ Biome formats and checks web frontend files. 1. **Contributor License Agreement (CLA)**: All contributors sign the CLA. A CLA bot posts instructions on the PR after it is opened. 2. **Commit messages**: Conventional Commits, with a concise subject line (50 characters max). -3. **Commit signing**: Commits are signed by the Devsy GitHub App. Push or create commits through the app installation (see [GitHub App Authentication](#github-app-authentication)); GitHub signs commits made through the app. +3. **Commit signing**: Commits are signed by the Devsy GitHub App. Use `task github:app:sign-commit` to create verified commits as the app — never run `git commit` locally (local commits are unsigned and fail the signature check). + + ```bash + # commit all working-tree changes (auto-creates remote branch if needed) + task github:app:sign-commit -- -m "fix: concise subject" -b "body text" + + # commit + open a draft PR in one step + task github:app:sign-commit -- -m "fix: concise subject" -b "body text" -pr + + # print the app installation token (for other API calls) + task github:app:sign-commit -- -token + ``` + 4. **Pre-commit checks**: Linters, checkers, and relevant unit tests run before pushing. `prek` (a fast pre-commit hook manager) manages them. - Installation (Linux and macOS): diff --git a/Taskfile.yml b/Taskfile.yml index 0b3ec7d178..1c7c9a7092 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -251,12 +251,12 @@ tasks: github:app:sign-commit: # usage: task github:app:sign-commit -- -m "subject" [-b "body"] [files...] + # task github:app:sign-commit -- -m "subject" -pr (commit + open draft PR) + # task github:app:sign-commit -- -m "subject" -pr-only (open draft PR only) # task github:app:sign-commit -- -token (print the app installation token) desc: | create a commit signed by the Devsy GitHub App (devsy-app[bot]) on the - current branch. Authenticates as the app installation and creates the - commit through GitHub so it signs it (committer: web-flow, verified). - With no file paths, commits all changed files vs origin/main. + current branch. Pass -token to print the app installation token instead (used to open PRs as the app without GITHUB_TOKEN). Reuses DEVSY_GITHUB_APP_ID / DEVSY_GITHUB_APP_PRIVATE_KEY[_PATH]. diff --git a/hack/analytics/analyze_runs.py b/hack/analytics/analyze_runs.py new file mode 100644 index 0000000000..b0017add49 --- /dev/null +++ b/hack/analytics/analyze_runs.py @@ -0,0 +1,983 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "typer>=0.12", +# "numpy>=1.26", +# "scikit-learn>=1.4", +# "pandas>=2.2", +# "seaborn>=0.13", +# "matplotlib>=3.8", +# "prettytable>=3.10", +# ] +# /// +"""Deterministic agent-fleet run analyzer. + +Gathers the previous day's agent runs from the OpenHands Cloud API (read-only), +normalizes them into tabular run records, clusters failure signatures with +scikit-learn, ranks recurring errors, renders a seaborn visualization, and emits +a human-readable daily report with concrete recommendations. + +The goal is to reduce the cognitive load agents spend recovering from errors by +codifying the recurring failure modes the data surfaces, so an operator (or the +agent-analytics agent itself) can ship one deterministic intervention per day. + +Run with uv (deps are declared inline above, so no venv or pip is needed): + + uv run hack/analytics/analyze_runs.py --sample --out-dir dist/analytics + uv run hack/analytics/analyze_runs.py --since yesterday --until today + +Auth uses OPENHANDS_API_KEY (Bearer). When the key is absent or no runs are +found in the window, pass --sample to exercise the full pipeline against bundled +synthetic data so the outputs are always producible offline. + +Outputs (under --out-dir, default dist/analytics): + runs.csv, failure_clusters.csv, top_errors.csv, daily_report.md, + failure_heatmap.png +""" + +from __future__ import annotations + +import csv +import json +import os +import re +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zipfile +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +import typer +from prettytable import PrettyTable +from sklearn.cluster import KMeans + +app = typer.Typer(add_completion=False, help="Deterministic agent-fleet run analyzer.") + +FEATURE_BUCKETS: list[str] = [ + "missing-tool", "test-failure", "cmd-exit-nonzero", "permission-denied", + "timeout", "git-baseline", "lint-issue", "commit-signing", + "frontend-check", "go-panic", "other", "none", +] +WORD_TO_UNIT: dict[str, str] = { + "day": "d", "hour": "h", "minute": "m", "second": "s", + "week": "w", "month": "mo", "year": "y", +} +EPOCH_UNITS: set[str] = {"y", "mo", "w", "d", "h", "m", "s"} +ISO_FORMATS: tuple[str, ...] = ( + "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S%z", +) +AGENT_BRANCH_RE = re.compile(r"([a-z][a-z0-9-]*?)/") +COMPACT_RE = re.compile(r"^(-?\d+)\s*([a-zA-Z]+)$") +WORD_RE = re.compile(r"^(\d+)\s+([a-zA-Z]+)(?:\s+ago)?$") +ERROR_RULES: list[tuple[tuple[str, ...], str]] = [ + (("command not found", "not installed", "no such file or directory"), "missing-tool"), + (("exit code", "test"), "test-failure"), + (("exit code",), "cmd-exit-nonzero"), + (("permission denied", "403", "forbidden"), "permission-denied"), + (("timeout", "timed out"), "timeout"), + (("merge-base", "origin/main", "git fetch"), "git-baseline"), + (("lint", "golangci", "biome"), "lint-issue"), + (("sign_commit", "verified=false", "createcommitonbranch"), "commit-signing"), + (("svelte-check", "vitest"), "frontend-check"), + (("panic", "nil pointer"), "go-panic"), +] +SAMPLE_ROWS: tuple[tuple[Any, ...], ...] = ( + ("lint-fixer", "error", 1, 420.0, 38, 22, 3, "missing-tool", "golangci-lint: command not found"), + ("lint-fixer", "ran", 0, 300.0, 30, 18, 0, "none", ""), + ("ci-optimizer", "error", 1, 510.0, 44, 25, 2, "missing-tool", "act: command not found"), + ("ci-optimizer", "error", 1, 490.0, 40, 21, 2, "git-baseline", "git merge-base origin/main: ambiguous"), + ("pkg-core-agent", "error", 1, 600.0, 52, 30, 4, "test-failure", "TestRepoClone: exit code 1"), + ("pkg-core-agent", "error", 1, 620.0, 55, 31, 4, "test-failure", "TestRepoClone: exit code 1"), + ("pkg-core-agent", "ran", 0, 360.0, 28, 16, 0, "none", ""), + ("pkg-ssh-git", "error", 1, 380.0, 33, 19, 1, "permission-denied", "permission denied (publickey)"), + ("pkg-container", "error", 1, 700.0, 60, 35, 5, "cmd-exit-nonzero", "docker: Cannot connect to the Docker daemon"), + ("pkg-container", "error", 1, 720.0, 62, 36, 5, "cmd-exit-nonzero", "docker: Cannot connect to the Docker daemon"), + ("ui-polish", "error", 1, 290.0, 26, 14, 2, "frontend-check", "svelte-check: type error in App.svelte"), + ("ui-polish", "ran", 0, 240.0, 22, 12, 0, "none", ""), + ("docs-keeper", "ran", 0, 180.0, 16, 9, 0, "none", ""), + ("codefactor", "error", 1, 410.0, 34, 20, 1, "lint-issue", "golangci-lint: SA9003: return value"), + ("integration-test", "error", 1, 800.0, 70, 40, 3, "timeout", "context deadline exceeded"), + ("devcontainer-spec", "error", 1, 330.0, 29, 16, 2, "commit-signing", "verified=false"), +) + + +@dataclass +class RunRecord: + """One normalized row per agent run.""" + + conversation_id: str + agent: str + status: str + failed: int + duration_s: float + event_count: int + tool_call_count: int + error_count: int + top_error_bucket: str + error_buckets: str + sample_error: str + title: str + + +@dataclass +class ErrorStat: + """Aggregated counts derived while scanning a run's events.""" + + tool_counts: Counter[str] = field(default_factory=Counter) + error_buckets: Counter[str] = field(default_factory=Counter) + error_texts: list[str] = field(default_factory=list) + error_events: int = 0 + + +@dataclass +class AnalysisBundle: + """Everything the output writers need, bundled to keep arg counts low.""" + + runs: list[RunRecord] + clusters: list[dict[str, Any]] + tops: list[dict[str, Any]] + window: tuple[str, str] + source: str + + +@dataclass +class Options: + """Resolved CLI options shared across helpers.""" + + out_dir: str + base_url: str + token: str + limit: int + sample: bool + verbose: bool + + +@dataclass +class OutputPaths: + """Bundle of output file paths to keep writer arg counts low.""" + + runs_csv: str + clusters_csv: str + top_csv: str + report_md: str + heatmap_png: str + + +@dataclass +class UploadResult: + """Outcome of a temp.sh upload (None link when upload is skipped/failed).""" + + link: str | None + reason: str + + +def parse_when(value: str, now: datetime) -> datetime: + """Parse a date/datetime or relative phrase into a timezone-aware datetime.""" + low = value.strip().lower() + named = named_when(low, now) + if named is not None: + return named + compact = match_compact(low, now) + if compact is not None: + return compact + worded = match_words(low, now) + if worded is not None: + return worded + iso_dt = parse_iso(value) + if iso_dt is not None: + return iso_dt + raise typer.BadParameter(f"unparseable date/time: {value!r}") + + +def named_when(low: str, now: datetime) -> datetime | None: + """Resolve the named aliases: now/today/yesterday/tomorrow.""" + table: dict[str, datetime] = { + "now": now, + "today": now.replace(hour=0, minute=0, second=0, microsecond=0), + "yesterday": (now - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), + "tomorrow": (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), + "eod": (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0), + } + return table.get(low) + + +def match_compact(low: str, now: datetime) -> datetime | None: + """Resolve compact signed relatives like -2d, 3h, -1w.""" + m = COMPACT_RE.match(low) + if not m: + return None + unit = WORD_TO_UNIT.get(m.group(2).rstrip("s"), m.group(2).rstrip("s")) + if unit not in EPOCH_UNITS: + return None + return add_relative(now, int(m.group(1)), unit) + + +def match_words(low: str, now: datetime) -> datetime | None: + """Resolve word relatives like '2 days ago', '3 hours', '1 week ago'.""" + m = WORD_RE.match(low) + if not m: + return None + unit = WORD_TO_UNIT.get(m.group(2).rstrip("s"), m.group(2).rstrip("s")) + if unit not in EPOCH_UNITS: + return None + return add_relative(now, -int(m.group(1)), unit) + + +def parse_iso(value: str) -> datetime | None: + """Parse an ISO 8601 date/datetime string, defaulting to UTC when naive.""" + for fmt in ISO_FORMATS: + try: + dt = datetime.strptime(value, fmt) + except ValueError: + continue + return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) + return None + + +def add_relative(base: datetime, n: int, unit: str) -> datetime: + """Add a relative offset to a base datetime, keyed by epoch unit.""" + deltas: dict[str, timedelta] = { + "d": timedelta(days=n), "h": timedelta(hours=n), "m": timedelta(minutes=n), + "s": timedelta(seconds=n), "w": timedelta(weeks=n), + "mo": timedelta(days=30 * n), "y": timedelta(days=365 * n), + } + delta = deltas.get(unit) + if delta is None: + raise typer.BadParameter(f"unknown relative unit: {unit!r}") + return base + delta + + +def iso(dt: datetime) -> str: + """Render a datetime as a UTC ISO 8601 string.""" + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class CloudClient: + """Minimal read-only OpenHands Cloud API client.""" + + def __init__(self, base_url: str, token: str, timeout: int = 30, verbose: bool = False) -> None: + self.base = base_url.rstrip("/") + self.token = token + self.timeout = timeout + self.verbose = verbose + + def search_conversations(self, limit: int = 100) -> list[dict[str, Any]]: + """List recent app conversations.""" + data = self.get("/api/v1/app-conversations/search", {"limit": limit}) + return normalize_items(data) + + def conversation_events(self, conv_id: str, limit: int = 500) -> list[dict[str, Any]]: + """Fetch events for one conversation.""" + path = f"/api/v1/conversation/{conv_id}/events/search" + return normalize_items(self.get(path, {"limit": limit})) + + def get(self, path: str, params: dict[str, Any]) -> Any: + """Issue a GET request and decode JSON, raising on HTTP/parse errors.""" + url = self.build_url(path, params) + if self.verbose: + typer.echo(f"GET {url}", err=True) + req = urllib.request.Request(url, headers=self.headers()) + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + body = resp.read() + except urllib.error.HTTPError as exc: + raise typer.Exit(1) from SystemExit(f"HTTP {exc.code} for {url}") + return decode_json(url, body) + + def build_url(self, path: str, params: dict[str, Any]) -> str: + """Build a query-stringed URL for a path.""" + qs = urllib.parse.urlencode(params) + return f"{self.base}{path}?{qs}" + + def headers(self) -> dict[str, str]: + """Build auth + accept headers.""" + out: dict[str, str] = {"Accept": "application/json"} + if self.token: + out["Authorization"] = f"Bearer {self.token}" + return out + + +def normalize_items(data: Any) -> list[dict[str, Any]]: + """Normalize an API response into a list of item dicts.""" + if isinstance(data, dict): + items = data.get("items", []) + else: + items = data or [] + return [i for i in items if isinstance(i, dict)] + + +def decode_json(url: str, body: bytes) -> Any: + """Decode JSON bytes, raising on empty or malformed payloads.""" + if not body: + return None + try: + return json.loads(body) + except json.JSONDecodeError as exc: + raise typer.Exit(1) from SystemExit(f"non-JSON response from {url}: {exc}") + + +def infer_agent(conversation: dict[str, Any]) -> str: + """Derive the owning agent from branch/title/git context.""" + for key in ("git_branch_name", "gitBranchName", "branch", "head_branch"): + m = AGENT_BRANCH_RE.match(str(conversation.get(key) or "")) + if m: + return m.group(1) + title = conversation.get("title") or conversation.get("name") or "" + m = AGENT_BRANCH_RE.search(str(title)) + if m: + return m.group(1) + return fallback_agent(conversation) + + +def fallback_agent(conversation: dict[str, Any]) -> str: + """Return a last-resort agent identifier from agent_id/repo.""" + return str(conversation.get("agent_id") or conversation.get("repo") or "unknown") + + +def parse_ts(value: str | None) -> datetime | None: + """Parse an ISO timestamp string into a timezone-aware datetime.""" + if not value: + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) + + +def classify_error(text: str) -> str: + """Bucket a raw error string into a canonical failure signature.""" + low = (text or "").lower() + for needles, label in ERROR_RULES: + if all(n in low for n in needles): + return label + if text: + return "other" + return "none" + + +def extract_error_text(event: dict[str, Any]) -> str: + """Pull the most useful error text from an event's common shapes.""" + for key in ("error", "detail", "message", "stderr"): + v = event.get(key) + if isinstance(v, str) and v: + return v + nested = nested_dict(event, "observation") + for key in ("stderr", "stdout", "content", "message"): + v = nested.get(key) + if isinstance(v, str) and v: + return v + action = nested_dict(event, "action") + cmd = action.get("command") + return cmd if isinstance(cmd, str) else "" + + +def nested_dict(event: dict[str, Any], key: str) -> dict[str, Any]: + """Return event[key] as a dict, or an empty dict when absent/not a dict.""" + v = event.get(key) + return v if isinstance(v, dict) else {} + + +def build_run_record(conversation: dict[str, Any], events: list[dict[str, Any]]) -> RunRecord: + """Normalize a conversation + its events into a RunRecord.""" + stat = scan_events(events) + started = parse_ts(conversation.get("created_at") or conversation.get("createdAt")) + ended = parse_ts(conversation.get("updated_at") or conversation.get("updatedAt")) + duration = duration_s(started, ended) + status = resolve_status(conversation, stat.error_events) + return RunRecord( + conversation_id=str(conversation.get("id") or conversation.get("app_conversation_id") or ""), + agent=infer_agent(conversation), + status=status, + failed=int(is_failed(status, stat.error_events)), + duration_s=round(duration, 1), + event_count=len(events), + tool_call_count=sum(stat.tool_counts.values()), + error_count=stat.error_events, + top_error_bucket=stat.error_buckets.most_common(1)[0][0] if stat.error_buckets else "none", + error_buckets=json.dumps(stat.error_buckets, separators=(",", ":")), + sample_error=stat.error_texts[0] if stat.error_texts else "", + title=str(conversation.get("title") or ""), + ) + + +def scan_events(events: list[dict[str, Any]]) -> ErrorStat: + """Tally tool calls and error signals across a run's events.""" + stat = ErrorStat() + for event in events: + kind = event.get("kind") or "" + tool = event.get("tool_name") or "" + if tool: + stat.tool_counts[str(tool)] += 1 + if kind in {"ErrorEvent", "ObservationEvent"}: + maybe_count_error(event, stat) + return stat + + +def maybe_count_error(event: dict[str, Any], stat: ErrorStat) -> None: + """Record an error bucket/text when an event looks like a failure.""" + text = extract_error_text(event) + exit_code = event_exit_code(event) + if looks_like_error(event.get("kind"), text, exit_code): + stat.error_events += 1 + stat.error_buckets[classify_error(text)] += 1 + if text: + stat.error_texts.append(text[:300]) + + +def event_exit_code(event: dict[str, Any]) -> int | None: + """Read the exit code from either the observation or the event top level.""" + obs = nested_dict(event, "observation") + code = obs.get("exit_code", event.get("exit_code")) + return to_int(code) + + +def to_int(value: Any) -> int | None: + """Coerce a value to int, returning None when not convertible.""" + if isinstance(value, bool) or value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def looks_like_error(kind: Any, text: str, exit_code: int | None) -> bool: + """Decide whether an event represents a failure.""" + if kind == "ErrorEvent": + return True + if text and exit_code not in (None, 0): + return True + return "error" in (text or "").lower() + + +def duration_s(started: datetime | None, ended: datetime | None) -> float: + """Compute a run duration in seconds from its bounds.""" + if not started or not ended: + return 0.0 + return (ended - started).total_seconds() + + +def resolve_status(conversation: dict[str, Any], error_events: int) -> str: + """Resolve a human-readable run status.""" + status = conversation.get("execution_status") or conversation.get("status") or "unknown" + if status == "unknown" and error_events == 0: + return "stopped" if conversation.get("stopped") else "ran" + return str(status) + + +def is_failed(status: str, error_events: int) -> bool: + """Decide whether a run counts as failed.""" + return status in {"error", "stopped", "aborted", "failed"} or error_events > 0 + + +def fetch_runs(client: CloudClient, window: tuple[datetime, datetime], limit: int) -> list[RunRecord]: + """Fetch and normalize all runs in a window from the Cloud API.""" + since, until = window + runs: list[RunRecord] = [] + for conv in client.search_conversations(limit=limit): + if in_window(conv, since, until): + runs.append(run_from_conv(client, conv)) + time.sleep(0.05) + return runs + + +def in_window(conv: dict[str, Any], since: datetime, until: datetime) -> bool: + """Check whether a conversation was created inside the window.""" + created = parse_ts(conv.get("created_at") or conv.get("createdAt")) + return bool(created and since <= created < until) + + +def run_from_conv(client: CloudClient, conv: dict[str, Any]) -> RunRecord: + """Build a RunRecord for a conversation, fetching its events.""" + cid = conv.get("id") or conv.get("app_conversation_id") + if not cid: + return build_run_record(conv, []) + events = client.conversation_events(str(cid), limit=500) + return build_run_record(conv, events) + + +def synthetic_runs() -> list[RunRecord]: + """Bundled synthetic sample so the pipeline produces outputs offline.""" + out: list[RunRecord] = [] + for i, row in enumerate(SAMPLE_ROWS): + agent, status, failed, dur, ev, tools, errs, bucket, sample = row + out.append(RunRecord( + conversation_id=f"sample-{i:03d}", + agent=agent, + status=status, + failed=failed, + duration_s=dur, + event_count=ev, + tool_call_count=tools, + error_count=errs, + top_error_bucket=bucket, + error_buckets=json.dumps({bucket: errs} if errs else {}, separators=(",", ":")), + sample_error=sample, + title=f"{agent} run {i}", + )) + return out + + +def to_row(r: RunRecord) -> dict[str, Any]: + """Convert a RunRecord into a plain dict for CSV writing.""" + return { + "conversation_id": r.conversation_id, "agent": r.agent, "status": r.status, + "failed": r.failed, "duration_s": r.duration_s, "event_count": r.event_count, + "tool_call_count": r.tool_call_count, "error_count": r.error_count, + "top_error_bucket": r.top_error_bucket, "error_buckets": r.error_buckets, + "sample_error": r.sample_error, "title": r.title, + } + + +def bucket_map() -> tuple[list[str], dict[str, int]]: + """Return the bucket list and an index map for the feature vector.""" + return FEATURE_BUCKETS, {b: i for i, b in enumerate(FEATURE_BUCKETS)} + + +def feature_vector(r: RunRecord, b_idx: dict[str, int]) -> np.ndarray: + """Build the failure-bucket feature vector for one run.""" + vec = np.zeros(len(b_idx), dtype=float) + try: + eb = json.loads(r.error_buckets) if r.error_buckets else {} + except json.JSONDecodeError: + eb = {} + for bucket, count in eb.items(): + if bucket in b_idx: + vec[b_idx[bucket]] += float(count) + if not r.error_count and "none" in b_idx: + vec[b_idx["none"]] = 1.0 + return vec + + +def cluster_failures(runs: list[RunRecord]) -> list[dict[str, Any]]: + """KMeans-cluster runs by their failure-bucket feature vector.""" + if not runs: + return [] + _, b_idx = bucket_map() + matrix = np.vstack([feature_vector(r, b_idx) for r in runs]) + labels = fit_labels(matrix) + return [cluster_row(runs[i], int(labels[i])) for i in range(len(runs))] + + +def fit_labels(matrix: np.ndarray) -> np.ndarray: + """Fit KMeans (or fall back to a single cluster) and return labels.""" + if matrix.shape[0] <= 1 or matrix.sum() == 0: + return np.zeros(matrix.shape[0], dtype=int) + k = max(2, min(5, matrix.shape[0] - 1)) + km = KMeans(n_clusters=k, n_init=10, random_state=7) + return km.fit_predict(matrix) + + +def cluster_row(r: RunRecord, cluster: int) -> dict[str, Any]: + """Build one failure_clusters.csv row.""" + return { + "conversation_id": r.conversation_id, "agent": r.agent, "cluster": cluster, + "error_count": r.error_count, "top_error_bucket": r.top_error_bucket, + "sample_error": r.sample_error[:200], + } + + +def top_errors(runs: list[RunRecord]) -> list[dict[str, Any]]: + """Rank recurring failure signatures by run count.""" + sigs: Counter[str] = Counter() + agents: dict[str, set[str]] = {} + for r in runs: + if not r.error_count: + continue + sigs[r.top_error_bucket] += 1 + agents.setdefault(r.top_error_bucket, set()).add(r.agent) + failed_total = sum(1 for r in runs if r.error_count) + return [top_row(sig, count, agents, failed_total) for sig, count in sigs.most_common()] + + +def top_row(sig: str, count: int, agents: dict[str, set[str]], failed_total: int) -> dict[str, Any]: + """Build one top_errors.csv row.""" + return { + "signature": sig, "run_count": count, + "affected_agents": ",".join(sorted(agents.get(sig, set()))), + "share_of_failed": round(count / max(1, failed_total), 3), + } + + +def write_csv(path: str, rows: list[dict[str, Any]]) -> None: + """Write a list of dicts to a CSV file.""" + fields = list(rows[0].keys()) if rows else [] + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fields) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +def agent_index(runs: list[RunRecord]) -> tuple[list[str], dict[str, int]]: + """Return sorted agent names and their row index.""" + agents = sorted({r.agent for r in runs}) + return agents, {a: i for i, a in enumerate(agents)} + + +def bucket_matrix(runs: list[RunRecord]) -> tuple[list[str], list[str], np.ndarray]: + """Build the (agents, buckets, matrix) counts of failures by agent x bucket.""" + agents, a_idx = agent_index(runs) + _, b_idx = bucket_map() + matrix = np.zeros((len(agents), len(b_idx)), dtype=int) + for r in runs: + accumulate(r, a_idx, b_idx, matrix) + return agents, FEATURE_BUCKETS, matrix + + +def accumulate(r: RunRecord, a_idx: dict[str, int], b_idx: dict[str, int], matrix: np.ndarray) -> None: + """Add one run's bucket counts into the agent x bucket matrix.""" + if r.agent not in a_idx: + return + try: + eb = json.loads(r.error_buckets) if r.error_buckets else {} + except json.JSONDecodeError: + eb = {} + for bucket, count in eb.items(): + if bucket in b_idx: + matrix[a_idx[r.agent], b_idx[bucket]] += int(count) + + +def render_heatmap(path: str, runs: list[RunRecord]) -> None: + """Render the failure-mode x agent seaborn heatmap to a PNG.""" + if not runs: + return + agents, buckets, matrix = bucket_matrix(runs) + if not agents: + return + df = pd.DataFrame(matrix, index=agents, columns=buckets) + fig, ax = plt.subplots(figsize=fig_size(buckets, agents)) + sns.heatmap(df, annot=True, fmt="d", cmap="rocket_r", ax=ax, cbar=True) + ax.set_title("agent-fleet failure modes (yesterday)") + ax.set_xlabel("failure signature") + ax.set_ylabel("agent") + fig.tight_layout() + fig.savefig(path, dpi=140) + plt.close(fig) + + +def fig_size(buckets: list[str], agents: list[str]) -> tuple[float, float]: + """Pick a readable figure size from the matrix dimensions.""" + return (max(8.0, len(buckets) * 0.7), max(4.0, len(agents) * 0.5)) + + +def make_table(columns: list[str], rows: list[list[Any]]) -> PrettyTable: + """Build a PrettyTable with a left-aligned first column.""" + table = PrettyTable() + table.field_names = columns + table.align = "l" + for row in rows: + table.add_row(row) + return table + + +def write_report(path: str, bundle: AnalysisBundle) -> None: + """Write the human-readable daily report markdown (footer appended later).""" + lines = report_header(bundle) + lines.append(report_agent_table(bundle.runs)) + lines.append(report_top_table(bundle.tops)) + lines.append(report_cluster_table(bundle.clusters)) + lines.append(report_recommendations(bundle.tops)) + with open(path, "w") as f: + f.write("\n\n".join(lines)) + + +def report_header(bundle: AnalysisBundle) -> list[str]: + """Build the report title + headline metrics.""" + total = len(bundle.runs) + failed = sum(1 for r in bundle.runs if r.error_count) + rate = round(failed / total, 3) if total else 0.0 + return [ + "# agent-fleet daily report", + "", + f"- window: `{bundle.window[0]}` -> `{bundle.window[1]}`", + f"- source: {bundle.source}", + f"- runs: {total}", + f"- failed runs: {failed}", + f"- failure rate: {rate}", + ] + + +def report_agent_table(runs: list[RunRecord]) -> str: + """Build the per-agent breakdown as a PrettyTable block.""" + by = per_agent(runs) + rows = [[agent, d["runs"], d["failed"], d["errors"]] for agent, d in sorted(by.items())] + table = make_table(["agent", "runs", "failed", "errors"], rows) + return "## per-agent breakdown\n\n```\n" + str(table.get_string()) + "\n```" + + +def per_agent(runs: list[RunRecord]) -> dict[str, dict[str, int]]: + """Aggregate per-agent run/failed/error counts.""" + out: dict[str, dict[str, int]] = {} + for r in runs: + d = out.setdefault(r.agent, {"runs": 0, "failed": 0, "errors": 0}) + d["runs"] += 1 + d["failed"] += 1 if r.error_count else 0 + d["errors"] += int(r.error_count) + return out + + +def report_top_table(tops: list[dict[str, Any]]) -> str: + """Build the top recurring failure signatures as a PrettyTable block.""" + rows = [[t["signature"], t["run_count"], t["affected_agents"], t["share_of_failed"]] for t in tops] + table = make_table(["signature", "run_count", "affected_agents", "share_of_failed"], rows) + return "## top recurring failure signatures\n\n```\n" + str(table.get_string()) + "\n```" + + +def report_cluster_table(clusters: list[dict[str, Any]]) -> str: + """Build the failure clusters as a PrettyTable block.""" + rows = [[c["conversation_id"], c["agent"], c["cluster"], c["error_count"], c["top_error_bucket"]] for c in clusters] + table = make_table(["conversation_id", "agent", "cluster", "error_count", "top_error_bucket"], rows) + return "## failure clusters\n\n```\n" + str(table.get_string()) + "\n```" + + +def report_recommendations(tops: list[dict[str, Any]]) -> str: + """Build the recommendations section from the ranked signatures.""" + if not tops: + return "## recommendations\n\n- no actionable failure mode found today." + top = tops[0] + lines = [ + "## recommendations", + "", + f"- highest-impact: **{top['signature']}** affects {top['run_count']} runs across " + f"{top['affected_agents']} (share_of_failed={top['share_of_failed']}). " + "Pick ONE deterministic intervention that codifies recovery for this signature " + "(missing-tool pre-flight check, prompt clarification, or small code fix) so agent " + "reasoning budget stays on the primary task.", + ] + for t in tops[1:3]: + lines.append(f"- secondary: **{t['signature']}** ({t['run_count']} runs, agents: {t['affected_agents']}).") + return "\n".join(lines) + + +def gather_runs(opts: Options, window: tuple[datetime, datetime]) -> tuple[list[RunRecord], str]: + """Return runs + their source label, honoring --sample or the live API.""" + if opts.sample: + return synthetic_runs(), "synthetic-sample (--sample)" + if not opts.token: + typer.echo("OPENHANDS_API_KEY not set; pass --sample for offline mode.", err=True) + raise typer.Exit(2) + client = CloudClient(opts.base_url, opts.token, verbose=opts.verbose) + runs = fetch_runs(client, window, opts.limit) + if not runs: + typer.echo("no runs found in window; retry with --sample for offline mode.", err=True) + raise typer.Exit(3) + return runs, "openhands-cloud-api" + + +def is_actionable(tops: list[dict[str, Any]], min_runs: int = 2, min_share: float = 0.1) -> bool: + """Decide whether the data justifies a PR (a recurring, non-trivial signature).""" + if not tops: + return False + top = tops[0] + enough_runs = top.get("run_count", 0) >= min_runs + enough_share = top.get("share_of_failed", 0.0) >= min_share + return bool(enough_runs and enough_share) + + +def zip_outputs(paths: OutputPaths) -> str: + """Zip the five output files into a temp archive and return its path.""" + handle, tmp = tempfile.mkstemp(suffix=".zip") + os.close(handle) + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as zf: + for p in (paths.runs_csv, paths.clusters_csv, paths.top_csv, paths.report_md, paths.heatmap_png): + if os.path.exists(p): + zf.write(p, arcname=os.path.basename(p)) + return tmp + + +def upload_to_tempsh(zip_path: str) -> UploadResult: + """Upload the zipped outputs to temp.sh and return the share link.""" + try: + with open(zip_path, "rb") as f: + body = f.read() + except OSError as exc: + return UploadResult(link=None, reason=f"read failed: {exc}") + boundary = uuid.uuid4().hex + parts = [ + f"--{boundary}".encode(), + f'Content-Disposition: form-data; name="file"; filename="{os.path.basename(zip_path)}"'.encode(), + b"Content-Type: application/zip", + b"", + body, + f"--{boundary}--".encode(), + b"", + ] + payload = b"\r\n".join(parts) + headers = { + "Content-Type": f"multipart/form-data; boundary={boundary}", + "User-Agent": "devsy-analytics/1.0", + "Accept": "text/plain, */*", + } + req = urllib.request.Request("https://temp.sh/upload", data=payload, headers=headers, method="POST") + try: + with no_redirect_opener().open(req, timeout=60) as resp: + return upload_response(resp) + except urllib.error.HTTPError as exc: + return UploadResult(link=None, reason=upload_http_error(exc)) + except (urllib.error.URLError, OSError) as exc: + return UploadResult(link=None, reason=f"upload failed: {exc}") + + +def upload_response(resp: Any) -> UploadResult: + """Inspect the temp.sh response (plain-text URL on success, short message on rejection).""" + code = resp.getcode() + raw = resp.read().decode("utf-8", errors="replace").strip() + if code != 200: + return UploadResult(link=None, reason=f"temp.sh returned HTTP {code}: {raw[:200]}") + if not raw: + return UploadResult(link=None, reason="temp.sh returned an empty body") + if raw.startswith("http://") or raw.startswith("https://"): + return UploadResult(link=raw, reason="uploaded to temp.sh") + return UploadResult(link=None, reason=f"temp.sh rejected: {raw[:200]}") + + +def no_redirect_opener() -> urllib.request.OpenerDirector: + """Build a minimal urllib opener (redirects are rejected by upload_response via code checks).""" + opener = urllib.request.OpenerDirector() + for handler in (urllib.request.HTTPSHandler(), urllib.request.HTTPHandler()): + opener.add_handler(handler) + return opener + + +def upload_http_error(exc: urllib.error.HTTPError) -> str: + """Render a clear skip reason for a temp.sh HTTP error status.""" + return f"temp.sh returned HTTP {exc.code}" + + +def annotate_report_actionability(path: str, actionable: bool, upload: UploadResult) -> None: + """Append the actionability + upload verdict and footer to the daily report.""" + verdict = "ACTIONABLE" if actionable else "NOT-ACTIONABLE" + block = ["", "## pr-gate", "", f"- verdict: **{verdict}**"] + if actionable: + block.append("- a PR is warranted: ship the recommended intervention and reference the metric above.") + else: + block.append("- no PR: the data does not surface a recurring, high-share failure mode today.") + if upload.link: + block.append(f"- uploaded outputs: {upload.link}") + else: + block.append(f"- upload: skipped ({upload.reason})") + block.append("") + block.append("_generated by hack/analytics/analyze_runs.py_") + block.append("") + with open(path, "a") as f: + f.write("\n".join(block)) + + +def write_outputs(out_dir: str, bundle: AnalysisBundle) -> OutputPaths: + """Write all five deliverables to disk and report what was written.""" + paths = output_paths(out_dir) + write_csv(paths.runs_csv, [to_row(r) for r in bundle.runs]) + write_csv(paths.clusters_csv, bundle.clusters) + write_csv(paths.top_csv, bundle.tops) + render_heatmap(paths.heatmap_png, bundle.runs) + write_report(paths.report_md, bundle) + echo_writes(paths, bundle) + return paths + + +def finalize_outputs(paths: OutputPaths, tops: list[dict[str, Any]]) -> None: + """Upload the zipped outputs and append the actionability verdict to the report.""" + actionable = is_actionable(tops) + upload = skip_upload() if no_upload_env() else upload_bundle(paths) + annotate_report_actionability(paths.report_md, actionable, upload) + echo_verdict(actionable, upload) + + +def no_upload_env() -> bool: + """Read the ANALYTICS_NO_UPLOAD env flag.""" + return os.environ.get("ANALYTICS_NO_UPLOAD", "").lower() in {"1", "true", "yes"} + + +def skip_upload() -> UploadResult: + """Return an UploadResult indicating the upload was intentionally skipped.""" + return UploadResult(link=None, reason="ANALYTICS_NO_UPLOAD=1") + + +def upload_bundle(paths: OutputPaths) -> UploadResult: + """Zip the outputs, upload to temp.sh, and clean up the temp archive.""" + zip_path = zip_outputs(paths) + try: + return upload_to_tempsh(zip_path) + finally: + try: + os.remove(zip_path) + except OSError: + pass + + +def echo_verdict(actionable: bool, upload: UploadResult) -> None: + """Print the PR-gate verdict and upload outcome.""" + verdict = "ACTIONABLE" if actionable else "NOT-ACTIONABLE" + typer.echo(f"pr-gate: {verdict}") + if upload.link: + typer.echo(f"uploaded outputs: {upload.link}") + else: + typer.echo(f"upload: skipped ({upload.reason})", err=True) + + +def output_paths(out_dir: str) -> OutputPaths: + """Build the OutputPaths bundle for one output directory.""" + return OutputPaths( + runs_csv=os.path.join(out_dir, "runs.csv"), + clusters_csv=os.path.join(out_dir, "failure_clusters.csv"), + top_csv=os.path.join(out_dir, "top_errors.csv"), + report_md=os.path.join(out_dir, "daily_report.md"), + heatmap_png=os.path.join(out_dir, "failure_heatmap.png"), + ) + + +def echo_writes(paths: OutputPaths, bundle: AnalysisBundle) -> None: + """Print a summary of the written outputs.""" + typer.echo(f"wrote {paths.runs_csv} ({len(bundle.runs)} runs)") + typer.echo(f"wrote {paths.clusters_csv} ({len(bundle.clusters)} clusters)") + typer.echo(f"wrote {paths.top_csv} ({len(bundle.tops)} signatures)") + typer.echo(f"wrote {paths.report_md}") + typer.echo(f"wrote {paths.heatmap_png}") + + +def default_options(out_dir: str, sample: bool) -> Options: + """Build Options from the CLI flags plus environment-derived values.""" + base = os.environ.get("OPENHANDS_HOST", "https://app.all-hands.dev") + token = os.environ.get("OPENHANDS_API_KEY") or os.environ.get("OPENHANDS_CLOUD_API_KEY", "") + verbose = os.environ.get("ANALYTICS_VERBOSE", "").lower() in {"1", "true", "yes"} + return Options(out_dir=out_dir, base_url=base, token=token, limit=100, sample=sample, verbose=verbose) + + +@app.command() +def analyze( + since: str = typer.Option("yesterday", help="window start: yesterday|today|ISO|'2 days ago'|'-2d' (use = for signed)."), + until: str = typer.Option("today", help="window end: today|now|ISO|'1 day ago'|'-1d' (use = for signed)."), + out_dir: str = typer.Option("dist/analytics", help="output directory."), + sample: bool = typer.Option(False, "--sample", help="use bundled synthetic runs (offline)."), +) -> None: + """Gather, cluster, rank, visualize, upload, and report on agent-fleet runs.""" + opts = default_options(out_dir, sample) + os.makedirs(opts.out_dir, exist_ok=True) + now = datetime.now(timezone.utc) + window_dt = (parse_when(since, now), parse_when(until, now)) + window = (iso(window_dt[0]), iso(window_dt[1])) + runs, source = gather_runs(opts, window_dt) + clusters = cluster_failures(runs) + tops = top_errors(runs) + bundle = AnalysisBundle(runs=runs, clusters=clusters, tops=tops, window=window, source=source) + paths = write_outputs(opts.out_dir, bundle) + finalize_outputs(paths, tops) + + +if __name__ == "__main__": + app() diff --git a/hack/automations/agents.yaml b/hack/automations/agents.yaml index 42c156ee7a..f3281b212d 100644 --- a/hack/automations/agents.yaml +++ b/hack/automations/agents.yaml @@ -5,7 +5,6 @@ agents: - id: pkg-container branch_prefix: pkg-container - git_add_glob: commit_subject: '"" ""' pr_title: job_name: package review job @@ -13,16 +12,10 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 3 pr_step_number: 4 + verify_type: go intro: You are a package reviewer for Devsy's container/Docker category. The devsy-org/devsy repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE package, review it, make ONE small focused improvement, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Pick ONE package from the scope. Review for: daemon lifecycle correctness, credential handling, image pull/cache logic, error propagation, and edge-case tests (missing daemon, registry auth failure). Never echo registry tokens or docker config credentials; mask credentials in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. - - STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. pr_body_must_include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test) constraints: ONE package per run. Never echo registry tokens or docker config credentials. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the Docker, OCI, image, and compose packages for small focused improvements. @@ -36,7 +29,6 @@ agents: `agent.md` for human review when scope boundaries shift. - id: pkg-core-agent branch_prefix: pkg-core-agent - git_add_glob: commit_subject: '"" ""' pr_title: job_name: package review job @@ -44,16 +36,10 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 3 pr_step_number: 4 + verify_type: go intro: You are a package reviewer for Devsy's agent-runtime core category. The devsy-org/devsy repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE package, review it, make ONE small focused improvement, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Pick ONE package from the scope. Review for: option resolution correctness (see pkg/options/resolve.go precedence: DEVSY_AGENT_URL -> AGENT_URL context option -> GitHub release default), config defaults, workspace path handling, template edge cases, and lowercase logging. Respect the DEVSY_AGENT_URL / AGENT_URL / GitHub-release resolution order in resolve.go. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. - - STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. pr_body_must_include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test) constraints: ONE package per run. Respect the DEVSY_AGENT_URL / AGENT_URL / GitHub-release resolution order in resolve.go. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the agent runtime, options, config, and workspace packages for small focused improvements. @@ -67,7 +53,6 @@ agents: `agent.md` for human review when scope boundaries shift. - id: pkg-ssh-git branch_prefix: pkg-ssh-git - git_add_glob: commit_subject: '"" ""' pr_title: job_name: package review job @@ -75,16 +60,10 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 3 pr_step_number: 4 + verify_type: go intro: You are a package reviewer for Devsy's SSH/git/credentials category. The devsy-org/devsy repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE package, review it, make ONE small focused improvement, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- - STEP 1 — Pick ONE package from the scope. Review for: error handling, resource leaks (file/goroutine), lowercase logging, test coverage of edge cases (auth failure, key rotation, permission errors). Never log or print secret/key material; mask credentials in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. - - STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. + STEP 1 — Pick ONE package from the scope. Review for: error handling, resource leaks (file/goroutine), lowercase logging, test coverage of edge cases (auth failure, key rotation, permission errors). Never log or print secret/key material; mask credentials in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. If your change touched pkg/git, ensure you didn't introduce NEW failures. pr_body_must_include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test) constraints: ONE package per run. Never log or print secret/key material; mask credentials in logs. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the SSH, git, credentials, and shell packages for small focused improvements. @@ -98,7 +77,6 @@ agents: for human review when scope boundaries shift. - id: pkg-system-platform branch_prefix: pkg-system-platform - git_add_glob: commit_subject: '"" ""' pr_title: job_name: package review job @@ -106,16 +84,10 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 3 pr_step_number: 4 + verify_type: go intro: You are a package reviewer for Devsy's system/platform/utility category (the largest category — rotate packages across runs). The devsy-org/devsy repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE package, review it, make ONE small focused improvement, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- - STEP 1 — Pick ONE package from the scope (rotate across runs). Review for: cross-platform correctness (Linux/macOS/Windows), path handling, lowercase logging, secret masking in pkg/secrets/pkg/token, and edge-case tests. Never print secret/token values; mask in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. - - STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. + STEP 1 — Pick ONE package from the scope (rotate across runs). Review for: cross-platform correctness (Linux/macOS/Windows), path handling, lowercase logging, secret masking in pkg/secrets/pkg/token, and edge-case tests. Never print secret/token values; mask in logs. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. If your change touched pkg/git, ensure you didn't introduce NEW failures. pr_body_must_include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test) constraints: ONE package per run. Never print secret/token values; mask in logs. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the OS, platform, and utility packages for small focused improvements. @@ -135,7 +107,6 @@ agents: `agent.md` for human review when scope boundaries shift. - id: pkg-tunnel-network branch_prefix: pkg-tunnel-network - git_add_glob: commit_subject: '"" ""' pr_title: job_name: package review job @@ -143,17 +114,11 @@ agents: step0_title: Check for and install the Go toolchain + protoc if missing commit_step_number: 3 pr_step_number: 4 + verify_type: go + verify_extras: "If tunnel proto changed: task cli:build:grpc" intro: You are a package reviewer for Devsy's tunneling/networking category. The devsy-org/devsy repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE package, review it, make ONE small focused improvement, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Pick ONE package from the scope. Review for: connection/goroutine leaks, context cancellation handling, port conflict edge cases, gRPC error codes, and lowercase logging. Do not regenerate gRPC stubs without running task cli:build:grpc. Prefer adding/fixing a test over a behavioral change. Make ONE small, focused change reviewable in 15 minutes. - - STEP 2 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - - If tunnel proto changed: task cli:build:grpc - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. pr_body_must_include: the package reviewed, the issue found, the change, the verification performed (lint:ci, cli:test, cli:build:grpc if proto touched) constraints: ONE package per run. Do not regenerate gRPC stubs without running task cli:build:grpc. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the tunneling, HTTP, and networking packages for small focused improvements. @@ -167,7 +132,6 @@ agents: `agent.md` for human review when scope boundaries shift. - id: cmd-reviewer branch_prefix: cmd-reviewer - git_add_glob: commit_subject: '"" ""' pr_title: job_name: CLI review job @@ -175,18 +139,12 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 4 pr_step_number: 5 + verify_type: go intro: You are the CLI command reviewer for the devsy-org/devsy repository. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Review ONE cmd/ subdirectory per run and open ONE app-signed PR with a small improvement. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Pick ONE cmd/ subdirectory to review. Enumerate with: ls cmd/. Pick one. Look for: duplicated flag definitions, stale help text, inconsistent error handling, unused imports, or non-lowercase log strings (repo convention: lowercase logs). Respect existing flag aliases registered in cmd/flag_aliases_test.go. STEP 2 — Make ONE small, focused improvement reviewable in ~20 minutes. Prefer correctness/help-text fixes over behavior changes. - - STEP 3 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # checks ONLY new issues introduced by your change (mirrors CI). Must be 0 issues. - - task cli:test # unit tests. - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, that is the baseline; proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. pr_body_must_include: the cmd/ subdirectory reviewed, the issue found, the minimal change, the verification performed (task cli:lint:ci result, task cli:test result) constraints: ONE cmd/ subdirectory, ONE improvement, ONE commit, ONE PR per run (plus at most ONE follow-up fix commit if a status check fails — see the ensure-status-checks step). Keep it reviewable in ~20 minutes. If no actionable issue found, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the cmd/ CLI directory for small improvements to command structure, flags, and help text. @@ -199,7 +157,6 @@ agents: `agent.md` for human review when review priorities change. - id: lint-fixer branch_prefix: lint-fixer - git_add_glob: commit_subject: '"style: " ""' pr_title: 'style: ' job_name: lint fix job @@ -207,6 +164,8 @@ agents: step0_title: Check for and install the Go toolchain + Node if missing commit_step_number: 4 pr_step_number: 5 + verify_type: go+desktop + verify_extras: "(cd desktop && npm run check) # svelte-check / type check, if web files touched" intro: You are the lint fixer for the devsy-org/devsy repository. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE category of lint findings, apply mechanical fixes, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |2- (cd desktop && npm install --ignore-scripts >/dev/null 2>&1 || true) @@ -216,14 +175,6 @@ agents: - (cd desktop && npx biome check) # web: biome STEP 2 — Pick ONE category of findings. Apply mechanical fixes via task cli:lint:fix (Go) or (cd desktop && npx biome check --write) (web). For non-mechanical findings, fix at most 1-3 occurrences so the diff stays reviewable in 15 minutes. No behavioral changes. Do NOT disable linters to silence findings. - - STEP 3 — Verify (CRITICAL — use CI-equivalent lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues introduced. - - task cli:test - - (cd desktop && npm run check) # svelte-check / type check, if web files touched - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different category. pr_body_must_include: the linter + category, the findings fixed (count), the fix method (mechanical/manual), the verification performed (lint:ci, cli:test, desktop check if web touched) constraints: No behavioral changes — formatting and lint fixes only. Do not disable linters. If no lint findings, do nothing and report "no lint findings to fix". Never use GITHUB_TOKEN for the commit or PR. description: Detects linting errors across the repo and makes small focused fixes. @@ -238,7 +189,6 @@ agents: for human review when lint rules change. - id: devcontainer-spec branch_prefix: devcontainer-spec - git_add_glob: commit_subject: '"" ""' pr_title: job_name: devcontainer spec job @@ -246,6 +196,8 @@ agents: step0_title: Check for and install the Go toolchain + devcontainer CLI if missing commit_step_number: 5 pr_step_number: 6 + verify_type: go + verify_extras: "If feasible: task cli:test:e2e:suite -- \"devcontainer\"" intro: You are the devcontainer specification reviewer for the devsy-org/devsy repository. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Find ONE declared-but-unimplemented devcontainer spec requirement or stale feature lock/integrity and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Read .devcontainer/devcontainer.json and the devcontainer spec (https://raw.githubusercontent.com/devcontainers/spec/refs/heads/main/schemas/devContainer.base.schema.json, https://raw.githubusercontent.com/devcontainers/spec/refs/heads/main/schemas/devContainerFeature.schema.json, https://containers.dev/implementors/json_schema/). Read .devcontainer/devcontainer-lock.json, .github/workflows/devcontainer.yml, and e2e/devcontainer-feature.json. @@ -253,14 +205,6 @@ agents: STEP 2 — Identify ONE spec requirement that is declared but not implemented, OR a feature whose lock/integrity (SHA) is stale. Keep the change reviewable in 15 minutes. Do NOT bump feature versions without verifying the SHA/integrity. Pick exactly ONE. STEP 3 — Implement or correct it. If features changed, regenerate the lock via: devcontainer build --workspace-folder . --image-name /tmp/dc-test 2>/dev/null || true (best-effort; lock regeneration may require docker). - - STEP 4 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test - - If feasible: task cli:test:e2e:suite -- "devcontainer" - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different improvement. pr_body_must_include: the spec requirement, the gap found, the change, the verification performed (lint:ci, cli:test, e2e result) constraints: ONE requirement per run. Do not bump feature versions without verifying the SHA/integrity. If no actionable requirement found, do nothing and report "no actionable requirement found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the devcontainer specification, finds unimplemented requirements, and attempts focused implementation. @@ -274,7 +218,6 @@ agents: for human review when spec compliance scope changes. - id: integration-test branch_prefix: integration-test - git_add_glob: commit_subject: '"test(e2e): " ""' pr_title: 'test(e2e): ' job_name: e2e review job @@ -282,6 +225,8 @@ agents: step0_title: Check for and install the Go toolchain if missing (Ginkgo comes with the Go toolchain) commit_step_number: 4 pr_step_number: 5 + verify_type: go + verify_extras: "Run the focused suite: task cli:test:e2e:suite -- \"\" (or task cli:test:e2e:focus -- \"\")" intro: You are the integration-test reviewer for the devsy-org/devsy repository. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Pick ONE e2e/test suite, fix flakiness or add an edge-case test, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Authenticate gh read-only with the Devsy GitHub App token to inspect past runs: @@ -290,14 +235,6 @@ agents: Pick ONE failing or flaky test suite from e2e/tests/. Inspect its logs via gh run view --repo devsy-org/devsy --log | tail -200. STEP 2 — Fix the flaky test OR add an edge-case test the logs reveal as uncovered. Use real code paths, not mocks. Use the e2e framework helpers (e2e/framework/). Keep the change reviewable in 15 minutes. - - STEP 3 — Verify (CRITICAL): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # Must be 0 new issues. - - task cli:test # unit tests (exclude e2e). - - Run the focused suite: task cli:test:e2e:suite -- "" (or task cli:test:e2e:focus -- "") - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different test. pr_body_must_include: the suite/test, the logs inspected (run id), the fix/edge-case added, the verification performed (lint:ci, cli:test, focused e2e suite result) constraints: ONE suite/test per run. Use real code paths, not mocks. If no actionable flakiness or uncovered edge case found, do nothing and report "no actionable e2e issue found". Never use GITHUB_TOKEN for the commit or PR. description: Reviews the e2e integration tests, fixes issues, and adds edge-case tests using past run logs as reference. @@ -310,7 +247,6 @@ agents: to this `agent.md` for human review when review priorities change. - id: docs-keeper branch_prefix: docs-keeper - git_add_glob: commit_subject: '"docs: " ""' pr_title: 'docs: ' job_name: docs job @@ -341,7 +277,6 @@ agents: review when documentation scope changes. - id: ci-optimizer branch_prefix: ci-optimizer - git_add_glob: commit_subject: '"" ""' pr_title: job_name: CI optimization job @@ -390,7 +325,6 @@ agents: propose a `description` amendment to this `agent.md` for human review. - id: codefactor branch_prefix: fix/codefactor - git_add_glob: commit_subject: '"fix: " ""' pr_title: 'fix: ' job_name: Codefactor issue fix job @@ -398,6 +332,7 @@ agents: step0_title: Check for and install the Go toolchain if missing (Go, task, golangci-lint) commit_step_number: 5 pr_step_number: 6 + verify_type: go intro: You are the Codefactor issue fixer for the devsy-org/devsy repository. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Fix exactly ONE Codefactor issue per run and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. review_steps: |- STEP 1 — Fetch ALL Codefactor issues (deterministic, paginated, no auth): @@ -425,14 +360,7 @@ agents: STEP 2 — Pick ONE issue (any category: Maintainability, Duplication, Complexity). Prefer the most contained change reviewable in ~20 minutes. Pick exactly ONE. (Open PRs were already listed in STEP 0 — do not pick an issue addressed by one of them.) - STEP 3 — Fix it: read FilePath around StartLine..EndLine, make the minimal correct change. Do NOT disable linters or add workaround comments. - - STEP 4 — Verify (CRITICAL — use CI-equivalent lint, not the full lint): - - mkdir -p dist - - git fetch --quiet origin main - - task cli:lint:ci # checks ONLY new issues introduced by your change (mirrors CI). Must be 0 issues. - - task cli:test # unit tests. - KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, that is the baseline; proceed. If your change touched pkg/git, ensure you didn't introduce NEW failures. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different issue. + STEP 3 — Fix it: read FilePath around StartLine..EndLine, make the minimal correct change. Do NOT disable linters or add workaround comments. If your change touched pkg/git, ensure you didn't introduce NEW failures. pr_body_must_include: the Codefactor issue Key, file:line, the Codefactor message, the fix summary, and that task cli:lint:ci passed (0 new issues) + task cli:test status. Add label "codefactor" if it exists. constraints: ONE issue, ONE commit, ONE PR per run (plus at most ONE follow-up fix commit if a status check fails — see the ensure-status-checks step). Keep it reviewable in ~20 minutes. If no fixable issue today, do nothing and report "no actionable issue found". Never use GITHUB_TOKEN for commit/PR. description: Fixes one Codefactor issue per run by making a minimal, lint-clean change and opening an app-signed PR. @@ -447,7 +375,6 @@ agents: issue-selection priorities change. - id: ui-polish branch_prefix: ui-polish - git_add_glob: commit_subject: '"fix(ui): " ""' pr_title: 'fix(ui): ' job_name: UI polish job @@ -487,3 +414,61 @@ agents: fragility, svelte-check gotchas) to the running automation via the automation service — not the git repo — using the service's agentic memory if available. Propose a `description` amendment to this `agent.md` for human review when polish priorities change. +- id: agent-analytics + branch_prefix: agent-analytics + commit_subject: '"" ""' + pr_title: + job_name: agent analytics job + toolchain: go+python + step0_title: Check for and install the Go toolchain + uv if missing + commit_step_number: 6 + pr_step_number: 7 + intro: You are the analytics engineer for the devsy-org/devsy agent fleet. The repo is cloned in your workspace on branch main. Follow AGENTS.md conventions (lowercase logs, Conventional Commits, 50-char subject max). Review the previous day's agent runs, reduce the errors and failures agents encounter by applying deterministic methods that lower cognitive load, and open ONE app-signed PR. Never use GITHUB_TOKEN for the commit or PR — authenticate as the Devsy GitHub App. + review_steps: |- + STEP 1 — Gather yesterday's run data deterministically. The repo ships hack/analytics/analyze_runs.py, a self-contained uv script (PEP 723 inline deps: typer, numpy, scikit-learn, pandas, seaborn, matplotlib, prettytable). Run it ONLY with uv run (never pip/venv) and yesterday's date window: + uv run hack/analytics/analyze_runs.py --since yesterday --until today \ + --out-dir dist/analytics + It authenticates with OPENHANDS_API_KEY (read-only), lists app-conversations created in the window, pulls each conversation's events, and writes: + - dist/analytics/runs.csv (one row per run: agent, status, duration, event/tool counts, error count) + - dist/analytics/failure_clusters.csv (sklearn KMeans clusters of failure signatures) + - dist/analytics/top_errors.csv (ranked recurring error signatures) + - dist/analytics/daily_report.md (human-readable summary with recommendations) + - dist/analytics/failure_heatmap.png (seaborn visualization of failure mode x agent) + If OPENHANDS_API_KEY is unavailable OR no runs exist for the window, fall back to the bundled synthetic sample so the pipeline still produces outputs: add `--sample`. Treat the API as the source of truth when available; the sample is only for offline development. + The analyzer also zips the five outputs and uploads them to https://temp.sh/upload (3-day expiry, returns a plain-text share URL), then writes a `## pr-gate` verdict at the end of daily_report.md: ACTIONABLE (top failure signature recurs across >=2 runs at >=10% share of failed runs) or NOT-ACTIONABLE. Set ANALYTICS_NO_UPLOAD=1 to skip the upload (e.g. in an offline sandbox). If temp.sh is unreachable or rejects the upload the upload is recorded as skipped with the reason; the pipeline still completes. + + STEP 2 — Analyze the daily_report.md and top_errors.csv. Identify ONE high-impact, deterministic intervention that reduces the cognitive load agents spend recovering from errors, so reasoning budget stays on the primary task. Prefer, in order of reviewability: + - A script/automation fix in hack/analytics/ or .agents/ that codifies a recurring recovery (e.g. a missing-tool check, a PR-duplicate guard, a deterministic pre-flight). + - A prompt (agent.md) clarification that removes an ambiguity the data shows agents repeatedly stumble on (stale assertion, wrong resolution order, missing masking rule). Prompt edits must be regenerated through hack/automations/agents.yaml — never hand-edit a generated agent.md. + - A small code fix in pkg/ or cmd/ that eliminates a recurring failure mode the data pinpoints. + Pick exactly ONE. The intervention must be justified by a number from the analytics output (cluster size, error count, recurrence rate). Quote the metric in the PR body. + + STEP 3 — Implement the ONE intervention. Keep it minimal and reviewable in ~20 minutes. If it touches a generated agent prompt, edit hack/automations/agents.yaml and regenerate: `go run ./hack/automations` then confirm `go run ./hack/automations -check` is clean. + + STEP 4 — Verify (CRITICAL — use CI-equivalent lint): + - mkdir -p dist + - git fetch --quiet origin main + - task cli:lint:ci # Must be 0 new issues. + - task cli:test + - uv run hack/analytics/analyze_runs.py --sample --out-dir /tmp/analytics-verify # smoke test the pipeline (sets ANALYTICS_NO_UPLOAD=1 to skip the live upload in CI) + - uvx --from mypy mypy --strict --ignore-missing-imports --disable-error-code untyped-decorator hack/analytics/analyze_runs.py # static typing (must be 0 issues) + - uvx --from radon radon cc -s hack/analytics/analyze_runs.py | grep -oE '\([0-9]+\)' | tr -d '()' | sort -n -r | head -1 # cyclomatic complexity; must be < 10 + KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on origin/main already — a stale assertion, NOT caused by your change. If ONLY pkg/git fails and your change did not touch pkg/git, proceed. If your change introduces any NEW lint issue or test failure, fix the root cause or pick a different intervention. + + STEP 5 — PR-GATE DECISION (read the `## pr-gate` verdict in daily_report.md): + - If the verdict is NOT-ACTIONABLE, DO NOT create a PR. Report "no actionable failure mode found — pr-gate NOT-ACTIONABLE" and stop. A run that opens a PR against a NOT-ACTIONABLE verdict is a FAILED run. + - If the verdict is ACTIONABLE, write the PR body to /tmp/pr_body.md. It MUST contain: the analytics window (since/until dates), the pr-gate verdict, the headline metric(s) from daily_report.md that motivated the change (e.g. "cluster_0: 7 runs, signature: missing-tool 'golangci-lint'"), the ONE intervention, the verification performed (lint:ci, cli:test, uv smoke run, mypy, radon complexity), the temp.sh upload link (or the skip reason if temp.sh was unreachable/rejected), and attach/reference the visualization (failure_heatmap.png path or PR-attached image). State the expected reduction in failure recurrence. + pr_body_must_include: the analytics window, the pr-gate verdict, the headline metric(s) from daily_report.md, the ONE intervention, the verification performed (lint:ci, cli:test, uv smoke run, mypy, radon complexity), the temp.sh upload link or skip reason, the visualization reference + constraints: ONE intervention per run, justified by a metric from the analytics output. Do not invent metrics. Run the script only via `uv run` (never pip/venv). If OPENHANDS_API_KEY is unavailable, use --sample and say so. ONLY create a PR when the pr-gate verdict in daily_report.md is ACTIONABLE; if NOT-ACTIONABLE, do nothing and report "no actionable failure mode found — pr-gate NOT-ACTIONABLE". Never use GITHUB_TOKEN for the commit or PR. + description: Reviews the previous day's agent-fleet runs with Python data-science tooling, clusters failure modes, and ships one deterministic intervention (script, prompt, or code fix) that lowers agent cognitive load and reduces recurring errors. + scope: |- + - `hack/analytics/` — the deterministic run-analysis pipeline (analyze_runs.py + outputs) + - `.agents/agents/`, `.agents/tasks/`, `hack/automations/agents.yaml` — agent prompts/schedules + - `pkg/`, `cmd/` — small code fixes only when the data pinpoints a recurring failure mode + - OpenHands Cloud API (`/api/v1/app-conversations/search`, `/api/v1/conversation/{id}/events/search`) read-only, via OPENHANDS_API_KEY + self_improvement: |- + At the end of each run, persist findings (recurring failure signatures, cluster centroids + that map to actionable fixes, metric baselines) to the running automation via the + automation service — not the git repo — using the service's agentic memory if available. + Propose a `description` amendment to this `agent.md` for human review when the analysis + method or intervention priorities change. diff --git a/hack/automations/main.go b/hack/automations/main.go index 8aa7f15957..4554677f12 100644 --- a/hack/automations/main.go +++ b/hack/automations/main.go @@ -32,7 +32,6 @@ const ( type agent struct { ID string `yaml:"id"` BranchPrefix string `yaml:"branch_prefix"` - GitAddGlob string `yaml:"git_add_glob"` CommitSubject string `yaml:"commit_subject"` PRTitle string `yaml:"pr_title"` JobName string `yaml:"job_name"` @@ -42,6 +41,8 @@ type agent struct { PRStepNumber int `yaml:"pr_step_number"` Intro string `yaml:"intro"` ReviewSteps string `yaml:"review_steps"` + VerifyType string `yaml:"verify_type"` + VerifyExtras string `yaml:"verify_extras"` PRBodyMustInclude string `yaml:"pr_body_must_include"` Constraints string `yaml:"constraints"` Description string `yaml:"description"` @@ -56,6 +57,7 @@ type config struct { type renderView struct { Agent agent Step0Body string + VerifyBlock string PRTitleJSON string StatusStepNumber int } @@ -120,6 +122,7 @@ func renderAgent(tmpl *template.Template, a agent) string { view := renderView{ Agent: a, Step0Body: installBlock(a.Toolchain), + VerifyBlock: verifyBlock(a), PRTitleJSON: escapeJSONString(a.PRTitle), StatusStepNumber: a.PRStepNumber + 1, } @@ -172,6 +175,8 @@ func installBlock(toolchain string) string { return joinLines(goInstallBlock(), goVerify, cdTidy) case "go+node": return joinLines(goInstallBlock(), nodeInstallBlock(), goVerifyNode(), cdTidy) + case "go+python": + return joinLines(goInstallBlock(), pythonInstallBlock(), goVerifyPython(), cdTidy) case "go+protoc": return joinLines(goInstallBlock(), protocInstall(), goVerify, cdTidy) case "go+devcontainer": @@ -185,6 +190,31 @@ func installBlock(toolchain string) string { } } +const knownGitFailure = "KNOWN PRE-EXISTING FAILURE: pkg/git tests (TestRepoClone*) fail on " + + "origin/main already — a stale assertion, NOT caused by your change. " + + "If ONLY pkg/git fails and your change did not touch pkg/git, proceed. " + + "If your change introduces any NEW lint issue or test failure, " + + "fix the root cause or pick a different improvement." + +func verifyBlock(a agent) string { + if a.VerifyType == "" { + return "" + } + stepNum := a.CommitStepNumber - 1 + lines := []string{ + fmt.Sprintf("STEP %d — Verify (CRITICAL — use CI-equivalent lint):", stepNum), + " - mkdir -p dist", + " - git fetch --quiet origin main", + " - task cli:lint:ci # Must be 0 new issues.", + " - task cli:test", + } + if a.VerifyExtras != "" { + lines = append(lines, " - "+a.VerifyExtras) + } + lines = append(lines, " "+knownGitFailure) + return joinLines(lines...) +} + func goInstallBlock() string { golangciExtract := "tar -C /tmp -xzf /tmp/gcl.tgz && sudo mv /tmp/golangci-lint-" + golangciVersion + "-linux-amd64/golangci-lint /usr/local/bin/ && " + @@ -221,6 +251,20 @@ func goVerifyNode() string { return " go version && task --version && golangci-lint --version && node --version" } +func pythonInstallBlock() string { + return joinLines( + " # Check for uv — install only if missing (scripts declare deps inline via PEP 723)", + " if ! command -v uv >/dev/null 2>&1; then", + " curl -LsSf https://astral.sh/uv/install.sh | sh", + " fi", + ) +} + +func goVerifyPython() string { + return " go version && task --version && golangci-lint --version && " + + "uv --version && uv run hack/analytics/analyze_runs.py --sample --out-dir /tmp/analytics-verify >/dev/null" +} + func protocInstall() string { return joinLines( " # Check for protoc — install only if missing", diff --git a/hack/automations/main_test.go b/hack/automations/main_test.go index 8fa963e76f..ee14418b4f 100644 --- a/hack/automations/main_test.go +++ b/hack/automations/main_test.go @@ -46,7 +46,10 @@ func TestGenerateWritesAllAgents(t *testing.T) { "ci-optimizer", "cmd-reviewer", "devcontainer-spec", "docs-keeper", "integration-test", "lint-fixer", "pkg-container", "pkg-core-agent", "pkg-ssh-git", "pkg-system-platform", "pkg-tunnel-network", + "agent-analytics", } + + pythonAgents := map[string]bool{"agent-analytics": true} for _, id := range ids { p := agentFilePath(t, id) b, err := os.ReadFile(p) @@ -57,10 +60,9 @@ func TestGenerateWritesAllAgents(t *testing.T) { s := string(b) for _, want := range []string{ "Do NOT run `git commit`", - "Pre-PR check (CRITICAL)", - "exactly ONE commit", - "createCommitOnBranch", + "verified=true", "sign_commit", + "-pr-only", "---\nid:", "## Scope", "## Self-improvement", @@ -73,7 +75,13 @@ func TestGenerateWritesAllAgents(t *testing.T) { t.Errorf("%s: still references app_commit.py", id) } if strings.Contains(s, "pip install") { - t.Errorf("%s: still installs pip packages", id) + t.Errorf("%s: still uses pip install; use uv run with PEP 723 inline deps", id) + } + if strings.Contains(s, "python3 -m venv") || strings.Contains(s, "venv/bin/python") { + t.Errorf("%s: still uses a venv; use uv run only", id) + } + if pythonAgents[id] && !strings.Contains(s, "uv run hack/analytics/analyze_runs.py") { + t.Errorf("%s: expected uv run invocation for the analytics script", id) } } } diff --git a/hack/automations/template.md.tmpl b/hack/automations/template.md.tmpl index fc56d472c4..1187d36467 100644 --- a/hack/automations/template.md.tmpl +++ b/hack/automations/template.md.tmpl @@ -26,8 +26,13 @@ flight. If every candidate is already covered, do nothing and report "no actiona found — all candidates already have open PRs". {{.Agent.ReviewSteps}} - +{{if .VerifyBlock}} +{{.VerifyBlock}} +{{end}} FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): + Write self-documenting code: clear names, small functions, obvious structure. + Avoid wordy comments — prefer no comments unless the code expresses something + genuinely unintuitive (a non-obvious invariant, workaround, or trade-off). - git fetch --quiet origin main - task cli:format # auto-format Go code (gofmt, gci, gofumpt via golangci-lint fmt) - task cli:lint:ci # Must be 0 new issues. If any issue appears, fix it and re-run. @@ -37,39 +42,22 @@ FORMATTING GATE (CRITICAL — run BEFORE committing, must pass): Common formatting failures: gci (import ordering), gofumpt (struct/function spacing), golines (line length > 120). Running task cli:format first auto-fixes most of these. -STEP {{.Agent.CommitStepNumber}} — Commit via the Devsy GitHub App (signed, verified). The repo -ships a Go tool (hack/sign_commit, JWT via golang-jwt/jwt/v5) that authenticates as the -app installation and creates the commit through the GraphQL createCommitOnBranch mutation, -so GitHub signs it (committer: GitHub, verified). No Python, no GITHUB_TOKEN. +STEP {{.Agent.CommitStepNumber}} — Commit via the Devsy GitHub App (signed, verified). +The repo ships a Go tool (hack/sign_commit) that authenticates as the app installation +and creates the commit through GitHub's GraphQL API, so GitHub signs it (committer: +web-flow, verified). It auto-detects all working-tree changes (staged, unstaged, and +untracked files) and auto-creates the remote branch from origin/main if needed. +Do NOT run `git commit` locally — it produces an unsigned commit that fails the signature +check. - git fetch --quiet origin main - git checkout -b {{.Agent.BranchPrefix}}/ origin/main - - git add {{.Agent.GitAddGlob}} - - WARNING: Do NOT run `git commit` locally. The sign_commit tool creates the commit via - the GitHub `createCommitOnBranch` mutation (committer: GitHub, verified). A local - `git commit` would produce an UNSIGNED commit that the mutation - does NOT replace — both commits would land on the branch and the unsigned one fails the - signature status check. Only `git add` to stage. - task github:app:sign-commit -- -m {{.Agent.CommitSubject}} - Confirm output: verified=true. - - Pre-PR check (CRITICAL): confirm the branch has exactly ONE commit ahead of - origin/main and it is verified. Run: - git log --oneline origin/main..HEAD - If there is MORE than one commit (e.g. an extra unsigned local commit), reset and - redo cleanly: - git reset --hard origin/main - # re-apply your edit, then: git add {{.Agent.GitAddGlob}} - task github:app:sign-commit -- -m {{.Agent.CommitSubject}} - git log --oneline origin/main..HEAD # must show exactly ONE commit - Only proceed to open the PR once exactly one verified commit is on the branch. STEP {{.Agent.PRStepNumber}} — Open the PR as the app (no GITHUB_TOKEN): - - TOKEN=$(task github:app:sign-commit -- -token) - Write the PR body to /tmp/pr_body.md. It MUST include: {{.Agent.PRBodyMustInclude}}, a line "This PR was created by an AI agent as part of an automated daily {{.Agent.JobName}}." - - PR_BODY=$(python3 -c 'import json,sys;print(json.dumps(open("/tmp/pr_body.md").read()))') - - PR_JSON=$(curl -s -X POST -H "Authorization: bearer $TOKEN" -H "Accept: application/vnd.github+json" https://api.github.com/repos/devsy-org/devsy/pulls -d "{\"title\":\"{{.PRTitleJSON}}\",\"head\":\"{{.Agent.BranchPrefix}}/\",\"base\":\"main\",\"draft\":true,\"body\":${PR_BODY}}") - - PR_NUMBER=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["number"])' <<< "$PR_JSON") - - PR_URL=$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["html_url"])' <<< "$PR_JSON") - - Report the PR URL and commit SHA. A run that does not produce a PR URL is a FAILED run. + - task github:app:sign-commit -- -pr-only -title {{.PRTitleJSON}} -b "$(cat /tmp/pr_body.md)" + - Report the PR URL from the output. A run that does not produce a PR URL is a FAILED run. STEP {{.StatusStepNumber}} — Ensure status checks pass (lint failures are the most common reason a daily PR needs a follow-up): GitHub runs the `Lint` job (golangci-lint-action, `only-new-issues: true`) and, for desktop @@ -94,7 +82,6 @@ STEP {{.StatusStepNumber}} — Ensure status checks pass (lint failures are the - Go files: `task cli:format && task cli:lint:ci` (0 new issues) and, if relevant, `task cli:test`. - Desktop files: `cd desktop && npx biome check --write && npm run check`. 4. Stage the fix and push a follow-up commit via the app (same flow as the original commit): - - git add - task github:app:sign-commit -- -m "" "" - git log --oneline origin/main..HEAD # the fix is a SECOND commit on the branch 5. Re-poll `gh pr checks "$PR_NUMBER"` until the check that failed is now `SUCCESS`/`NEUTRAL`. diff --git a/hack/sign_commit/main.go b/hack/sign_commit/main.go index be4241fc15..e0f3be0e2a 100644 --- a/hack/sign_commit/main.go +++ b/hack/sign_commit/main.go @@ -16,8 +16,9 @@ // Usage: // // task github:app:sign-commit -- -m "subject" [-b "body"] [files...] -// -// With no file paths, changed files vs origin/main are committed. +// task github:app:sign-commit -- -m "subject" -pr (commit + open draft PR) +// task github:app:sign-commit -- -m "subject" -pr-only (open draft PR only) +// task github:app:sign-commit -- -token (print the app installation token) package main import ( @@ -46,6 +47,8 @@ const ( apiBase = "https://api.github.com" maxExpiration = 10 * time.Minute issuedAtSkew = 60 * time.Second + mainBranch = "main" + bodyKey = "body" ) type addition struct { @@ -90,6 +93,10 @@ type options struct { branch string all bool token bool + pr bool + prOnly bool + title string + draft bool paths []string } @@ -103,8 +110,23 @@ func main() { body := flag.String("b", "", "commit body (optional)") repo := flag.String("repo", defaultRepo, "owner/name repository") branch := flag.String("branch", "", "branch (default: current)") - all := flag.Bool("all", true, "commit all changed files vs origin/main when no paths given") + all := flag.Bool( + "all", + true, + "commit all working-tree changes vs origin/main when no paths given", + ) token := flag.Bool("token", false, "print the app installation token and exit (no commit)") + pr := flag.Bool( + "pr", + false, + "create a draft PR after committing (uses -title or -m for the PR title)", + ) + prOnly := flag.Bool( + "pr-only", false, + "create a draft PR without committing (uses -title or -m for title, -b for body)", + ) + title := flag.String("title", "", "PR title (default: commit subject from -m)") + draft := flag.Bool("draft", true, "create the PR as draft (use -draft=false for a ready PR)") flag.Parse() opts := options{ @@ -114,6 +136,10 @@ func main() { branch: *branch, all: *all, token: *token, + pr: *pr, + prOnly: *prOnly, + title: *title, + draft: *draft, paths: flag.Args(), } if err := run(opts); err != nil { @@ -136,12 +162,36 @@ func run(o options) error { fmt.Println(token) return nil } - return commitFiles(o, token) + if o.prOnly { + return createPRFromOptions(token, o) + } + _, err = commitFiles(o, token) + if err != nil { + return err + } + if o.pr { + return createPR(token, o, o.branch) + } + return nil } -func commitFiles(o options, token string) error { +func createPRFromOptions(token string, o options) error { + branch := o.branch + if branch == "" { + b, err := currentBranch() + if err != nil { + return fmt.Errorf("detect branch: %w", err) + } + branch = b + } + return createPR(token, o, branch) +} + +func commitFiles(o options, token string) (string, error) { if o.message == "" { - return fmt.Errorf("-m commit subject is required (or pass -token to print the app token)") + return "", fmt.Errorf( + "-m commit subject is required (or pass -token to print the app token)", + ) } branch := o.branch @@ -149,40 +199,41 @@ func commitFiles(o options, token string) error { var err error branch, err = currentBranch() if err != nil { - return fmt.Errorf("detect branch: %w", err) + return "", fmt.Errorf("detect branch: %w", err) } + o.branch = branch } paths, err := resolvePaths(o.all, o.paths) if err != nil { - return err + return "", err } - head, err := refSHA(token, o.repo, branch) + head, err := ensureBranch(token, o.repo, branch) if err != nil { - return fmt.Errorf("read branch head: %w", err) + return "", err } adds, dels, err := fileChanges(paths, os.ReadFile) if err != nil { - return err + return "", err } vars := commitVars(o, branch, head, fileChange{Additions: adds, Deletions: dels}) return publishCommit(token, o.repo, vars) } -func publishCommit(token, repo string, vars createCommitVars) error { +func publishCommit(token, repo string, vars createCommitVars) (string, error) { sha, err := createCommit(token, vars) if err != nil { - return fmt.Errorf("create commit: %w", err) + return "", fmt.Errorf("create commit: %w", err) } verified, err := commitVerified(token, repo, sha) if err != nil { - return fmt.Errorf("verify commit: %w", err) + return "", fmt.Errorf("verify commit: %w", err) } fmt.Printf("%s verified=%v\n", sha, verified) - return nil + return sha, nil } func appToken(owner string) (string, error) { @@ -367,6 +418,70 @@ func refSHA(token, repo, branch string) (string, error) { return sha, nil } +// ensureBranch reads the remote branch SHA, auto-creating the branch from +// origin/main when it does not yet exist. This removes the need for agents to +// manually create the remote ref before committing. +func ensureBranch(token, repo, branch string) (string, error) { + sha, err := refSHA(token, repo, branch) + if err == nil { + return sha, nil + } + mainSHA, mErr := refSHA(token, repo, mainBranch) + if mErr != nil { + return "", fmt.Errorf( + "branch %q missing and cannot read origin/main to create it: %w", + branch, + mErr, + ) + } + if cErr := createBranchRef(token, repo, branch, mainSHA); cErr != nil { + return "", fmt.Errorf("auto-create branch %q: %w", branch, cErr) + } + fmt.Printf("created remote branch %s from origin/main (%s)\n", branch, mainSHA) + return mainSHA, nil +} + +func createBranchRef(token, repo, branch, sha string) error { + payload := map[string]any{ + "ref": "refs/heads/" + branch, + "sha": sha, + } + _, err := ghPost(token, fmt.Sprintf("%s/repos/%s/git/refs", apiBase, repo), payload) + return err +} + +// createPR opens a pull request for the branch after a successful commit. +func createPR(token string, o options, branch string) error { + title := o.title + if title == "" { + title = o.message + } + prBody := o.body + if prBody == "" { + prBody = "This PR was created by an AI agent on behalf of the user." + } + payload := map[string]any{ + "title": title, + "head": resolveBranchName(branch), + "base": mainBranch, + bodyKey: prBody, + "draft": o.draft, + } + body, err := ghPost(token, fmt.Sprintf("%s/repos/%s/pulls", apiBase, o.repo), payload) + if err != nil { + return fmt.Errorf("create PR: %w", err) + } + url, _ := body["html_url"].(string) + num, _ := body["number"].(float64) + fmt.Printf("PR #%d: %s (draft=%v)\n", int(num), url, o.draft) + return nil +} + +// resolveBranchName returns the branch name without a refs/heads/ prefix. +func resolveBranchName(branch string) string { + return strings.TrimPrefix(branch, "refs/heads/") +} + func commitVerified(token, repo, sha string) (bool, error) { body, err := ghGet(token, fmt.Sprintf("%s/repos/%s/commits/%s", apiBase, repo, sha)) if err != nil { @@ -476,13 +591,29 @@ func currentBranch() (string, error) { func changedFiles() ([]string, error) { base, err := gitOutput("merge-base", "HEAD", "origin/main") if err != nil { - return nil, err + base = "origin/main" } - out, err := gitOutput("diff", "--name-only", base, "HEAD") + tracked, err := gitOutput("diff", "--name-only", base) if err != nil { - return nil, err + return nil, fmt.Errorf("list tracked changes: %w", err) + } + untracked, err := gitOutput("ls-files", "--others", "--exclude-standard") + if err != nil { + return nil, fmt.Errorf("list untracked files: %w", err) } - return splitLines(out), nil + seen := make(map[string]bool) + var files []string + for _, f := range splitLines(tracked + "\n" + untracked) { + if f == "" || seen[f] { + continue + } + seen[f] = true + files = append(files, f) + } + if len(files) == 0 { + return nil, nil + } + return files, nil } func gitOutput(args ...string) (string, error) { diff --git a/hack/sign_commit/main_test.go b/hack/sign_commit/main_test.go index e506872790..65d6e72207 100644 --- a/hack/sign_commit/main_test.go +++ b/hack/sign_commit/main_test.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "encoding/pem" "os" + "os/exec" "path/filepath" "reflect" "strings" @@ -362,6 +363,77 @@ func TestUrlOrErr(t *testing.T) { } } +const testMainBranch = "main" + +func TestResolveBranchName(t *testing.T) { + const plain = "feature/foo" + cases := []struct { + name, input, want string + }{ + {"plain name", plain, plain}, + {"refs prefix", "refs/heads/" + plain, plain}, + {testMainBranch, "refs/heads/" + testMainBranch, testMainBranch}, + {"empty", "", ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := resolveBranchName(c.input); got != c.want { + t.Errorf("got %q, want %q", got, c.want) + } + }) + } +} + +func gitRun(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if err := cmd.Run(); err != nil { + t.Fatalf("git %v: %v", args, err) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestChangedFilesDetectsWorkingTree(t *testing.T) { + dir := t.TempDir() + gitRun(t, dir, "init", "--quiet") + gitRun(t, dir, "config", "user.email", "t@t.com") + gitRun(t, dir, "config", "user.name", "test") + gitRun(t, dir, "checkout", "-b", testMainBranch) + writeFile(t, filepath.Join(dir, "committed.txt"), "x") + writeFile(t, filepath.Join(dir, "staged.txt"), "s") + gitRun(t, dir, "add", ".") + gitRun(t, dir, "commit", "--quiet", "-m", "init") + gitRun(t, dir, "branch", "origin/main") + + writeFile(t, filepath.Join(dir, "modified.txt"), "hello") + writeFile(t, filepath.Join(dir, "committed.txt"), "modified") + gitRun(t, dir, "add", "staged.txt") + writeFile(t, filepath.Join(dir, "staged.txt"), "staged-mod") + + t.Chdir(dir) + files, err := changedFiles() + if err != nil { + t.Fatalf("changedFiles: %v", err) + } + got := make(map[string]bool) + for _, f := range files { + got[f] = true + } + want := []string{"committed.txt", "modified.txt", "staged.txt"} + for _, f := range want { + if !got[f] { + t.Errorf("expected %q in changed files, got %v", f, got) + } + } +} + func rsaPEM(t *testing.T, key *rsa.PrivateKey) []byte { t.Helper() der := x509.MarshalPKCS1PrivateKey(key)