diff --git a/.github/workflows/mypy-primer-analysis.lock.yml b/.github/workflows/mypy-primer-analysis.lock.yml index 44600cff3262..7e82ea7cff96 100644 --- a/.github/workflows/mypy-primer-analysis.lock.yml +++ b/.github/workflows/mypy-primer-analysis.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"dc4534f06f52affa0b2d9652d802049c655753d3b9c2c3aa7d3234ae44eef382","body_hash":"c556f78e4001fd9a12dff004bfbd395f50d9ec024e3d193e3013dc467ca3d31b","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-luna","detection_agent_id":"copilot","detection_agent_model":"detection","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"2925d6bb869d200f10327f82e923685622ae7d6ad419ed8d0ac4b5c09c6c0e6e","body_hash":"f0e8d12c9e674082ed44e124f78b9604501b234f37b5c76a3cbd10440501ffe2","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol","detection_agent_id":"copilot","detection_agent_model":"detection","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["actions_get","actions_list","get_commit","get_file_contents","get_job_logs","get_latest_release","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","list_branches","list_commits","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["publish_primer_analysis"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -148,7 +148,7 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: "gpt-5.6-luna" + GH_AW_INFO_MODEL: "gpt-5.6-sol" GH_AW_INFO_VERSION: "1.0.80" GH_AW_INFO_AGENT_VERSION: "1.0.80" GH_AW_INFO_CLI_VERSION: "v0.88.7" @@ -547,7 +547,7 @@ jobs: env: GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" - GH_AW_SAFE_OUTPUTS_CONFIG: "{\"publish-primer-analysis\":{\"description\":\"Required for every analysis. Submit one JSON report covering every manifest project. This queues validation by the trusted publisher, not a direct GitHub write.\",\"inputs\":{\"report\":{\"default\":null,\"description\":\"JSON object with a projects array in the format described in the workflow instructions.\",\"required\":true,\"type\":\"string\"}}},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"publish-primer-analysis\":{\"description\":\"Required once for each manifest project. Submit one project's JSON report per call, with its evidence. All projects must be submitted before analysis is complete. This queues validation by the trusted publisher, not a direct GitHub write.\",\"inputs\":{\"report\":{\"default\":null,\"description\":\"JSON object with a projects array containing exactly one project in the required format. Maximum 10240 UTF-8 bytes per call; preserve citations.\",\"required\":true,\"type\":\"string\"}},\"max\":100},\"report_incomplete\":{}}" with: script: | const path = require('path'); @@ -564,12 +564,12 @@ jobs: "repo_params": {}, "dynamic_tools": [ { - "description": "Required for every analysis. Submit one JSON report covering every manifest project. This queues validation by the trusted publisher, not a direct GitHub write.", + "description": "Required once for each manifest project. Submit one project's JSON report per call, with its evidence. All projects must be submitted before analysis is complete. This queues validation by the trusted publisher, not a direct GitHub write.", "inputSchema": { "additionalProperties": false, "properties": { "report": { - "description": "JSON object with a projects array in the format described in the workflow instructions.", + "description": "JSON object with a projects array containing exactly one project in the required format. Maximum 10240 UTF-8 bytes per call; preserve citations.", "type": "string" } }, @@ -789,7 +789,7 @@ jobs: export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) # shellcheck disable=SC2016 - printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.28.14/awf-config.schema.json","network":{"allowDomains":["*.githubusercontent.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","codeload.github.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","docs.github.com","github-cloud.githubusercontent.com","github-cloud.s3.amazonaws.com","github.blog","github.com","github.githubassets.com","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","lfs.github.com","objects.githubusercontent.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","patch-diff.githubusercontent.com","patchdiff.githubusercontent.com","ppa.launchpad.net","raw.githubusercontent.com","s.symcb.com","s.symcd.com","security.ubuntu.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"],"isolation":true,"topologyAttach":["awmg-mcpg"]},"apiProxy":{"enabled":true,"enableTokenSteering":false,"maxRuns":30,"maxCacheMisses":5,"maxAiCredits":25,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.5","gpt-5.6","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"auto":["copilot/auto","large"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex","kimi"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"detection":["small"],"evals":["small"],"fable":["copilot/*fable*","anthropic/*fable*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","google/nano-banana*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-3.6-flash":["copilot/gemini-3.6*flash*","google/gemini-3.6*flash*","gemini/gemini-3.6*flash*"],"gemini-3.7-flash":["copilot/gemini-3.7*flash*","google/gemini-3.7*flash*","gemini/gemini-3.7*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-omni":["copilot/gemini-omni*","google/gemini-omni*","gemini/gemini-omni*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.1":["copilot/gpt-5.1*","openai/gpt-5.1*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"gpt-5.6":["copilot/gpt-5.6*","openai/gpt-5.6*"],"grok":["copilot/*grok*","openai/*grok*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"image-generation":["copilot/gpt-image*","openai/gpt-image*","openai/chatgpt-image*","copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","google/imagen*"],"kimi":["copilot/kimi*","openai/kimi*"],"kiwi":["copilot/kiwi*","openai/kiwi*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"lyria":["google/lyria*","gemini/lyria*","copilot/lyria*"],"mai-code":["copilot/MAI-Code*","copilot/mai-code*","openai/MAI-Code*"],"mai-code-1-flash-picker":["copilot/MAI-Code-1-Flash-picker*","copilot/mai-code-1-flash-picker*","openai/MAI-Code-1-Flash-picker*"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"nano-banana":["copilot/nano-banana*","google/nano-banana*","gemini/nano-banana*"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"raptor-mini":["copilot/raptor*","openai/raptor*"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"small-agent":["haiku","gpt-5-mini","gemini-flash"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4.5*","copilot/*sonnet-4.6*","copilot/*sonnet-5*","copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*","anthropic/*sonnet-5*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"veo":["google/veo*","gemini/veo*"],"vision":["copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.28.14,squid=sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5,agent=sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98,api-proxy=sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5,cli-proxy=sha256:3a379c5e96e29499c815e9dd2a71334d01c326a9b73991c76544fda9cae35c34"},"logging":{"proxyLogsDir":"/tmp/gh-aw/sandbox/firewall/logs","auditDir":"/tmp/gh-aw/sandbox/firewall/audit"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.28.14/awf-config.schema.json","network":{"allowDomains":["*.githubusercontent.com","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","codeload.github.com","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","docs.github.com","github-cloud.githubusercontent.com","github-cloud.s3.amazonaws.com","github.blog","github.com","github.githubassets.com","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","lfs.github.com","objects.githubusercontent.com","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","patch-diff.githubusercontent.com","patchdiff.githubusercontent.com","ppa.launchpad.net","raw.githubusercontent.com","s.symcb.com","s.symcd.com","security.ubuntu.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com"],"isolation":true,"topologyAttach":["awmg-mcpg"]},"apiProxy":{"enabled":true,"enableTokenSteering":false,"maxRuns":30,"maxCacheMisses":5,"maxAiCredits":200,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.5","gpt-5.6","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"auto":["copilot/auto","large"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex","kimi"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"detection":["small"],"evals":["small"],"fable":["copilot/*fable*","anthropic/*fable*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","google/nano-banana*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-3.6-flash":["copilot/gemini-3.6*flash*","google/gemini-3.6*flash*","gemini/gemini-3.6*flash*"],"gemini-3.7-flash":["copilot/gemini-3.7*flash*","google/gemini-3.7*flash*","gemini/gemini-3.7*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-omni":["copilot/gemini-omni*","google/gemini-omni*","gemini/gemini-omni*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.1":["copilot/gpt-5.1*","openai/gpt-5.1*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"gpt-5.6":["copilot/gpt-5.6*","openai/gpt-5.6*"],"grok":["copilot/*grok*","openai/*grok*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"image-generation":["copilot/gpt-image*","openai/gpt-image*","openai/chatgpt-image*","copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","google/imagen*"],"kimi":["copilot/kimi*","openai/kimi*"],"kiwi":["copilot/kiwi*","openai/kiwi*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"lyria":["google/lyria*","gemini/lyria*","copilot/lyria*"],"mai-code":["copilot/MAI-Code*","copilot/mai-code*","openai/MAI-Code*"],"mai-code-1-flash-picker":["copilot/MAI-Code-1-Flash-picker*","copilot/mai-code-1-flash-picker*","openai/MAI-Code-1-Flash-picker*"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"nano-banana":["copilot/nano-banana*","google/nano-banana*","gemini/nano-banana*"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"raptor-mini":["copilot/raptor*","openai/raptor*"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"small-agent":["haiku","gpt-5-mini","gemini-flash"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4.5*","copilot/*sonnet-4.6*","copilot/*sonnet-5*","copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*","anthropic/*sonnet-5*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"veo":["google/veo*","gemini/veo*"],"vision":["copilot/gemini-*image*","google/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.28.14,squid=sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5,agent=sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98,api-proxy=sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5,cli-proxy=sha256:3a379c5e96e29499c815e9dd2a71334d01c326a9b73991c76544fda9cae35c34"},"logging":{"proxyLogsDir":"/tmp/gh-aw/sandbox/firewall/logs","auditDir":"/tmp/gh-aw/sandbox/firewall/audit"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" GH_AW_DOCKER_HOST="" @@ -814,7 +814,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: gpt-5.6-luna + COPILOT_MODEL: gpt-5.6-sol GH_AW_LLM_PROVIDER: github GH_AW_MAX_TURNS: 30 GH_AW_PHASE: agent @@ -1317,7 +1317,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} - GH_AW_MAX_AI_CREDITS: "25" + GH_AW_MAX_AI_CREDITS: "200" GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} diff --git a/.github/workflows/mypy-primer-analysis.md b/.github/workflows/mypy-primer-analysis.md index d070039c7cd9..fed6098f39a9 100644 --- a/.github/workflows/mypy-primer-analysis.md +++ b/.github/workflows/mypy-primer-analysis.md @@ -29,12 +29,12 @@ permissions: engine: id: copilot version: 1.0.80 - model: gpt-5.6-luna + model: gpt-5.6-sol harness: mypyPrimerCopilotHarness.cjs args: [--deny-tool, write, --deny-tool, shell] timeout-minutes: 15 max-turns: 30 -max-ai-credits: 25 +max-ai-credits: 200 sandbox: agent: token-steering: false @@ -226,7 +226,8 @@ safe-outputs: jobs: publish-primer-analysis: if: github.event_name != 'workflow_dispatch' - description: Required for every analysis. Submit one JSON report covering every manifest project. This queues validation by the trusted publisher, not a direct GitHub write. + description: Required once for each manifest project. Submit one project's JSON report per call, with its evidence. All projects must be submitted before analysis is complete. This queues validation by the trusted publisher, not a direct GitHub write. + max: 100 runs-on: ubuntu-latest permissions: contents: read @@ -234,7 +235,7 @@ safe-outputs: pull-requests: write inputs: report: - description: JSON object with a projects array in the format described in the workflow instructions. + description: JSON object with a projects array containing exactly one project in the required format. Maximum 10240 UTF-8 bytes per call; preserve citations. required: true type: string steps: @@ -287,6 +288,12 @@ Help the Pyright reviewer understand the ecosystem effect of the exact PR commit identified in `/tmp/gh-aw/primer-input/manifest.json`. This is advisory analysis, not a PR approval, a merge recommendation, or an instruction to modify code. +Your deliverable is a regression investigation, not a description of the diff. +Explain what checking or useful type information changed, why the PR could cause +that change, and what a downstream user can no longer rely on. Diagnostic counts +and project inventories are already available; do not use them as your summary. +A type-checker regression does not require a Python runtime bug. + Use native file-reading tools for local artifacts and MCP tools for GitHub reads and report submission. Shell execution and file editing are disabled; do not invoke `github` or `safeoutputs` as shell commands. @@ -304,7 +311,7 @@ the recorded primer run; its PR may already be merged or closed. Submit the same required report rather than stopping because of the PR's current state. The trusted workflow renders a preview and never posts a PR comment in this mode. -## Evidence and coverage +## Regression investigation 1. Read the manifest first. Its `projects` inventory and counts are deterministic and include every changed project across all eight shards, including projects @@ -316,6 +323,12 @@ trusted workflow renders a preview and never posts a PR comment in this mode. These summaries are not a replacement for the complete raw evidence. Raw diffs are under `/tmp/gh-aw/primer-input/raw/mypy_primer_diffs_N/diff_N.txt`. Do not analyze the truncated PR comment instead. + `regressionSignals`, when present, contains independently extracted leads: + `type-erasure`, `assertion-failure`, `removed-check`, and `gradual-detail`. + Investigate non-SymPy type-erasure and assertion failures first, then possible + suppressed checks and new gradual types. These are warning signals, not proof + of causation. Address them explicitly even if you think the change is intentional. + Do not spend most of the budget describing a large noisy SymPy diff. 2. Read the PR diff and relevant Pyright implementation/tests. `source.headSha` identifies the PR branch, but primer normally analyzes GitHub's synthetic merge commit. Use the recorded new/base commits in the source run's logs when @@ -323,7 +336,7 @@ trusted workflow renders a preview and never posts a PR comment in this mode. Do not assume current main, a published release, or the PR head is identical to that checkout. Use read-only GitHub tools; do not execute or install PR or ecosystem code. -3. Group changes by diagnostic rule and inferred-type transition. Distinguish +3. Group changes by a common causal mechanism, not just diagnostic rule. Distinguish added/removed diagnostics from message rewrites at the same location. Explain representative groups, including any group you could not investigate, for every changed project. Do not let one large project hide the others. Concise diffs @@ -332,6 +345,10 @@ trusted workflow renders a preview and never posts a PR comment in this mode. Zero added/removed headers does not mean no change. Explain detail-only type transitions and disclose missing context; do not fabricate diagnostic counts or locations. Treat detail-only precision loss as potentially significant. + For each important mechanism, inspect a representative affected call site and + its relevant overloads, annotations, or protocol declarations. Determine what + the old and new checker paths do with the same inputs. An assertion's expected + type is not itself an observed baseline reveal; label that distinction. 4. Prioritize added `reportAssertTypeFailure` diagnostics, loss of precision to `Any`/`Unknown`, and removed attribute/argument/operator errors that might have disappeared because of lost precision. More errors can also mean restored @@ -341,13 +358,29 @@ trusted workflow renders a preview and never posts a PR comment in this mode. Pin source citations to commits where possible. If the exact ecosystem revision or environment is unavailable, say so; current upstream source is not proof of what the recorded run analyzed. + A `likely-pr` attribution must cite a relevant implementation line at the + manifest's exact head SHA. Explain the changed branch or constraint decision + and the path from that decision to the observed result; a link alone is not an + explanation. If the causal path cannot be established, use `unclear`. 6. For overload changes, consider whether inferred results preserve operations supported by the retained candidate returns. Do not call a loss of supported operations an expected improvement merely because a container was preserved. + Also check the opposite failure: results such as Index[Any], ndarray, or a + shape-specialized object becoming bare Any/Unknown lose the outer API, not + merely an element parameter. Inspect whether members, argument checks, + operators, or inferred constructor types stop being checked. Propose a small + positive/negative control that would distinguish the old and new behavior. 7. Never infer correctness from a successful primer job, PR description, historical diagnostic count, or absence of `assert_type` failures. Separate static annotation gaps from actual runtime bugs. Do not claim a reproduction or historical comparison you did not perform. Use `needs-review` when evidence is insufficient. + In particular, do not dismiss new assertion failures or erased container types + as an "exposed typing issue" merely because the PR claims conformance or adds + tests expecting Any. Intentional behavior can still regress downstream checking. + Blaming a stub requires evidence that the declaration or expectation is wrong, + including consideration of counterevidence. Otherwise flag `possible-regression` + and state the attribution uncertainty. An uninspected overload set cannot + justify a high-confidence claim that the stubs need to change. 8. SymPy frequently has noisy primer differences. Do not attribute its changes to the PR solely because they appear in the diff. When only SymPy changed, the publisher uses a short non-blocking-noise notice, but you must still submit a @@ -361,7 +394,8 @@ the PR and rejects stale or incomplete reports. ## Output -Call `publish_primer_analysis` exactly once with `report` containing a JSON string: +Call `publish_primer_analysis` once for each manifest project, with `report` +containing a JSON string whose `projects` array has exactly one entry: ```json { @@ -370,9 +404,13 @@ Call `publish_primer_analysis` exactly once with `report` containing a JSON stri "name": "the exact project name from the manifest", "assessment": "needs-review", "confidence": "low", - "summary": "A concise explanation for the PR table, at most 240 characters.", - "explanation": "Evidence-backed explanation of the meaningful groups. Aim for 1800 characters; longer explanations are preserved in the full report.", - "unresolved": "Uninvestigated groups, missing provenance, or remaining questions; at most 800 characters. Use None identified only when justified.", + "attribution": "unclear", + "summary": "A specific regression hypothesis or investigation conclusion, not counts; at most 240 characters.", + "before": "Baseline behavior and its evidence. Distinguish observed types from assertion expectations or static inference; at most 1200 characters.", + "after": "Changed inferred type or checking behavior, with a representative location; at most 1200 characters.", + "impact": "What downstream operation, error check, API, or valid program is affected, not just that diagnostics changed; at most 1200 characters.", + "explanation": "Trace the PR's changed implementation through the affected declarations to the result. Explain counterevidence and why this may or may not be a checker regression. Aim for 1800 characters; longer analysis is preserved in the full report.", + "unresolved": "Remaining uncertainty and the exact next check or minimal positive/negative control needed. Do not claim it was run. At most 800 characters.", "evidence": [ { "url": "https://github.com/owner/repo/blob/COMMIT/path#L123", @@ -384,18 +422,37 @@ Call `publish_primer_analysis` exactly once with `report` containing a JSON stri } ``` +Each call has a 10240-byte UTF-8 input limit. Do not put all projects in one call. +Shorten redundant prose if needed, not required evidence. A successful call only +queues that project's analysis: continue until every manifest project has been +submitted exactly once. The trusted workflow assembles and validates the complete +report; missing or duplicate projects still fail. The combined report payload is +limited to 1000000 characters. + Include every manifest project exactly once, even when you run out of investigation -budget. Assessments are `expected-improvement`, `exposed-typing-issue`, +budget. Before each submission, check its required fields and citations: +non-`needs-review` assessments require evidence, and `likely-pr` requires a +line-pinned source citation at the manifest's exact head SHA. If the evidence is +unavailable, use `needs-review` and `unclear` rather than dropping citations from a +stronger claim. +Assessments are `expected-improvement`, `exposed-typing-issue`, `possible-regression`, or `needs-review`. Confidence is `low`, `medium`, or `high`. Use `possible-regression` for a credible problem needing investigation, not only proven defects. Mixed or incompletely investigated projects should not receive a blanket expected classification. Supply up to five evidence links per project, using `github.com` or `typing.python.org`; non-`needs-review` assessments require evidence. Do not emit placeholder URLs from this example. +Attribution is `likely-pr`, `unclear`, or `unlikely-pr`; it is separate from the +assessment and its confidence. For an uninvestigated project, explicitly say what +is unknown rather than inventing before/after behavior. Reserve enough budget to +submit the complete report; use the investigation budget on the highest-risk +mechanisms rather than exhaustive narration of routine changes. The publisher supplies the authoritative counts, commit, run links, and report -heading. It posts a compact per-project table and expandable explanations when -they fit. Each explanation is previewed up to 1800 characters in the comment, +heading. The comment leads with potential regressions and unresolved investigations, +including recorded warning signals even when the AI assessment is benign. Counts +remain in the full artifact rather than occupying the main comment. +Each causal explanation is previewed up to 1800 characters in the comment, with an explicit notice if truncated. The complete explanations and limitations are always retained in the `mypy-primer-analysis-report` artifact. Missing or empty explanations are still invalid; the overall report payload remains bounded. diff --git a/build/ci/mypyPrimerAnalysis.ts b/build/ci/mypyPrimerAnalysis.ts index a9ccd06212d3..73b9e945f152 100644 --- a/build/ci/mypyPrimerAnalysis.ts +++ b/build/ci/mypyPrimerAnalysis.ts @@ -39,6 +39,13 @@ interface ProjectDiff { detailsRemoved: DiffLine[]; } +interface RegressionSignal { + kind: 'type-erasure' | 'assertion-failure' | 'removed-check' | 'gradual-detail'; + count: number; + examplesAdded: string[]; + examplesRemoved: string[]; +} + interface Manifest extends ReturnType { repository: string; source: SourceRun; @@ -56,6 +63,33 @@ const assessments = new Map([ ['possible-regression', 'Possible regression'], ['needs-review', 'Needs human review'], ]); +const attributions = new Map([ + ['likely-pr', 'Likely caused by the PR'], + ['unclear', 'Not established'], + ['unlikely-pr', 'Likely unrelated to the PR'], +]); +const signalLabels: Record = { + 'type-erasure': 'New assertions receive bare Any/Unknown instead of their expected type', + 'assertion-failure': 'New type assertions fail', + 'removed-check': 'Type-checking diagnostics disappear without replacements at the same locations', + 'gradual-detail': 'Changed diagnostic details introduce Any/Unknown', +}; +const checkingRules = new Set([ + 'reportArgumentType', + 'reportAssignmentType', + 'reportAttributeAccessIssue', + 'reportCallIssue', + 'reportGeneralTypeIssues', + 'reportIndexIssue', + 'reportOperatorIssue', + 'reportOptionalSubscript', + 'reportOptionalMemberAccess', + 'reportOptionalCall', + 'reportOptionalIterable', + 'reportOptionalContextManager', + 'reportOptionalOperand', + 'reportReturnType', +]); function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object' && !Array.isArray(value); @@ -346,8 +380,67 @@ function groupChanges(added: DiffLine[], removed: DiffLine[]) { return Object.fromEntries(groups); } +function getRegressionSignals(project: ProjectDiff): RegressionSignal[] { + const signals: RegressionSignal[] = []; + const addSignal = (kind: RegressionSignal['kind'], added: DiffLine[], removed: DiffLine[], count: number) => { + if (count) { + signals.push({ + kind, + count, + examplesAdded: added.slice(0, 3).map((entry) => entry.text.slice(0, 600)), + examplesRemoved: removed.slice(0, 3).map((entry) => entry.text.slice(0, 600)), + }); + } + }; + const assertions = project.added.filter((entry) => entry.rule === 'reportAssertTypeFailure'); + const erased = assertions.filter((entry) => { + const match = /"assert_type" mismatch: expected "(.+)" but received "(Any|Unknown)"/.exec(entry.text); + return match && !['Any', 'Unknown'].includes(match[1]); + }); + const erasedSet = new Set(erased); + const otherAssertions = assertions.filter((entry) => !erasedSet.has(entry)); + addSignal('type-erasure', erased, [], erased.length); + addSignal('assertion-failure', otherAssertions, [], otherAssertions.length); + + const locationKey = (entry: Diagnostic) => { + const match = /^(.+:\d+:\d+) - (?:error|warning|information):/.exec(entry.text); + return match ? `${match[1]}:${entry.rule}` : undefined; + }; + const replacements = new Set(project.added.map(locationKey)); + const removedChecks = project.removed.filter((entry) => { + const key = locationKey(entry); + return key !== undefined && checkingRules.has(entry.rule) && !replacements.has(key); + }); + addSignal('removed-check', [], removedChecks, removedChecks.length); + + // Detail-only diffs have no reliable location. These are investigation leads, + // not a pairing of before/after types or proof of lost precision. + const occurrences = (entries: DiffLine[], pattern: RegExp) => + entries.reduce((total, entry) => total + (entry.text.match(pattern)?.length ?? 0), 0); + const newGradualTypes = [/\bAny\b/g, /\bUnknown\b/g] + .filter( + (pattern) => + occurrences([...project.added, ...project.detailsAdded], pattern) > + occurrences([...project.removed, ...project.detailsRemoved], pattern) + ) + .map((pattern) => new RegExp(pattern.source)); + const gradualDetails = project.detailsAdded.filter((entry) => + newGradualTypes.some((pattern) => pattern.test(entry.text)) + ); + addSignal('gradual-detail', gradualDetails, project.detailsRemoved, gradualDetails.length); + return signals; +} + export function summarizeDiffs(projects: ProjectDiff[]) { + const regressionSignals = new Map(); + for (const project of projects) { + const signals = getRegressionSignals(project); + if (signals.length) { + regressionSignals.set(project.name, signals); + } + } return { + ...(regressionSignals.size ? { regressionSignals: Object.fromEntries(regressionSignals) } : {}), projects: projects.map((project) => ({ name: project.name, url: project.url, @@ -407,6 +500,7 @@ function escapeMarkdown(value: string): string { export function renderReport(manifest: Manifest, report: unknown, analysisRunId: number) { const reports = array(record(report).projects).map(record); + const signalsByProject = new Map(Object.entries(manifest.regressionSignals ?? {})); const names = reports.map((item) => text(item.name, 80)); if ( names.length !== manifest.projects.length || @@ -438,8 +532,13 @@ export function renderReport(manifest: Manifest, report: unknown, analysisRunId: '| Project | Added | Removed | Detail lines + / - | Assessment | Confidence | Explanation |', '| --- | ---: | ---: | ---: | --- | --- | --- |', ]; - const details: string[] = []; - const fullDetails: string[] = []; + const analyses: { + rank: number; + category: 'risk' | 'unresolved' | 'other'; + compact: string; + details: string[]; + fullDetails: string[]; + }[] = []; for (const project of manifest.projects) { const item = reports.find((candidate) => candidate.name === project.name)!; const assessment = assessments.get(text(item.assessment)); @@ -447,10 +546,18 @@ export function renderReport(manifest: Manifest, report: unknown, analysisRunId: if (!assessment || !['low', 'medium', 'high'].includes(confidence)) { throw new Error(`Invalid assessment or confidence for ${project.name}`); } + const attribution = attributions.get(text(item.attribution)); + if (!attribution) { + throw new Error(`Invalid PR attribution for ${project.name}`); + } + const before = escapeMarkdown(text(item.before, 1200)); + const after = escapeMarkdown(text(item.after, 1200)); + const impact = escapeMarkdown(text(item.impact, 1200)); const summary = escapeMarkdown(text(item.summary, 240)); const explanation = text(item.explanation, maxReportLength); const explanationPreview = Array.from(explanation).slice(0, 1800).join(''); const unresolved = escapeMarkdown(text(item.unresolved, 800)); + let citesPrCode = false; const evidence = array(item.evidence).map((entry) => { const source = record(entry); const url = new URL(text(source.url, 1000)); @@ -463,45 +570,122 @@ export function renderReport(manifest: Manifest, report: unknown, analysisRunId: ) { throw new Error(`Unsupported evidence URL for ${project.name}`); } + if ( + url.hostname === 'github.com' && + /^#L\d+(?:-L\d+)?$/.test(url.hash) && + [manifest.repository, manifest.source.headRepository].some((repository) => + url.pathname + .toLowerCase() + .startsWith(`/${repository}/blob/${manifest.source.headSha}/`.toLowerCase()) + ) + ) { + citesPrCode = true; + } return `- ${escapeMarkdown(text(source.detail, 400))}: <${url.href}>`; }); if (evidence.length > 5 || (!evidence.length && item.assessment !== 'needs-review')) { throw new Error(`Expected cited evidence for ${project.name}`); } + if (item.attribution === 'likely-pr' && !citesPrCode) { + throw new Error(`PR attribution requires a line-pinned citation at the analyzed head for ${project.name}`); + } rows.push( `| ${escapeMarkdown(project.name)} | ${project.added} | ${project.removed} | ` + `${project.detailLinesAdded} / ${project.detailLinesRemoved} | ` + `${assessment} | ${confidence} | ${summary} |` ); - const projectHeading = [`### ${escapeMarkdown(project.name)}`, '']; - const projectEvidence = ['', `**Unresolved / coverage limits:** ${unresolved}`, '', ...evidence, '']; - details.push( + const signals = signalsByProject.get(project.name) ?? []; + const risk = item.assessment === 'possible-regression' || signals.length > 0; + const category = risk ? 'risk' : item.assessment === 'needs-review' ? 'unresolved' : 'other'; + const projectHeading = [ + `### ${escapeMarkdown(project.name)}: ${summary}`, + '', + ...(signals.length + ? [ + '**Regression warning signals require review, regardless of the AI assessment:**', + ...signals.map((signal) => `- ${signalLabels[signal.kind]}.`), + '', + ] + : []), + `**AI assessment:** ${assessment} (${confidence} confidence). **PR attribution:** ${attribution}.`, + '', + `**Before / baseline evidence:** ${before}`, + '', + `**After / PR evidence:** ${after}`, + '', + `**Why this matters:** ${impact}`, + '', + '**Causal analysis:**', + '', + ]; + const projectEvidence = ['', `**Uncertainty / next check:** ${unresolved}`, '', ...evidence, '']; + const details = [ ...projectHeading, escapeMarkdown(explanationPreview), ...(explanationPreview.length < explanation.length ? ['', '**Explanation truncated; see the full report artifact linked below.**'] : []), - ...projectEvidence - ); - fullDetails.push(...projectHeading, escapeMarkdown(explanation), ...projectEvidence); + ...projectEvidence, + ]; + analyses.push({ + category, + rank: signals.some((signal) => signal.kind === 'type-erasure') + ? 0 + : risk + ? 1 + : category === 'unresolved' + ? 2 + : 3, + compact: + `- **${escapeMarkdown(project.name)}: ${risk ? 'Regression review required' : assessment}.** ` + + `${summary} PR attribution: ${attribution}.`, + details, + fullDetails: [...projectHeading, escapeMarkdown(explanation), ...projectEvidence], + }); } + analyses.sort((a, b) => a.rank - b.rank); const footer = [ `**Full evidence and limitations:** download \`mypy-primer-analysis-report\` from [this analysis run](${analysisUrl}).`, 'The original raw primer comment is unchanged. Treat uncertainty and possible regressions as requests for human review.', ]; - const compactBody = [...heading, ...rows, '', ...footer].join('\n'); - const expandedBody = [ - ...heading, - ...rows, + const risks = analyses.filter((analysis) => analysis.category === 'risk'); + const unresolved = analyses.filter((analysis) => analysis.category === 'unresolved'); + const other = analyses.filter((analysis) => analysis.category === 'other'); + const overview = risks.length + ? '**Regression review required.** Potential regressions or recorded warning signals are listed first; causation must be established separately.' + : unresolved.length + ? '**Regression analysis incomplete.** No concrete regression has been established; unresolved investigations follow.' + : '**No potential regression identified in the investigated changes.** This is not proof that the PR is regression-free.'; + const compactBody = [ + ...provenance, + overview, '', - '
', - 'Evidence and limitations by project', + 'Full causal analyses are available in the report artifact.', '', - ...details, - '
', + ...analyses.map((analysis) => analysis.compact), '', ...footer, ].join('\n'); + const expandedBody = [ + ...provenance, + overview, + '', + ...(risks.length ? ['## Potential regressions', '', ...risks.flatMap((analysis) => analysis.details)] : []), + ...(unresolved.length + ? ['## Unresolved investigations', '', ...unresolved.flatMap((analysis) => analysis.details)] + : []), + ...(other.length + ? [ + '
', + 'Other project assessments', + '', + ...other.flatMap((analysis) => analysis.details), + '
', + '', + ] + : []), + ...footer, + ].join('\n'); let body = expandedBody.length <= 60000 ? expandedBody : compactBody; const onlyProject = manifest.projects.length === 1 ? manifest.projects[0] : undefined; if (onlyProject?.name === 'sympy' && onlyProject.url === 'https://github.com/sympy/sympy') { @@ -518,7 +702,10 @@ export function renderReport(manifest: Manifest, report: unknown, analysisRunId: if (body.length > 60000) { throw new Error('The report exceeds the comment limit; refusing to omit projects'); } - return { body, fullReport: [...heading, ...rows, '', ...fullDetails].join('\n') }; + return { + body, + fullReport: [...heading, ...rows, '', ...analyses.flatMap((analysis) => analysis.fullDetails)].join('\n'), + }; } export function getStagedMode(activationInfo: unknown, repository: string, runId: number, runAttempt: number): boolean { @@ -540,13 +727,27 @@ export function validateSubmittedReport(manifest: Manifest, agentOutput: unknown const items = array(record(agentOutput).items) .map(record) .filter((item) => item.type === 'publish_primer_analysis'); - if (items.length !== 1) { + if (!items.length || items.length > manifest.projects.length) { throw new Error( - `Expected exactly one primer analysis report; found ${items.length}. ` + + `Expected 1 to ${manifest.projects.length} primer analysis submissions; found ${items.length}. ` + 'Submit publish_primer_analysis, not noop or report_incomplete.' ); } - return renderReport(manifest, JSON.parse(text(items[0].report, maxReportLength)), analysisRunId); + let combinedLength = 0; + const projects = items.flatMap((item) => { + const content = text(item.report, maxReportLength); + combinedLength += content.length; + if (combinedLength > maxReportLength) { + throw new Error(`The combined primer analysis report exceeds ${maxReportLength} characters`); + } + const projects = array(record(JSON.parse(content)).projects); + // Retain compatibility with historical, single-submission reports. + if (items.length > 1 && projects.length !== 1) { + throw new Error('Each per-project submission must contain exactly one project'); + } + return projects; + }); + return renderReport(manifest, { projects }, analysisRunId); } export function validateSubmittedReportFile(manifest: Manifest, outputPath: string, analysisRunId: number) { diff --git a/docs/mypy-primer-analysis.md b/docs/mypy-primer-analysis.md index 5a43adc1f3ec..4f0f9baf0ca5 100644 --- a/docs/mypy-primer-analysis.md +++ b/docs/mypy-primer-analysis.md @@ -5,9 +5,11 @@ differences** analyzes the complete eight-shard output and posts a separate, commit-specific explanation. It is advisory: it does not approve PRs, change labels, merge changes, or make mypy_primer diagnostic differences fail CI. -The PR comment lists every changed project, deterministic added/removed diagnostic -counts, an assessment, confidence, and a short explanation, followed by expandable -evidence and limitations when they fit. The linked workflow run's +The PR comment leads with potential regressions and unresolved investigations, +not a diagnostic-count table. Each project analysis states baseline evidence, +changed behavior, downstream impact, a causal explanation, PR attribution, and +the remaining uncertainty or next check. Type-erasure signals are ranked first; +other assessed changes are collapsed. The linked workflow run's `mypy-primer-analysis-report` artifact always contains the complete explanation, evidence, and limitations for every project. The original raw diff remains available. Added/removed counts refer to diagnostic headers; message rewrites can @@ -18,7 +20,7 @@ Explanations longer than 1,800 characters do not prevent publication. The commen shows the first 1,800 Unicode code points with an explicit truncation notice, while the artifact retains the complete explanation. Truncation happens before Markdown escaping, without splitting surrogate pairs. The 60,000-character comment budget -still falls back to the compact table when expanded details do not fit. +still falls back to a compact regression outline when expanded details do not fit. Nonempty explanations and the overall 1,000,000-character input bound remain required. Other field limits, project coverage, assessment/evidence validation, and stale-PR checks are unchanged. @@ -31,11 +33,28 @@ explicitly present on that line, or `unspecified` when absent. Both retain bound examples, with complete evidence in the raw artifacts. Detail-only projects still require an assessment, even with zero added/removed headers. +The collector also extracts generic `regressionSignals`: new assertion failures +returning bare `Any`/`Unknown`, other assertion failures, removed checking diagnostics +without replacements at the same locations/rules, and newly appearing gradual +types in diagnostic details. These are investigation leads, not confirmed bugs or +proof that the PR caused them. Detail-only signals do not pair unrelated lines. +Signals stay visible in the regression section even if the model labels a project +an improvement or a stub issue; such a label cannot silently dismiss the warning. +This is not an exhaustive detector, and absence of signals does not establish safety. + Assessments distinguish expected improvements, newly exposed typing issues, possible regressions, and changes needing human review. Neither fewer diagnostics nor a successful primer job establishes correctness. The agent is specifically instructed to investigate assertion failures, type precision loss, and diagnostics that may have disappeared because a result became `Any` or `Unknown`. +PR attribution is separate: `likely-pr`, `unclear`, or `unlikely-pr`. A likely +attribution requires an implementation citation pinned to the analyzed head and +line, as well as a causal explanation. Source links do not independently prove the +reasoning, so these remain advisory findings. Conformance intent and newly changed +test expectations are not evidence that a loss of downstream checking is harmless. +New assertion failures must not be blamed on stubs without inspecting the relevant +declarations. The agent cannot execute reproductions and must label static reasoning +and proposed follow-up checks accordingly. When the only changed project is the canonical `sympy` project, the comment is a short notice that treats these differences as non-blocking primer noise. It keeps @@ -78,25 +97,34 @@ policy disallows this inference permission, a repository administrator must reso that configuration before the workflow can run. See the [Copilot authentication reference](https://github.github.com/gh-aw/reference/auth/). -The initial limits are 25 AI credits, 30 agent turns, and 15 minutes for the agent. +The limits are 200 AI credits, 30 agent turns, and 15 minutes for the agent. They can be adjusted in `.github/workflows/mypy-primer-analysis.md`. The agent must mark insufficiently investigated projects as needing review rather than inventing evidence to meet its budget. -The analysis uses `engine.model: gpt-5.6-luna`, a newer lightweight GPT model with -lower standard token rates than the previous GPT-5.4 mini; consult the +The analysis uses `engine.model: gpt-5.6-sol` rather than the lightweight Luna +model, with a larger budget for causal regression investigation; consult the [Copilot pricing table](https://docs.github.com/en/copilot/reference/copilot-billing/models-and-pricing) for current rates. Keep `sandbox.agent.token-steering: false` so the proxy preserves the selected model rather than steering requests to another one. -The 25-credit cap is unchanged; a cheaper model does not guarantee completion. +The 200-credit analysis budget is approximately $2; it does not guarantee completion +or detection of every regression. The model must be enabled for the organization, and its advisory assessments still require human review. Threat detection explicitly retains its existing `detection` model alias rather than inheriting the analysis model. -Every activated analysis must submit exactly one `publish_primer_analysis` report. -This safe-output submission queues the trusted publisher; it is not a direct GitHub -write. `noop` is disabled, and an independent post-step validates successful agent -executions after safe-output ingestion, using the normalized `agent_output.json` +Every activated analysis must call `publish_primer_analysis` once per changed +project, with exactly one entry in each payload's `projects` array. The pinned +framework limits each string input to 10240 UTF-8 bytes; per-project submissions +avoid squeezing every investigation into that single allowance. Up to 100 calls +are allowed, matching the maximum project inventory. The validator assembles all +projects, preserves the combined 1000000-character payload limit, and rejects +missing or duplicate projects. Historical single-submission reports remain +readable. Individual tool successes do not mean the complete report is valid. + +These safe-output submissions queue the trusted publisher; they are not direct +GitHub writes. `noop` is disabled, and an independent post-step validates successful +agent executions after safe-output ingestion, using the normalized `agent_output.json` and the same report validator as the publisher. Empty, no-op-only, incomplete, duplicate, or malformed submissions fail the agent job rather than leaving a misleading green run with publication skipped. The validator and manifest diff --git a/packages/pyright-internal/src/tests/mypyPrimerAnalysis.test.ts b/packages/pyright-internal/src/tests/mypyPrimerAnalysis.test.ts index 8017c0ba8b87..c69c8630af0b 100644 --- a/packages/pyright-internal/src/tests/mypyPrimerAnalysis.test.ts +++ b/packages/pyright-internal/src/tests/mypyPrimerAnalysis.test.ts @@ -83,6 +83,16 @@ const expectedDetailOnlyProject = { ], }; const expectedDetailOnlySummary = { + regressionSignals: { + example: [ + { + kind: 'gradual-detail', + count: 1, + examplesAdded: [expectedDetailOnlyProject.detailsAdded[0].text], + examplesRemoved: expectedDetailOnlyProject.detailsRemoved.map((entry) => entry.text), + }, + ], + }, projects: [ { name: 'example', @@ -136,11 +146,21 @@ const expectedReportHeading = [ const expectedReportRow = '| example | 1 | 1 | 1 / 1 | Needs human review | low | The argument error changed its wording. |'; const expectedReportDetails = [ - '### example', + '### example: The argument error changed its wording.', + '', + '**AI assessment:** Needs human review (low confidence). **PR attribution:** Not established.', + '', + '**Before / baseline evidence:** An argument error was reported.', + '', + '**After / PR evidence:** An argument error remains.', + '', + '**Why this matters:** No changed checking has been established.', + '', + '**Causal analysis:**', '', 'The error remains at the same location. Its cause has not been established.', '', - '**Unresolved / coverage limits:** Exact dependency revision unavailable.', + '**Uncertainty / next check:** Exact dependency revision unavailable.', '', '', ]; @@ -149,22 +169,51 @@ const expectedReportFooter = [ '[this analysis run](https://github.com/microsoft/pyright/actions/runs/100).', 'The original raw primer comment is unchanged. Treat uncertainty and possible regressions as requests for human review.', ]; +const incompleteOverview = + '**Regression analysis incomplete.** No concrete regression has been established; unresolved investigations follow.'; +const riskOverview = + '**Regression review required.** Potential regressions or recorded warning signals are listed first; causation must be established separately.'; const expectedReport = { body: [ - ...expectedReportHeading, - expectedReportRow, + ...expectedReportProvenance, + incompleteOverview, '', - '
', - 'Evidence and limitations by project', + '## Unresolved investigations', '', ...expectedReportDetails, - '
', - '', ...expectedReportFooter, ].join('\n'), fullReport: [...expectedReportHeading, expectedReportRow, '', ...expectedReportDetails].join('\n'), }; +function expectedLargeReport(projectCount: number, explanation: string) { + const escapedNames = Array.from({ length: projectCount }, (_, index) => `project\\_${index}`); + const rows = escapedNames.map((name) => expectedReportRow.replace('example', name)); + const details = escapedNames.flatMap((name) => + expectedReportDetails.map((line) => + line + .replace('### example:', `### ${name}:`) + .replace('The error remains at the same location. Its cause has not been established.', explanation) + ) + ); + return { + body: [ + ...expectedReportProvenance, + incompleteOverview, + '', + 'Full causal analyses are available in the report artifact.', + '', + ...escapedNames.map( + (name) => + `- **${name}: Needs human review.** The argument error changed its wording. PR attribution: Not established.` + ), + '', + ...expectedReportFooter, + ].join('\n'), + fullReport: [...expectedReportHeading, ...rows, '', ...details].join('\n'), + }; +} + function fixture(contents = sample, headRepository = repository) { const source = { runId: 42, @@ -232,14 +281,23 @@ function fixture(contents = sample, headRepository = repository) { name: 'example', assessment: 'needs-review', confidence: 'low', + attribution: 'unclear', summary: 'The argument error changed its wording.', + before: 'An argument error was reported.', + after: 'An argument error remains.', + impact: 'No changed checking has been established.', explanation: 'The error remains at the same location. Its cause has not been established.', unresolved: 'Exact dependency revision unavailable.', evidence: [], }, ], }; - const output = () => ({ items: [{ type: 'publish_primer_analysis', report: JSON.stringify(report) }] }); + const output = (perProject = false) => ({ + items: (perProject ? report.projects.map((project) => ({ projects: [project] })) : [report]).map((report) => ({ + type: 'publish_primer_analysis', + report: JSON.stringify(report), + })), + }); return { source, manifest, pr, run, artifacts, comments, associated, request, report, output }; } @@ -506,6 +564,7 @@ describe('mypy_primer analysis', () => { throw new Error('Preparation unexpectedly skipped the detail-only project'); } f.report.projects[0] = { + ...f.report.projects[0], name: 'example', assessment: 'needs-review', confidence: 'low', @@ -515,26 +574,36 @@ describe('mypy_primer analysis', () => { evidence: [], }; const expectedDetails = [ - '### example', + '### example: Only diagnostic details changed.', + '', + '**Regression warning signals require review, regardless of the AI assessment:**', + '- Changed diagnostic details introduce Any/Unknown.', + '', + '**AI assessment:** Needs human review (low confidence). **PR attribution:** Not established.', + '', + '**Before / baseline evidence:** An argument error was reported.', + '', + '**After / PR evidence:** An argument error remains.', + '', + '**Why this matters:** No changed checking has been established.', + '', + '**Causal analysis:**', '', 'The return type changed from Match\\[str\\] \\| None to Unknown.', '', - '**Unresolved / coverage limits:** Diagnostic headers and locations are absent from the concise diff.', + '**Uncertainty / next check:** Diagnostic headers and locations are absent from the concise diff.', '', '', ]; const expectedRow = '| example | 0 | 0 | 2 / 2 | Needs human review | low | Only diagnostic details changed. |'; const expected = { body: [ - ...expectedReportHeading, - expectedRow, + ...expectedReportProvenance, + riskOverview, '', - '
', - 'Evidence and limitations by project', + '## Potential regressions', '', ...expectedDetails, - '
', - '', ...expectedReportFooter, ].join('\n'), fullReport: [...expectedReportHeading, expectedRow, '', ...expectedDetails].join('\n'), @@ -687,6 +756,272 @@ describe('mypy_primer analysis', () => { expect(result).toStrictEqual(expectedReport); }); + test('extracts type-erasure and other assertion signals without inferring baseline reveals', () => { + const erased = [ + '.../projects/pandas-stubs/test.py:1:1 - error: "assert_type" mismatch: expected "Index[Any]" but received "Any" (reportAssertTypeFailure)', + '.../projects/pandas-stubs/test.py:2:1 - error: "assert_type" mismatch: expected "Rotation[tuple[Any, ...]]" but received "Unknown" (reportAssertTypeFailure)', + ]; + const assertion = + '.../projects/pandas-stubs/test.py:3:1 - error: "assert_type" mismatch: expected "int" but received "Literal[1]" (reportAssertTypeFailure)'; + const diff = [ + 'pandas-stubs (https://github.com/pandas-dev/pandas-stubs)', + ...[...erased, assertion].map((line) => `+ ${line}`), + '- 0 errors, 0 warnings, 0 informations', + '+ 3 errors, 0 warnings, 0 informations', + ].join('\n'); + expect(summarizeDiffs(parseDiff(diff, 0)).regressionSignals).toStrictEqual({ + 'pandas-stubs': [ + { kind: 'type-erasure', count: 2, examplesAdded: erased, examplesRemoved: [] }, + { kind: 'assertion-failure', count: 1, examplesAdded: [assertion], examplesRemoved: [] }, + ], + }); + }); + + test('flags removed checks but not replacements or unrelated lint removals', () => { + const removed = '.../projects/example/test.py:2:1 - error: Missing member (reportAttributeAccessIssue)'; + const diff = [ + 'example (https://github.com/example/project)', + '- .../projects/example/test.py:1:1 - error: Old argument error (reportArgumentType)', + '+ .../projects/example/test.py:1:1 - error: Reworded argument error (reportArgumentType)', + `- ${removed}`, + '+ .../projects/example/test.py:2:1 - error: Different argument error (reportArgumentType)', + '- .../projects/example/test.py:3:1 - warning: Unused import (reportUnusedImport)', + '- 2 errors, 1 warning, 0 informations', + '+ 2 errors, 0 warnings, 0 informations', + ].join('\n'); + expect(summarizeDiffs(parseDiff(diff, 0)).regressionSignals).toStrictEqual({ + example: [{ kind: 'removed-check', count: 1, examplesAdded: [], examplesRemoved: [removed] }], + }); + expect(summarizeDiffs(parseDiff(sample, 0)).regressionSignals).toBeUndefined(); + }); + + test('detects new Unknown returns despite pre-existing Unknown parameters', () => { + const before = ' Return type mismatch: override returns type "(value: Unknown) -> int"'; + const after = ' Return type mismatch: override returns type "(value: Unknown) -> Unknown"'; + const diff = ['pip (https://github.com/pypa/pip)', `-${before}`, `+${after}`].join('\n'); + expect(summarizeDiffs(parseDiff(diff, 0)).regressionSignals).toStrictEqual({ + pip: [{ kind: 'gradual-detail', count: 1, examplesAdded: [after], examplesRemoved: [before] }], + }); + }); + + test('does not flag a gradual type merely moving from a header into a detail', () => { + const diff = [ + 'example (https://github.com/example/project)', + '- .../projects/example/test.py:1:1 - error: Argument of type Unknown is incompatible (reportArgumentType)', + '+ .../projects/example/test.py:1:1 - error: Argument is incompatible (reportArgumentType)', + '+ Type Unknown is incompatible (reportArgumentType)', + ].join('\n'); + expect(summarizeDiffs(parseDiff(diff, 0)).regressionSignals).toBeUndefined(); + }); + + test('bounds warning examples without undercounting the observed signal', () => { + const lines = Array.from( + { length: 5 }, + (_, index) => + `.../projects/example/test.py:${index + 1}:1 - error: "assert_type" mismatch: expected "${'X'.repeat( + 800 + )}" but received "Any" (reportAssertTypeFailure)` + ); + const diff = [ + 'example (https://github.com/example/project)', + ...lines.map((line) => `+ ${line}`), + '- 0 errors, 0 warnings, 0 informations', + '+ 5 errors, 0 warnings, 0 informations', + ].join('\n'); + expect(summarizeDiffs(parseDiff(diff, 0)).regressionSignals).toStrictEqual({ + example: [ + { + kind: 'type-erasure', + count: 5, + examplesAdded: lines.slice(0, 3).map((line) => line.slice(0, 600)), + examplesRemoved: [], + }, + ], + }); + }); + + test.each([ + ['__proto__', '\\_\\_proto\\_\\_'], + ['constructor', 'constructor'], + ['toString', 'toString'], + ])('treats project name %s as data rather than an inherited key', (name, escapedName) => { + const f = fixture(sample.replace(/example/g, name)); + f.report.projects[0].name = name; + expect(renderReport({ ...f.manifest, regressionSignals: {} }, f.report, 100)).toStrictEqual({ + body: expectedReport.body.replace(/example/g, escapedName), + fullReport: expectedReport.fullReport.replace(/example/g, escapedName), + }); + const line = `.../projects/${name}/test.py:1:1 - error: "assert_type" mismatch: expected "int" but received "Any" (reportAssertTypeFailure)`; + const diff = [ + `${name} (https://github.com/example/project)`, + `+ ${line}`, + '- 0 errors, 0 warnings, 0 informations', + '+ 1 error, 0 warnings, 0 informations', + ].join('\n'); + // Compare entries so Jest does not treat a constructor data key as a class identity. + expect( + Object.entries(JSON.parse(JSON.stringify(summarizeDiffs(parseDiff(diff, 0)).regressionSignals))) + ).toStrictEqual([[name, [{ kind: 'type-erasure', count: 1, examplesAdded: [line], examplesRemoved: [] }]]]); + }); + + test('keeps type-erasure risks first even when the model blames stubs', () => { + const diff = [ + sample, + 'pandas-stubs (https://github.com/pandas-dev/pandas-stubs)', + '+ .../projects/pandas-stubs/test.py:1:1 - error: "assert_type" mismatch: expected "Index[Any]" but received "Any" (reportAssertTypeFailure)', + '- 0 errors, 0 warnings, 0 informations', + '+ 1 error, 0 warnings, 0 informations', + ].join('\n'); + const f = fixture(diff); + const pandas = { + ...f.report.projects[0], + name: 'pandas-stubs', + assessment: 'exposed-typing-issue', + confidence: 'high', + summary: 'The model claims that the stub expectation is wrong.', + before: 'The assertion expects Index[Any]; the baseline reveal is not recorded.', + after: 'The new assertion receives Any.', + impact: 'The expected outer container type is absent from the reported result.', + explanation: 'Conformance intent alone does not establish that the lost type information is harmless.', + evidence: [ + { + url: 'https://typing.python.org/en/latest/spec/overload.html', + detail: 'Specification context, not proof of a bad stub.', + }, + ], + }; + const pandasDetails = [ + '### pandas-stubs: The model claims that the stub expectation is wrong.', + '', + '**Regression warning signals require review, regardless of the AI assessment:**', + '- New assertions receive bare Any/Unknown instead of their expected type.', + '', + '**AI assessment:** Exposed typing issue (high confidence). **PR attribution:** Not established.', + '', + '**Before / baseline evidence:** The assertion expects Index\\[Any\\]; the baseline reveal is not recorded.', + '', + '**After / PR evidence:** The new assertion receives Any.', + '', + '**Why this matters:** The expected outer container type is absent from the reported result.', + '', + '**Causal analysis:**', + '', + 'Conformance intent alone does not establish that the lost type information is harmless.', + '', + '**Uncertainty / next check:** Exact dependency revision unavailable.', + '', + '- Specification context, not proof of a bad stub.: ', + '', + ]; + expect(renderReport(f.manifest, { projects: [...f.report.projects, pandas] }, 100)).toStrictEqual({ + body: [ + ...expectedReportProvenance, + riskOverview, + '', + '## Potential regressions', + '', + ...pandasDetails, + '## Unresolved investigations', + '', + ...expectedReportDetails, + ...expectedReportFooter, + ].join('\n'), + fullReport: [ + ...expectedReportHeading, + expectedReportRow, + '| pandas-stubs | 1 | 0 | 0 / 0 | Exposed typing issue | high | The model claims that the stub expectation is wrong. |', + '', + ...pandasDetails, + ...expectedReportDetails, + ].join('\n'), + }); + }); + + test.each([ + `https://github.com/microsoft/pyright/blob/main/file.ts#L1`, + `https://github.com/microsoft/pyright/blob/${'b'.repeat(40)}/file.ts#L1`, + `https://github.com/microsoft/pyright/blob/${headSha}/file.ts`, + `https://github.com/other/repo/blob/${headSha}/file.ts#L1`, + 'https://typing.python.org/en/latest/spec/overload.html#step-5', + ])('rejects likely PR attribution without exact head and line evidence: %s', (url) => { + const f = fixture(); + const report = { + projects: [ + { + ...f.report.projects[0], + attribution: 'likely-pr', + evidence: [{ url, detail: 'Insufficient attribution evidence.' }], + }, + ], + }; + expect(() => renderReport(f.manifest, report, 100)).toThrow( + new Error('PR attribution requires a line-pinned citation at the analyzed head for example') + ); + }); + + test.each([ + { headRepository: repository, evidenceRepository: repository }, + { headRepository: 'contributor/pyright', evidenceRepository: repository }, + { headRepository: 'contributor/pyright', evidenceRepository: 'contributor/pyright' }, + ])( + 'accepts $evidenceRepository evidence for a PR from $headRepository', + ({ headRepository, evidenceRepository }) => { + const f = fixture(sample, headRepository); + const url = `https://github.com/${evidenceRepository}/blob/${headSha}/file.ts#L1-L5`; + const report = { + projects: [ + { + ...f.report.projects[0], + attribution: 'likely-pr', + evidence: [{ url, detail: 'Changed branch.' }], + }, + ], + }; + const details = [...expectedReportDetails.slice(0, -1), `- Changed branch.: <${url}>`, ''] + .join('\n') + .replace('PR attribution:** Not established.', 'PR attribution:** Likely caused by the PR.'); + expect(renderReport(f.manifest, report, 100)).toStrictEqual({ + body: expectedReport.body.replace(expectedReportDetails.join('\n'), details), + fullReport: expectedReport.fullReport.replace(expectedReportDetails.join('\n'), details), + }); + } + ); + + test.each([ + `https://github.com/contributor/pyright/blob/${'b'.repeat(40)}/file.ts#L1`, + `https://github.com/contributor/pyright/blob/${headSha}/file.ts`, + `https://github.com/contributor/pyright-extra/blob/${headSha}/file.ts#L1`, + `https://github.com/other/pyright/blob/${headSha}/file.ts#L1`, + ])('rejects mismatched attribution evidence for a fork PR: %s', (url) => { + const f = fixture(sample, 'contributor/pyright'); + const report = { + projects: [ + { + ...f.report.projects[0], + attribution: 'likely-pr', + evidence: [{ url, detail: 'Insufficient attribution evidence.' }], + }, + ], + }; + expect(() => renderReport(f.manifest, report, 100)).toThrow( + new Error('PR attribution requires a line-pinned citation at the analyzed head for example') + ); + }); + + test.each(['before', 'after', 'impact', 'attribution'])('requires the %s investigation field', (field) => { + const f = fixture(); + const report = { projects: [{ ...f.report.projects[0], [field]: undefined }] }; + expect(() => renderReport(f.manifest, report, 100)).toThrow( + new Error(`Expected nonempty text of at most ${field === 'attribution' ? 2000 : 1200} characters`) + ); + }); + + test('rejects unsupported PR attribution classifications', () => { + const f = fixture(); + f.report.projects[0].attribution = 'definitely-caused-by-pr'; + expect(() => renderReport(f.manifest, f.report, 100)).toThrow(new Error('Invalid PR attribution for example')); + }); + test('uses a short non-blocking notice only for canonical SymPy-only changes', () => { const f = fixture( sample @@ -729,15 +1064,12 @@ describe('mypy_primer analysis', () => { ]; expect(renderReport(manifest, { projects: [sympy, ...f.report.projects] }, 100)).toStrictEqual({ body: [ - ...expectedReportHeading, - ...rows, + ...expectedReportProvenance, + incompleteOverview, '', - '
', - 'Evidence and limitations by project', + '## Unresolved investigations', '', ...details, - '
', - '', ...expectedReportFooter, ].join('\n'), fullReport: [...expectedReportHeading, ...rows, '', ...details].join('\n'), @@ -810,6 +1142,9 @@ describe('mypy_primer analysis', () => { ['summary', 'x'.repeat(241)], ['unresolved', 'x'.repeat(801)], ['explanation', 'x'.repeat(1000001)], + ['before', 'x'.repeat(1201)], + ['after', 'x'.repeat(1201)], + ['impact', 'x'.repeat(1201)], ]) { const report = { projects: [{ ...f.report.projects[0], [field]: value }] }; expect(() => renderReport(f.manifest, report, 100)).toThrow('Expected nonempty text'); @@ -860,14 +1195,7 @@ describe('mypy_primer analysis', () => { const report = { projects: names.map((name) => ({ ...f.report.projects[0], name, explanation })) }; const result = renderReport(manifest, report, 100); expect(result.body.length).toBeLessThanOrEqual(60000); - expect(result.body).not.toContain('
'); - expect(result.body).toContain(expectedReportFooter[0]); - expect(result.fullReport).not.toContain('Explanation truncated'); - for (const name of names) { - const escaped = name.replace('_', '\\_'); - expect(result.body).toContain(`| ${escaped} |`); - expect(result.fullReport).toContain(`### ${escaped}\n\n${explanation}\n`); - } + expect(result).toStrictEqual(expectedLargeReport(names.length, explanation)); }); test('large reports retain every project in the comment and full artifact', () => { @@ -883,23 +1211,7 @@ describe('mypy_primer analysis', () => { const result = renderReport(manifest, report, 100); expect(result.body.length).toBeLessThan(60000); expect(result.fullReport.length).toBeGreaterThan(60000); - const expectedRows = names.map( - (_, index) => - `| project\\_${index} | 1 | 1 | 1 / 1 | Needs human review | low | The argument error changed its wording. |` - ); - const expectedDetails = names.flatMap((_, index) => [ - `### project\\_${index}`, - '', - 'x'.repeat(1800), - '', - '**Unresolved / coverage limits:** Exact dependency revision unavailable.', - '', - '', - ]); - expect(result).toStrictEqual({ - body: [...expectedReportHeading, ...expectedRows, '', ...expectedReportFooter].join('\n'), - fullReport: [...expectedReportHeading, ...expectedRows, '', ...expectedDetails].join('\n'), - }); + expect(result).toStrictEqual(expectedLargeReport(names.length, 'x'.repeat(1800))); }); test('rejects missing projects, unsupported classifications, and uncited certainty', () => { @@ -919,8 +1231,8 @@ describe('mypy_primer analysis', () => { f.report.projects[0].summary = '