From ad0f026907fe1a40b16d11fa7303edb019088231 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 06:28:21 +0000 Subject: [PATCH 1/2] Add notify-target guidance for experiment readiness comments Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily-experiment-report.lock.yml | 34 +++++++++++-------- .github/workflows/daily-experiment-report.md | 22 ++++++++---- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index 21cf9eda54c..227896cf780 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"db66e7460236226c5875221eb830ef24b7367f31cfaa66262afed864b40dd6ac","body_hash":"d13f5ff30176b972edd960f204060ef112ff96cb40716a1594422d8267704d6b","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"30a96670d4158c5a60299d6ef5c3d32e74fd5fb28c0d8fe62b3636528d7015e7","body_hash":"de26a4a9956ab4d2d70f21d0409065b4e4e4355978b9ec07c19b4671cfd1b20d","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) @@ -222,29 +222,30 @@ jobs: GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} GH_AW_GITHUB_ACTOR: ${{ github.actor }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} # poutine:ignore untrusted_checkout_exec run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_5965dcdb8eab7fa5_EOF' + cat << 'GH_AW_PROMPT_f4b1126dbb418f62_EOF' - GH_AW_PROMPT_5965dcdb8eab7fa5_EOF + GH_AW_PROMPT_f4b1126dbb418f62_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/cache_memory_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_5965dcdb8eab7fa5_EOF' + cat << 'GH_AW_PROMPT_f4b1126dbb418f62_EOF' Tools: add_comment(max:10), create_discussion, add_labels(max:10), upload_asset(max:10), missing_tool, missing_data, noop upload_asset: provide a file path; returns a URL; assets are published after the workflow completes (safeoutputs). - GH_AW_PROMPT_5965dcdb8eab7fa5_EOF + GH_AW_PROMPT_f4b1126dbb418f62_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_5965dcdb8eab7fa5_EOF' + cat << 'GH_AW_PROMPT_f4b1126dbb418f62_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -273,15 +274,15 @@ jobs: {{/if}} - GH_AW_PROMPT_5965dcdb8eab7fa5_EOF + GH_AW_PROMPT_f4b1126dbb418f62_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_5965dcdb8eab7fa5_EOF' + cat << 'GH_AW_PROMPT_f4b1126dbb418f62_EOF' {{#runtime-import .github/workflows/shared/otlp.md}} {{#runtime-import .github/workflows/shared/trending-charts-simple.md}} {{#runtime-import .github/workflows/shared/reporting.md}} {{#runtime-import .github/workflows/daily-experiment-report.md}} - GH_AW_PROMPT_5965dcdb8eab7fa5_EOF + GH_AW_PROMPT_f4b1126dbb418f62_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -289,6 +290,7 @@ jobs: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} with: script: | @@ -309,6 +311,7 @@ jobs: GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} GH_AW_GITHUB_ACTOR: ${{ github.actor }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` โ€” run `safeoutputs --help` to see available tools' @@ -332,6 +335,7 @@ jobs: GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_REPOSITORY_OWNER: process.env.GH_AW_GITHUB_REPOSITORY_OWNER, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST @@ -555,9 +559,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_11c5fb2718418053_EOF' - {"add_comment":{"max":10},"add_labels":{"max":10},"create_discussion":{"category":"audits","close_older_discussions":true,"expires":72,"fallback_to_issue":true,"max":1,"title_prefix":"[experiments] "},"create_report_incomplete_issue":{},"max_bot_mentions":1,"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{},"upload_asset":{"allowed-exts":[".png",".jpg",".jpeg",".svg"],"branch":"assets/${GITHUB_WORKFLOW}","max":10,"max-size":10240}} - GH_AW_SAFE_OUTPUTS_CONFIG_11c5fb2718418053_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_36b329e0edf6efad_EOF' + {"add_comment":{"max":10},"add_labels":{"max":10},"create_discussion":{"category":"audits","close_older_discussions":true,"expires":72,"fallback_to_issue":true,"max":1,"title_prefix":"[experiments] "},"create_report_incomplete_issue":{},"max_bot_mentions":1,"mentions":{"enabled":true},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{},"upload_asset":{"allowed-exts":[".png",".jpg",".jpeg",".svg"],"branch":"assets/${GITHUB_WORKFLOW}","max":10,"max-size":10240}} + GH_AW_SAFE_OUTPUTS_CONFIG_36b329e0edf6efad_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -816,7 +820,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_6db808d75aa14117_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_eb5773d666e80e96_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -862,7 +866,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_6db808d75aa14117_EOF + GH_AW_MCP_CONFIG_eb5773d666e80e96_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -1630,7 +1634,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.grafana.net,*.pythonhosted.org,*.sentry.io,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":10},\"add_labels\":{\"max\":10},\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":72,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[experiments] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":false},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{},\"upload_asset\":{\"allowed-exts\":[\".png\",\".jpg\",\".jpeg\",\".svg\"],\"branch\":\"assets/${{ github.workflow }}\",\"max\":10,\"max-size\":10240}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":10},\"add_labels\":{\"max\":10},\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":72,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[experiments] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":true},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{},\"upload_asset\":{\"allowed-exts\":[\".png\",\".jpg\",\".jpeg\",\".svg\"],\"branch\":\"assets/${{ github.workflow }}\",\"max\":10,\"max-size\":10240}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/daily-experiment-report.md b/.github/workflows/daily-experiment-report.md index c6768f6c972..4a6cd31ed1a 100644 --- a/.github/workflows/daily-experiment-report.md +++ b/.github/workflows/daily-experiment-report.md @@ -44,7 +44,7 @@ safe-outputs: max: 10 add-labels: max: 10 - mentions: false + mentions: true allowed-github-references: [] max-bot-mentions: 1 @@ -117,6 +117,7 @@ by the experiments CLI: - Primary metric (`metric:` field), if set - Secondary metrics (`secondary_metrics:` list), if set - Tracking issue number, if an `issue:` field is set +- Notify target (`notify.issue` and/or `notify.discussion`), if set If no workflows declare `experiments:`, append the following to `$GITHUB_STEP_SUMMARY` and exit: @@ -530,16 +531,19 @@ After the discussion is created, also write a one-line summary to `$GITHUB_STEP_ Daily experiment report: N experiments analysed, M reached significance (p < 0.05). Discussion: ``` -## Step 8 โ€” Notify Tracking Issues +## Step 8 โ€” Notify Experiment Owners -For each experiment that has a `issue:` field set, post a comment to that tracking issue when any -of the following conditions are met **for the first time today**: +For each experiment that has a `notify.issue` or `notify.discussion` target set, post a comment to +that target when any of the following conditions are met **for the first time today**. Every +notification comment MUST include a mention for the experiment owner as `@${{ github.repository_owner }}`. **Condition A โ€” All variants reached `min_samples`:** Post a comment: ``` ๐Ÿงช **Experiment `` is ready for analysis!** +Owner: @${{ github.repository_owner }} + All variants have reached the minimum sample size of `` runs: : / ... @@ -553,6 +557,8 @@ Post a comment: ``` ๐Ÿ“Š **Experiment `` has reached statistical significance (p = )** +Owner: @${{ github.repository_owner }} + Recommendation: **** @@ -564,15 +570,17 @@ Post a comment: ``` โš ๏ธ **Guardrail violation in experiment ``** +Owner: @${{ github.repository_owner }} + The following guardrail metric failed: - `` expected ``, got `` for variant `` Recommendation: **ABANDON** โ€” investigate immediately. ``` -Use the `add-comment` safe-output tool to post comments. Skip experiments with no -`issue:` field. Do not post duplicate comments if the same condition was already reported in a -previous run today. +Use the `add-comment` safe-output tool to post comments. Select the target in this order: +1) `notify.issue` (when present), 2) `notify.discussion` (when present), 3) skip. +Do not post duplicate comments if the same condition was already reported in a previous run today. ## Step 9 โ€” Update Experiment Lifecycle Labels From 17d1ccf6462396cf4bfafec482190a66eb8ffa0b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 06:32:02 +0000 Subject: [PATCH 2/2] Refine experiments.notify notification instructions and owner mentions Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/daily-experiment-report.lock.yml | 2 +- .github/workflows/daily-experiment-report.md | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index 227896cf780..b9d476fe88f 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"30a96670d4158c5a60299d6ef5c3d32e74fd5fb28c0d8fe62b3636528d7015e7","body_hash":"de26a4a9956ab4d2d70f21d0409065b4e4e4355978b9ec07c19b4671cfd1b20d","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"30a96670d4158c5a60299d6ef5c3d32e74fd5fb28c0d8fe62b3636528d7015e7","body_hash":"e5acb07e5db1e9b5cc1ac6fd4dbe993bbaad4d9abcddc583d5be43f6a3816ff1","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/setup-python","sha":"a309ff8b426b58ec0e2a45f0f869d46889d02405","version":"v6.2.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) diff --git a/.github/workflows/daily-experiment-report.md b/.github/workflows/daily-experiment-report.md index 4a6cd31ed1a..17741a39110 100644 --- a/.github/workflows/daily-experiment-report.md +++ b/.github/workflows/daily-experiment-report.md @@ -117,7 +117,7 @@ by the experiments CLI: - Primary metric (`metric:` field), if set - Secondary metrics (`secondary_metrics:` list), if set - Tracking issue number, if an `issue:` field is set -- Notify target (`notify.issue` and/or `notify.discussion`), if set +- Notify target (`notify.issue` and/or `notify.discussion`), if set, to determine where readiness/significance/guardrail notifications should be posted If no workflows declare `experiments:`, append the following to `$GITHUB_STEP_SUMMARY` and exit: @@ -534,8 +534,9 @@ Daily experiment report: N experiments analysed, M reached significance (p < 0.0 ## Step 8 โ€” Notify Experiment Owners For each experiment that has a `notify.issue` or `notify.discussion` target set, post a comment to -that target when any of the following conditions are met **for the first time today**. Every -notification comment MUST include a mention for the experiment owner as `@${{ github.repository_owner }}`. +that target when any of the following conditions are met **for the first time today**. In this +workflow, the experiment owner is defined by convention as the repository owner, so every +notification comment MUST include `@${{ github.repository_owner }}`. **Condition A โ€” All variants reached `min_samples`:** Post a comment: @@ -579,7 +580,9 @@ Recommendation: **ABANDON** โ€” investigate immediately. ``` Use the `add-comment` safe-output tool to post comments. Select the target in this order: -1) `notify.issue` (when present), 2) `notify.discussion` (when present), 3) skip. +1) `notify.issue` (when present), 2) `notify.discussion` (when present), 3) skip. If both are +present, post to `notify.issue` only so each notification is emitted once and tracking stays tied to +the canonical experiment thread. Do not post duplicate comments if the same condition was already reported in a previous run today. ## Step 9 โ€” Update Experiment Lifecycle Labels