From 0f18d6dc2d1fde6edc804abaea9c3ac51cd6be75 Mon Sep 17 00:00:00 2001 From: Sudhanshu Sali Date: Mon, 13 Jul 2026 15:38:32 -0400 Subject: [PATCH 1/2] Default to Opus 4.8; freeze v0 and adopt the v1.x release track Two changes bundled for the v1.8 release: 1. Bump the built-in default model opus-4-7 -> opus-4-8 for the reasoning stages (Investigator/Critic). Reporter/issue path stays Haiku 4.5 (Opus rejects outputConfig.textFormat over Bedrock). The sampling-param strip pattern already covers opus-4-8, so no call-path change is needed. Routed every default through config._DEFAULT_MODEL / _DEFAULT_REPORTER_MODEL (main.py provenance + pricing fallback, doctor.py) so the literal lives in exactly one place and a future bump can't drift the early-exit paths out of sync. 2. Freeze the moving v0 tag and document the v1.x numbered-release track. v0 had silently stopped advancing at the v1.6 vintage, so the README's 'v0 = latest' contract was already false. Convert every @v0 / /v0/ / default-v0 reference (README, caller example, action.yml, reusable workflow default, CFN template) to v1.8, and rewrite the versioning + supply-chain prose to recommend a numbered tag or SHA pin. Also add the missing BEDROCK_ISSUE_MODEL_ID row to the README env table. 538 tests pass. --- .github/workflows/shadow-review.yml | 4 +-- README.md | 39 +++++++++++++------------- SECURITY.md | 10 ++++--- action.yml | 12 ++++---- examples/caller-workflow.yml | 10 ++++--- examples/shadow.example.yml | 4 +-- examples/shadow.java.yml | 4 +-- infrastructure/shadow-iam-stack.yaml | 9 +++--- requirements.txt | 2 +- src/scripts/shadow/bedrock_client.py | 2 +- src/scripts/shadow/config.py | 6 ++-- src/scripts/shadow/doctor.py | 10 ++++--- src/scripts/shadow/main.py | 18 ++++++------ tests/unit/test_cost.py | 3 +- tests/unit/test_provenance_from_env.py | 2 +- 15 files changed, 72 insertions(+), 63 deletions(-) diff --git a/.github/workflows/shadow-review.yml b/.github/workflows/shadow-review.yml index 9be14dd..507e9d7 100644 --- a/.github/workflows/shadow-review.yml +++ b/.github/workflows/shadow-review.yml @@ -14,7 +14,7 @@ on: inputs: shadow_ref: type: string - default: v0 + default: v1.8 description: "git ref of sudsali/shadow to clone (tag, branch, or sha)" pr_number: type: string @@ -155,7 +155,7 @@ jobs: EVENT_AFTER: ${{ github.event.pull_request.head.sha || github.event.after }} KB_S3_BUCKET: ${{ secrets.KB_S3_BUCKET }} KB_S3_KEY: ${{ secrets.KB_S3_KEY }} - BEDROCK_MODEL_ID: ${{ secrets.BEDROCK_MODEL_ID || 'us.anthropic.claude-opus-4-7' }} + BEDROCK_MODEL_ID: ${{ secrets.BEDROCK_MODEL_ID || 'us.anthropic.claude-opus-4-8' }} BEDROCK_REPORTER_MODEL_ID: ${{ secrets.BEDROCK_REPORTER_MODEL_ID || 'us.anthropic.claude-haiku-4-5-20251001-v1:0' }} BEDROCK_CRITIC_MODEL_ID: ${{ secrets.BEDROCK_CRITIC_MODEL_ID }} BEDROCK_ISSUE_MODEL_ID: ${{ secrets.BEDROCK_ISSUE_MODEL_ID }} diff --git a/README.md b/README.md index b890d74..43f9e01 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Three agents. The Investigator emits structured findings with hypothesis, eviden This is generator–verifier as the **core architecture**, not a re-rank step. A single agent can't disprove itself; the pipeline forces independent verification before anything reaches your repo. -**Cost-aware model split:** Investigator and Critic run on Opus 4.7 (deep search + independent verification). Reporter runs on Haiku 4.5 (cheap structured-JSON formatting). The two reasoning stages dominate token spend; the Reporter is a thin formatting pass. +**Cost-aware model split:** Investigator and Critic run on Opus 4.8 (deep search + independent verification). Reporter runs on Haiku 4.5 (cheap structured-JSON formatting). The two reasoning stages dominate token spend; the Reporter is a thin formatting pass. --- @@ -104,7 +104,7 @@ permissions: issues: write jobs: shadow: - uses: sudsali/shadow/.github/workflows/shadow-review.yml@v0 + uses: sudsali/shadow/.github/workflows/shadow-review.yml@v1.8 secrets: AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} GUARDRAIL_ID: ${{ secrets.GUARDRAIL_ID }} @@ -197,8 +197,8 @@ bot: # Env vars BEDROCK_MODEL_ID / BEDROCK_REPORTER_MODEL_ID / BEDROCK_CRITIC_MODEL_ID / # BEDROCK_ISSUE_MODEL_ID take precedence. models: - investigator: us.anthropic.claude-opus-4-7 - critic: us.anthropic.claude-opus-4-7 + investigator: us.anthropic.claude-opus-4-8 + critic: us.anthropic.claude-opus-4-8 reporter: us.anthropic.claude-haiku-4-5-20251001-v1:0 # PR-review JSON formatting only # issue: us.anthropic.claude-sonnet-4-6 # issue/followup answers; defaults to `reporter`. # # Must support structured output (Haiku 4.5 / Sonnet 4.6). @@ -224,9 +224,10 @@ models: | `BOT_GITHUB_ACTOR` | `github-actions[bot]` | GitHub login Shadow's comments appear under. **Set this to a unique value** if your repo has other workflows that also post as `github-actions[bot]` (e.g., PR-overlap detectors, claim-checkers). Otherwise Shadow's `already_commented` dedup matches their comments and silently SKIPs every PR. | | `BOT_REQUIRE_GUARDRAIL` | `true` | Production runs (`DRY_RUN=false`) refuse to start when `GUARDRAIL_ID` is unset — Shadow won't run without prompt-injection defense. On the reusable workflow, drive this via the `require_guardrail` **input** (`with: require_guardrail: 'false'`) — accepts `0`/`false`/`no`/`off`. `DRY_RUN=true` bypasses the gate regardless. | | `DRY_RUN` | `false` | When `true`, Shadow writes the artifact but doesn't post comments. Bypasses the `BOT_REQUIRE_GUARDRAIL` gate. | -| `BEDROCK_MODEL_ID` | `us.anthropic.claude-opus-4-7` | Investigator model. | -| `BEDROCK_REPORTER_MODEL_ID` | `us.anthropic.claude-haiku-4-5-20251001-v1:0` | Reporter model. Default is Haiku because Bedrock's `outputConfig.textFormat` is Haiku-only over Converse today; Opus 4.7 rejects it. | +| `BEDROCK_MODEL_ID` | `us.anthropic.claude-opus-4-8` | Investigator model. | +| `BEDROCK_REPORTER_MODEL_ID` | `us.anthropic.claude-haiku-4-5-20251001-v1:0` | Reporter model. Default is Haiku because Bedrock's `outputConfig.textFormat` is Haiku-only over Converse today; Opus 4.7/4.8 reject it. | | `BEDROCK_CRITIC_MODEL_ID` | falls back to `BEDROCK_MODEL_ID` | Critic model. | +| `BEDROCK_ISSUE_MODEL_ID` | falls back to `BEDROCK_REPORTER_MODEL_ID` | Issue/followup answer model. Decoupled from the PR Reporter so you can answer issues on a stronger model while keeping PR-review JSON formatting cheap. **Must support structured output** — Haiku 4.5 or Sonnet 4.6 (Opus 4.7/4.8 and Sonnet 5 reject it). | | `GUARDRAIL_ID` / `GUARDRAIL_VERSION` | unset | Bedrock Guardrail ID + version. **The CFN Launch Stack provisions one by default**; copy the `GuardrailId`/`GuardrailVersion` outputs into these repo secrets after stack deploy. **Required for production runs** — with `require_guardrail` defaulting to `true`, a `DRY_RUN=false` run refuses to start when these are unset (pass `require_guardrail: 'false'` under `with:` to opt out). When set, the guardrail wraps every Converse call as a server-side prompt-injection scanner on top of the local sanitizer + prompt constraints. | | `KB_S3_BUCKET` / `KB_S3_KEY` | unset | Optional S3-hosted knowledge base appended to the Investigator's system prompt. Useful for project-specific conventions. The IAM role needs `s3:GetObject` on that bucket; CFN doesn't grant this — extend the role yourself. | | `SLACK_WEBHOOK_URL` | unset | Slack channel webhook for escalation pings. Set to a [Slack incoming webhook URL](https://api.slack.com/messaging/webhooks); the bot posts a one-line summary to that channel on every ESCALATE. | @@ -242,7 +243,7 @@ Shadow ships language-agnostic default prompts (`prompts/*.txt`). To run your ow ```yaml jobs: shadow: - uses: sudsali/shadow/.github/workflows/shadow-review.yml@ # replace with an audited ref (or @v0) + uses: sudsali/shadow/.github/workflows/shadow-review.yml@ # replace with an audited ref (or a @v1.x tag) with: prompt_sm_prefix: my-bot # SM namespace holding your prompts aws_region: us-east-1 # region of the secrets + Bedrock (match your CFN BedrockRegion) @@ -278,7 +279,7 @@ Shadow is BYO-AWS today. The bot calls Bedrock from **your** account, you get th ### Recommended: one-click CloudFormation Launch Stack -[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https://raw.githubusercontent.com/sudsali/shadow/v0/infrastructure/shadow-iam-stack.yaml&stackName=shadow-bot) +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https://raw.githubusercontent.com/sudsali/shadow/v1.8/infrastructure/shadow-iam-stack.yaml&stackName=shadow-bot) The button opens AWS Console with [`infrastructure/shadow-iam-stack.yaml`](infrastructure/shadow-iam-stack.yaml) pre-loaded. Fill in: @@ -287,8 +288,8 @@ The button opens AWS Console with [`infrastructure/shadow-iam-stack.yaml`](infra | **GitHubOrg** | Your GitHub org or username | | **GitHubRepo** | Repo name. No default — pick one. Pass `*` only if you've audited every repo in the org. | | **ShadowSourceRepo** | `sudsali/shadow` (default) — or your fork's `owner/repo` if you maintain a hardened private copy | -| **ShadowWorkflowRef** | `*` for quick start, or a 40-char SHA to pin trust to one audited revision | -| **BedrockRegion** | Where Bedrock will be invoked. `us-east-1` / `us-west-2` / `us-east-2` are the validated combinations; other regions work if both Opus 4.7 and Haiku 4.5 are available there ([model-region matrix](https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html)). The region you pick here must match where you enable model access in the next step. | +| **ShadowWorkflowRef** | `*` for quick start, a `refs/tags/v1.x` release tag (e.g. `refs/tags/v1.8`), or a 40-char SHA to pin trust to one audited revision | +| **BedrockRegion** | Where Bedrock will be invoked. `us-east-1` / `us-west-2` / `us-east-2` are the validated combinations; other regions work if both Opus 4.8 and Haiku 4.5 are available there ([model-region matrix](https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html)). The region you pick here must match where you enable model access in the next step. | | **ExistingOidcProviderArn** | Leave blank if your account has no GitHub OIDC provider yet. **If your account already uses GitHub Actions OIDC, paste the existing provider ARN** (`aws iam list-open-id-connect-providers`). Leaving blank when one exists fails with `EntityAlreadyExists`. | | **MonthlyBudgetLimit** + **BudgetEmailAddress** | Optional. Set both to enable an AWS Budget that emails at 80% / 100% of the cap. `0` / blank skips the alarm. | | **ProvisionGuardrail** | Default `true`. Provisions a Bedrock Guardrail with prompt-attack defense + PII blocks (see [Security model](#security-model)). Set to `false` only if you maintain a custom guardrail and want to point Shadow at it via the `GUARDRAIL_ID`/`GUARDRAIL_VERSION` secrets. | @@ -301,7 +302,7 @@ The stack creates the OIDC provider (if needed), an IAM role with the canonical You still need to **enable Bedrock model access** (the stack can't do this for you): -> AWS Console → Bedrock (in your `BedrockRegion`) → Model access → enable `anthropic.claude-opus-4-7` AND `anthropic.claude-haiku-4-5`. Auto-subscribes in ≤ 15 min. +> AWS Console → Bedrock (in your `BedrockRegion`) → Model access → enable `anthropic.claude-opus-4-8` AND `anthropic.claude-haiku-4-5`. Auto-subscribes in ≤ 15 min. After the stack is up: @@ -309,7 +310,7 @@ After the stack is up: python -m shadow.doctor --role-arn $ARN --region $REGION ``` -> **Heads-up on the template URL:** the Launch Stack button resolves `…/sudsali/shadow/v0/infrastructure/shadow-iam-stack.yaml` at click time. It tracks the moving `v0` tag. If you re-launch the stack later, AWS fetches whatever is at `v0` *then* — not what you saw before. For reproducible IAM provisioning, download the YAML at a specific SHA and upload it manually. +> **Heads-up on the template URL:** the Launch Stack button resolves `…/sudsali/shadow/v1.8/infrastructure/shadow-iam-stack.yaml` at click time — pinned to the `v1.8` release tag, so it's stable as long as you use this button. If you bump to a newer Shadow release, grab that release's Launch Stack URL. For fully reproducible IAM provisioning, download the YAML at a specific SHA and upload it manually. ### Manual setup (alternative) @@ -385,10 +386,10 @@ You're letting a bot read your repo and post on your behalf. Here's the trust bo Pick the trade-off: -- **`@v0`** (moving tag) — auto-updates to whatever upstream tags as `v0` next. **Lowest friction; you don't control which version reviews your code.** Suitable for trying the bot. -- **`@<40-char SHA>`** — frozen at the version you audited. Manual update required. **Recommended for production.** Add a Dependabot config so SHA bumps land as PRs you can review: +- **`@v1.8`** (numbered release tag) — a specific, immutable version. Bump it yourself when you want a newer one. **Recommended starting point** — readable, and it won't change under you. Browse the [Releases page](https://github.com/sudsali/shadow/releases) for the changelog. +- **`@<40-char SHA>`** — frozen at the exact commit you audited. Strongest supply-chain guarantee (a tag *could* in principle be re-pointed; a SHA can't). **Recommended for production.** Add a Dependabot config so SHA/tag bumps land as PRs you can review: ```yaml - # .github/dependabot.yml — bumps @ pins; silently inert for @v0 + # .github/dependabot.yml — bumps @ and @v1.x pins as reviewable PRs version: 2 updates: - package-ecosystem: github-actions @@ -397,7 +398,7 @@ Pick the trade-off: ``` - **Fork into your org and pin to your fork's SHA** — full org control. Recommended when your trust boundary is the org, not an individual GitHub account. -> **`v0` tag stability.** The `v0` tag is currently force-pushed as the project iterates — adopters pinning `@v0` get the latest revision on every workflow run. The first numbered release (`v1.0`) will freeze the tag-version contract: from then on, `v0` will not move and follow-on changes ship as semver releases (`v1.1`, `v1.2`, …). Until that release, treat `@v0` as "latest" semantics. SHA-pinned adopters are insulated from any tag movement. +> **Versioning.** Shadow ships as numbered releases on the `v1.x` track (`v1.6`, `v1.7`, `v1.8`, …) — each an immutable tag with a matching [GitHub Release](https://github.com/sudsali/shadow/releases). Pin to a specific `v1.x` tag (or its SHA) and bump deliberately; nothing moves under you. The old `@v0` "latest" tag is **frozen** at its final pre-`v1.x` revision and no longer advances — if you pinned `@v0`, move to a numbered tag to get current fixes. SHA-pinned adopters are unaffected either way. --- @@ -440,13 +441,13 @@ If you previously tried to set `SHADOW_DISABLED` as a Secret rather than a Varia | Symptom | Likely cause | Fix | |---|---|---| | `Could not assume role` from configure-aws-credentials | OIDC trust policy mismatch | Verify `sub` matches `repo:YOUR_ORG/YOUR_REPO:*` and `job_workflow_ref` includes the Shadow workflow path. Run `aws sts get-caller-identity` from a minimal workflow first. | -| `AccessDeniedException` on Bedrock | Model access not enabled in region | AWS Console → Bedrock → Model access → enable Opus 4.7 + Haiku 4.5. | -| `ValidationException` on Bedrock call | Wrong model ID format | Check `BEDROCK_MODEL_ID` is `us.anthropic.claude-opus-4-7` (no `-v1` suffix on 4.7). | +| `AccessDeniedException` on Bedrock | Model access not enabled in region | AWS Console → Bedrock → Model access → enable Opus 4.8 + Haiku 4.5. | +| `ValidationException` on Bedrock call | Wrong model ID format | Check `BEDROCK_MODEL_ID` is `us.anthropic.claude-opus-4-8` (no `-v1` suffix on 4.7/4.8). | | `BOT_REQUIRE_GUARDRAIL=true and DRY_RUN=false but GUARDRAIL_ID is unset`, run exits 1 | Production run started without a guardrail wired in | Paste the stack's `GuardrailId`/`GuardrailVersion` outputs into the `GUARDRAIL_ID`/`GUARDRAIL_VERSION` repo secrets and forward them in the caller workflow's `secrets:`. To run without one, pass `require_guardrail: 'false'` under `with:`. | | `AccessDeniedException` on `bedrock:ApplyGuardrail` | IAM role lacks `bedrock:ApplyGuardrail` for the configured guardrail | Manual-setup only — add the `bedrock:ApplyGuardrail` statement scoped to your guardrail ARN (see [Manual setup](#manual-setup-alternative)). The Launch Stack grants it automatically. | | `AccessDeniedException` on `secretsmanager:GetSecretValue` (or every PR/issue/followup escalates with `prompt_load_failed` after setting `prompt_sm_prefix`) | Using [BYO prompts](#bring-your-own-prompts) but the IAM role can't read the secrets | Add a `secretsmanager:GetSecretValue` statement scoped to `arn:aws:secretsmanager:::secret:/*-prompt*` (covers all eight prompt secrets — the five `pr-*` plus `issue-classify`/`issue-respond`/`followup`). The CFN Launch Stack does NOT grant this — you extend the role. Also confirm the secrets exist in `aws_region`. | | `[WARN] SM prompt(s) configured but fell back to bundled defaults` from `shadow doctor` | An `SM_*` prompt secret is missing/unreadable/in the wrong region, so a commit prompt silently used the bundled default | Verify the secret names exist and the role has `GetSecretValue` in `aws_region`. Core prompts fail closed loudly; commit prompts fall back silently, which is why the doctor flags it. | -| `Required prompt missing: prompts/pr-investigator.txt` | `@v0` (or pinned SHA) doesn't include `prompts/` | Verify the ref in `sudsali/shadow` includes the `prompts/` directory. If you forked, ensure your tag does too. | +| `Required prompt missing: prompts/pr-investigator.txt` | pinned ref (`@v1.x` or SHA) doesn't include `prompts/` | Verify the ref in `sudsali/shadow` includes the `prompts/` directory. If you forked, ensure your tag does too. | | No comments posted, workflow green | `dry_run: true` | Set `dry_run: false` in caller-workflow inputs. | | Every PR escalates with `prompt_load_failed` | `prompts/` not present at the pinned `shadow_ref` | Same as above. | | `existing_feedback` always empty | Caller workflow missing `pull-requests: read` | Reusable workflow declares this; if you customized the caller, ensure permissions include `pull-requests: read`. | diff --git a/SECURITY.md b/SECURITY.md index d78d6aa..d0281d1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ The bot's privileged operations are: inputs include the diff, codebase tree, and existing PR/issue comments. 2. **Posting on GitHub as the workflow's bot identity** — inline review comments, labels, optional Slack pings. -3. **Calling Bedrock from your AWS account** — Anthropic Opus 4.7 + Haiku 4.5 +3. **Calling Bedrock from your AWS account** — Anthropic Opus 4.8 + Haiku 4.5 inferences, scoped to your IAM role's permissions. ### Adversaries @@ -25,9 +25,11 @@ The bot's privileged operations are: workflow file, or Shadow source. - **Existing-repo collaborator.** Same as above plus can author issue/comment content under any GitHub login the bot is configured to recognize. -- **Compromised upstream `sudsali/shadow`.** A malicious commit to the moving - `v0` tag would propagate to every adopter pinning `@v0` on their next workflow - run. Adopters who SHA-pin (`@<40-char SHA>`) are insulated. +- **Compromised upstream `sudsali/shadow`.** A `v1.x` release tag is immutable + by convention, but a tag *can* be force-repointed at the source repo; if that + happened, adopters pinning that tag (`@v1.8`) would pick up the malicious + revision on their next workflow run. Adopters who SHA-pin (`@<40-char SHA>`) + are insulated — a SHA cannot be repointed. - **Compromised Anthropic / AWS Bedrock.** Out of scope; same trust boundary as the adopter's other Bedrock usage. diff --git a/action.yml b/action.yml index f57afc1..9a15578 100644 --- a/action.yml +++ b/action.yml @@ -14,11 +14,11 @@ branding: # and forwards to the reusable workflow; all real work happens there. # # Adopters can use either shape: -# uses: sudsali/shadow@v0 +# uses: sudsali/shadow@v1.8 # with: # aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} # or directly: -# uses: sudsali/shadow/.github/workflows/shadow-review.yml@v0 +# uses: sudsali/shadow/.github/workflows/shadow-review.yml@v1.8 # secrets: # AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} # @@ -60,9 +60,9 @@ inputs: required: false default: "false" shadow_ref: - description: Pin to a Shadow release (`v0` or a 40-char SHA). + description: Pin to a Shadow release (a `v1.x` tag or a 40-char SHA). required: false - default: "v0" + default: "v1.8" outputs: shadow_result: @@ -79,13 +79,13 @@ outputs: # # jobs: # shadow: -# uses: sudsali/shadow/.github/workflows/shadow-review.yml@v0 +# uses: sudsali/shadow/.github/workflows/shadow-review.yml@v1.8 # # When GitHub Actions adds composite-to-reusable-workflow chaining # (https://github.com/orgs/community/discussions/56086), this file flips # from stub to real wrapper. Until then, this exists so: # 1. Marketplace search finds Shadow -# 2. `uses: sudsali/shadow@v0` doesn't 404 +# 2. `uses: sudsali/shadow@v1.8` doesn't 404 # 3. The branding badge renders on PRs runs: using: "composite" diff --git a/examples/caller-workflow.yml b/examples/caller-workflow.yml index 6d8fdb6..ba91113 100644 --- a/examples/caller-workflow.yml +++ b/examples/caller-workflow.yml @@ -29,8 +29,8 @@ on: type: boolean default: true shadow_ref: - default: v0 - description: "Shadow ref to pin to: v0 (moving tag), or a 40-char commit SHA" + default: v1.8 + description: "Shadow ref to pin to: a v1.x release tag, or a 40-char commit SHA" permissions: contents: read @@ -43,11 +43,13 @@ permissions: # disable both jobs without deleting this workflow file. jobs: shadow: - uses: sudsali/shadow/.github/workflows/shadow-review.yml@v0 + uses: sudsali/shadow/.github/workflows/shadow-review.yml@v1.8 with: pr_number: ${{ inputs.pr_number }} dry_run: ${{ inputs.dry_run && 'true' || 'false' }} - shadow_ref: ${{ inputs.shadow_ref || 'v0' }} + # Keep this in sync with the `uses:` pin above — GitHub forbids + # expressions in `uses:`, so the two can't share a variable. + shadow_ref: ${{ inputs.shadow_ref || 'v1.8' }} # Cost levers — uncomment to override the defaults (10/8). Lower # numbers cap Bedrock spend at the cost of review depth. # investigator_max_tool_calls: "5" diff --git a/examples/shadow.example.yml b/examples/shadow.example.yml index 28dae5b..e0c6449 100644 --- a/examples/shadow.example.yml +++ b/examples/shadow.example.yml @@ -20,8 +20,8 @@ bot: # Env vars (BEDROCK_MODEL_ID, BEDROCK_REPORTER_MODEL_ID, BEDROCK_CRITIC_MODEL_ID) # take precedence over yaml. models: - investigator: us.anthropic.claude-opus-4-7 - critic: us.anthropic.claude-opus-4-7 + investigator: us.anthropic.claude-opus-4-8 + critic: us.anthropic.claude-opus-4-8 reporter: us.anthropic.claude-haiku-4-5-20251001-v1:0 # PR-review JSON formatting # issue: # issue/followup answers; defaults to `reporter`. Must # # support structured output (Haiku 4.5 or Sonnet 4.6). diff --git a/examples/shadow.java.yml b/examples/shadow.java.yml index d9a13fe..4503ad4 100644 --- a/examples/shadow.java.yml +++ b/examples/shadow.java.yml @@ -10,6 +10,6 @@ bot: escalate_label: needs-human models: - investigator: us.anthropic.claude-opus-4-7 - critic: us.anthropic.claude-opus-4-7 + investigator: us.anthropic.claude-opus-4-8 + critic: us.anthropic.claude-opus-4-8 reporter: us.anthropic.claude-haiku-4-5-20251001-v1:0 diff --git a/infrastructure/shadow-iam-stack.yaml b/infrastructure/shadow-iam-stack.yaml index d68ab3d..9d660f0 100644 --- a/infrastructure/shadow-iam-stack.yaml +++ b/infrastructure/shadow-iam-stack.yaml @@ -23,7 +23,7 @@ Metadata: GitHubOrg: { default: "Your GitHub org or user (e.g., awslabs, sudsali)" } GitHubRepo: { default: "Your repo name (use * for org-wide; audit first)" } ShadowSourceRepo: { default: "Shadow source repo (sudsali/shadow upstream, or your fork)" } - ShadowWorkflowRef: { default: "Shadow workflow ref pin (use * for v0; SHA for prod)" } + ShadowWorkflowRef: { default: "Shadow workflow ref pin (use * for quick start; tag or SHA for prod)" } BedrockRegion: { default: "Bedrock region (us-east-1, us-west-2, us-east-2)" } MonthlyBudgetLimit: { default: "Monthly Bedrock spend limit in USD (0 = disabled)" } BudgetEmailAddress: { default: "Email for budget threshold notifications" } @@ -65,9 +65,10 @@ Parameters: Description: >- `job_workflow_ref` pin for Shadow's reusable workflow. Use `*` for a quick start (any version of the workflow can assume this role). - For production, pin to a 40-char SHA you've audited: - `refs/tags/v0` or ``. The trailing wildcard lets the - same role work across Shadow patch releases until you re-pin. + For production, pin to a release tag or an audited 40-char SHA: + `refs/tags/v1.8` or ``. A trailing wildcard (e.g. + `refs/tags/v1.*`) lets the same role work across Shadow releases + until you re-pin. Default: "*" MinLength: 1 MaxLength: 200 diff --git a/requirements.txt b/requirements.txt index 1642524..4367226 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# Version-pinned for reproducibility. Production hardening (post-v0): +# Version-pinned for reproducibility. Production hardening (planned): # regenerate with `pip-compile --generate-hashes` and call `pip install # --require-hashes` in the workflow to defend against PyPI compromise. requests==2.33.1 diff --git a/src/scripts/shadow/bedrock_client.py b/src/scripts/shadow/bedrock_client.py index 1330ad3..212fd1b 100644 --- a/src/scripts/shadow/bedrock_client.py +++ b/src/scripts/shadow/bedrock_client.py @@ -229,7 +229,7 @@ def invoke(self, system_prompt, user_prompt, max_tokens=4096, write premium with zero reuse. Issue/followup callers pass True since their prefix is KB + codebase_map, stable across calls. - model_id: override the default model for this call. Issue triage - uses Haiku because Opus 4.7 doesn't accept outputConfig.textFormat + uses Haiku because Opus (4.7/4.8) doesn't accept outputConfig.textFormat (structured output) over Bedrock today; Haiku does. """ target_model = model_id or self._model_id diff --git a/src/scripts/shadow/config.py b/src/scripts/shadow/config.py index 91b58b4..504ad35 100644 --- a/src/scripts/shadow/config.py +++ b/src/scripts/shadow/config.py @@ -11,7 +11,7 @@ # Built-in default model. Module-level so the cost-pricing test can assert # the pricing table covers it without re-encoding the literal in two places. -_DEFAULT_MODEL = "us.anthropic.claude-opus-4-7" +_DEFAULT_MODEL = "us.anthropic.claude-opus-4-8" # Reporter built-in default. Haiku because the Reporter is JSON-formatting # only — Opus's reasoning depth is wasted there, and Haiku is the model # documented as the default in the README. @@ -40,8 +40,8 @@ def __init__(self): _DEFAULT_MODEL, ), _DEFAULT_MODEL) # Reporter is JSON-formatting only; Haiku handles structured output - # well enough AND is the model the issue path REQUIRES (Opus 4.7 - # rejects outputConfig.textFormat over Bedrock today). Adopters who + # well enough AND is the model the issue path REQUIRES (Opus 4.7/4.8 + # reject outputConfig.textFormat over Bedrock today). Adopters who # override BEDROCK_MODEL_ID (Investigator) without setting Reporter # silently get Haiku here — log loud at INFO so the asymmetry is # visible in workflow logs rather than discovered via "why does my diff --git a/src/scripts/shadow/doctor.py b/src/scripts/shadow/doctor.py index 059214f..051ca74 100644 --- a/src/scripts/shadow/doctor.py +++ b/src/scripts/shadow/doctor.py @@ -117,12 +117,14 @@ def _check_bedrock_access(args, result): # which model families reject sampling params. from .bedrock_client import _build_inference_config from . import shadow_config + from .config import _DEFAULT_MODEL, _DEFAULT_REPORTER_MODEL # Validate the models the adopter actually configured, not hardcoded ones — - # otherwise doctor greens an install whose real model (e.g. an Opus 4.8 + # otherwise doctor greens an install whose real model (e.g. a Sonnet 4.6 # override) is unreachable. Resolve exactly as Config does: env > .shadow.yml - # > default, so a yaml-only override is validated too. - _default_opus = "us.anthropic.claude-opus-4-7" - _default_haiku = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + # > default, so a yaml-only override is validated too. Defaults come from + # config so a default-model bump can't drift doctor out of sync. + _default_opus = _DEFAULT_MODEL + _default_haiku = _DEFAULT_REPORTER_MODEL yml = shadow_config.load(args.repo_root or os.getenv("SHADOW_REPO_ROOT", ".")) investigator = shadow_config.env_or( "BEDROCK_MODEL_ID", shadow_config.get(yml, "models", "investigator"), _default_opus) diff --git a/src/scripts/shadow/main.py b/src/scripts/shadow/main.py index 1e0dd25..ee5aac2 100644 --- a/src/scripts/shadow/main.py +++ b/src/scripts/shadow/main.py @@ -17,7 +17,7 @@ import time import uuid -from .config import Config +from .config import Config, _DEFAULT_MODEL, _DEFAULT_REPORTER_MODEL from .bedrock_client import BedrockClient from .github_client import GitHubClient from .knowledge_base import KnowledgeBase @@ -170,7 +170,7 @@ def analyze(): # The model id stamped on this item's artifacts — and interpolated into the # user-facing "Generated by AI (model: …)" footer act() posts. The - # issue/followup surface invokes the reporter model (Haiku; Opus 4.7 rejects + # issue/followup surface invokes the reporter model (Haiku; Opus 4.7/4.8 reject # structured output over Bedrock), the PR pipeline invokes the investigator # model. Name the model that actually runs on THIS surface so the footer is # truthful. Computed once here so the many early-exit artifacts below can't @@ -1837,7 +1837,7 @@ def _finalize_metrics(metrics, inv, crit): # Bedrock per-1M-token list prices for the cross-region inference profiles # Shadow defaults to. Adopters who override BEDROCK_*_MODEL_ID get cost # fallback to the default so the line is still informative — under-counts -# only when the override is more expensive than Opus 4.7. +# only when the override is more expensive than Opus 4.8. _MODEL_PRICING_PER_M_TOKENS = { "us.anthropic.claude-opus-4-7": {"input": 5.00, "output": 25.00}, "us.anthropic.claude-opus-4-8": {"input": 5.00, "output": 25.00}, @@ -1849,8 +1849,8 @@ def _finalize_metrics(metrics, inv, crit): def _stage_cost(stage_metrics): if not isinstance(stage_metrics, dict) or stage_metrics.get("skipped"): return 0.0 - model_id = stage_metrics.get("model_id") or "us.anthropic.claude-opus-4-7" - pricing = _MODEL_PRICING_PER_M_TOKENS.get(model_id) or _MODEL_PRICING_PER_M_TOKENS["us.anthropic.claude-opus-4-7"] + model_id = stage_metrics.get("model_id") or _DEFAULT_MODEL + pricing = _MODEL_PRICING_PER_M_TOKENS.get(model_id) or _MODEL_PRICING_PER_M_TOKENS[_DEFAULT_MODEL] in_tok = stage_metrics.get("input_tokens", 0) or 0 out_tok = stage_metrics.get("output_tokens", 0) or 0 cache_read = stage_metrics.get("cache_read_tokens", 0) or 0 @@ -2032,10 +2032,10 @@ def _build_provenance_from_env(): in early-exit artifacts while runtime actually ran Haiku. (yaml-only model overrides are not visible here — this path has no Config; those artifacts are the cheap SKIP/ESCALATE ones, and the full path uses _build_provenance(cfg).)""" - _opus = "us.anthropic.claude-opus-4-7" - _haiku = "us.anthropic.claude-haiku-4-5-20251001-v1:0" - investigator = os.getenv("BEDROCK_MODEL_ID", _opus) - reporter = (os.getenv("BEDROCK_REPORTER_MODEL_ID") or "").strip() or _haiku + # Mirror Config's defaults from the single source of truth so a future + # default-model bump can't drift this early-exit path out of sync. + investigator = os.getenv("BEDROCK_MODEL_ID", _DEFAULT_MODEL) + reporter = (os.getenv("BEDROCK_REPORTER_MODEL_ID") or "").strip() or _DEFAULT_REPORTER_MODEL return { "schema_version": 1, "shadow_ref": os.getenv("SHADOW_BOT_REF", "").strip() or "unknown", diff --git a/tests/unit/test_cost.py b/tests/unit/test_cost.py index 52be2a2..cd3594a 100644 --- a/tests/unit/test_cost.py +++ b/tests/unit/test_cost.py @@ -79,7 +79,8 @@ def test_cache_writes_are_one_point_two_five_input_price(): def test_unknown_model_falls_back_to_opus_pricing(): """Adopter who overrides BEDROCK_MODEL_ID to something unrecognized still - gets a cost number — pricing table fallback to Opus 4.7.""" + gets a cost number — pricing table fallback to the default model (Opus, + $5/1M input).""" stage = { "skipped": False, "model_id": "anthropic.claude-9000-not-yet", "input_tokens": 1_000_000, "output_tokens": 0, diff --git a/tests/unit/test_provenance_from_env.py b/tests/unit/test_provenance_from_env.py index e478440..3e3192f 100644 --- a/tests/unit/test_provenance_from_env.py +++ b/tests/unit/test_provenance_from_env.py @@ -7,7 +7,7 @@ from shadow import main as m -_OPUS = "us.anthropic.claude-opus-4-7" +_OPUS = "us.anthropic.claude-opus-4-8" _HAIKU = "us.anthropic.claude-haiku-4-5-20251001-v1:0" _MODEL_ENVS = ("BEDROCK_MODEL_ID", "BEDROCK_REPORTER_MODEL_ID", "BEDROCK_CRITIC_MODEL_ID", "BEDROCK_ISSUE_MODEL_ID") From 98173305444bd4957df0f5ce06108043445ba7c8 Mon Sep 17 00:00:00 2001 From: Sudhanshu Sali Date: Mon, 13 Jul 2026 16:19:46 -0400 Subject: [PATCH 2/2] Address review-deep findings on the v1.8 change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full 10-angle review of the default-model + v0-freeze diff surfaced no correctness bugs but several items worth fixing (four self-introduced): - main.py _build_provenance_from_env: the investigator line used the 2-arg os.getenv(name, default), which stamps "" for a set-but-empty BEDROCK_MODEL_ID (the normal result of forwarding an unset secret), diverging from Config.env_or. Its 3 siblings already used the robust '(getenv or "").strip() or default' idiom; align investigator too so the comment's 'mirrors the single source of truth' claim is actually true. Add a parametrized empty/whitespace regression test. - infrastructure: drop the 'refs/tags/v1.*' family-wildcard suggestion for the OIDC trust pin — it auto-trusts every future v1.x tag (incl. a malicious one an upstream attacker could push), defeating the pin. - README/versioning prose: a git tag is not 'immutable' (it can be re-pointed); reword so the tag-vs-SHA tradeoff is honest and doesn't undercut the SHA-is-stronger guidance. - requirements.txt: 'post-v0' -> present-tense 'not hash-pinned yet' so the hardening note isn't an unanchored rotting TODO. - test_config_init: add a bare-Config() test asserting the built-in model defaults resolve (was only indirectly pinned via provenance). 541 tests pass (was 538). --- README.md | 4 ++-- infrastructure/shadow-iam-stack.yaml | 9 +++++---- requirements.txt | 6 +++--- src/scripts/shadow/main.py | 7 +++++-- tests/integration/test_config_init.py | 22 ++++++++++++++++++++++ tests/unit/test_provenance_from_env.py | 16 ++++++++++++++++ 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 43f9e01..35a97fd 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,7 @@ You're letting a bot read your repo and post on your behalf. Here's the trust bo Pick the trade-off: -- **`@v1.8`** (numbered release tag) — a specific, immutable version. Bump it yourself when you want a newer one. **Recommended starting point** — readable, and it won't change under you. Browse the [Releases page](https://github.com/sudsali/shadow/releases) for the changelog. +- **`@v1.8`** (numbered release tag) — a specific version, fixed by release convention (not force-moved like `v0` was). Bump it yourself when you want a newer one. **Recommended starting point** — readable and stable. Note a tag is not cryptographically immutable — it *can* be re-pointed by whoever controls the source repo — so for the strongest guarantee, pin the SHA below. Browse the [Releases page](https://github.com/sudsali/shadow/releases) for the changelog. - **`@<40-char SHA>`** — frozen at the exact commit you audited. Strongest supply-chain guarantee (a tag *could* in principle be re-pointed; a SHA can't). **Recommended for production.** Add a Dependabot config so SHA/tag bumps land as PRs you can review: ```yaml # .github/dependabot.yml — bumps @ and @v1.x pins as reviewable PRs @@ -398,7 +398,7 @@ Pick the trade-off: ``` - **Fork into your org and pin to your fork's SHA** — full org control. Recommended when your trust boundary is the org, not an individual GitHub account. -> **Versioning.** Shadow ships as numbered releases on the `v1.x` track (`v1.6`, `v1.7`, `v1.8`, …) — each an immutable tag with a matching [GitHub Release](https://github.com/sudsali/shadow/releases). Pin to a specific `v1.x` tag (or its SHA) and bump deliberately; nothing moves under you. The old `@v0` "latest" tag is **frozen** at its final pre-`v1.x` revision and no longer advances — if you pinned `@v0`, move to a numbered tag to get current fixes. SHA-pinned adopters are unaffected either way. +> **Versioning.** Shadow ships as numbered releases on the `v1.x` track (`v1.6`, `v1.7`, `v1.8`, …) — each a fixed tag (not force-moved) with a matching [GitHub Release](https://github.com/sudsali/shadow/releases). Pin to a specific `v1.x` tag (or, for a guarantee no tag re-point can break, its SHA) and bump deliberately. The old `@v0` "latest" tag is **frozen** at its final pre-`v1.x` revision and no longer advances — if you pinned `@v0`, move to a numbered tag to get current fixes. SHA-pinned adopters are unaffected either way. --- diff --git a/infrastructure/shadow-iam-stack.yaml b/infrastructure/shadow-iam-stack.yaml index 9d660f0..211429c 100644 --- a/infrastructure/shadow-iam-stack.yaml +++ b/infrastructure/shadow-iam-stack.yaml @@ -65,10 +65,11 @@ Parameters: Description: >- `job_workflow_ref` pin for Shadow's reusable workflow. Use `*` for a quick start (any version of the workflow can assume this role). - For production, pin to a release tag or an audited 40-char SHA: - `refs/tags/v1.8` or ``. A trailing wildcard (e.g. - `refs/tags/v1.*`) lets the same role work across Shadow releases - until you re-pin. + For production, pin to a specific release tag or an audited 40-char + SHA: `refs/tags/v1.8` or ``. Avoid a family wildcard like + `refs/tags/v1.*` for the trust pin — it auto-trusts every FUTURE v1.x + tag, including one an attacker who compromised upstream could push, so + it defeats the point of pinning. A SHA is strongest (immutable). Default: "*" MinLength: 1 MaxLength: 200 diff --git a/requirements.txt b/requirements.txt index 4367226..c35af55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -# Version-pinned for reproducibility. Production hardening (planned): -# regenerate with `pip-compile --generate-hashes` and call `pip install -# --require-hashes` in the workflow to defend against PyPI compromise. +# Version-pinned for reproducibility. Not hash-pinned yet: to defend against +# PyPI compromise, regenerate with `pip-compile --generate-hashes` and call +# `pip install --require-hashes` in the workflow. requests==2.33.1 boto3==1.42.94 PyYAML==6.0.2 diff --git a/src/scripts/shadow/main.py b/src/scripts/shadow/main.py index ee5aac2..ece8076 100644 --- a/src/scripts/shadow/main.py +++ b/src/scripts/shadow/main.py @@ -2033,8 +2033,11 @@ def _build_provenance_from_env(): overrides are not visible here — this path has no Config; those artifacts are the cheap SKIP/ESCALATE ones, and the full path uses _build_provenance(cfg).)""" # Mirror Config's defaults from the single source of truth so a future - # default-model bump can't drift this early-exit path out of sync. - investigator = os.getenv("BEDROCK_MODEL_ID", _DEFAULT_MODEL) + # default-model bump can't drift this early-exit path out of sync. Use the + # `(getenv or "").strip() or default` idiom on every line so an empty or + # whitespace env value falls back to the default, matching Config.env_or + # (a bare `getenv(name, default)` would stamp "" when the var is set-empty). + investigator = (os.getenv("BEDROCK_MODEL_ID") or "").strip() or _DEFAULT_MODEL reporter = (os.getenv("BEDROCK_REPORTER_MODEL_ID") or "").strip() or _DEFAULT_REPORTER_MODEL return { "schema_version": 1, diff --git a/tests/integration/test_config_init.py b/tests/integration/test_config_init.py index a02d0dc..afa641e 100644 --- a/tests/integration/test_config_init.py +++ b/tests/integration/test_config_init.py @@ -82,6 +82,28 @@ def _clear_model_env(monkeypatch): monkeypatch.delenv(v, raising=False) +def test_defaults_resolve_to_builtin_models(monkeypatch): + """A bare Config() (no env, no yaml) must resolve the reasoning stages to + _DEFAULT_MODEL and the reporter/issue stages to _DEFAULT_REPORTER_MODEL. + Asserts against the constants (not literals) so it survives a deliberate + default bump, but still catches a silent wiring regression — a stale + literal on the env_or line, _scrub_model_id mangling a clean default, or + env_or's 3rd-arg being dropped — that test_provenance_from_env (which + guards main.py's separate os.getenv path) would not.""" + _set_required_env(monkeypatch) + _clear_model_env(monkeypatch) + with tempfile.TemporaryDirectory() as tmp: + monkeypatch.setenv("SHADOW_REPO_ROOT", tmp) + from shadow.config import ( + Config, _DEFAULT_MODEL, _DEFAULT_REPORTER_MODEL, + ) + cfg = Config() + assert cfg.bedrock_model_id == _DEFAULT_MODEL + assert cfg.critic_model_id == _DEFAULT_MODEL # critic follows investigator + assert cfg.reporter_model_id == _DEFAULT_REPORTER_MODEL + assert cfg.issue_model_id == _DEFAULT_REPORTER_MODEL # issue follows reporter + + def test_yaml_models_block_overrides_per_stage(monkeypatch): """models.investigator/critic/reporter in .shadow.yml should override the built-in defaults — adopters tune cost/quality without env vars.""" diff --git a/tests/unit/test_provenance_from_env.py b/tests/unit/test_provenance_from_env.py index 3e3192f..e986336 100644 --- a/tests/unit/test_provenance_from_env.py +++ b/tests/unit/test_provenance_from_env.py @@ -45,6 +45,22 @@ def test_opus_investigator_does_not_drag_reporter_or_issue_to_opus(monkeypatch): assert mo["issue"] == _HAIKU +@pytest.mark.parametrize("empty_value", ["", " "]) +def test_empty_or_whitespace_env_falls_back_to_defaults(monkeypatch, empty_value): + # A set-but-empty/whitespace env var (e.g. `${{ secrets.X }}` where X is + # unset -> GitHub passes "") must fall back to the default on EVERY stage, + # matching Config.env_or. Guards the investigator line specifically: a bare + # `getenv(name, default)` would stamp "" here while runtime resolves the + # real default, mislabeling the audit provenance. + for var in _MODEL_ENVS: + monkeypatch.setenv(var, empty_value) + mo = _models() + assert mo["investigator"] == _OPUS + assert mo["critic"] == _OPUS + assert mo["reporter"] == _HAIKU + assert mo["issue"] == _HAIKU + + def test_issue_follows_reporter_env(monkeypatch): monkeypatch.setenv("BEDROCK_REPORTER_MODEL_ID", "us.anthropic.claude-sonnet-4-6") assert _models()["issue"] == "us.anthropic.claude-sonnet-4-6"