diff --git a/changes/2026-10-23-issue-62-model-selection/brainstorm.md b/changes/2026-10-23-issue-62-model-selection/brainstorm.md new file mode 100644 index 0000000..680f4fe --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/brainstorm.md @@ -0,0 +1,31 @@ +# Brainstorm + +> **Issue 62 model selection** · approaches compared before design. Not a final design, and not an acceptance. + +## A. Call the decider inside matchRoute + +Every ask, including `Two roles own this.`, would reach a model. `takeRole` and `chooseLead` would start calling one too. Existing router tests that expect those asks with no model would fail. A rules table inside the match would also invent a role ranking the issue does not name. + +Rejected. The decider stays outside `matchRoute`. + +## B. Store overrides on team.yaml + +One file would hold bots and model slots. `TeamSchema` is strict, and the team tests reject unknown keys. Secrets would sit next to the team roster. The issue allows this file, and Q2 records why this change does not use it. + +Rejected for this change. A later issue can add a team-file slot if the schema grows on purpose. + +## C. A separate models.yaml, consulted only on "No rule matched." + +`selectModel` is pure precedence. `decide` is the runtime, and only `routeRole` calls it, and only for that one reason, and only when the file exists. Rules return `human` and do not rank roles. Command and Ollama are the only model calls. Decision tokens sit in their own array so the existing report header and provider enum stay. + +Chosen. It keeps today's route and `SDD_ROLE_MODEL` behavior when the file is absent, and it does not pull escalation, rooms, or the evidence gate into this change. + +## Comparison + +| | A | B | C | +|---|---|---|---| +| No models.yaml matches today | No | No | Yes | +| Two owners stay with the human | No | Yes | Yes | +| Team schema stays strict | Yes | No | Yes | +| Report header stays | Unspecified | Unspecified | Yes | +| Ollama stays behind config | Unspecified | Unspecified | Yes | diff --git a/changes/2026-10-23-issue-62-model-selection/clarify-design.md b/changes/2026-10-23-issue-62-model-selection/clarify-design.md new file mode 100644 index 0000000..57fb41f --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/clarify-design.md @@ -0,0 +1,59 @@ +# Clarify design + +> **Issue 62 model selection** · architect seams checked against the reading. Not an acceptance. + +## Q1. What is the metrics key for a decision call? + +The issue says to log every decision call with tokens and cost. It does not name the JSON key. The key is `decisions`, beside `requests`. A rules decision writes no row. Command and Ollama rows store tokens, model, work type, kind, and provider. Dollars stay out of the file. + +## Q2. Are the secret length and the probability tolerance part of the issue text? + +No. They are proposal bounds so the check is testable. PEM (`-----BEGIN`), a `name=value` assignment, or a no-whitespace value of at least 32 characters that is not an env-var name fails `sdd team check`. Probabilities must sit in `[0, 1]` and sum to within `0.01` of `1`. An env-var name such as `SDD_ROLE_MODEL` stays legal. + +## Q3. Does routeRole grow a required dependency argument? + +No. The third argument is optional: `{ env, fetch, runCommand, now }`. Callers that omit it keep today's path when `models.yaml` is absent. When the file exists and the reason is `No rule matched.`, a missing `runCommand` or `fetch` resolves to human. Core does not read `process.env`. + +## Q4. Does Ollama assume a default host? + +No. `base_url_env` must name a variable, and that variable must be set in the env the CLI passes. There is no default URL and no socket in tests. + +## Q5. Does a decision write need a file lock? + +No. `decide` awaits each `recordDecisionMetric` before the next write. That matches `startPhase`, which can already drop a row on overlap. This change does not add a lock. + +## Q6. What does the route decider see? + +The state is a JSON string of `stage`, `artifact`, `skill`, and `id`, each null when absent. The question is exactly `No rule matched.` Options are the role names sorted with `localeCompare`, then `human` once. A kept role prints `role ` and `reason Model picked .` There is no rule line. Every human result of that path prints `Ask the human.` and `No rule matched.` + +## Q7. Where do the token counts come from? + +A command reply may include `input_tokens`, `cache_read_tokens`, `cache_write_tokens`, and `output_tokens`. Ollama uses `prompt_eval_count` and `eval_count`. When the reply JSON itself names token fields, those override the Ollama counters. A missing count stays null. A command model is null unless the reply names one. An Ollama model is the response model, or the configured name such as `qwen2.5:7b`. The choice id is not stored as a model. + +## Q8. Which Ollama request is sent? + +`POST {base}/api/chat`, with `stream: false` and a JSON object format that requires `pick` and `probs`. The user content is JSON of `{ state, question, options }`. There is no `/api/generate` and no default host. An invalid base URL is not called. A throw or a non-OK response is a call with null tokens and confidence 0. The function is `ollamaDecision`. + +## Q9. Which change receives the row? + +`recordDecisionMetric` uses the active change id and honors `metrics.enabled`, the same as `startPhase`. No active change, or a config that cannot be read, skips the write and still routes. + +## Q10. What do `sdd models show` and `sdd models which` print when the file is bad? + +A missing file exits 1 and prints `No models.yaml.` A parse error, a schema error, or a secret uses the `loadModels` message. `which` without a known work type prints `Work type is invalid.` and exits 1. + +## Q11. What is a failed command, and when does it escalate? + +A throw, a non-zero exit, or a non-JSON reply is confidence 0, recorded with null tokens. It escalates only when a minimum is set. A missing env name, a missing `runCommand`, or a missing `fetch` asks the human, does not escalate, and writes no row. + +## Q12. How is the fallback judged when it has no minimum of its own? + +The fallback uses its own `min_confidence` when it has one. Otherwise it uses the first model's minimum. A still-low or invalid fallback asks the human. A second failure does not call a third model. A direct call with no minimum keeps a valid probability of `0.01` and does not escalate an invalid reply. The sum check is `abs(sum - 1) > 0.01 + 1e-9`, so a true difference of `0.01` is kept and `1.02` is not. + +## Q13. Where does `--bot` go? + +`--bot` is an option on `sdd role route` only. `sdd role take` keeps its positional bot and stays model-free. + +## Open + +No design question remains for this change. These answers are a proposal. The person has not accepted the reading. diff --git a/changes/2026-10-23-issue-62-model-selection/clarify-intent.md b/changes/2026-10-23-issue-62-model-selection/clarify-intent.md new file mode 100644 index 0000000..3c1451f --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/clarify-intent.md @@ -0,0 +1,35 @@ +# Clarify intent + +> **Issue 62 model selection** · proposals from the research assistant, product manager, product owner, and architect. Not an acceptance. + +## Q1. Does draft default to the comment's fast model or the issue's strong model? + +The owner's comment example sets `draft: fast`. The issue says a strong model handles drafting. The reading follows the issue. `draft`, `design`, `code`, and `review` default to the strong reason model. The comment's `draft: fast` is not the default. + +## Q2. Do overrides live in models.yaml or team.yaml? + +The issue says `.sdd/models.yaml` or `team.yaml`. `TeamSchema` is strict, so a model map in `team.yaml` fails today's check. This change reads `.sdd/models.yaml` only. It does not add model keys to the team file and it does not weaken the schema. If both files set a slot, `models.yaml` is the only file that counts. + +## Q3. When may the decider run? + +Only inside `routeRole`, and only when `matchRoute` returns ask with the reason exactly `No rule matched.`, and only when `models.yaml` exists. A routing rule, one `owns` or `skills` hit, a stage owner, or `Two roles own this.` never calls a model. `matchRoute`, `takeRole`, and `chooseLead` stay model-free. A no-rule take still fails with ask and writes nothing. + +## Q4. What happens below the confidence minimum? + +Confidence is the probability of `pick`. A reply that fails the Zod check has confidence `0`. Below `min_confidence`, ask `fallback_reason` once in decide mode. If that reply is still low, or the command or fetch is missing, ask the human. No `min_confidence` means no escalation. The rules provider never reads confidence. A missing env, `runCommand`, or `fetch` is human. Core does not read `process.env`. The CLI passes env in. + +## Q5. Where do decision tokens go? + +Do not add `rules`, `command`, or `ollama` to the token-usage provider enum, and do not map them onto `grok`. Do not change the report header `source phase provider model requests input cache_read cache_write output usd inr`. Add a `decisions` array. When it is non-empty, the report prints a second section. Dollars stay computed by `report.ts` from `prices.yaml`. They are not stored in `metrics.json`. + +## Q6. Is the Ollama preset in this change? + +Yes, behind config, as the issue says it comes right after the rules and command providers. Tests inject `fetch`. There is no live server, no `ollama` CLI spawn, and no change to the `ollama run` launcher. Hosted System-One stays out. + +## Q7. Which directory holds the new modules? + +The owner's comment names `packages/core/src/team/`. That directory does not exist. The router is `packages/core/src/role/routing.ts`. New modules go under `packages/core/src/role/models/`. + +## Open + +No intent question remains for this change. The person has not accepted the reading. diff --git a/changes/2026-10-23-issue-62-model-selection/clarify-tasks.md b/changes/2026-10-23-issue-62-model-selection/clarify-tasks.md new file mode 100644 index 0000000..33ffb1a --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/clarify-tasks.md @@ -0,0 +1,47 @@ +# Clarify tasks + +> **Issue 62 model selection** · task check against the design. Not an acceptance. + +## Q1. Does any task need a behavior the design does not name? + +No. The task list follows the seam table. `matchRoute`, `takeRole`, and `chooseLead` stay model-free. `decide` has one production caller, `routeRole`. + +## Q2. Does the report task change the existing cost rows? + +No. The token header stays. Decision rows are a second section, and only when `decisions` is non-empty. Dollars are computed at report time. + +## Q3. Is the Ollama task a live server? + +No. The task is an HTTP preset tested with an injected `fetch`. `launch-agent.ts` stays as it is. + +## Q4. What is in the decide state? + +The dev lead asked. The state is JSON of `stage`, `artifact`, `skill`, and `id`. The question text is exactly `No rule matched.` + +## Q5. Which numbers are stored? + +Command replies may name the four token fields. Ollama stores `prompt_eval_count` and `eval_count` unless the reply JSON names token fields, which then win. Missing stays null. The stored model is the reply's model, or the Ollama model name. It is not the choice id. + +## Q6. Which HTTP call does the Ollama task make? + +`POST {base}/api/chat` with `stream: false` and a JSON schema that requires `pick` and `probs`. No default host. + +## Q7. When does the metrics task write? + +It uses the active change and `metrics.enabled`. If either is missing, the route still returns and the file is left alone. + +## Q8. What failure text do the command tasks print? + +Missing file: `No models.yaml.` Bad work type: `Work type is invalid.` A secret or a bad file uses the loader message and does not print the secret. + +## Q9. What does a broken command count as? + +A throw, a non-zero exit, or non-JSON is confidence 0. It escalates only when a minimum is set. A missing command or fetch does not escalate. + +## Q10. What minimum does the fallback task use? + +The fallback's own minimum, or the first model's minimum when the fallback has none. A still-low strong reply asks the human. The sum tolerance keeps a difference of `0.01` and rejects `1.02`, using a `1e-9` epsilon so a binary `0.01` is not rejected. + +## Open + +No task question remains for this change. The dev lead's questions are answered here as a proposal. The person has not accepted the reading. diff --git a/changes/2026-10-23-issue-62-model-selection/design.md b/changes/2026-10-23-issue-62-model-selection/design.md new file mode 100644 index 0000000..d9b5449 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/design.md @@ -0,0 +1,79 @@ +# Design + +> **Issue 62 model selection** · architect seams. A proposal. The person has not accepted the reading. + +## Approach + +Approach C from the brainstorm. Add model selection beside the router. `matchRoute` stays as it is. `routeRole` may call `decide` only after that function asks with the reason exactly `No rule matched.`, and only when `.sdd/models.yaml` loads. No file means today's ask text and no model spawn. + +## Config + +`.sdd/models.yaml` is the only config. `sdd team init` does not write it. `.sdd/models/` stays the embedding cache. + +```yaml +version: 1 +models: + rules: { kind: decide, provider: rules } + local: { kind: decide, provider: command, command_env: SDD_DECIDE_MODEL, min_confidence: 0.7 } + strong: { kind: reason, provider: command, command_env: SDD_ROLE_MODEL } + nearby: { kind: decide, provider: ollama, model: qwen2.5:7b, base_url_env: OLLAMA_HOST, min_confidence: 0.7 } +defaults: + by_work_type: + route: rules + classify: local + check: local + draft: strong + design: strong + code: strong + review: strong + fallback_reason: strong + fallback_decide: rules +overrides: + bots: { bob-dev: { code: strong } } + stages: { pr_review: { review: strong } } + roles: { architect: { review: strong } } +``` + +There is no `command:` field, so a shell string is not stored. `command_env` and `base_url_env` are names only. The schema is strict. A missing file returns null. Ids in defaults and overrides must exist, and the model's `kind` must match the slot: decide for route, classify, and check; reason for draft, design, code, and review. + +Precedence inside one work type is `overrides.bots[bot]`, then `overrides.stages[stage]`, then `overrides.roles[role]`, then `defaults.by_work_type`, then the fallback for that kind. A missing work-type key uses `fallback_decide` for route, classify, and check, and `fallback_reason` for the strong kinds. + +## Seams + +| Piece | Change | +|---|---| +| `packages/core/src/role/types.ts` | Add `WorkType` and `DecisionModel` beside `RoleModel` | +| `packages/core/src/role/models/schema.ts` | `ModelsConfigSchema`, `loadModels`, `checkModels`. Strict. Missing file returns null | +| `packages/core/src/role/models/select.ts` | `selectModel` is pure precedence and returns id, kind, provider, and why | +| `packages/core/src/role/models/decide.ts` | `decide`. Tests call it. The only production caller is `routeRole` | +| `packages/core/src/role/models/command.ts` | `modelFromEnv(env)` moves here. Reason stdin stays `{ roleId, prompt }`. Decide stdin is `{ mode: "decide", state, question, options }` | +| `packages/core/src/role/models/rules.ts` | `rulesDecision` returns `human` when that option is present. No role table | +| `packages/core/src/role/models/ollama.ts` | `ollamaDecision({ baseUrl, model, fetch })` only when `provider: ollama`. `POST {base}/api/chat`. No default URL | +| `packages/core/src/role/routing.ts` | `routeRole(root, input, deps?)`. After `matchRoute`, call `decide` only for ask plus `No rule matched.` when `loadModels` is non-null. Options are sorted role names plus `human` | +| `packages/core/src/role/index.ts` | Export the new functions and types | +| `packages/cli/src/role-cmd.ts` | `modelFromEnv(process.env)`. Pass `env` into `routeRole` | +| `packages/cli/src/models-cmd.ts` | `sdd models show` and `sdd models which`. Register `models` in `commands.ts` | +| `packages/core/src/role/team.ts` and `packages/cli/src/team-cmd.ts` | `checkTeam` reads the team, then `checkModels`. `sdd team check` | +| `packages/core/src/metrics.ts` | Optional `decisions` array and `recordDecisionMetric`. Do not edit `importSessionMetrics` | +| `packages/core/src/report.ts` | Second section when `decisions` is non-empty. Cost from `prices.yaml`. Header unchanged | + +`routeRole` deps are `{ env, fetch, runCommand, now }`. Existing callers omit them. Core does not read `process.env`. A missing env, `runCommand`, or `fetch` resolves to human. + +A decide reply is accepted only when `pick` is one of `options`, every option has a probability in `[0, 1]`, and the probabilities sum to within `0.01` of `1`. The comparison is `abs(sum - 1) > 0.01 + 1e-9`. Otherwise confidence is `0`. Below `min_confidence`, call `fallback_reason` once with mode decide. The fallback uses its own minimum, or the first model's minimum when it has none. Still low resolves to human. A missing command or fetch resolves to human and does not escalate. A throw, a non-zero exit, or non-JSON is confidence 0 and escalates only when a minimum is set. Rules never read confidence. A direct call with no `min_confidence` does not escalate. + +Secret check, proposal bounds: PEM (`-----BEGIN`), a `name=value` assignment, or a no-whitespace value of at least 32 characters that is not an env-var name. Env names such as `SDD_ROLE_MODEL` stay legal. Check that file's values only. Do not print the secret. + +`docs/examples/role-model.mjs` stays a reason example. It returns `{ text }` and fails a decide check on purpose. + +Host stubs in `agents.ts` are not lengthened. `takeRole` and `chooseLead` do not gain a model. Ordinary `runRole` kinds stay `reading`, `code`, `scenario`, and `release`. + +## Report + +Decision rows are separate from `requests`. Suggested row fields: `workType`, `model` (nullable), `kind` (`reason` or `decide`), `provider` (`rules`, `command`, or `ollama`), token counts, and `startedAt`. Rules rows are not written. Dollars are computed at report time. `importSessionMetrics` writes the parsed object back, so a round-trip must keep `decisions`. An old file with no `decisions` key still parses. + +## Risks + +1. The decision provider cannot go on `TokenUsageMetricSchema.provider`. A separate array avoids a false provider and keeps the cost rows. The issue never named the key `decisions`. +2. A rules table would be invented. `rulesDecision` returns human and does not rank roles. `decide` is not inside `matchRoute`. +3. `role-model.mjs` will not satisfy the decide check. One fallback, then human. Point `SDD_DECIDE_MODEL` at a decide command. Do not change the example. +4. `metrics.json` is read-modify-write and can drop a row, which is already true of `startPhase`. `decide` awaits each `recordDecisionMetric` before the next write and does not add a lock. The secret length `32` and the sum tolerance `0.01` are proposal bounds. diff --git a/changes/2026-10-23-issue-62-model-selection/feature.md b/changes/2026-10-23-issue-62-model-selection/feature.md new file mode 100644 index 0000000..54c5b4b --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/feature.md @@ -0,0 +1,62 @@ +# Feature + +> **Issue 62 model selection** · `2026-10-23-issue-62-model-selection` · 2026-10-23 + +## Reading + +SDD picks a model by the type of work. Routing, classify, and check use a small or local decider. Architecture, drafting, code, and reviews use a strong model. A person overrides that per bot, per stage, or per role in `.sdd/models.yaml`. With no `models.yaml`, a role run still uses `SDD_ROLE_MODEL`, and `sdd role route` still does not call a model. + +`decide({ state, question, options })` returns `{ pick, probs }` from a rules provider or a generic command provider. An Ollama HTTP preset is available behind that same config. The decider may answer only when the router would say `No rule matched.` A matched rule, one owner, a stage owner, or `Two roles own this.` does not call it. Below the confidence minimum the cheap pick is not kept. The strong model is asked once. If that reply is still low, or the command is missing, the human is asked. Every decision call records tokens so `sdd report` can show cost. + +Bots still cannot accept a spec, run `sdd next`, or merge. + +## Source + +Issue #62, as filed, plus the owner's implementation comment on that issue. Where the comment and the issue disagree, this reading follows the issue. The research assistant gathered the file paths. The product manager proposes this reading. The product owner and the architect reviewed the same brief. It is not an acceptance. The person accepts or corrects it. + +## Scope + +- Work types are exactly `route`, `classify`, `check`, `draft`, `design`, `code`, and `review`. Design is the architecture work named in the issue. +- Defaults: `route` uses rules. `classify` and `check` use a local command decider. `draft`, `design`, `code`, and `review` use the strong reason model (`SDD_ROLE_MODEL`). +- Precedence is bot, then stage, then role, then the work-type default, then the fallback. A missing work-type key uses `fallback_decide` for route, classify, and check, and `fallback_reason` for the strong kinds. Every id must exist and its kind must match the slot. +- The rules provider is deterministic. It returns `human` when that option is present. It does not invent a role and it does not read confidence. +- A command or Ollama reply is kept only when `pick` is one of the options, every option has a probability in `[0, 1]`, and the probabilities sum to within `0.01` of `1`. Otherwise confidence is `0`. Confidence is the probability of the pick. No `min_confidence` means no escalation. +- `sdd models show` prints each resolved row and why. `sdd models which` takes `--work-type`, and optional `--role`, `--stage`, and `--bot`. +- `sdd team check` rejects a literal secret in `models.yaml` and accepts an env-var name such as `SDD_ROLE_MODEL`. It does not print the secret on success. It checks that file only. +- Ollama is an HTTP preset, used only when a slot says `provider: ollama`. Tests inject `fetch`. There is no live server and no `ollama` CLI spawn. +- Decision rows live in a `decisions` array on `metrics.json`. The existing token table header stays. Dollars stay computed by the report from `prices.yaml`. + +## Product owner + +Proposed checks, not a recorded decision: + +- No `models.yaml`: a reason call uses `SDD_ROLE_MODEL`. Unset exits 1 and prints `No role model is configured.` `sdd role route` with no match prints `Ask the human.` and `No rule matched.` and does not spawn a model. +- A matching routing rule, one `owns` or `skills` role, or a stage owner still prints the role, the reason, and the rule. The decide command is not spawned. +- Two owners print `Ask the human.` and `Two roles own this.` The decide command is not spawned. +- When the no-rule path names a command decider, a Zod-valid pick at or above `min_confidence` prints that role. A probability below the minimum, or a reply that fails the check, calls the strong model once. If that reply is still low or invalid, the command prints `Ask the human.` and no role line. +- `sdd models which --work-type draft` (and design, code, review) resolves to strong. Route, classify, and check resolve to the cheap decider. `--bot` beats `--stage`, which beats `--role`, which beats the work-type default. `sdd models show` prints each row and a why. +- `sdd team check` exits 1 when `models.yaml` holds a literal secret, and exits 0 when the slot is an env-var name. A model block in `team.yaml` is not read. `TeamSchema` stays strict. +- An injected Ollama body that passes the schema can be selected. An invalid body follows the low-confidence path. The test does not spawn `ollama` and does not open a socket. +- A command or Ollama call records tokens, model, and work type. `sdd report` shows them in a new section. Existing cost rows stay, and `sdd cost` still matches those rows. A rules decision adds no model request row. +- An old `metrics.json` still parses. The file stores tokens, not dollars. + +## Non-goals + +- No hosted System-One adapter and no auto-tuning. +- No escalation, no rooms, and no evidence gate. Those are issues #63, #65, and #66. +- No change to the existing `ollama run` launcher. +- No model keys on `team.yaml`. The issue text allows that file. Today's schema rejects unknown keys, so this change does not weaken it. +- `sdd team init` does not write `models.yaml`. +- No edit to `docs/examples/role-model.mjs`. Point `SDD_DECIDE_MODEL` at a decide command. +- Ordinary role runs stay on `SDD_ROLE_MODEL`. They do not switch onto `selectModel`, and they still do not write `metrics.json`. +- No QA role on the software template. That is issue #71. +- The research assistant is a local dogfood role. This change does not ship it. +- Epic #58 stays open. This change does not start issues #63 through #66. +- A role still cannot accept a spec, record a QA result, or generate a release. + +## Success criteria + +- With no `models.yaml`, route output and reason calls match today. +- With a config file, only the exact reason `No rule matched.` can call a decider. +- Precedence, `sdd models show`, `sdd models which`, and the secret check behave as the product owner checks say. +- The report keeps its current header and adds a second section only when decision rows exist. diff --git a/changes/2026-10-23-issue-62-model-selection/local-test-plan.md b/changes/2026-10-23-issue-62-model-selection/local-test-plan.md new file mode 100644 index 0000000..6114387 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/local-test-plan.md @@ -0,0 +1,22 @@ +# Local test plan + +> **Issue 62 model selection** · `2026-10-23-issue-62-model-selection` + +Node on this machine is v20.19.3. The `EBADENGINE` warning for `engines.node >= 24` is expected. + +QA proposes these checks. They are not a recorded QA result. The person has not signed them. + +| # | Check | Expected | +|---|--------|----------| +| 1 | No `models.yaml`. Reason call with `SDD_ROLE_MODEL`. Route with no match. | Reason uses that command and writes no decision row. Unset prints `No role model is configured.` and exits 1. Route prints `Ask the human.` and `No rule matched.` and does not spawn. | +| 2 | Rule, one owns phrase, skill `implement one change pack`, stage owner, two owners. Route set to a local decider. | None of them spawn. Two owners print `Two roles own this.` A no-rule take writes no inbox. | +| 3 | Command pick at 0.8 with minimum 0.7. Pick at 0.69. Strong reply also 0.69. | The 0.8 pick prints `role developer` once. 0.69 calls strong once in decide mode. A still-low strong reply asks the human. | +| 4 | `selectModel` and `sdd models which` with bot, stage, and role. | Bot beats stage, stage beats role, role beats the work-type default. Draft, design, code, and review default to strong. | +| 5 | `sdd team check` and `loadModels`. | A missing models file is valid. A PEM, a `TOKEN=` assignment, or a 35-character no-whitespace value fails and is not printed. A 31-character env-shaped value passes. A secret in a role file passes. A `models:` key on `team.yaml` fails as `team.yaml is invalid`. A missing `command_env` fails the schema. | +| 6 | Injected Ollama `fetch` to `http://127.0.0.1:9/api/chat`, model `qwen2.5:7b`. | A valid body is kept and stores `prompt_eval_count`. An invalid body calls strong once. An unset host does not call `fetch`. | +| 7 | Report and metrics. | The request header and the `3.0000` / `252.00` row stay. A second section appears only when `decisions` is non-empty. An empty array prints no second section. Command model is null. Ollama model is `qwen2.5:7b`. The file has no dollar fields. An old file and a session import keep or omit `decisions` as written. Rules write no row. | +| 8 | Omitted `routeRole` deps, probability edges, a pick outside the options. | Omitted deps ignore `process.env`. Sum `1.01` is kept. Sum `1.02` and `1.5` escalate. An omitted option escalates. No minimum keeps a valid `0.01` and does not escalate an invalid reply. | +| 9 | `pnpm --filter @structured-vibe-coding/core exec tsc --noEmit`, CLI `tsc --noEmit` after the core build, and `biome check` on the edited sources. | Exit 0. | +| 10 | `sdd verify` | Workspace tests (`pnpm test`) exit 0. | + +Explicitly not in this plan: a live Ollama server, a hosted System-One adapter, escalation, rooms, the evidence gate, model keys on `team.yaml` as a feature, decide mode in `docs/examples/role-model.mjs`, and host-stub length. diff --git a/changes/2026-10-23-issue-62-model-selection/local-test-results.md b/changes/2026-10-23-issue-62-model-selection/local-test-results.md new file mode 100644 index 0000000..8c7ecb9 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/local-test-results.md @@ -0,0 +1,31 @@ +# Local test results + +> **Issue 62 model selection** · `2026-10-23-issue-62-model-selection` + +Node v20.19.3. `EBADENGINE` warnings for `engines.node >= 24` are expected. + +| # | Result | +|---|--------| +| 1 | PASS. `role-decide`, `role-models`, `report`, `metrics`, `router`, `role`, and `team`: 63 tests, 7 files, exit 0. After the extra assertions, `role-decide`, `role-models`, and `report`: 22 tests, 3 files, exit 0. No models file does not spawn. A rules decider asks the human and writes no decision row. | +| 2 | PASS. A rule, one owner, the skill `implement one change pack`, a stage owner, and two owners do not spawn. A confident 0.8 pick prints the role once. 0.69 calls the strong model once. A still-low strong reply asks the human. | +| 3 | PASS. Precedence is bot, then stage, then role, then the work-type default. PEM, `TOKEN=…`, and the 35-character token are rejected and not printed. A 31-character value passes. A secret in `developer.md` leaves `sdd team check` at exit 0. A `models:` key on `team.yaml` is `team.yaml is invalid`. A missing `command_env` fails the schema. | +| 4 | PASS. Injected Ollama `fetch` hits `http://127.0.0.1:9/api/chat` with the configured model `qwen2.5:7b`, stores the response model `qwen2.5:latest` and 11 input tokens, and does not fetch when the host is unset. An invalid body calls strong once. The decide test for that row passed after the review fix: 6 tests, exit 0. | +| 5 | PASS. The report keeps `source phase provider model requests` and the `3.0000` / `252.00` request row. `Decisions:` appears for a non-empty array and not for `[]`. The session import keeps the decision row. The metrics JSON has no `usd`. | +| 6 | PASS. Omitted deps ignore `process.env`. Sum `1.01` stays. Sum `1.02` and `1.5`, an omitted option, and a pick outside the options escalate. No minimum keeps `0.01` and does not call strong for an invalid reply. | +| 7 | PASS. CLI `models.integration` and `role.integration`: 5 tests, 2 files, exit 0. `sdd models which` prints the draft line. A secret is rejected and not printed. Route without a file asks the human. Route with a decide script prints `role developer`. `sdd role run` still prints `hello builder`. Unset `SDD_ROLE_MODEL` is covered by `role.test.ts` and `role.integration.test.ts`. | +| 8 | PASS. Core `tsc --noEmit` exit 0. CLI `tsc --noEmit` exit 0 after the core build. `biome check` on the edited sources exit 0. | +| 9 | PASS. `sdd verify` ran `pnpm test` and recorded exit 0. | + +QA proposes this result. The person has not recorded it. A role does not accept the spec. Epic #58 stays open. Issues #63, #65, and #66 are not started. The research assistant stays a local role and is not in this change. + +The stamp below is the verify run. It records the git HEAD before this change is committed. + +--- + +## Latest run (2026-10-23T12:58:07.782Z) + +> Overall: PASS +> Stage: local_verify + + +- **Workspace tests**: exit 0 (`pnpm test`) diff --git a/changes/2026-10-23-issue-62-model-selection/meta.yaml b/changes/2026-10-23-issue-62-model-selection/meta.yaml new file mode 100644 index 0000000..58f1325 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/meta.yaml @@ -0,0 +1,25 @@ +id: 2026-10-23-issue-62-model-selection +title: Issue 62 model selection +workflow: feature +created: 2026-10-23T12:17:16.362Z +updated: 2026-10-23T12:58:07.783Z +status: in_progress +stage: local_verify +epic: issue-58 +depends: [] +flags: {} +overrides: + skip_stages: [] + gates: {} + extra_stages: [] +skipped: [] +gates: {} +verify_results: + local_verify: + ok: true + status: pass + head_sha: b11dfd5377ed51868fba6f3d00df89959f20b011 + at: 2026-10-23T12:58:07.782Z + results: + - name: Workspace tests + exitCode: 0 diff --git a/changes/2026-10-23-issue-62-model-selection/metrics.json b/changes/2026-10-23-issue-62-model-selection/metrics.json new file mode 100644 index 0000000..79a80a0 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/metrics.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "phases": [ + { + "phase": "intent", + "startedAt": "2026-10-23T12:17:16.362Z", + "endedAt": "2026-10-23T12:33:04.363Z" + }, + { + "phase": "clarify_intent", + "startedAt": "2026-10-23T12:33:04.363Z", + "endedAt": "2026-10-23T12:33:05.327Z" + }, + { + "phase": "brainstorm", + "startedAt": "2026-10-23T12:33:05.327Z", + "endedAt": "2026-10-23T12:33:06.171Z" + }, + { + "phase": "design", + "startedAt": "2026-10-23T12:33:06.171Z", + "endedAt": "2026-10-23T12:33:06.967Z" + }, + { + "phase": "clarify_design", + "startedAt": "2026-10-23T12:33:06.967Z", + "endedAt": "2026-10-23T12:33:07.806Z" + }, + { + "phase": "tasks", + "startedAt": "2026-10-23T12:33:07.806Z", + "endedAt": "2026-10-23T12:37:34.177Z" + }, + { + "phase": "clarify_tasks", + "startedAt": "2026-10-23T12:37:34.177Z", + "endedAt": "2026-10-23T12:37:34.995Z" + }, + { + "phase": "implement", + "startedAt": "2026-10-23T12:37:34.995Z", + "endedAt": "2026-10-23T12:56:39.087Z" + }, + { + "phase": "local_verify", + "startedAt": "2026-10-23T12:56:39.087Z" + } + ], + "requests": [] +} diff --git a/changes/2026-10-23-issue-62-model-selection/tasks.md b/changes/2026-10-23-issue-62-model-selection/tasks.md new file mode 100644 index 0000000..1591e13 --- /dev/null +++ b/changes/2026-10-23-issue-62-model-selection/tasks.md @@ -0,0 +1,38 @@ +# Tasks + +> **Issue 62 model selection** · dev-lead proposal from the design seams. Not an acceptance. + +## Breakdown + +- [x] Add `WorkType` and `DecisionModel` beside `RoleModel`, and add a strict `models.yaml` schema with `loadModels` and `checkModels`. +- [x] Add `selectModel` with precedence bot, stage, role, work-type default, then fallback, and require each id to exist and match kind. +- [x] Move `modelFromEnv` into `role/models/command.ts` so it takes env. Add the rules provider and the Ollama HTTP preset. Do not change `launch-agent.ts` or `docs/examples/role-model.mjs`. +- [x] Add `decide`. Keep a reply only when the pick is an option, every option has a probability, and the probabilities sum to within 0.01 of 1. Below `min_confidence`, call `fallback_reason` once. A missing command or fetch asks the human. Rules return human and write no metric. +- [x] Call `decide` from `routeRole` only when the reason is exactly `No rule matched.` and `models.yaml` loads. Keep `matchRoute`, `takeRole`, and `chooseLead` model-free. Pass optional `{ env, fetch, runCommand, now }`. +- [x] Add `sdd models show`, `sdd models which`, and `sdd team check`. Register `models` in `commands.ts`. Reject a literal secret without printing it. +- [x] Add an optional `decisions` array and `recordDecisionMetric`. Print a second report section when it is non-empty. Leave the token header and the provider enum unchanged. +- [x] Lock the behavior with Vitest: no file matches today, two owners and a rule do not spawn, precedence, confidence, injected Ollama, secrets, old metrics, and the new report section. +- [x] Update the CLI reference sentence that says `sdd role route` never calls a model. + +## Traceability + +| Task | Design ref | +|---|---| +| Schema and types | Design: config and seams for schema.ts and types.ts | +| Precedence | Design: bot, stage, role, work type, fallback | +| Providers | Design: rules, command, ollama. Reason stdin stays `{ roleId, prompt }` | +| decide | Design: confidence, one fallback, human when the command or fetch is missing | +| routeRole | Design: only `No rule matched.` and only when the file exists | +| Commands | Design: `sdd models show`, `sdd models which`, `sdd team check` | +| Report | Design: `decisions` array, header unchanged, dollars from prices.yaml | +| Tests | Product owner checks in feature.md | +| Docs | Design: no file still does not call a model | + +## Out of this change + +- Escalation, rooms, and the evidence gate. Issues #63, #65, and #66. +- Hosted System-One and auto-tuning. +- Model keys on `team.yaml`. +- A live Ollama server and any edit to the `ollama run` launcher. +- Shipping the research assistant. +- Host stub text. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3a4b184..ea5213f 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -92,6 +92,9 @@ Commit `.sdd/config.yaml`, workflows, templates, `protocol.md`, `memory/`, and c | `sdd role` | Create, list, and run a product role. `sdd role create --template software/developer` copies a shipped role | One job and one stop | | `sdd team init --template software` | Write `.sdd/team.yaml`, that template's roles, and `routing.yaml` when the template has one. Also `solo` and `research`. Does not overwrite | Model a team | | `sdd team show` | Print bots, human approvers, the rooms commit switch, and the branch pattern | See the team | +| `sdd team check` | Read `team.yaml`, then reject a literal secret in `.sdd/models.yaml`. A missing models file is allowed. An env-var name is allowed | Before a team runs | +| `sdd models show` | Print each model and which work type uses it | See the pick | +| `sdd models which --work-type draft` | Print the model for one work type. `--bot` beats `--stage`, which beats `--role` | Explain one slot | | `sdd graph --epic ` | Print which stories can run together and which must wait | Epic planning | | `sdd branch ` | Create that story branch from `main` | Start one story | | `sdd pr` | Push the change branch and open a pull request | Story is ready | @@ -163,13 +166,13 @@ sdd role create builder --template software/developer `software` ships product-manager, architect, product-owner, dev-lead, and developer. `solo` ships one developer. `research` ships one researcher. `developer` alone is ambiguous. Pass `software/developer` or `solo/developer`. Init writes nothing when `.sdd/team.yaml`, `.sdd/routing.yaml` (when the template ships one), or one of the template's role files already exists. A human approver whose name matches `git config user.name` is marked `(you)`. The rooms commit switch is stored and stays off in the shipped templates. The branch pattern includes `{bot}`. -`sdd role route` prints the role, the reason, and the rule for one item. With no flags it uses the active change. Rules in `.sdd/routing.yaml` are tried first. Then one role whose `owns` list names the artifact, or whose `skills` list names the skill. Then the active workflow stage's `owner`, when that role file exists. No match prints `Ask the human.` This command does not call a model. `sdd role queue` prints each bot inbox at `.sdd/agents//inbox.jsonl`. `sdd role take` writes one active item onto a bot of that role that is under its `wip` limit and under `role_wip` when the team sets one. A bot at either limit is not given another item. A handoff appends the same typed note to both inboxes. Those notes do not count toward the limit. +`sdd role route` prints the role, the reason, and the rule for one item. With no flags it uses the active change. Rules in `.sdd/routing.yaml` are tried first. Then one role whose `owns` list names the artifact, or whose `skills` list names the skill. Then the active workflow stage's `owner`, when that role file exists. No match prints `Ask the human.` With no `.sdd/models.yaml`, this command does not call a model. When that file exists, only the no-rule ask may call a decider. A matched rule, one owner, a stage owner, or two owners do not. `sdd team init` does not write `models.yaml`. Model overrides live in that file, not in `team.yaml`. `sdd role queue` prints each bot inbox at `.sdd/agents//inbox.jsonl`. `sdd role take` writes one active item onto a bot of that role that is under its `wip` limit and under `role_wip` when the team sets one. A bot at either limit is not given another item. A handoff appends the same typed note to both inboxes. Those notes do not count toward the limit. A role file at `.sdd/roles/.md` is still not a host stub. The four host stubs (`sdd`, `sdd-planner`, `sdd-implementer`, `sdd-reviewer`) each state what they own, what they never do, what they check, and when they ask. `sdd role run` stops in plain language before it writes a file. Allow once writes only the named file. Always allow remembers that file and those inputs for that role. Connectors and routines are not in this slice. `sdd role messaging off` stops a handoff before any model call. `sdd role handoff --to --change --section --task "..."` writes the same handoff record on both conversations and calls only the receiver. `sdd role group --change ` lists accepted owners. `sdd role network` prints one service, a connection with the lead absent, or asks the lead for a plan of depth at most 2. Nobody writes until `sdd role network accept`. `sdd role network run` runs a node only after its prerequisites, and the parent stays open until every node has a result. -A local adapter shows the three passes. It reads `{"roleId","prompt"}` and writes one JSON object. It does not call a network. +A local adapter shows the three passes. It reads `{"roleId","prompt"}` and writes one JSON object. It does not call a network. A decide command is separate. Set `SDD_DECIDE_MODEL` to a command that reads `{"mode":"decide","state","question","options"}` and writes `{"pick","probs"}`. The example adapter stays a reason command. ```bash SDD_ROLE_MODEL="node docs/examples/role-model.mjs" sdd role run builder --kind code --change --section
--task "..." diff --git a/packages/cli/src/__tests__/models.integration.test.ts b/packages/cli/src/__tests__/models.integration.test.ts new file mode 100644 index 0000000..6b9291c --- /dev/null +++ b/packages/cli/src/__tests__/models.integration.test.ts @@ -0,0 +1,152 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; + +const temps: string[] = []; +const here = dirname(fileURLToPath(import.meta.url)); +const sddBin = join(here, "../../dist/index.js"); +const token = "abcdEFGH-0123456789abcdef0123456789"; + +function runSdd( + cwd: string, + args: string[], + env: Record = {}, +): { status: number | null; stdout: string; stderr: string } { + const result = spawnSync(process.execPath, [sddBin, ...args], { + cwd, + encoding: "utf8", + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + USER: process.env.USER, + LANG: process.env.LANG, + CI: "1", + ...env, + }, + }); + return { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; +} + +beforeAll(async () => { + await import("node:fs/promises").then((fs) => fs.readFile(sddBin, "utf8")); +}); + +afterEach(async () => { + while (temps.length) { + const dir = temps.pop(); + if (dir) await rm(dir, { recursive: true, force: true }); + } +}); + +describe("sdd models and team check", () => { + it("explains a work type and rejects a secret without printing it", async () => { + const root = await mkdtemp(join(tmpdir(), "sdd-models-cli-")); + temps.push(root); + expect(runSdd(root, ["init", "--here", "--no-agent"]).status).toBe(0); + expect(runSdd(root, ["team", "init", "--template", "software"]).status).toBe(0); + const missing = runSdd(root, ["models", "which", "--work-type", "draft"]); + expect(missing.status).toBe(1); + expect(missing.stdout.trim()).toBe("No models.yaml."); + + const yaml = [ + "version: 1", + "models:", + " rules: { kind: decide, provider: rules }", + " strong: { kind: reason, provider: command, command_env: SDD_ROLE_MODEL }", + "defaults:", + " by_work_type:", + " route: rules", + " draft: strong", + " fallback_reason: strong", + " fallback_decide: rules", + "", + ].join("\n"); + await writeFile(join(root, ".sdd/models.yaml"), yaml, "utf8"); + const which = runSdd(root, ["models", "which", "--work-type", "draft"]); + expect(which.status).toBe(0); + expect(which.stdout).toContain("draft strong reason command work-type default"); + const checked = runSdd(root, ["team", "check"]); + expect(checked.status).toBe(0); + expect(checked.stdout.trim()).toBe("Team config is valid."); + + await writeFile(join(root, ".sdd/models.yaml"), yaml.replace("SDD_ROLE_MODEL", token), "utf8"); + const secret = runSdd(root, ["team", "check"]); + expect(secret.status).toBe(1); + expect(secret.stdout).toContain("models.yaml has a secret value."); + expect(secret.stdout).not.toContain(token); + expect(secret.stderr).not.toContain(token); + }); + + it("routes with a decide command and leaves a reason call on SDD_ROLE_MODEL", async () => { + const root = await mkdtemp(join(tmpdir(), "sdd-models-route-")); + temps.push(root); + expect(runSdd(root, ["init", "--here", "--no-agent"]).status).toBe(0); + expect(runSdd(root, ["team", "init", "--template", "software"]).status).toBe(0); + const script = join(root, "fake-model.js"); + await writeFile( + script, + `let raw = ""; +process.stdin.setEncoding("utf8"); +process.stdin.on("data", (chunk) => { raw += chunk; }); +process.stdin.on("end", () => { + const input = JSON.parse(raw); + if (input.mode === "decide") { + const probs = {}; + for (const option of input.options) probs[option] = 0; + probs.developer = 0.8; + probs.human = 0.2; + process.stdout.write(JSON.stringify({ pick: "developer", probs, input_tokens: 3, output_tokens: 1 })); + return; + } + process.stdout.write(JSON.stringify({ text: "hello " + input.roleId })); +}); +`, + "utf8", + ); + const command = `${process.execPath} ${JSON.stringify(script)}`; + const asked = runSdd(root, ["role", "route", "--artifact", "no-such-owns-phrase"], { + SDD_DECIDE_MODEL: command, + SDD_ROLE_MODEL: command, + }); + expect(asked.status).toBe(0); + expect(asked.stdout).toContain("Ask the human."); + expect(asked.stdout).toContain("No rule matched."); + + await writeFile( + join(root, ".sdd/models.yaml"), + [ + "version: 1", + "models:", + " local: { kind: decide, provider: command, command_env: SDD_DECIDE_MODEL, min_confidence: 0.7 }", + " strong: { kind: reason, provider: command, command_env: SDD_ROLE_MODEL }", + "defaults:", + " by_work_type: { route: local }", + " fallback_reason: strong", + " fallback_decide: local", + "", + ].join("\n"), + "utf8", + ); + const routed = runSdd(root, ["role", "route", "--artifact", "no-such-owns-phrase"], { + SDD_DECIDE_MODEL: command, + SDD_ROLE_MODEL: command, + }); + expect(routed.status).toBe(0); + expect(routed.stdout).toContain("role developer"); + expect(routed.stdout).not.toContain("Ask the human."); + + expect( + runSdd(root, ["role", "create", "--name", "builder", "--label", "Builder", "--job", "Build"]) + .status, + ).toBe(0); + const ran = runSdd(root, ["role", "run", "builder", "--task", "say hi"], { + SDD_ROLE_MODEL: command, + }); + expect(ran.status).toBe(0); + expect(ran.stdout.trim()).toBe("hello builder"); + }); +}); diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 88266e1..5179288 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -48,6 +48,7 @@ import { runSpeckitStyleInit, selectIntegration } from "./init-flow.js"; import { launchConfiguredAgent, reportAgentLaunch } from "./launch-agent.js"; import { mcpCmd } from "./mcp-cmd.js"; import { metricsCmd } from "./metrics-cmd.js"; +import { modelsCmd } from "./models-cmd.js"; import { isNodeVersionSupported } from "./node-support.js"; import { projectRoot, withInitialized, withProject } from "./project.js"; import { costCmd, reportCmd } from "./report-cmd.js"; @@ -1373,6 +1374,7 @@ export const commands = { agents, role: roleCmd, team: teamCmd, + models: modelsCmd, graph: graphCmd, branch: branchCmd, pr: prCmd, diff --git a/packages/cli/src/models-cmd.ts b/packages/cli/src/models-cmd.ts new file mode 100644 index 0000000..0f86aa4 --- /dev/null +++ b/packages/cli/src/models-cmd.ts @@ -0,0 +1,74 @@ +import { + type WorkType, + explainModel, + formatModels, + isWorkType, + loadModels, +} from "@structured-vibe-coding/core"; +import { defineCommand } from "citty"; +import { withProject } from "./project.js"; + +function textArg(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +async function loadedConfig(root: string) { + const loaded = await loadModels(root); + if (loaded === null) { + process.stdout.write("No models.yaml.\n"); + process.exit(1); + } + if ("error" in loaded) { + process.stdout.write(`${loaded.error}\n`); + process.exit(1); + } + return loaded.config; +} + +const show = defineCommand({ + meta: { name: "show", description: "Print each model and the work-type defaults" }, + async run() { + await withProject(async ({ root }) => { + const config = await loadedConfig(root); + process.stdout.write(`${formatModels(config)}\n`); + }); + }, +}); + +const which = defineCommand({ + meta: { name: "which", description: "Print the model for one work type" }, + args: { + "work-type": { + type: "string", + description: "route, classify, check, draft, design, code, or review", + required: true, + }, + role: { type: "string", description: "Role override" }, + stage: { type: "string", description: "Stage override" }, + bot: { type: "string", description: "Bot override" }, + }, + async run({ args }) { + const workType = textArg(args["work-type"]); + if (!workType || !isWorkType(workType)) { + process.stdout.write("Work type is invalid.\n"); + process.exit(1); + } + await withProject(async ({ root }) => { + const config = await loadedConfig(root); + const line = explainModel(config, workType as WorkType, { + role: textArg(args.role), + stage: textArg(args.stage), + bot: textArg(args.bot), + }); + process.stdout.write(`${line}\n`); + }); + }, +}); + +export const modelsCmd = defineCommand({ + meta: { + name: "models", + description: "Show which model a kind of work uses", + }, + subCommands: { show, which }, +}); diff --git a/packages/cli/src/role-cmd.ts b/packages/cli/src/role-cmd.ts index 556298b..d5a0e15 100644 --- a/packages/cli/src/role-cmd.ts +++ b/packages/cli/src/role-cmd.ts @@ -1,6 +1,4 @@ -import { spawn } from "node:child_process"; import { - type RoleModel, acceptNetwork, allowRole, createRole, @@ -10,6 +8,7 @@ import { editRole, handoffRole, hideRole, + modelFromEnv, pinRole, planNetwork, queueRole, @@ -20,6 +19,7 @@ import { setRoleMessaging, showRoleGroup, showRoles, + spawnModelCommand, takeRole, } from "@structured-vibe-coding/core"; import { defineCommand } from "citty"; @@ -43,42 +43,6 @@ function boolArg(value: unknown): boolean | undefined | "bad" { return "bad"; } -function modelFromEnv(): RoleModel | undefined { - const command = process.env.SDD_ROLE_MODEL?.trim(); - if (!command) return undefined; - return { - async complete(input) { - const stdout = await new Promise((resolve, reject) => { - const child = spawn(command, { shell: true, stdio: ["pipe", "pipe", "pipe"] }); - const out: Buffer[] = []; - child.stdout?.on("data", (chunk: Buffer) => out.push(chunk)); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0) { - reject(new Error(`role model exited ${code ?? "unknown"}`)); - return; - } - const text = Buffer.concat(out).toString("utf8").trim(); - try { - const parsed = JSON.parse(text) as unknown; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - reject(new Error("role model stdout is not a JSON object")); - return; - } - } catch { - reject(new Error("role model stdout is not JSON")); - return; - } - resolve(text); - }); - child.stdin?.write(`${JSON.stringify({ roleId: input.roleId, prompt: input.prompt })}\n`); - child.stdin?.end(); - }); - return stdout; - }, - }; -} - const create = defineCommand({ meta: { name: "create", description: "Write a new role file" }, args: { @@ -209,7 +173,7 @@ const run = defineCommand({ section: textArg(args.section), change: textArg(args.change), stop: args.stop === true, - model: args.stop === true ? undefined : modelFromEnv(), + model: args.stop === true ? undefined : modelFromEnv(process.env), }), ); }); @@ -257,7 +221,7 @@ const handoff = defineCommand({ task: textArg(args.task), change: textArg(args.change), section: textArg(args.section), - model: modelFromEnv(), + model: modelFromEnv(process.env), }), ); }); @@ -305,7 +269,7 @@ const networkRun = defineCommand({ await runNetwork(root, { change: textArg(args.change), section: textArg(args.section), - model: modelFromEnv(), + model: modelFromEnv(process.env), }), ); }); @@ -321,7 +285,7 @@ const networkRebuild = defineCommand({ await rebuildNetwork(root, { change: textArg(args.change), section: textArg(args.section), - model: modelFromEnv(), + model: modelFromEnv(process.env), }), ); }); @@ -341,7 +305,7 @@ const network = defineCommand({ change: textArg(args.change), section: textArg(args.section), task: textArg(args.task), - model: modelFromEnv(), + model: modelFromEnv(process.env), }), ); }); @@ -377,16 +341,29 @@ function routeArgs() { const route = defineCommand({ meta: { name: "route", description: "Route one item to the role that owns it" }, - args: routeArgs(), + args: { + ...routeArgs(), + bot: { type: "string" as const, description: "Bot name for a model override" }, + }, async run({ args }) { await withProject(async ({ root }) => { emit( - await routeRole(root, { - stage: textArg(args.stage), - artifact: textArg(args.artifact), - skill: textArg(args.skill), - id: textArg(args.id), - }), + await routeRole( + root, + { + stage: textArg(args.stage), + artifact: textArg(args.artifact), + skill: textArg(args.skill), + id: textArg(args.id), + bot: textArg(args.bot), + }, + { + env: process.env, + fetch: globalThis.fetch, + runCommand: spawnModelCommand, + now: () => new Date(), + }, + ), ); }); }, diff --git a/packages/cli/src/team-cmd.ts b/packages/cli/src/team-cmd.ts index 354646b..369d0ff 100644 --- a/packages/cli/src/team-cmd.ts +++ b/packages/cli/src/team-cmd.ts @@ -1,4 +1,4 @@ -import { gitUserName, initTeam, showTeam } from "@structured-vibe-coding/core"; +import { checkTeam, gitUserName, initTeam, showTeam } from "@structured-vibe-coding/core"; import { defineCommand } from "citty"; import { withProject } from "./project.js"; @@ -25,6 +25,15 @@ const init = defineCommand({ }, }); +const check = defineCommand({ + meta: { name: "check", description: "Check team.yaml and reject a secret in models.yaml" }, + async run() { + await withProject(async ({ root }) => { + emit(await checkTeam(root)); + }); + }, +}); + const show = defineCommand({ meta: { name: "show", description: "Print the team config" }, async run() { @@ -39,5 +48,5 @@ export const teamCmd = defineCommand({ name: "team", description: "Install a role team from a template and show it", }, - subCommands: { init, show }, + subCommands: { init, show, check }, }); diff --git a/packages/core/src/__tests__/metrics.test.ts b/packages/core/src/__tests__/metrics.test.ts index 6ded9cc..34c4ced 100644 --- a/packages/core/src/__tests__/metrics.test.ts +++ b/packages/core/src/__tests__/metrics.test.ts @@ -12,6 +12,7 @@ import { loadConfig, pathExists, readChangeMetrics, + recordDecisionMetric, skipStage, startPhase, switchWorkflow, @@ -529,4 +530,56 @@ stages: const metrics = await readChangeMetrics(root, config, change.id); expect(metrics.requests).toEqual([]); }); + + it("keeps decision rows when a session import writes the file back", async () => { + const { root, config, change } = await setup(); + const startedAt = (await readChangeMetrics(root, config, change.id)).phases[0]!.startedAt; + await recordDecisionMetric(root, config, change.id, { + workType: "route", + kind: "decide", + provider: "command", + model: null, + startedAt, + inputTokens: 4, + cacheReadTokens: null, + cacheWriteTokens: null, + outputTokens: 2, + }); + const logPath = join(root, "private-session.jsonl"); + await writeFile( + logPath, + `${JSON.stringify({ + type: "assistant", + timestamp: startedAt, + requestId: "claude-request-keep", + message: { + id: "claude-message-keep", + usage: { + input_tokens: 100, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + output_tokens: 30, + }, + content: [{ type: "text", text: "PRIVATE PROMPT CONTENT" }], + }, + })}\n`, + ); + + const imported = await importSessionMetrics(root, config, change.id, logPath); + const metrics = await readChangeMetrics(root, config, change.id); + const serialized = await readFile(join(change.path, "metrics.json"), "utf8"); + + expect(imported.imported).toBe(1); + expect(metrics.decisions).toEqual([ + expect.objectContaining({ + workType: "route", + provider: "command", + model: null, + inputTokens: 4, + }), + ]); + expect(metrics.requests).toHaveLength(1); + expect(serialized).not.toContain("usd"); + expect(serialized).not.toContain("PRIVATE PROMPT CONTENT"); + }); }); diff --git a/packages/core/src/__tests__/report.test.ts b/packages/core/src/__tests__/report.test.ts index ce7d2f3..19190f2 100644 --- a/packages/core/src/__tests__/report.test.ts +++ b/packages/core/src/__tests__/report.test.ts @@ -344,6 +344,60 @@ describe("token cost report", () => { expect(missing).toContain("No estimate: .sdd/handoff.md is missing."); expect(missing).not.toContain("estimated"); }); + + it("adds a decision section without changing the request rows", () => { + const table = prices(); + const text = render( + { + ...metrics( + [phase("intent", "2026-01-01T00:00:00.000Z", "2026-01-01T00:10:00.000Z")], + [request({ provider: "codex", requestId: "codex-1", inputTokens: 1_000_000 })], + ), + decisions: [ + { + workType: "route", + kind: "decide", + provider: "command", + model: null, + startedAt: "2026-01-01T00:00:00.000Z", + inputTokens: 1_000_000, + cacheReadTokens: null, + cacheWriteTokens: null, + outputTokens: null, + }, + ], + }, + { prices: table, handoffCodePoints: 99 }, + ); + expect(text).toContain("source phase provider model requests"); + expect(text).toContain( + "measured intent codex n/a 1 1000000 n/a n/a n/a 3.0000 252.00", + ); + expect(text).toContain("measured total all all 1 1000000 n/a n/a n/a 3.0000 252.00"); + expect(text).toContain("Decisions:"); + expect(text).toContain( + "work_type kind provider model calls input cache_read cache_write output usd inr", + ); + expect(text).toContain( + "route decide command n/a 1 1000000 n/a n/a n/a 3.0000 252.00", + ); + expect(text).not.toContain("measured intent command"); + + const empty = render( + { + ...metrics( + [phase("intent", "2026-01-01T00:00:00.000Z", "2026-01-01T00:10:00.000Z")], + [request({ provider: "codex", requestId: "codex-1", inputTokens: 1_000_000 })], + ), + decisions: [], + }, + { prices: table, handoffCodePoints: 99 }, + ); + expect(empty).toContain( + "measured intent codex n/a 1 1000000 n/a n/a n/a 3.0000 252.00", + ); + expect(empty).not.toContain("Decisions:"); + }); }); describe("report files", () => { diff --git a/packages/core/src/__tests__/role-decide.test.ts b/packages/core/src/__tests__/role-decide.test.ts new file mode 100644 index 0000000..0fefb95 --- /dev/null +++ b/packages/core/src/__tests__/role-decide.test.ts @@ -0,0 +1,402 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "pathe"; +import { afterEach, describe, expect, it } from "vitest"; +import { pathExists } from "../fs.js"; +import { + createChange, + initProject, + loadConfig, + readChangeMetrics, + setActiveChange, +} from "../index.js"; +import { createRole, initTeam } from "../role/index.js"; +import { routeRole, takeRole } from "../role/routing.js"; + +const temps: string[] = []; +const options = ["architect", "dev-lead", "developer", "product-manager", "product-owner", "human"]; + +afterEach(async () => { + while (temps.length) { + const dir = temps.pop(); + if (dir) await rm(dir, { recursive: true, force: true }); + } +}); + +function reply(pick: string, probability: number, sum = 1, tokens = true): string { + const probs: Record = {}; + for (const option of options) probs[option] = 0; + const other = options.find((option) => option !== pick) ?? pick; + probs[pick] = probability; + probs[other] = (probs[other] ?? 0) + (sum - probability); + const body: Record = { pick, probs }; + if (tokens) { + body.input_tokens = 4; + body.output_tokens = 2; + } + return JSON.stringify(body); +} + +function modelsYaml(route = "rules", min = "0.7"): string { + return [ + "version: 1", + "models:", + ` rules: { kind: decide, provider: rules }`, + ` local: { kind: decide, provider: command, command_env: SDD_DECIDE_MODEL, min_confidence: ${min} }`, + " strong: { kind: reason, provider: command, command_env: SDD_ROLE_MODEL }", + " nearby: { kind: decide, provider: ollama, model: qwen2.5:7b, base_url_env: OLLAMA_HOST, min_confidence: 0.7 }", + "defaults:", + " by_work_type:", + ` route: ${route}`, + " classify: local", + " check: local", + " draft: strong", + " design: strong", + " code: strong", + " review: strong", + " fallback_reason: strong", + " fallback_decide: rules", + ].join("\n"); +} + +async function project(): Promise { + const root = await mkdtemp(join(tmpdir(), "sdd-decide-")); + temps.push(root); + await initProject({ projectRoot: root, agents: false }); + expect((await initTeam(root, "software", "Ada Lovelace")).exitCode).toBe(0); + const config = await loadConfig(root); + const change = await createChange({ + projectRoot: root, + config, + title: "Pick a model", + workflowName: "feature", + }); + await setActiveChange(root, config, change.id); + return root; +} + +describe("route decides only when no rule matched", () => { + it("keeps today's ask when models.yaml is absent", async () => { + const root = await project(); + const calls: string[] = []; + const routed = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async (_command, stdin) => { + calls.push(stdin); + return reply("developer", 0.9); + }, + }, + ); + expect(routed).toMatchObject({ + exitCode: 0, + stdout: "Ask the human.\nNo rule matched.", + }); + expect(calls).toEqual([]); + const config = await loadConfig(root); + const active = (await readFile(join(root, "changes/.active"), "utf8")).trim(); + const metrics = await readChangeMetrics(root, config, active); + expect(metrics.decisions).toBeUndefined(); + + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("rules"), "utf8"); + const ruled = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide" }, + runCommand: async (_command, stdin) => { + calls.push(stdin); + return reply("developer", 0.9); + }, + }, + ); + expect(ruled.stdout).toBe("Ask the human.\nNo rule matched."); + expect(calls).toEqual([]); + const afterRules = await readChangeMetrics(root, config, active); + expect(afterRules.decisions ?? []).toEqual([]); + }); + + it("does not call a model for a rule, one owner, a skill, a stage owner, or two owners", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("local"), "utf8"); + const calls: string[] = []; + const runCommand = async (_command: string, stdin: string) => { + calls.push(stdin); + return reply("developer", 0.9); + }; + const deps = { env: { SDD_DECIDE_MODEL: "decide" }, runCommand }; + expect((await routeRole(root, { stage: "implement" }, deps)).stdout).toContain( + "rule implement", + ); + expect( + (await routeRole(root, { artifact: "the change pack for one service" }, deps)).stdout, + ).toContain("rule owns"); + expect((await routeRole(root, { skill: "implement one change pack" }, deps)).stdout).toContain( + "rule skill", + ); + await rm(join(root, ".sdd/routing.yaml")); + const config = await loadConfig(root); + const change = await createChange({ + projectRoot: root, + config, + title: "Stage owner", + workflowName: "team-software", + }); + await setActiveChange(root, config, change.id); + expect((await routeRole(root, {}, deps)).stdout).toContain("rule stage-owner"); + await createRole(root, { + name: "other-dev", + label: "Other", + job: "Build the other pack", + owns: ["the change pack for one service"], + }); + const shared = await routeRole(root, { artifact: "the change pack for one service" }, deps); + expect(shared.stdout).toBe("Ask the human.\nTwo roles own this."); + expect(calls).toEqual([]); + const taken = await takeRole(root, { artifact: "no-such-owns-phrase" }); + expect(taken.exitCode).toBe(1); + expect(taken.stdout).toContain("Ask the human."); + expect(await pathExists(join(root, ".sdd/agents/developer/inbox.jsonl"))).toBe(false); + }); + + it("keeps a confident command pick, escalates once, and records tokens", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("local"), "utf8"); + const calls: { command: string; stdin: string }[] = []; + const runCommand = async (command: string, stdin: string) => { + calls.push({ command, stdin }); + if (command === "decide") return reply("developer", 0.8); + return reply("architect", 0.9); + }; + const picked = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand, + now: () => new Date("2026-01-01T00:00:00.000Z"), + }, + ); + expect(picked.exitCode).toBe(0); + expect(picked.stdout).toBe("role developer\nreason Model local picked developer."); + expect(picked.stdout).not.toContain("rule "); + expect(calls).toHaveLength(1); + const body = JSON.parse(calls[0]?.stdin ?? "") as { mode: string; options: string[] }; + expect(body.mode).toBe("decide"); + expect(body.options).toEqual(options); + + calls.length = 0; + const low = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async (command, stdin) => { + calls.push({ command, stdin }); + return command === "decide" ? reply("developer", 0.69) : reply("architect", 0.9); + }, + now: () => new Date("2026-01-01T00:00:00.000Z"), + }, + ); + expect(low.stdout).toBe("role architect\nreason Model strong picked architect."); + expect(calls.map((call) => call.command)).toEqual(["decide", "strong"]); + + const still = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async () => reply("developer", 0.69), + }, + ); + expect(still.stdout).toBe("Ask the human.\nNo rule matched."); + expect(still.stdout).not.toContain("role "); + + const config = await loadConfig(root); + const active = (await readFile(join(root, "changes/.active"), "utf8")).trim(); + const metrics = await readChangeMetrics(root, config, active); + expect(metrics.decisions?.some((row) => row.provider === "command" && row.model === null)).toBe( + true, + ); + expect(metrics.decisions?.every((row) => row.inputTokens === 4)).toBe(true); + expect(JSON.stringify(metrics)).not.toContain("usd"); + }); + + it("does not read process.env when deps are omitted and does not escalate without a minimum", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("local"), "utf8"); + const previous = process.env.SDD_DECIDE_MODEL; + process.env.SDD_DECIDE_MODEL = "definitely-not-a-command"; + try { + const routed = await routeRole(root, { artifact: "no-such-owns-phrase" }); + expect(routed.stdout).toBe("Ask the human.\nNo rule matched."); + } finally { + if (previous === undefined) Reflect.deleteProperty(process.env, "SDD_DECIDE_MODEL"); + else process.env.SDD_DECIDE_MODEL = previous; + } + + await writeFile( + join(root, ".sdd/models.yaml"), + modelsYaml("local", "null").replace("min_confidence: null", "min_confidence: 0.7"), + "utf8", + ); + const bare = modelsYaml("local").replace(" min_confidence: 0.7", ""); + await writeFile(join(root, ".sdd/models.yaml"), bare, "utf8"); + const calls: string[] = []; + const kept = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async (command, stdin) => { + calls.push(command); + return stdin.includes("nope") ? reply("developer", 0.01) : reply("developer", 0.01); + }, + }, + ); + expect(kept.stdout).toContain("role developer"); + expect(calls).toEqual(["decide"]); + + const invalid = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async (command) => { + calls.push(command); + return JSON.stringify({ pick: "not-a-role", probs: { "not-a-role": 1 } }); + }, + }, + ); + expect(invalid.stdout).toBe("Ask the human.\nNo rule matched."); + expect(invalid.stdout).not.toContain("not-a-role"); + expect(calls.filter((command) => command === "strong")).toEqual([]); + }); + + it("uses an injected Ollama response and follows a bad body to the strong model", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("nearby"), "utf8"); + const fetches: string[] = []; + const commands: string[] = []; + const fetchImpl: typeof fetch = async (input, init) => { + fetches.push(String(input)); + expect(String(init?.body)).toContain("qwen2.5:7b"); + return new Response( + JSON.stringify({ + model: "qwen2.5:latest", + message: { content: reply("developer", 0.9, 1, false) }, + prompt_eval_count: 11, + eval_count: 5, + }), + { status: 200 }, + ); + }; + const picked = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { OLLAMA_HOST: "http://127.0.0.1:9", SDD_ROLE_MODEL: "strong" }, + fetch: fetchImpl, + runCommand: async (command) => { + commands.push(command); + return reply("architect", 0.9); + }, + now: () => new Date("2026-01-01T00:00:00.000Z"), + }, + ); + expect(picked.stdout).toContain("role developer"); + expect(fetches).toEqual(["http://127.0.0.1:9/api/chat"]); + expect(commands).toEqual([]); + + const bad = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { OLLAMA_HOST: "http://127.0.0.1:9", SDD_ROLE_MODEL: "strong" }, + fetch: async () => + new Response(JSON.stringify({ message: { content: '{"pick":"nope"}' } }), { + status: 200, + }), + runCommand: async (command) => { + commands.push(command); + return JSON.stringify({ text: "not a decision" }); + }, + }, + ); + expect(commands).toEqual(["strong"]); + expect(bad.stdout).toBe("Ask the human.\nNo rule matched."); + + let fetched = false; + const missing = await routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_ROLE_MODEL: "strong" }, + fetch: async () => { + fetched = true; + throw new Error("socket"); + }, + runCommand: async (command) => { + commands.push(command); + return reply("architect", 0.9); + }, + }, + ); + expect(fetched).toBe(false); + expect(missing.stdout).toBe("Ask the human.\nNo rule matched."); + + const config = await loadConfig(root); + const active = (await readFile(join(root, "changes/.active"), "utf8")).trim(); + const metrics = await readChangeMetrics(root, config, active); + expect( + metrics.decisions?.some((row) => row.model === "qwen2.5:latest" && row.inputTokens === 11), + ).toBe(true); + }); + + it("rejects a probability total outside the tolerance and a pick outside the options", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml("local"), "utf8"); + const commands: string[] = []; + const run = (body: string) => + routeRole( + root, + { artifact: "no-such-owns-phrase" }, + { + env: { SDD_DECIDE_MODEL: "decide", SDD_ROLE_MODEL: "strong" }, + runCommand: async (command) => { + commands.push(command); + return command === "decide" ? body : reply("architect", 0.9); + }, + }, + ); + commands.length = 0; + const wide = await run(reply("developer", 0.8, 1.02)); + expect(wide.stdout).toContain("role architect"); + expect(commands).toEqual(["decide", "strong"]); + + commands.length = 0; + const close = await run(reply("developer", 0.8, 1.01)); + expect(close.stdout).toContain("role developer"); + expect(commands).toEqual(["decide"]); + + commands.length = 0; + const outside = await run(JSON.stringify({ pick: "not-a-role", probs: { developer: 1 } })); + expect(outside.stdout).not.toContain("not-a-role"); + expect(commands[0]).toBe("decide"); + expect(commands).toContain("strong"); + + commands.length = 0; + const far = await run(reply("developer", 0.8, 1.5)); + expect(far.stdout).toContain("role architect"); + expect(commands).toEqual(["decide", "strong"]); + + commands.length = 0; + const omitted = await run( + JSON.stringify({ pick: "developer", probs: { developer: 0.8, human: 0.2 } }), + ); + expect(omitted.stdout).not.toContain("role developer"); + expect(commands).toContain("strong"); + }); +}); diff --git a/packages/core/src/__tests__/role-models.test.ts b/packages/core/src/__tests__/role-models.test.ts new file mode 100644 index 0000000..dbb2479 --- /dev/null +++ b/packages/core/src/__tests__/role-models.test.ts @@ -0,0 +1,193 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "pathe"; +import { afterEach, describe, expect, it } from "vitest"; +import { ChangeMetricsSchema } from "../metrics.js"; +import { + checkTeam, + explainModel, + formatModels, + initTeam, + loadModels, + selectModel, +} from "../role/index.js"; +import { rulesDecision } from "../role/models/rules.js"; +import { ModelsConfigSchema } from "../role/models/schema.js"; + +const temps: string[] = []; +const token = "abcdEFGH-0123456789abcdef0123456789"; + +afterEach(async () => { + while (temps.length) { + const dir = temps.pop(); + if (dir) await rm(dir, { recursive: true, force: true }); + } +}); + +function modelsYaml(): string { + return [ + "version: 1", + "models:", + " rules: { kind: decide, provider: rules }", + " local: { kind: decide, provider: command, command_env: SDD_DECIDE_MODEL, min_confidence: 0.7 }", + " strong: { kind: reason, provider: command, command_env: SDD_ROLE_MODEL }", + " nearby: { kind: decide, provider: ollama, model: qwen2.5:7b, base_url_env: OLLAMA_HOST, min_confidence: 0.7 }", + " by-role: { kind: reason, provider: command, command_env: SDD_BY_ROLE }", + " by-stage: { kind: reason, provider: command, command_env: SDD_BY_STAGE }", + " by-bot: { kind: reason, provider: command, command_env: SDD_BY_BOT }", + "defaults:", + " by_work_type:", + " route: rules", + " classify: local", + " check: local", + " draft: strong", + " design: strong", + " code: strong", + " review: strong", + " fallback_reason: strong", + " fallback_decide: rules", + "overrides:", + " bots:", + " bob-dev: { code: by-bot }", + " stages:", + " pr_review: { code: by-stage }", + " roles:", + " architect: { code: by-role }", + ].join("\n"); +} + +async function project(): Promise { + const root = await mkdtemp(join(tmpdir(), "sdd-models-")); + temps.push(root); + await mkdir(join(root, ".sdd"), { recursive: true }); + return root; +} + +describe("model config", () => { + it("loads a missing file as null and rejects a bad file, a secret, and a kind mismatch", async () => { + const root = await project(); + expect(await loadModels(root)).toBeNull(); + await writeFile(join(root, ".sdd/models.yaml"), "version: [\n", "utf8"); + expect(await loadModels(root)).toEqual({ + error: "models.yaml is invalid: models could not be parsed", + }); + + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml(), "utf8"); + const loaded = await loadModels(root); + expect(loaded && "config" in loaded).toBe(true); + + await writeFile( + join(root, ".sdd/models.yaml"), + modelsYaml().replace("SDD_ROLE_MODEL", "-----BEGIN FAKE-----"), + "utf8", + ); + const secret = await loadModels(root); + expect(secret).toEqual({ error: "models.yaml has a secret value." }); + expect(JSON.stringify(secret)).not.toContain("BEGIN"); + expect(JSON.stringify(secret)).not.toContain(token); + + await writeFile( + join(root, ".sdd/models.yaml"), + modelsYaml().replace("SDD_DECIDE_MODEL", `TOKEN=${token}`), + "utf8", + ); + expect(await loadModels(root)).toEqual({ error: "models.yaml has a secret value." }); + + await writeFile( + join(root, ".sdd/models.yaml"), + modelsYaml().replace("SDD_DECIDE_MODEL", token), + "utf8", + ); + expect(await loadModels(root)).toEqual({ error: "models.yaml has a secret value." }); + + await writeFile( + join(root, ".sdd/models.yaml"), + modelsYaml().replace("SDD_DECIDE_MODEL", "abcdefghijklmnopqrstuvwxyz01234"), + "utf8", + ); + expect(await loadModels(root)).toMatchObject({ config: { version: 1 } }); + + const mismatch = ModelsConfigSchema.safeParse({ + version: 1, + models: { + rules: { kind: "decide", provider: "rules" }, + strong: { kind: "reason", provider: "command", command_env: "SDD_ROLE_MODEL" }, + }, + defaults: { + by_work_type: { route: "strong" }, + fallback_reason: "strong", + fallback_decide: "rules", + }, + }); + expect(mismatch.success).toBe(false); + + const missingEnv = ModelsConfigSchema.safeParse({ + version: 1, + models: { + rules: { kind: "decide", provider: "rules" }, + strong: { kind: "reason", provider: "command" }, + }, + defaults: { + by_work_type: { draft: "strong" }, + fallback_reason: "strong", + fallback_decide: "rules", + }, + }); + expect(missingEnv.success).toBe(false); + }); + + it("picks bot, then stage, then role, then the work type, then the fallback", async () => { + const root = await project(); + await writeFile(join(root, ".sdd/models.yaml"), modelsYaml(), "utf8"); + const loaded = await loadModels(root); + if (!loaded || !("config" in loaded)) throw new Error("expected config"); + const config = loaded.config; + expect(selectModel(config, { workType: "draft" })).toMatchObject({ + id: "strong", + kind: "reason", + why: "work-type default", + }); + expect(selectModel(config, { workType: "route" }).id).toBe("rules"); + expect(selectModel(config, { workType: "classify" }).id).toBe("local"); + expect(selectModel(config, { workType: "code", role: "architect" }).why).toBe("role architect"); + expect( + selectModel(config, { workType: "code", role: "architect", stage: "pr_review" }).id, + ).toBe("by-stage"); + expect( + selectModel(config, { + workType: "code", + role: "architect", + stage: "pr_review", + bot: "bob-dev", + }).why, + ).toBe("bot bob-dev"); + expect(explainModel(config, "review", {})).toBe( + "review strong reason command work-type default", + ); + expect(formatModels(config)).toContain("fallback_reason strong reason command fallback"); + expect(formatModels(config)).toContain("fallback_decide rules decide rules fallback"); + }); + + it("asks the human from rules and keeps a team file strict", async () => { + expect(rulesDecision(["developer", "human"])).toEqual({ pick: "human" }); + expect(rulesDecision(["developer"])).toEqual({ pick: "human" }); + const root = await project(); + expect((await initTeam(root, "software", "Ada Lovelace")).exitCode).toBe(0); + expect(await checkTeam(root)).toMatchObject({ + exitCode: 0, + stdout: "Team config is valid.", + }); + await writeFile(join(root, ".sdd/roles/developer.md"), "\n-----BEGIN FAKE-----\n", { + flag: "a", + }); + expect((await checkTeam(root)).exitCode).toBe(0); + const team = await readFile(join(root, ".sdd/team.yaml"), "utf8"); + await writeFile(join(root, ".sdd/team.yaml"), `${team}models:\n code: strong\n`, "utf8"); + const checked = await checkTeam(root); + expect(checked.exitCode).toBe(1); + expect(checked.stdout).toContain("team.yaml is invalid"); + expect(ChangeMetricsSchema.parse({ version: 1, phases: [], requests: [] }).decisions).toBe( + undefined, + ); + }); +}); diff --git a/packages/core/src/metrics.ts b/packages/core/src/metrics.ts index 2590105..6e721d3 100644 --- a/packages/core/src/metrics.ts +++ b/packages/core/src/metrics.ts @@ -5,6 +5,7 @@ import { createInterface } from "node:readline"; import { z } from "zod"; import { pathExists, readText, writeText } from "./fs.js"; import { changePath } from "./paths.js"; +import { WORK_TYPES } from "./role/types.js"; import type { Config } from "./schemas.js"; const UtcTimestampSchema = z.string().datetime(); @@ -30,14 +31,28 @@ export const TokenUsageMetricSchema = z.object({ measured: z.literal(true), }); +export const DecisionMetricSchema = z.object({ + workType: z.enum(WORK_TYPES), + kind: z.enum(["reason", "decide"]), + provider: z.enum(["command", "ollama"]), + model: z.string().min(1).nullable(), + startedAt: UtcTimestampSchema, + inputTokens: z.number().int().nonnegative().nullable(), + cacheReadTokens: z.number().int().nonnegative().nullable(), + cacheWriteTokens: z.number().int().nonnegative().nullable(), + outputTokens: z.number().int().nonnegative().nullable(), +}); + export const ChangeMetricsSchema = z.object({ version: z.literal(1), phases: z.array(PhaseMetricSchema), requests: z.array(TokenUsageMetricSchema), + decisions: z.array(DecisionMetricSchema).optional(), }); export type PhaseMetric = z.infer; export type TokenUsageMetric = z.infer; +export type DecisionMetric = z.infer; export type ChangeMetrics = z.infer; export interface MetricsImportResult { @@ -91,6 +106,18 @@ async function writeChangeMetrics( await writeText(path, `${JSON.stringify(validated, null, 2)}\n`); } +export async function recordDecisionMetric( + projectRoot: string, + config: Config, + changeId: string, + draft: DecisionMetric, +): Promise { + if (!config.metrics.enabled) return; + const metrics = await readChangeMetrics(projectRoot, config, changeId); + const decisions = [...(metrics.decisions ?? []), DecisionMetricSchema.parse(draft)]; + await writeChangeMetrics(projectRoot, config, changeId, { ...metrics, decisions }); +} + export async function startPhase( projectRoot: string, config: Config, diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index 433e773..bafdf91 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -74,6 +74,9 @@ export interface RenderChangeReportInput { const TOKEN_HEADER = "source phase provider model requests input cache_read cache_write output usd inr"; +const DECISION_HEADER = + "work_type kind provider model calls input cache_read cache_write output usd inr"; + function finiteMs(startIso: string, endMs: number): number { const start = Date.parse(startIso); if (!Number.isFinite(start) || !Number.isFinite(endMs)) return 0; @@ -354,10 +357,79 @@ export function renderChangeReport(input: RenderChangeReportInput): string { for (const request of input.metrics.requests) addRequest(total, request); lines.push(` ${totalLine(total, input.metrics, input.prices)}`); } + const decisions = decisionSection(input.metrics, input.prices); + if (decisions.length > 0) { + lines.push("Decisions:"); + lines.push(` ${DECISION_HEADER}`); + for (const line of decisions) lines.push(` ${line}`); + } if (input.prices === null) lines.push("prices.yaml is missing"); return lines.join("\n"); } +function groupDecisions(metrics: ChangeMetrics): DecisionGroup[] { + const groups = new Map(); + for (const row of metrics.decisions ?? []) { + const model = row.model ?? null; + const key = `${row.workType}\0${row.kind}\0${row.provider}\0${model ?? ""}`; + let group = groups.get(key); + if (!group) { + group = { + workType: row.workType, + kind: row.kind, + provider: row.provider, + model, + totals: emptyTotals(), + }; + groups.set(key, group); + } + group.totals.requests += 1; + group.totals.input = addCount(group.totals.input, row.inputTokens); + group.totals.cacheRead = addCount(group.totals.cacheRead, row.cacheReadTokens); + group.totals.cacheWrite = addCount(group.totals.cacheWrite, row.cacheWriteTokens); + group.totals.output = addCount(group.totals.output, row.outputTokens); + } + const order = ["route", "classify", "check", "draft", "design", "code", "review"]; + return [...groups.values()].sort((left, right) => { + const workDelta = order.indexOf(left.workType) - order.indexOf(right.workType); + if (workDelta !== 0) return workDelta; + const kindDelta = left.kind.localeCompare(right.kind); + if (kindDelta !== 0) return kindDelta; + const providerDelta = left.provider.localeCompare(right.provider); + if (providerDelta !== 0) return providerDelta; + return (left.model ?? "").localeCompare(right.model ?? ""); + }); +} + +function decisionSection(metrics: ChangeMetrics, prices: Prices | null): string[] { + return groupDecisions(metrics).map((group) => { + const card = prices ? cardFor(prices, group.provider, group.model) : null; + const cost = + prices && card ? costFor(group.totals, card, prices.usd_to_inr) : { usd: "n/a", inr: "n/a" }; + return [ + group.workType, + group.kind, + group.provider, + modelCell(group.model), + String(group.totals.requests), + cell(group.totals.input), + cell(group.totals.cacheRead), + cell(group.totals.cacheWrite), + cell(group.totals.output), + cost.usd, + cost.inr, + ].join(" "); + }); +} + +interface DecisionGroup { + workType: string; + kind: string; + provider: string; + model: string | null; + totals: Totals; +} + function requestUsdNumerator(request: ChangeMetrics["requests"][number], card: PriceCard): bigint { const parts: Array<[number | null, number]> = [ [request.inputTokens, card.input_per_million_usd], diff --git a/packages/core/src/role/index.ts b/packages/core/src/role/index.ts index a49a040..51cfc70 100644 --- a/packages/core/src/role/index.ts +++ b/packages/core/src/role/index.ts @@ -20,9 +20,18 @@ export { } from "./profile.js"; export { TeamSchema } from "./schema.js"; export type { TeamConfig } from "./schema.js"; +export { decide } from "./models/decide.js"; +export type { DecideInput, DecisionOutcome } from "./models/decide.js"; +export { modelFromEnv, spawnModelCommand } from "./models/command.js"; +export { loadModels, checkModels, ModelsConfigSchema } from "./models/schema.js"; +export type { ModelsConfig, ModelEntry } from "./models/schema.js"; +export { selectModel } from "./models/select.js"; +export type { ModelContext } from "./models/select.js"; +export { explainModel, formatModels } from "./models/show.js"; export { queueRole, routeRole, takeRole } from "./routing.js"; -export type { RouteHit, RouteItem } from "./routing.js"; +export type { RouteDeps, RouteHit, RouteItem } from "./routing.js"; export { + checkTeam, createRoleFromTemplate, formatTeam, gitUserName, @@ -32,4 +41,13 @@ export { readTeamConfig, showTeam, } from "./team.js"; -export type { RoleCommandResult, RoleLists, RoleModel, RoleProfile } from "./types.js"; +export type { + DecisionModel, + ModelChoice, + RoleCommandResult, + RoleLists, + RoleModel, + RoleProfile, + WorkType, +} from "./types.js"; +export { WORK_TYPES, isWorkType, workKind } from "./types.js"; diff --git a/packages/core/src/role/models/command.ts b/packages/core/src/role/models/command.ts new file mode 100644 index 0000000..e54df41 --- /dev/null +++ b/packages/core/src/role/models/command.ts @@ -0,0 +1,59 @@ +import { spawn } from "node:child_process"; +import type { RoleModel } from "../types.js"; + +export function decideStdin(input: { + state: string; + question: string; + options: readonly string[]; +}): string { + return `${JSON.stringify({ + mode: "decide", + state: input.state, + question: input.question, + options: input.options, + })}\n`; +} + +export function spawnModelCommand(command: string, stdin: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, { shell: true, stdio: ["pipe", "pipe", "ignore"] }); + const out: Buffer[] = []; + child.stdout?.on("data", (chunk: Buffer) => out.push(chunk)); + child.on("error", reject); + child.stdin?.on("error", () => undefined); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(`role model exited ${code ?? "unknown"}`)); + return; + } + resolve(Buffer.concat(out).toString("utf8").trim()); + }); + child.stdin?.write(stdin); + child.stdin?.end(); + }); +} + +export function modelFromEnv( + env: Readonly>, +): RoleModel | undefined { + const command = env.SDD_ROLE_MODEL?.trim(); + if (!command) return undefined; + return { + async complete(input) { + const text = await spawnModelCommand( + command, + `${JSON.stringify({ roleId: input.roleId, prompt: input.prompt })}\n`, + ); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error("role model stdout is not JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("role model stdout is not a JSON object"); + } + return text; + }, + }; +} diff --git a/packages/core/src/role/models/decide.ts b/packages/core/src/role/models/decide.ts new file mode 100644 index 0000000..68c29be --- /dev/null +++ b/packages/core/src/role/models/decide.ts @@ -0,0 +1,302 @@ +import { z } from "zod"; +import { recordDecisionMetric } from "../../metrics.js"; +import type { Config } from "../../schemas.js"; +import type { ModelChoice, WorkType } from "../types.js"; +import { decideStdin } from "./command.js"; +import { ollamaDecision } from "./ollama.js"; +import { rulesDecision } from "./rules.js"; +import type { ModelEntry, ModelsConfig } from "./schema.js"; +import { type ModelContext, selectModel } from "./select.js"; + +const tokenField = z.number().int().nonnegative().nullable().optional(); + +const DecisionReplySchema = z + .object({ + pick: z.string(), + probs: z.record(z.string(), z.number()), + model: z.string().trim().min(1).optional(), + input_tokens: tokenField, + cache_read_tokens: tokenField, + cache_write_tokens: tokenField, + output_tokens: tokenField, + }) + .strict(); + +type DecisionReply = z.infer; + +function minimumOf(entry: ModelEntry | undefined): number | undefined { + if (!entry || entry.provider === "rules") return undefined; + return entry.min_confidence; +} + +export interface DecideInput { + models: ModelsConfig | null; + ctx: ModelContext & { workType: WorkType }; + state: string; + question: string; + options: readonly T[]; + env: Readonly>; + fetch?: typeof fetch; + runCommand?: (command: string, stdin: string) => Promise; + now?: () => Date; + metrics?: { projectRoot: string; config: Config; changeId: string }; +} + +export interface DecisionOutcome { + pick: T | "human"; + source: "rules" | "decide" | "reason" | "human"; + choice: ModelChoice | null; + confidence: number | null; +} + +interface Attempt { + kept: boolean; + missing: boolean; + escalate: boolean; + confidence: number | null; + outcome: DecisionOutcome; +} + +interface Usage { + model?: string; + inputTokens: number | null; + cacheReadTokens: number | null; + cacheWriteTokens: number | null; + outputTokens: number | null; +} + +function humanOutcome( + choice: ModelChoice | null, + confidence: number | null, +): DecisionOutcome { + return { pick: "human", source: "human", choice, confidence }; +} + +function emptyUsage(): Usage { + return { + inputTokens: null, + cacheReadTokens: null, + cacheWriteTokens: null, + outputTokens: null, + }; +} + +function usageFrom(reply: DecisionReply, base: Usage): Usage { + return { + model: reply.model ?? base.model, + inputTokens: reply.input_tokens === undefined ? base.inputTokens : reply.input_tokens, + cacheReadTokens: + reply.cache_read_tokens === undefined ? base.cacheReadTokens : reply.cache_read_tokens, + cacheWriteTokens: + reply.cache_write_tokens === undefined ? base.cacheWriteTokens : reply.cache_write_tokens, + outputTokens: reply.output_tokens === undefined ? base.outputTokens : reply.output_tokens, + }; +} + +function scoreReply( + raw: string, + options: readonly T[], + base: Usage, +): { ok: true; pick: T; confidence: number; usage: Usage } | { ok: false; usage: Usage } { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return { ok: false, usage: base }; + } + const parsed = DecisionReplySchema.safeParse(value); + if (!parsed.success) return { ok: false, usage: base }; + const reply = parsed.data; + const usage = usageFrom(reply, base); + if (!options.includes(reply.pick as T)) return { ok: false, usage }; + const keys = Object.keys(reply.probs); + if (keys.length !== options.length) return { ok: false, usage }; + let sum = 0; + for (const option of options) { + const probability = reply.probs[option]; + if (typeof probability !== "number" || probability < 0 || probability > 1) { + return { ok: false, usage }; + } + sum += probability; + } + if (Math.abs(sum - 1) > 0.01 + 1e-9) return { ok: false, usage }; + const confidence = reply.probs[reply.pick]; + if (typeof confidence !== "number") return { ok: false, usage }; + return { ok: true, pick: reply.pick as T, confidence, usage }; +} + +function storedModel(entry: ModelEntry, usage: Usage): string | null { + if (usage.model) return usage.model; + if (entry.provider === "ollama") return entry.model; + return null; +} + +async function recordCall( + input: DecideInput, + entry: ModelEntry, + usage: Usage, +): Promise { + if (!input.metrics) return; + if (entry.provider === "rules") return; + await recordDecisionMetric( + input.metrics.projectRoot, + input.metrics.config, + input.metrics.changeId, + { + workType: input.ctx.workType, + kind: entry.kind, + provider: entry.provider, + model: storedModel(entry, usage), + startedAt: (input.now ?? (() => new Date()))().toISOString(), + inputTokens: usage.inputTokens, + cacheReadTokens: usage.cacheReadTokens, + cacheWriteTokens: usage.cacheWriteTokens, + outputTokens: usage.outputTokens, + }, + ); +} + +function judge( + choice: ModelChoice, + entry: ModelEntry, + scored: { ok: true; pick: T; confidence: number } | { ok: false }, + floor?: number, +): Attempt { + const minimum = minimumOf(entry) ?? floor; + if (!scored.ok) { + return { + kept: false, + missing: false, + escalate: minimum !== undefined, + confidence: 0, + outcome: humanOutcome(choice, 0), + }; + } + if (minimum !== undefined && scored.confidence < minimum) { + return { + kept: false, + missing: false, + escalate: true, + confidence: scored.confidence, + outcome: humanOutcome(choice, scored.confidence), + }; + } + const source = choice.kind === "reason" ? "reason" : "decide"; + return { + kept: true, + missing: false, + escalate: false, + confidence: scored.confidence, + outcome: { pick: scored.pick, source, choice, confidence: scored.confidence }, + }; +} + +async function runOne( + input: DecideInput, + choice: ModelChoice, + floor?: number, +): Promise> { + const entry = input.models?.models[choice.id]; + if (!entry) { + return { + kept: false, + missing: true, + escalate: false, + confidence: null, + outcome: humanOutcome(choice, null), + }; + } + if (entry.provider === "rules") { + rulesDecision(input.options); + return { + kept: true, + missing: false, + escalate: false, + confidence: null, + outcome: { pick: "human", source: "rules", choice, confidence: null }, + }; + } + if (entry.provider === "command") { + const command = input.env[entry.command_env]?.trim(); + if (!command || !input.runCommand) { + return { + kept: false, + missing: true, + escalate: false, + confidence: null, + outcome: humanOutcome(choice, null), + }; + } + let stdout = ""; + try { + stdout = await input.runCommand(command, decideStdin(input)); + } catch { + await recordCall(input, entry, emptyUsage()); + return judge(choice, entry, { ok: false }, floor); + } + const scored = scoreReply(stdout, input.options, emptyUsage()); + await recordCall(input, entry, scored.usage); + return judge(choice, entry, scored, floor); + } + const baseUrl = input.env[entry.base_url_env]?.trim(); + if (!baseUrl || !input.fetch) { + return { + kept: false, + missing: true, + escalate: false, + confidence: null, + outcome: humanOutcome(choice, null), + }; + } + const remote = await ollamaDecision({ + baseUrl, + model: entry.model, + state: input.state, + question: input.question, + options: input.options, + fetch: input.fetch, + }); + if (!remote.called) { + return { + kept: false, + missing: true, + escalate: false, + confidence: null, + outcome: humanOutcome(choice, null), + }; + } + const base: Usage = { + model: remote.model, + inputTokens: remote.inputTokens, + cacheReadTokens: null, + cacheWriteTokens: null, + outputTokens: remote.outputTokens, + }; + const scored = remote.content + ? scoreReply(remote.content, input.options, base) + : { ok: false as const, usage: base }; + await recordCall(input, entry, scored.usage); + return judge(choice, entry, scored, floor); +} + +export async function decide(input: DecideInput): Promise> { + if (!input.models) return { pick: "human", source: "human", choice: null, confidence: null }; + const choice = selectModel(input.models, input.ctx); + const first = await runOne(input, choice); + if (first.kept) return first.outcome; + if (first.missing || !first.escalate) return first.outcome; + const floor = minimumOf(input.models.models[choice.id]); + const fallbackId = input.models.defaults.fallback_reason; + if (choice.id === fallbackId) return first.outcome; + const fallbackEntry = input.models.models[fallbackId]; + if (!fallbackEntry) return first.outcome; + const fallback: ModelChoice = { + id: fallbackId, + kind: fallbackEntry.kind, + provider: fallbackEntry.provider, + why: "fallback", + }; + const second = await runOne(input, fallback, floor); + if (second.kept) return second.outcome; + return second.outcome; +} diff --git a/packages/core/src/role/models/ollama.ts b/packages/core/src/role/models/ollama.ts new file mode 100644 index 0000000..09a119e --- /dev/null +++ b/packages/core/src/role/models/ollama.ts @@ -0,0 +1,122 @@ +function count(value: unknown): number | null { + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null; +} + +export interface OllamaResult { + called: boolean; + model: string; + content: string | null; + inputTokens: number | null; + outputTokens: number | null; +} + +function chatUrl(base: string): string | null { + try { + const withSlash = base.endsWith("/") ? base : `${base}/`; + return new URL("api/chat", withSlash).toString(); + } catch { + return null; + } +} + +export async function ollamaDecision(input: { + baseUrl: string; + model: string; + state: string; + question: string; + options: readonly string[]; + fetch: typeof fetch; +}): Promise { + const url = chatUrl(input.baseUrl); + if (!url) { + return { + called: false, + model: input.model, + content: null, + inputTokens: null, + outputTokens: null, + }; + } + const body = JSON.stringify({ + model: input.model, + stream: false, + format: { + type: "object", + properties: { + pick: { type: "string" }, + probs: { type: "object", additionalProperties: { type: "number" } }, + }, + required: ["pick", "probs"], + }, + messages: [ + { + role: "user", + content: JSON.stringify({ + state: input.state, + question: input.question, + options: input.options, + }), + }, + ], + }); + try { + const response = await input.fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + }); + const text = await response.text(); + if (!response.ok) { + return { + called: true, + model: input.model, + content: null, + inputTokens: null, + outputTokens: null, + }; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return { + called: true, + model: input.model, + content: null, + inputTokens: null, + outputTokens: null, + }; + } + const record = + parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + const message = + record.message && typeof record.message === "object" && !Array.isArray(record.message) + ? (record.message as Record) + : {}; + const content = + typeof message.content === "string" + ? message.content + : message.content && typeof message.content === "object" + ? JSON.stringify(message.content) + : null; + const model = + typeof record.model === "string" && record.model.trim() ? record.model : input.model; + return { + called: true, + model, + content, + inputTokens: count(record.prompt_eval_count), + outputTokens: count(record.eval_count), + }; + } catch { + return { + called: true, + model: input.model, + content: null, + inputTokens: null, + outputTokens: null, + }; + } +} diff --git a/packages/core/src/role/models/rules.ts b/packages/core/src/role/models/rules.ts new file mode 100644 index 0000000..6dee644 --- /dev/null +++ b/packages/core/src/role/models/rules.ts @@ -0,0 +1,4 @@ +/** Deterministic decider. It returns human and does not rank roles. */ +export function rulesDecision(options: readonly string[]): { pick: "human" } { + return { pick: options.includes("human") ? "human" : "human" }; +} diff --git a/packages/core/src/role/models/schema.ts b/packages/core/src/role/models/schema.ts new file mode 100644 index 0000000..8746dd1 --- /dev/null +++ b/packages/core/src/role/models/schema.ts @@ -0,0 +1,178 @@ +import { join } from "pathe"; +import { parse as parseYaml } from "yaml"; +import { z } from "zod"; +import { pathExists, readText } from "../../fs.js"; +import { WORK_TYPES, type WorkType, isWorkType, workKind } from "../types.js"; + +const idSchema = z.string().trim().min(1); +const envName = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/); +const confidence = z.number().gte(0).lte(1); + +const WorkTypeSchema = z.enum(WORK_TYPES); +const workSlots = z.record(WorkTypeSchema, idSchema); + +const ModelEntrySchema = z.discriminatedUnion("provider", [ + z + .object({ + kind: z.literal("decide"), + provider: z.literal("rules"), + }) + .strict(), + z + .object({ + kind: z.enum(["decide", "reason"]), + provider: z.literal("command"), + command_env: envName, + min_confidence: confidence.optional(), + }) + .strict(), + z + .object({ + kind: z.literal("decide"), + provider: z.literal("ollama"), + model: z.string().trim().min(1), + base_url_env: envName, + min_confidence: confidence.optional(), + }) + .strict(), +]); + +export type ModelEntry = z.infer; + +export const ModelsConfigSchema = z + .object({ + version: z.literal(1), + models: z.record(idSchema, ModelEntrySchema), + defaults: z + .object({ + by_work_type: workSlots, + fallback_reason: idSchema, + fallback_decide: idSchema, + }) + .strict(), + overrides: z + .object({ + bots: z.record(idSchema, workSlots).optional(), + stages: z.record(idSchema, workSlots).optional(), + roles: z.record(idSchema, workSlots).optional(), + }) + .strict() + .optional(), + }) + .strict() + .superRefine((config, ctx) => { + if (Object.keys(config.models).length === 0) { + ctx.addIssue({ code: "custom", path: ["models"], message: "needs a model" }); + } + expectSlot(ctx, config.models, config.defaults.fallback_reason, "reason", [ + "defaults", + "fallback_reason", + ]); + expectSlot(ctx, config.models, config.defaults.fallback_decide, "decide", [ + "defaults", + "fallback_decide", + ]); + expectMap(ctx, config.models, config.defaults.by_work_type, ["defaults", "by_work_type"]); + expectNamed(ctx, config.models, config.overrides?.bots, ["overrides", "bots"]); + expectNamed(ctx, config.models, config.overrides?.stages, ["overrides", "stages"]); + expectNamed(ctx, config.models, config.overrides?.roles, ["overrides", "roles"]); + }); + +export type ModelsConfig = z.infer; + +const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=\S+$/; + +function expectSlot( + ctx: z.RefinementCtx, + models: Record, + id: string | undefined, + kind: "reason" | "decide", + path: (string | number)[], +): void { + if (!id) return; + const model = models[id]; + if (!model) { + ctx.addIssue({ code: "custom", path, message: "unknown model" }); + return; + } + if (model.kind !== kind) { + ctx.addIssue({ code: "custom", path, message: `kind must be ${kind}` }); + } +} + +function expectMap( + ctx: z.RefinementCtx, + models: Record, + map: Partial> | undefined, + path: (string | number)[], +): void { + if (!map) return; + for (const [work, id] of Object.entries(map)) { + if (!isWorkType(work)) continue; + expectSlot(ctx, models, id, workKind(work), [...path, work]); + } +} + +function expectNamed( + ctx: z.RefinementCtx, + models: Record, + named: Record>> | undefined, + path: (string | number)[], +): void { + if (!named) return; + for (const [name, map] of Object.entries(named)) { + expectMap(ctx, models, map, [...path, name]); + } +} + +export function formatModelsError(error: z.ZodError): string { + const issue = error.issues[0]; + const path = issue?.path.join(".") || "models"; + const message = issue?.message ?? "invalid"; + return `models.yaml is invalid: ${path} ${message}`; +} + +function looksSecret(value: string): boolean { + if (value.includes("-----BEGIN")) return true; + if (!/\s/.test(value) && ASSIGNMENT.test(value)) return true; + if (!/\s/.test(value) && value.length >= 32 && !ENV_NAME.test(value)) return true; + return false; +} + +function containsSecret(value: unknown): boolean { + if (typeof value === "string") return looksSecret(value); + if (Array.isArray(value)) return value.some(containsSecret); + if (value && typeof value === "object") return Object.values(value).some(containsSecret); + return false; +} + +export function modelsPath(root: string): string { + return join(root, ".sdd", "models.yaml"); +} + +export async function loadModels( + root: string, +): Promise<{ config: ModelsConfig } | { error: string } | null> { + const path = modelsPath(root); + if (!(await pathExists(path))) return null; + let raw: unknown; + try { + raw = parseYaml(await readText(path)); + } catch { + return { error: "models.yaml is invalid: models could not be parsed" }; + } + if (containsSecret(raw)) return { error: "models.yaml has a secret value." }; + const parsed = ModelsConfigSchema.safeParse(raw); + if (!parsed.success) return { error: formatModelsError(parsed.error) }; + return { config: parsed.data }; +} + +export async function checkModels( + root: string, +): Promise<{ ok: true } | { ok: false; message: string }> { + const loaded = await loadModels(root); + if (loaded === null) return { ok: true }; + if ("error" in loaded) return { ok: false, message: loaded.error }; + return { ok: true }; +} diff --git a/packages/core/src/role/models/select.ts b/packages/core/src/role/models/select.ts new file mode 100644 index 0000000..cae0dae --- /dev/null +++ b/packages/core/src/role/models/select.ts @@ -0,0 +1,52 @@ +import { type ModelChoice, type WorkType, workKind } from "../types.js"; +import type { ModelsConfig } from "./schema.js"; + +export interface ModelContext { + workType: WorkType; + role?: string; + bot?: string; + stage?: string; +} + +function slot( + map: Partial> | undefined, + workType: WorkType, +): string | undefined { + const id = map?.[workType]?.trim(); + return id || undefined; +} + +/** Pure precedence: bot, stage, role, work-type default, then the fallback for that kind. */ +export function selectModel(config: ModelsConfig, ctx: ModelContext): ModelChoice { + const bot = ctx.bot?.trim(); + const stage = ctx.stage?.trim(); + const role = ctx.role?.trim(); + const fromBot = bot ? slot(config.overrides?.bots?.[bot], ctx.workType) : undefined; + const fromStage = stage ? slot(config.overrides?.stages?.[stage], ctx.workType) : undefined; + const fromRole = role ? slot(config.overrides?.roles?.[role], ctx.workType) : undefined; + const fromWork = slot(config.defaults.by_work_type, ctx.workType); + let id: string; + let why: string; + if (fromBot && bot) { + id = fromBot; + why = `bot ${bot}`; + } else if (fromStage && stage) { + id = fromStage; + why = `stage ${stage}`; + } else if (fromRole && role) { + id = fromRole; + why = `role ${role}`; + } else if (fromWork) { + id = fromWork; + why = "work-type default"; + } else { + id = + workKind(ctx.workType) === "decide" + ? config.defaults.fallback_decide + : config.defaults.fallback_reason; + why = "fallback"; + } + const model = config.models[id]; + if (!model) return { id, kind: workKind(ctx.workType), provider: "rules", why }; + return { id, kind: model.kind, provider: model.provider, why }; +} diff --git a/packages/core/src/role/models/show.ts b/packages/core/src/role/models/show.ts new file mode 100644 index 0000000..f35c3b4 --- /dev/null +++ b/packages/core/src/role/models/show.ts @@ -0,0 +1,55 @@ +import { type ModelChoice, WORK_TYPES, type WorkType } from "../types.js"; +import type { ModelEntry, ModelsConfig } from "./schema.js"; +import { selectModel } from "./select.js"; + +export function formatModelChoice(workType: string, choice: ModelChoice): string { + return `${workType} ${choice.id} ${choice.kind} ${choice.provider} ${choice.why}`; +} + +function confidence(model: ModelEntry): string { + return "min_confidence" in model && model.min_confidence !== undefined + ? String(model.min_confidence) + : "n/a"; +} + +function formatEntry(id: string, model: ModelEntry): string { + if (model.provider === "rules") return `${id} decide rules`; + if (model.provider === "command") { + return `${id} ${model.kind} command command_env ${model.command_env} min_confidence ${confidence(model)}`; + } + return `${id} decide ollama model ${model.model} base_url_env ${model.base_url_env} min_confidence ${confidence(model)}`; +} + +export function formatModels(config: ModelsConfig): string { + const lines = Object.keys(config.models) + .sort() + .map((id) => formatEntry(id, config.models[id] as ModelEntry)); + for (const workType of WORK_TYPES) { + lines.push(formatModelChoice(workType, selectModel(config, { workType }))); + } + const reason = config.models[config.defaults.fallback_reason]; + const decide = config.models[config.defaults.fallback_decide]; + if (reason) { + lines.push( + `fallback_reason ${config.defaults.fallback_reason} ${reason.kind} ${reason.provider} fallback`, + ); + } + if (decide) { + lines.push( + `fallback_decide ${config.defaults.fallback_decide} ${decide.kind} ${decide.provider} fallback`, + ); + } + return lines.join("\n"); +} + +export function explainModel( + config: ModelsConfig, + workType: WorkType, + ctx: { + role?: string; + bot?: string; + stage?: string; + }, +): string { + return formatModelChoice(workType, selectModel(config, { workType, ...ctx })); +} diff --git a/packages/core/src/role/routing.ts b/packages/core/src/role/routing.ts index b1576c1..ae635bf 100644 --- a/packages/core/src/role/routing.ts +++ b/packages/core/src/role/routing.ts @@ -4,6 +4,8 @@ import { z } from "zod"; import { buildContext, getActiveChangeId } from "../change-context.js"; import { loadConfig } from "../config.js"; import { listDirs, listFiles, pathExists, readText } from "../fs.js"; +import { type DecideInput, decide } from "./models/decide.js"; +import { loadModels } from "./models/schema.js"; import { canonicalRoleName, parseRoleDocument } from "./profile.js"; import { appendJson } from "./run.js"; import { readTeamConfig } from "./team.js"; @@ -59,6 +61,13 @@ export interface RouteItem { bot?: string; } +export interface RouteDeps { + env?: Readonly>; + fetch?: typeof fetch; + runCommand?: (command: string, stdin: string) => Promise; + now?: () => Date; +} + export type RouteHit = | { kind: "role"; role: string; reason: string; rule: string } | { kind: "ask"; reason: string }; @@ -99,10 +108,12 @@ function cleanItem(input: RouteItem): RouteItem { const stage = input.stage?.trim(); const artifact = input.artifact?.trim(); const skill = input.skill?.trim(); + const bot = input.bot?.trim(); if (id) item.id = id; if (stage) item.stage = stage; if (artifact) item.artifact = artifact; if (skill) item.skill = skill; + if (bot) item.bot = bot; return item; } @@ -297,7 +308,10 @@ async function prepareItem(root: string, input: RouteItem): Promise { +function routeState(item: RouteItem): string { + return JSON.stringify({ + stage: item.stage ?? null, + artifact: item.artifact ?? null, + skill: item.skill ?? null, + id: item.id ?? null, + }); +} + +async function routeMetrics(root: string): Promise["metrics"]> { + try { + const config = await loadConfig(root); + const changeId = await getActiveChangeId(root, config); + if (!changeId) return undefined; + return { projectRoot: root, config, changeId }; + } catch { + return undefined; + } +} + +export async function routeRole( + root: string, + input: RouteItem, + deps?: RouteDeps, +): Promise { const item = await prepareItem(root, input); if ("error" in item) return fail(item.error); const matched = await matchRoute(root, item); if (!matched.ok) return fail(matched.message); - return ok(formatHit(matched.match)); + if (matched.match.kind !== "ask" || matched.match.reason !== "No rule matched.") { + return ok(formatHit(matched.match)); + } + const loaded = await loadModels(root); + if (loaded === null) return ok(formatHit(matched.match)); + if ("error" in loaded) return fail(loaded.error); + const roles = await loadRoles(root); + if ("error" in roles) return fail(roles.error); + const names = roles.map((role) => role.name).sort((left, right) => left.localeCompare(right)); + const options = names.includes("human") ? names : [...names, "human"]; + const decision = await decide({ + models: loaded.config, + ctx: { workType: "route", stage: item.stage, bot: item.bot }, + state: routeState(item), + question: "No rule matched.", + options, + env: deps?.env ?? {}, + fetch: deps?.fetch, + runCommand: deps?.runCommand, + now: deps?.now, + metrics: await routeMetrics(root), + }); + if (decision.pick !== "human" && names.includes(decision.pick)) { + const id = decision.choice?.id ?? "model"; + return ok(`role ${decision.pick}\nreason Model ${id} picked ${decision.pick}.`); + } + return ok("Ask the human.\nNo rule matched."); } async function readInbox(root: string, name: string): Promise { diff --git a/packages/core/src/role/team.ts b/packages/core/src/role/team.ts index 33e6754..dbbdb54 100644 --- a/packages/core/src/role/team.ts +++ b/packages/core/src/role/team.ts @@ -3,6 +3,7 @@ import { join } from "pathe"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; import { defaultTeamsDir } from "../defaults.js"; import { listDirs, listFiles, pathExists, readText, writeText } from "../fs.js"; +import { checkModels } from "./models/schema.js"; import { createRole, parseRoleDocument, roleFilePath } from "./profile.js"; import { type TeamConfig, TeamSchema, formatTeamError } from "./schema.js"; import type { RoleCommandResult } from "./types.js"; @@ -176,6 +177,14 @@ export async function showTeam(root: string, gitUser: string | null): Promise { + const loaded = await readTeamConfig(root); + if ("error" in loaded) return fail(loaded.error); + const models = await checkModels(root); + if (!models.ok) return fail(models.message); + return ok("Team config is valid."); +} + export async function resolveRoleTemplate( template: string, ): Promise<{ id: string; raw: string } | { error: string }> { diff --git a/packages/core/src/role/types.ts b/packages/core/src/role/types.ts index df8eac4..f4d4370 100644 --- a/packages/core/src/role/types.ts +++ b/packages/core/src/role/types.ts @@ -42,6 +42,44 @@ export interface RoleModel { complete(input: { roleId: string; prompt: string }): Promise; } +export const WORK_TYPES = [ + "route", + "classify", + "check", + "draft", + "design", + "code", + "review", +] as const; + +export type WorkType = (typeof WORK_TYPES)[number]; + +export function isWorkType(value: string): value is WorkType { + return (WORK_TYPES as readonly string[]).includes(value); +} + +/** Route, classify, and check are cheap decisions. The other work types reason. */ +export function workKind(workType: WorkType): "reason" | "decide" { + return workType === "route" || workType === "classify" || workType === "check" + ? "decide" + : "reason"; +} + +export interface DecisionModel { + decide(input: { + state: string; + question: string; + options: readonly T[]; + }): Promise<{ pick: T; probs: Record }>; +} + +export interface ModelChoice { + id: string; + kind: "reason" | "decide"; + provider: "rules" | "command" | "ollama"; + why: string; +} + export interface RoleCommandResult { exitCode: number; stdout: string;