diff --git a/backend/scripts/probe_gpt56_cache_rates.py b/backend/scripts/probe_gpt56_cache_rates.py index 744d8f56..a0f303c9 100644 --- a/backend/scripts/probe_gpt56_cache_rates.py +++ b/backend/scripts/probe_gpt56_cache_rates.py @@ -38,7 +38,14 @@ Then, once Cost Explorer has settled (~24h), recover the rates: AWS_PROFILE=dev-ai uv run python scripts/probe_gpt56_cache_rates.py \ - --rates-only --since 2026-09-05 + --rates-only --since 2026-09-05 \ + --table dev-boisestateai-v2-sessions-metadata + +⚠️ Cost Explorer bills these models through AWS Marketplace, under usage types +that name the token bucket and the service tier but NOT the model. Every +OpenAI-family model in the account shares those four rows. A derived rate is +therefore only a given model's rate on a day when it was the ONLY OpenAI-family +model to run — which is what ``--table`` checks and prints. """ from __future__ import annotations @@ -47,6 +54,7 @@ import asyncio import json import os +import re import sys import time from dataclasses import dataclass, field @@ -259,11 +267,90 @@ def print_verdicts(results: List[ArmResult]) -> None: print(f" {key:<24} {ta.get(key,0):>9,} {tb.get(key,0):>9,}") -def derive_rates(since: str, until: Optional[str], region: str) -> int: +# Cost Explorer names no model. OpenAI-family models on Bedrock bill through +# AWS Marketplace, under usage types that carry the token bucket and the +# service tier but NOT the model id — every OpenAI model in the account lands +# in the same four rows. Verified 2026-09-06 against USAGE_TYPE grouped by +# OPERATION and by BILLING_ENTITY; no finer dimension exists. +_MARKETPLACE_TOKEN_USAGE = re.compile( + r"MP:\w+?_(?Pinput_tokens|output_tokens|cache_read_tokens|cache_write_tokens)" + r"_(?P[A-Za-z0-9-]+)-Units$" +) +# The PascalCase twin is the Converse-family (Claude) naming. Matched only so a +# run can SAY it saw them — attributing these to a GPT model is the exact +# mistake this guard exists to prevent. +_CONVERSE_TOKEN_USAGE = re.compile(r"MP:\w+?_(?:Cache(?:Read|Write)Input|Input|Output)TokenCount-Units$") + +_BUCKET_TO_USAGE_KEY = { + "input_tokens": "inputTokens", + "output_tokens": "outputTokens", + "cache_read_tokens": "cacheReadInputTokens", + "cache_write_tokens": "cacheWriteInputTokens", +} + + +def _is_openai_family(model_id: str) -> bool: + lowered = model_id.lower() + return "openai" in lowered or "gpt" in lowered + + +def models_that_ran(table_name: str, region: str, since: str, until: str) -> Dict[str, Dict[str, float]]: + """Per-model token totals we recorded, for the same window. + + This is the attribution guard. Cost Explorer cannot say which model spent + the money, so a derived rate is only trustworthy on a day when exactly one + OpenAI-family model ran. + """ + import boto3 + + table = boto3.resource("dynamodb", region_name=region).Table(table_name) + totals: Dict[str, Dict[str, float]] = {} + kwargs: Dict[str, Any] = { + "FilterExpression": "begins_with(GSI_SK, :c) AND #ts BETWEEN :s AND :u", + "ExpressionAttributeValues": {":c": "C#", ":s": since, ":u": until}, + "ExpressionAttributeNames": {"#ts": "timestamp"}, + "ProjectionExpression": "modelInfo, tokenUsage", + } + start_key = None + while True: + if start_key: + kwargs["ExclusiveStartKey"] = start_key + response = table.scan(**kwargs) + for item in response.get("Items", []): + info = item.get("modelInfo") or {} + model_id = info.get("modelId") or info.get("model") or "(unknown)" + usage = item.get("tokenUsage") or {} + bucket = totals.setdefault(model_id, {"calls": 0.0}) + bucket["calls"] += 1 + for key in _BUCKET_TO_USAGE_KEY.values(): + try: + bucket[key] = bucket.get(key, 0.0) + float(usage.get(key) or 0) + except (TypeError, ValueError): + continue + start_key = response.get("LastEvaluatedKey") + if not start_key: + break + return totals + + +def derive_rates( + since: str, + until: Optional[str], + region: str, + table_name: Optional[str] = None, +) -> int: """Recover $/MTok per bucket from Cost Explorer usage + cost. - Rate = unblended cost / usage quantity, per usage type. This is the step the - Price List API cannot supply for these models. + Rate = unblended cost / usage quantity, per usage type. The Price List API + cannot supply this: there is no Marketplace service code in it at all + (checked 2026-09-06 — 269 service codes, none for Marketplace), and the + four Bedrock codes carry no commercial GPT-5.6 rows. + + The unit is read from Cost Explorer's own ``Unit`` field rather than + assumed. Marketplace token rows report ``1M tokens``; the natively-billed + Bedrock rows (Nova, Titan, Mantle-served models) report ``1K tokens``. An + earlier version of this function assumed 1K for everything, which + overstated every Marketplace rate by 1000x. """ import boto3 @@ -273,43 +360,113 @@ def derive_rates(since: str, until: Optional[str], region: str) -> int: ce = boto3.client("ce", region_name="us-east-1") resp = ce.get_cost_and_usage( TimePeriod={"Start": since, "End": end}, - Granularity="MONTHLY", + Granularity="DAILY", Metrics=["UnblendedCost", "UsageQuantity"], - Filter={"Dimensions": {"Key": "SERVICE", "Values": ["Amazon Bedrock"]}}, GroupBy=[{"Type": "DIMENSION", "Key": "USAGE_TYPE"}], ) - rows = [] + per_day: Dict[str, List[Dict[str, Any]]] = {} + converse_days: Dict[str, float] = {} for period in resp.get("ResultsByTime", []): + day = period["TimePeriod"]["Start"] for group in period.get("Groups", []): usage_type = group["Keys"][0] - if "gpt-5.6" not in usage_type.lower(): - continue cost = float(group["Metrics"]["UnblendedCost"]["Amount"]) qty = float(group["Metrics"]["UsageQuantity"]["Amount"]) - rows.append((usage_type, qty, cost, (cost / qty) if qty else None)) - - if not rows: + if _CONVERSE_TOKEN_USAGE.search(usage_type): + converse_days[day] = converse_days.get(day, 0.0) + cost + continue + match = _MARKETPLACE_TOKEN_USAGE.search(usage_type) + if not match or not qty: + continue + unit = group["Metrics"]["UsageQuantity"].get("Unit", "") + per_mtok = _to_per_mtok(cost / qty, unit) + per_day.setdefault(day, []).append({ + "bucket": match.group("bucket"), + "tier": match.group("tier"), + "usage_type": usage_type, + "qty": qty, + "unit": unit, + "cost": cost, + "per_mtok": per_mtok, + }) + + if not per_day: print( - f"No GPT-5.6 usage types in Cost Explorer for {since}..{end}.\n" - "Cost Explorer lags ~24h — if the turns ran today, try again tomorrow." + f"No Marketplace token usage types in Cost Explorer for {since}..{end}.\n" + "Marketplace line items settle slower than native AWS ones — allow " + "24-48h, not 24h." ) return 1 - print(f"\nGPT-5.6 usage, {since}..{end}") - print(f"{'usage type':<62} {'qty':>14} {'cost USD':>10} {'$/MTok':>10}") - for usage_type, qty, cost, rate in sorted(rows): - # Cost Explorer reports Bedrock token usage in units of 1K tokens. - per_mtok = f"{rate * 1000:.4f}" if rate is not None else "n/a" - print(f"{usage_type:<62} {qty:>14,.0f} {cost:>10.4f} {per_mtok:>10}") + for day in sorted(per_day): + print(f"\n▸ {day}") + attribution = _print_attribution(table_name, region, day) + print(f" {'bucket':<20}{'tier':<10}{'unit':>12}{'qty':>14}{'cost USD':>11}{'$/MTok':>11}") + for row in sorted(per_day[day], key=lambda r: r["bucket"]): + rate = f"{row['per_mtok']:.4f}" if row["per_mtok"] is not None else "unit?" + print( + f" {row['bucket']:<20}{row['tier']:<10}{row['unit']:>12}" + f"{row['qty']:>14,.6f}{row['cost']:>11.4f}{rate:>11}" + ) + if converse_days.get(day): + print( + f" (also ${converse_days[day]:.4f} of Converse-family " + "*TokenCount rows that day — Claude, not GPT; excluded)" + ) + if attribution is False: + print( + " ⚠️ NOT ATTRIBUTABLE. More than one OpenAI-family model ran " + "this day and Cost Explorer does not break the buckets down by " + "model. Re-run the probe on a day when only one model runs." + ) + print( - "\n⚠️ Confirm the usage UNIT before trusting $/MTok: the column above " - "assumes Cost Explorer reports these in 1K-token units. Cross-check one " - "row against the token totals this script printed when it ran the turns." + "\nMethod: these rows are model-agnostic, so a rate is only a given " + "model's rate on a day when that model was the only OpenAI-family " + "model to run. Check the attribution line above before using a number." ) return 0 +def _to_per_mtok(rate_per_unit: float, unit: str) -> Optional[float]: + """Convert a $/unit rate to $/MTok using Cost Explorer's declared unit.""" + normalized = (unit or "").strip().lower() + if normalized in ("1m tokens", "1m token"): + return rate_per_unit + if normalized in ("1k tokens", "1k token"): + return rate_per_unit * 1000 + if normalized in ("tokens", "token"): + return rate_per_unit * 1_000_000 + return None + + +def _print_attribution(table_name: Optional[str], region: str, day: str) -> Optional[bool]: + """Print which models we recorded that day. Returns False if ambiguous.""" + if not table_name: + print(" models that ran: unknown (pass --table to attribute)") + return None + from datetime import date, timedelta + + nxt = (date.fromisoformat(day) + timedelta(days=1)).isoformat() + try: + ran = models_that_ran(table_name, region, day, nxt) + except Exception as exc: # noqa: BLE001 - diagnostic only + print(f" models that ran: lookup failed ({exc})") + return None + + openai_models = sorted(m for m in ran if _is_openai_family(m)) + others = sorted(m for m in ran if not _is_openai_family(m)) + if not openai_models: + print(" models that ran: no OpenAI-family model recorded " + "(usage may be from a direct-transport probe, which is not recorded)") + else: + print(f" models that ran: {', '.join(openai_models)}") + if others: + print(f" (also non-OpenAI: {', '.join(others)} — billed separately)") + return len(openai_models) == 1 + + async def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model-id", default="us.openai.gpt-5.6-sol") @@ -354,13 +511,19 @@ async def main() -> int: help="Skip the turns; just read Cost Explorer and derive rates.", ) parser.add_argument("--since", default=None, help="YYYY-MM-DD for --rates-only") + parser.add_argument( + "--table", + default=None, + help="sessions-metadata table, for the --rates-only attribution guard " + "(e.g. dev-boisestateai-v2-sessions-metadata).", + ) parser.add_argument("--until", default=None) args = parser.parse_args() if args.rates_only: if not args.since: parser.error("--rates-only requires --since YYYY-MM-DD") - return derive_rates(args.since, args.until, args.region) + return derive_rates(args.since, args.until, args.region, args.table) modes = ["explicit", "implicit"] if args.mode == "both" else [args.mode] approx_calls = args.turns * len(modes) diff --git a/docs/specs/gpt-5-6-prompt-caching.md b/docs/specs/gpt-5-6-prompt-caching.md index 0965ae43..b5cc9805 100644 --- a/docs/specs/gpt-5-6-prompt-caching.md +++ b/docs/specs/gpt-5-6-prompt-caching.md @@ -1,6 +1,6 @@ # Plan: prompt caching for OpenAI GPT-5.6 on Bedrock -**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) remains BLOCKED — no commercial rates in the Price List API — so dollar figures are provisional. +**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) is UNBLOCKED as of 2026-09-06: the rates are published on each model's card in the Bedrock User Guide — they are absent from the pricing APIs, which is what the earlier BLOCKED finding was actually measuring. All three dev GPT-5.6 rows were wrong and over-charged by exactly 20% (Terra and Luna carried GovCloud rates; Sol's output came from an inferred 6x ratio that is really 5x); corrected 2026-09-06T15:59Z. The tier/long-context modelling gap is resolved too: Priority and Flex are not supported for these models, and `maxInputTokens: 272000` pins us inside short-context pricing. **Author:** (drafted with Claude) **Date:** 2026-09-04 **Related:** `agents/main_agent/core/model_config.py`, `agents/main_agent/core/agent_factory.py`, @@ -236,6 +236,20 @@ API-key `/chat/api-converse` handler. Don't fork the build logic. ### PR-3 — Catalog entry and pricing +> **SHIPPED 2026-09-06.** `CURATED_BEDROCK_RESPONSES_MODELS` carries Sol, Terra +> and Luna at their published Geo CRIS short-context rates, behind a new +> "Bedrock Responses" catalog tab. `supportsCaching` is pinned true and +> `maxInputTokens` to 272,000 by test, because both are pricing correctness +> rather than preference. The curated `openai.gpt-5.4` Mantle entry was fixed in +> the same pass — it inherited `mantleDefaults()`' `supportsCaching: false` and +> so one-click-created exactly the mis-priced row that had to be repaired by +> hand in prod. `supportedParams` is deliberately omitted: AWS publishes no +> parameter table for GPT-5.6, and a declared spec flips the #915 guard +> restrictive, so a guess would silently block valid params. +> +> The original plan below is kept for the reasoning; where it conflicts with +> the RESOLVED section, the RESOLVED section is right. + - Add GPT-5.6 to `CURATED_MANTLE_MODELS`' sibling set (or a new `CURATED_RUNTIME_OPENAI_MODELS` if the transport field warrants a separate tab). - `supportsCaching: true`, plus verified `cacheReadPricePerMillionTokens` and @@ -247,7 +261,79 @@ API-key `/chat/api-converse` handler. Don't fork the build logic. $0 rather than inventing waste. - Re-verify every rate against the Price List API, per the ⚠️ above. -#### ⛔ BLOCKED — the Price List API does not publish these rates +#### ✅ RESOLVED 2026-09-06 — the rates were published all along, in the model cards + +Everything below this section was written while looking for these rates in +*pricing APIs*. They are not there, and that finding stands. But they are +published, in prose, on each model's card in the Bedrock User Guide: + +- [GPT-5.6 Sol](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html) +- [GPT-5.6 Terra](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html) +- [GPT-5.6 Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html) +- [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) + +**Published rates, Geo CRIS, Short Context (272K)** — Geo CRIS is the row that +applies to the `us.*` inference profiles we actually call. All figures $/MTok. + +| model | input | 30m cache write | cache read | output | +|---|---:|---:|---:|---:| +| `us.openai.gpt-5.6-sol` | 4.40 | 5.50 | 0.44 | 22.00 | +| `us.openai.gpt-5.6-terra` | 2.20 | 2.75 | 0.22 | 13.20 | +| `us.openai.gpt-5.6-luna` | 0.22 | 0.275 | 0.022 | 1.32 | +| `openai.gpt-5.4` (Mantle, In-Region) | 2.75 | — (no write fee) | 0.275 | 16.50 | + +**Every dev row was wrong, and every error over-charged by exactly 20%.** +Corrected in the dev catalog 2026-09-06T15:59Z: + +| model | field | was | now | +|---|---|---:|---:| +| sol | output | 26.40 | **22.00** | +| terra | input / output / cache read / cache write | 2.64 / 15.84 / 0.264 / 3.30 | **2.20 / 13.20 / 0.22 / 2.75** | +| luna | input / output / cache read / cache write | 0.264 / 1.584 / 0.0264 / 0.33 | **0.22 / 1.32 / 0.022 / 0.275** | + +The `1.2x` is not a coincidence: Terra and Luna had been sourced wholesale from +the **GovCloud** Price List rows, which are exactly 1.2x commercial. Sol's +output was the one number with no source at all — it came from a `6x` input +ratio inferred from GovCloud, and the real ratio is `5x`. `openai.gpt-5.4` was +already correct, including the empty cache-write cell. + +**The modelling gap is resolved, not merely downgraded.** + +- **Service tiers do not apply.** Every card states it outright: *"Priority and + Flex tiers are not supported for this model."* Only Standard exists, so the + 0.5x/2x tier dimension `CuratedModel` cannot represent is not a dimension for + these models at all. +- **Long context is real, and the earlier "2x twin" was wrong.** The threshold + is 272K (the models' own window is 1M). Above it, **input is 2x but output is + only 1.5x** — Sol 4.40→8.80 and 22.00→33.00, and the same 2x/1.5x split holds + for Terra and Luna. A flat 2x assumption would have over-priced long-context + output by a third. +- **We do not reach it.** All four rows carry `maxInputTokens: 272000`, pinned + at the short-context boundary, and compaction runs at 100K. Short-context + rates are therefore the correct single rate for our traffic, and the cap is + what keeps that true — do not raise it without also modelling the second + price card. + +**For prod: use Global CRIS.** `global.openai.gpt-5.6-*` prices **9.1% below** +the `us.*` Geo CRIS rates across every bucket (Sol 4.00 / 5.00 / 0.40 / 20.00). +Prod already runs Claude on `global.*`; dev cannot, because of the dev-only SCP. +So the prod rows should be `global.*` ids with the Global CRIS rate card — not +copies of the dev rows. + +**What this means for the empirical work below.** It is no longer the source of +truth, but it is not wasted: it is now the *audit* of these published numbers, +and the tooling fixes it produced (the 1000x unit bug, daily-vs-monthly, the +attribution guard) are what make that audit trustworthy. The 2026-09-06 Sol +window still reads on schedule; it should now reproduce 4.40 / 0.44 / 5.50 / +22.00 rather than discover them. + +**Process lesson.** The search was run entirely against pricing *APIs* — Price +List, then Marketplace Catalog — and concluded "no source exists" without ever +checking the model's own documentation page. Check the model card first; it is +the primary source for rates, caching support, context windows, service tiers, +and endpoint/API support, and it is where AWS documents all of them together. + +#### ⛔ SUPERSEDED — the Price List API does not publish these rates Checked 2026-09-05 against dev-ai (490617140655) with SSO credentials, across every Bedrock service code: @@ -276,6 +362,96 @@ Options when it is picked back up, in preference order: 3. Ship model-card rates explicitly labelled unverified, in code and in the PR. Last resort: these rows price real spend against faculty quotas. + +#### 2026-09-06 — Option 2 attempted: what Cost Explorer can and cannot say + +Option 2 was run against dev-ai. It is viable, but only under a constraint the +plan above did not anticipate, and it closed off Option 1 in the process. + +⚠️ **The conclusion this paragraph originally drew was wrong — see the RESOLVED +section above.** What holds: these models bill through **AWS Marketplace**, the +Price List API has no Marketplace service code (all 269 enumerated 2026-09-06), +and the Marketplace Catalog API is seller-side and returns nothing for a +subscriber. What does not hold is the inference drawn from that — "therefore no +source exists, derive them empirically." The rates were published in the model +cards the whole time. Absence from an API is not absence from the docs. + +**Cost Explorer has the dollars, but names no model.** Usage types look like +`USW2-MP:USW2_cache_read_tokens_standard-Units` — they carry the token bucket +and the service tier, never the model id. Every OpenAI-family model in the +account shares the same four rows. There is no finer dimension: checked +`USAGE_TYPE` grouped by `OPERATION` (all `InvokeModelStreamingInference`) and +by `BILLING_ENTITY` (all `AWS Marketplace`). + +**Therefore a rate is attributable only on a single-model day.** That is the +method, and it works — dev has near-zero organic OpenAI traffic, so clean days +are easy to claim. `probe_gpt56_cache_rates.py --rates-only --table ` now prints which models we recorded that day and refuses to vouch for a +number when more than one OpenAI-family model ran. + +Two traps, both hit and both now guarded in the script: + +- **Read it DAILY, not monthly.** Daily rows come back as exact round numbers; + a multi-day window silently blends models into a meaningless average. August + shows two distinct price cards — `$5.50 / $27.50` and `$2.20 / $11.00` — and + 2026-08-31 is visibly a blend of the two (`$4.3780` input). A monthly read + would have reported that blend as if it were a rate. +- **The unit is `1M tokens`, not `1K`.** Cost Explorer declares it in the + `Unit` field, and it differs by billing path: Marketplace rows are `1M + tokens`, natively-billed rows (Nova, Titan, Mantle-served models) are `1K + tokens`. The script previously assumed 1K for everything, which overstated + every Marketplace rate by 1000x. It now reads the declared unit. + +**The cache ratios hold, independently confirmed.** On every clean day, in both +price cards, cache read is exactly `0.1x` input and cache write exactly +`1.25x`. This is commercial-region billing data, and it corroborates the +GovCloud ratio finding below from a completely different source. + +**Bearing on the modelling gap.** Every row observed is `_standard` and no +`-long-ctx` usage type has ever appeared in this account. The model cards +explain why: Priority and Flex are *not supported* for these models, so +`_standard` is the only tier that can appear; and our `maxInputTokens: 272000` +keeps every request inside the short-context price card. The billing data and +the cards agree. + +**Controlled window claimed: 2026-09-06, `us.openai.gpt-5.6-sol` only.** Dev +had zero recorded model calls that day before the probe. Expected totals, to be +divided into that day's Cost Explorer dollars: + +| bucket | tokens | +|---|---| +| `inputTokens` | 2,660 | +| `cacheReadInputTokens` | 17,496 | +| `cacheWriteInputTokens` | 5,868 | +| `outputTokens` | 50 | + +Two arms produced these: an 8,000-token prefix over 4 turns for the cache +buckets, and a 600-token prefix over 6 turns — deliberately under the +1024-token minimum cacheable prefix, so nothing caches and input tokens +accumulate as the rate anchor. Read it once Marketplace settles (allow 24-48h, +not 24h) and Terra and Luna need their own single-model days. + +**What the read was expected to settle.** The dev rows carried, at the time this was written (all since corrected — see the RESOLVED section above): + +| model | input | output | cache read | cache write | +|---|---:|---:|---:|---:| +| `us.openai.gpt-5.6-sol` | 4.40 | 26.40 | 0.44 | 5.50 | +| `us.openai.gpt-5.6-terra` | 2.64 | 15.84 | 0.264 | 3.30 | +| `us.openai.gpt-5.6-luna` | 0.264 | 1.584 | 0.0264 | 0.33 | + +The cache columns are not the risk — they are `0.1x` and `1.25x` of input, now +confirmed from two independent sources. The risk is concentrated in two places: + +- **Every output rate is a guess.** They are `6x` input, a ratio taken from the + GovCloud Terra row. Commercial daily billing shows both price cards running + at `1:5`, not `1:6` — so if GPT-5.6 also prices at `1:5`, Sol's output rate is + overstated by 20%. Output is the largest per-token number in each row. +- **Sol's input rate has no source at all.** Terra and Luna at least descend + from GovCloud Price List rows; `sol` is absent from the Price List entirely, + so `4.40` came from the model card. + +Both were settled by the model cards instead (Sol output is `22.00`, not `26.40`; every Terra and Luna figure was a GovCloud rate). The single-model day now serves as an audit of the published numbers rather than the source of them. + What the GovCloud rows *do* establish, and can be relied on: - The **ratios are exact**. Terra standard: input `2.64`, cache read `0.264` diff --git a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts index e5d9448a..aa83ecc8 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts @@ -7,7 +7,11 @@ import { ModelCatalogPage } from './model-catalog.page'; import { ManagedModelsService } from './services/managed-models.service'; import { CuratedModelPrefillService } from './services/curated-model-prefill.service'; import { AddCuratedModelDialogComponent } from './components/add-curated-model-dialog.component'; -import { CURATED_BEDROCK_MODELS, CURATED_MANTLE_MODELS } from './models/curated-models'; +import { + CURATED_BEDROCK_MODELS, + CURATED_BEDROCK_RESPONSES_MODELS, + CURATED_MANTLE_MODELS, +} from './models/curated-models'; function createMockManagedModelsService(overrides: Partial<{ isModelAdded: (modelId: string) => boolean; @@ -239,15 +243,19 @@ describe('ModelCatalogPage', () => { // named a `us.*` (Regional/CRIS) inference profile, which prices ~10% higher. // Nothing failed — the numbers were merely wrong, everywhere downstream. describe('curated Bedrock pricing', () => { + // The CRIS tier a model id resolves to drives its rate card, so the two + // must agree on every list that declares a tier — not just Bedrock's. + const tieredModels = [...CURATED_BEDROCK_MODELS, ...CURATED_BEDROCK_RESPONSES_MODELS]; + it('declares a pricingTier that matches the tier its modelId names', () => { - for (const model of CURATED_BEDROCK_MODELS) { + for (const model of tieredModels) { const expected = model.template.modelId.startsWith('global.') ? 'global' : 'regional'; expect(`${model.key}:${model.pricingTier}`).toBe(`${model.key}:${expected}`); } }); it('derives cache rates from base input at Bedrock\'s published multipliers', () => { - for (const model of CURATED_BEDROCK_MODELS) { + for (const model of tieredModels) { const t = model.template; if (!t.supportsCaching) continue; const input = t.inputPricePerMillionTokens; @@ -256,4 +264,63 @@ describe('ModelCatalogPage', () => { } }); }); + + describe('curated bedrock-responses (GPT-5.6) entries', () => { + it('renders them on their own tab', () => { + const page = createComponent(); + page.selectTab('bedrock-responses'); + + expect(page.visibleModels().map(m => m.key)).toEqual( + CURATED_BEDROCK_RESPONSES_MODELS.map(m => m.key), + ); + expect(CURATED_BEDROCK_RESPONSES_MODELS.length).toBeGreaterThan(0); + }); + + it('never ships supportsCaching false — the provider forces it true', () => { + // `false` here is not a preference but a false statement: these models + // cache implicitly server-side and it cannot be turned off. Its only + // effect would be to clear the cache rates, pricing cached tokens at + // $0.00 while AWS bills them in full. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.supportsCaching}`).toBe(`${model.key}:true`); + } + }); + + it('pins maxInputTokens to the 272K short-context boundary', () => { + // Load-bearing pricing, not just a cap: above 272K these models bill + // input at 2x and output at 1.5x, and a CuratedModel holds one flat rate + // per bucket. Raising this silently opens the second price card. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.maxInputTokens}`).toBe(`${model.key}:272000`); + } + }); + + it('routes over the Responses API, which is the only surface that caches', () => { + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.apiMode}`).toBe(`${model.key}:responses`); + expect(`${model.key}:${model.template.provider}`).toBe(`${model.key}:bedrock-responses`); + } + }); + + it('declares no supportedParams rather than an invented one', () => { + // AWS publishes no parameter table for GPT-5.6. A declared spec flips the + // #915 guard from permissive to restrictive, so a guessed one would + // silently block params the model actually accepts. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(model.template.supportedParams ?? null).toBeNull(); + } + }); + }); + + it('curates GPT-5.4 on Mantle with caching on and no write fee', () => { + // Its model card publishes a cache-read rate with an em dash for cache + // write. Inheriting mantleDefaults()' supportsCaching:false priced its + // cached tokens at $0.00 while AWS billed them — the bug that had to be + // fixed by hand in prod. + const gpt54 = CURATED_MANTLE_MODELS.find(m => m.key === 'gpt-5-4'); + + expect(gpt54?.template.supportsCaching).toBe(true); + expect(gpt54?.template.cacheReadPricePerMillionTokens).toBeCloseTo(0.275, 6); + expect(gpt54?.template.cacheWritePricePerMillionTokens).toBe(0); + }); }); diff --git a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts index cf7734cd..2849f8ac 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts @@ -31,6 +31,10 @@ interface ProviderTab { const PROVIDER_TABS: ProviderTab[] = [ { id: 'bedrock', label: 'Bedrock' }, { id: 'mantle', label: 'Bedrock Mantle' }, + // The OpenAI Responses API on bedrock-runtime. A separate tab from 'openai' + // (which is direct-to-OpenAI and unpopulated) because the transport, not the + // vendor, is what differs — and it is the only surface where GPT-5.6 caches. + { id: 'bedrock-responses', label: 'Bedrock Responses' }, { id: 'openai', label: 'OpenAI' }, { id: 'gemini', label: 'Gemini' }, ]; diff --git a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts index 4d5a073b..9739d283 100644 --- a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts +++ b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts @@ -5,9 +5,21 @@ import { ManagedModelFormData, ModelProvider } from './managed-model.model'; * one-click create a fully-configured managed model — including pricing and * per-param specs — plus a small amount of presentation metadata for the card. * - * NOTE — Claude rates below were read from the **AWS Price List API** - * (`AmazonBedrockFoundationModels`, us-west-2, published 2026-09-01), not the - * pricing page. Re-verify there when bumping a model id: + * NOTE — **the model card is the primary source for rates.** Each model's page + * in the Bedrock User Guide publishes its full rate card (per inference option + * and context window) alongside caching support, context windows, service + * tiers and endpoint support: + * + * docs.aws.amazon.com/bedrock/latest/userguide/model-card--.html + * + * Check it first. The OpenAI-family rates are absent from the Price List API + * entirely — those models bill through AWS Marketplace, which no pricing API + * covers — and reading that absence as "unpublished" put three rows into the + * dev catalog at GovCloud prices, over-charging by 20%. + * + * Claude rates below were read from the **AWS Price List API** + * (`AmazonBedrockFoundationModels`, us-west-2, published 2026-09-01), which + * does carry them. Re-verify there when bumping a model id: * * aws pricing get-products --region us-east-1 \ * --service-code AmazonBedrockFoundationModels \ @@ -66,17 +78,22 @@ const claude4xDefaults = (): Pick< }); /** - * Bedrock publishes a Claude model's cache rates as fixed multiples of its base - * input rate: 5-minute cache write is **1.25x**, cache read is **0.1x** (the - * 1-hour write we do not use is 2x). Deriving them removes the two fields most - * likely to drift — the ratios were the one thing the old table got right. + * Bedrock publishes cache rates as fixed multiples of a model's base input + * rate: cache write is **1.25x**, cache read is **0.1x** (the 1-hour Claude + * write we do not use is 2x). Deriving them removes the two fields most likely + * to drift — the ratios were the one thing the old table got right. + * + * Not Claude-specific: the GPT-5.6 model cards publish exactly the same two + * multipliers (Sol 4.40 -> 5.50 / 0.44), and commercial Cost Explorer billing + * reproduces them to four decimals on every clean day. Two model families, two + * independent sources, same ratios. * * `input` and `output` are the only independently published numbers, and both * are TIER-SPECIFIC. Pass the rates for the tier the `modelId` names, and set * `pricingTier` to match; a `us.*` id costs ~10% more than the `global.*` rates * for the same model, which is exactly how the two drifted apart before. */ -const claudeRates = ( +const ratesWithDerivedCache = ( input: number, output: number, ): Pick< @@ -110,7 +127,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ modelName: 'Claude Opus 4.7', maxOutputTokens: 64_000, // Regional (CRIS): $5.50 / $27.50. Global is $5.00 / $25.00. - ...claudeRates(5.5, 27.5), + ...ratesWithDerivedCache(5.5, 27.5), knowledgeCutoffDate: '2025-10-01', supportedParams: { params: { @@ -137,7 +154,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ maxOutputTokens: 128_000, // Global: $2.00 / $10.00 — correct as declared, this id really is // `global.*`. Regional would be $2.20 / $11.00. - ...claudeRates(2.0, 10.0), + ...ratesWithDerivedCache(2.0, 10.0), knowledgeCutoffDate: null, supportedParams: { params: { @@ -162,7 +179,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ modelName: 'Claude Sonnet 4.6', maxOutputTokens: 64_000, // Regional (CRIS): $3.30 / $16.50. Global is $3.00 / $15.00. - ...claudeRates(3.3, 16.5), + ...ratesWithDerivedCache(3.3, 16.5), knowledgeCutoffDate: '2025-07-01', supportedParams: { params: { @@ -187,7 +204,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ maxOutputTokens: 64_000, // Regional (CRIS): $1.10 / $5.50. Global is $1.00 / $5.00. This is the // platform default model, so this is the row every cost number rides on. - ...claudeRates(1.1, 5.5), + ...ratesWithDerivedCache(1.1, 5.5), knowledgeCutoffDate: '2025-02-01', supportedParams: { params: { @@ -205,10 +222,13 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ /** * Shared defaults for Bedrock Mantle (OpenAI-compatible open-weight) models. * - * Caching is intentionally absent: prompt caching on Bedrock is model-bound - * to Anthropic Claude + a small set of Amazon Nova models, none of which run - * through the Mantle provider, so these never cache and carry no cache - * pricing. `apiMode` (Chat Completions vs Responses) and an optional `region` + * `supportsCaching: false` is the right DEFAULT here — most Mantle models are + * open-weight and genuinely never cache — but it is not universal, so any + * entry for a model that does cache must override it. `openai.gpt-5.4` is the + * one below that does: its model card publishes a cache-read rate (0.1x input) + * with no write fee. Inheriting the default there priced its cached tokens at + * $0.00 while AWS billed them, which is exactly the bug that had to be fixed + * by hand in prod. `apiMode` (Chat Completions vs Responses) and an optional `region` * are the Mantle-specific fields — sourced from each model card (there is no * API that exposes them). The base path is derived by the SDK from the model id. */ @@ -255,8 +275,16 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ // _OPENAI_PATH_MODEL_PREFIXES, so one-click create routes correctly // (unlike the Gemma case noted below). apiMode: 'responses', + // Model card, In-Region: $2.75 / $16.50, cache read $0.275 (0.1x input), + // and the cache-write cell is an em dash — there is NO write fee on this + // model. A literal 0 is the correct rate, not a missing value: it makes + // `compute_wasted_usd` see a non-positive premium and return $0 instead + // of inventing waste. inputPricePerMillionTokens: 2.75, outputPricePerMillionTokens: 16.5, + supportsCaching: true, + cacheReadPricePerMillionTokens: 0.275, + cacheWritePricePerMillionTokens: 0, }, }, { @@ -293,22 +321,129 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ // Use the `google.gemma-4-` prefix, NOT `google.gemma-`: Gemma 3 is on `/v1`. ]; + /** - * Provider-keyed lookup for the catalog tabs. Bedrock + Mantle are populated; - * OpenAI/Gemini are intentional empty arrays — the page renders a - * 'Coming soon' empty state when the active tab has no entries. + * Shared defaults for `bedrock-responses` — the OpenAI **Responses** API on + * `bedrock-runtime`. + * + * `supportsCaching: true` is not a preference here, it is the only truthful + * value. These models cache implicitly and server-side with no way to turn it + * off, so `false` would be a false statement whose only effect is to clear the + * cache-rate fields — pricing cached tokens at $0.00 while AWS bills them in + * full. On a warm conversation nearly every input token is a cached one, so + * that is close to total under-reporting. The backend normalizes it the same + * way (`_resolve_supports_caching`, forced for this provider), as it does + * `apiMode: 'responses'`. * - * `bedrock-responses` is empty for now: the transport landed ahead of its - * catalog rows, whose rates have to be verified against the Price List API - * first. Adding a model there today means the escape-hatch form, which works. - * NOTE for whoever curates it: `mantleDefaults()` hardcodes - * `supportsCaching: false` — a GPT-5.6 row must not inherit that, since - * caching is the entire reason this transport exists. + * `maxInputTokens: 272_000` is **load-bearing pricing**, not just a cap. These + * models have a 1M window, but AWS prices them on two cards: above 272K, input + * costs 2x and output 1.5x. `CuratedModel` holds one flat rate per bucket, so + * the cap is what keeps that single rate correct. Raising it silently opens + * the second price card and under-charges every long turn. + */ +const bedrockResponsesDefaults = (): Pick< + ManagedModelFormData, + | 'provider' + | 'providerName' + | 'inputModalities' + | 'outputModalities' + | 'responseStreamingSupported' + | 'maxInputTokens' + | 'maxOutputTokens' + | 'allowedAppRoles' + | 'availableToRoles' + | 'enabled' + | 'isDefault' + | 'supportsCaching' + | 'apiMode' +> => ({ + provider: 'bedrock-responses', + providerName: 'OpenAI', + inputModalities: ['TEXT', 'IMAGE'], + outputModalities: ['TEXT'], + responseStreamingSupported: true, + maxInputTokens: 272_000, + // The cards publish no output cap ("Max output tokens: N/A"), so claim none + // rather than invent one — this value is only a ceiling on the configured + // max_tokens param and is never sent to the provider. + maxOutputTokens: null, + allowedAppRoles: [], + availableToRoles: [], + enabled: true, + isDefault: false, + supportsCaching: true, + apiMode: 'responses', +}); + +/** + * GPT-5.6 on `bedrock-runtime` via the Responses API. + * + * Rates are the **Geo CRIS, Short Context (272K)** row from each model card — + * Geo CRIS is the tier the `us.*` inference profiles resolve to, and these + * models are inference-profile-only (no ON_DEMAND). Verified 2026-09-06. + * + * `supportedParams` is deliberately absent. AWS publishes no parameter table + * for these models (`model-parameters-openai.html` documents only the + * open-weight gpt-oss family), and an invented spec would be worse than none: + * a declared spec flips the #915 guard from permissive to restrictive, so a + * wrong entry silently blocks a parameter the model actually accepts. Add one + * only from published or measured evidence. + */ +export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ + { + key: 'gpt-5-6-sol', + tagline: 'OpenAI\'s most capable model — frontier reasoning and agentic work.', + capabilities: ['Reasoning', 'Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-sol', + modelName: 'GPT-5.6 Sol', + // Geo CRIS: $4.40 / $22.00. Global CRIS is $4.00 / $20.00. + ...ratesWithDerivedCache(4.4, 22.0), + knowledgeCutoffDate: null, + }, + }, + { + key: 'gpt-5-6-terra', + tagline: 'Balanced everyday model — strong performance per dollar.', + capabilities: ['Reasoning', 'Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-terra', + modelName: 'GPT-5.6 Terra', + // Geo CRIS: $2.20 / $13.20. Global CRIS is $2.00 / $12.00. + ...ratesWithDerivedCache(2.2, 13.2), + knowledgeCutoffDate: null, + }, + }, + { + key: 'gpt-5-6-luna', + tagline: 'Fast and affordable — for classification, routing and high volume.', + capabilities: ['Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-luna', + modelName: 'GPT-5.6 Luna', + // Geo CRIS: $0.22 / $1.32. Global CRIS is $0.20 / $1.20. + ...ratesWithDerivedCache(0.22, 1.32), + knowledgeCutoffDate: null, + }, + }, +]; + +/** + * Provider-keyed lookup for the catalog tabs. Bedrock, Mantle and + * bedrock-responses are populated; OpenAI/Gemini are intentional empty arrays + * — the page renders a 'Coming soon' empty state when the active tab has no + * entries. */ export const CURATED_MODELS_BY_PROVIDER: Record = { bedrock: CURATED_BEDROCK_MODELS, openai: [], gemini: [], mantle: CURATED_MANTLE_MODELS, - 'bedrock-responses': [], + 'bedrock-responses': CURATED_BEDROCK_RESPONSES_MODELS, };