From e30c7e3439a5cd667eb8b3c744c3ec567d58988f Mon Sep 17 00:00:00 2001 From: Phil Merrell Date: Sun, 6 Sep 2026 09:43:03 -0600 Subject: [PATCH 1/3] fix(costs): derive GPT-5.6 rates from a single-model day, not a 1000x-wrong blend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--rates-only` could never have produced a usable number. Three defects, all found by actually running it against dev-ai: 1. It filtered usage types on the substring `gpt-5.6`. No usage type contains a model id, so the filter matched nothing and the script reported "Cost Explorer lags ~24h" — a lag message for a search that was never going to match, which is the worst possible failure mode for a tool whose whole job is to answer "have the numbers landed yet?". 2. It multiplied every rate by 1000 to convert from 1K-token units. These models bill through AWS Marketplace in units of **1M tokens**, and Cost Explorer declares the unit in its own `Unit` field. Every derived rate was overstated 1000x. It now reads the declared unit and converts accordingly. 3. It read MONTHLY. Daily rows come back as exact round numbers; a multi-day window silently blends models into an average that looks like a rate. The blend is not hypothetical, and it is why this needed a guard rather than a fix. Marketplace usage types carry the token bucket and the service tier but never the model, so every OpenAI-family model in the account shares the same four rows — verified against USAGE_TYPE grouped by OPERATION and by BILLING_ENTITY; no finer dimension exists. August shows two distinct price cards ($5.50/$27.50 and $2.20/$11.00) and 2026-08-31 is visibly a blend of the two. A rate is therefore only a given model's rate on a day when it was the sole OpenAI-family model to run, so `--table` now reconciles against what we recorded in sessions-metadata and refuses to vouch for a number otherwise. I nearly shipped the mistake this guard prevents: a first read of Aug 20-31 gave a cache-read rate matching gpt-5.4's 0.1x to four decimals, and a reconcile then showed zero GPT calls in that window. The match was coincidence. This also closes off the spec's Option 1. The Price List API has no Marketplace service code at all (all 269 enumerated), and the Marketplace Catalog API is seller-side. These rates are not unpublished-yet; they are unpublishable through any pricing API while they bill this way, so waiting will not produce them. Bearing on the tier/long-context modelling gap PR-3 must resolve: every row ever seen in this account is `_standard` and no `-long-ctx` usage type has appeared, so a flat standard rate is correct for current traffic and a change would show up as a new usage type. That makes the gap monitorable rather than blocking. Controlled window claimed 2026-09-06 for `us.openai.gpt-5.6-sol` (dev had zero recorded calls beforehand); expected token totals are recorded in the spec so the read is a verification rather than a guess. Co-Authored-By: Claude Opus 5 --- backend/scripts/probe_gpt56_cache_rates.py | 211 ++++++++++++++++++--- docs/specs/gpt-5-6-prompt-caching.md | 93 ++++++++- 2 files changed, 279 insertions(+), 25 deletions(-) diff --git a/backend/scripts/probe_gpt56_cache_rates.py b/backend/scripts/probe_gpt56_cache_rates.py index 744d8f56..a0f303c9 100644 --- a/backend/scripts/probe_gpt56_cache_rates.py +++ b/backend/scripts/probe_gpt56_cache_rates.py @@ -38,7 +38,14 @@ Then, once Cost Explorer has settled (~24h), recover the rates: AWS_PROFILE=dev-ai uv run python scripts/probe_gpt56_cache_rates.py \ - --rates-only --since 2026-09-05 + --rates-only --since 2026-09-05 \ + --table dev-boisestateai-v2-sessions-metadata + +⚠️ Cost Explorer bills these models through AWS Marketplace, under usage types +that name the token bucket and the service tier but NOT the model. Every +OpenAI-family model in the account shares those four rows. A derived rate is +therefore only a given model's rate on a day when it was the ONLY OpenAI-family +model to run — which is what ``--table`` checks and prints. """ from __future__ import annotations @@ -47,6 +54,7 @@ import asyncio import json import os +import re import sys import time from dataclasses import dataclass, field @@ -259,11 +267,90 @@ def print_verdicts(results: List[ArmResult]) -> None: print(f" {key:<24} {ta.get(key,0):>9,} {tb.get(key,0):>9,}") -def derive_rates(since: str, until: Optional[str], region: str) -> int: +# Cost Explorer names no model. OpenAI-family models on Bedrock bill through +# AWS Marketplace, under usage types that carry the token bucket and the +# service tier but NOT the model id — every OpenAI model in the account lands +# in the same four rows. Verified 2026-09-06 against USAGE_TYPE grouped by +# OPERATION and by BILLING_ENTITY; no finer dimension exists. +_MARKETPLACE_TOKEN_USAGE = re.compile( + r"MP:\w+?_(?Pinput_tokens|output_tokens|cache_read_tokens|cache_write_tokens)" + r"_(?P[A-Za-z0-9-]+)-Units$" +) +# The PascalCase twin is the Converse-family (Claude) naming. Matched only so a +# run can SAY it saw them — attributing these to a GPT model is the exact +# mistake this guard exists to prevent. +_CONVERSE_TOKEN_USAGE = re.compile(r"MP:\w+?_(?:Cache(?:Read|Write)Input|Input|Output)TokenCount-Units$") + +_BUCKET_TO_USAGE_KEY = { + "input_tokens": "inputTokens", + "output_tokens": "outputTokens", + "cache_read_tokens": "cacheReadInputTokens", + "cache_write_tokens": "cacheWriteInputTokens", +} + + +def _is_openai_family(model_id: str) -> bool: + lowered = model_id.lower() + return "openai" in lowered or "gpt" in lowered + + +def models_that_ran(table_name: str, region: str, since: str, until: str) -> Dict[str, Dict[str, float]]: + """Per-model token totals we recorded, for the same window. + + This is the attribution guard. Cost Explorer cannot say which model spent + the money, so a derived rate is only trustworthy on a day when exactly one + OpenAI-family model ran. + """ + import boto3 + + table = boto3.resource("dynamodb", region_name=region).Table(table_name) + totals: Dict[str, Dict[str, float]] = {} + kwargs: Dict[str, Any] = { + "FilterExpression": "begins_with(GSI_SK, :c) AND #ts BETWEEN :s AND :u", + "ExpressionAttributeValues": {":c": "C#", ":s": since, ":u": until}, + "ExpressionAttributeNames": {"#ts": "timestamp"}, + "ProjectionExpression": "modelInfo, tokenUsage", + } + start_key = None + while True: + if start_key: + kwargs["ExclusiveStartKey"] = start_key + response = table.scan(**kwargs) + for item in response.get("Items", []): + info = item.get("modelInfo") or {} + model_id = info.get("modelId") or info.get("model") or "(unknown)" + usage = item.get("tokenUsage") or {} + bucket = totals.setdefault(model_id, {"calls": 0.0}) + bucket["calls"] += 1 + for key in _BUCKET_TO_USAGE_KEY.values(): + try: + bucket[key] = bucket.get(key, 0.0) + float(usage.get(key) or 0) + except (TypeError, ValueError): + continue + start_key = response.get("LastEvaluatedKey") + if not start_key: + break + return totals + + +def derive_rates( + since: str, + until: Optional[str], + region: str, + table_name: Optional[str] = None, +) -> int: """Recover $/MTok per bucket from Cost Explorer usage + cost. - Rate = unblended cost / usage quantity, per usage type. This is the step the - Price List API cannot supply for these models. + Rate = unblended cost / usage quantity, per usage type. The Price List API + cannot supply this: there is no Marketplace service code in it at all + (checked 2026-09-06 — 269 service codes, none for Marketplace), and the + four Bedrock codes carry no commercial GPT-5.6 rows. + + The unit is read from Cost Explorer's own ``Unit`` field rather than + assumed. Marketplace token rows report ``1M tokens``; the natively-billed + Bedrock rows (Nova, Titan, Mantle-served models) report ``1K tokens``. An + earlier version of this function assumed 1K for everything, which + overstated every Marketplace rate by 1000x. """ import boto3 @@ -273,43 +360,113 @@ def derive_rates(since: str, until: Optional[str], region: str) -> int: ce = boto3.client("ce", region_name="us-east-1") resp = ce.get_cost_and_usage( TimePeriod={"Start": since, "End": end}, - Granularity="MONTHLY", + Granularity="DAILY", Metrics=["UnblendedCost", "UsageQuantity"], - Filter={"Dimensions": {"Key": "SERVICE", "Values": ["Amazon Bedrock"]}}, GroupBy=[{"Type": "DIMENSION", "Key": "USAGE_TYPE"}], ) - rows = [] + per_day: Dict[str, List[Dict[str, Any]]] = {} + converse_days: Dict[str, float] = {} for period in resp.get("ResultsByTime", []): + day = period["TimePeriod"]["Start"] for group in period.get("Groups", []): usage_type = group["Keys"][0] - if "gpt-5.6" not in usage_type.lower(): - continue cost = float(group["Metrics"]["UnblendedCost"]["Amount"]) qty = float(group["Metrics"]["UsageQuantity"]["Amount"]) - rows.append((usage_type, qty, cost, (cost / qty) if qty else None)) - - if not rows: + if _CONVERSE_TOKEN_USAGE.search(usage_type): + converse_days[day] = converse_days.get(day, 0.0) + cost + continue + match = _MARKETPLACE_TOKEN_USAGE.search(usage_type) + if not match or not qty: + continue + unit = group["Metrics"]["UsageQuantity"].get("Unit", "") + per_mtok = _to_per_mtok(cost / qty, unit) + per_day.setdefault(day, []).append({ + "bucket": match.group("bucket"), + "tier": match.group("tier"), + "usage_type": usage_type, + "qty": qty, + "unit": unit, + "cost": cost, + "per_mtok": per_mtok, + }) + + if not per_day: print( - f"No GPT-5.6 usage types in Cost Explorer for {since}..{end}.\n" - "Cost Explorer lags ~24h — if the turns ran today, try again tomorrow." + f"No Marketplace token usage types in Cost Explorer for {since}..{end}.\n" + "Marketplace line items settle slower than native AWS ones — allow " + "24-48h, not 24h." ) return 1 - print(f"\nGPT-5.6 usage, {since}..{end}") - print(f"{'usage type':<62} {'qty':>14} {'cost USD':>10} {'$/MTok':>10}") - for usage_type, qty, cost, rate in sorted(rows): - # Cost Explorer reports Bedrock token usage in units of 1K tokens. - per_mtok = f"{rate * 1000:.4f}" if rate is not None else "n/a" - print(f"{usage_type:<62} {qty:>14,.0f} {cost:>10.4f} {per_mtok:>10}") + for day in sorted(per_day): + print(f"\n▸ {day}") + attribution = _print_attribution(table_name, region, day) + print(f" {'bucket':<20}{'tier':<10}{'unit':>12}{'qty':>14}{'cost USD':>11}{'$/MTok':>11}") + for row in sorted(per_day[day], key=lambda r: r["bucket"]): + rate = f"{row['per_mtok']:.4f}" if row["per_mtok"] is not None else "unit?" + print( + f" {row['bucket']:<20}{row['tier']:<10}{row['unit']:>12}" + f"{row['qty']:>14,.6f}{row['cost']:>11.4f}{rate:>11}" + ) + if converse_days.get(day): + print( + f" (also ${converse_days[day]:.4f} of Converse-family " + "*TokenCount rows that day — Claude, not GPT; excluded)" + ) + if attribution is False: + print( + " ⚠️ NOT ATTRIBUTABLE. More than one OpenAI-family model ran " + "this day and Cost Explorer does not break the buckets down by " + "model. Re-run the probe on a day when only one model runs." + ) + print( - "\n⚠️ Confirm the usage UNIT before trusting $/MTok: the column above " - "assumes Cost Explorer reports these in 1K-token units. Cross-check one " - "row against the token totals this script printed when it ran the turns." + "\nMethod: these rows are model-agnostic, so a rate is only a given " + "model's rate on a day when that model was the only OpenAI-family " + "model to run. Check the attribution line above before using a number." ) return 0 +def _to_per_mtok(rate_per_unit: float, unit: str) -> Optional[float]: + """Convert a $/unit rate to $/MTok using Cost Explorer's declared unit.""" + normalized = (unit or "").strip().lower() + if normalized in ("1m tokens", "1m token"): + return rate_per_unit + if normalized in ("1k tokens", "1k token"): + return rate_per_unit * 1000 + if normalized in ("tokens", "token"): + return rate_per_unit * 1_000_000 + return None + + +def _print_attribution(table_name: Optional[str], region: str, day: str) -> Optional[bool]: + """Print which models we recorded that day. Returns False if ambiguous.""" + if not table_name: + print(" models that ran: unknown (pass --table to attribute)") + return None + from datetime import date, timedelta + + nxt = (date.fromisoformat(day) + timedelta(days=1)).isoformat() + try: + ran = models_that_ran(table_name, region, day, nxt) + except Exception as exc: # noqa: BLE001 - diagnostic only + print(f" models that ran: lookup failed ({exc})") + return None + + openai_models = sorted(m for m in ran if _is_openai_family(m)) + others = sorted(m for m in ran if not _is_openai_family(m)) + if not openai_models: + print(" models that ran: no OpenAI-family model recorded " + "(usage may be from a direct-transport probe, which is not recorded)") + else: + print(f" models that ran: {', '.join(openai_models)}") + if others: + print(f" (also non-OpenAI: {', '.join(others)} — billed separately)") + return len(openai_models) == 1 + + async def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--model-id", default="us.openai.gpt-5.6-sol") @@ -354,13 +511,19 @@ async def main() -> int: help="Skip the turns; just read Cost Explorer and derive rates.", ) parser.add_argument("--since", default=None, help="YYYY-MM-DD for --rates-only") + parser.add_argument( + "--table", + default=None, + help="sessions-metadata table, for the --rates-only attribution guard " + "(e.g. dev-boisestateai-v2-sessions-metadata).", + ) parser.add_argument("--until", default=None) args = parser.parse_args() if args.rates_only: if not args.since: parser.error("--rates-only requires --since YYYY-MM-DD") - return derive_rates(args.since, args.until, args.region) + return derive_rates(args.since, args.until, args.region, args.table) modes = ["explicit", "implicit"] if args.mode == "both" else [args.mode] approx_calls = args.turns * len(modes) diff --git a/docs/specs/gpt-5-6-prompt-caching.md b/docs/specs/gpt-5-6-prompt-caching.md index 0965ae43..ec163bc8 100644 --- a/docs/specs/gpt-5-6-prompt-caching.md +++ b/docs/specs/gpt-5-6-prompt-caching.md @@ -1,6 +1,6 @@ # Plan: prompt caching for OpenAI GPT-5.6 on Bedrock -**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) remains BLOCKED — no commercial rates in the Price List API — so dollar figures are provisional. +**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) remains BLOCKED, but the blocker changed on 2026-09-06: these models bill through AWS Marketplace, which no pricing API covers, so no amount of waiting will publish them. The empirical route is open instead — Cost Explorer carries the dollars but names no model, so rates are only attributable on a single-model day. First controlled window claimed 2026-09-06 (Sol). Dollar figures stay provisional until it is read. **Author:** (drafted with Claude) **Date:** 2026-09-04 **Related:** `agents/main_agent/core/model_config.py`, `agents/main_agent/core/agent_factory.py`, @@ -276,6 +276,97 @@ Options when it is picked back up, in preference order: 3. Ship model-card rates explicitly labelled unverified, in code and in the PR. Last resort: these rows price real spend against faculty quotas. + +#### 2026-09-06 — Option 2 attempted: what Cost Explorer can and cannot say + +Option 2 was run against dev-ai. It is viable, but only under a constraint the +plan above did not anticipate, and it closed off Option 1 in the process. + +**Option 1 is not a waiting game.** These models bill through **AWS +Marketplace**, and the Price List API has no Marketplace service code — all 269 +service codes were enumerated on 2026-09-06 and none covers Marketplace. The +Marketplace Catalog API is seller-side and returns nothing for a subscriber. +So the earlier "not published yet" reading was wrong: there is no pricing API +that *could* carry these rates while they bill this way. Waiting will not +produce them. + +**Cost Explorer has the dollars, but names no model.** Usage types look like +`USW2-MP:USW2_cache_read_tokens_standard-Units` — they carry the token bucket +and the service tier, never the model id. Every OpenAI-family model in the +account shares the same four rows. There is no finer dimension: checked +`USAGE_TYPE` grouped by `OPERATION` (all `InvokeModelStreamingInference`) and +by `BILLING_ENTITY` (all `AWS Marketplace`). + +**Therefore a rate is attributable only on a single-model day.** That is the +method, and it works — dev has near-zero organic OpenAI traffic, so clean days +are easy to claim. `probe_gpt56_cache_rates.py --rates-only --table ` now prints which models we recorded that day and refuses to vouch for a +number when more than one OpenAI-family model ran. + +Two traps, both hit and both now guarded in the script: + +- **Read it DAILY, not monthly.** Daily rows come back as exact round numbers; + a multi-day window silently blends models into a meaningless average. August + shows two distinct price cards — `$5.50 / $27.50` and `$2.20 / $11.00` — and + 2026-08-31 is visibly a blend of the two (`$4.3780` input). A monthly read + would have reported that blend as if it were a rate. +- **The unit is `1M tokens`, not `1K`.** Cost Explorer declares it in the + `Unit` field, and it differs by billing path: Marketplace rows are `1M + tokens`, natively-billed rows (Nova, Titan, Mantle-served models) are `1K + tokens`. The script previously assumed 1K for everything, which overstated + every Marketplace rate by 1000x. It now reads the declared unit. + +**The cache ratios hold, independently confirmed.** On every clean day, in both +price cards, cache read is exactly `0.1x` input and cache write exactly +`1.25x`. This is commercial-region billing data, and it corroborates the +GovCloud ratio finding below from a completely different source. + +**Bearing on the modelling gap.** Every row observed is `_standard`; no +`-long-ctx` usage type has ever appeared in this account. So the 0.5x/2x tier +and 2x long-context dimensions are not currently being billed against us, and a +flat standard rate is correct *for our present traffic*. That downgrades the +gap from "silently mis-prices" to "mis-prices only if traffic changes, and Cost +Explorer will show a new usage type when it does" — which is a monitorable +condition rather than a blocking unknown. + +**Controlled window claimed: 2026-09-06, `us.openai.gpt-5.6-sol` only.** Dev +had zero recorded model calls that day before the probe. Expected totals, to be +divided into that day's Cost Explorer dollars: + +| bucket | tokens | +|---|---| +| `inputTokens` | 2,660 | +| `cacheReadInputTokens` | 17,496 | +| `cacheWriteInputTokens` | 5,868 | +| `outputTokens` | 50 | + +Two arms produced these: an 8,000-token prefix over 4 turns for the cache +buckets, and a 600-token prefix over 6 turns — deliberately under the +1024-token minimum cacheable prefix, so nothing caches and input tokens +accumulate as the rate anchor. Read it once Marketplace settles (allow 24-48h, +not 24h) and Terra and Luna need their own single-model days. + +**What the read will actually settle.** The dev rows currently carry: + +| model | input | output | cache read | cache write | +|---|---:|---:|---:|---:| +| `us.openai.gpt-5.6-sol` | 4.40 | 26.40 | 0.44 | 5.50 | +| `us.openai.gpt-5.6-terra` | 2.64 | 15.84 | 0.264 | 3.30 | +| `us.openai.gpt-5.6-luna` | 0.264 | 1.584 | 0.0264 | 0.33 | + +The cache columns are not the risk — they are `0.1x` and `1.25x` of input, now +confirmed from two independent sources. The risk is concentrated in two places: + +- **Every output rate is a guess.** They are `6x` input, a ratio taken from the + GovCloud Terra row. Commercial daily billing shows both price cards running + at `1:5`, not `1:6` — so if GPT-5.6 also prices at `1:5`, Sol's output rate is + overstated by 20%. Output is the largest per-token number in each row. +- **Sol's input rate has no source at all.** Terra and Luna at least descend + from GovCloud Price List rows; `sol` is absent from the Price List entirely, + so `4.40` came from the model card. + +The single-model day yields input and output directly, which settles both. + What the GovCloud rows *do* establish, and can be relied on: - The **ratios are exact**. Terra standard: input `2.64`, cache read `0.264` From 9f54623932cefa5e609709c75845c595d4a24573 Mon Sep 17 00:00:00 2001 From: Phil Merrell Date: Sun, 6 Sep 2026 10:01:03 -0600 Subject: [PATCH 2/3] =?UTF-8?q?fix(costs):=20correct=20every=20GPT-5.6=20r?= =?UTF-8?q?ate=20=E2=80=94=20they=20were=20published=20in=20the=20model=20?= =?UTF-8?q?cards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I concluded yesterday that these rates existed in no source and had to be derived empirically. That was wrong, and the error was one of scope: the search ran against pricing *APIs* — Price List, then Marketplace Catalog — and stopped there. AWS publishes them in prose on each model's card in the Bedrock User Guide, alongside caching support, context windows, service tiers and endpoint support. Absence from an API is not absence from the docs. Every dev GPT-5.6 row was wrong, and every error over-charged by exactly 20% (corrected in the dev catalog 2026-09-06T15:59Z): sol output 26.40 -> 22.00 terra in/out/cache read/write 2.64 / 15.84 / 0.264 / 3.30 -> 2.20 / 13.20 / 0.22 / 2.75 luna in/out/cache read/write 0.264 / 1.584 / 0.0264 / 0.33 -> 0.22 / 1.32 / 0.022 / 0.275 The 1.2x is not coincidence: Terra and Luna were sourced wholesale from the GovCloud Price List rows, which are exactly 1.2x commercial. Sol's output was the one figure with no source at all — a 6x input ratio inferred from GovCloud, where the real ratio is 5x. `openai.gpt-5.4` was already correct, empty cache-write cell included, so yesterday's prod fix is confirmed by the card. This also resolves the tier/long-context gap PR-3 was blocked on, rather than merely downgrading it as the previous commit claimed: - Service tiers do not apply. Every card says Priority and Flex are not supported for these models, so the 0.5x/2x dimension does not exist here. - Long context is real, and the spec's "2x twin" was wrong: above the 272K threshold input is 2x but output is only 1.5x. A flat 2x would have over-priced long-context output by a third. - We do not reach it. All rows carry maxInputTokens 272000, pinned at the short-context boundary, and compaction runs at 100K — so one short-context rate is correct, and that cap is what keeps it correct. Noted for the prod rows: `global.openai.gpt-5.6-*` prices 9.1% below the `us.*` Geo CRIS card across every bucket, and prod already runs Claude on `global.*`. Prod should not be a copy of the dev rows. The empirical work in the previous commit is not wasted — it is now the audit of these published numbers instead of the source of them, and the 2026-09-06 Sol window should reproduce 4.40 / 0.44 / 5.50 / 22.00 rather than discover it. Co-Authored-By: Claude Opus 5 --- docs/specs/gpt-5-6-prompt-caching.md | 107 ++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/docs/specs/gpt-5-6-prompt-caching.md b/docs/specs/gpt-5-6-prompt-caching.md index ec163bc8..ef1652fc 100644 --- a/docs/specs/gpt-5-6-prompt-caching.md +++ b/docs/specs/gpt-5-6-prompt-caching.md @@ -1,6 +1,6 @@ # Plan: prompt caching for OpenAI GPT-5.6 on Bedrock -**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) remains BLOCKED, but the blocker changed on 2026-09-06: these models bill through AWS Marketplace, which no pricing API covers, so no amount of waiting will publish them. The empirical route is open instead — Cost Explorer carries the dollars but names no model, so rates are only attributable on a single-model day. First controlled window claimed 2026-09-06 (Sol). Dollar figures stay provisional until it is read. +**Status:** Shipped and VERIFIED LIVE 2026-09-05 — PR-1 (#945), PR-2 (#949), PR-5 (#951), PR-4 (#954, shipped OFF via #956) and the IAM fix (#959). Caching confirmed working end-to-end through the agent loop: warm turns cost 10.6x less than cold. PR-3 (catalog rates) is UNBLOCKED as of 2026-09-06: the rates are published on each model's card in the Bedrock User Guide — they are absent from the pricing APIs, which is what the earlier BLOCKED finding was actually measuring. All three dev GPT-5.6 rows were wrong and over-charged by exactly 20% (Terra and Luna carried GovCloud rates; Sol's output came from an inferred 6x ratio that is really 5x); corrected 2026-09-06T15:59Z. The tier/long-context modelling gap is resolved too: Priority and Flex are not supported for these models, and `maxInputTokens: 272000` pins us inside short-context pricing. **Author:** (drafted with Claude) **Date:** 2026-09-04 **Related:** `agents/main_agent/core/model_config.py`, `agents/main_agent/core/agent_factory.py`, @@ -247,7 +247,79 @@ API-key `/chat/api-converse` handler. Don't fork the build logic. $0 rather than inventing waste. - Re-verify every rate against the Price List API, per the ⚠️ above. -#### ⛔ BLOCKED — the Price List API does not publish these rates +#### ✅ RESOLVED 2026-09-06 — the rates were published all along, in the model cards + +Everything below this section was written while looking for these rates in +*pricing APIs*. They are not there, and that finding stands. But they are +published, in prose, on each model's card in the Bedrock User Guide: + +- [GPT-5.6 Sol](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html) +- [GPT-5.6 Terra](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html) +- [GPT-5.6 Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html) +- [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) + +**Published rates, Geo CRIS, Short Context (272K)** — Geo CRIS is the row that +applies to the `us.*` inference profiles we actually call. All figures $/MTok. + +| model | input | 30m cache write | cache read | output | +|---|---:|---:|---:|---:| +| `us.openai.gpt-5.6-sol` | 4.40 | 5.50 | 0.44 | 22.00 | +| `us.openai.gpt-5.6-terra` | 2.20 | 2.75 | 0.22 | 13.20 | +| `us.openai.gpt-5.6-luna` | 0.22 | 0.275 | 0.022 | 1.32 | +| `openai.gpt-5.4` (Mantle, In-Region) | 2.75 | — (no write fee) | 0.275 | 16.50 | + +**Every dev row was wrong, and every error over-charged by exactly 20%.** +Corrected in the dev catalog 2026-09-06T15:59Z: + +| model | field | was | now | +|---|---|---:|---:| +| sol | output | 26.40 | **22.00** | +| terra | input / output / cache read / cache write | 2.64 / 15.84 / 0.264 / 3.30 | **2.20 / 13.20 / 0.22 / 2.75** | +| luna | input / output / cache read / cache write | 0.264 / 1.584 / 0.0264 / 0.33 | **0.22 / 1.32 / 0.022 / 0.275** | + +The `1.2x` is not a coincidence: Terra and Luna had been sourced wholesale from +the **GovCloud** Price List rows, which are exactly 1.2x commercial. Sol's +output was the one number with no source at all — it came from a `6x` input +ratio inferred from GovCloud, and the real ratio is `5x`. `openai.gpt-5.4` was +already correct, including the empty cache-write cell. + +**The modelling gap is resolved, not merely downgraded.** + +- **Service tiers do not apply.** Every card states it outright: *"Priority and + Flex tiers are not supported for this model."* Only Standard exists, so the + 0.5x/2x tier dimension `CuratedModel` cannot represent is not a dimension for + these models at all. +- **Long context is real, and the earlier "2x twin" was wrong.** The threshold + is 272K (the models' own window is 1M). Above it, **input is 2x but output is + only 1.5x** — Sol 4.40→8.80 and 22.00→33.00, and the same 2x/1.5x split holds + for Terra and Luna. A flat 2x assumption would have over-priced long-context + output by a third. +- **We do not reach it.** All four rows carry `maxInputTokens: 272000`, pinned + at the short-context boundary, and compaction runs at 100K. Short-context + rates are therefore the correct single rate for our traffic, and the cap is + what keeps that true — do not raise it without also modelling the second + price card. + +**For prod: use Global CRIS.** `global.openai.gpt-5.6-*` prices **9.1% below** +the `us.*` Geo CRIS rates across every bucket (Sol 4.00 / 5.00 / 0.40 / 20.00). +Prod already runs Claude on `global.*`; dev cannot, because of the dev-only SCP. +So the prod rows should be `global.*` ids with the Global CRIS rate card — not +copies of the dev rows. + +**What this means for the empirical work below.** It is no longer the source of +truth, but it is not wasted: it is now the *audit* of these published numbers, +and the tooling fixes it produced (the 1000x unit bug, daily-vs-monthly, the +attribution guard) are what make that audit trustworthy. The 2026-09-06 Sol +window still reads on schedule; it should now reproduce 4.40 / 0.44 / 5.50 / +22.00 rather than discover them. + +**Process lesson.** The search was run entirely against pricing *APIs* — Price +List, then Marketplace Catalog — and concluded "no source exists" without ever +checking the model's own documentation page. Check the model card first; it is +the primary source for rates, caching support, context windows, service tiers, +and endpoint/API support, and it is where AWS documents all of them together. + +#### ⛔ SUPERSEDED — the Price List API does not publish these rates Checked 2026-09-05 against dev-ai (490617140655) with SSO credentials, across every Bedrock service code: @@ -282,13 +354,13 @@ Options when it is picked back up, in preference order: Option 2 was run against dev-ai. It is viable, but only under a constraint the plan above did not anticipate, and it closed off Option 1 in the process. -**Option 1 is not a waiting game.** These models bill through **AWS -Marketplace**, and the Price List API has no Marketplace service code — all 269 -service codes were enumerated on 2026-09-06 and none covers Marketplace. The -Marketplace Catalog API is seller-side and returns nothing for a subscriber. -So the earlier "not published yet" reading was wrong: there is no pricing API -that *could* carry these rates while they bill this way. Waiting will not -produce them. +⚠️ **The conclusion this paragraph originally drew was wrong — see the RESOLVED +section above.** What holds: these models bill through **AWS Marketplace**, the +Price List API has no Marketplace service code (all 269 enumerated 2026-09-06), +and the Marketplace Catalog API is seller-side and returns nothing for a +subscriber. What does not hold is the inference drawn from that — "therefore no +source exists, derive them empirically." The rates were published in the model +cards the whole time. Absence from an API is not absence from the docs. **Cost Explorer has the dollars, but names no model.** Usage types look like `USW2-MP:USW2_cache_read_tokens_standard-Units` — they carry the token bucket @@ -321,13 +393,12 @@ price cards, cache read is exactly `0.1x` input and cache write exactly `1.25x`. This is commercial-region billing data, and it corroborates the GovCloud ratio finding below from a completely different source. -**Bearing on the modelling gap.** Every row observed is `_standard`; no -`-long-ctx` usage type has ever appeared in this account. So the 0.5x/2x tier -and 2x long-context dimensions are not currently being billed against us, and a -flat standard rate is correct *for our present traffic*. That downgrades the -gap from "silently mis-prices" to "mis-prices only if traffic changes, and Cost -Explorer will show a new usage type when it does" — which is a monitorable -condition rather than a blocking unknown. +**Bearing on the modelling gap.** Every row observed is `_standard` and no +`-long-ctx` usage type has ever appeared in this account. The model cards +explain why: Priority and Flex are *not supported* for these models, so +`_standard` is the only tier that can appear; and our `maxInputTokens: 272000` +keeps every request inside the short-context price card. The billing data and +the cards agree. **Controlled window claimed: 2026-09-06, `us.openai.gpt-5.6-sol` only.** Dev had zero recorded model calls that day before the probe. Expected totals, to be @@ -346,7 +417,7 @@ buckets, and a 600-token prefix over 6 turns — deliberately under the accumulate as the rate anchor. Read it once Marketplace settles (allow 24-48h, not 24h) and Terra and Luna need their own single-model days. -**What the read will actually settle.** The dev rows currently carry: +**What the read was expected to settle.** The dev rows carried, at the time this was written (all since corrected — see the RESOLVED section above): | model | input | output | cache read | cache write | |---|---:|---:|---:|---:| @@ -365,7 +436,7 @@ confirmed from two independent sources. The risk is concentrated in two places: from GovCloud Price List rows; `sol` is absent from the Price List entirely, so `4.40` came from the model card. -The single-model day yields input and output directly, which settles both. +Both were settled by the model cards instead (Sol output is `22.00`, not `26.40`; every Terra and Luna figure was a GovCloud rate). The single-model day now serves as an audit of the published numbers rather than the source of them. What the GovCloud rows *do* establish, and can be relied on: From a3ab6cd8a03dcf12269ebeffeec767e564f44066 Mon Sep 17 00:00:00 2001 From: Phil Merrell Date: Sun, 6 Sep 2026 10:15:52 -0600 Subject: [PATCH 3/3] feat(models): curate GPT-5.6 Sol, Terra and Luna (PR-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `CURATED_BEDROCK_RESPONSES_MODELS` behind a new "Bedrock Responses" catalog tab, so the three GPT-5.6 models are one-click-creatable instead of requiring the escape-hatch form. Rates are the published Geo CRIS short-context row from each AWS model card — Geo CRIS is the tier the `us.*` inference profiles resolve to, and these models are inference-profile-only. Two values are pinned by test because they are pricing correctness, not preference: - `supportsCaching: true`. These models cache implicitly server-side with no way to turn it off, so `false` is not a preference but a false statement, and its only effect is to clear the cache-rate fields — pricing cached tokens at $0.00 while AWS bills them in full. On a warm conversation nearly every input token is a cached one. - `maxInputTokens: 272_000`. These have a 1M window but AWS prices them on two cards: above 272K, input costs 2x and output 1.5x. A CuratedModel holds one flat rate per bucket, so this cap is what keeps that single rate honest. Raising it silently opens the second price card. Fixes the curated `openai.gpt-5.4` Mantle entry in the same pass. It inherited `mantleDefaults()`' `supportsCaching: false`, so one-click-creating it produced exactly the mis-priced row that had to be repaired by hand in prod last night. Its card publishes a cache-read rate at 0.1x input and an em dash for cache write, so caching is on with a literal 0 write rate — 0 is the correct value rather than a missing one, because it makes `compute_wasted_usd` see a non-positive premium and return $0 instead of inventing waste. The `mantleDefaults()` comment claiming Bedrock caching is model-bound to Claude+Nova was simply wrong and is corrected. `claudeRates` becomes `ratesWithDerivedCache`: the 1.25x write / 0.1x read multipliers are not Claude-specific. The GPT-5.6 cards publish the same two, and commercial Cost Explorer billing reproduces them to four decimals — two model families, two independent sources, same ratios. `supportedParams` is deliberately absent from the new entries. AWS publishes no parameter table for GPT-5.6 (`model-parameters-openai.html` covers only the open-weight gpt-oss family), and a declared spec flips the #915 guard from permissive to restrictive — so an invented one would silently block parameters the model actually accepts. Better none than a guess. Not browser-verified: the page is admin-gated against the dev backend, so an unmerged frontend change cannot be signed in to. Layout risk is low — the tab strip is `flex-wrap` and the card grid is unchanged — but the visual check is worth doing on dev after merge. Co-Authored-By: Claude Opus 5 --- docs/specs/gpt-5-6-prompt-caching.md | 14 ++ .../manage-models/model-catalog.page.spec.ts | 73 ++++++- .../admin/manage-models/model-catalog.page.ts | 4 + .../manage-models/models/curated-models.ts | 187 +++++++++++++++--- 4 files changed, 249 insertions(+), 29 deletions(-) diff --git a/docs/specs/gpt-5-6-prompt-caching.md b/docs/specs/gpt-5-6-prompt-caching.md index ef1652fc..b5cc9805 100644 --- a/docs/specs/gpt-5-6-prompt-caching.md +++ b/docs/specs/gpt-5-6-prompt-caching.md @@ -236,6 +236,20 @@ API-key `/chat/api-converse` handler. Don't fork the build logic. ### PR-3 — Catalog entry and pricing +> **SHIPPED 2026-09-06.** `CURATED_BEDROCK_RESPONSES_MODELS` carries Sol, Terra +> and Luna at their published Geo CRIS short-context rates, behind a new +> "Bedrock Responses" catalog tab. `supportsCaching` is pinned true and +> `maxInputTokens` to 272,000 by test, because both are pricing correctness +> rather than preference. The curated `openai.gpt-5.4` Mantle entry was fixed in +> the same pass — it inherited `mantleDefaults()`' `supportsCaching: false` and +> so one-click-created exactly the mis-priced row that had to be repaired by +> hand in prod. `supportedParams` is deliberately omitted: AWS publishes no +> parameter table for GPT-5.6, and a declared spec flips the #915 guard +> restrictive, so a guess would silently block valid params. +> +> The original plan below is kept for the reasoning; where it conflicts with +> the RESOLVED section, the RESOLVED section is right. + - Add GPT-5.6 to `CURATED_MANTLE_MODELS`' sibling set (or a new `CURATED_RUNTIME_OPENAI_MODELS` if the transport field warrants a separate tab). - `supportsCaching: true`, plus verified `cacheReadPricePerMillionTokens` and diff --git a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts index e5d9448a..aa83ecc8 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.spec.ts @@ -7,7 +7,11 @@ import { ModelCatalogPage } from './model-catalog.page'; import { ManagedModelsService } from './services/managed-models.service'; import { CuratedModelPrefillService } from './services/curated-model-prefill.service'; import { AddCuratedModelDialogComponent } from './components/add-curated-model-dialog.component'; -import { CURATED_BEDROCK_MODELS, CURATED_MANTLE_MODELS } from './models/curated-models'; +import { + CURATED_BEDROCK_MODELS, + CURATED_BEDROCK_RESPONSES_MODELS, + CURATED_MANTLE_MODELS, +} from './models/curated-models'; function createMockManagedModelsService(overrides: Partial<{ isModelAdded: (modelId: string) => boolean; @@ -239,15 +243,19 @@ describe('ModelCatalogPage', () => { // named a `us.*` (Regional/CRIS) inference profile, which prices ~10% higher. // Nothing failed — the numbers were merely wrong, everywhere downstream. describe('curated Bedrock pricing', () => { + // The CRIS tier a model id resolves to drives its rate card, so the two + // must agree on every list that declares a tier — not just Bedrock's. + const tieredModels = [...CURATED_BEDROCK_MODELS, ...CURATED_BEDROCK_RESPONSES_MODELS]; + it('declares a pricingTier that matches the tier its modelId names', () => { - for (const model of CURATED_BEDROCK_MODELS) { + for (const model of tieredModels) { const expected = model.template.modelId.startsWith('global.') ? 'global' : 'regional'; expect(`${model.key}:${model.pricingTier}`).toBe(`${model.key}:${expected}`); } }); it('derives cache rates from base input at Bedrock\'s published multipliers', () => { - for (const model of CURATED_BEDROCK_MODELS) { + for (const model of tieredModels) { const t = model.template; if (!t.supportsCaching) continue; const input = t.inputPricePerMillionTokens; @@ -256,4 +264,63 @@ describe('ModelCatalogPage', () => { } }); }); + + describe('curated bedrock-responses (GPT-5.6) entries', () => { + it('renders them on their own tab', () => { + const page = createComponent(); + page.selectTab('bedrock-responses'); + + expect(page.visibleModels().map(m => m.key)).toEqual( + CURATED_BEDROCK_RESPONSES_MODELS.map(m => m.key), + ); + expect(CURATED_BEDROCK_RESPONSES_MODELS.length).toBeGreaterThan(0); + }); + + it('never ships supportsCaching false — the provider forces it true', () => { + // `false` here is not a preference but a false statement: these models + // cache implicitly server-side and it cannot be turned off. Its only + // effect would be to clear the cache rates, pricing cached tokens at + // $0.00 while AWS bills them in full. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.supportsCaching}`).toBe(`${model.key}:true`); + } + }); + + it('pins maxInputTokens to the 272K short-context boundary', () => { + // Load-bearing pricing, not just a cap: above 272K these models bill + // input at 2x and output at 1.5x, and a CuratedModel holds one flat rate + // per bucket. Raising this silently opens the second price card. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.maxInputTokens}`).toBe(`${model.key}:272000`); + } + }); + + it('routes over the Responses API, which is the only surface that caches', () => { + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(`${model.key}:${model.template.apiMode}`).toBe(`${model.key}:responses`); + expect(`${model.key}:${model.template.provider}`).toBe(`${model.key}:bedrock-responses`); + } + }); + + it('declares no supportedParams rather than an invented one', () => { + // AWS publishes no parameter table for GPT-5.6. A declared spec flips the + // #915 guard from permissive to restrictive, so a guessed one would + // silently block params the model actually accepts. + for (const model of CURATED_BEDROCK_RESPONSES_MODELS) { + expect(model.template.supportedParams ?? null).toBeNull(); + } + }); + }); + + it('curates GPT-5.4 on Mantle with caching on and no write fee', () => { + // Its model card publishes a cache-read rate with an em dash for cache + // write. Inheriting mantleDefaults()' supportsCaching:false priced its + // cached tokens at $0.00 while AWS billed them — the bug that had to be + // fixed by hand in prod. + const gpt54 = CURATED_MANTLE_MODELS.find(m => m.key === 'gpt-5-4'); + + expect(gpt54?.template.supportsCaching).toBe(true); + expect(gpt54?.template.cacheReadPricePerMillionTokens).toBeCloseTo(0.275, 6); + expect(gpt54?.template.cacheWritePricePerMillionTokens).toBe(0); + }); }); diff --git a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts index cf7734cd..2849f8ac 100644 --- a/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts +++ b/frontend/ai.client/src/app/admin/manage-models/model-catalog.page.ts @@ -31,6 +31,10 @@ interface ProviderTab { const PROVIDER_TABS: ProviderTab[] = [ { id: 'bedrock', label: 'Bedrock' }, { id: 'mantle', label: 'Bedrock Mantle' }, + // The OpenAI Responses API on bedrock-runtime. A separate tab from 'openai' + // (which is direct-to-OpenAI and unpopulated) because the transport, not the + // vendor, is what differs — and it is the only surface where GPT-5.6 caches. + { id: 'bedrock-responses', label: 'Bedrock Responses' }, { id: 'openai', label: 'OpenAI' }, { id: 'gemini', label: 'Gemini' }, ]; diff --git a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts index 4d5a073b..9739d283 100644 --- a/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts +++ b/frontend/ai.client/src/app/admin/manage-models/models/curated-models.ts @@ -5,9 +5,21 @@ import { ManagedModelFormData, ModelProvider } from './managed-model.model'; * one-click create a fully-configured managed model — including pricing and * per-param specs — plus a small amount of presentation metadata for the card. * - * NOTE — Claude rates below were read from the **AWS Price List API** - * (`AmazonBedrockFoundationModels`, us-west-2, published 2026-09-01), not the - * pricing page. Re-verify there when bumping a model id: + * NOTE — **the model card is the primary source for rates.** Each model's page + * in the Bedrock User Guide publishes its full rate card (per inference option + * and context window) alongside caching support, context windows, service + * tiers and endpoint support: + * + * docs.aws.amazon.com/bedrock/latest/userguide/model-card--.html + * + * Check it first. The OpenAI-family rates are absent from the Price List API + * entirely — those models bill through AWS Marketplace, which no pricing API + * covers — and reading that absence as "unpublished" put three rows into the + * dev catalog at GovCloud prices, over-charging by 20%. + * + * Claude rates below were read from the **AWS Price List API** + * (`AmazonBedrockFoundationModels`, us-west-2, published 2026-09-01), which + * does carry them. Re-verify there when bumping a model id: * * aws pricing get-products --region us-east-1 \ * --service-code AmazonBedrockFoundationModels \ @@ -66,17 +78,22 @@ const claude4xDefaults = (): Pick< }); /** - * Bedrock publishes a Claude model's cache rates as fixed multiples of its base - * input rate: 5-minute cache write is **1.25x**, cache read is **0.1x** (the - * 1-hour write we do not use is 2x). Deriving them removes the two fields most - * likely to drift — the ratios were the one thing the old table got right. + * Bedrock publishes cache rates as fixed multiples of a model's base input + * rate: cache write is **1.25x**, cache read is **0.1x** (the 1-hour Claude + * write we do not use is 2x). Deriving them removes the two fields most likely + * to drift — the ratios were the one thing the old table got right. + * + * Not Claude-specific: the GPT-5.6 model cards publish exactly the same two + * multipliers (Sol 4.40 -> 5.50 / 0.44), and commercial Cost Explorer billing + * reproduces them to four decimals on every clean day. Two model families, two + * independent sources, same ratios. * * `input` and `output` are the only independently published numbers, and both * are TIER-SPECIFIC. Pass the rates for the tier the `modelId` names, and set * `pricingTier` to match; a `us.*` id costs ~10% more than the `global.*` rates * for the same model, which is exactly how the two drifted apart before. */ -const claudeRates = ( +const ratesWithDerivedCache = ( input: number, output: number, ): Pick< @@ -110,7 +127,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ modelName: 'Claude Opus 4.7', maxOutputTokens: 64_000, // Regional (CRIS): $5.50 / $27.50. Global is $5.00 / $25.00. - ...claudeRates(5.5, 27.5), + ...ratesWithDerivedCache(5.5, 27.5), knowledgeCutoffDate: '2025-10-01', supportedParams: { params: { @@ -137,7 +154,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ maxOutputTokens: 128_000, // Global: $2.00 / $10.00 — correct as declared, this id really is // `global.*`. Regional would be $2.20 / $11.00. - ...claudeRates(2.0, 10.0), + ...ratesWithDerivedCache(2.0, 10.0), knowledgeCutoffDate: null, supportedParams: { params: { @@ -162,7 +179,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ modelName: 'Claude Sonnet 4.6', maxOutputTokens: 64_000, // Regional (CRIS): $3.30 / $16.50. Global is $3.00 / $15.00. - ...claudeRates(3.3, 16.5), + ...ratesWithDerivedCache(3.3, 16.5), knowledgeCutoffDate: '2025-07-01', supportedParams: { params: { @@ -187,7 +204,7 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ maxOutputTokens: 64_000, // Regional (CRIS): $1.10 / $5.50. Global is $1.00 / $5.00. This is the // platform default model, so this is the row every cost number rides on. - ...claudeRates(1.1, 5.5), + ...ratesWithDerivedCache(1.1, 5.5), knowledgeCutoffDate: '2025-02-01', supportedParams: { params: { @@ -205,10 +222,13 @@ export const CURATED_BEDROCK_MODELS: CuratedModel[] = [ /** * Shared defaults for Bedrock Mantle (OpenAI-compatible open-weight) models. * - * Caching is intentionally absent: prompt caching on Bedrock is model-bound - * to Anthropic Claude + a small set of Amazon Nova models, none of which run - * through the Mantle provider, so these never cache and carry no cache - * pricing. `apiMode` (Chat Completions vs Responses) and an optional `region` + * `supportsCaching: false` is the right DEFAULT here — most Mantle models are + * open-weight and genuinely never cache — but it is not universal, so any + * entry for a model that does cache must override it. `openai.gpt-5.4` is the + * one below that does: its model card publishes a cache-read rate (0.1x input) + * with no write fee. Inheriting the default there priced its cached tokens at + * $0.00 while AWS billed them, which is exactly the bug that had to be fixed + * by hand in prod. `apiMode` (Chat Completions vs Responses) and an optional `region` * are the Mantle-specific fields — sourced from each model card (there is no * API that exposes them). The base path is derived by the SDK from the model id. */ @@ -255,8 +275,16 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ // _OPENAI_PATH_MODEL_PREFIXES, so one-click create routes correctly // (unlike the Gemma case noted below). apiMode: 'responses', + // Model card, In-Region: $2.75 / $16.50, cache read $0.275 (0.1x input), + // and the cache-write cell is an em dash — there is NO write fee on this + // model. A literal 0 is the correct rate, not a missing value: it makes + // `compute_wasted_usd` see a non-positive premium and return $0 instead + // of inventing waste. inputPricePerMillionTokens: 2.75, outputPricePerMillionTokens: 16.5, + supportsCaching: true, + cacheReadPricePerMillionTokens: 0.275, + cacheWritePricePerMillionTokens: 0, }, }, { @@ -293,22 +321,129 @@ export const CURATED_MANTLE_MODELS: CuratedModel[] = [ // Use the `google.gemma-4-` prefix, NOT `google.gemma-`: Gemma 3 is on `/v1`. ]; + /** - * Provider-keyed lookup for the catalog tabs. Bedrock + Mantle are populated; - * OpenAI/Gemini are intentional empty arrays — the page renders a - * 'Coming soon' empty state when the active tab has no entries. + * Shared defaults for `bedrock-responses` — the OpenAI **Responses** API on + * `bedrock-runtime`. + * + * `supportsCaching: true` is not a preference here, it is the only truthful + * value. These models cache implicitly and server-side with no way to turn it + * off, so `false` would be a false statement whose only effect is to clear the + * cache-rate fields — pricing cached tokens at $0.00 while AWS bills them in + * full. On a warm conversation nearly every input token is a cached one, so + * that is close to total under-reporting. The backend normalizes it the same + * way (`_resolve_supports_caching`, forced for this provider), as it does + * `apiMode: 'responses'`. * - * `bedrock-responses` is empty for now: the transport landed ahead of its - * catalog rows, whose rates have to be verified against the Price List API - * first. Adding a model there today means the escape-hatch form, which works. - * NOTE for whoever curates it: `mantleDefaults()` hardcodes - * `supportsCaching: false` — a GPT-5.6 row must not inherit that, since - * caching is the entire reason this transport exists. + * `maxInputTokens: 272_000` is **load-bearing pricing**, not just a cap. These + * models have a 1M window, but AWS prices them on two cards: above 272K, input + * costs 2x and output 1.5x. `CuratedModel` holds one flat rate per bucket, so + * the cap is what keeps that single rate correct. Raising it silently opens + * the second price card and under-charges every long turn. + */ +const bedrockResponsesDefaults = (): Pick< + ManagedModelFormData, + | 'provider' + | 'providerName' + | 'inputModalities' + | 'outputModalities' + | 'responseStreamingSupported' + | 'maxInputTokens' + | 'maxOutputTokens' + | 'allowedAppRoles' + | 'availableToRoles' + | 'enabled' + | 'isDefault' + | 'supportsCaching' + | 'apiMode' +> => ({ + provider: 'bedrock-responses', + providerName: 'OpenAI', + inputModalities: ['TEXT', 'IMAGE'], + outputModalities: ['TEXT'], + responseStreamingSupported: true, + maxInputTokens: 272_000, + // The cards publish no output cap ("Max output tokens: N/A"), so claim none + // rather than invent one — this value is only a ceiling on the configured + // max_tokens param and is never sent to the provider. + maxOutputTokens: null, + allowedAppRoles: [], + availableToRoles: [], + enabled: true, + isDefault: false, + supportsCaching: true, + apiMode: 'responses', +}); + +/** + * GPT-5.6 on `bedrock-runtime` via the Responses API. + * + * Rates are the **Geo CRIS, Short Context (272K)** row from each model card — + * Geo CRIS is the tier the `us.*` inference profiles resolve to, and these + * models are inference-profile-only (no ON_DEMAND). Verified 2026-09-06. + * + * `supportedParams` is deliberately absent. AWS publishes no parameter table + * for these models (`model-parameters-openai.html` documents only the + * open-weight gpt-oss family), and an invented spec would be worse than none: + * a declared spec flips the #915 guard from permissive to restrictive, so a + * wrong entry silently blocks a parameter the model actually accepts. Add one + * only from published or measured evidence. + */ +export const CURATED_BEDROCK_RESPONSES_MODELS: CuratedModel[] = [ + { + key: 'gpt-5-6-sol', + tagline: 'OpenAI\'s most capable model — frontier reasoning and agentic work.', + capabilities: ['Reasoning', 'Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-sol', + modelName: 'GPT-5.6 Sol', + // Geo CRIS: $4.40 / $22.00. Global CRIS is $4.00 / $20.00. + ...ratesWithDerivedCache(4.4, 22.0), + knowledgeCutoffDate: null, + }, + }, + { + key: 'gpt-5-6-terra', + tagline: 'Balanced everyday model — strong performance per dollar.', + capabilities: ['Reasoning', 'Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-terra', + modelName: 'GPT-5.6 Terra', + // Geo CRIS: $2.20 / $13.20. Global CRIS is $2.00 / $12.00. + ...ratesWithDerivedCache(2.2, 13.2), + knowledgeCutoffDate: null, + }, + }, + { + key: 'gpt-5-6-luna', + tagline: 'Fast and affordable — for classification, routing and high volume.', + capabilities: ['Vision', 'Long context', 'Prompt caching'], + pricingTier: 'regional', + template: { + ...bedrockResponsesDefaults(), + modelId: 'us.openai.gpt-5.6-luna', + modelName: 'GPT-5.6 Luna', + // Geo CRIS: $0.22 / $1.32. Global CRIS is $0.20 / $1.20. + ...ratesWithDerivedCache(0.22, 1.32), + knowledgeCutoffDate: null, + }, + }, +]; + +/** + * Provider-keyed lookup for the catalog tabs. Bedrock, Mantle and + * bedrock-responses are populated; OpenAI/Gemini are intentional empty arrays + * — the page renders a 'Coming soon' empty state when the active tab has no + * entries. */ export const CURATED_MODELS_BY_PROVIDER: Record = { bedrock: CURATED_BEDROCK_MODELS, openai: [], gemini: [], mantle: CURATED_MANTLE_MODELS, - 'bedrock-responses': [], + 'bedrock-responses': CURATED_BEDROCK_RESPONSES_MODELS, };