From 095652a38a352858899c96220f5a46e5008ca139 Mon Sep 17 00:00:00 2001 From: Amaan Date: Mon, 27 Jul 2026 19:06:04 +0530 Subject: [PATCH] feat/new-models: Add gpt-oss-120b, qwen3-30b-a3b-fp8, and the domain IAB classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings the CLI in line with the models published in the API reference. - `chat` gains `-m, --model` (default `LFM2.5-1.2B-Instruct`) covering `gpt-oss-120b`, `qwen3-30b-a3b-fp8`, and `LFM2.5-1.2B-Thinking`, plus `-r, --reasoning` to print the trace reasoning models return. `qwen3-30b-a3b-fp8` has no Responses endpoint, so it is routed to `/v1/chat/completions` transparently. - New `classify_domain` command for `zlm-v1-iab-domain-classifier`, which classifies a bare hostname with no page fetch. - `classify_iab_enriched` now sends `zlm-v2-iab-classify-edge-enriched`; the docs renamed the model from `zlm-v1-...`. - Reasoning models emit a `reasoning` output item ahead of the assistant message, so response parsing now selects the `message` item instead of assuming `output[0]`. - Refresh `ZGPU_PRICING` against the published catalog — the edge models are $0.02/$0.05 per 1M, not the $0.05/$0.40 the table still carried. Verified against the live API: all four model paths return expected output, and the reasoning trace is separated from the answer. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 30 ++++++- docs/ADDING_COMMANDS.md | 7 +- docs/DOCUMENTATION.md | 82 ++++++++++++++++-- src/cli.ts | 2 + src/commands/chat.ts | 125 +++++++++++++++++++++++----- src/commands/classifyDomain.ts | 72 ++++++++++++++++ src/commands/classifyIabEnriched.ts | 2 +- src/lib/chatCompletions.ts | 37 ++++++++ src/lib/responses.ts | 36 ++++++-- src/lib/savings.ts | 24 +++--- tests/responses.test.ts | 83 ++++++++++++++++++ tests/savings.test.ts | 8 +- 12 files changed, 457 insertions(+), 51 deletions(-) create mode 100644 src/commands/classifyDomain.ts create mode 100644 src/lib/chatCompletions.ts create mode 100644 tests/responses.test.ts diff --git a/README.md b/README.md index 4ad6ded..c68e5a3 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The official command-line interface for [ZeroGPU](https://zerogpu.ai) — run fa - [Classification](#classification) - [`classify_iab`](#classify_iab) - [`classify_iab_enriched`](#classify_iab_enriched) + - [`classify_domain`](#classify_domain) - [`classify_structured`](#classify_structured) - [`classify_zero_shot`](#classify_zero_shot) - [Extraction](#extraction) @@ -122,16 +123,33 @@ zerogpu status #### `chat` -Chat with the **LFM2.5-1.2B-Instruct** model. +Chat with a ZeroGPU text-generation model. Defaults to **LFM2.5-1.2B-Instruct**; use `--model` to reach for a larger reasoning model. ```bash zerogpu chat "Explain edge inference in one sentence." zerogpu chat "Translate to French: Good morning." -i "You are a precise translator." + +# Frontier-grade reasoning, with the trace printed alongside the answer +zerogpu chat "Why does my API keep getting rate-limited?" -m gpt-oss-120b -r + +# Multilingual reasoning +zerogpu chat "Explique la mise en cache en une phrase." -m qwen3-30b-a3b-fp8 ``` | Option | Description | |---|---| | `-i, --instructions ` | System instructions that steer the assistant's behavior. | +| `-m, --model ` | Model to use (default `LFM2.5-1.2B-Instruct`). | +| `-r, --reasoning` | Also print the reasoning trace, for models that return one. | + +| Model | Notes | +|---|---| +| `LFM2.5-1.2B-Instruct` | Default. Fast edge chat. | +| `LFM2.5-1.2B-Thinking` | Compact reasoning model. | +| `gpt-oss-120b` | 117B MoE, 131K context, reasoning + function calling. | +| `qwen3-30b-a3b-fp8` | 30.5B MoE, 100+ languages, reasoning + function calling. | + +`qwen3-30b-a3b-fp8` is served by the Chat Completions API rather than the Responses API; the CLI routes it automatically. #### `chat_thinking` @@ -177,6 +195,16 @@ Same as `classify_iab` but returns enriched output: audience, topics, keywords, zerogpu classify_iab_enriched "Best running shoes for marathon training." ``` +#### `classify_domain` + +Classify a **domain name** against the IAB taxonomy using the ZeroGPU domain edge model. Takes only the hostname — no page fetch or body text — so it suits bidstream enrichment and inventory-level targeting. + +```bash +zerogpu classify_domain nytimes.com +``` + +Returns scored content categories plus topics, keywords, and inferred user intent for the domain. When you have the page text and need per-URL precision, use `classify_iab` instead. + #### `classify_structured` Classify text against a **structured schema** of categories and allowed labels using GLiNER2. diff --git a/docs/ADDING_COMMANDS.md b/docs/ADDING_COMMANDS.md index 1e93954..2098554 100644 --- a/docs/ADDING_COMMANDS.md +++ b/docs/ADDING_COMMANDS.md @@ -6,7 +6,8 @@ This guide explains how to add a new CLI command to the ZeroGPU CLI. - `src/commands/` — one file per command, each exporting a `registerCommand(program)` function. - `src/cli.ts` — wires every command into the root program. -- `src/lib/responses.ts` — shared `RESPONSES_ENDPOINT` and `ResponsesApiResponse` for `/v1/responses` calls. +- `src/lib/responses.ts` — shared `RESPONSES_ENDPOINT`, `ResponsesApiResponse`, and the `extractOutputText` / `extractReasoningText` helpers for `/v1/responses` calls. +- `src/lib/chatCompletions.ts` — the same for `/v1/chat/completions`, used by models the platform serves only there (currently `qwen3-30b-a3b-fp8`), plus `toResponsesUsage` to normalize token counts for savings tracking. - `src/lib/auth.ts` — `getApiKey()` for authenticated requests. ## Steps @@ -15,7 +16,9 @@ This guide explains how to add a new CLI command to the ZeroGPU CLI. - Export `registerXxxCommand(program: Command): void`. - Define the command name, args, and `.description(...)`. - In `.action(...)`, validate auth, call the API, and print the result. - - For Responses API models, import `RESPONSES_ENDPOINT` and `ResponsesApiResponse` from `../lib/responses.js` — do not hardcode the endpoint or redefine the type. + - For Responses API models, import `RESPONSES_ENDPOINT` and `ResponsesApiResponse` from `../lib/responses.js` — do not hardcode the endpoint or redefine the type. Use `extractOutputText(data)` rather than reading `output[0]`: reasoning models put a `reasoning` item ahead of the assistant message. + - For Chat Completions models, import `CHAT_COMPLETIONS_ENDPOINT` and `ChatCompletionsApiResponse` from `../lib/chatCompletions.js`, and pass `toResponsesUsage(data.usage)` to `recordAndMaybeNotify`. + - Add the model's published input/output price to `ZGPU_PRICING` in `src/lib/savings.ts`, so savings are computed at the real rate rather than the fallback. - Keep only command-specific constants local (e.g. `const MODEL = "..."`). 2. **Register it** in `src/cli.ts`: diff --git a/docs/DOCUMENTATION.md b/docs/DOCUMENTATION.md index 85b44a2..43c0369 100644 --- a/docs/DOCUMENTATION.md +++ b/docs/DOCUMENTATION.md @@ -2,10 +2,12 @@ ## 1. Project Description -`zerogpu-cli` is the official command-line interface for [ZeroGPU](https://zerogpu.ai), a distributed / edge inference platform for small language models (SLMs) and nano language models. The CLI is a thin, OpenAI-compatible client around the ZeroGPU **Responses API** (`https://api.zerogpu.ai/v1/responses`) that lets you call a curated set of edge-optimized models directly from your terminal for common NLP workloads: +`zerogpu-cli` is the official command-line interface for [ZeroGPU](https://zerogpu.ai), a distributed / edge inference platform for small language models (SLMs) and nano language models. The CLI is a thin, OpenAI-compatible client around the ZeroGPU **Responses API** (`https://api.zerogpu.ai/v1/responses`) — and, for models served only there, the **Chat Completions API** (`https://api.zerogpu.ai/v1/chat/completions`) — that lets you call a curated set of edge-optimized models directly from your terminal for common NLP workloads: - Conversational chat (`LFM2.5-1.2B-Instruct`, `LFM2.5-1.2B-Thinking`) -- IAB content/audience classification (`zlm-v1-iab-classify-edge`, `…-enriched`) +- Reasoning and tool-use chat (`gpt-oss-120b`, `qwen3-30b-a3b-fp8`) +- IAB content/audience classification (`zlm-v1-iab-classify-edge`, `zlm-v2-iab-classify-edge-enriched`) +- Domain-level IAB classification (`zlm-v1-iab-domain-classifier`) - Zero-shot classification (`deberta-v3-small`) - Structured / schema-driven classification and JSON extraction (`gliner2-base-v1`) - Named-entity recognition with custom labels (`gliner2-base-v1`) @@ -89,7 +91,7 @@ The CLI exposes the following commands: |---|---| | [`login`](#41-login) | Sign in and persist API key | | [`status`](#42-status) | Show current sign-in status | -| [`chat`](#43-chat) | Chat with `LFM2.5-1.2B-Instruct` | +| [`chat`](#43-chat) | Chat with a text-generation model (default `LFM2.5-1.2B-Instruct`) | | [`chat_thinking`](#44-chat_thinking) | Chat with the Thinking variant (returns reasoning) | | [`classify_iab`](#45-classify_iab) | IAB taxonomy classification | | [`classify_iab_enriched`](#46-classify_iab_enriched) | IAB enriched (topics / keywords / intent) | @@ -101,6 +103,7 @@ The CLI exposes the following commands: | [`extract_json`](#412-extract_json) | Schema-driven structured JSON extraction | | [`summarize`](#413-summarize) | Summarize text with `llama-3.1-8b-instruct-fast` | | [`generate_followups`](#414-generate_followups) | Generate follow-up questions | +| [`classify_domain`](#415-classify_domain) | Domain-level IAB classification | ### Common exit codes | Code | Meaning | @@ -182,11 +185,11 @@ zerogpu status ### 4.3 `chat` -Chat with the `LFM2.5-1.2B-Instruct` model. +Chat with a ZeroGPU text-generation model. Defaults to `LFM2.5-1.2B-Instruct`. **Synopsis** ``` -zerogpu chat [-i ] +zerogpu chat [-i ] [-m ] [-r] ``` **Parameters** @@ -195,11 +198,26 @@ zerogpu chat [-i ] |---|---|---|---| | `text` (positional) | string | yes | The user message / prompt to send. | | `-i`, `--instructions ` | string | optional | System instructions that steer the assistant's behavior. | +| `-m`, `--model ` | string | optional | Model to use. Default `LFM2.5-1.2B-Instruct`. Matched case-insensitively; an unknown id exits `1` and lists the valid ones. | +| `-r`, `--reasoning` | boolean | optional | Also print the reasoning trace, for models that return one. Ignored when the model returns none. | + +**Models** + +| Model | API | Notes | +|---|---|---| +| `LFM2.5-1.2B-Instruct` | Responses | Default. Fast edge chat. | +| `LFM2.5-1.2B-Thinking` | Responses | Compact reasoning model. | +| `gpt-oss-120b` | Responses | 117B MoE, 131K context, reasoning + function calling. | +| `qwen3-30b-a3b-fp8` | Chat Completions | 30.5B MoE, 100+ languages, reasoning + function calling. | + +`qwen3-30b-a3b-fp8` has no Responses endpoint, so the CLI posts it to `/v1/chat/completions` instead, mapping `--instructions` to a `system` message and normalizing `prompt_tokens` / `completion_tokens` back to Responses token names for savings tracking. This routing is transparent — the command and its output are identical either way. **Example** ```bash zerogpu chat "Explain WebSockets in two sentences." \ -i "You are a concise technical writer." + +zerogpu chat "Why is my p99 latency spiking?" -m gpt-oss-120b -r ``` **Expected output** @@ -281,7 +299,7 @@ zerogpu classify_iab "The Lakers signed a new point guard ahead of the playoffs. ### 4.6 `classify_iab_enriched` -Classify text with the **enriched** IAB edge model (`zlm-v1-iab-classify-edge-enriched`) — returns audience categories plus topics, keywords, and inferred intent. +Classify text with the **enriched** IAB edge model (`zlm-v2-iab-classify-edge-enriched`) — returns audience categories plus topics, keywords, and inferred intent. **Synopsis** ``` @@ -611,6 +629,47 @@ zerogpu generate_followups \ --- +### 4.15 `classify_domain` + +Classify a domain name against the IAB taxonomy using `zlm-v1-iab-domain-classifier`. It infers the categories that characterize a site as a whole from the hostname alone — no crawl, no page text — which keeps payloads roughly 10x smaller than page-level classification. Use it for bidstream enrichment, allow/deny-list scoring, and inventory-level targeting; when you have the page text and need per-URL precision, use [`classify_iab`](#45-classify_iab) instead. + +**Synopsis** +``` +zerogpu classify_domain +``` + +**Parameters** + +| Name | Type | Required | Description | +|---|---|---|---| +| `domain` (positional) | string | yes | Bare domain name to classify, e.g. `nytimes.com`. | + +**Example** +```bash +zerogpu classify_domain nytimes.com +``` + +**Expected output (illustrative)** +```json +{ + "content": { + "iab_1_0": [{ "code": "IAB12", "name": "News", "score": 0.8124 }], + "iab_2_2": [{ "name": "News and Politics", "score": 0.8124 }] + }, + "topics": [{ "name": "news", "score": 0.8124 }], + "keywords": ["news", "politics", "world", "business"], + "user_intent": { + "name": "reading news and current events", + "category": "informational", + "score": 0.5 + } +} +``` + +**Outcomes** — same as the common table. + +--- + ## 5. Network & API Contract All inference commands POST to: @@ -621,6 +680,14 @@ Content-Type: application/json x-api-key: ``` +The one exception is `chat --model qwen3-30b-a3b-fp8`, which the ZeroGPU platform serves only through the OpenAI-compatible Chat Completions endpoint: + +``` +POST https://api.zerogpu.ai/v1/chat/completions +Content-Type: application/json +x-api-key: +``` + The request body is: ```jsonc { @@ -635,12 +702,15 @@ The CLI parses the response as: ```ts interface ResponsesApiResponse { output?: Array<{ + type?: string; content?: Array<{ type?: string; text?: string }>; }>; } ``` It picks the first `content` entry whose `type === "output_text"` (falling back to `content[0]`), then prints `text` — pretty-printed if it parses as JSON, otherwise as a raw string. +Reasoning models emit a `reasoning` output item **ahead of** the assistant message, so `chat` selects the item whose `type === "message"` rather than assuming `output[0]`, and reads the trace from the `reasoning` item when `--reasoning` is passed. On the Chat Completions path the answer is `choices[0].message.content` and the trace is `choices[0].message.reasoning`. + --- ## 6. Version Compatibility & Update Behavior diff --git a/src/cli.ts b/src/cli.ts index 3b7c92d..86bcc6d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,6 +5,7 @@ import { Command } from "commander"; import updateNotifier from "update-notifier"; import { registerChatCommand } from "./commands/chat.js"; import { registerChatThinkingCommand } from "./commands/chatThinking.js"; +import { registerClassifyDomainCommand } from "./commands/classifyDomain.js"; import { registerClassifyIabCommand } from "./commands/classifyIab.js"; import { registerClassifyIabEnrichedCommand } from "./commands/classifyIabEnriched.js"; import { registerClassifyStructuredCommand } from "./commands/classifyStructured.js"; @@ -95,6 +96,7 @@ export function buildProgram(): Command { registerStatusCommand(program); registerClassifyIabCommand(program); registerClassifyIabEnrichedCommand(program); + registerClassifyDomainCommand(program); registerClassifyStructuredCommand(program); registerClassifyZeroShotCommand(program); registerGenerateFollowupsCommand(program); diff --git a/src/commands/chat.ts b/src/commands/chat.ts index db1da49..4e1df5b 100644 --- a/src/commands/chat.ts +++ b/src/commands/chat.ts @@ -1,23 +1,73 @@ import { Command } from "commander"; import { getApiKey } from "../lib/auth.js"; +import { + CHAT_COMPLETIONS_ENDPOINT, + toResponsesUsage, + type ChatCompletionsApiResponse, +} from "../lib/chatCompletions.js"; import { RESPONSES_ENDPOINT, + extractOutputText, + extractReasoningText, type ResponsesApiResponse, + type ResponsesUsage, } from "../lib/responses.js"; import { recordAndMaybeNotify } from "../lib/savings.js"; -const MODEL = "LFM2.5-1.2B-Instruct"; +const DEFAULT_MODEL = "LFM2.5-1.2B-Instruct"; + +// Text-generation models `--model` accepts, and the API each one speaks. +// qwen3-30b-a3b-fp8 is Chat Completions only — it has no Responses endpoint. +// Source: https://docs.zerogpu.ai/docs/text-generation +const CHAT_MODELS: Record = { + "LFM2.5-1.2B-Instruct": "responses", + "LFM2.5-1.2B-Thinking": "responses", + "gpt-oss-120b": "responses", + "qwen3-30b-a3b-fp8": "chat-completions", +}; + +// Model ids are case-sensitive to the API but not to the person typing them. +function resolveModel(name: string): string | undefined { + return Object.keys(CHAT_MODELS).find( + (id) => id.toLowerCase() === name.toLowerCase(), + ); +} export function registerChatCommand(program: Command): void { program .command("chat ") - .description("Chat with the LFM2.5 instruct model.") + .description( + `Chat with a ZeroGPU text-generation model (default ${DEFAULT_MODEL}).`, + ) .option( "-i, --instructions ", "System instructions that steer the assistant's behavior.", ) + .option( + "-m, --model ", + `Model to use: ${Object.keys(CHAT_MODELS).join(", ")}.`, + DEFAULT_MODEL, + ) + .option( + "-r, --reasoning", + "Also print the reasoning trace, for models that return one.", + ) .action( - async (text: string, opts: { instructions?: string }) => { + async ( + text: string, + opts: { instructions?: string; model: string; reasoning?: boolean }, + ) => { + const model = resolveModel(opts.model); + + if (!model) { + console.error( + `Unknown model '${opts.model}'. Available models: ${Object.keys( + CHAT_MODELS, + ).join(", ")}.`, + ); + process.exit(1); + } + const apiKey = getApiKey(); if (!apiKey) { @@ -27,22 +77,36 @@ export function registerChatCommand(program: Command): void { process.exit(1); } - const body: Record = { - model: MODEL, - input: text, - }; - if (opts.instructions) body.instructions = opts.instructions; + const useChatCompletions = CHAT_MODELS[model] === "chat-completions"; + + const body: Record = useChatCompletions + ? { + model, + messages: [ + ...(opts.instructions + ? [{ role: "system", content: opts.instructions }] + : []), + { role: "user", content: text }, + ], + } + : { model, input: text }; + if (!useChatCompletions && opts.instructions) { + body.instructions = opts.instructions; + } let response: Response; try { - response = await fetch(RESPONSES_ENDPOINT, { - method: "POST", - headers: { - "content-type": "application/json", - "x-api-key": apiKey.apiKey, + response = await fetch( + useChatCompletions ? CHAT_COMPLETIONS_ENDPOINT : RESPONSES_ENDPOINT, + { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey.apiKey, + }, + body: JSON.stringify(body), }, - body: JSON.stringify(body), - }); + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(`Request failed: ${message}`); @@ -56,16 +120,35 @@ export function registerChatCommand(program: Command): void { process.exit(1); } - const data = (await response.json()) as ResponsesApiResponse; - const content = data.output?.[0]?.content?.find( - (c) => c.type === "output_text", - )?.text ?? data.output?.[0]?.content?.[0]?.text; + const payload: unknown = await response.json(); + + let content: string | undefined; + let reasoning: string | undefined; + let usage: ResponsesUsage | undefined; + + if (useChatCompletions) { + const data = payload as ChatCompletionsApiResponse; + const message = data.choices?.[0]?.message; + content = message?.content; + reasoning = message?.reasoning; + usage = toResponsesUsage(data.usage); + } else { + const data = payload as ResponsesApiResponse; + content = extractOutputText(data); + reasoning = extractReasoningText(data); + usage = data.usage; + } + if (!content) { console.error("Response did not contain any chat content."); - console.error(JSON.stringify(data, null, 2)); + console.error(JSON.stringify(payload, null, 2)); process.exit(1); } + if (opts.reasoning && reasoning) { + console.log(`Reasoning:\n${reasoning}\n`); + } + try { const parsed = JSON.parse(content); console.log(JSON.stringify(parsed, null, 2)); @@ -73,7 +156,7 @@ export function registerChatCommand(program: Command): void { console.log(content); } - recordAndMaybeNotify({ model: MODEL, usage: data.usage }); + recordAndMaybeNotify({ model, usage }); }, ); } diff --git a/src/commands/classifyDomain.ts b/src/commands/classifyDomain.ts new file mode 100644 index 0000000..cbbbd95 --- /dev/null +++ b/src/commands/classifyDomain.ts @@ -0,0 +1,72 @@ +import { Command } from "commander"; +import { getApiKey } from "../lib/auth.js"; +import { + RESPONSES_ENDPOINT, + extractOutputText, + type ResponsesApiResponse, +} from "../lib/responses.js"; +import { recordAndMaybeNotify } from "../lib/savings.js"; + +const MODEL = "zlm-v1-iab-domain-classifier"; + +export function registerClassifyDomainCommand(program: Command): void { + program + .command("classify_domain ") + .alias("classify-domain") + .description( + "Classify a domain name with the IAB domain edge model (content, topics, keywords, intent).", + ) + .action(async (domain: string) => { + const apiKey = getApiKey(); + + if (!apiKey) { + console.error( + "You're not fully signed in yet. Run 'zerogpu login' to set your API key.", + ); + process.exit(1); + } + + let response: Response; + try { + response = await fetch(RESPONSES_ENDPOINT, { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey.apiKey, + }, + body: JSON.stringify({ + model: MODEL, + input: domain, + }), + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`Request failed: ${message}`); + process.exit(1); + } + + if (!response.ok) { + const body = await response.text(); + console.error(`Request failed with status ${response.status}.`); + if (body) console.error(body); + process.exit(1); + } + + const data = (await response.json()) as ResponsesApiResponse; + const content = extractOutputText(data); + if (!content) { + console.error("Response did not contain any classification content."); + console.error(JSON.stringify(data, null, 2)); + process.exit(1); + } + + try { + const parsed = JSON.parse(content); + console.log(JSON.stringify(parsed, null, 2)); + } catch { + console.log(content); + } + + recordAndMaybeNotify({ model: MODEL, usage: data.usage }); + }); +} diff --git a/src/commands/classifyIabEnriched.ts b/src/commands/classifyIabEnriched.ts index 748a155..9d93c0a 100644 --- a/src/commands/classifyIabEnriched.ts +++ b/src/commands/classifyIabEnriched.ts @@ -6,7 +6,7 @@ import { } from "../lib/responses.js"; import { recordAndMaybeNotify } from "../lib/savings.js"; -const MODEL = "zlm-v1-iab-classify-edge-enriched"; +const MODEL = "zlm-v2-iab-classify-edge-enriched"; export function registerClassifyIabEnrichedCommand(program: Command): void { program diff --git a/src/lib/chatCompletions.ts b/src/lib/chatCompletions.ts new file mode 100644 index 0000000..53b436f --- /dev/null +++ b/src/lib/chatCompletions.ts @@ -0,0 +1,37 @@ +import type { ResponsesUsage } from "./responses.js"; + +export const CHAT_COMPLETIONS_ENDPOINT = + "https://api.zerogpu.ai/v1/chat/completions"; + +export interface ChatCompletionsUsage { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; +} + +export interface ChatCompletionsApiResponse { + model?: string; + usage?: ChatCompletionsUsage; + choices?: Array<{ + index?: number; + finish_reason?: string; + message?: { + role?: string; + content?: string; + // Reasoning models return their trace alongside the answer. + reasoning?: string; + }; + }>; +} + +// Savings bookkeeping speaks the Responses API's token names. +export function toResponsesUsage( + usage: ChatCompletionsUsage | undefined, +): ResponsesUsage | undefined { + if (!usage) return undefined; + return { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + total_tokens: usage.total_tokens, + }; +} diff --git a/src/lib/responses.ts b/src/lib/responses.ts index d7321a3..2cc5aa0 100644 --- a/src/lib/responses.ts +++ b/src/lib/responses.ts @@ -8,13 +8,37 @@ export interface ResponsesUsage { output_tokens_details?: { reasoning_tokens?: number }; } +export interface ResponsesContentPart { + type?: string; + text?: string; +} + +export interface ResponsesOutputItem { + type?: string; + content?: ResponsesContentPart[]; +} + export interface ResponsesApiResponse { model?: string; usage?: ResponsesUsage; - output?: Array<{ - content?: Array<{ - type?: string; - text?: string; - }>; - }>; + output?: ResponsesOutputItem[]; +} + +// Reasoning models (gpt-oss-120b) emit a `reasoning` item ahead of the +// assistant message, so the answer is not always output[0]. +export function extractOutputText( + data: ResponsesApiResponse, +): string | undefined { + const items = data.output ?? []; + const message = items.find((item) => item.type === "message") ?? items[0]; + const parts = message?.content ?? []; + return parts.find((p) => p.type === "output_text")?.text ?? parts[0]?.text; +} + +export function extractReasoningText( + data: ResponsesApiResponse, +): string | undefined { + const parts = + data.output?.find((item) => item.type === "reasoning")?.content ?? []; + return parts.find((p) => p.type === "reasoning_text")?.text ?? parts[0]?.text; } diff --git a/src/lib/savings.ts b/src/lib/savings.ts index 382b833..59899b1 100644 --- a/src/lib/savings.ts +++ b/src/lib/savings.ts @@ -19,18 +19,22 @@ const DEFAULT_BASELINE = "claude-opus-4-8"; // Actual ZeroGPU pricing per 1M tokens (in / out), per model. // Source: https://docs.zerogpu.ai model catalog. const ZGPU_PRICING: Record = { + "gpt-oss-120b": { in: 0.03, out: 0.1 }, + "qwen3-30b-a3b-fp8": { in: 0.05, out: 0.3 }, "llama-3.1-8b-instruct-fast": { in: 0.02, out: 0.05 }, - "zlm-v1-iab-classify-edge": { in: 0.05, out: 0.4 }, - "zlm-v1-iab-classify-edge-enriched": { in: 0.05, out: 0.4 }, - "zlm-v1-followup-questions-edge": { in: 0.05, out: 0.4 }, - "gliner-multi-pii-v1": { in: 0.05, out: 0.4 }, - "gliner2-base-v1": { in: 0.05, out: 0.4 }, - "deberta-v3-small": { in: 0.04, out: 0.1 }, - "LFM2.5-1.2B-Thinking": { in: 0.02, out: 0.1 }, - "LFM2.5-1.2B-Instruct": { in: 0.02, out: 0.1 }, + "zlm-v1-iab-classify-edge": { in: 0.02, out: 0.05 }, + "zlm-v2-iab-classify-edge-enriched": { in: 0.02, out: 0.05 }, + "zlm-v1-iab-domain-classifier": { in: 0.02, out: 0.05 }, + "zlm-v1-followup-questions-edge": { in: 0.02, out: 0.05 }, + "gliner-multi-pii-v1": { in: 0.02, out: 0.05 }, + "gliner2-base-v1": { in: 0.02, out: 0.05 }, + "deberta-v3-small": { in: 0.02, out: 0.05 }, + "LFM2.5-1.2B-Thinking": { in: 0.02, out: 0.05 }, + "LFM2.5-1.2B-Instruct": { in: 0.02, out: 0.05 }, }; -// Conservative fallback for any model id not in the table above. -const ZGPU_FALLBACK = { in: 0.05, out: 0.4 }; +// Conservative fallback for any model id not in the table above: the priciest +// published rate, so an unlisted model never overstates savings. +const ZGPU_FALLBACK = { in: 0.05, out: 0.3 }; // Cadence: aim to show the savings note roughly once every 2–3 routed calls, // never twice in a row, and guaranteed within a bounded window. diff --git a/tests/responses.test.ts b/tests/responses.test.ts new file mode 100644 index 0000000..c2dd0ce --- /dev/null +++ b/tests/responses.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from "vitest"; +import { + extractOutputText, + extractReasoningText, + type ResponsesApiResponse, +} from "../src/lib/responses.js"; +import { toResponsesUsage } from "../src/lib/chatCompletions.js"; + +// gpt-oss-120b puts its reasoning trace ahead of the assistant message. +const reasoningPayload: ResponsesApiResponse = { + model: "gpt-oss-120b", + output: [ + { + type: "reasoning", + content: [{ type: "reasoning_text", text: "Think it through first." }], + }, + { + type: "message", + content: [{ type: "output_text", text: "The final answer." }], + }, + ], +}; + +describe("extractOutputText", () => { + it("skips the reasoning item and returns the assistant message", () => { + expect(extractOutputText(reasoningPayload)).toBe("The final answer."); + }); + + it("reads a single-message response", () => { + expect( + extractOutputText({ + output: [ + { type: "message", content: [{ type: "output_text", text: "Hi." }] }, + ], + }), + ).toBe("Hi."); + }); + + it("falls back to the first content part when parts are untyped", () => { + expect(extractOutputText({ output: [{ content: [{ text: "Hi." }] }] })).toBe( + "Hi.", + ); + }); + + it("returns undefined for an empty response", () => { + expect(extractOutputText({})).toBeUndefined(); + expect(extractOutputText({ output: [] })).toBeUndefined(); + }); +}); + +describe("extractReasoningText", () => { + it("returns the reasoning trace when present", () => { + expect(extractReasoningText(reasoningPayload)).toBe( + "Think it through first.", + ); + }); + + it("returns undefined when the model returned no reasoning", () => { + expect( + extractReasoningText({ + output: [ + { type: "message", content: [{ type: "output_text", text: "Hi." }] }, + ], + }), + ).toBeUndefined(); + }); +}); + +describe("toResponsesUsage", () => { + it("maps Chat Completions token names onto Responses token names", () => { + expect( + toResponsesUsage({ + prompt_tokens: 52, + completion_tokens: 143, + total_tokens: 195, + }), + ).toEqual({ input_tokens: 52, output_tokens: 143, total_tokens: 195 }); + }); + + it("passes through an absent usage block", () => { + expect(toResponsesUsage(undefined)).toBeUndefined(); + }); +}); diff --git a/tests/savings.test.ts b/tests/savings.test.ts index 2f7724c..ab855c6 100644 --- a/tests/savings.test.ts +++ b/tests/savings.test.ts @@ -60,14 +60,14 @@ describe("computeCallSavings", () => { const sonnet = computeCallSavings(740, 280, "claude-sonnet-4-6", "gliner2-base-v1").savingsUsd; expect(sonnet).toBeLessThan(opus); const claude = (740 * 3 + 280 * 15) / 1e6; // Sonnet: 0.00642 - const zgpu = (740 * 0.05 + 280 * 0.4) / 1e6; // gliner2 real cost + const zgpu = (740 * 0.02 + 280 * 0.05) / 1e6; // gliner2 real cost expect(sonnet).toBeCloseTo(claude - zgpu, 8); }); it("uses the per-model ZeroGPU rate (different models, different cost)", () => { // Same Claude baseline + same tokens, but a pricier ZeroGPU model → smaller savings. const cheap = computeCallSavings(740, 280, "claude-opus-4-8", "LFM2.5-1.2B-Instruct").savingsUsd; - const pricier = computeCallSavings(740, 280, "claude-opus-4-8", "gliner2-base-v1").savingsUsd; + const pricier = computeCallSavings(740, 280, "claude-opus-4-8", "qwen3-30b-a3b-fp8").savingsUsd; expect(cheap).toBeGreaterThan(pricier); }); @@ -78,9 +78,9 @@ describe("computeCallSavings", () => { }); it("falls back to a default ZeroGPU rate for an unknown model", () => { - const known = computeCallSavings(740, 280, "claude-opus-4-8", "gliner2-base-v1").savingsUsd; + const known = computeCallSavings(740, 280, "claude-opus-4-8", "qwen3-30b-a3b-fp8").savingsUsd; const unknown = computeCallSavings(740, 280, "claude-opus-4-8", "some-new-zgpu-model").savingsUsd; - // fallback equals gliner2's {0.05, 0.4} rate + // fallback equals the priciest published rate, qwen3's {0.05, 0.3} expect(unknown).toBeCloseTo(known, 10); }); });