From 33b2bfef2c3b21d0998e9ddc43ef50e8f56073d2 Mon Sep 17 00:00:00 2001 From: Jeremy Smith Date: Wed, 2 Sep 2026 07:33:24 +0100 Subject: [PATCH] Validate ids and token counts, add required items, hygiene for 0.2.0 - Reject duplicate item ids - Reject tokens fields and tokenizer results that are negative, NaN, Infinity or not numbers - Add optional required flag on Item; fit() throws naming the item and the token shortfall when a required item or pair group does not fit, and pair groups must agree on required - Add examples/demo.ts (runs with tsx) and stop ignoring examples/ - Correct the README on Anthropic token counting and document the new validation errors - Add CI workflow, CHANGELOG, homepage, bugs, sideEffects and engines - Bump to 0.2.0 Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 18 ++ .gitignore | 1 - CHANGELOG.md | 44 ++++ README.md | 90 +++++-- examples/demo.ts | 69 +++++ package-lock.json | 541 ++++++++++++++++++++++++++++++++++++--- package.json | 13 +- src/index.ts | 83 +++++- tests/fit.test.ts | 137 ++++++++++ 9 files changed, 939 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 examples/demo.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..903114c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm test + - run: npm run build diff --git a/.gitignore b/.gitignore index bf8e8ab..a8fdd94 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ node_modules/ dist/ *.tsbuildinfo -examples/ .env .env.* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..389718a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.2.0] - 2026-09-02 + +### Added + +- `required?: boolean` on `Item`. When a required item (or the pair group containing it) cannot be included, `fit()` throws `[snug] Required item "id" does not fit ...` naming the item and the token shortfall. Required items are still placed by priority. +- Validation that all items in a pair group agree on `required`. +- `examples/demo.ts`, a short runnable demo, and a `demo` script that runs it with `tsx`. +- `CHANGELOG.md` and a GitHub Actions CI workflow (test and build on Node 20). +- `homepage`, `bugs`, `sideEffects: false` and `engines.node >= 18` in `package.json`. + +### Fixed + +- Duplicate item ids are now rejected with `[snug] Duplicate item id "x"`. Previously costs and inclusion were keyed by id with no uniqueness check, so two items sharing an id could both be excluded even when one of them fitted. +- Token counts are validated. A `tokens` field or tokenizer result that is negative, `NaN`, `Infinity` or not a number now throws. Previously `tokens: -50` produced a negative `tokensUsed` and a `tokensRemaining` above the budget, and `tokens: NaN` silently never fitted. +- `examples/` is no longer gitignored, so the `demo` script works for anyone cloning the repository. + +### Changed + +- README: the token counting section no longer claims tiktoken counts Anthropic tokens. Anthropic has its own tokenizer and a `count_tokens` endpoint; pass a count from the API via the `tokens` field or accept an approximation. +- README documents `required` and every validation error. + +## [0.1.1] - 2026-04-05 + +Initial release, published to npm as `@jeremysnr/snug`. The repository history begins at this version; 0.1.0 was an earlier publish of the same code with no separate record. + +### Added + +- `fit(items, options)`: greedy selection by descending priority within a token budget, preserving input order. +- `pairId` for atomic pair groups (for example `tool_use` and `tool_result`), with validation that a group shares one priority. +- `tokens` field for pre-counted costs, `reserve` option, and a character-based fallback tokenizer with a suppressible warning. +- Validation of `budget`, `reserve` and `priority`. + +[Unreleased]: https://github.com/JeremySNR/snug/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/JeremySNR/snug/compare/v0.1.1...v0.2.0 +[0.1.1]: https://github.com/JeremySNR/snug/releases/tag/v0.1.1 diff --git a/README.md b/README.md index 846cc55..3f10c1c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **Fit prioritised content into a token budget.** -Every LLM application has the same problem: you have a context window of N tokens and need to fit a system prompt, conversation history, retrieved documents, and tool definitions into it — with space left for the model's reply. Every team writes their own solution from scratch. +Every LLM application has the same problem: you have a context window of N tokens and need to fit a system prompt, conversation history, retrieved documents, and tool definitions into it, with space left for the model's reply. Every team writes their own solution from scratch. `snug` is a single function that solves this once. @@ -16,15 +16,15 @@ import { fit } from '@jeremysnr/snug'; const { included } = fit( [ - { id: 'system', content: systemPrompt, priority: 100 }, + { id: 'system', content: systemPrompt, priority: 100, required: true }, { id: 'history', content: chatHistory, priority: 60 }, { id: 'rag', content: retrievedDocs, priority: 40 }, ], { budget: 8192, reserve: 1024, tokenizer: myTokenizer }, ); -// included — items that fit, in original input order -// excluded — items that didn't fit +// included: items that fit, in original input order +// excluded: items that didn't fit ``` Items are selected greedily in descending priority order. The result preserves original input order. Zero dependencies. Works in Node, Deno, Bun, and edge runtimes. @@ -47,17 +47,18 @@ fit(items: Item[], options: FitOptions): FitResult | Field | Type | Description | |-------|------|-------------| -| `id` | `string` | Unique identifier | -| `content` | `unknown` | Your content — not inspected by snug | +| `id` | `string` | Unique identifier. Duplicates throw. | +| `content` | `unknown` | Your content, not inspected by snug | | `priority` | `number` | Higher = included first | -| `tokens` | `number` | Pre-counted cost (optional — see below) | -| `pairId` | `string` | Atomic pair group (optional — see below) | +| `tokens` | `number` | Pre-counted cost (optional, see below) | +| `pairId` | `string` | Atomic pair group (optional, see below) | +| `required` | `boolean` | Throw instead of excluding if it does not fit (optional, see below) | **FitOptions** | Field | Type | Default | Description | |-------|------|---------|-------------| -| `budget` | `number` | — | Token limit for included items | +| `budget` | `number` | (required) | Token limit for included items | | `tokenizer` | `(text: string) => number` | built-in approx | Your token counter | | `reserve` | `number` | `0` | Tokens to hold back (e.g. for model response) | | `suppressApproximationWarning` | `boolean` | `false` | Silence the no-tokenizer warning | @@ -75,7 +76,7 @@ fit(items: Item[], options: FitOptions): FitResult ## Pair constraints -Anthropic's API requires strict 1:1 pairing between `tool_use` and `tool_result` messages — orphaning either half causes a 400 error. Mark paired items with a shared `pairId` and snug treats them as an atomic unit: both are included or neither is. +Anthropic's API requires strict 1:1 pairing between `tool_use` and `tool_result` messages. Orphaning either half causes a 400 error. Mark paired items with a shared `pairId` and snug treats them as an atomic unit: both are included or neither is. ```ts fit( @@ -87,20 +88,68 @@ fit( ); ``` -All items in a pair group must share the same `priority`. +All items in a pair group must share the same `priority` and the same `required` value. + +## Required items + +Some content must always be sent: a system prompt, the user's latest message, a tool result the model is waiting on. Mark it `required: true` and snug throws if it cannot be included, rather than silently dropping it and letting the request go out incomplete. + +```ts +fit( + [ + { id: 'system', content: systemPrompt, priority: 100, required: true }, + { id: 'latest', content: latestTurn, priority: 90, required: true }, + { id: 'rag', content: docs, priority: 40 }, + ], + { budget: 4096, reserve: 512, tokenizer }, +); +// Error: [snug] Required item "latest" does not fit: it needs 700 tokens but only +// 120 remain after higher-priority items (short by 580). Budget 4096, reserve 512. +``` + +Required items are still placed by priority. The flag does not promote an item above higher-priority optional items; it only changes what happens when the item does not fit. Give required items the highest priorities if they must be placed before optional content. + +If any item in a pair group is required, all items in that group must be required, and the error names the group. + +## Validation errors + +`fit()` throws an `Error` whose message starts with `[snug]` when the input cannot be trusted. Catch these during development; they indicate a bug in the calling code rather than a tight budget. + +| Condition | Message | +|-----------|---------| +| Two items share an `id` | `Duplicate item id "x". Item ids must be unique.` | +| `tokens` is negative, `NaN`, `Infinity`, or not a number | `Item "x" has an invalid \`tokens\` value: -50. Token counts must be finite numbers >= 0.` | +| The tokenizer returns a negative, `NaN`, `Infinity`, or non-number value | `Item "x" received an invalid token count from the tokenizer: NaN. ...` | +| `priority` is not finite | `Item "x" has a non-finite priority: Infinity` | +| Items in a pair group have different priorities | `All items in pair group "p" must have the same priority. Found 90 and 50.` | +| Items in a pair group disagree on `required` | `All items in pair group "p" must agree on \`required\`. Found true and false.` | +| A required item or pair group does not fit | `Required item "x" does not fit: it needs N tokens but only M remain after higher-priority items (short by S). Budget B, reserve R.` | +| `content` is not a string and `tokens` is missing | `Item "x" has no \`tokens\` field and its \`content\` is not a string.` | +| `budget` is not a positive finite number | `budget must be a positive finite number. Got: 0` | +| `reserve` is negative or not less than `budget` | `reserve (100) must be less than budget (100).` | ## Token counting -Pass any `(text: string) => number` function: +Pass any `(text: string) => number` function. + +**OpenAI models** use tiktoken. Pass the model name so tiktoken picks the right encoding (`gpt-4o` and newer use `o200k_base`; `gpt-4` and `gpt-3.5-turbo` use `cl100k_base`): ```ts -// tiktoken (OpenAI / Anthropic) import { encoding_for_model } from 'tiktoken'; const enc = encoding_for_model('gpt-4o'); const tokenizer = (text: string) => enc.encode(text).length; ``` -If you already have a token count (e.g. from an API usage response), pass it directly via the `tokens` field and skip counting entirely: +**Anthropic models** do not use tiktoken. Claude has its own tokenizer, which Anthropic does not publish as a library, and a tiktoken count will be off by a variable margin. For an exact count call the [count_tokens endpoint](https://docs.anthropic.com/en/api/messages-count-tokens) (`client.messages.countTokens(...)` in the SDK) and pass the result through the `tokens` field so snug never needs to count: + +```ts +const { input_tokens } = await client.messages.countTokens({ model, messages: [msg] }); +{ id: 'msg', content: msg, priority: 50, tokens: input_tokens } +``` + +If a network round trip per item is too expensive, use a tiktoken or character-based count as an approximation and keep a healthy `reserve` to absorb the error. + +If you already have a token count from any source (for example an API usage response), pass it via the `tokens` field and skip counting entirely: ```ts { id: 'msg', content: msg, priority: 50, tokens: 342 } @@ -108,12 +157,21 @@ If you already have a token count (e.g. from an API usage response), pass it dir When no tokenizer is supplied, snug falls back to `Math.ceil(text.length / 4)` and prints a warning. This is useful for prototyping but can be off by up to 37% in production. +## Demo + +``` +npm install +npm run demo +``` + +Runs [`examples/demo.ts`](./examples/demo.ts) with `tsx`. + ## Ecosystem | Package | What it does | |---------|-------------| -| `@jeremysnr/snug` | Zero-dependency core — bring your own tokenizer | -| [`@jeremysnr/snug-tiktoken`](https://github.com/JeremySNR/snug-tiktoken) | Pre-wired with tiktoken, model-agnostic | +| `@jeremysnr/snug` | Zero-dependency core, bring your own tokenizer | +| [`@jeremysnr/snug-tiktoken`](https://github.com/JeremySNR/snug-tiktoken) | Pre-wired with tiktoken for OpenAI encodings | | [`@jeremysnr/snug-openai`](https://github.com/JeremySNR/snug-openai) | Accepts OpenAI SDK message arrays directly | | [`@jeremysnr/snug-anthropic`](https://github.com/JeremySNR/snug-anthropic) | Accepts Anthropic SDK message arrays, auto-pairs tool messages | diff --git a/examples/demo.ts b/examples/demo.ts new file mode 100644 index 0000000..875bee3 --- /dev/null +++ b/examples/demo.ts @@ -0,0 +1,69 @@ +/** + * Runnable demo for snug. Run with `npm run demo` (uses tsx). + * + * Uses a deliberately crude tokenizer (one token per word) so the numbers are + * easy to follow. In real code, pass a tiktoken encoder or a count from your + * provider's API via the `tokens` field. + */ +import { fit } from '../src/index.js'; +import type { Item } from '../src/index.js'; + +const wordTokenizer = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length; + +const items: Item[] = [ + { + id: 'system', + content: 'You are a concise travel assistant. Answer in British English.', + priority: 100, + required: true, + }, + { + id: 'latest-user', + content: 'Which of the three hotels you mentioned has the best pool?', + priority: 90, + required: true, + }, + { + id: 'tool-use', + content: 'search_hotels({ city: "Lisbon", stars: 4 })', + priority: 70, + pairId: 'call-1', + }, + { + id: 'tool-result', + content: 'Found: Hotel Avenida (pool, rooftop), Casa do Bairro (no pool), Tejo Suites (indoor pool).', + priority: 70, + pairId: 'call-1', + }, + { + id: 'old-history', + content: + 'User: Hi, I am planning a long weekend in Lisbon in October with my partner. ' + + 'Assistant: Lovely choice. October is warm and quieter than the summer months.', + priority: 40, + }, + { + id: 'rag-doc', + content: + 'Lisbon guide: The city has seven hills, a historic tram network, and excellent seafood. ' + + 'Neighbourhoods include Alfama, Baixa, Chiado, Bairro Alto and Belem.', + priority: 30, + }, +]; + +function show(label: string, budget: number, reserve: number): void { + console.log(`\n== ${label}: budget ${budget}, reserve ${reserve}`); + try { + const result = fit(items, { budget, reserve, tokenizer: wordTokenizer }); + console.log(` included : ${result.included.map(i => i.id).join(', ')}`); + console.log(` excluded : ${result.excluded.map(i => i.id).join(', ') || '(none)'}`); + console.log(` used ${result.tokensUsed}, remaining ${result.tokensRemaining}`); + } catch (err) { + console.log(` threw: ${(err as Error).message}`); + } +} + +show('Everything fits', 200, 20); +show('Tight: low-priority items drop, tool pair stays together', 60, 10); +show('Very tight: the tool pair is dropped as a unit', 40, 5); +show('Too tight for a required item', 15, 5); diff --git a/package-lock.json b/package-lock.json index acd23b5..fa87be4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "snug", - "version": "0.1.1", + "name": "@jeremysnr/snug", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "snug", - "version": "0.1.1", + "name": "@jeremysnr/snug", + "version": "0.2.0", "license": "MIT", "devDependencies": { "@types/jest": "^29.5.12", @@ -14,7 +14,11 @@ "jest": "^29.7.0", "ts-jest": "^29.1.5", "tsup": "^8.5.1", + "tsx": "^4.19.0", "typescript": "^5.5.0" + }, + "engines": { + "node": ">=18" } }, "node_modules/@babel/code-frame": { @@ -2769,20 +2773,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/get-tsconfig": { - "version": "4.13.7", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.7.tgz", - "integrity": "sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -4400,17 +4390,6 @@ "node": ">=8" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "optional": true, - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/resolve.exports": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", @@ -4961,6 +4940,510 @@ "node": ">= 12" } }, + "node_modules/tsx": { + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", diff --git a/package.json b/package.json index bce663a..49c796f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@jeremysnr/snug", - "version": "0.1.1", + "version": "0.2.0", "description": "Fit prioritised content into a token budget. Zero dependencies.", "keywords": [ "llm", @@ -11,11 +11,19 @@ ], "license": "MIT", "author": "Jeremy Smith", + "homepage": "https://github.com/JeremySNR/snug#readme", + "bugs": { + "url": "https://github.com/JeremySNR/snug/issues" + }, "repository": { "type": "git", "url": "git+https://github.com/JeremySNR/snug.git" }, "type": "module", + "sideEffects": false, + "engines": { + "node": ">=18" + }, "main": "./dist/index.cjs", "module": "./dist/index.js", "types": "./dist/index.d.ts", @@ -34,7 +42,7 @@ "build": "tsup", "test": "node node_modules/jest/bin/jest.js", "prepublishOnly": "npm run build", - "demo": "npx tsx examples/demo.ts" + "demo": "tsx examples/demo.ts" }, "devDependencies": { "@types/jest": "^29.5.12", @@ -42,6 +50,7 @@ "jest": "^29.7.0", "ts-jest": "^29.1.5", "tsup": "^8.5.1", + "tsx": "^4.19.0", "typescript": "^5.5.0" } } diff --git a/src/index.ts b/src/index.ts index b4d0cd1..0703bfa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,12 @@ export interface Item { priority: number; /** Items sharing a pairId are included or excluded as a unit. */ pairId?: string; + /** + * When true, fit() throws instead of excluding this item (or the pair group + * containing it) if it cannot be included. Required items are still placed + * by priority; the flag only changes what happens when they do not fit. + */ + required?: boolean; } export interface FitOptions { @@ -28,33 +34,68 @@ const APPROX_WARNING = '(~4 chars/token). This can be off by up to 37% on large payloads. ' + 'Pass a real tokenizer via options.tokenizer for production use.'; -// ~4 chars/token. Not accurate — fallback only. +// ~4 chars/token. Not accurate. Fallback only. export function approximateTokens(text: string): number { return Math.ceil(text.length / 4); } +function assertTokenCount(item: Item, value: unknown, source: 'tokens' | 'tokenizer'): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + const where = source === 'tokens' + ? 'has an invalid `tokens` value' + : 'received an invalid token count from the tokenizer'; + throw new Error( + `[snug] Item "${item.id}" ${where}: ${String(value)}. ` + + 'Token counts must be finite numbers >= 0.', + ); + } + return value; +} + function resolveTokens(item: Item, tokenizer: Tokenizer): number { - if (item.tokens !== undefined) return item.tokens; - if (typeof item.content === 'string') return tokenizer(item.content); + if (item.tokens !== undefined) return assertTokenCount(item, item.tokens, 'tokens'); + if (typeof item.content === 'string') { + return assertTokenCount(item, tokenizer(item.content), 'tokenizer'); + } throw new Error( `[snug] Item "${item.id}" has no \`tokens\` field and its \`content\` is not a string.`, ); } function validateItems(items: Item[]): void { + const seenIds = new Set(); const pairPriority = new Map(); + const pairRequired = new Map(); + for (const item of items) { + if (seenIds.has(item.id)) { + throw new Error(`[snug] Duplicate item id "${item.id}". Item ids must be unique.`); + } + seenIds.add(item.id); + if (!Number.isFinite(item.priority)) { throw new Error(`[snug] Item "${item.id}" has a non-finite priority: ${item.priority}`); } + if (item.pairId !== undefined) { - const existing = pairPriority.get(item.pairId); - if (existing === undefined) { + const existingPriority = pairPriority.get(item.pairId); + if (existingPriority === undefined) { pairPriority.set(item.pairId, item.priority); - } else if (existing !== item.priority) { + } else if (existingPriority !== item.priority) { throw new Error( `[snug] All items in pair group "${item.pairId}" must have the same priority. ` + - `Found ${existing} and ${item.priority}.`, + `Found ${existingPriority} and ${item.priority}.`, + ); + } + + const required = item.required === true; + const existingRequired = pairRequired.get(item.pairId); + if (existingRequired === undefined) { + pairRequired.set(item.pairId, required); + } else if (existingRequired !== required) { + throw new Error( + `[snug] All items in pair group "${item.pairId}" must agree on \`required\`. ` + + `Found ${existingRequired} and ${required}.`, ); } } @@ -86,14 +127,26 @@ export function fit(items: T[], options: FitOptions): FitResult< items.map(item => [item.id, resolveTokens(item, tokenizer)]), ); - interface Group { items: T[]; totalTokens: number; priority: number; firstIndex: number } + interface Group { + items: T[]; + totalTokens: number; + priority: number; + firstIndex: number; + required: boolean; + } const groupMap = new Map(); for (let i = 0; i < items.length; i++) { const item = items[i]; const key = item.pairId ?? item.id; if (!groupMap.has(key)) { - groupMap.set(key, { items: [], totalTokens: 0, priority: item.priority, firstIndex: i }); + groupMap.set(key, { + items: [], + totalTokens: 0, + priority: item.priority, + firstIndex: i, + required: item.required === true, + }); } const g = groupMap.get(key)!; g.items.push(item); @@ -111,6 +164,18 @@ export function fit(items: T[], options: FitOptions): FitResult< if (tokensUsed + g.totalTokens <= effectiveBudget) { for (const item of g.items) includedIds.add(item.id); tokensUsed += g.totalTokens; + } else if (g.required) { + const remaining = effectiveBudget - tokensUsed; + const shortfall = g.totalTokens - remaining; + const first = g.items[0]; + const label = g.items.length > 1 + ? `Required item "${first.id}" (pair group "${first.pairId}")` + : `Required item "${first.id}"`; + throw new Error( + `[snug] ${label} does not fit: it needs ${g.totalTokens} tokens but only ` + + `${remaining} remain after higher-priority items (short by ${shortfall}). ` + + `Budget ${budget}, reserve ${reserve}.`, + ); } } diff --git a/tests/fit.test.ts b/tests/fit.test.ts index f87c483..2a2cea1 100644 --- a/tests/fit.test.ts +++ b/tests/fit.test.ts @@ -159,6 +159,143 @@ describe('validation', () => { const items: Item[] = [{ id: 'x', content: { nested: true }, priority: 10 }]; expect(() => fit(items, { budget: 100, tokenizer: t })).toThrow(/no `tokens` field/); }); + + test('throws on duplicate item ids', () => { + expect(() => + fit([itemT('x', '', 10, 5), itemT('x', '', 10, 9)], { budget: 6, tokenizer: t }), + ).toThrow('[snug] Duplicate item id "x"'); + }); + + test('throws on a negative tokens field', () => { + expect(() => fit([itemT('neg', '', 10, -50)], { budget: 100, tokenizer: t })) + .toThrow(/Item "neg" has an invalid `tokens` value: -50/); + }); + + test('throws on a NaN tokens field', () => { + expect(() => fit([itemT('nan', '', 10, NaN)], { budget: 100, tokenizer: t })) + .toThrow(/Item "nan" has an invalid `tokens` value: NaN/); + }); + + test('throws on an Infinity tokens field', () => { + expect(() => fit([itemT('inf', '', 10, Infinity)], { budget: 100, tokenizer: t })) + .toThrow(/invalid `tokens` value: Infinity/); + }); + + test('throws when the tokenizer returns NaN', () => { + expect(() => fit([item('a', 'hi', 10)], { budget: 100, tokenizer: () => NaN })) + .toThrow(/Item "a" received an invalid token count from the tokenizer: NaN/); + }); + + test('throws when the tokenizer returns a negative number', () => { + expect(() => fit([item('a', 'hi', 10)], { budget: 100, tokenizer: () => -1 })) + .toThrow(/invalid token count from the tokenizer: -1/); + }); + + test('throws when the tokenizer returns a non-number', () => { + const bad = (() => '3') as unknown as (text: string) => number; + expect(() => fit([item('a', 'hi', 10)], { budget: 100, tokenizer: bad })) + .toThrow(/invalid token count from the tokenizer: 3/); + }); + + test('accepts a zero token count', () => { + const result = fit([itemT('empty', '', 10, 0)], { budget: 5, tokenizer: t }); + expect(result.included.map(i => i.id)).toEqual(['empty']); + expect(result.tokensUsed).toBe(0); + }); +}); + +describe('required items', () => { + const req = (id: string, content: string, priority: number, pairId?: string): Item => + ({ id, content, priority, pairId, required: true }); + + test('required item is included when it fits', () => { + const result = fit([req('sys', 'aaaaa', 100), item('opt', 'bbb', 10)], { + budget: 20, + tokenizer: t, + }); + expect(result.included.map(i => i.id)).toEqual(['sys', 'opt']); + }); + + test('throws when a required item does not fit', () => { + expect(() => fit([req('sys', 'a'.repeat(12), 100)], { budget: 10, tokenizer: t })) + .toThrow('[snug] Required item "sys" does not fit'); + }); + + test('error names the item and the token shortfall', () => { + expect(() => fit([req('sys', 'a'.repeat(12), 100)], { budget: 10, tokenizer: t })) + .toThrow(/needs 12 tokens but only 10 remain.*short by 2/); + }); + + test('shortfall accounts for higher-priority items already placed', () => { + expect(() => + fit([item('high', 'a'.repeat(6), 100), req('must', 'b'.repeat(6), 50)], { + budget: 10, + tokenizer: t, + }), + ).toThrow(/needs 6 tokens but only 4 remain.*short by 2/); + }); + + test('required items are still placed by priority', () => { + // 'high' is not required but outranks 'must'; it is placed first and + // consumes the budget, so 'must' cannot fit and fit() throws rather than + // silently promoting the required item. + expect(() => + fit([item('high', 'a'.repeat(8), 100), req('must', 'b'.repeat(5), 50)], { + budget: 10, + tokenizer: t, + }), + ).toThrow(/Required item "must"/); + + // When there is room after the higher-priority item, both are included. + const result = fit([item('high', 'a'.repeat(5), 100), req('must', 'b'.repeat(5), 50)], { + budget: 10, + tokenizer: t, + }); + expect(result.included.map(i => i.id)).toEqual(['high', 'must']); + }); + + test('optional items are still excluded normally alongside required ones', () => { + const result = fit( + [req('sys', 'aaaaa', 100), item('big', 'b'.repeat(50), 60), item('small', 'cc', 40)], + { budget: 10, tokenizer: t }, + ); + expect(result.included.map(i => i.id)).toEqual(['sys', 'small']); + expect(result.excluded.map(i => i.id)).toEqual(['big']); + }); + + test('required pair group throws when the pair does not fit', () => { + expect(() => + fit([req('use', 'a'.repeat(6), 80, 'p1'), req('result', 'b'.repeat(6), 80, 'p1')], { + budget: 10, + tokenizer: t, + }), + ).toThrow(/Required item "use" \(pair group "p1"\) does not fit: it needs 12 tokens/); + }); + + test('required pair group is included when it fits', () => { + const result = fit( + [req('use', 'abc', 80, 'p1'), req('result', 'def', 80, 'p1')], + { budget: 10, tokenizer: t }, + ); + expect(result.included.map(i => i.id)).toEqual(['use', 'result']); + }); + + test('throws when items in a pair group disagree on required', () => { + expect(() => + fit([req('use', 'abc', 80, 'p1'), item('result', 'def', 80, 'p1')], { + budget: 100, + tokenizer: t, + }), + ).toThrow(/pair group "p1" must agree on `required`/); + }); + + test('required: false is treated the same as omitted in a pair group', () => { + const items: Item[] = [ + { id: 'use', content: 'abc', priority: 80, pairId: 'p1', required: false }, + { id: 'result', content: 'def', priority: 80, pairId: 'p1' }, + ]; + expect(() => fit(items, { budget: 100, tokenizer: t })).not.toThrow(); + }); }); describe('no tokenizer', () => {