From 34da3b23b08e96667533651713f2c70f5056a11b Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Mon, 19 Oct 2026 21:11:32 -0500 Subject: [PATCH 1/7] Add embedding layer: titler crate, C API, Python package, npm package app/titler is a thread-safe facade over core and the pure-Rust CPU backend. app/capi exposes it as a C library with include/titler.h; app/python wraps that with ctypes and builds platform wheels; app/npm packages the WebAssembly module as titler-wasm. EMBEDDING.md explains each. Tests: Rust (4), C smoke, Python (7), Node (3). CI and release workflows build and test all of them. Load time 32 ms, 25 MB memory. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 37 ++++++++++ .github/workflows/release.yml | 70 +++++++++++++++++++ .gitignore | 9 +++ app/Cargo.lock | 15 ++++ app/Cargo.toml | 2 +- app/EMBEDDING.md | 91 ++++++++++++++++++++++++ app/README.md | 6 ++ app/capi/Cargo.toml | 13 ++++ app/capi/include/titler.h | 53 ++++++++++++++ app/capi/src/lib.rs | 119 ++++++++++++++++++++++++++++++++ app/capi/tests/run-smoke.sh | 20 ++++++ app/capi/tests/smoke.c | 34 +++++++++ app/npm/README.md | 16 +++++ app/npm/build.sh | 9 +++ app/npm/index.d.ts | 13 ++++ app/npm/index.js | 41 +++++++++++ app/npm/package.json | 19 +++++ app/npm/test/titler.test.js | 26 +++++++ app/python/README.md | 15 ++++ app/python/build_wheel.py | 43 ++++++++++++ app/python/pyproject.toml | 23 ++++++ app/python/tests/test_titler.py | 56 +++++++++++++++ app/python/titler/__init__.py | 100 +++++++++++++++++++++++++++ app/titler/Cargo.toml | 13 ++++ app/titler/src/lib.rs | 53 ++++++++++++++ app/titler/tests/titles.rs | 48 +++++++++++++ 26 files changed, 943 insertions(+), 1 deletion(-) create mode 100644 app/EMBEDDING.md create mode 100644 app/capi/Cargo.toml create mode 100644 app/capi/include/titler.h create mode 100644 app/capi/src/lib.rs create mode 100644 app/capi/tests/run-smoke.sh create mode 100644 app/capi/tests/smoke.c create mode 100644 app/npm/README.md create mode 100644 app/npm/build.sh create mode 100644 app/npm/index.d.ts create mode 100644 app/npm/index.js create mode 100644 app/npm/package.json create mode 100644 app/npm/test/titler.test.js create mode 100644 app/python/README.md create mode 100644 app/python/build_wheel.py create mode 100644 app/python/pyproject.toml create mode 100644 app/python/tests/test_titler.py create mode 100644 app/python/titler/__init__.py create mode 100644 app/titler/Cargo.toml create mode 100644 app/titler/src/lib.rs create mode 100644 app/titler/tests/titles.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bb3abf..2d23d17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,40 @@ jobs: - name: WebAssembly build working-directory: app run: cargo build --release -p titler-wasm --target wasm32-unknown-unknown + + bindings: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + workspaces: app + - uses: taiki-e/install-action@v2 + with: + tool: wasm-pack + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Rust API tests + working-directory: app + run: cargo test --release -p titler + - name: C API smoke test + working-directory: app + run: ./capi/tests/run-smoke.sh + - name: Python package tests + working-directory: app/python + run: | + pip install pytest + TITLER_LIB="$GITHUB_WORKSPACE/app/target/release/libtitler_c.so" python -m pytest tests -q + - name: npm package tests + working-directory: app + run: | + wasm-pack build wasm --target web --release --out-dir ../npm/pkg --out-name titler_wasm + cp model/titler-v3-q4_k_m.gguf npm/titler.gguf + cd npm && npm test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57b0ceb..38a49fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,3 +32,73 @@ jobs: - uses: softprops/action-gh-release@v2 with: files: titler-${{ matrix.name }}* + + libraries: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, name: linux-x86_64 } + - { os: windows-latest, name: windows-x86_64 } + - { os: macos-latest, name: macos-arm64 } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: app + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build the C library + working-directory: app + run: cargo build --release -p titler-capi + - name: Zip the C library with its header + shell: bash + working-directory: app + run: | + files="capi/include/titler.h" + for f in target/release/titler_c.dll target/release/titler_c.dll.lib target/release/titler_c.lib target/release/libtitler_c.so target/release/libtitler_c.dylib target/release/libtitler_c.a; do + [ -f "$f" ] && files="$files $f" + done + python -m zipfile -c "../titler-capi-${{ matrix.name }}.zip" $files + - name: Build the Python wheel + shell: bash + working-directory: app/python + run: | + pip install build wheel setuptools + python build_wheel.py + cp dist/*.whl ../.. + - uses: softprops/action-gh-release@v2 + with: + files: | + titler-capi-${{ matrix.name }}.zip + titler-*.whl + + npm-and-model: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + workspaces: app + - uses: taiki-e/install-action@v2 + with: + tool: wasm-pack + - name: Build the npm package + working-directory: app + run: | + wasm-pack build wasm --target web --release --out-dir ../npm/pkg --out-name titler_wasm + cp model/titler-v3-q4_k_m.gguf npm/titler.gguf + cd npm && npm pack + cp titler-wasm-*.tgz ../.. + cp ../model/titler-v3-q4_k_m.gguf ../../titler-v3-q4_k_m.gguf + - uses: softprops/action-gh-release@v2 + with: + files: | + titler-wasm-*.tgz + titler-v3-q4_k_m.gguf diff --git a/.gitignore b/.gitignore index 3be86b0..b57bd48 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,12 @@ node_modules/ # Rust build output app/target/ app/website/dist/ +app/npm/pkg/ +app/npm/titler.gguf +app/python/dist/ +app/python/build/ +app/python/*.egg-info/ +app/python/titler/*.dll +app/python/titler/*.so +app/python/titler/*.dylib +app/python/titler/*.gguf diff --git a/app/Cargo.lock b/app/Cargo.lock index 88928fd..28639df 100644 --- a/app/Cargo.lock +++ b/app/Cargo.lock @@ -442,6 +442,21 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "titler" +version = "0.1.0" +dependencies = [ + "titler-core", + "titler-cpu", +] + +[[package]] +name = "titler-capi" +version = "0.1.0" +dependencies = [ + "titler", +] + [[package]] name = "titler-cli" version = "0.1.0" diff --git a/app/Cargo.toml b/app/Cargo.toml index 3cde4cf..04c9584 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["core", "cpu", "llama", "cli", "wasm"] +members = ["core", "cpu", "titler", "llama", "cli", "wasm", "capi"] [workspace.package] version = "0.1.0" diff --git a/app/EMBEDDING.md b/app/EMBEDDING.md new file mode 100644 index 0000000..30c65d7 --- /dev/null +++ b/app/EMBEDDING.md @@ -0,0 +1,91 @@ +# Use titler in your app + +titler turns a prompt into a short title (8 words at most). The model is one 4 MB file. It runs on the CPU, offline, +with no telemetry. Pick the way that fits your app: + +| Your app is | Use | Install | +|---|---|---| +| Rust | the `titler` crate | `titler = { git = "https://github.com/Cyronius/titler" }` | +| C, C++, C#, Go, Swift, Java (anything that loads a DLL) | the C library | `titler-capi-.zip` from the [releases](https://github.com/Cyronius/titler/releases) | +| Python | the `titler` wheel | `pip install titler-0.1.0-py3-none-.whl` from the releases | +| A web page or Node | `titler-wasm` | `npm install titler-wasm-0.1.0.tgz` from the releases | +| A script or shell | the `titler` command | `titler-` from the releases | + +The model file is `titler-v3-q4_k_m.gguf` in each release and in `app/model/`. The wheel, the npm package and the +command-line tool already contain it. + +## Rust + +```rust +let titler = titler::Titler::from_path("titler-v3-q4_k_m.gguf")?; // load once +let title = titler.title("how do I sort a list of dicts by a key in python")?; +``` + +`Titler` is `Send + Sync`. Share one between threads (in an `Arc`); `title` takes `&self`. `title_with_beams(prompt, 1)` is +the fast setting. For a faster native build, use `titler-llama` (llama.cpp, needs a C++ compiler) in place of `titler`. + +## C and everything that can call C + +Header: `capi/include/titler.h`. Library: `titler_c.dll`, `libtitler_c.so` or `libtitler_c.dylib` (and a static library). + +```c +titler_t *t = titler_new_from_file("titler-v3-q4_k_m.gguf"); +char out[256]; +int64_t n = titler_title(t, "how do I sort a list of dicts by a key in python", 0 /* default beams */, out, sizeof out); +if (n < 0) fprintf(stderr, "%s\n", titler_last_error()); +titler_free(t); +``` + +- Strings are UTF-8. A 256-byte buffer holds any title. The return value is the full length; if it is `cap` or more, the title was cut. +- `titler_title` can be called from many threads at once on one handle. Free the handle only when no call is running. +- Errors return NULL or -1. The message is in `titler_last_error()` for the calling thread. +- C#: `[DllImport("titler_c")]` with the same signatures. Go: cgo or `purego`. Both call plain C functions. + +## Python + +```python +from titler import Titler +t = Titler() # the model inside the wheel; or Titler("path.gguf") +print(t.title("how do I sort a list of dicts by a key in python")) +``` + +A `Titler` can be shared between threads. The wheel is built per platform (the C library is inside it). + +## Web and Node + +```js +import { createTitler } from "titler-wasm"; +const titler = await createTitler(); // loads the 4 MB model +titler.title("how do I sort a list of dicts by a key in python"); +``` + +In a browser, run it in a Web Worker so the page does not freeze; `website/worker.js` is a complete example. +Host `pkg/titler_wasm_bg.wasm` and `titler.gguf` as static files next to your script. + +## What you get + +| | | +|---|---| +| Output | 3 to 8 words in Title Case, for example `Sorting Dicts By Key in Python`. Same input, same output. | +| Input | Any text. Only the first 256 tokens (about 1,000 characters of English) are read. English works best. | +| Speed, 4 beams | Native Rust (one thread): median 40-70 ms, slowest 5% 200-275 ms (long prompts). llama.cpp: 18 ms / 48 ms. Browser: 50-175 ms. | +| Speed, 1 beam | About half of that, with slightly worse titles. | +| Memory | About 25 MB after loading (weights are held as 32-bit numbers). Load time about 30 ms. | +| Quality | Judged against the previous titler (v1.5): the new titles win about 65% of the pairs that have a winner. | +| Beams | 4 is the default the model was judged with. The maximum is 8. | + +## Limits + +- It titles; it does not summarize, translate or answer. +- It sometimes drops an apostrophe (`Nick Wildes Judy Hopp Dialog Line`). Weak inputs are non-English text, vague one-liners and pasted data. +- The training data and its licenses are listed in `model/NOTICE.md`. + +## Build from source + +``` +cd app +cargo build --release -p titler-capi # C library -> target/release/ +cargo build --release -p titler-cli # command-line tool +python python/build_wheel.py # Python wheel (from app/python) +npm/build.sh # npm package (needs wasm-bindgen-cli, see README.md) +``` diff --git a/app/README.md b/app/README.md index 5531d11..c539a98 100644 --- a/app/README.md +++ b/app/README.md @@ -5,6 +5,10 @@ A 4 MB model that turns a prompt into a short title. It runs on the CPU in about | Folder | Content | |---|---| | `core/` | Tokenizer, beam search, title cleanup. No I/O. The model side is a `Backend` trait. | +| `titler/` | The `titler` crate: load a model, call `title`. Shares one model between threads. | +| `capi/` | C library and `include/titler.h`. | +| `python/` | Python package (ctypes over the C library) and wheel builder. | +| `npm/` | The `titler-wasm` npm package. | | `cpu/` | `Backend` in plain Rust (no C++, no dependencies). It runs in the browser. | | `llama/` | `Backend` for llama.cpp (through `llama-cpp-2`). Faster on a desktop. | | `cli/` | The `titler` command. The model is built into the binary. | @@ -12,6 +16,8 @@ A 4 MB model that turns a prompt into a short title. It runs on the CPU in about | `website/` | The demo page. `build-site.sh` builds it into `website/dist/`. | | `model/` | The shipped GGUF (Q4_K_M, 5.84M parameters) and `NOTICE.md` (data licenses). | +To use titler in your own app (Rust, C, Python, JavaScript), see [EMBEDDING.md](EMBEDDING.md). + ## Use ``` diff --git a/app/capi/Cargo.toml b/app/capi/Cargo.toml new file mode 100644 index 0000000..fa4d039 --- /dev/null +++ b/app/capi/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "titler-capi" +description = "C interface to titler (include/titler.h)" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +name = "titler_c" +crate-type = ["cdylib", "staticlib", "rlib"] + +[dependencies] +titler = { path = "../titler" } diff --git a/app/capi/include/titler.h b/app/capi/include/titler.h new file mode 100644 index 0000000..c31a085 --- /dev/null +++ b/app/capi/include/titler.h @@ -0,0 +1,53 @@ +/* titler: short titles for prompts, from a 4 MB model that runs on the CPU. + * + * Load the model once with titler_new_*, then call titler_title from any number of threads at the same time. + * Strings are UTF-8. Functions that can fail return NULL or -1 and set a message for titler_last_error(). + */ +#ifndef TITLER_H +#define TITLER_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN32) && defined(TITLER_DLL_IMPORT) +#define TITLER_API __declspec(dllimport) +#else +#define TITLER_API +#endif + +typedef struct TitlerHandle titler_t; + +/* Load the model from a GGUF file. Returns NULL on error. */ +TITLER_API titler_t *titler_new_from_file(const char *path); + +/* Load the model from GGUF bytes already in memory. The bytes are copied; free them afterwards if you like. */ +TITLER_API titler_t *titler_new_from_memory(const uint8_t *data, size_t len); + +/* Free a handle. NULL is fine. Do not use the handle afterwards, and do not free it while another thread uses it. */ +TITLER_API void titler_free(titler_t *t); + +/* Write the title for a NUL-terminated prompt into out (at most cap bytes, including the NUL). + * beams: 4 is the default (use 0 or less for it); 1 is fastest; the maximum is 8. + * Returns the full length of the title in bytes, not counting the NUL, or -1 on error. + * If the return value is cap or more, the title was cut short at a character boundary. A buffer of 256 bytes is enough + * for any title (8 words). Invalid UTF-8 in the prompt is replaced, not rejected. */ +TITLER_API int64_t titler_title(const titler_t *t, const char *prompt, int beams, char *out, size_t cap); + +/* Same, for a prompt that is not NUL-terminated or contains NUL bytes. */ +TITLER_API int64_t titler_title_n(const titler_t *t, const uint8_t *prompt, size_t len, int beams, char *out, size_t cap); + +/* The message for the last failure on the calling thread. Valid until the next titler call on this thread. */ +TITLER_API const char *titler_last_error(void); + +/* Library version, for example "0.1.0". */ +TITLER_API const char *titler_version(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/app/capi/src/lib.rs b/app/capi/src/lib.rs new file mode 100644 index 0000000..c4610aa --- /dev/null +++ b/app/capi/src/lib.rs @@ -0,0 +1,119 @@ +//! C interface. See include/titler.h for the contract. + +use std::cell::RefCell; +use std::ffi::{c_char, c_int, CStr, CString}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::slice; + +pub struct TitlerHandle(titler::Titler); + +thread_local! { + static LAST_ERROR: RefCell = RefCell::new(CString::default()); +} + +fn set_error(message: impl Into>) { + let c = CString::new(message).unwrap_or_else(|_| CString::new("error").unwrap()); + LAST_ERROR.with(|e| *e.borrow_mut() = c); +} + +fn guard(fail: T, f: impl FnOnce() -> Result) -> T { + match catch_unwind(AssertUnwindSafe(f)) { + Ok(Ok(v)) => v, + Ok(Err(e)) => { + set_error(e); + fail + } + Err(_) => { + set_error("internal error (panic)"); + fail + } + } +} + +fn boxed(t: titler::Result) -> Result<*mut TitlerHandle, String> { + t.map(|t| Box::into_raw(Box::new(TitlerHandle(t)))).map_err(|e| e.to_string()) +} + +/// # Safety +/// `path` must be a valid NUL-terminated string. +#[no_mangle] +pub unsafe extern "C" fn titler_new_from_file(path: *const c_char) -> *mut TitlerHandle { + guard(std::ptr::null_mut(), || { + if path.is_null() { + return Err("path is NULL".into()); + } + let path = CStr::from_ptr(path).to_string_lossy().into_owned(); + boxed(titler::Titler::from_path(path)) + }) +} + +/// # Safety +/// `data` must point to `len` readable bytes. +#[no_mangle] +pub unsafe extern "C" fn titler_new_from_memory(data: *const u8, len: usize) -> *mut TitlerHandle { + guard(std::ptr::null_mut(), || { + if data.is_null() { + return Err("data is NULL".into()); + } + boxed(titler::Titler::from_gguf_bytes(slice::from_raw_parts(data, len))) + }) +} + +/// # Safety +/// `t` must come from a `titler_new_*` call and not be used afterwards. NULL is allowed. +#[no_mangle] +pub unsafe extern "C" fn titler_free(t: *mut TitlerHandle) { + if !t.is_null() { + drop(Box::from_raw(t)); + } +} + +unsafe fn title_into(t: *const TitlerHandle, prompt: &[u8], beams: c_int, out: *mut c_char, cap: usize) -> i64 { + guard(-1, || { + let handle = t.as_ref().ok_or("titler handle is NULL")?; + let beams = if beams <= 0 { titler::DEFAULT_BEAMS } else { beams as usize }; + let title = handle.0.title_with_beams(&String::from_utf8_lossy(prompt), beams).map_err(|e| e.to_string())?; + if !out.is_null() && cap > 0 { + let mut n = title.len().min(cap - 1); + while !title.is_char_boundary(n) { + n -= 1; + } + std::ptr::copy_nonoverlapping(title.as_ptr(), out as *mut u8, n); + *out.add(n) = 0; + } + Ok(title.len() as i64) + }) +} + +/// # Safety +/// `t` is a live handle, `prompt` a NUL-terminated string, `out` has room for `cap` bytes. +#[no_mangle] +pub unsafe extern "C" fn titler_title(t: *const TitlerHandle, prompt: *const c_char, beams: c_int, out: *mut c_char, cap: usize) -> i64 { + if prompt.is_null() { + set_error("prompt is NULL"); + return -1; + } + title_into(t, CStr::from_ptr(prompt).to_bytes(), beams, out, cap) +} + +/// # Safety +/// `t` is a live handle, `prompt` has `len` readable bytes, `out` has room for `cap` bytes. +#[no_mangle] +pub unsafe extern "C" fn titler_title_n(t: *const TitlerHandle, prompt: *const u8, len: usize, beams: c_int, out: *mut c_char, cap: usize) -> i64 { + if prompt.is_null() && len > 0 { + set_error("prompt is NULL"); + return -1; + } + let bytes = if len == 0 { &[][..] } else { slice::from_raw_parts(prompt, len) }; + title_into(t, bytes, beams, out, cap) +} + +#[no_mangle] +pub extern "C" fn titler_last_error() -> *const c_char { + LAST_ERROR.with(|e| e.borrow().as_ptr()) +} + +#[no_mangle] +pub extern "C" fn titler_version() -> *const c_char { + concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char +} diff --git a/app/capi/tests/run-smoke.sh b/app/capi/tests/run-smoke.sh new file mode 100644 index 0000000..c31c395 --- /dev/null +++ b/app/capi/tests/run-smoke.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Builds the C library and runs capi/tests/smoke.c against it. Run from app/. +set -euo pipefail +cd "$(dirname "$0")/../.." +cargo build --release -p titler-capi +out=$(mktemp -d) +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + out=$(cygpath -m "$out") + # Needs a Visual Studio developer shell (cl.exe and the Windows headers on the path). + cl -nologo -Icapi/include -DTITLER_DLL_IMPORT capi/tests/smoke.c target/release/titler_c.dll.lib -Fe:"$out/smoke.exe" -Fo:"$out/" + cp target/release/titler_c.dll "$out/" + "$out/smoke.exe" model/titler-v3-q4_k_m.gguf ;; + Darwin) + cc -Icapi/include capi/tests/smoke.c -Ltarget/release -ltitler_c -o "$out/smoke" + DYLD_LIBRARY_PATH=target/release "$out/smoke" model/titler-v3-q4_k_m.gguf ;; + *) + cc -Icapi/include capi/tests/smoke.c -Ltarget/release -ltitler_c -o "$out/smoke" + LD_LIBRARY_PATH=target/release "$out/smoke" model/titler-v3-q4_k_m.gguf ;; +esac diff --git a/app/capi/tests/smoke.c b/app/capi/tests/smoke.c new file mode 100644 index 0000000..4a5a87c --- /dev/null +++ b/app/capi/tests/smoke.c @@ -0,0 +1,34 @@ +/* Built and run by capi/tests/run-smoke.sh. Usage: smoke MODEL.gguf */ +#include +#include +#include + +#define CHECK(cond, msg) do { if (!(cond)) { fprintf(stderr, "FAIL: %s (%s)\n", msg, titler_last_error()); return 1; } } while (0) + +int main(int argc, char **argv) { + CHECK(argc == 2, "usage: smoke MODEL.gguf"); + CHECK(titler_new_from_file("no-such-file.gguf") == NULL, "missing file must fail"); + CHECK(strlen(titler_last_error()) > 0, "error message set"); + + titler_t *t = titler_new_from_file(argv[1]); + CHECK(t != NULL, "load model"); + + char out[256]; + int64_t n = titler_title(t, "how do I sort a list of dicts by a key in python", 0, out, sizeof out); + CHECK(n > 0 && n < (int64_t)sizeof out, "title length"); + CHECK(strlen(out) == (size_t)n, "title is NUL-terminated at its length"); + printf("%s\n", out); + + char tiny[8]; + n = titler_title(t, "how do I sort a list of dicts by a key in python", 1, tiny, sizeof tiny); + CHECK(n >= (int64_t)sizeof tiny && strlen(tiny) == sizeof tiny - 1, "short buffer is cut, full length returned"); + + CHECK(titler_title(t, NULL, 4, out, sizeof out) == -1, "NULL prompt fails"); + CHECK(titler_title(NULL, "x", 4, out, sizeof out) == -1, "NULL handle fails"); + CHECK(titler_title_n(t, (const uint8_t *)"", 0, 4, out, sizeof out) >= 0, "empty prompt works"); + + titler_free(t); + titler_free(NULL); + printf("titler %s ok\n", titler_version()); + return 0; +} diff --git a/app/npm/README.md b/app/npm/README.md new file mode 100644 index 0000000..617cdf3 --- /dev/null +++ b/app/npm/README.md @@ -0,0 +1,16 @@ +# titler-wasm + +A 4 MB model that turns a prompt into a short title. It runs on the CPU with WebAssembly, in the browser and in Node. +Nothing is sent to a server. + +```js +import { createTitler } from "titler-wasm"; + +const titler = await createTitler(); // loads the model in this package (4 MB) +console.log(titler.title("how do I sort a list of dicts by a key in python")); // Sorting Dicts By Key in Python +console.log(titler.title("a long prompt…", { beams: 1 })); // 1 beam is fastest +``` + +- `createTitler({ model })` takes a URL, a file path (Node) or the GGUF bytes, if you host the model yourself. +- A title takes about 50-150 ms. Run it in a Web Worker so that the page stays smooth (see `app/website/worker.js` in the repository). +- With a bundler, make sure `pkg/titler_wasm_bg.wasm` and `titler.gguf` are copied as files; the package finds them with `new URL(..., import.meta.url)`. diff --git a/app/npm/build.sh b/app/npm/build.sh new file mode 100644 index 0000000..cb1952b --- /dev/null +++ b/app/npm/build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Builds the npm package in app/npm: the WebAssembly module, its JavaScript glue and the model. Run from anywhere. +set -euo pipefail +cd "$(dirname "$0")/.." +cargo build --release -p titler-wasm --target wasm32-unknown-unknown +rm -rf npm/pkg +wasm-bindgen --target web --out-dir npm/pkg --out-name titler_wasm target/wasm32-unknown-unknown/release/titler_wasm.wasm +cp model/titler-v3-q4_k_m.gguf npm/titler.gguf +echo "npm package ready in app/npm" diff --git a/app/npm/index.d.ts b/app/npm/index.d.ts new file mode 100644 index 0000000..a36452f --- /dev/null +++ b/app/npm/index.d.ts @@ -0,0 +1,13 @@ +export interface Titler { + /** A short title for the prompt. `beams`: 4 is the default, 1 is fastest, the maximum is 8. */ + title(prompt: string, options?: { beams?: number }): string; + /** Release the model's memory. The object cannot be used afterwards. */ + free(): void; +} + +export interface CreateOptions { + /** URL, file path (Node), or GGUF bytes. Default: the model file in this package. */ + model?: string | URL | ArrayBuffer | Uint8Array; +} + +export function createTitler(options?: CreateOptions): Promise; diff --git a/app/npm/index.js b/app/npm/index.js new file mode 100644 index 0000000..0f7e0ec --- /dev/null +++ b/app/npm/index.js @@ -0,0 +1,41 @@ +import init, { initSync, WasmTitler } from "./pkg/titler_wasm.js"; + +const isNode = typeof process !== "undefined" && !!process.versions?.node; + +async function readBytes(source) { + if (source instanceof Uint8Array) return source; + if (source instanceof ArrayBuffer) return new Uint8Array(source); + const url = source instanceof URL ? source : new URL(source, typeof location !== "undefined" ? location.href : import.meta.url); + if (isNode && url.protocol === "file:") { + const fs = "node:fs/promises"; + const { fileURLToPath } = await import(/* @vite-ignore */ "node:url"); + return new Uint8Array(await (await import(/* @vite-ignore */ fs)).readFile(fileURLToPath(url))); + } + const response = await fetch(url); + if (!response.ok) throw new Error(`could not load ${url}: ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); +} + +let ready; +function loadWasm() { + // Browsers refuse a synchronous compile on the main thread, so only Node takes the sync path. + ready ??= isNode ? readBytes(new URL("./pkg/titler_wasm_bg.wasm", import.meta.url)).then((bytes) => initSync({ module: bytes })) : init(); + return ready; +} + +/** + * Load the model. `model` is a URL, a file path (Node), or the GGUF bytes; the default is the file in this package. + * Returns an object with `title(prompt, { beams })` and `free()`. + */ +export async function createTitler({ model = new URL("./titler.gguf", import.meta.url) } = {}) { + const [, bytes] = await Promise.all([loadWasm(), readBytes(model)]); + const inner = new WasmTitler(bytes); + return { + title(prompt, { beams = 4 } = {}) { + return inner.title(String(prompt), beams); + }, + free() { + inner.free(); + }, + }; +} diff --git a/app/npm/package.json b/app/npm/package.json new file mode 100644 index 0000000..6e93e60 --- /dev/null +++ b/app/npm/package.json @@ -0,0 +1,19 @@ +{ + "name": "titler-wasm", + "version": "0.1.0", + "description": "A 4 MB model that turns a prompt into a short title. Runs in the browser and in Node with WebAssembly.", + "type": "module", + "main": "index.js", + "types": "index.d.ts", + "exports": { + ".": { "types": "./index.d.ts", "default": "./index.js" }, + "./titler.gguf": "./titler.gguf" + }, + "files": ["index.js", "index.d.ts", "pkg/titler_wasm.js", "pkg/titler_wasm.d.ts", "pkg/titler_wasm_bg.wasm", "titler.gguf", "README.md"], + "keywords": ["llm", "title", "wasm", "gguf"], + "repository": "github:Cyronius/titler", + "scripts": { + "build": "bash build.sh", + "test": "node --test" + } +} diff --git a/app/npm/test/titler.test.js b/app/npm/test/titler.test.js new file mode 100644 index 0000000..d84bebe --- /dev/null +++ b/app/npm/test/titler.test.js @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { createTitler } from "../index.js"; + +test("titles a prompt with the packaged model", async () => { + const t = await createTitler(); + const title = t.title("how do I sort a list of dicts by a key in python"); + assert.match(title.toLowerCase(), /sort/); + assert.match(title.toLowerCase(), /python/); + assert.ok(title.split(" ").length <= 8); + t.free(); +}); + +test("accepts model bytes and a beam count", async () => { + const bytes = await readFile(new URL("../titler.gguf", import.meta.url)); + const t = await createTitler({ model: new Uint8Array(bytes) }); + assert.ok(t.title("write a poem about autumn rain", { beams: 1 }).length > 0); + assert.ok(t.title("", { beams: 99 }).length >= 0); + t.free(); +}); + +test("a missing model file is an error", async () => { + await assert.rejects(createTitler({ model: new URL("./nope.gguf", import.meta.url) })); +}); diff --git a/app/python/README.md b/app/python/README.md new file mode 100644 index 0000000..6f278ae --- /dev/null +++ b/app/python/README.md @@ -0,0 +1,15 @@ +# titler for Python + +A 4 MB model that turns a prompt into a short title. It runs on the CPU in about 15-100 ms. + +```python +from titler import Titler + +t = Titler() # the model inside the wheel +print(t.title("how do I sort a list of dicts by a key in python")) # Sorting Dicts By Key in Python +``` + +`Titler("path/to/model.gguf")` loads another model file. `t.title(text, beams=4)`: 1 beam is fastest, 4 is the default. +One `Titler` can be used from many threads at the same time. + +This package is a thin layer over the C library in the same repository (`app/capi`). diff --git a/app/python/build_wheel.py b/app/python/build_wheel.py new file mode 100644 index 0000000..6471c89 --- /dev/null +++ b/app/python/build_wheel.py @@ -0,0 +1,43 @@ +"""Builds a platform wheel that holds the C library and the model (2026-10-20). + + cargo build --release -p titler-capi # from app/ + python build_wheel.py # from app/python; writes dist/titler-*.whl +""" + +import shutil +import subprocess +import sys +import sysconfig +from pathlib import Path + +HERE = Path(__file__).parent +TARGET = HERE.parent / "target" / "release" +LIBS = {"win32": "titler_c.dll", "darwin": "libtitler_c.dylib"} +MODEL = HERE.parent / "model" / "titler-v3-q4_k_m.gguf" + + +def main() -> None: + lib = TARGET / LIBS.get(sys.platform, "libtitler_c.so") + if not lib.exists(): + sys.exit(f"{lib} is missing: run `cargo build --release -p titler-capi` in app/ first") + package = HERE / "titler" + staged = [package / lib.name, package / "titler.gguf"] + shutil.copy(lib, staged[0]) + shutil.copy(MODEL, staged[1]) + dist = HERE / "dist" + shutil.rmtree(dist, ignore_errors=True) + try: + subprocess.run([sys.executable, "-m", "build", "--wheel", "--outdir", str(dist)], cwd=HERE, check=True) + plat = sysconfig.get_platform().replace("-", "_").replace(".", "_") + wheel = next(dist.glob("*.whl")) + subprocess.run([sys.executable, "-m", "wheel", "tags", "--remove", "--python-tag", "py3", "--abi-tag", "none", "--platform-tag", plat, str(wheel)], check=True) + finally: + for f in staged: + f.unlink(missing_ok=True) + shutil.rmtree(HERE / "build", ignore_errors=True) + shutil.rmtree(HERE / "titler.egg-info", ignore_errors=True) + print(*dist.glob("*.whl")) + + +if __name__ == "__main__": + main() diff --git a/app/python/pyproject.toml b/app/python/pyproject.toml new file mode 100644 index 0000000..ddb7dd7 --- /dev/null +++ b/app/python/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "titler" +version = "0.1.0" +description = "A 4 MB model that turns a prompt into a short title. Runs on the CPU." +readme = "README.md" +requires-python = ">=3.9" +keywords = ["llm", "title", "gguf"] + +[project.urls] +Source = "https://github.com/Cyronius/titler" + +[tool.setuptools] +packages = ["titler"] + +[tool.setuptools.package-data] +titler = ["*.dll", "*.so", "*.dylib", "*.gguf"] + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/app/python/tests/test_titler.py b/app/python/tests/test_titler.py new file mode 100644 index 0000000..cffa05b --- /dev/null +++ b/app/python/tests/test_titler.py @@ -0,0 +1,56 @@ +import concurrent.futures +import os +from pathlib import Path + +import pytest + +from titler import Titler, TitlerError, version + +MODEL = Path(__file__).parents[2] / "model" / "titler-v3-q4_k_m.gguf" + + +@pytest.fixture(scope="module") +def titler(): + with Titler(MODEL) as t: + yield t + + +def test_titles_a_prompt(titler): + title = titler.title("how do I sort a list of dicts by a key in python") + assert "python" in title.lower() and "sort" in title.lower() + assert len(title.split()) <= 8 + + +def test_non_ascii_prompt_round_trips(titler): + assert titler.title("日本語のテキストを要約してください。東京の天気について教えて").strip() != "" + + +def test_threads_agree_with_one_thread(titler): + prompts = ["write a poem about autumn rain", "what is the capital of australia", "fix this rust borrow checker error"] + alone = [titler.title(p) for p in prompts] + with concurrent.futures.ThreadPoolExecutor(3) as pool: + together = list(pool.map(titler.title, prompts)) + assert alone == together + + +def test_model_from_bytes(): + with Titler(MODEL.read_bytes()) as t: + assert t.title("hello there, how are you today").strip() != "" + + +def test_errors_are_exceptions(): + with pytest.raises(TitlerError): + Titler("no-such-file.gguf") + with pytest.raises(TitlerError): + Titler(b"not a gguf") + + +def test_closed_titler_raises(): + t = Titler(MODEL) + t.close() + with pytest.raises(TitlerError): + t.title("x") + + +def test_version(): + assert version() == "0.1.0" diff --git a/app/python/titler/__init__.py b/app/python/titler/__init__.py new file mode 100644 index 0000000..d2b8463 --- /dev/null +++ b/app/python/titler/__init__.py @@ -0,0 +1,100 @@ +"""Short titles for prompts. See README.md.""" + +import ctypes +import os +import sys +from pathlib import Path + +__all__ = ["Titler", "TitlerError"] + +_HERE = Path(__file__).parent +_LIB_NAMES = {"win32": "titler_c.dll", "darwin": "libtitler_c.dylib"} + + +class TitlerError(RuntimeError): + pass + + +def _load_library() -> ctypes.CDLL: + name = _LIB_NAMES.get(sys.platform, "libtitler_c.so") + candidates = [os.environ.get("TITLER_LIB"), _HERE / name] + for c in candidates: + if c and Path(c).exists(): + lib = ctypes.CDLL(str(c)) + break + else: + raise TitlerError(f"cannot find {name}; build it with `cargo build --release -p titler-capi` and set TITLER_LIB") + lib.titler_new_from_file.argtypes = [ctypes.c_char_p] + lib.titler_new_from_file.restype = ctypes.c_void_p + lib.titler_new_from_memory.argtypes = [ctypes.c_char_p, ctypes.c_size_t] + lib.titler_new_from_memory.restype = ctypes.c_void_p + lib.titler_free.argtypes = [ctypes.c_void_p] + lib.titler_free.restype = None + lib.titler_title_n.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_char_p, ctypes.c_size_t] + lib.titler_title_n.restype = ctypes.c_int64 + lib.titler_last_error.argtypes = [] + lib.titler_last_error.restype = ctypes.c_char_p + lib.titler_version.argtypes = [] + lib.titler_version.restype = ctypes.c_char_p + return lib + + +_lib = None + + +def _library() -> ctypes.CDLL: + global _lib + if _lib is None: + _lib = _load_library() + return _lib + + +def _error() -> str: + return (_library().titler_last_error() or b"unknown error").decode("utf-8", "replace") + + +class Titler: + """A loaded model. Safe to share between threads.""" + + def __init__(self, model=None): + lib = _library() + if model is None: + model = _HERE / "titler.gguf" + if not model.exists(): + raise TitlerError("this install has no bundled model; pass the path of a .gguf file") + if isinstance(model, (bytes, bytearray, memoryview)): + data = bytes(model) + self._handle = lib.titler_new_from_memory(data, len(data)) + else: + self._handle = lib.titler_new_from_file(os.fspath(model).encode("utf-8")) + if not self._handle: + raise TitlerError(_error()) + + def title(self, prompt: str, beams: int = 4) -> str: + if not self._handle: + raise TitlerError("this Titler is closed") + lib = _library() + data = prompt.encode("utf-8", "replace") + buf = ctypes.create_string_buffer(512) + n = lib.titler_title_n(self._handle, data, len(data), beams, buf, len(buf)) + if n < 0: + raise TitlerError(_error()) + return buf.value.decode("utf-8") + + def close(self) -> None: + if getattr(self, "_handle", None): + _library().titler_free(self._handle) + self._handle = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + self.close() + + +def version() -> str: + return _library().titler_version().decode() diff --git a/app/titler/Cargo.toml b/app/titler/Cargo.toml new file mode 100644 index 0000000..a748d6e --- /dev/null +++ b/app/titler/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "titler" +description = "A 4 MB model that turns a prompt into a short title. Runs on the CPU, no C++ needed." +version.workspace = true +edition.workspace = true +license.workspace = true +repository = "https://github.com/Cyronius/titler" +keywords = ["llm", "title", "gguf", "wasm"] +categories = ["text-processing"] + +[dependencies] +titler-core = { path = "../core", version = "0.1.0" } +titler-cpu = { path = "../cpu", version = "0.1.0" } diff --git a/app/titler/src/lib.rs b/app/titler/src/lib.rs new file mode 100644 index 0000000..f744721 --- /dev/null +++ b/app/titler/src/lib.rs @@ -0,0 +1,53 @@ +//! Short titles for prompts, from a 4 MB model that runs on the CPU. +//! +//! ```no_run +//! let titler = titler::Titler::from_path("titler-v3-q4_k_m.gguf")?; +//! println!("{}", titler.title("how do I sort a list of dicts by a key in python")?); +//! # Ok::<(), titler::Error>(()) +//! ``` +//! +//! Load the model once and share the `Titler` between threads: `title` takes `&self`, so any number of threads can +//! title at the same time. + +use std::path::Path; +use std::sync::Arc; + +use titler_cpu::{Model, Session}; + +pub use titler_core::{Error, Options, Result}; + +pub const DEFAULT_BEAMS: usize = 4; +const MAX_BEAMS: usize = 8; + +pub struct Titler { + core: titler_core::Titler, + model: Arc, +} + +impl Titler { + pub fn from_gguf_bytes(bytes: &[u8]) -> Result { + Titler::with_options(bytes, Options::default()) + } + + pub fn from_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + let bytes = std::fs::read(path).map_err(|e| Error(format!("{}: {e}", path.display())))?; + Titler::from_gguf_bytes(&bytes) + } + + pub fn with_options(bytes: &[u8], options: Options) -> Result { + Ok(Titler { core: titler_core::Titler::from_gguf_bytes(bytes, options)?, model: Arc::new(Model::from_gguf_bytes(bytes)?) }) + } + + /// Title with the default 4 beams, the setting the model was judged with. + pub fn title(&self, prompt: &str) -> Result { + self.title_with_beams(prompt, self.core.options().beams) + } + + /// 1 beam is fastest; more beams cost time and give slightly better titles. Clamped to 1..=8. + pub fn title_with_beams(&self, prompt: &str, beams: usize) -> Result { + let beams = beams.clamp(1, MAX_BEAMS); + let mut session = Session::new(self.model.clone(), beams); + self.core.title_with_beams(&mut session, prompt, beams) + } +} diff --git a/app/titler/tests/titles.rs b/app/titler/tests/titles.rs new file mode 100644 index 0000000..0aefb31 --- /dev/null +++ b/app/titler/tests/titles.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::thread; + +use titler::Titler; + +fn model() -> Titler { + Titler::from_path(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../model/titler-v3-q4_k_m.gguf")).unwrap() +} + +#[test] +fn titles_a_prompt() { + let title = model().title("how do I sort a list of dicts by a key in python").unwrap(); + assert!(title.to_lowercase().contains("python") && title.to_lowercase().contains("sort"), "{title}"); + assert!(title.split_whitespace().count() <= 8, "{title}"); +} + +#[test] +fn threads_can_title_at_once_and_agree_with_one_thread() { + let titler = Arc::new(model()); + let prompts = ["write a poem about autumn rain", "what is the capital of australia", "fix this rust borrow checker error", "summarize the plot of hamlet"]; + let alone: Vec = prompts.iter().map(|p| titler.title(p).unwrap()).collect(); + let handles: Vec<_> = prompts + .iter() + .map(|p| { + let t = titler.clone(); + let p = p.to_string(); + thread::spawn(move || t.title(&p).unwrap()) + }) + .collect(); + let together: Vec = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert_eq!(alone, together); +} + +#[test] +fn bad_input_is_an_error_not_a_crash() { + assert!(Titler::from_gguf_bytes(b"not a gguf file").is_err()); + assert!(Titler::from_path("does-not-exist.gguf").is_err()); +} + +#[test] +fn empty_and_huge_prompts_work() { + let t = model(); + t.title("").unwrap(); + t.title(&"word ".repeat(3000)).unwrap(); + t.title_with_beams("hello there", 0).unwrap(); + t.title_with_beams("hello there", 99).unwrap(); +} From cb99880ee76126e547937019d3f7ffddf90f6626 Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Mon, 19 Oct 2026 21:12:52 -0500 Subject: [PATCH 2/7] Make shell scripts executable Co-Authored-By: Claude Sonnet 5.5 --- app/build-site.sh | 0 app/capi/tests/run-smoke.sh | 0 app/npm/build.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 app/build-site.sh mode change 100644 => 100755 app/capi/tests/run-smoke.sh mode change 100644 => 100755 app/npm/build.sh diff --git a/app/build-site.sh b/app/build-site.sh old mode 100644 new mode 100755 diff --git a/app/capi/tests/run-smoke.sh b/app/capi/tests/run-smoke.sh old mode 100644 new mode 100755 diff --git a/app/npm/build.sh b/app/npm/build.sh old mode 100644 new mode 100755 From 349c1f440ea6c7112fdc55bedc30b4b6def6916d Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Mon, 19 Oct 2026 21:19:40 -0500 Subject: [PATCH 3/7] Add MIT license (code and model weights) Co-Authored-By: Claude Sonnet 5.5 --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d98eb02 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Cyrus Attoun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From aaea624ee9aa48d5eb90368bab0b1c5ea6a6b5b3 Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Mon, 19 Oct 2026 21:26:34 -0500 Subject: [PATCH 4/7] Add decode and parity requirements to the title spec; point README to v2 and the embedding guide Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++++++++- specs/title/spec.md | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3409e96..0900bb1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ # titler -A tiny T5 model that turns a prompt into a short title. +A tiny model that turns a prompt into a short title. + +> **New: titler v2** is a standard Llama-architecture GGUF (5.8M parameters, **4 MB**) that runs on the CPU in 15-100 ms. It wins about +> 65% of the head-to-head pairs against the v1.5 model described below. +> Try it in your browser: **https://cyronius.github.io/titler/** (runs locally with WebAssembly, nothing is sent anywhere). +> Use it in your own app (Rust, C, Python, JavaScript, command line): [app/EMBEDDING.md](app/EMBEDDING.md). +> Code and model: [app/](app/). Training and evaluation scripts are in [scripts/](scripts/). The sections below describe v1.5. + **7.6M parameters · 10.3 MB int8 ONNX · CPU-only · browser / Node / Python** diff --git a/specs/title/spec.md b/specs/title/spec.md index d670234..a9f0889 100644 --- a/specs/title/spec.md +++ b/specs/title/spec.md @@ -64,3 +64,40 @@ Unicode cases (see the plan's round-trip count). - `decase_title("Fixing My Children's Bike")` → `fixing my children's bike` - `decase_title("iPhone SQL Setup Guide")` → `iPhone SQL setup guide` - `clean_title(decase_title(t)) == t` for `Fixing My Children's Bike`, `Python vs Rust for CLI Tools`, `A Guide to mRNA Vaccines` + +### TITLE-DECODE: The shipped model is decoded with 4-beam search, a title-only repeat ban and a filler penalty +**Applies to:** titler, app/core +**Verification:** test +**External tests:** `app/core/tests/golden.rs` (`beam_search_matches_huggingface`), cases made by `scripts/24_make_app_golden.py` + +Titles come from beam search that follows Hugging Face `generate(num_beams=N)` (length penalty 1, no early stop): 4 beams by +default, at most 40 new tokens after `BOS + first 256 prompt tokens + `. Inside the title (not the prompt) a token pair is +never repeated and a word-start token never follows itself. The log-probability of "request" and "inquiry" is lowered by 3.0. The +Python code (`scripts/15_gen_v2_titles.py`) and the Rust code (`app/core`) give the same tokens. + +**Acceptance criteria:** +- 24 toy-model cases (1, 2, 4 and 6 beams; ban on and off; penalty 0 and 3.0; stop by end token and by length) → Rust token ids equal the Hugging Face ids + +### TITLE-TOKENIZE-PARITY: The Rust tokenizer equals the training tokenizer +**Applies to:** app/core +**Verification:** test +**External tests:** `app/core/tests/golden.rs` (`tokenizer_matches_python`) + +The Rust tokenizer reads the vocabulary from the GGUF and gives the same ids as the Hugging Face tokenizer (`tokenizer.json`) for +any text, including text with characters outside the vocabulary (spelled out as byte tokens). A split of a long run such as +`____` may differ when both splits have the same score; the decoded text is then the same. + +**Acceptance criteria:** +- 1,684 eval, non-English and edge-case texts → identical ids, except at most 3 equal-text ties +- `""` → no tokens + +### TITLE-CLEAN-PARITY: Rust `clean_title` equals Python `clean_title` +**Applies to:** app/core +**Verification:** test +**External tests:** `app/core/tests/golden.rs` (`clean_title_matches_python`) + +The Rust port gives the same output as `clean_title` (TITLE-PRODUCT-CLAMP, TITLE-APOSTROPHE, TITLE-CASING), except that it does +not strip teacher-only `` tags. + +**Acceptance criteria:** +- 1,533 decased training titles and 30 edge cases (quotes, markdown, leaked format ranges, line breaks) → identical strings From d8ad3c370c847566565ab22635b4c0de98119ee8 Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Mon, 19 Oct 2026 21:34:26 -0500 Subject: [PATCH 5/7] Add per-type win-rate report and record the known limits in the title spec Co-Authored-By: Claude Sonnet 5.5 --- scripts/25_hard_input_report.py | 62 +++++++++++++++++++++++++++++++++ specs/title/spec.md | 4 +++ 2 files changed, 66 insertions(+) create mode 100644 scripts/25_hard_input_report.py diff --git a/scripts/25_hard_input_report.py b/scripts/25_hard_input_report.py new file mode 100644 index 0000000..b3bbdf5 --- /dev/null +++ b/scripts/25_hard_input_report.py @@ -0,0 +1,62 @@ +"""Win rate against v1.5 by input type, plus the failing rows for each type (2026-10-20). + + uv run python scripts/25_hard_input_report.py --dump data/eval/judge_rust_q4_vs_v15.jsonl --out data/eval/hard_failures.json +""" + +import argparse +import collections +import json +import re + + +def non_english(p: str) -> bool: + letters = [c for c in p if c.isalpha()] + if not letters: + return False + if sum(1 for c in letters if ord(c) > 0x024F) / len(letters) > 0.3: + return True + accented = re.search(r"[¿¡ñãõçßäöüéèêàâîôûœ]", p) + return bool(accented) and len(re.findall(r"\b(the|and|is|to|of|a|in|for|you|what)\b", p.lower())) < 2 + + +def kinds(row: dict) -> list[str]: + p, bucket = row["prompt"], row.get("bucket", "normal") + out = [] + if non_english(p): + out.append("non_english") + if bucket in ("code_paste", "sql", "log_paste", "markdown", "long"): + out.append("pasted_" + bucket) + if bucket in ("very_short", "vague") or len(p.split()) <= 4: + out.append("short_or_vague") + return out or [bucket if bucket in ("hard", "gratitude", "hostile") else "other"] + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--dump", required=True, help="judge dump with the new titles as A and v1.5 as B") + ap.add_argument("--eval-set", default="data/eval/eval_set.jsonl") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + rows = [json.loads(l) for l in open(args.eval_set, encoding="utf-8") if l.strip()] + judged = {d["id"]: d for d in map(json.loads, open(args.dump, encoding="utf-8")) if d} + stats: dict[str, collections.Counter] = collections.defaultdict(collections.Counter) + failing: dict[str, list] = collections.defaultdict(list) + for r in rows: + d = judged.get(r["id"]) + verdict = d["verdict"] if d else "same" + for k in kinds(r): + stats[k]["rows"] += 1 + stats[k][verdict] += 1 + if verdict in ("b", "tie"): + failing[k].append({"id": r["id"], "verdict": "v1.5 wins" if verdict == "b" else "tie", "prompt": r["prompt"][:600], "new": d["a"], "v15": d["b"]}) + print(f"{'type':16} {'rows':>5} {'same':>5} {'new':>4} {'v1.5':>5} {'tie':>4} new share of decided") + for k, c in sorted(stats.items(), key=lambda x: -x[1]["rows"]): + decided = c["a"] + c["b"] + print(f"{k:16} {c['rows']:5} {c['same']:5} {c['a']:4} {c['b']:5} {c['tie']:4} {c['a'] / decided:.1%}" if decided else f"{k:16} {c['rows']:5} {c['same']:5}") + json.dump(failing, open(args.out, "w", encoding="utf-8"), ensure_ascii=False, indent=1) + print("failing rows (v1.5 wins or tie):", {k: len(v) for k, v in failing.items()}, "->", args.out) + + +if __name__ == "__main__": + main() diff --git a/specs/title/spec.md b/specs/title/spec.md index a9f0889..074d474 100644 --- a/specs/title/spec.md +++ b/specs/title/spec.md @@ -101,3 +101,7 @@ not strip teacher-only `` tags. **Acceptance criteria:** - 1,533 decased training titles and 30 edge cases (quotes, markdown, leaked format ranges, line breaks) → identical strings + +### Known limits +- Dropped apostrophes in possessives (`Nick Wildes`): 1 of 1,064 eval titles and 4.5% of training titles that could drop one (2026-10-20), so not worth a retrain. +- Hard inputs (non-English, logs, short prompts) are not yet measured well enough: see `.claude/plans/titler-v3-hard-inputs.md`. From 24271baa1b83f6ec3cecf0e2ed506d6ff33bf593 Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Tue, 20 Oct 2026 02:45:06 -0500 Subject: [PATCH 6/7] State that titler is English only Co-Authored-By: Claude Sonnet 5.5 --- app/EMBEDDING.md | 4 ++-- specs/title/spec.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/EMBEDDING.md b/app/EMBEDDING.md index 30c65d7..d88870b 100644 --- a/app/EMBEDDING.md +++ b/app/EMBEDDING.md @@ -67,7 +67,7 @@ Host `pkg/titler_wasm_bg.wasm` and `titler.gguf` as static files next to your sc | | | |---|---| | Output | 3 to 8 words in Title Case, for example `Sorting Dicts By Key in Python`. Same input, same output. | -| Input | Any text. Only the first 256 tokens (about 1,000 characters of English) are read. English works best. | +| Input | Any text. Only the first 256 tokens (about 1,000 characters of English) are read. English only: other languages are not supported. | | Speed, 4 beams | Native Rust (one thread): median 40-70 ms, slowest 5% 200-275 ms (long prompts). llama.cpp: 18 ms / 48 ms. Browser: 50-175 ms. | | Speed, 1 beam | About half of that, with slightly worse titles. | | Memory | About 25 MB after loading (weights are held as 32-bit numbers). Load time about 30 ms. | @@ -77,7 +77,7 @@ Host `pkg/titler_wasm_bg.wasm` and `titler.gguf` as static files next to your sc ## Limits - It titles; it does not summarize, translate or answer. -- It sometimes drops an apostrophe (`Nick Wildes Judy Hopp Dialog Line`). Weak inputs are non-English text, vague one-liners and pasted data. +- It sometimes drops an apostrophe (`Nick Wildes Judy Hopp Dialog Line`). Weak inputs are vague one-liners and pasted logs. - The training data and its licenses are listed in `model/NOTICE.md`. ## Build from source diff --git a/specs/title/spec.md b/specs/title/spec.md index 074d474..5c4711e 100644 --- a/specs/title/spec.md +++ b/specs/title/spec.md @@ -104,4 +104,5 @@ not strip teacher-only `` tags. ### Known limits - Dropped apostrophes in possessives (`Nick Wildes`): 1 of 1,064 eval titles and 4.5% of training titles that could drop one (2026-10-20), so not worth a retrain. -- Hard inputs (non-English, logs, short prompts) are not yet measured well enough: see `.claude/plans/titler-v3-hard-inputs.md`. +- English only: non-English prompts are out of scope (owner, 2026-10-20). +- Hard inputs (logs, the mixed hard bucket) are weaker than the rest: see `.claude/plans/titler-v3-hard-inputs.md`. From 1a24903b2bb7c58d450481cddd5221173ac0c873 Mon Sep 17 00:00:00 2001 From: Cyrus Attoun Date: Tue, 20 Oct 2026 03:26:42 -0500 Subject: [PATCH 7/7] Add hard-input eval builder and --eval-set option for the judge Co-Authored-By: Claude Sonnet 5.5 --- scripts/11_pairwise_judge.py | 7 ++++- scripts/26_build_hard_eval.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 scripts/26_build_hard_eval.py diff --git a/scripts/11_pairwise_judge.py b/scripts/11_pairwise_judge.py index 42feb0b..d08505a 100644 --- a/scripts/11_pairwise_judge.py +++ b/scripts/11_pairwise_judge.py @@ -81,8 +81,11 @@ def report(pairs: list[dict], verdicts: list[str]) -> None: print(f" {bucket}: A {bc['a']} B {bc['b']} tie {bc['tie']}") +EVAL_SET = "data/eval/eval_set.jsonl" + + def read_eval_set() -> list[dict]: - with Path("data/eval/eval_set.jsonl").open(encoding="utf-8") as f: + with Path(EVAL_SET).open(encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] @@ -108,10 +111,12 @@ def main() -> None: ap.add_argument("--b") ap.add_argument("--limit", type=int) ap.add_argument("--bucket", help="only this bucket, e.g. hard") + ap.add_argument("--eval-set", default=EVAL_SET, help="JSONL with id, bucket, prompt") ap.add_argument("--calibrate", action="store_true") ap.add_argument("--extra", default="{}", help="JSON merged into the request, e.g. reasoning_effort") ap.add_argument("--dump", help="write per-pair verdicts here") args = ap.parse_args() + globals()["EVAL_SET"] = args.eval_set pairs = calibration_pairs(args.limit or 100) if args.calibrate else load_pairs(args.a, args.b, args.limit, args.bucket) verdicts = asyncio.run(judge_pairs(args.judge, pairs, json.loads(args.extra))) report(pairs, verdicts) diff --git a/scripts/26_build_hard_eval.py b/scripts/26_build_hard_eval.py new file mode 100644 index 0000000..8c51f08 --- /dev/null +++ b/scripts/26_build_hard_eval.py @@ -0,0 +1,54 @@ +"""Eval set with 150 held-out rows per hard input type, plus v1.5 titles for them (2026-10-20). + + uv run python scripts/26_build_hard_eval.py +""" + +import importlib.util +import json +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from eval import PPL_FALLBACK_THRESHOLD, fallback_title, generate, load_model # noqa: E402 + +PER_GROUP = 150 +GROUPS = { + "short": ("very_short", "vague"), + "code": ("code_paste",), + "log": ("log_paste",), + "sql": ("sql",), + "markdown": ("markdown",), + "long": ("long",), +} + + +def read(path: str) -> list[dict]: + return [json.loads(l) for l in open(path, encoding="utf-8") if l.strip()] + + +def main() -> None: + spec = importlib.util.spec_from_file_location("strat", "scripts/03_stratify.py") + strat = importlib.util.module_from_spec(spec) + spec.loader.exec_module(strat) + held_out = read("data/v5/test.jsonl") + read("data/v5/val.jsonl") + rng = random.Random(26) + rows = [] + for group, buckets in GROUPS.items(): + pool = [r for r in held_out if (r.get("bucket") or strat.bucket(r["prompt"])) in buckets] + rng.shuffle(pool) + rows += [{"id": r["id"], "bucket": group, "prompt": r["prompt"]} for r in pool[:PER_GROUP]] + rows += [{"id": r["id"], "bucket": "hard", "prompt": r["prompt"]} for r in read("data/hardcases.jsonl")] + out = Path("data/eval") + (out / "hard_eval.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows), encoding="utf-8") + tokenizer, model = load_model("model/titler-v15-scratch-web") + titles = {} + for r in rows: + title, ppl = generate(tokenizer, model, r["prompt"]) + titles[r["id"]] = fallback_title(r["prompt"]) if ppl > PPL_FALLBACK_THRESHOLD else title + (out / "hard_titles_v15.json").write_text(json.dumps(titles, ensure_ascii=False), encoding="utf-8") + print(len(rows), "rows;", {g: sum(r["bucket"] == g for r in rows) for g in [*GROUPS, "hard"]}) + + +if __name__ == "__main__": + main()