From 697dc28e8c853422c9845083f6541cf7ceb51d83 Mon Sep 17 00:00:00 2001 From: Parallel Entrepreneur <40026523+ParallelEntrepreneur@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:13:03 +0000 Subject: [PATCH] feat(worker): harness composition, D1/KV/R2 bindings, core table migrations and wrangler.toml (#15) Refs #15 Co-Authored-By: Colonizer Settlers <331648616+colonizer-settlers@users.noreply.github.com> --- crates/tokker-worker/src/lib.rs | 43 ++++++++++-- migrations/0001_core_tables.sql | 107 ++++++++++++++++++++++++++++ tools/smoke.mjs | 66 ++++++++++++++++++ wrangler.toml | 120 ++++++++++++++++++++++++++++++++ 4 files changed, 330 insertions(+), 6 deletions(-) create mode 100644 migrations/0001_core_tables.sql create mode 100644 tools/smoke.mjs create mode 100644 wrangler.toml diff --git a/crates/tokker-worker/src/lib.rs b/crates/tokker-worker/src/lib.rs index de8e024..224d549 100644 --- a/crates/tokker-worker/src/lib.rs +++ b/crates/tokker-worker/src/lib.rs @@ -9,32 +9,55 @@ //! serve(harness, runtime, req, env, ctx).await //! } //! ``` +//! +//! Bindings follow plan.md §5.1: D1 (`DB`) for the price tables, KV (`KV`) +//! for hot JSON responses, R2 (`BUCKET`) for evidence snapshots and CSV +//! downloads, and the Workers rate limiting binding (`RATE_LIMITER`) for +//! anonymous IP limits. The crons live in `wrangler.toml` `[triggers]`, +//! hand-maintained until Cratefield's manifest cron field lands +//! (plan.md §5.2 G7a). #![forbid(unsafe_code)] use std::sync::OnceLock; use cratefield_core::{Harness, Venture}; -use cratefield_runtime_cloudflare::{Cloudflare, serve}; +use cratefield_runtime_cloudflare::{Cloudflare, serve, serve_scheduled}; use tokker_api::TokkerApi; -use worker::{Context, Env, Request, Response, event}; +use worker::{Context, Env, Request, Response, ScheduleContext, ScheduledEvent, event}; static INSTANCE: OnceLock<(Harness, Cloudflare)> = OnceLock::new(); +/// The harness and its runtime, built once per isolate. +/// +/// The runtime is constructed once and cloned into the builder — +/// `Cloudflare` is `Clone` (it carries binding names, not bindings) — so +/// the fetch and scheduled entry points share one composition. fn instance() -> &'static (Harness, Cloudflare) { INSTANCE.get_or_init(|| { + let runtime = Cloudflare::new() + .db("DB") + .kv("KV") + .blob("BUCKET") + .rate_limiter("RATE_LIMITER"); let harness = Harness::builder() - .venture(Venture::new("tokker", "tokker.dev").public_url("https://tokker.dev")) + .venture( + Venture::new("tokker", "tokker.dev") + .public_url("https://tokker.dev") + // The site that embeds the API (plan.md §5): build() + // refuses a venture with no CORS origin at all. + .cors_origins(["https://tokker.dev"]), + ) .module(TokkerApi) - .runtime(Cloudflare::new().db("DB")) + .runtime(runtime.clone()) .build() .expect("tokker harness is valid"); - let runtime = Cloudflare::new().db("DB"); (harness, runtime) }) } -/// Worker fetch entry point. +/// Worker fetch entry point: the harness router answers `/v1/*` and the +/// built-in probes (`/__health`, `/__ready`, `/__surface`). /// /// # Errors /// @@ -44,3 +67,11 @@ pub async fn fetch(req: Request, env: Env, ctx: Context) -> worker::Result/" + provider_id TEXT NOT NULL REFERENCES providers(id), + model_id TEXT NOT NULL REFERENCES models(id), + creator TEXT NOT NULL, -- model_creator; the /v1/offers filter + provider_type TEXT NOT NULL CHECK (provider_type IN ('first_party', 'aggregator', 'inference_host', 'cloud')), + open_weights TEXT NOT NULL CHECK (open_weights IN ('true', 'false', 'unknown')), + currency TEXT NOT NULL, -- ISO 4217; native currency first (CLAUDE.md rule 4) + input_per_mtok REAL, -- native currency; NULL when "unknown" + output_per_mtok REAL, + usd_input_per_mtok REAL, -- the row's derived usd block, at fx.date + usd_output_per_mtok REAL, + usd_blended_3to1 REAL, -- the 3:1 input:output blend used for ranking + context_window INTEGER, + max_output INTEGER, + batch_discount_pct REAL, + region_notes TEXT, + source TEXT NOT NULL, -- the exact URL the row came from + confidence TEXT NOT NULL CHECK (confidence IN ('official_page', 'official_docs', 'secondary')), + last_verified_at TEXT NOT NULL, -- ISO date; shown everywhere (plan.md §3) + json TEXT NOT NULL +); + +CREATE INDEX offers_creator ON offers(creator); +CREATE INDEX offers_provider ON offers(provider_id); +CREATE INDEX offers_model ON offers(model_id); +CREATE INDEX offers_open_weights ON offers(open_weights); +CREATE INDEX offers_usd_input ON offers(usd_input_per_mtok); +CREATE INDEX offers_cheapest ON offers(model_id, usd_blended_3to1); -- /v1/models/{slug}/cheapest + +CREATE TABLE plans ( + id TEXT PRIMARY KEY, -- "/" + vendor_id TEXT NOT NULL REFERENCES providers(id), + vendor_name TEXT NOT NULL, + product TEXT NOT NULL, + plan_name TEXT NOT NULL, + category TEXT NOT NULL CHECK (category IN ('chat', 'coding_agent', 'ide', 'token_plan', 'api_credit_bundle')), + currency TEXT NOT NULL, + price_month REAL, -- native currency; NULL when "unknown" + price_year REAL, + usd_price_month REAL, + usd_price_year REAL, + last_verified_at TEXT NOT NULL, + json TEXT NOT NULL +); + +CREATE INDEX plans_vendor ON plans(vendor_id); +CREATE INDEX plans_category ON plans(category); + +CREATE TABLE sources ( + id TEXT PRIMARY KEY, -- "/"; sources[] has no id in the dataset, so the loader assigns this + provider_id TEXT NOT NULL, + url TEXT NOT NULL, + method TEXT NOT NULL CHECK (method IN ('json_api', 'html_static', 'html_js_rendered', 'docs_markdown', 'pdf', 'manual')), + endpoint TEXT NOT NULL, + volatility TEXT NOT NULL CHECK (volatility IN ('high', 'medium', 'low')), + fetch_ok INTEGER NOT NULL, -- boolean + part TEXT NOT NULL, -- the part file that produced the entry + notes TEXT, + json TEXT NOT NULL +); + +CREATE INDEX sources_provider ON sources(provider_id); + +CREATE TABLE fx ( + currency TEXT PRIMARY KEY, -- ISO 4217; one row per fx.rates entry + rate REAL NOT NULL CHECK (rate > 0), -- one unit of base in that currency + base TEXT NOT NULL, + date TEXT NOT NULL, -- the rate date for the whole dataset + json TEXT NOT NULL +); diff --git a/tools/smoke.mjs b/tools/smoke.mjs new file mode 100644 index 0000000..cc16343 --- /dev/null +++ b/tools/smoke.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// Smoke-test a running Tokker Worker: the harness built-ins (/__health, +// /__ready, /__surface) and the 404 fallback. Start the Worker first +// (`npx wrangler dev --local` from the repo root), then run +// `node tools/smoke.mjs [base-url]`. SMOKE_URL or the first argument set the +// base URL (default http://127.0.0.1:8787); prints one PASS/FAIL line per +// check and exits 1 if any check fails. + +const BASE = process.env.SMOKE_URL || process.argv[2] || 'http://127.0.0.1:8787'; + +let failed = false; + +/** Run one check: name plus an async function answering {ok, detail}. */ +async function check(name, fn) { + try { + const { ok, detail } = await fn(); + console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); + if (!ok) failed = true; + } catch (err) { + console.log(`FAIL ${name} — ${err.message}`); + failed = true; + } +} + +/** GET a path, returning the response and its text (empty on a read error). */ +async function get(path) { + const res = await fetch(BASE + path, { redirect: 'manual' }); + const body = await res.text().catch(() => ''); + return { res, body }; +} + +/** {ok, detail} for a plain status expectation. */ +function atStatus(res, want, body) { + const ok = want(res.status); + return { ok, detail: `status ${res.status}${ok ? '' : `: ${body.slice(0, 200)}`}` }; +} + +await check('/__health answers 2xx', async () => { + const { res, body } = await get('/__health'); + return atStatus(res, (s) => s >= 200 && s < 300, body); +}); + +// Locally the D1 binding is wired, so /__ready's SELECT 1 answers 200. +await check('/__ready answers 2xx (SELECT 1 through the DB port)', async () => { + const { res, body } = await get('/__ready'); + return atStatus(res, (s) => s >= 200 && s < 300, body); +}); + +await check('/__surface answers 2xx JSON', async () => { + const { res, body } = await get('/__surface'); + let json = null; + try { + json = JSON.parse(body); + } catch { + // left null; the detail below says so + } + const ok = res.status >= 200 && res.status < 300 && json !== null; + return { ok, detail: `status ${res.status}, ${json === null ? 'body is not JSON' : 'JSON'}` }; +}); + +await check('unknown path answers 404', async () => { + const { res, body } = await get('/__smoke-not-a-route'); + return atStatus(res, (s) => s === 404, body); +}); + +process.exit(failed ? 1 : 0); diff --git a/wrangler.toml b/wrangler.toml new file mode 100644 index 0000000..1e15cf8 --- /dev/null +++ b/wrangler.toml @@ -0,0 +1,120 @@ +# The Tokker API Worker (plan.md §5): the Cratefield harness on +# cratefield-runtime-cloudflare, served from crates/tokker-worker. +# +# Build: worker-build is expected on PATH (`cargo install worker-build`); +# output lands in the crate's gitignored `build/`. +# +# Secrets — set per environment with `wrangler secret put`, or in the +# untracked `.dev.vars` for local dev; never values in this file: +# HARNESS_SECRET >= 32 bytes; provides the harness Signer port +# ADMIN_TOKEN unlocks the admin variant of GET /__surface +# HARNESS_SECRET_PREVIOUS only during key rotation + +name = "tokker-api" +main = "crates/tokker-worker/build/worker/shim.mjs" +compatibility_date = "2026-09-01" +workers_dev = true + +[build] +command = "worker-build --release crates/tokker-worker" +watch_dir = ["crates/tokker-worker/src"] + +[observability] +enabled = true + +# Hand-maintained until Cratefield's manifest cron field lands +# (plan.md §5.2 G7a, Cratefield/harness#774). +[triggers] +crons = ["*/30 * * * *"] + +# Default (local/dev) bindings; every deploy environment re-declares its own. +[[d1_databases]] +binding = "DB" +database_name = "tokker" +# Local dev keys the database on the name; remote deploys use the env blocks +# below, which carry the real ids. +database_id = "local" +migrations_dir = "migrations" + +[[kv_namespaces]] +binding = "KV" +id = "local" + +[[r2_buckets]] +binding = "BUCKET" +bucket_name = "tokker-local" + +# Anonymous per-IP limit for the public read API (plan.md §5.1). Workers +# rate limiting binding; counters are per colo, per namespace. +[[ratelimits]] +name = "RATE_LIMITER" +namespace_id = "1001" +simple = { limit = 120, period = 60 } + +# --- staging ------------------------------------------------------------- + +[env.staging] +name = "tokker-api-staging" +workers_dev = true + +[env.staging.vars] +ENV = "staging" + +[env.staging.triggers] +crons = ["*/30 * * * *"] + +[[env.staging.d1_databases]] +binding = "DB" +database_name = "tokker-staging" +# Owner: create with `wrangler d1 create tokker-staging` and paste the id. +database_id = "REPLACE_WITH_STAGING_D1_ID" +migrations_dir = "migrations" + +[[env.staging.kv_namespaces]] +binding = "KV" +# Owner: create with `wrangler kv namespace create KV --env staging`. +id = "REPLACE_WITH_STAGING_KV_ID" + +[[env.staging.r2_buckets]] +binding = "BUCKET" +bucket_name = "tokker-staging" + +[[env.staging.ratelimits]] +name = "RATE_LIMITER" +namespace_id = "2001" +simple = { limit = 120, period = 60 } + +# --- production ---------------------------------------------------------- + +[env.production] +name = "tokker-api-production" +# No routes or custom domains yet; workers.dev only. Disable workers_dev +# when the custom domain lands. +workers_dev = true + +[env.production.vars] +ENV = "production" + +[env.production.triggers] +crons = ["*/30 * * * *"] + +[[env.production.d1_databases]] +binding = "DB" +database_name = "tokker-production" +# Owner: create with `wrangler d1 create tokker-production` and paste the id. +database_id = "REPLACE_WITH_PRODUCTION_D1_ID" +migrations_dir = "migrations" + +[[env.production.kv_namespaces]] +binding = "KV" +# Owner: create with `wrangler kv namespace create KV --env production`. +id = "REPLACE_WITH_PRODUCTION_KV_ID" + +[[env.production.r2_buckets]] +binding = "BUCKET" +bucket_name = "tokker-production" + +[[env.production.ratelimits]] +name = "RATE_LIMITER" +namespace_id = "3001" +simple = { limit = 120, period = 60 }