From 611bfc150ce63de6de78902a23902e5f0415b2c5 Mon Sep 17 00:00:00 2001 From: howdeploy Date: Sun, 20 Sep 2026 14:09:40 +0300 Subject: [PATCH] fix: complete demo, OpenRouter and TypeSafe provider routing --- CHANGELOG.md | 9 ++ README.md | 2 + README.ru.md | 2 + README.zh-CN.md | 2 + docs/PROVIDERS.md | 67 ++++++++++ integrations/hermes/__init__.py | 4 +- integrations/omp/extension.js | 4 +- src/browser.mjs | 3 +- src/cli.mjs | 2 +- src/config.mjs | 12 +- src/mcp-server.mjs | 3 + src/providers/index.mjs | 12 ++ src/providers/typesafe.mjs | 12 +- src/route.mjs | 20 ++- src/supervision.mjs | 14 +- test/provider-transports.test.mjs | 205 ++++++++++++++++++++++++++++++ test/providers.test.mjs | 126 ++++++++++++++++++ 17 files changed, 464 insertions(+), 35 deletions(-) create mode 100644 docs/PROVIDERS.md create mode 100644 src/providers/index.mjs create mode 100644 test/provider-transports.test.mjs create mode 100644 test/providers.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index c7816a1..3927a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to jev-layer are recorded here. Entries describe public behavior and compatibility impact; provider-specific experiments are listed only when they affect a public contract. +## Unreleased + +### Fixed + +- Route `typesafe` through the official System One provider; share provider resolution with supervision and preserve fail-open decisions for unsupported providers. +- Apply configured provider environment names in CLI/MCP, including supervision and browser decisions; let Hermes/OMP honor project provider selection. +- Send OpenRouter's optional app title and record the configured model in provider metadata; allow the Hermes process to outlive the provider HTTP timeout. +- Cover demo, OpenRouter, and TypeSafe routing with offline provider and CLI/MCP/adapter regression checks; document setup and configuration precedence. + ## [0.1.0] First public-release candidate. This version is prepared but has not been pushed, released on GitHub, or published to npm. diff --git a/README.md b/README.md index c83ce58..bbf59c9 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ export OPENROUTER_API_KEY='provided-by-your-secret-store' jev doctor --project /path/to/workspace ``` +All three modes (`demo`, `openrouter`, and direct `typesafe`), their endpoints, and configuration precedence are documented in the [provider guide](docs/PROVIDERS.md). + ## Core surfaces - **Routing:** `jev_route` selects one capability from the host-supplied candidate set. Selection is advisory; the host validates the id and permissions. diff --git a/README.ru.md b/README.ru.md index cc73344..3abeb27 100644 --- a/README.ru.md +++ b/README.ru.md @@ -70,6 +70,8 @@ export OPENROUTER_API_KEY='provided-by-your-secret-store' jev doctor --project /path/to/workspace ``` +Все три режима (`demo`, `openrouter` и прямой `typesafe`), их endpoints и порядок выбора конфигурации описаны в [руководстве по провайдерам](docs/PROVIDERS.md). + ## Основные поверхности - **Routing:** `jev_route` выбирает одну capability из набора, предоставленного host. Host повторно проверяет id и permissions. diff --git a/README.zh-CN.md b/README.zh-CN.md index d543a73..643d452 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -70,6 +70,8 @@ export OPENROUTER_API_KEY='provided-by-your-secret-store' jev doctor --project /path/to/workspace ``` +三种模式（`demo`、`openrouter` 和直连 `typesafe`）、端点及配置优先级见[提供商指南](docs/PROVIDERS.md)。 + ## 核心能力 - **Routing：** `jev_route` 从 host 提供的候选集合中选择一个 capability。host 会再次验证 id 和权限。 diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md new file mode 100644 index 0000000..05f1ada --- /dev/null +++ b/docs/PROVIDERS.md @@ -0,0 +1,67 @@ +# Routing providers + +CLI, stdio MCP, and the Hermes/OMP adapters use the same three providers. Routing, supervision, and browser decisions share provider resolution. The host still owns execution and approval. + +| Provider | Default endpoint | Default model | Credentials | +| --- | --- | --- | --- | +| `demo` | None; deterministic word matching, no HTTP | `jev-demo` | None | +| `openrouter` | `https://openrouter.ai/api/alpha/decisions` | `typesafe/jev-1.13` | `OPENROUTER_API_KEY` | +| `typesafe` | `https://api.typesafe.ai/v1/systemone` | `jev-latest` | `TYPESAFE_API_KEY` | + +Both remote providers send `{ model, state, questions }` with a Choice question and Bearer authentication. They use the [OpenRouter Decisions API](https://openrouter.ai/docs/api/api-reference/alphadecisions/submit-a-decisions-questions-and-answers-request) and the [official TypeSafe API](https://docs.typesafe.ai/api), respectively. + +## Select a provider + +For a stdio MCP process, export the selected provider's API key through your secret store into the harness environment, then choose one command: + +```sh +JEV_LAYER_PROVIDER=demo jev mcp +JEV_LAYER_PROVIDER=openrouter jev mcp +JEV_LAYER_PROVIDER=typesafe jev mcp +``` + +For a single CLI request from the repository root: + +```sh +jev cli --provider demo --input examples/route-request.json +jev cli --provider openrouter --input examples/route-request.json +jev cli --provider typesafe --input examples/route-request.json +``` + +Selection precedence is: request `provider` → CLI `--provider` (CLI only) → `JEV_LAYER_PROVIDER` → `.jev/config.json` `provider` → `demo`. MCP launcher `env` entries count as process environment; update them when switching providers. Hermes and OMP delegate this choice to the CLI instead of forcing `demo`. The Hermes adapter's default process timeout is 10 seconds, allowing either provider's HTTP timeout to complete; `JEV_LAYER_TIMEOUT_S` overrides it. + +`jev add --provider typesafe` generates configuration for TypeSafe in the target workspace. Use `--provider openrouter` or `--provider demo` for the other modes. Keep keys outside generated configuration. + +## Endpoints, models, and custom environment names + +- TypeSafe: `TYPESAFE_ENDPOINT`, `TYPESAFE_MODEL`. +- OpenRouter: `OPENROUTER_DECISIONS_ENDPOINT`, `OPENROUTER_DECISIONS_MODEL`; optional attribution headers use `OPENROUTER_HTTP_REFERER` and `OPENROUTER_APP_TITLE`. +- Endpoints are complete request URLs, including `/v1/systemone` or `/api/alpha/decisions`. +- Default HTTP timeouts are 2 seconds for TypeSafe and 5 seconds for OpenRouter. There are no provider retries; failures return control to the host. + +The CLI and MCP read `.jev/config.json` from their working directory. `JEV_CONFIG` selects another file. To use secret-store-specific environment names, set only their names in the config: + +```json +{ + "provider": "typesafe", + "providers": { + "typesafe": { + "api_key_env": "MY_TYPESAFE_KEY", + "endpoint_env": "MY_TYPESAFE_ENDPOINT", + "model_env": "MY_TYPESAFE_MODEL" + } + } +} +``` + +Omitted fields retain the default environment names. A missing configured key returns `fallback` without an HTTP call, even if `TYPESAFE_API_KEY` is set. The same mapping is supported under `providers.openrouter`. + +## Verification + +`jev doctor` checks configuration and key presence; it does not run inference or validate a key with the provider. + +```sh +node --test test/providers.test.mjs test/provider-transports.test.mjs +``` + +These offline checks cover all three provider selections, CLI/MCP precedence, custom environment names, Hermes/OMP transport, and fail-open behavior. Remote-provider requests use injected responses or a loopback HTTP server with fixture credentials. No paid inference, agent model, or real browser is invoked. Live provider/account availability requires a separately configured environment. diff --git a/integrations/hermes/__init__.py b/integrations/hermes/__init__.py index 70e26b2..e0dd364 100644 --- a/integrations/hermes/__init__.py +++ b/integrations/hermes/__init__.py @@ -29,11 +29,11 @@ def jev_route(args: dict, **kwargs) -> str: """Return a bounded decision as JSON; never execute the selected target.""" try: completed = subprocess.run( - [os.environ.get("JEV_NODE", "node"), str(CLI), "--provider", os.environ.get("JEV_LAYER_PROVIDER", "demo")], + [os.environ.get("JEV_NODE", "node"), str(CLI)], input=json.dumps(args) + "\n", capture_output=True, text=True, - timeout=float(os.environ.get("JEV_LAYER_TIMEOUT_S", "3")), + timeout=float(os.environ.get("JEV_LAYER_TIMEOUT_S", "10")), check=False, ) if completed.returncode != 0: diff --git a/integrations/omp/extension.js b/integrations/omp/extension.js index 1664f78..664a057 100644 --- a/integrations/omp/extension.js +++ b/integrations/omp/extension.js @@ -36,7 +36,7 @@ async function runCore(payload, signal) { } async function runCoreWithBun(payload, signal) { - const child = Bun.spawn([process.env.JEV_NODE ?? "node", CORE, "--provider", process.env.JEV_LAYER_PROVIDER ?? "demo"], { + const child = Bun.spawn([process.env.JEV_NODE ?? "node", CORE], { cwd: process.cwd(), env: process.env, stdin: "pipe", @@ -58,7 +58,7 @@ async function runCoreWithBun(payload, signal) { function runCoreWithNode(payload, signal) { return new Promise((resolve) => { - const child = spawn(process.env.JEV_NODE ?? "node", [CORE, "--provider", process.env.JEV_LAYER_PROVIDER ?? "demo"], { + const child = spawn(process.env.JEV_NODE ?? "node", [CORE], { cwd: process.cwd(), env: process.env, stdio: ["pipe", "pipe", "pipe"], diff --git a/src/browser.mjs b/src/browser.mjs index 671343b..668cf36 100644 --- a/src/browser.mjs +++ b/src/browser.mjs @@ -106,6 +106,7 @@ export async function decideBrowserStep(input, options = {}) { } const decision = await routeRequest(request, { provider: options.provider ?? "demo", + config: options.config, engine: options.engine ?? "native", policy: options.policy, maxContextBytes: options.maxContextBytes, @@ -566,4 +567,4 @@ function elapsed(started) { function message(error) { return error instanceof Error ? error.message : String(error); -} \ No newline at end of file +} diff --git a/src/cli.mjs b/src/cli.mjs index 647547f..10e9ae9 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -14,7 +14,7 @@ for await (const line of lines) { try { const request = JSON.parse(line); const { provider = args.provider ?? configuredProvider(config), engine = args.engine ?? "native", ...payload } = request; - const decision = await routeRequest(payload, { provider, engine }); + const decision = await routeRequest(payload, { provider, engine, config }); process.stdout.write(`${JSON.stringify(decision)}\n`); } catch (error) { process.stdout.write(`${JSON.stringify({ status: "error", reason: error instanceof Error ? error.message : String(error), execution: { enabled: false, status: "not_started" } })}\n`); diff --git a/src/config.mjs b/src/config.mjs index fa20ffb..816d967 100644 --- a/src/config.mjs +++ b/src/config.mjs @@ -46,6 +46,16 @@ export function configuredProvider(config, explicitProvider) { return explicitProvider ?? process.env.JEV_LAYER_PROVIDER ?? config?.provider ?? DEFAULT_CONFIG.provider; } +export function configuredProviderOptions(config, provider) { + if (!config?.providers?.[provider]) return {}; + const names = { ...DEFAULT_CONFIG.providers[provider], ...config.providers[provider] }; + return { + apiKey: process.env[names.api_key_env] ?? "", + endpoint: process.env[names.endpoint_env], + model: process.env[names.model_env], + }; +} + export function configuredReplayPath(config) { return process.env.JEV_REPLAY_CASES ?? config?.replay_cases ?? DEFAULT_CONFIG.replay_cases; } @@ -72,4 +82,4 @@ function mergeConfig(parsed) { ...(parsed.providers && typeof parsed.providers === "object" ? parsed.providers : {}), }, }; -} \ No newline at end of file +} diff --git a/src/mcp-server.mjs b/src/mcp-server.mjs index adff87f..e311528 100755 --- a/src/mcp-server.mjs +++ b/src/mcp-server.mjs @@ -125,6 +125,7 @@ async function route(args, id) { const decision = await routeRequest(request, { provider, engine, + config, contextFilterMode: request.policy?.context_filter_mode ?? config.features?.context_filter, }); return persistDecision(request, decision, id); @@ -133,6 +134,7 @@ async function route(args, id) { async function browserStep(args, id) { const { provider: requestedProvider, enabled, ...input } = args; const routed = await decideBrowserStep(input, { + config, enabled: enabled ?? config.features?.browser_fast_path, provider: configuredProvider(config, requestedProvider), }); @@ -152,6 +154,7 @@ async function supervise(args, id) { const { provider: requestedProvider, enabled, ...input } = args; const result = await superviseWork({ ...input, + config, enabled: enabled ?? config.features?.supervision, provider: configuredProvider(config, requestedProvider), receiptPath: CASES_PATH, diff --git a/src/providers/index.mjs b/src/providers/index.mjs new file mode 100644 index 0000000..cc8d3a1 --- /dev/null +++ b/src/providers/index.mjs @@ -0,0 +1,12 @@ +import { configuredProviderOptions } from "../config.mjs"; +import { DemoProvider } from "./demo.mjs"; +import { OpenRouterDecisionsProvider, TypeSafeProvider } from "./typesafe.mjs"; + +export function resolveProvider(provider, options = {}) { + if (provider && typeof provider === "object") return provider; + if (provider === "demo") return new DemoProvider(); + const settings = { ...configuredProviderOptions(options.config, provider), ...options[provider] }; + if (provider === "openrouter") return new OpenRouterDecisionsProvider(settings); + if (provider === "typesafe") return new TypeSafeProvider(settings); + throw new Error(`unsupported provider: ${provider}`); +} diff --git a/src/providers/typesafe.mjs b/src/providers/typesafe.mjs index 4056564..3f37e92 100644 --- a/src/providers/typesafe.mjs +++ b/src/providers/typesafe.mjs @@ -1,13 +1,12 @@ -import { stableJson } from "../contract.mjs"; - export class TypeSafeProvider { name = "typesafe"; - constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-latest", timeoutMs = 2_000 } = {}) { + constructor({ apiKey = process.env.TYPESAFE_API_KEY, endpoint = process.env.TYPESAFE_ENDPOINT ?? "https://api.typesafe.ai/v1/systemone", model = process.env.TYPESAFE_MODEL ?? "jev-latest", timeoutMs = 2_000, fetchImpl = globalThis.fetch } = {}) { this.apiKey = apiKey; this.endpoint = endpoint; this.model = model; this.timeoutMs = timeoutMs; + this.fetchImpl = fetchImpl; } async decide({ state, candidates }) { @@ -31,7 +30,8 @@ export class TypeSafeProvider { async evaluate({ state, questions }) { if (!this.apiKey) throw new Error("TYPESAFE_API_KEY is not configured"); - const response = await fetch(this.endpoint, { + if (typeof this.fetchImpl !== "function") throw new Error("fetch is unavailable"); + const response = await this.fetchImpl(this.endpoint, { method: "POST", headers: { Authorization: `Bearer ${this.apiKey}`, @@ -57,8 +57,6 @@ export class TypeSafeProvider { } export class OpenRouterDecisionsProvider { - name = "openrouter:typesafe/jev-1.13"; - constructor({ apiKey = process.env.OPENROUTER_API_KEY, endpoint = process.env.OPENROUTER_DECISIONS_ENDPOINT ?? "https://openrouter.ai/api/alpha/decisions", @@ -68,12 +66,14 @@ export class OpenRouterDecisionsProvider { httpReferer = process.env.OPENROUTER_HTTP_REFERER, appTitle = process.env.OPENROUTER_APP_TITLE, } = {}) { + this.name = `openrouter:${model}`; this.apiKey = apiKey; this.endpoint = endpoint; this.model = model; this.timeoutMs = timeoutMs; this.fetchImpl = fetchImpl; this.httpReferer = httpReferer; + this.appTitle = appTitle; } async decide({ state, candidates }) { const criteria = Object.fromEntries(candidates.map((candidate) => [candidate.id, { diff --git a/src/route.mjs b/src/route.mjs index fd5f225..39aab81 100644 --- a/src/route.mjs +++ b/src/route.mjs @@ -2,8 +2,7 @@ import { byteLength, decisionEnvelope, normalizeRequest } from "./contract.mjs"; import { projectState } from "./context-filter.mjs"; import { discoverCapabilities } from "./discovery.mjs"; import { decisionCandidates, DEFAULT_POLICY, deterministicCandidate, filterCapabilities, normalizeCapabilities } from "./registry.mjs"; -import { DemoProvider } from "./providers/demo.mjs"; -import { OpenRouterDecisionsProvider, TypeSafeProvider } from "./providers/typesafe.mjs"; +import { resolveProvider } from "./providers/index.mjs"; export async function routeRequest(input, options = {}) { const discovered = options.discovery ? discoverCapabilities(options.discovery) : []; @@ -15,9 +14,8 @@ export async function routeRequest(input, options = {}) { const capabilities = normalizeCapabilities(request.capabilities); const { policy, assessments, eligible } = filterCapabilities(capabilities, request, effectivePolicy); const projection = projectState(request, eligible, options.maxContextBytes ?? 6_000, { mode: options.contextFilterMode }); - const provider = resolveProvider(options.provider ?? "demo", options); const base = { - provider: provider.name, + provider: typeof options.provider === "string" ? options.provider : options.provider?.name ?? "jev-demo", candidateCount: eligible.length, contextBytes: projection.context_bytes, started, @@ -26,6 +24,13 @@ export async function routeRequest(input, options = {}) { if (process.env.JEV_LAYER_ENABLED === "0") { return fallback(base, assessments, "disabled", "Jev layer disabled by JEV_LAYER_ENABLED=0", projection.state); } + let provider; + try { + provider = resolveProvider(options.provider ?? "demo", options); + base.provider = provider.name; + } catch (error) { + return fallback(base, assessments, "provider_error", error instanceof Error ? error.message : String(error), projection.state); + } if (eligible.length === 0) { return decisionEnvelope({ status: "no_decision", @@ -123,13 +128,6 @@ export async function routeRequest(input, options = {}) { }); } -function resolveProvider(provider, options) { - if (provider && typeof provider === "object" && typeof provider.decide === "function") return provider; - if (provider === "demo") return new DemoProvider(); - if (provider === "openrouter") return new OpenRouterDecisionsProvider(options.openrouter); - throw new Error(`unsupported provider: ${provider}`); -} - function readChoiceAnswer(raw) { const answers = raw?.answers; if (!answers || typeof answers !== "object") throw new Error("response has no answers object"); diff --git a/src/supervision.mjs b/src/supervision.mjs index ac6bb38..561ba21 100644 --- a/src/supervision.mjs +++ b/src/supervision.mjs @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; import { stableJson } from "./contract.mjs"; -import { DemoProvider } from "./providers/demo.mjs"; -import { OpenRouterDecisionsProvider, TypeSafeProvider } from "./providers/typesafe.mjs"; +import { resolveProvider } from "./providers/index.mjs"; import { appendSupervisionReceipt } from "./receipts.mjs"; export const SUPERVISION_ENV = "JEV_SUPERVISION"; @@ -104,6 +103,7 @@ export async function superviseWork({ actor_permissions, policy, provider = "demo", + config, enabled, attempts = 0, receiptPath, @@ -124,7 +124,7 @@ export async function superviseWork({ if (!supervisionEnabled({ enabled })) return finalize(base, started, "disabled", 0); let resolved; try { - resolved = resolveProvider(provider); + resolved = resolveProvider(provider, { config }); if (!resolved || typeof resolved.evaluate !== "function") throw new Error("provider does not support supervision evaluation"); } catch (error) { return finalize(base, started, "provider_error", 1, error); @@ -177,14 +177,6 @@ export async function superviseWork({ return result; } -function resolveProvider(provider) { - if (provider && typeof provider === "object") return provider; - if (provider === "demo") return new DemoProvider(); - if (provider === "openrouter") return new OpenRouterDecisionsProvider(); - if (provider === "typesafe") return new TypeSafeProvider(); - throw new Error(`unsupported provider: ${provider}`); -} - function finalize(result, started, reason, jevCalls, error = null) { const message = error instanceof Error ? error.message : error ? String(error) : reason; return { diff --git a/test/provider-transports.test.mjs b/test/provider-transports.test.mjs new file mode 100644 index 0000000..2ffdfff --- /dev/null +++ b/test/provider-transports.test.mjs @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { routeRequest } from "../src/route.mjs"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const request = { + intent: "Search repository files", + actor_permissions: ["read"], + capabilities: [ + { id: "search", name: "Repository search", description: "Search repository files", risk: "low", permissions: ["read"] }, + { id: "clock", name: "Clock", description: "Report the current time", risk: "low", permissions: ["read"] }, + ], +}; +const cleanEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^(JEV_|TYPESAFE_|OPENROUTER_)/.test(key))); +const hasPython = spawnSync("python3", ["--version"]).status === 0; + +test("provider configuration reaches CLI, MCP and adapters over local HTTP", { timeout: 30_000 }, async (t) => { + const workspace = await mkdtemp(join(tmpdir(), "jev-providers-")); + t.after(() => rm(workspace, { recursive: true, force: true })); + const calls = []; + const server = createServer(async (incoming, outgoing) => { + let text = ""; + for await (const chunk of incoming) text += chunk; + const body = JSON.parse(text); + calls.push({ url: incoming.url, headers: incoming.headers, body }); + if (incoming.url === "/timeout") return; + const answers = Object.fromEntries(Object.entries(body.questions).map(([id, question]) => { + if (question.type === "noul") return [id, { type: "noul", noul: 0.1 }]; + const ids = Object.keys(question.criteria); + const choice = ids.includes("search") ? "search" : ids[0]; + return [id, { type: "choice", choice, confidence: 1, probabilities: Object.fromEntries(ids.map((key) => [key, key === choice ? 1 : 0])) }]; + })); + outgoing.setHeader("Content-Type", "application/json"); + outgoing.end(JSON.stringify({ model: body.model, answers, usage: { input_tokens: 10, output_tokens: 2 } })); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + t.after(() => { server.closeAllConnections(); server.close(); }); + const endpoint = `http://127.0.0.1:${server.address().port}`; + const configPath = join(workspace, "config.json"); + const casesPath = join(workspace, "cases.jsonl"); + const env = { + ...cleanEnv, + JEV_CONFIG: configPath, + JEV_REPLAY_CASES: casesPath, + TYPESAFE_API_KEY: "fixture-typesafe", + TYPESAFE_ENDPOINT: `${endpoint}/typesafe`, + TYPESAFE_MODEL: "fixture-typesafe-model", + OPENROUTER_API_KEY: "fixture-openrouter", + OPENROUTER_DECISIONS_ENDPOINT: `${endpoint}/openrouter`, + OPENROUTER_DECISIONS_MODEL: "fixture-openrouter-model", + JEV_NODE: process.execPath, + PYTHONDONTWRITEBYTECODE: "1", + }; + + async function run(args, input, overrides = {}, command = process.execPath) { + const child = spawn(command, args, { cwd: workspace, env: { ...env, ...overrides }, stdio: ["pipe", "pipe", "pipe"], timeout: 10_000 }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.stdin.end(input); + const [code] = await once(child, "close"); + assert.equal(code, 0, stderr); + assert.equal(stderr, ""); + return stdout.trim().split(/\r?\n/).map((line) => JSON.parse(line)); + } + + async function cli(payload = request, args = [], overrides = {}) { + return (await run([join(root, "bin/jev.mjs"), "cli", ...args], JSON.stringify(payload), overrides))[0]; + } + + async function mcp(payload = request, overrides = {}, name = "jev_route") { + const messages = [ + { jsonrpc: "2.0", id: 1, method: "initialize", params: {} }, + { jsonrpc: "2.0", id: 2, method: "tools/call", params: { name, arguments: payload } }, + ]; + const responses = await run([join(root, "bin/jev.mjs"), "mcp"], messages.map((message) => JSON.stringify(message)).join("\n") + "\n", overrides); + assert.equal(responses[0].result.serverInfo.name, "jev-layer"); + assert.equal(responses[1].error, undefined); + return responses[1].result.structuredContent; + } + + function selected(decision, provider, before) { + assert.equal(decision.status, "selected"); + assert.equal(decision.selected, "search"); + assert.ok(decision.correlation_id); + assert.deepEqual(decision.execution, { enabled: false, status: "not_started" }); + assert.equal(calls.length, before + (provider === "demo" ? 0 : 1)); + if (provider === "demo") return; + assert.equal(calls.at(-1).url, `/${provider}`); + assert.equal(calls.at(-1).headers.authorization, `Bearer fixture-${provider}`); + assert.equal(calls.at(-1).body.model, `fixture-${provider}-model`); + if (provider === "openrouter") assert.equal(decision.receipt.provider, "openrouter:fixture-openrouter-model"); + } + + for (const provider of ["demo", "openrouter", "typesafe"]) { + for (const transport of ["cli", "mcp"]) { + await t.test(`${provider} via ${transport}: explicit selection overrides environment and config`, async () => { + await writeFile(configPath, JSON.stringify({ provider: "unknown" })); + const before = calls.length; + const decision = transport === "cli" + ? await cli({ ...request, provider }, ["--provider", "unknown"], { JEV_LAYER_PROVIDER: "unknown" }) + : await mcp({ ...request, provider }, { JEV_LAYER_PROVIDER: "unknown" }); + selected(decision, provider, before); + }); + await t.test(`${provider} via ${transport}: environment overrides config`, async () => { + await writeFile(configPath, JSON.stringify({ provider: "unknown" })); + const before = calls.length; + const decision = transport === "cli" + ? await cli(request, [], { JEV_LAYER_PROVIDER: provider }) + : await mcp(request, { JEV_LAYER_PROVIDER: provider }); + selected(decision, provider, before); + }); + await t.test(`${provider} via ${transport}: project configuration`, async () => { + await writeFile(configPath, JSON.stringify({ provider })); + const before = calls.length; + selected(await (transport === "cli" ? cli() : mcp()), provider, before); + }); + } + await t.test(`${provider} via CLI flag overrides environment and config`, async () => { + await writeFile(configPath, JSON.stringify({ provider: "unknown" })); + const before = calls.length; + selected(await cli(request, ["--provider", provider], { JEV_LAYER_PROVIDER: "unknown" }), provider, before); + }); + await t.test(`${provider} remains disabled without HTTP`, async () => { + const before = calls.length; + for (const decision of [await cli({ ...request, provider }, [], { JEV_LAYER_ENABLED: "0" }), await mcp({ ...request, provider }, { JEV_LAYER_ENABLED: "0" })]) { + assert.equal(decision.fallback.type, "disabled"); + assert.equal(decision.execution.enabled, false); + } + assert.equal(calls.length, before); + }); + } + + for (const provider of ["openrouter", "typesafe"]) { + await writeFile(configPath, JSON.stringify({ provider, providers: { [provider]: { + api_key_env: "FIXTURE_KEY", endpoint_env: "FIXTURE_ENDPOINT", model_env: "FIXTURE_MODEL", + } } })); + const aliases = { FIXTURE_KEY: `fixture-${provider}`, FIXTURE_ENDPOINT: `${endpoint}/${provider}`, FIXTURE_MODEL: `fixture-${provider}-model` }; + const guardedEnv = { ...aliases, TYPESAFE_API_KEY: "", OPENROUTER_API_KEY: "", TYPESAFE_ENDPOINT: `${endpoint}/wrong`, OPENROUTER_DECISIONS_ENDPOINT: `${endpoint}/wrong` }; + for (const transport of ["cli", "mcp"]) { + await t.test(`${provider} via ${transport}: configured environment variable names`, async () => { + const before = calls.length; + selected(await (transport === "cli" ? cli(request, [], guardedEnv) : mcp(request, guardedEnv)), provider, before); + }); + await t.test(`${provider} via ${transport}: missing configured key fails open`, async () => { + const before = calls.length; + const decision = await (transport === "cli" ? cli(request) : mcp()); + assert.equal(decision.fallback.type, "provider_error"); + assert.equal(calls.length, before); + }); + } + await t.test(`${provider} configuration reaches MCP supervision and browser decisions`, async () => { + const before = calls.length; + const supervised = await mcp({ enabled: true, job: {}, observation: {} }, guardedEnv, "jev_supervise"); + assert.equal(supervised.status, "judged"); + const browser = await mcp({ enabled: true, goal: "Scroll down", observation: { scroll: { down: true } } }, guardedEnv, "jev_browser_step"); + assert.equal(browser.status, "selected"); + assert.equal(browser.selected, "browser:scroll:down"); + assert.equal(browser.execution.enabled, false); + assert.equal(calls.length, before + 2); + assert.equal(calls.at(-1).url, `/${provider}`); + }); + await t.test(`${provider} times out an actual stalled HTTP request`, async () => { + const decision = await routeRequest(request, { provider, [provider]: { apiKey: "fixture-key", endpoint: `${endpoint}/timeout`, timeoutMs: 50 } }); + assert.equal(decision.fallback.type, "provider_error"); + assert.match(decision.reason, /timed out/); + assert.equal(decision.execution.enabled, false); + }); + + await t.test(`${provider} project configuration reaches the OMP Node adapter`, async () => { + const before = calls.length; + const code = ` + import extension from ${JSON.stringify(new URL("../integrations/omp/extension.js", import.meta.url).href)}; + const field = { optional() { return this; } }; + const zod = { string: () => field, unknown: () => field, array: () => field, object: () => field }; + let tool; + extension({ zod, registerTool(value) { tool = value; } }); + console.log(JSON.stringify((await tool.execute("fixture", ${JSON.stringify(request)})).details)); + `; + selected((await run(["--input-type=module", "-e", code], "", guardedEnv))[0], provider, before); + }); + await t.test(`${provider} project configuration reaches the Hermes adapter`, { skip: !hasPython }, async () => { + const before = calls.length; + const code = "import importlib.util, json, pathlib, sys\np = pathlib.Path(sys.argv[1])\nspec = importlib.util.spec_from_file_location('fixture_hermes', p, submodule_search_locations=[str(p.parent)])\nmodule = importlib.util.module_from_spec(spec)\nsys.modules[spec.name] = module\nspec.loader.exec_module(module)\nprint(module.jev_route(json.load(sys.stdin)))"; + selected((await run(["-c", code, join(root, "integrations/hermes/__init__.py")], JSON.stringify(request), guardedEnv, "python3"))[0], provider, before); + }); + } + + await t.test("MCP persists routing cases with their correlation ids", async () => { + const records = (await readFile(casesPath, "utf8")).trim().split(/\r?\n/).map(JSON.parse); + assert.ok(records.some((record) => record.decision?.receipt?.provider === "typesafe")); + for (const record of records.filter((record) => record.record_type === "routing_case")) { + assert.equal(record.case_id, record.correlation_id); + } + }); +}); diff --git a/test/providers.test.mjs b/test/providers.test.mjs new file mode 100644 index 0000000..d2d844e --- /dev/null +++ b/test/providers.test.mjs @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { routeRequest } from "../src/route.mjs"; + +// Keep these offline checks independent of a developer's provider environment. +for (const key of Object.keys(process.env)) { + if (/^(JEV_|TYPESAFE_|OPENROUTER_)/.test(key)) delete process.env[key]; +} + +const request = { + intent: "Search repository files", + actor_permissions: ["read"], + capabilities: [ + { id: "search", name: "Repository search", description: "Search repository files", permissions: ["read"], risk: "low" }, + { id: "clock", name: "Clock", description: "Report the current time", permissions: ["read"], risk: "low" }, + ], +}; +const answer = { type: "choice", choice: "search", probabilities: { search: 0.9, clock: 0.1 }, confidence: 0.8 }; + +test("demo routes deterministically without a network call or credentials", async (t) => { + t.mock.method(globalThis, "fetch", () => { throw new Error("demo must remain offline"); }); + const first = await routeRequest(request, { provider: "demo" }); + const second = await routeRequest(request, { provider: "demo" }); + assert.equal(first.status, "selected"); + assert.equal(first.selected, "search"); + assert.deepEqual(first.probabilities, second.probabilities); + assert.equal(first.receipt.provider, "jev-demo"); + assert.equal(first.execution.enabled, false); + assert.equal(globalThis.fetch.mock.callCount(), 0); +}); + +for (const [provider, endpoint, model] of [ + ["typesafe", "https://api.typesafe.ai/v1/systemone", "jev-latest"], + ["openrouter", "https://openrouter.ai/api/alpha/decisions", "typesafe/jev-1.13"], +]) { + test(`${provider} routes with the documented endpoint, model, auth and Choice contract`, async () => { + let calls = 0; + const decision = await routeRequest(request, { + provider, + [provider]: { + apiKey: "fixture-key", + httpReferer: "https://example.test", + appTitle: "Jev fixture", + async fetchImpl(url, init) { + calls += 1; + assert.equal(url, endpoint); + assert.equal(init.method, "POST"); + assert.equal(init.headers.Authorization, "Bearer fixture-key"); + assert.equal(init.headers["Content-Type"], "application/json"); + assert.ok(init.signal instanceof AbortSignal); + if (provider === "openrouter") { + assert.equal(init.headers["HTTP-Referer"], "https://example.test"); + assert.equal(init.headers["X-Title"], "Jev fixture"); + } + const body = JSON.parse(init.body); + assert.equal(body.model, model); + assert.equal(body.state.intent, request.intent); + assert.equal(body.questions.tool.type, "choice"); + assert.deepEqual(Object.keys(body.questions.tool.criteria), ["search", "clock"]); + assert.equal(body.questions.tool.criteria.search.description, request.capabilities[0].description); + assert.equal("messages" in body, false); + return Response.json({ model, answers: { tool: answer }, usage: { input_tokens: 10, output_tokens: 2 } }); + }, + }, + }); + assert.equal(calls, 1); + assert.equal(decision.status, "selected"); + assert.equal(decision.selected, "search"); + assert.deepEqual(decision.probabilities, answer.probabilities); + assert.equal(decision.confidence, answer.confidence); + assert.deepEqual(decision.execution, { enabled: false, status: "not_started" }); + }); + + test(`${provider} never calls HTTP when credentials are missing or routing is deterministic`, async () => { + let calls = 0; + const options = { provider, [provider]: { apiKey: "", fetchImpl: async () => { calls += 1; throw new Error("must not call HTTP"); } } }; + const missing = await routeRequest(request, options); + assert.equal(missing.status, "fallback"); + assert.equal(missing.fallback.type, "provider_error"); + assert.match(missing.reason, /API_KEY is not configured/); + const deterministic = await routeRequest({ ...request, policy: { deterministic: true, deterministic_capability_id: "search" } }, options); + assert.equal(deterministic.status, "selected"); + assert.equal(calls, 0); + }); + + for (const status of [401, 422, 429, 529]) { + test(`${provider} fails open on HTTP ${status} without retries`, async () => { + let calls = 0; + const decision = await routeRequest(request, { provider, [provider]: { + apiKey: "fixture-key", + async fetchImpl() { calls += 1; return new Response("fixture error", { status }); }, + } }); + assert.equal(calls, 1); + assert.equal(decision.status, "fallback"); + assert.equal(decision.fallback.type, "provider_error"); + assert.match(decision.reason, new RegExp(`HTTP ${status}`)); + assert.equal(decision.execution.enabled, false); + }); + } + + for (const [name, response, fallback] of [ + ["invalid JSON", "{", "provider_error"], + ["missing answers", {}, "provider_error"], + ["wrong answer type", { answers: { tool: { type: "score" } } }, "malformed_response"], + ["unknown capability", { answers: { tool: { ...answer, choice: "unknown" } } }, "invalid_selection"], + ["low confidence", { answers: { tool: { ...answer, confidence: 0.1 } } }, "low_confidence"], + ]) { + test(`${provider} fails open on ${name}`, async () => { + const decision = await routeRequest(request, { provider, [provider]: { + apiKey: "fixture-key", + fetchImpl: async () => new Response(typeof response === "string" ? response : JSON.stringify(response)), + } }); + assert.equal(decision.status, "fallback"); + assert.equal(decision.fallback.type, fallback); + assert.equal(decision.execution.enabled, false); + }); + } +} + +test("unsupported providers preserve the routing fallback envelope", async () => { + const decision = await routeRequest(request, { provider: "unknown" }); + assert.equal(decision.status, "fallback"); + assert.equal(decision.fallback.type, "provider_error"); + assert.ok(decision.correlation_id); + assert.equal(decision.execution.enabled, false); +});