diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..53fbaa7 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,54 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + # Windows: the same OS the tests were written and verified on (the path-sandbox tests use Windows paths). + runs-on: windows-latest + defaults: + run: + shell: bash + env: + TESTCOLLAB_TOKEN: ${{ secrets.TESTCOLLAB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Unit and integration tests (bun test) + working-directory: testing + run: | + mkdir -p ci-results + bun test --coverage --reporter=junit --reporter-outfile=ci-results/junit-ci.xml + + - name: API tests (Postman collections run by Newman) + if: ${{ !cancelled() }} + run: bun testing/automation/api-tests.ts + + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: test-results + path: | + testing/ci-results/junit-ci.xml + testing/automation/results/*.xml + + # Runs only when the repository has a TESTCOLLAB_TOKEN secret (never on forks). + # Optional repository variable TESTCOLLAB_PROJECT overrides the project id. + - name: Report results to TestCollab + if: ${{ !cancelled() && env.TESTCOLLAB_TOKEN != '' }} + working-directory: testing + run: npx --yes @testcollab/cli report --project "${{ vars.TESTCOLLAB_PROJECT || '19282' }}" --format junit --result-file ci-results/junit-ci.xml --auto-create diff --git a/.gitignore b/.gitignore index dcc3b48..68a1e51 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,8 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # /brag launch video output (generated locally; not committed) brag-output*/ + +# test run outputs (generated) +testing/ci-results/ +testing/automation/results/ +testing/tmp/ diff --git a/bun.lock b/bun.lock index 8e0dda2..ed0113c 100644 --- a/bun.lock +++ b/bun.lock @@ -35,6 +35,7 @@ }, "devDependencies": { "@codeflow/server": "workspace:*", + "@codeflow/shared": "workspace:*", "@types/react": "^19.2.17", }, "peerDependencies": { diff --git a/packages/cli/package.json b/packages/cli/package.json index 10b68a0..fb94926 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -33,6 +33,7 @@ }, "devDependencies": { "@codeflow/server": "workspace:*", + "@codeflow/shared": "workspace:*", "@types/react": "^19.2.17" }, "peerDependencies": { diff --git a/packages/cli/src/lib/local-tools.ts b/packages/cli/src/lib/local-tools.ts index c882903..e8925f8 100644 --- a/packages/cli/src/lib/local-tools.ts +++ b/packages/cli/src/lib/local-tools.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, readdir, stat, writeFile } from "fs/promises"; -import { dirname, isAbsolute, join, relative, resolve } from "path"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "path"; import { toolInputSchemas, Mode, type ModeType } from "@codeflow/shared"; const MAX_FILE_SIZE = 10_000; @@ -13,7 +13,7 @@ function resolveInsideCwd(path: string) { const resolved = resolve(cwd, path); const rel = relative(cwd, resolved); - if (rel.startsWith("..") || isAbsolute(rel)) { + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { throw new Error("Path is outside the project directory"); } @@ -87,7 +87,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M "-E", ]; if (include) args.push(`--include=${include}`); - args.push(pattern, resolved); + args.push("-e", pattern, resolved); const proc = Bun.spawn(["grep", ...args], { cwd, stdout: "pipe", stderr: "pipe" }); const [stdout, stderr] = await Promise.all([ @@ -140,7 +140,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M if (occurrences === 0) throw new Error("oldString not found in file"); if (occurrences > 1) throw new Error(`oldString is ambiguous; found ${occurrences} matches`); - await writeFile(resolved, content.replace(oldString, newString), "utf-8"); + await writeFile(resolved, content.replace(oldString, () => newString), "utf-8"); return { success: true as const, path: relative(cwd, resolved) }; } case "bash": { diff --git a/packages/server/src/lib/credits.ts b/packages/server/src/lib/credits.ts index 17a7d1f..1645864 100644 --- a/packages/server/src/lib/credits.ts +++ b/packages/server/src/lib/credits.ts @@ -79,7 +79,7 @@ function convertUsdToCredits(estimatedCostUsd: number) { // If a request costs any non-zero amount, charge at least 1 credit, then // round up so partial credits always become a whole credit. - return Math.max(1, Math.ceil(estimatedCostUsd / USD_PER_CREDIT)); + return Math.max(1, Math.ceil(estimatedCostUsd / USD_PER_CREDIT - 1e-9)); }; diff --git a/testing/automation/api-tests.ts b/testing/automation/api-tests.ts new file mode 100644 index 0000000..341b8fe --- /dev/null +++ b/testing/automation/api-tests.ts @@ -0,0 +1,89 @@ +// Postman / Newman harness for the CodeFlow API. +// Starts the REAL Hono server (packages/server/src/index.ts) on a local port with Clerk, Polar, PostgreSQL +// and the model API replaced by stubs, so no outside service is called. The bearer token picks the stub user: +// user-ok (100 credits) user-zero (0) user-neg (-1) user-one (1) user-down (credits look-up fails) +// anything else = invalid token. Session "s1" exists; every other id is "not found". +// +// bun testing/automation/api-tests.ts start the stub server, run both collections with Newman, stop +// bun testing/automation/api-tests.ts --serve only keep the stub server running (use it from the Postman app) +import { mock } from "bun:test"; +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; + +const here = import.meta.dir; +const serverSrc = join(here, "../../packages/server/src"); +const port = Number(process.env.API_TEST_PORT ?? 3101); + +const USERS: Record = { + "user-ok": { id: "user_ok", balance: 100 }, + "user-zero": { id: "user_zero", balance: 0 }, + "user-neg": { id: "user_neg", balance: -1 }, + "user-one": { id: "user_one", balance: 1 }, + "user-down": { id: "user_down", balance: "down" }, +}; +const session = (userId: string) => ({ id: "s1", userId, title: "demo session", messages: [], createdAt: new Date("2026-01-01") }); + +const aiPath = Bun.resolveSync("ai", serverSrc); +const dbClientPath = Bun.resolveSync("@codeflow/database/client", serverSrc); +const realAi = await import(aiPath); + +mock.module(join(serverSrc, "lib/auth"), () => ({ + authenticateOAuthRequest: async (req: Request) => { + const user = USERS[(req.headers.get("authorization") ?? "").replace(/^Bearer /, "")]; + return user ? { userId: user.id } : null; + }, +})); +mock.module(join(serverSrc, "lib/polar"), () => ({ + getAvailableCreditsBalance: async (userId: string) => { + const user = Object.values(USERS).find((u) => u.id === userId); + if (!user || user.balance === "down") throw new Error("Polar unreachable"); + return user.balance; + }, + ingestAiUsage: async () => {}, + createCheckoutUrl: async () => "https://example.invalid/checkout", + createCustomerPortalUrl: async () => "https://example.invalid/portal", +})); +mock.module(dbClientPath, () => ({ + db: { + session: { + findUnique: async ({ where }: any) => (where.id === "s1" ? session(where.userId) : null), + findMany: async ({ where }: any) => [session(where.userId)], + create: async ({ data }: any) => ({ id: "new1", messages: [], createdAt: new Date(), updatedAt: new Date(), ...data }), + update: async ({ where }: any) => session(where.userId), + }, + }, +})); +mock.module(aiPath, () => ({ + ...realAi, + streamText: () => ({ + toUIMessageStreamResponse: () => + new Response('data: {"type":"finish"}\n\n', { headers: { "content-type": "text/event-stream" } }), + }), +})); + +const { default: app } = await import(join(serverSrc, "index")); +const server = Bun.serve({ port, fetch: app.fetch, idleTimeout: app.idleTimeout }); +console.log(`CodeFlow API (stubbed) listening on http://localhost:${port}`); + +if (process.argv.includes("--serve")) { + console.log("Serving for the Postman app. Press Ctrl+C to stop."); +} else { + const results = join(here, "results"); + mkdirSync(results, { recursive: true }); + const runs: [string, string[], string][] = [ + ["codeflow-api.postman_collection.json", [], "newman-api.xml"], + ["codeflow-chat-gate-data-driven.postman_collection.json", ["-d", join(here, "data/chat-gate.csv")], "newman-chat-gate.xml"], + ]; + let failed = 0; + for (const [collection, extra, xml] of runs) { + const proc = Bun.spawn( + ["bunx", "newman@6", "run", join(here, collection), "-e", join(here, "local.postman_environment.json"), + "--env-var", `baseUrl=http://localhost:${port}`, ...extra, "-r", "cli,junit", "--reporter-junit-export", join(results, xml)], + { stdio: ["inherit", "inherit", "inherit"] }, + ); + if ((await proc.exited) !== 0) failed++; + } + server.stop(true); + console.log(failed ? `\n${failed} Newman run(s) FAILED` : "\nAll Newman runs passed"); + process.exit(failed ? 1 : 0); +} diff --git a/testing/automation/codeflow-api.postman_collection.json b/testing/automation/codeflow-api.postman_collection.json new file mode 100644 index 0000000..2a4da53 --- /dev/null +++ b/testing/automation/codeflow-api.postman_collection.json @@ -0,0 +1,860 @@ +{ + "info": { + "name": "CodeFlow API", + "description": "Black-box API tests of the CodeFlow server. Run against the stub server: `bun testing/automation/api-tests.ts --serve`. Tokens are stub users: tokenOk (100 credits), tokenZero (0), tokenNeg (-1), tokenOne (1), tokenDown (credits look-up fails).", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is under 2000 ms\", function () { pm.expect(pm.response.responseTime).to.be.below(2000); });" + ] + } + } + ], + "item": [ + { + "name": "Auth callback (GET /auth/callback)", + "item": [ + { + "name": "AC-01 OAuth error is returned to the user", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 400\", function () { pm.response.to.have.status(400); });", + "pm.test(\"Body is the error description\", function () { pm.expect(pm.response.text()).to.eql(\"Access was denied\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/auth/callback?error=access_denied&error_description=Access%20was%20denied", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "auth", + "callback" + ], + "query": [ + { + "key": "error", + "value": "access_denied" + }, + { + "key": "error_description", + "value": "Access%20was%20denied" + } + ] + } + } + }, + { + "name": "AC-02 Missing code and state", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 400\", function () { pm.response.to.have.status(400); });", + "pm.test(\"Body explains what is missing\", function () { pm.expect(pm.response.text()).to.eql(\"Missing authorization code or state\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/auth/callback", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "auth", + "callback" + ] + } + } + }, + { + "name": "AC-03 Malformed state", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 400\", function () { pm.response.to.have.status(400); });", + "pm.test(\"Body is 'Invalid authentication state'\", function () { pm.expect(pm.response.text()).to.eql(\"Invalid authentication state\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/auth/callback?code=abc&state=not-valid", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "auth", + "callback" + ], + "query": [ + { + "key": "code", + "value": "abc" + }, + { + "key": "state", + "value": "not-valid" + } + ] + } + } + }, + { + "name": "AC-04 Valid state redirects to the CLI's loopback port", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 302\", function () { pm.response.to.have.status(302); });", + "pm.test(\"Redirects to http://localhost:5555/callback with code and state\", function () { pm.expect(pm.response.headers.get(\"Location\")).to.eql(\"http://localhost:5555/callback?code=abc&state=eyJwb3J0Ijo1NTU1fQ.sig\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/auth/callback?code=abc&state=eyJwb3J0Ijo1NTU1fQ.sig", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "auth", + "callback" + ], + "query": [ + { + "key": "code", + "value": "abc" + }, + { + "key": "state", + "value": "eyJwb3J0Ijo1NTU1fQ.sig" + } + ] + } + }, + "protocolProfileBehavior": { + "followRedirects": false + } + } + ] + }, + { + "name": "Billing", + "item": [ + { + "name": "BL-01 Checkout without a token", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 401\", function () { pm.response.to.have.status(401); });", + "pm.test(\"Error message is 'Unauthorized. Run /login to continue.'\", function () { pm.expect(pm.response.json().error).to.eql(\"Unauthorized. Run /login to continue.\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [], + "url": { + "raw": "{{baseUrl}}/billing/checkout", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "billing", + "checkout" + ] + } + } + }, + { + "name": "BL-02 Checkout with a valid token", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Response is JSON\", function () { pm.response.to.have.header(\"Content-Type\"); pm.expect(pm.response.headers.get(\"Content-Type\")).to.include(\"application/json\"); });", + "pm.test(\"Returns a checkout url\", function () { pm.expect(pm.response.json().url).to.be.a(\"string\").and.not.empty; });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + } + ], + "url": { + "raw": "{{baseUrl}}/billing/checkout", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "billing", + "checkout" + ] + } + } + }, + { + "name": "BL-03 Billing portal with a valid token", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Returns a portal url\", function () { pm.expect(pm.response.json().url).to.be.a(\"string\").and.not.empty; });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + } + ], + "url": { + "raw": "{{baseUrl}}/billing/portal", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "billing", + "portal" + ] + } + } + }, + { + "name": "BL-04 Success page needs no sign-in", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Tells the user they can close the tab\", function () { pm.expect(pm.response.text()).to.include(\"You can close this tab\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/billing/success", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "billing", + "success" + ] + } + } + } + ] + }, + { + "name": "Sessions", + "item": [ + { + "name": "SE-01 List sessions without a token", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 401\", function () { pm.response.to.have.status(401); });", + "pm.test(\"Error message is 'Unauthorized. Run /login to continue.'\", function () { pm.expect(pm.response.json().error).to.eql(\"Unauthorized. Run /login to continue.\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/sessions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions" + ] + } + } + }, + { + "name": "SE-02 List own sessions", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Response is JSON\", function () { pm.response.to.have.header(\"Content-Type\"); pm.expect(pm.response.headers.get(\"Content-Type\")).to.include(\"application/json\"); });", + "pm.test(\"Returns an array with id and title\", function () { const b = pm.response.json(); pm.expect(b).to.be.an(\"array\").and.not.empty; pm.expect(b[0]).to.have.keys([\"id\", \"title\", \"createdAt\", \"userId\", \"messages\"]); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions" + ] + } + } + }, + { + "name": "SE-03 Open an existing session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Returns session s1\", function () { pm.expect(pm.response.json().id).to.eql(\"s1\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions/s1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions", + "s1" + ] + } + } + }, + { + "name": "SE-04 Open a session that does not exist", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 404\", function () { pm.response.to.have.status(404); });", + "pm.test(\"Error message is 'Session not found'\", function () { pm.expect(pm.response.json().error).to.eql(\"Session not found\"); });" + ] + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions/missing", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions", + "missing" + ] + } + } + }, + { + "name": "SE-05 Create a session", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 201\", function () { pm.response.to.have.status(201); });", + "pm.test(\"Returns the new session with the title\", function () { const b = pm.response.json(); pm.expect(b.id).to.be.a(\"string\"); pm.expect(b.title).to.eql(\"API test session\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"title\": \"API test session\"\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "SE-06 Create a session with no credits", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 402\", function () { pm.response.to.have.status(402); });", + "pm.test(\"Error message is 'No credits remaining. Run /upgrade to buy more credits.'\", function () { pm.expect(pm.response.json().error).to.eql(\"No credits remaining. Run /upgrade to buy more credits.\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenZero}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"title\": \"x\"\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "SE-07 Create a session with an invalid body", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 400\", function () { pm.response.to.have.status(400); });", + "pm.test(\"Error message is 'Invalid request body'\", function () { pm.expect(pm.response.json().error).to.eql(\"Invalid request body\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/sessions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "sessions" + ] + }, + "body": { + "mode": "raw", + "raw": "{}", + "options": { + "raw": { + "language": "json" + } + } + } + } + } + ] + }, + { + "name": "Chat gate (POST /chat), one request per decision-table rule", + "item": [ + { + "name": "DT-01 Invalid token is refused", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 401\", function () { pm.response.to.have.status(401); });", + "pm.test(\"Error message is 'Unauthorized. Run /login to continue.'\", function () { pm.expect(pm.response.json().error).to.eql(\"Unauthorized. Run /login to continue.\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenForged}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"s1\",\n \"mode\": \"BUILD\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "DT-02 Credits look-up failure gives 503", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 503\", function () { pm.response.to.have.status(503); });", + "pm.test(\"Error message is 'Unable to verify credits balance right now.'\", function () { pm.expect(pm.response.json().error).to.eql(\"Unable to verify credits balance right now.\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenDown}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"s1\",\n \"mode\": \"BUILD\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "DT-03 Zero balance gives 402", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 402\", function () { pm.response.to.have.status(402); });", + "pm.test(\"Error message is 'No credits remaining. Run /upgrade to buy more credits.'\", function () { pm.expect(pm.response.json().error).to.eql(\"No credits remaining. Run /upgrade to buy more credits.\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenZero}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"s1\",\n \"mode\": \"BUILD\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "DT-04 Unknown mode gives 400", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 400\", function () { pm.response.to.have.status(400); });", + "pm.test(\"Error message is 'Invalid request body'\", function () { pm.expect(pm.response.json().error).to.eql(\"Invalid request body\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"s1\",\n \"mode\": \"DEBUG\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "DT-05 Unknown session gives 404", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 404\", function () { pm.response.to.have.status(404); });", + "pm.test(\"Error message is 'Session not found'\", function () { pm.expect(pm.response.json().error).to.eql(\"Session not found\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"missing\",\n \"mode\": \"BUILD\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + }, + { + "name": "DT-06 Valid request streams a reply", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status is 200\", function () { pm.response.to.have.status(200); });", + "pm.test(\"Content type is text/event-stream\", function () { pm.expect(pm.response.headers.get(\"Content-Type\")).to.include(\"text/event-stream\"); });" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{tokenOk}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"s1\",\n \"mode\": \"BUILD\",\n \"model\": \"claude-haiku-4-5\",\n \"messages\": [\n {\n \"id\": \"m1\",\n \"role\": \"user\",\n \"parts\": [\n {\n \"type\": \"text\",\n \"text\": \"hello\"\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + } + } + ] + } + ] +} diff --git a/testing/automation/codeflow-chat-gate-data-driven.postman_collection.json b/testing/automation/codeflow-chat-gate-data-driven.postman_collection.json new file mode 100644 index 0000000..6c666bf --- /dev/null +++ b/testing/automation/codeflow-chat-gate-data-driven.postman_collection.json @@ -0,0 +1,69 @@ +{ + "info": { + "name": "CodeFlow chat gate (data-driven)", + "description": "One POST /chat request run once per row of data/chat-gate.csv. Columns: caseId, token, mode, model, sessionId, expectedStatus, expectedError.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is under 2000 ms\", function () { pm.expect(pm.response.responseTime).to.be.below(2000); });" + ] + } + } + ], + "item": [ + { + "name": "POST /chat - {{caseId}}", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const id = pm.iterationData.get(\"caseId\");", + "const expected = Number(pm.iterationData.get(\"expectedStatus\"));", + "const expectedError = pm.iterationData.get(\"expectedError\");", + "pm.test(id + \" returns \" + expected, function () { pm.response.to.have.status(expected); });", + "if (expectedError) { pm.test(id + \" error message\", function () { pm.expect(pm.response.json().error).to.eql(expectedError); }); }" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{token}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"{{sessionId}}\",\n \"mode\": \"{{mode}}\",\n \"model\": \"{{model}}\",\n \"messages\": [{ \"id\": \"m1\", \"role\": \"user\", \"parts\": [{ \"type\": \"text\", \"text\": \"hello\" }] }]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/chat", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "chat" + ] + } + } + } + ] +} diff --git a/testing/automation/data/chat-gate.csv b/testing/automation/data/chat-gate.csv new file mode 100644 index 0000000..a555b0a --- /dev/null +++ b/testing/automation/data/chat-gate.csv @@ -0,0 +1,23 @@ +caseId,token,mode,model,sessionId,expectedStatus,expectedError +DT-01,forged-token,BUILD,claude-haiku-4-5,s1,401,Unauthorized. Run /login to continue. +DT-02,user-down,BUILD,claude-haiku-4-5,s1,503,Unable to verify credits balance right now. +DT-03,user-zero,BUILD,claude-haiku-4-5,s1,402,No credits remaining. Run /upgrade to buy more credits. +DT-04,user-ok,DEBUG,claude-haiku-4-5,s1,400,Invalid request body +DT-05,user-ok,BUILD,claude-haiku-4-5,missing,404,Session not found +DT-06,user-ok,BUILD,claude-haiku-4-5,s1,200, +BVA-24,user-neg,BUILD,claude-haiku-4-5,s1,402,No credits remaining. Run /upgrade to buy more credits. +BVA-25,user-zero,BUILD,claude-haiku-4-5,s1,402,No credits remaining. Run /upgrade to buy more credits. +BVA-26,user-one,BUILD,claude-haiku-4-5,s1,200, +ECP-17,user-ok,BUILD,gpt-4,s1,400,Invalid request body +MX-01,user-ok,PLAN,claude-sonnet-4-6,s1,200, +MX-02,user-ok,PLAN,claude-haiku-4-5,s1,200, +MX-03,user-ok,PLAN,claude-opus-4-6,s1,200, +MX-04,user-ok,PLAN,gpt-5.4,s1,200, +MX-05,user-ok,PLAN,gpt-5.4-mini,s1,200, +MX-06,user-ok,PLAN,gpt-5.4-nano,s1,200, +MX-07,user-ok,BUILD,claude-sonnet-4-6,s1,200, +MX-08,user-ok,BUILD,claude-haiku-4-5,s1,200, +MX-09,user-ok,BUILD,claude-opus-4-6,s1,200, +MX-10,user-ok,BUILD,gpt-5.4,s1,200, +MX-11,user-ok,BUILD,gpt-5.4-mini,s1,200, +MX-12,user-ok,BUILD,gpt-5.4-nano,s1,200, diff --git a/testing/automation/local.postman_environment.json b/testing/automation/local.postman_environment.json new file mode 100644 index 0000000..bfe3f15 --- /dev/null +++ b/testing/automation/local.postman_environment.json @@ -0,0 +1,49 @@ +{ + "id": "codeflow-local", + "name": "CodeFlow local (stub server)", + "_postman_variable_scope": "environment", + "values": [ + { + "key": "baseUrl", + "value": "http://localhost:3101", + "type": "default", + "enabled": true + }, + { + "key": "tokenOk", + "value": "user-ok", + "type": "default", + "enabled": true + }, + { + "key": "tokenZero", + "value": "user-zero", + "type": "default", + "enabled": true + }, + { + "key": "tokenNeg", + "value": "user-neg", + "type": "default", + "enabled": true + }, + { + "key": "tokenOne", + "value": "user-one", + "type": "default", + "enabled": true + }, + { + "key": "tokenDown", + "value": "user-down", + "type": "default", + "enabled": true + }, + { + "key": "tokenForged", + "value": "forged-token", + "type": "default", + "enabled": true + } + ] +} diff --git a/testing/tests/cli-units.test.ts b/testing/tests/cli-units.test.ts new file mode 100644 index 0000000..08fc574 --- /dev/null +++ b/testing/tests/cli-units.test.ts @@ -0,0 +1,31 @@ +// Unit tests for CLI helpers. Importing the command list does not open a browser, +// read credentials, or call the server; actions only run when a command is chosen. +import { describe, expect, test } from "bun:test"; +import { getfilteredCommands } from "../../packages/cli/src/components/command-menu/filter-commands"; +import { getErrorMessage } from "../../packages/cli/src/lib/http-errors"; + +const names = (query: string) => getfilteredCommands(query).map((c) => c.name); + +describe("Unit: command menu filter (ECP on the typed query)", () => { + test("UT-07 empty query returns all 10 commands", () => expect(names("")).toHaveLength(10)); + test("UT-08 prefix 'mo' returns only 'models'", () => expect(names("mo")).toEqual(["models"])); + test("UT-09 upper-case 'MO' is matched case-insensitively", () => expect(names("MO")).toEqual(["models"])); + test("UT-10 prefix 'l' returns 'login' and 'logout'", () => expect(names("l")).toEqual(["login", "logout"])); + test("UT-11 prefix 'zz' returns no commands", () => expect(names("zz")).toEqual([])); +}); + +describe("Unit: HTTP error message extraction", () => { + const response = (json: () => Promise, status: number, statusText: string) => ({ json, status, statusText }); + + test("UT-12 JSON body with an error string -> that message", async () => { + expect(await getErrorMessage(response(async () => ({ error: "No credits remaining" }), 402, "Payment Required"))) + .toBe("No credits remaining"); + }); + test("UT-13 body that is not JSON -> status text", async () => { + expect(await getErrorMessage(response(async () => { throw new SyntaxError("bad json"); }, 503, "Service Unavailable"))) + .toBe("Service Unavailable"); + }); + test("UT-14 empty error and empty status text -> generic message with status code", async () => { + expect(await getErrorMessage(response(async () => ({ error: "" }), 500, ""))).toBe("Request failed with status 500"); + }); +}); diff --git a/testing/tests/credits.test.ts b/testing/tests/credits.test.ts new file mode 100644 index 0000000..32c43fe --- /dev/null +++ b/testing/tests/credits.test.ts @@ -0,0 +1,120 @@ +// Unit tests for packages/server/src/lib/credits.ts (pure logic, no network). +// Model under test: claude-haiku-4-5 = $1 / M input tokens, $5 / M output tokens, +// so the exact expected charge is ceil((input + 5 * output) / 10000) credits. +import { describe, expect, test } from "bun:test"; +import { calculateCreditsForUsage } from "../../packages/server/src/lib/credits"; + +const haiku = (inputTokens: unknown, outputTokens: unknown) => + calculateCreditsForUsage({ + provider: "anthropic", + model: "claude-haiku-4-5", + usage: { inputTokens, outputTokens } as any, + }).credits; + +describe("BVA: input domain 4n+1", () => { + const cases: [string, number, number, number][] = [ + ["BVA-01", 0, 100000, 50], + ["BVA-02", 1, 100000, 51], + ["BVA-03", 100000, 100000, 60], + ["BVA-04", 199999, 100000, 70], + ["BVA-05", 200000, 100000, 70], + ["BVA-06", 100000, 0, 10], + ["BVA-07", 100000, 1, 11], + ["BVA-08", 100000, 199999, 110], + ["BVA-09", 100000, 200000, 110], + ]; + for (const [id, x, y, expected] of cases) { + test(`${id} input=${x} output=${y} -> ${expected} credits`, () => { + expect(haiku(x, y)).toBe(expected); + }); + } + + test("BVA-10 robust: input=-1 is rejected", () => { + expect(() => haiku(-1, 100000)).toThrow("Credit conversion requires input and output token counts"); + }); + test("BVA-11 robust: output=-1 is rejected", () => { + expect(() => haiku(100000, -1)).toThrow("Credit conversion requires input and output token counts"); + }); +}); + +describe("BVA: output domain", () => { + const cases: [string, number, number][] = [ + ["BVA-12", 0, 0], + ["BVA-13", 1, 1], + ["BVA-14", 9999, 1], + ["BVA-15", 10000, 1], + ["BVA-16", 10001, 2], + ["BVA-17", 70000, 7], + ]; + for (const [id, input, expected] of cases) { + test(`${id} input=${input} output=0 -> ${expected} credits`, () => { + expect(haiku(input, 0)).toBe(expected); + }); + } +}); + +describe("ECP: provider / model classes", () => { + const bill = (provider: string, model: string) => + calculateCreditsForUsage({ provider, model, usage: { inputTokens: 10, outputTokens: 10 } as any }); + + test("ECP-01 valid model + matching provider -> credits returned", () => { + expect(bill("anthropic", "claude-haiku-4-5").credits).toBe(1); + }); + test("ECP-02 valid model + wrong provider -> unsupported model", () => { + expect(() => bill("anthropic", "gpt-5.4")).toThrow("Unsupported billing model: gpt-5.4"); + }); + test("ECP-03 unknown provider -> unsupported provider", () => { + expect(() => bill("google", "gemini-pro")).toThrow("Unsupported billing provider: google"); + }); + test("ECP-04 known provider + unknown model -> unsupported model", () => { + expect(() => bill("openai", "gpt-4")).toThrow("Unsupported billing model: gpt-4"); + }); +}); + +describe("ECP: token count classes", () => { + test("ECP-05 non-integer tokens (1.5) are rejected", () => { + expect(() => haiku(1.5, 10)).toThrow(); + }); + test("ECP-06 missing tokens (undefined) are rejected", () => { + expect(() => haiku(undefined, 10)).toThrow(); + }); + test("ECP-07 non-finite tokens (Infinity) are rejected", () => { + expect(() => haiku(Infinity, 10)).toThrow(); + }); +}); + +describe("White box: MC/DC for the 8-condition guard in getTokenCounts", () => { + // Each row makes exactly one atomic condition true (short-circuit OR), plus the all-false row. + const rows: [string, string, unknown, unknown, boolean][] = [ + ["WB-01", "all conditions false (valid)", 10, 10, false], + ["WB-02", "C1 inputTokens == null", undefined, 10, true], + ["WB-03", "C2 outputTokens == null", 10, undefined, true], + ["WB-04", "C3 !isFinite(inputTokens)", Infinity, 10, true], + ["WB-05", "C4 !isFinite(outputTokens)", 10, Infinity, true], + ["WB-06", "C5 !isInteger(inputTokens)", 1.5, 10, true], + ["WB-07", "C6 !isInteger(outputTokens)", 10, 1.5, true], + ["WB-08", "C7 inputTokens < 0", -1, 10, true], + ["WB-09", "C8 outputTokens < 0", 10, -1, true], + ]; + for (const [id, label, x, y, throws] of rows) { + test(`${id} ${label} -> ${throws ? "throws" : "returns credits"}`, () => { + if (throws) expect(() => haiku(x, y)).toThrow(); + else expect(haiku(x, y)).toBe(1); + }); + } +}); + +describe("White box: basis paths of getModelPricing (V(G) = 3)", () => { + const bill = (provider: string, model: string) => + calculateCreditsForUsage({ provider, model, usage: { inputTokens: 1, outputTokens: 1 } as any }); + + test("WB-10 path 1-2-5: model found, provider matches -> pricing returned", () => { + expect(bill("openai", "gpt-5.4-nano").credits).toBe(1); + }); + test("WB-11 path 1-2-3-4: provider known, model mismatch -> 'Unsupported billing model'", () => { + expect(() => bill("openai", "claude-opus-4-6")).toThrow("Unsupported billing model"); + }); + test("WB-12 path 1-2-3-6: provider unknown -> 'Unsupported billing provider'", () => { + expect(() => bill("mistral", "claude-opus-4-6")).toThrow("Unsupported billing provider"); + }); +}); diff --git a/testing/tests/local-tools.test.ts b/testing/tests/local-tools.test.ts new file mode 100644 index 0000000..b3dcde6 --- /dev/null +++ b/testing/tests/local-tools.test.ts @@ -0,0 +1,151 @@ +// Bottom-up integration: the real executeLocalTool (packages/cli/src/lib/local-tools.ts) +// against a real file system. This test file is the DRIVER. Everything happens inside a +// throw-away temp directory, so the CodeFlow project itself is never read or written. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { executeLocalTool } from "../../packages/cli/src/lib/local-tools"; + +const originalCwd = process.cwd(); +let root: string; +let project: string; + +beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "codeflow-test-")); + project = join(root, "project"); + mkdirSync(join(project, "src"), { recursive: true }); + writeFileSync(join(root, "outside.txt"), "secret"); + writeFileSync(join(project, "src", "a.txt"), "hello"); + writeFileSync(join(project, "..notes.md"), "notes inside the project"); + writeFileSync(join(project, "f9999.txt"), "x".repeat(9999)); + writeFileSync(join(project, "f10000.txt"), "x".repeat(10000)); + writeFileSync(join(project, "f10001.txt"), "x".repeat(10001)); + process.chdir(project); +}); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(root, { recursive: true, force: true }); +}); + +describe("BVA: readFile truncation at MAX_FILE_SIZE = 10,000 chars", () => { + test("BVA-18 9,999 chars -> returned whole, not truncated", async () => { + const r: any = await executeLocalTool("readFile", { path: "f9999.txt" }, "PLAN"); + expect(r.content).toHaveLength(9999); + expect(r.truncated).toBeUndefined(); + }); + test("BVA-19 10,000 chars -> returned whole, not truncated", async () => { + const r: any = await executeLocalTool("readFile", { path: "f10000.txt" }, "PLAN"); + expect(r.content).toHaveLength(10000); + expect(r.truncated).toBeUndefined(); + }); + test("BVA-20 10,001 chars -> truncated to 10,000, totalLength 10,001", async () => { + const r: any = await executeLocalTool("readFile", { path: "f10001.txt" }, "PLAN"); + expect(r).toMatchObject({ truncated: true, totalLength: 10001 }); + expect(r.content).toHaveLength(10000); + }); +}); + +describe("ECP + MC/DC: path sandbox", () => { + test("ECP-12 path inside project 'src/a.txt' (A=F, B=F) -> file is read", async () => { + expect(await executeLocalTool("readFile", { path: "src/a.txt" }, "PLAN")).toEqual({ content: "hello" }); + }); + test("ECP-13 parent escape '../outside.txt' (A=T) -> rejected", async () => { + await expect(executeLocalTool("readFile", { path: "../outside.txt" }, "PLAN")).rejects.toThrow("Path is outside the project directory"); + }); + test("ECP-14 path on another drive 'Z:/secret.txt' (A=F, B=T) -> rejected", async () => { + await expect(executeLocalTool("readFile", { path: "Z:/secret.txt" }, "PLAN")).rejects.toThrow("Path is outside the project directory"); + }); + test("ECP-15 file inside project whose name starts with '..' ('..notes.md') -> file is read", async () => { + expect(await executeLocalTool("readFile", { path: "..notes.md" }, "PLAN")).toEqual({ content: "notes inside the project" }); + }); +}); + +describe("Decision table: mode x tool class in executeLocalTool", () => { + test("DT-07 PLAN + read-only tool (listDirectory) -> allowed", async () => { + const r: any = await executeLocalTool("listDirectory", { path: "src" }, "PLAN"); + expect(r.entries).toEqual([{ name: "a.txt", type: "file" }]); + }); + test("DT-08 PLAN + write tool (writeFile) -> rejected, nothing written", async () => { + await expect(executeLocalTool("writeFile", { path: "plan.txt", content: "x" }, "PLAN")).rejects.toThrow("Tool writeFile is not available in PLAN mode"); + expect(() => readFileSync(join(project, "plan.txt"))).toThrow(); + }); + test("DT-09 PLAN + unknown tool -> rejected by the PLAN guard", async () => { + await expect(executeLocalTool("deleteFile", {}, "PLAN")).rejects.toThrow("Tool deleteFile is not available in PLAN mode"); + }); + test("DT-10 BUILD + read-only tool (readFile) -> allowed", async () => { + expect(await executeLocalTool("readFile", { path: "src/a.txt" }, "BUILD")).toEqual({ content: "hello" }); + }); + test("DT-11 BUILD + write tool (writeFile) -> file written", async () => { + const r: any = await executeLocalTool("writeFile", { path: "out/b.txt", content: "built" }, "BUILD"); + expect(r).toMatchObject({ success: true, bytesWritten: 5 }); + expect(readFileSync(join(project, "out", "b.txt"), "utf-8")).toBe("built"); + }); + test("DT-12 BUILD + unknown tool -> 'Unknown tool'", async () => { + await expect(executeLocalTool("deleteFile", {}, "BUILD")).rejects.toThrow("Unknown tool: deleteFile"); + }); +}); + +describe("White box: editFile paths", () => { + test("WB-13 occurrences = 0 -> 'oldString not found in file'", async () => { + writeFileSync(join(project, "e.txt"), "alpha beta"); + await expect(executeLocalTool("editFile", { path: "e.txt", oldString: "gamma", newString: "x" }, "BUILD")).rejects.toThrow("oldString not found in file"); + }); + test("WB-14 occurrences = 2 -> 'ambiguous; found 2 matches', file unchanged", async () => { + writeFileSync(join(project, "e.txt"), "beta beta"); + await expect(executeLocalTool("editFile", { path: "e.txt", oldString: "beta", newString: "x" }, "BUILD")).rejects.toThrow("oldString is ambiguous; found 2 matches"); + expect(readFileSync(join(project, "e.txt"), "utf-8")).toBe("beta beta"); + }); + test("WB-15 occurrences = 1 -> text replaced", async () => { + writeFileSync(join(project, "e.txt"), "alpha beta"); + await executeLocalTool("editFile", { path: "e.txt", oldString: "beta", newString: "gamma" }, "BUILD"); + expect(readFileSync(join(project, "e.txt"), "utf-8")).toBe("alpha gamma"); + }); + test("WB-16 newString containing '$$' (shell PID) is written literally", async () => { + writeFileSync(join(project, "run.sh"), "echo PID"); + await executeLocalTool("editFile", { path: "run.sh", oldString: "PID", newString: "$$" }, "BUILD"); + expect(readFileSync(join(project, "run.sh"), "utf-8")).toBe("echo $$"); + }); +}); + +// On Windows, grep is not on the PATH of a plain PowerShell window, and Bun does not see a PATH +// change made while it runs. Git for Windows ships a grep.exe, so when grep is missing and Git's +// copy exists, the tool's `Bun.spawn(["grep", ...])` call is pointed at that file for this test run. +const gitGrep = "C:\\Program Files\\Git\\usr\\bin\\grep.exe"; +const useGitGrep = Bun.which("grep") == null && process.platform === "win32" && existsSync(gitGrep); +if (useGitGrep) { + const realSpawn = Bun.spawn.bind(Bun) as (...args: any[]) => unknown; + (Bun as any).spawn = (cmd: unknown, ...rest: unknown[]) => + realSpawn(Array.isArray(cmd) && cmd[0] === "grep" ? [gitGrep, ...cmd.slice(1)] : cmd, ...rest); +} +const hasGrep = Bun.which("grep") != null || useGitGrep; + +describe.skipIf(!hasGrep)("BVA + ECP: grep cap (needs grep)", () => { + const grepIn = async (lines: number) => { + mkdirSync(join(project, "g"), { recursive: true }); + writeFileSync(join(project, "g", "g.txt"), Array.from({ length: lines }, (_, i) => `needle ${i}`).join("\n")); + return (await executeLocalTool("grep", { pattern: "needle", path: "g" }, "PLAN")) as any; + }; + test("BVA-21 49 matches -> 49 returned, not truncated", async () => { + const r = await grepIn(49); + expect(r.matches).toHaveLength(49); + expect(r.truncated).toBeUndefined(); + }); + test("BVA-22 50 matches -> 50 returned, not truncated", async () => { + const r = await grepIn(50); + expect(r.matches).toHaveLength(50); + expect(r.truncated).toBeUndefined(); + }); + test("BVA-23 51 matches -> 50 returned, truncated, totalMatches 51", async () => { + const r = await grepIn(51); + expect(r.matches).toHaveLength(50); + expect(r).toMatchObject({ truncated: true, totalMatches: 51 }); + }); + test("ECP-16 pattern that starts with '-' ('-v') is searched as text", async () => { + mkdirSync(join(project, "flags"), { recursive: true }); + writeFileSync(join(project, "flags", "usage.txt"), "pass -v for verbose\nplain line"); + const r = (await executeLocalTool("grep", { pattern: "-v", path: "flags" }, "PLAN")) as any; + expect(r.matches?.map((m: any) => m.content)).toEqual(["pass -v for verbose"]); + }); +}); diff --git a/testing/tests/server-integration.test.ts b/testing/tests/server-integration.test.ts new file mode 100644 index 0000000..16fd61b --- /dev/null +++ b/testing/tests/server-integration.test.ts @@ -0,0 +1,137 @@ +// Top-down integration: the real Hono server (packages/server/src/index.ts) with its real +// middleware chain (requireAuth -> requireCreditsBalance -> body validator -> route). +// External systems are replaced by STUBS, so no Clerk, Polar, Postgres or model API is called: +// Clerk auth -> lib/auth.authenticateOAuthRequest +// Polar -> lib/polar.getAvailableCreditsBalance / ingestAiUsage +// Postgres -> @codeflow/database/client db.session +// Model API -> ai.streamText +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { join } from "node:path"; + +// Resolve bare packages the same way the server does (they live in the server workspace). +const serverSrc = join(import.meta.dir, "../../packages/server/src"); +const aiPath = Bun.resolveSync("ai", serverSrc); +const dbClientPath = Bun.resolveSync("@codeflow/database/client", serverSrc); +const realAi = await import(aiPath); + +const stub = { + balance: 100 as number | Error, + session: { id: "s1", userId: "user_1", messages: [] } as object | null, +}; + +mock.module("../../packages/server/src/lib/auth", () => ({ + authenticateOAuthRequest: async (req: Request) => + req.headers.get("authorization") === "Bearer valid-token" ? { userId: "user_1" } : null, +})); + +mock.module("../../packages/server/src/lib/polar", () => ({ + getAvailableCreditsBalance: async () => { + if (stub.balance instanceof Error) throw stub.balance; + return stub.balance; + }, + ingestAiUsage: async () => {}, + createCheckoutUrl: async () => "https://example.invalid/checkout", + createCustomerPortalUrl: async () => "https://example.invalid/portal", +})); + +mock.module(dbClientPath, () => ({ + db: { + session: { + findUnique: async () => stub.session, + update: async () => stub.session, + findMany: async () => [], + create: async () => stub.session, + }, + }, +})); + +mock.module(aiPath, () => ({ + ...realAi, + streamText: () => ({ + toUIMessageStreamResponse: () => + new Response("data: {\"type\":\"finish\"}\n\n", { headers: { "content-type": "text/event-stream" } }), + }), +})); + +const { default: server } = await import("../../packages/server/src/index"); + +const validBody = { + id: "s1", + mode: "BUILD", + model: "claude-haiku-4-5", + messages: [{ id: "m1", role: "user", parts: [{ type: "text", text: "hello" }] }], +}; + +function postChat(body: unknown, token: string | null = "valid-token") { + const headers: Record = { "content-type": "application/json" }; + if (token) headers.authorization = `Bearer ${token}`; + return server.fetch(new Request("http://localhost/chat", { method: "POST", headers, body: JSON.stringify(body) })); +} + +beforeEach(() => { + stub.balance = 100; + stub.session = { id: "s1", userId: "user_1", messages: [] }; +}); + +describe("Decision table: POST /chat gate", () => { + test("DT-01 R1 no/invalid token (C1=F) -> 401 Unauthorized", async () => { + const res = await postChat(validBody, "forged-token"); + expect(res.status).toBe(401); + expect(await res.json()).toEqual({ error: "Unauthorized. Run /login to continue." }); + }); + test("DT-02 R2 token ok, billing service down (C2=F) -> 503", async () => { + stub.balance = new Error("Polar unreachable"); + const res = await postChat(validBody); + expect(res.status).toBe(503); + expect(await res.json()).toEqual({ error: "Unable to verify credits balance right now." }); + }); + test("DT-03 R3 balance not positive (C3=F) -> 402 Payment Required", async () => { + stub.balance = 0; + const res = await postChat(validBody); + expect(res.status).toBe(402); + expect(await res.json()).toEqual({ error: "No credits remaining. Run /upgrade to buy more credits." }); + }); + test("DT-04 R4 invalid body, mode 'DEBUG' (C4=F) -> 400", async () => { + const res = await postChat({ ...validBody, mode: "DEBUG" }); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: "Invalid request body" }); + }); + test("DT-05 R5 session missing or not owned (C5=F) -> 404", async () => { + stub.session = null; + const res = await postChat(validBody); + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: "Session not found" }); + }); + test("DT-06 R6 all causes true -> 200 event stream", async () => { + const res = await postChat(validBody); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toContain("text/event-stream"); + }); +}); + +describe("BVA: credit gate 'balance <= 0 -> 402'", () => { + test("BVA-24 balance = -1 -> 402", async () => { + stub.balance = -1; + expect((await postChat(validBody)).status).toBe(402); + }); + test("BVA-25 balance = 0 -> 402", async () => { + stub.balance = 0; + expect((await postChat(validBody)).status).toBe(402); + }); + test("BVA-26 balance = 1 -> allowed (200)", async () => { + stub.balance = 1; + expect((await postChat(validBody)).status).toBe(200); + }); +}); + +describe("ECP: request body classes", () => { + test("ECP-17 unsupported model 'gpt-4' -> 400", async () => { + expect((await postChat({ ...validBody, model: "gpt-4" })).status).toBe(400); + }); + test("ECP-18 empty messages array -> 400", async () => { + expect((await postChat({ ...validBody, messages: [] })).status).toBe(400); + }); + test("ECP-19 PLAN mode with a valid body -> 200", async () => { + expect((await postChat({ ...validBody, mode: "PLAN" })).status).toBe(200); + }); +}); diff --git a/testing/tests/shared-and-models.test.ts b/testing/tests/shared-and-models.test.ts new file mode 100644 index 0000000..c51e57c --- /dev/null +++ b/testing/tests/shared-and-models.test.ts @@ -0,0 +1,36 @@ +// Unit tests for packages/shared/src/schemas.ts and packages/server/src/lib/models.ts. +// resolveChatModel only builds a provider client object; no API key or network call is used. +import { describe, expect, test } from "bun:test"; +import { getToolContracts, modeSchema, Mode } from "../../packages/shared/src/schemas"; +import { isSupportedChatModel, resolveChatModel } from "../../packages/server/src/lib/models"; + +describe("ECP: mode values", () => { + test("ECP-08 'PLAN' is accepted", () => expect(modeSchema.safeParse("PLAN").success).toBe(true)); + test("ECP-09 'BUILD' is accepted", () => expect(modeSchema.safeParse("BUILD").success).toBe(true)); + test("ECP-10 'DEBUG' (unknown mode) is rejected", () => expect(modeSchema.safeParse("DEBUG").success).toBe(false)); + test("ECP-11 'plan' (wrong case) is rejected", () => expect(modeSchema.safeParse("plan").success).toBe(false)); +}); + +describe("Unit: tool contracts per mode", () => { + test("UT-01 PLAN mode exposes only the 4 read-only tools", () => { + expect(Object.keys(getToolContracts(Mode.PLAN)).sort()).toEqual(["glob", "grep", "listDirectory", "readFile"]); + }); + test("UT-02 BUILD mode exposes all 7 tools", () => { + expect(Object.keys(getToolContracts(Mode.BUILD)).sort()).toEqual( + ["bash", "editFile", "glob", "grep", "listDirectory", "readFile", "writeFile"], + ); + }); +}); + +describe("Unit: model registry", () => { + test("UT-03 isSupportedChatModel('gpt-5.4-mini') is true", () => expect(isSupportedChatModel("gpt-5.4-mini")).toBe(true)); + test("UT-04 isSupportedChatModel('gpt-4') is false", () => expect(isSupportedChatModel("gpt-4")).toBe(false)); + test("UT-05 resolveChatModel('claude-haiku-4-5') resolves to the anthropic provider", () => { + const resolved = resolveChatModel("claude-haiku-4-5"); + expect(resolved.provider).toBe("anthropic"); + expect(resolved.modelId).toBe("claude-haiku-4-5"); + }); + test("UT-06 resolveChatModel('unknown-model') throws", () => { + expect(() => resolveChatModel("unknown-model")).toThrow("Unsupported model: unknown-model"); + }); +});