Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
name: tests

on:
push:
branches: [main]
pull_request:

permissions:
contents: read

jobs:
test:
# Windows: the same OS the tests were written and verified on (the path-sandbox tests use Windows paths).
runs-on: windows-latest
defaults:
run:
shell: bash
env:
TESTCOLLAB_TOKEN: ${{ secrets.TESTCOLLAB_TOKEN }}
steps:
- uses: actions/checkout@v4

- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.14

- name: Install dependencies
run: bun install --frozen-lockfile

- name: Unit and integration tests (bun test)
working-directory: testing
run: |
mkdir -p ci-results
bun test --coverage --reporter=junit --reporter-outfile=ci-results/junit-ci.xml

- name: API tests (Postman collections run by Newman)
if: ${{ !cancelled() }}
run: bun testing/automation/api-tests.ts

- name: Upload test results
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: test-results
path: |
testing/ci-results/junit-ci.xml
testing/automation/results/*.xml

# Runs only when the repository has a TESTCOLLAB_TOKEN secret (never on forks).
# Optional repository variable TESTCOLLAB_PROJECT overrides the project id.
- name: Report results to TestCollab
if: ${{ !cancelled() && env.TESTCOLLAB_TOKEN != '' }}
working-directory: testing
run: npx --yes @testcollab/cli report --project "${{ vars.TESTCOLLAB_PROJECT || '19282' }}" --format junit --result-file ci-results/junit-ci.xml --auto-create
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,8 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

# /brag launch video output (generated locally; not committed)
brag-output*/

# test run outputs (generated)
testing/ci-results/
testing/automation/results/
testing/tmp/
1 change: 1 addition & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
},
"devDependencies": {
"@codeflow/server": "workspace:*",
"@codeflow/shared": "workspace:*",
"@types/react": "^19.2.17"
},
"peerDependencies": {
Expand Down
8 changes: 4 additions & 4 deletions packages/cli/src/lib/local-tools.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { mkdir, readFile, readdir, stat, writeFile } from "fs/promises";
import { dirname, isAbsolute, join, relative, resolve } from "path";
import { dirname, isAbsolute, join, relative, resolve, sep } from "path";
import { toolInputSchemas, Mode, type ModeType } from "@codeflow/shared";

const MAX_FILE_SIZE = 10_000;
Expand All @@ -13,7 +13,7 @@ function resolveInsideCwd(path: string) {
const resolved = resolve(cwd, path);
const rel = relative(cwd, resolved);

if (rel.startsWith("..") || isAbsolute(rel)) {
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
throw new Error("Path is outside the project directory");
}

Expand Down Expand Up @@ -87,7 +87,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M
"-E",
];
if (include) args.push(`--include=${include}`);
args.push(pattern, resolved);
args.push("-e", pattern, resolved);

const proc = Bun.spawn(["grep", ...args], { cwd, stdout: "pipe", stderr: "pipe" });
const [stdout, stderr] = await Promise.all([
Expand Down Expand Up @@ -140,7 +140,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M
if (occurrences === 0) throw new Error("oldString not found in file");
if (occurrences > 1) throw new Error(`oldString is ambiguous; found ${occurrences} matches`);

await writeFile(resolved, content.replace(oldString, newString), "utf-8");
await writeFile(resolved, content.replace(oldString, () => newString), "utf-8");
return { success: true as const, path: relative(cwd, resolved) };
}
case "bash": {
Expand Down
2 changes: 1 addition & 1 deletion packages/server/src/lib/credits.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ function convertUsdToCredits(estimatedCostUsd: number) {

// If a request costs any non-zero amount, charge at least 1 credit, then
// round up so partial credits always become a whole credit.
return Math.max(1, Math.ceil(estimatedCostUsd / USD_PER_CREDIT));
return Math.max(1, Math.ceil(estimatedCostUsd / USD_PER_CREDIT - 1e-9));
};


Expand Down
89 changes: 89 additions & 0 deletions testing/automation/api-tests.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// Postman / Newman harness for the CodeFlow API.
// Starts the REAL Hono server (packages/server/src/index.ts) on a local port with Clerk, Polar, PostgreSQL
// and the model API replaced by stubs, so no outside service is called. The bearer token picks the stub user:
// user-ok (100 credits) user-zero (0) user-neg (-1) user-one (1) user-down (credits look-up fails)
// anything else = invalid token. Session "s1" exists; every other id is "not found".
//
// bun testing/automation/api-tests.ts start the stub server, run both collections with Newman, stop
// bun testing/automation/api-tests.ts --serve only keep the stub server running (use it from the Postman app)
import { mock } from "bun:test";
import { mkdirSync } from "node:fs";
import { join } from "node:path";

const here = import.meta.dir;
const serverSrc = join(here, "../../packages/server/src");
const port = Number(process.env.API_TEST_PORT ?? 3101);

const USERS: Record<string, { id: string; balance: number | "down" }> = {
"user-ok": { id: "user_ok", balance: 100 },
"user-zero": { id: "user_zero", balance: 0 },
"user-neg": { id: "user_neg", balance: -1 },
"user-one": { id: "user_one", balance: 1 },
"user-down": { id: "user_down", balance: "down" },
};
const session = (userId: string) => ({ id: "s1", userId, title: "demo session", messages: [], createdAt: new Date("2026-01-01") });

const aiPath = Bun.resolveSync("ai", serverSrc);
const dbClientPath = Bun.resolveSync("@codeflow/database/client", serverSrc);
const realAi = await import(aiPath);

mock.module(join(serverSrc, "lib/auth"), () => ({
authenticateOAuthRequest: async (req: Request) => {
const user = USERS[(req.headers.get("authorization") ?? "").replace(/^Bearer /, "")];
return user ? { userId: user.id } : null;
},
}));
mock.module(join(serverSrc, "lib/polar"), () => ({
getAvailableCreditsBalance: async (userId: string) => {
const user = Object.values(USERS).find((u) => u.id === userId);
if (!user || user.balance === "down") throw new Error("Polar unreachable");
return user.balance;
},
ingestAiUsage: async () => {},
createCheckoutUrl: async () => "https://example.invalid/checkout",
createCustomerPortalUrl: async () => "https://example.invalid/portal",
}));
mock.module(dbClientPath, () => ({
db: {
session: {
findUnique: async ({ where }: any) => (where.id === "s1" ? session(where.userId) : null),
findMany: async ({ where }: any) => [session(where.userId)],
create: async ({ data }: any) => ({ id: "new1", messages: [], createdAt: new Date(), updatedAt: new Date(), ...data }),
update: async ({ where }: any) => session(where.userId),
},
},
}));
mock.module(aiPath, () => ({
...realAi,
streamText: () => ({
toUIMessageStreamResponse: () =>
new Response('data: {"type":"finish"}\n\n', { headers: { "content-type": "text/event-stream" } }),
}),
}));

const { default: app } = await import(join(serverSrc, "index"));
const server = Bun.serve({ port, fetch: app.fetch, idleTimeout: app.idleTimeout });
console.log(`CodeFlow API (stubbed) listening on http://localhost:${port}`);

if (process.argv.includes("--serve")) {
console.log("Serving for the Postman app. Press Ctrl+C to stop.");
} else {
const results = join(here, "results");
mkdirSync(results, { recursive: true });
const runs: [string, string[], string][] = [
["codeflow-api.postman_collection.json", [], "newman-api.xml"],
["codeflow-chat-gate-data-driven.postman_collection.json", ["-d", join(here, "data/chat-gate.csv")], "newman-chat-gate.xml"],
];
let failed = 0;
for (const [collection, extra, xml] of runs) {
const proc = Bun.spawn(
["bunx", "newman@6", "run", join(here, collection), "-e", join(here, "local.postman_environment.json"),
"--env-var", `baseUrl=http://localhost:${port}`, ...extra, "-r", "cli,junit", "--reporter-junit-export", join(results, xml)],
{ stdio: ["inherit", "inherit", "inherit"] },
);
if ((await proc.exited) !== 0) failed++;
}
server.stop(true);
console.log(failed ? `\n${failed} Newman run(s) FAILED` : "\nAll Newman runs passed");
process.exit(failed ? 1 : 0);
}
Loading
Loading