Skip to content

Commit ca9952a

Browse files
authored
test: add comprehensive E2E test suite for CLI wrapper (#30)
* test: add comprehensive E2E test suite for CLI wrapper P0 E2E Tests Added: - validate.e2e.test.ts: 8 tests for validation command - badge.e2e.test.ts: 9 tests for badge issue/verify commands - score.e2e.test.ts: 4 tests for scoring (via validation) - status.e2e.test.ts: 8 tests for CLI status/version checks Infrastructure: - vitest.config.e2e.ts: E2E test configuration - tests/e2e/setup.ts: Shared fixtures and CLI runner - tests/e2e/fixtures/: Test agent cards (valid, invalid, malformed) - .github/workflows/e2e.yml: E2E test workflow - package.json: Added test:e2e script Reorganized: - tests/unit/: Moved existing unit tests - tests/README.md: Test documentation - vitest.config.ts: Updated for unit tests only Total: 29 new E2E tests * fix(ci): add GHCR credentials for capiscio-server image pull Add credentials block to service container using REPO_ACCESS_TOKEN secret to authenticate with ghcr.io for pulling the private capiscio-server image. Also updated: - Node versions: 16 -> 22 - Package manager: npm -> pnpm * fix(ci): build capiscio-server from source instead of GHCR pull Switch from pulling ghcr.io/capiscio/capiscio-server:latest to building from source. This allows E2E tests to run without requiring a release, enabling downstream testing during development. Pattern matches capiscio-e2e-tests approach: - Checkout capiscio-server with REPO_ACCESS_TOKEN - Build via Dockerfile or Go directly - Start server with test configuration - Run E2E tests against localhost:8080 * fix(e2e): remove tests for non-existent CLI commands - Remove score.e2e.test.ts (CLI has no 'score' command) - Remove status.e2e.test.ts (CLI has no 'status' command) - Remove setup.ts (tests no longer need server) - Fix validate tests to use --schema-only for offline validation - Fix badge tests to use actual CLI flags (--self-sign, --accept-self-signed) - Update valid-agent-card.json to match A2A v1.3.0 schema - Simplify e2e.yml workflow (no longer needs server/postgres) Tests now run entirely offline using CLI features: - badge issue --self-sign - badge verify --accept-self-signed --offline - validate --schema-only * fix(e2e): remove setupFiles reference in vitest.config.e2e.ts * fix(e2e): handle CLI download messages in badge tests * refactor(e2e): address copilot review feedback - Extract runCapiscio/extractToken to shared helpers.ts - Rename malformed.json to malformed.txt to avoid linter issues - Update imports to use shared helpers - Remove unused artifact upload from e2e.yml - Update tests/README.md to match actual test coverage
1 parent fa39ac4 commit ca9952a

12 files changed

Lines changed: 491 additions & 28 deletions

.github/workflows/e2e.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: E2E Tests
2+
3+
on:
4+
push:
5+
branches: [main, develop]
6+
pull_request:
7+
branches: [main, develop]
8+
9+
jobs:
10+
e2e-tests:
11+
name: E2E Tests (Node ${{ matrix.node-version }})
12+
runs-on: ubuntu-latest
13+
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
node-version: ['18', '20', '22']
18+
19+
steps:
20+
- name: Checkout
21+
uses: actions/checkout@v4
22+
23+
- name: Setup Node.js ${{ matrix.node-version }}
24+
uses: actions/setup-node@v4
25+
with:
26+
node-version: ${{ matrix.node-version }}
27+
28+
- name: Install pnpm
29+
uses: pnpm/action-setup@v4
30+
with:
31+
version: 9
32+
33+
- name: Install dependencies
34+
run: pnpm install
35+
36+
- name: Build project
37+
run: pnpm build
38+
39+
- name: Run E2E tests
40+
run: pnpm test:e2e

package.json

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,17 @@
3838
"scripts": {
3939
"build": "tsup",
4040
"dev": "tsup --watch",
41-
"test": "vitest run",
42-
"test:watch": "vitest --watch",
43-
"test:coverage": "vitest run --coverage",
41+
"test": "vitest run --config vitest.config.ts --dir tests/unit",
42+
"test:unit": "vitest run --config vitest.config.ts --dir tests/unit",
43+
"test:e2e": "vitest run --config vitest.config.e2e.ts",
44+
"test:watch": "vitest --watch --config vitest.config.ts --dir tests/unit",
45+
"test:coverage": "vitest run --coverage --config vitest.config.ts --dir tests/unit",
46+
"test:all": "npm run test:unit && npm run test:e2e",
4447
"lint": "eslint src --ext .ts",
4548
"lint:fix": "eslint src --ext .ts --fix",
4649
"typecheck": "tsc --noEmit",
4750
"clean": "rm -rf dist",
48-
"prepublishOnly": "npm run clean && npm run build && npm test",
51+
"prepublishOnly": "npm run clean && npm run build && npm run test:unit",
4952
"start": "node bin/capiscio.js"
5053
},
5154
"dependencies": {

tests/README.md

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Tests for capiscio-node CLI
2+
3+
This directory contains unit and E2E tests for the `capiscio` CLI wrapper.
4+
5+
## Directory Structure
6+
7+
```
8+
tests/
9+
├── unit/ # Unit tests with mocks (no server required)
10+
│ └── cli.test.ts
11+
└── e2e/ # E2E tests (offline mode, no server required)
12+
├── fixtures/ # Test data files
13+
│ ├── valid-agent-card.json
14+
│ ├── invalid-agent-card.json
15+
│ └── malformed.json
16+
├── validate.e2e.test.ts # Validation command tests
17+
└── badge.e2e.test.ts # Badge issuance/verification tests
18+
```
19+
20+
## Running Tests
21+
22+
### Run All Tests
23+
24+
```bash
25+
pnpm test # Unit tests only (default)
26+
pnpm test:all # Both unit and E2E tests
27+
```
28+
29+
### Run Only Unit Tests
30+
31+
```bash
32+
pnpm test:unit
33+
```
34+
35+
### Run Only E2E Tests
36+
37+
```bash
38+
pnpm test:e2e
39+
```
40+
41+
### Run with Watch Mode
42+
43+
```bash
44+
pnpm test:watch
45+
```
46+
47+
### Run with Coverage
48+
49+
```bash
50+
pnpm test:coverage
51+
```
52+
53+
## E2E Test Design
54+
55+
The E2E tests are designed to run **offline** without requiring a server:
56+
57+
- **Validate tests**: Use `--schema-only` flag for local schema validation
58+
- **Badge tests**: Use `--self-sign` for issuance and `--accept-self-signed --offline` for verification
59+
60+
This approach allows E2E tests to run in CI without complex server infrastructure.
61+
62+
## Test Coverage
63+
64+
### Validate Command (`validate.e2e.test.ts`)
65+
66+
- ✅ Valid local agent card file (schema-only mode)
67+
- ✅ Invalid local agent card file
68+
- ✅ Malformed JSON file
69+
- ✅ Nonexistent file
70+
- ✅ JSON output format
71+
- ✅ Help command
72+
73+
### Badge Commands (`badge.e2e.test.ts`)
74+
75+
- ✅ Issue self-signed badge
76+
- ✅ Issue badge with custom expiration
77+
- ✅ Issue badge with audience restriction
78+
- ✅ Verify self-signed badge (offline)
79+
- ✅ Verify invalid token (error handling)
80+
- ✅ Help commands (badge, issue, verify)
81+
82+
## CI/CD Integration
83+
84+
The E2E tests run in GitHub Actions without server dependencies:
85+
86+
```yaml
87+
# See .github/workflows/e2e.yml
88+
- name: Run E2E tests
89+
run: pnpm test:e2e
90+
```
91+
92+
## Notes
93+
94+
- **Offline Mode**: All E2E tests run offline without server dependencies
95+
- **Timeouts**: Tests have 15-second timeouts to prevent hanging
96+
- **Download Messages**: On first run, the CLI may download the capiscio-core binary; tests handle this gracefully
97+
98+
## Troubleshooting
99+
100+
### TypeScript Build Errors
101+
102+
Ensure the project is built before running E2E tests:
103+
104+
```bash
105+
pnpm build
106+
pnpm test:e2e
107+
```
108+
109+
### Path Issues
110+
111+
Ensure you're running tests from the project root:
112+
113+
```bash
114+
cd /path/to/capiscio-node
115+
pnpm test:e2e
116+
```

tests/e2e/badge.e2e.test.ts

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/**
2+
* E2E tests for capiscio badge commands.
3+
*
4+
* Tests badge issuance and verification commands against the CLI.
5+
* These tests focus on the CLI interface itself, not the server.
6+
*/
7+
8+
import { describe, it, expect } from 'vitest';
9+
import { runCapiscio, extractToken } from './helpers';
10+
11+
describe('badge commands', () => {
12+
describe('badge issue', () => {
13+
it('should issue a self-signed badge', async () => {
14+
const result = await runCapiscio([
15+
'badge', 'issue', '--self-sign', '--domain', 'test.example.com'
16+
]);
17+
18+
// Self-signed badge issuance should succeed
19+
expect(result.exitCode).toBe(0);
20+
21+
// Output should contain a JWT token (has dots for header.payload.signature)
22+
// Get last line in case there are download messages
23+
const token = extractToken(result.stdout);
24+
expect(token.split('.').length).toBe(3); // JWT format
25+
}, 15000);
26+
27+
it('should issue badge with custom expiration', async () => {
28+
const result = await runCapiscio([
29+
'badge', 'issue', '--self-sign', '--exp', '10m'
30+
]);
31+
32+
expect(result.exitCode).toBe(0);
33+
const token = extractToken(result.stdout);
34+
expect(token.split('.').length).toBe(3);
35+
}, 15000);
36+
37+
it('should issue badge with audience restriction', async () => {
38+
const result = await runCapiscio([
39+
'badge', 'issue', '--self-sign', '--aud', 'https://api.example.com'
40+
]);
41+
42+
expect(result.exitCode).toBe(0);
43+
const token = extractToken(result.stdout);
44+
expect(token.split('.').length).toBe(3);
45+
}, 15000);
46+
47+
it('should display help for badge issue', async () => {
48+
const result = await runCapiscio(['badge', 'issue', '--help']);
49+
50+
expect(result.exitCode).toBe(0);
51+
const helpText = result.stdout.toLowerCase();
52+
expect(helpText.includes('issue')).toBe(true);
53+
expect(helpText.includes('self-sign') || helpText.includes('level')).toBe(true);
54+
}, 15000);
55+
});
56+
57+
describe('badge verify', () => {
58+
it('should verify a self-signed badge', async () => {
59+
// First issue a badge
60+
const issueResult = await runCapiscio([
61+
'badge', 'issue', '--self-sign', '--domain', 'test.example.com'
62+
]);
63+
expect(issueResult.exitCode).toBe(0);
64+
const token = extractToken(issueResult.stdout);
65+
66+
// Then verify it with --accept-self-signed
67+
const verifyResult = await runCapiscio([
68+
'badge', 'verify', token, '--accept-self-signed', '--offline'
69+
]);
70+
71+
expect(verifyResult.exitCode).toBe(0);
72+
const output = verifyResult.stdout.toLowerCase();
73+
expect(
74+
output.includes('valid') || output.includes('verified') || output.includes('ok')
75+
).toBe(true);
76+
}, 15000);
77+
78+
it('should fail for invalid token', async () => {
79+
const invalidToken = 'invalid.jwt.token';
80+
const result = await runCapiscio(['badge', 'verify', invalidToken, '--accept-self-signed']);
81+
82+
expect(result.exitCode).not.toBe(0);
83+
const errorOutput = (result.stderr + result.stdout).toLowerCase();
84+
expect(
85+
errorOutput.includes('invalid') ||
86+
errorOutput.includes('verify') ||
87+
errorOutput.includes('failed') ||
88+
errorOutput.includes('malformed') ||
89+
errorOutput.includes('error')
90+
).toBe(true);
91+
}, 15000);
92+
93+
it('should display help for badge verify', async () => {
94+
const result = await runCapiscio(['badge', 'verify', '--help']);
95+
96+
expect(result.exitCode).toBe(0);
97+
const helpText = result.stdout.toLowerCase();
98+
expect(helpText.includes('verify') || helpText.includes('token')).toBe(true);
99+
}, 15000);
100+
});
101+
102+
describe('badge help', () => {
103+
it('should display help for badge command', async () => {
104+
const result = await runCapiscio(['badge', '--help']);
105+
106+
expect(result.exitCode).toBe(0);
107+
const helpText = result.stdout.toLowerCase();
108+
expect(helpText.includes('badge')).toBe(true);
109+
expect(
110+
helpText.includes('issue') || helpText.includes('verify') || helpText.includes('usage')
111+
).toBe(true);
112+
}, 15000);
113+
});
114+
});
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"version": "1.0",
3+
"did": "invalid-did-format",
4+
"name": "Invalid Agent",
5+
"publicKey": {
6+
"kty": "WRONG",
7+
"x": "invalid-key-data"
8+
}
9+
}

tests/e2e/fixtures/malformed.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"version": "1.0",
3+
"did": "did:web:example.com",
4+
"trailing": "comma",
5+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"protocolVersion": "1.3.0",
3+
"version": "1.0.0",
4+
"name": "Test Agent",
5+
"description": "A test agent for E2E validation tests",
6+
"url": "https://example.com/.well-known/agent.json",
7+
"capabilities": {
8+
"streaming": false,
9+
"pushNotifications": false
10+
},
11+
"skills": [
12+
{
13+
"id": "test-skill",
14+
"name": "Test Skill",
15+
"description": "A skill for testing",
16+
"tags": ["test", "validation"]
17+
}
18+
],
19+
"provider": {
20+
"organization": "Test Organization",
21+
"url": "https://example.com"
22+
},
23+
"authentication": {
24+
"schemes": ["none"]
25+
}
26+
}

tests/e2e/helpers.ts

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/**
2+
* Shared test helpers for E2E tests.
3+
*/
4+
5+
import { exec } from 'child_process';
6+
import { promisify } from 'util';
7+
import path from 'path';
8+
9+
const execAsync = promisify(exec);
10+
11+
export const CLI_PATH = path.join(__dirname, '../../bin/capiscio.js');
12+
export const FIXTURES_DIR = path.join(__dirname, 'fixtures');
13+
14+
/**
15+
* Run the capiscio CLI with the given arguments.
16+
*/
17+
export async function runCapiscio(
18+
args: string[],
19+
env?: Record<string, string>
20+
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
21+
try {
22+
const { stdout, stderr } = await execAsync(`node "${CLI_PATH}" ${args.join(' ')}`, {
23+
env: { ...process.env, ...env },
24+
});
25+
return { stdout, stderr, exitCode: 0 };
26+
} catch (error: unknown) {
27+
const err = error as { stdout?: string; stderr?: string; code?: number };
28+
return {
29+
stdout: err.stdout || '',
30+
stderr: err.stderr || '',
31+
exitCode: err.code || 1,
32+
};
33+
}
34+
}
35+
36+
/**
37+
* Extract token from CLI output - handles potential download messages on first run.
38+
* The CLI may print download progress before the actual output.
39+
*/
40+
export function extractToken(stdout: string): string {
41+
const lines = stdout.trim().split('\n');
42+
return lines[lines.length - 1].trim();
43+
}

0 commit comments

Comments
 (0)