Skip to content

Commit 4c905c6

Browse files
beondebeonde
authored
test: add comprehensive E2E test suite for CLI wrapper (#7)
* test: add comprehensive E2E test suite for CLI wrapper P0 E2E Tests Added: - test_validate_e2e.py: 8 tests for validation command - test_badge_e2e.py: 9 tests for badge issue/verify commands - test_score_e2e.py: 4 tests for scoring (via validation) - test_status_e2e.py: 8 tests for CLI status/version checks Infrastructure: - pytest.ini: E2E test configuration with markers - tests/e2e/conftest.py: Shared fixtures and CLI runner - tests/e2e/fixtures/: Test agent cards (valid, invalid, malformed) - .github/workflows/e2e.yml: E2E test workflow - .github/workflows/test.yml: Updated unit test workflow Reorganized: - tests/unit/: Moved existing unit tests - tests/README.md: Test documentation Total: 29 new E2E tests * fix(e2e): remove tests for non-existent CLI commands - Remove test_score_e2e.py (CLI has no 'score' command) - Remove test_status_e2e.py (CLI has no 'status' command) - Update conftest.py to remove server dependencies - Fix test_validate_e2e.py to use --schema-only for offline validation - Fix test_badge_e2e.py to use --self-sign and --accept-self-signed - Update valid-agent-card.json to match A2A v1.3.0 schema - Simplify e2e.yml workflow (no longer needs server/postgres) Tests now run entirely offline using CLI features: - badge issue --self-sign - badge verify --accept-self-signed --offline - validate --schema-only * fix(config): correct pytest.ini syntax (use INI format, not TOML) * fix(e2e): handle CLI download messages in badge tests * refactor(e2e): address copilot review feedback - Remove unused imports (os, json) from test files - Rename malformed.json to malformed.txt to avoid linter issues - Update tests/README.md to match actual test coverage * fix(tests): use platform-agnostic path comparison in unit test The test was failing on Windows CI because Path('/bin/capiscio') normalizes to backslashes on Windows. Compare using str(binary_path) to ensure consistent comparison across platforms.
1 parent 035a569 commit 4c905c6

14 files changed

Lines changed: 1404 additions & 2 deletions

‎.github/workflows/e2e.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: E2E Tests
2+
3+
on:
4+
push:
5+
branches: [main, develop]
6+
pull_request:
7+
branches: [main, develop]
8+
9+
jobs:
10+
e2e-tests:
11+
name: E2E Tests (Python ${{ matrix.python-version }})
12+
runs-on: ubuntu-latest
13+
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
python-version: ["3.10", "3.11", "3.12"]
18+
19+
steps:
20+
- uses: actions/checkout@v4
21+
22+
- name: Set up Python ${{ matrix.python-version }}
23+
uses: actions/setup-python@v5
24+
with:
25+
python-version: ${{ matrix.python-version }}
26+
27+
- name: Install dependencies
28+
run: |
29+
python -m pip install --upgrade pip
30+
pip install -e ".[dev,test]"
31+
32+
- name: Run unit tests
33+
run: |
34+
pytest tests/unit/ -v --cov=capiscio --cov-report=xml --cov-report=term
35+
36+
- name: Run E2E tests
37+
run: |
38+
pytest tests/e2e/ -v --tb=short
39+
40+
- name: Upload coverage reports
41+
if: matrix.python-version == '3.11'
42+
uses: codecov/codecov-action@v4
43+
with:
44+
file: ./coverage.xml
45+
flags: unittests
46+
name: codecov-umbrella
47+
fail_ci_if_error: false

‎.github/workflows/test.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Unit Tests
2+
3+
on:
4+
push:
5+
branches: [main, develop]
6+
pull_request:
7+
branches: [main, develop]
8+
9+
jobs:
10+
test:
11+
name: Unit Tests (Python ${{ matrix.python-version }})
12+
runs-on: ${{ matrix.os }}
13+
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
os: [ubuntu-latest, macos-latest, windows-latest]
18+
python-version: ["3.10", "3.11", "3.12"]
19+
20+
steps:
21+
- uses: actions/checkout@v4
22+
23+
- name: Set up Python ${{ matrix.python-version }}
24+
uses: actions/setup-python@v5
25+
with:
26+
python-version: ${{ matrix.python-version }}
27+
28+
- name: Install dependencies
29+
run: |
30+
python -m pip install --upgrade pip
31+
pip install -e ".[dev,test]"
32+
33+
- name: Run unit tests
34+
run: |
35+
pytest tests/unit/ -v --cov=capiscio --cov-report=xml --cov-report=term
36+
37+
- name: Upload coverage
38+
if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest'
39+
uses: codecov/codecov-action@v4
40+
with:
41+
file: ./coverage.xml
42+
flags: unittests
43+
name: codecov-umbrella

‎pyproject.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ dev = [
3636
"build",
3737
"twine",
3838
]
39+
test = [
40+
"pytest>=7.0.0",
41+
"pytest-cov>=4.0.0",
42+
"requests>=2.31.0",
43+
]
3944

4045
[project.scripts]
4146
capiscio = "capiscio.cli:main"

‎pytest.ini‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[pytest]
2+
testpaths = tests
3+
python_files = test_*.py
4+
python_classes = Test*
5+
python_functions = test_*
6+
addopts = -v --strict-markers
7+
markers =
8+
e2e: marks tests as end-to-end tests requiring a live server
9+
unit: marks tests as unit tests (no external dependencies)
10+
11+
filterwarnings =
12+
ignore::DeprecationWarning

‎tests/README.md‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Tests for capiscio-python CLI
2+
3+
This directory contains unit and E2E tests for the `capiscio` CLI wrapper.
4+
5+
## Directory Structure
6+
7+
```
8+
tests/
9+
├── unit/ # Unit tests with mocks (no server required)
10+
│ ├── test_cli.py
11+
│ └── test_manager.py
12+
└── e2e/ # E2E tests (offline mode, no server required)
13+
├── conftest.py # Pytest fixtures and configuration
14+
├── fixtures/ # Test data files
15+
│ ├── valid-agent-card.json
16+
│ ├── invalid-agent-card.json
17+
│ └── malformed.txt
18+
├── test_validate_e2e.py # Validation command tests
19+
└── test_badge_e2e.py # Badge issuance/verification tests
20+
```
21+
22+
## Running Tests
23+
24+
### Run All Tests
25+
26+
```bash
27+
pytest # All tests
28+
pytest tests/unit/ # Unit tests only
29+
pytest tests/e2e/ # E2E tests only
30+
```
31+
32+
### Run Specific Test File
33+
34+
```bash
35+
pytest tests/e2scio --cov-report=html
36+
```
37+
38+
## E2E Test Design
39+
40+
The E2E tests are designed to run **offline** without requiring a server:
41+
42+
- **Validate tests**: Use `--schema-only` flag for local schema validation
43+
- **Badge tests**: Use `--self-sign` for issuance and `--accept-self-signed --offline` for verification
44+
45+
This approach allows E2E tests to run in CI without complex server infrastructure.
46+
47+
## Test Coverage
48+
49+
### Validate Command (`test_validate_e2e.py`)
50+
51+
- ✅ Valid local agent card file (schema-only mode)
52+
- ✅ Invalid local agent card file
53+
- ✅ Malformed JSON file
54+
- ✅ Nonexistent file
55+
- ✅ JSON output format
56+
- ✅ Help command
57+
58+
### ├── test_validate_e2e.sue self-signed badge
59+
- ✅ Issue badge with custom expiration
60+
- ✅ Issue badge with audience restriction
61+
- ✅ Verify self-signed badge (offline)
62+
- ✅ Verify invalid token (error handling)
63+
- ✅ Help commands (badge, issue, verify)
64+
65+
## CI/CD Integration
66+
67+
The E2E tests run in GitHub Actions without server dependencies:
68+
69+
```yaml
70+
# See .github/workflows/e2e.yml
71+
- name: Run E2E tests
72+
run: pytest tests/e2e/
73+
```
74+
75+
## Notes
76+
77+
- **Offline Mode**: All E2E tests run offline without server dependencies
78+
- **Download Messages**: On first run, the CLI may download the capiscio-core binary; tests handle this gracefully
79+
80+
## Troubleshooting
81+
82+
### Build/Install Issues
83+
84+
Ensure the project is installed:
85+
86+
```bash
87+
pip install -e .
88+
pytest tests/e2e/
89+
```
90+
91+
### Path Issues
92+
93+
Ensure you're running pytest from the project root:
94+
95+
```bash
96+
cd /path/to/capiscio-python
97+
pytest tests/e2e/
98+
```

‎tests/e2e/conftest.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
"""
2+
pytest configuration for capiscio-python CLI E2E tests.
3+
4+
Provides fixtures for testing the CLI offline using:
5+
- validate --schema-only
6+
- badge issue --self-sign
7+
- badge verify --accept-self-signed --offline
8+
"""
9+
10+
import pytest
11+
from pathlib import Path
12+
13+
14+
@pytest.fixture(scope="session")
15+
def fixtures_dir() -> Path:
16+
"""Get path to fixtures directory."""
17+
return Path(__file__).parent / "fixtures"
18+
19+
20+
@pytest.fixture
21+
def valid_agent_card_path(fixtures_dir: Path) -> Path:
22+
"""Path to valid agent card fixture."""
23+
return fixtures_dir / "valid-agent-card.json"
24+
25+
26+
@pytest.fixture
27+
def invalid_agent_card_path(fixtures_dir: Path) -> Path:
28+
"""Path to invalid agent card fixture."""
29+
return fixtures_dir / "invalid-agent-card.json"
30+
31+
32+
@pytest.fixture
33+
def malformed_json_path(fixtures_dir: Path) -> Path:
34+
"""Path to malformed JSON fixture."""
35+
return fixtures_dir / "malformed.txt"
36+
37+
38+
@pytest.fixture
39+
def nonexistent_path(fixtures_dir: Path) -> Path:
40+
"""Path to a file that doesn't exist."""
41+
return fixtures_dir / "does-not-exist.json"

‎tests/e2e/fixtures/invalid-agent-card.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"version": "1.0",
3+
"did": "invalid-did-format",
4+
"name": "Invalid Agent",
5+
"publicKey": {
6+
"kty": "WRONG",
7+
"x": "invalid-key-data"
8+
}
9+
}

‎tests/e2e/fixtures/malformed.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"version": "1.0",
3+
"did": "did:web:example.com"
4+
"missing": "comma above"

‎tests/e2e/fixtures/valid-agent-card.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"protocolVersion": "1.3.0",
3+
"version": "1.0.0",
4+
"name": "Test Agent",
5+
"description": "A test agent for E2E validation tests",
6+
"url": "https://example.com/.well-known/agent.json",
7+
"capabilities": {
8+
"streaming": false,
9+
"pushNotifications": false
10+
},
11+
"skills": [
12+
{
13+
"id": "test-skill",
14+
"name": "Test Skill",
15+
"description": "A skill for testing",
16+
"tags": ["test", "validation"]
17+
}
18+
],
19+
"provider": {
20+
"organization": "Test Organization",
21+
"url": "https://example.com"
22+
},
23+
"authentication": {
24+
"schemes": ["none"]
25+
}
26+
}

0 commit comments

Comments
 (0)