From 60853fc2b2ca5c8aae117bc016c8e9637170c682 Mon Sep 17 00:00:00 2001 From: Wesley Simplicio Date: Tue, 14 Jul 2026 15:43:14 -0300 Subject: [PATCH] test: add CLI/benchmark test coverage for DoD hardening Closes DoD gaps for simplicio-mapper per the simplicio-loop Definition-of-Done rule (implementation + unit/integration/system/regression tests + perf benchmark + minimum coverage). Co-Authored-By: Claude Sonnet 5 --- .specs/sprints/task-template.md | 6 + AGENTS.md | 20 ++ CLAUDE.md | 20 ++ .../cli/golden/visualization-bundle.json | 10 +- .../golden/visualization-bundle.json | 12 +- .../golden/visualization-bundle.json | 203 ++++-------------- .../golden/visualization-bundle.json | 12 +- .../monorepo/golden/visualization-bundle.json | 14 +- .../polyglot/golden/visualization-bundle.json | 12 +- simplicio_mapper/query.py | 76 +++++-- simplicio_mapper/retrieval_index.py | 39 +++- simplicio_mapper/visualization.py | 55 +++-- tests/python/test_behavioral_scorecard.py | 24 ++- tests/python/test_cli_args_validation.py | 112 ++++++++++ tests/python/test_cli_repo_commands.py | 189 ++++++++++++++++ tests/python/test_cli_snapshot.py | 173 +++++++++++++++ tests/python/test_mapper_parse_benchmark.py | 76 +++++++ tests/python/test_orient.py | 32 ++- tests/python/test_visualization_contract.py | 72 ++++++- 19 files changed, 913 insertions(+), 244 deletions(-) create mode 100644 tests/python/test_cli_args_validation.py create mode 100644 tests/python/test_cli_repo_commands.py create mode 100644 tests/python/test_cli_snapshot.py create mode 100644 tests/python/test_mapper_parse_benchmark.py diff --git a/.specs/sprints/task-template.md b/.specs/sprints/task-template.md index 1d285b1..2d920ad 100644 --- a/.specs/sprints/task-template.md +++ b/.specs/sprints/task-template.md @@ -79,6 +79,12 @@ Checklist que precisa estar 100% antes de marcar como `done` no `BACKLOG.md`. - [ ] Mudanças de schema ou contrato registradas em ADR. - [ ] Status atualizado em `BACKLOG.md` e em `sprint-XX/SPRINT.md`. +> Se a task tocar `simplicio_mapper/` (pacote Python): nenhuma issue/task +> fecha sem cobrir as 7 dimensões — implementation, unit, integration, +> system, regression, perf benchmark e coverage >= 85% (piso de CI real é +> 88%, ver `.github/workflows/python-ci.yml` e a seção "DoD específico do +> pacote Python" em `AGENTS.md`). + ## Pegadinhas conhecidas Liste armadilhas, dívida técnica encostada, comportamentos não óbvios. Atualize conforme o time descobre durante a execução. diff --git a/AGENTS.md b/AGENTS.md index 2794a73..a292f32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -314,6 +314,26 @@ PR só faz merge quando **todos** os itens abaixo estão marcados: CI bloqueia merge se DoD falhar (`.github/workflows/dod.yml`). +### DoD específico do pacote Python (`simplicio_mapper/`) + +O checklist acima é o DoD genérico do scaffold (Node/npm). Para trabalho dentro +de `simplicio_mapper/` (o pacote Python canônico), nenhuma issue/task fecha +sem cobrir as **7 dimensões** abaixo — regra explícita, não é opcional: + +- [ ] **Implementation** — o código de produção que a task pede, sem escopo extra. +- [ ] **Unit** — cobre a lógica isolada do módulo tocado (`tests/python/test_*.py`). +- [ ] **Integration** — cobre a interação entre módulos/CLI (ex.: dispatch via `main()`, `cli/_repo_commands.py`). +- [ ] **System** — cobre o fluxo ponta-a-ponta via entry point real (ex.: `simplicio-mapper snapshot ...` contra um repo de verdade), não só unidades isoladas. +- [ ] **Regression** — todo bug real corrigido ganha um teste que trava a regressão (não só o fix). +- [ ] **Perf benchmark** — quando a mudança toca caminho quente (retrieval, indexação, parsing em escala), roda `python3 scripts/runtime_scale_benchmark.py` (ou o benchmark relevante em `scripts/*_benchmark.py`) e reporta o número; não é "achismo". +- [ ] **Coverage >= 85%** — piso do pacote Python. O gate real de CI (`.github/workflows/python-ci.yml`, job `python-tests`) já roda `python -m pytest tests/python -q --cov=simplicio_mapper --cov-report=term --cov-fail-under=88`, ou seja, o piso de CI (88%) é mais estrito que este mínimo (85%) — nunca reduzir o `--cov-fail-under` do workflow para acomodar uma task. + +Comando de verificação local (mesmo usado no CI): + +```bash +python -m pytest tests/python -q --cov=simplicio_mapper --cov-report=term --cov-fail-under=88 +``` + --- ## Padrões de código diff --git a/CLAUDE.md b/CLAUDE.md index 85ad95b..560261f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -328,6 +328,26 @@ PR só faz merge quando **todos** os itens abaixo estão marcados: CI bloqueia merge se DoD falhar (`.github/workflows/dod.yml`). +### DoD específico do pacote Python (`simplicio_mapper/`) + +O checklist acima é o DoD genérico do scaffold (Node/npm). Para trabalho dentro +de `simplicio_mapper/` (o pacote Python canônico), nenhuma issue/task fecha +sem cobrir as **7 dimensões** abaixo — regra explícita, não é opcional: + +- [ ] **Implementation** — o código de produção que a task pede, sem escopo extra. +- [ ] **Unit** — cobre a lógica isolada do módulo tocado (`tests/python/test_*.py`). +- [ ] **Integration** — cobre a interação entre módulos/CLI (ex.: dispatch via `main()`, `cli/_repo_commands.py`). +- [ ] **System** — cobre o fluxo ponta-a-ponta via entry point real (ex.: `simplicio-mapper snapshot ...` contra um repo de verdade), não só unidades isoladas. +- [ ] **Regression** — todo bug real corrigido ganha um teste que trava a regressão (não só o fix). +- [ ] **Perf benchmark** — quando a mudança toca caminho quente (retrieval, indexação, parsing em escala), roda `python3 scripts/runtime_scale_benchmark.py` (ou o benchmark relevante em `scripts/*_benchmark.py`) e reporta o número; não é "achismo". +- [ ] **Coverage >= 85%** — piso do pacote Python. O gate real de CI (`.github/workflows/python-ci.yml`, job `python-tests`) já roda `python -m pytest tests/python -q --cov=simplicio_mapper --cov-report=term --cov-fail-under=88`, ou seja, o piso de CI (88%) é mais estrito que este mínimo (85%) — nunca reduzir o `--cov-fail-under` do workflow para acomodar uma task. + +Comando de verificação local (mesmo usado no CI): + +```bash +python -m pytest tests/python -q --cov=simplicio_mapper --cov-report=term --cov-fail-under=88 +``` + --- ## Padrões de código diff --git a/contracts/mapper-canvas/v1/fixtures/cli/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/cli/golden/visualization-bundle.json index 93b0211..e1fb597 100644 --- a/contracts/mapper-canvas/v1/fixtures/cli/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/cli/golden/visualization-bundle.json @@ -512,7 +512,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { "file_count": 3, "root": "." @@ -934,7 +934,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "cli/commands.py", @@ -942,7 +942,7 @@ "kind": "file", "language": "python", "metrics": { - "size_bytes": 43 + "size_bytes": 41 }, "name": "commands.py", "parent_id": "module:26f9252a7a7e4177", @@ -954,7 +954,7 @@ "kind": "file", "language": "python", "metrics": { - "size_bytes": 112 + "size_bytes": 105 }, "name": "main.py", "parent_id": "module:26f9252a7a7e4177", @@ -966,7 +966,7 @@ "kind": "file", "language": "toml", "metrics": { - "size_bytes": 51 + "size_bytes": 48 }, "name": "pyproject.toml", "parent_id": "module:5a219264bc5d86e9", diff --git a/contracts/mapper-canvas/v1/fixtures/event-driven/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/event-driven/golden/visualization-bundle.json index e98d7d7..193886b 100644 --- a/contracts/mapper-canvas/v1/fixtures/event-driven/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/event-driven/golden/visualization-bundle.json @@ -528,7 +528,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { "file_count": 4, "root": "." @@ -982,7 +982,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "package.json", @@ -990,7 +990,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 51 + "size_bytes": 50 }, "name": "package.json", "parent_id": "module:5a219264bc5d86e9", @@ -1002,7 +1002,7 @@ "kind": "file", "language": "javascript", "metrics": { - "size_bytes": 77 + "size_bytes": 75 }, "name": "consumer.js", "parent_id": "module:04164df2fbf7cf9b", @@ -1014,7 +1014,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 39 + "size_bytes": 38 }, "name": "events.json", "parent_id": "module:04164df2fbf7cf9b", @@ -1026,7 +1026,7 @@ "kind": "file", "language": "javascript", "metrics": { - "size_bytes": 125 + "size_bytes": 122 }, "name": "producer.js", "parent_id": "module:04164df2fbf7cf9b", diff --git a/contracts/mapper-canvas/v1/fixtures/incomplete/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/incomplete/golden/visualization-bundle.json index 36ad6a8..68937ec 100644 --- a/contracts/mapper-canvas/v1/fixtures/incomplete/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/incomplete/golden/visualization-bundle.json @@ -18,24 +18,21 @@ "edge_count": 0, "external_edge_count": 0, "id": "cluster:workspace:86fa3762d2eb6db6", - "importance": 0.3025, + "importance": 0.363333, "internal_edge_count": 0, "kind": "workspace", "language_mix": { "javascript": 1, "json": 1, - "markdown": 1, - "text": 1 + "markdown": 1 }, - "member_count": 4, + "member_count": 3, "member_ids": [ - "cluster:file:56c17ad5d9463672", "cluster:file:a451623ea511a512", "cluster:file:d7f6204f00b61069", "cluster:file:f5ed0571f3d5e126" ], "member_paths": [ - ".env", "README.md", "package.json", "src/app.js" @@ -51,22 +48,19 @@ "edge_count": 0, "external_edge_count": 0, "id": "cluster:directory:c22d9e7664332f0b", - "importance": 0.186667, + "importance": 0.22, "internal_edge_count": 0, "kind": "directory", "language_mix": { "json": 1, - "markdown": 1, - "text": 1 + "markdown": 1 }, - "member_count": 3, + "member_count": 2, "member_ids": [ - "cluster:file:56c17ad5d9463672", "cluster:file:a451623ea511a512", "cluster:file:d7f6204f00b61069" ], "member_paths": [ - ".env", "README.md", "package.json" ], @@ -105,22 +99,19 @@ "edge_count": 0, "external_edge_count": 0, "id": "cluster:package:710115d946c582c2", - "importance": 0.186667, + "importance": 0.22, "internal_edge_count": 0, "kind": "package", "language_mix": { "json": 1, - "markdown": 1, - "text": 1 + "markdown": 1 }, - "member_count": 3, + "member_count": 2, "member_ids": [ - "cluster:file:56c17ad5d9463672", "cluster:file:a451623ea511a512", "cluster:file:d7f6204f00b61069" ], "member_paths": [ - ".env", "README.md", "package.json" ], @@ -152,30 +143,6 @@ "overlapping": true, "recommended_collapse_level": "members" }, - { - "cohesion": 0.0, - "coupling": 0.0, - "dependency_density": 0.0, - "edge_count": 0, - "external_edge_count": 0, - "id": "cluster:namespace:ae7db4ac72c14b1f", - "importance": 0.12, - "internal_edge_count": 0, - "kind": "namespace", - "language_mix": { - "text": 1 - }, - "member_count": 1, - "member_ids": [ - "cluster:file:56c17ad5d9463672" - ], - "member_paths": [ - ".env" - ], - "name": "", - "overlapping": true, - "recommended_collapse_level": "expanded" - }, { "cohesion": 0.0, "coupling": 0.0, @@ -255,24 +222,21 @@ "edge_count": 0, "external_edge_count": 0, "id": "cluster:domain:883d523e9eba130e", - "importance": 0.3025, + "importance": 0.363333, "internal_edge_count": 0, "kind": "domain", "language_mix": { "javascript": 1, "json": 1, - "markdown": 1, - "text": 1 + "markdown": 1 }, - "member_count": 4, + "member_count": 3, "member_ids": [ - "cluster:file:56c17ad5d9463672", "cluster:file:a451623ea511a512", "cluster:file:d7f6204f00b61069", "cluster:file:f5ed0571f3d5e126" ], "member_paths": [ - ".env", "README.md", "package.json", "src/app.js" @@ -281,30 +245,6 @@ "overlapping": true, "recommended_collapse_level": "expanded" }, - { - "cohesion": 0.0, - "coupling": 0.0, - "dependency_density": 0.0, - "edge_count": 0, - "external_edge_count": 0, - "id": "cluster:layer:0db4f4feb847f612", - "importance": 0.12, - "internal_edge_count": 0, - "kind": "layer", - "language_mix": { - "text": 1 - }, - "member_count": 1, - "member_ids": [ - "cluster:file:56c17ad5d9463672" - ], - "member_paths": [ - ".env" - ], - "name": "asset", - "overlapping": true, - "recommended_collapse_level": "expanded" - }, { "cohesion": 0.0, "coupling": 0.0, @@ -425,7 +365,7 @@ { "cluster_id": "cluster:workspace:86fa3762d2eb6db6", "collapse_level": "expanded", - "importance": 0.3025, + "importance": 0.363333, "lane": "workspace", "lane_index": 0, "order": 0, @@ -434,7 +374,7 @@ { "cluster_id": "cluster:directory:c22d9e7664332f0b", "collapse_level": "expanded", - "importance": 0.186667, + "importance": 0.22, "lane": "directory", "lane_index": 2, "order": 1, @@ -452,7 +392,7 @@ { "cluster_id": "cluster:package:710115d946c582c2", "collapse_level": "expanded", - "importance": 0.186667, + "importance": 0.22, "lane": "package", "lane_index": 1, "order": 3, @@ -467,22 +407,13 @@ "order": 4, "rationale": "stable strategy order; renderer chooses coordinates" }, - { - "cluster_id": "cluster:namespace:ae7db4ac72c14b1f", - "collapse_level": "expanded", - "importance": 0.12, - "lane": "namespace", - "lane_index": 3, - "order": 5, - "rationale": "stable strategy order; renderer chooses coordinates" - }, { "cluster_id": "cluster:namespace:dc094dac44e7bb2e", "collapse_level": "expanded", "importance": 0.12, "lane": "namespace", "lane_index": 3, - "order": 6, + "order": 5, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -491,7 +422,7 @@ "importance": 0.32, "lane": "namespace", "lane_index": 3, - "order": 7, + "order": 6, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -500,25 +431,16 @@ "importance": 0.65, "lane": "namespace", "lane_index": 3, - "order": 8, + "order": 7, "rationale": "stable strategy order; renderer chooses coordinates" }, { "cluster_id": "cluster:domain:883d523e9eba130e", "collapse_level": "expanded", - "importance": 0.3025, + "importance": 0.363333, "lane": "domain", "lane_index": 4, - "order": 9, - "rationale": "stable strategy order; renderer chooses coordinates" - }, - { - "cluster_id": "cluster:layer:0db4f4feb847f612", - "collapse_level": "expanded", - "importance": 0.12, - "lane": "layer", - "lane_index": 5, - "order": 10, + "order": 8, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -527,7 +449,7 @@ "importance": 0.32, "lane": "layer", "lane_index": 5, - "order": 11, + "order": 9, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -536,7 +458,7 @@ "importance": 0.12, "lane": "layer", "lane_index": 5, - "order": 12, + "order": 10, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -545,7 +467,7 @@ "importance": 0.65, "lane": "layer", "lane_index": 5, - "order": 13, + "order": 11, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -554,13 +476,13 @@ "importance": 0.65, "lane": "flow", "lane_index": 6, - "order": 14, + "order": 12, "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { - "file_count": 4, + "file_count": 3, "root": "." }, "provenance": { @@ -601,7 +523,7 @@ "diagnostics": [ { "code": "heuristic-edge", - "count": 5 + "count": 4 }, { "code": "unresolved-relation", @@ -688,16 +610,6 @@ "target": "module:5a219264bc5d86e9", "type": "contains" }, - { - "confidence": 0.7, - "direction": "forward", - "language": null, - "provenance": "static-mapper", - "source": "repository:f39c4b97cb07d484", - "source_location": null, - "target": "layer:0db4f4feb847f612", - "type": "groups" - }, { "confidence": 0.7, "direction": "forward", @@ -807,7 +719,7 @@ { "cluster_id": "cluster:workspace:86fa3762d2eb6db6", "collapse_level": "expanded", - "importance": 0.3025, + "importance": 0.363333, "lane": "workspace", "lane_index": 0, "order": 0, @@ -816,7 +728,7 @@ { "cluster_id": "cluster:directory:c22d9e7664332f0b", "collapse_level": "expanded", - "importance": 0.186667, + "importance": 0.22, "lane": "directory", "lane_index": 2, "order": 1, @@ -834,7 +746,7 @@ { "cluster_id": "cluster:package:710115d946c582c2", "collapse_level": "expanded", - "importance": 0.186667, + "importance": 0.22, "lane": "package", "lane_index": 1, "order": 3, @@ -849,22 +761,13 @@ "order": 4, "rationale": "stable strategy order; renderer chooses coordinates" }, - { - "cluster_id": "cluster:namespace:ae7db4ac72c14b1f", - "collapse_level": "expanded", - "importance": 0.12, - "lane": "namespace", - "lane_index": 3, - "order": 5, - "rationale": "stable strategy order; renderer chooses coordinates" - }, { "cluster_id": "cluster:namespace:dc094dac44e7bb2e", "collapse_level": "expanded", "importance": 0.12, "lane": "namespace", "lane_index": 3, - "order": 6, + "order": 5, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -873,7 +776,7 @@ "importance": 0.32, "lane": "namespace", "lane_index": 3, - "order": 7, + "order": 6, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -882,25 +785,16 @@ "importance": 0.65, "lane": "namespace", "lane_index": 3, - "order": 8, + "order": 7, "rationale": "stable strategy order; renderer chooses coordinates" }, { "cluster_id": "cluster:domain:883d523e9eba130e", "collapse_level": "expanded", - "importance": 0.3025, + "importance": 0.363333, "lane": "domain", "lane_index": 4, - "order": 9, - "rationale": "stable strategy order; renderer chooses coordinates" - }, - { - "cluster_id": "cluster:layer:0db4f4feb847f612", - "collapse_level": "expanded", - "importance": 0.12, - "lane": "layer", - "lane_index": 5, - "order": 10, + "order": 8, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -909,7 +803,7 @@ "importance": 0.32, "lane": "layer", "lane_index": 5, - "order": 11, + "order": 9, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -918,7 +812,7 @@ "importance": 0.12, "lane": "layer", "lane_index": 5, - "order": 12, + "order": 10, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -927,7 +821,7 @@ "importance": 0.65, "lane": "layer", "lane_index": 5, - "order": 13, + "order": 11, "rationale": "stable strategy order; renderer chooses coordinates" }, { @@ -936,11 +830,11 @@ "importance": 0.65, "lane": "flow", "lane_index": 6, - "order": 14, + "order": 12, "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "README.md", @@ -948,7 +842,7 @@ "kind": "file", "language": "markdown", "metrics": { - "size_bytes": 70 + "size_bytes": 69 }, "name": "README.md", "parent_id": "module:5a219264bc5d86e9", @@ -960,7 +854,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 50 + "size_bytes": 49 }, "name": "package.json", "parent_id": "module:5a219264bc5d86e9", @@ -972,7 +866,7 @@ "kind": "file", "language": "javascript", "metrics": { - "size_bytes": 88 + "size_bytes": 86 }, "name": "app.js", "parent_id": "module:04164df2fbf7cf9b", @@ -988,17 +882,6 @@ "parent_id": "repository:f39c4b97cb07d484", "path": null }, - { - "canonical": "asset", - "confidence": 0.7, - "id": "layer:0db4f4feb847f612", - "kind": "layer", - "metrics": {}, - "name": "asset", - "parent_id": "repository:f39c4b97cb07d484", - "path": null, - "rule_id": "mapper.layer-path" - }, { "canonical": "config", "confidence": 0.7, diff --git a/contracts/mapper-canvas/v1/fixtures/layered-web/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/layered-web/golden/visualization-bundle.json index b1d3b18..41b41fe 100644 --- a/contracts/mapper-canvas/v1/fixtures/layered-web/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/layered-web/golden/visualization-bundle.json @@ -623,7 +623,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { "file_count": 4, "root": "." @@ -1202,7 +1202,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "package.json", @@ -1210,7 +1210,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 54 + "size_bytes": 53 }, "name": "package.json", "parent_id": "module:5a219264bc5d86e9", @@ -1222,7 +1222,7 @@ "kind": "file", "language": "html", "metrics": { - "size_bytes": 59 + "size_bytes": 57 }, "name": "index.html", "parent_id": "module:77af9af43a396f96", @@ -1234,7 +1234,7 @@ "kind": "file", "language": "javascript", "metrics": { - "size_bytes": 204 + "size_bytes": 200 }, "name": "routes.js", "parent_id": "module:04164df2fbf7cf9b", @@ -1246,7 +1246,7 @@ "kind": "file", "language": "javascript", "metrics": { - "size_bytes": 117 + "size_bytes": 114 }, "name": "server.js", "parent_id": "module:04164df2fbf7cf9b", diff --git a/contracts/mapper-canvas/v1/fixtures/monorepo/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/monorepo/golden/visualization-bundle.json index e7a0824..89b0aca 100644 --- a/contracts/mapper-canvas/v1/fixtures/monorepo/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/monorepo/golden/visualization-bundle.json @@ -730,7 +730,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { "file_count": 5, "root": "." @@ -1296,7 +1296,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "apps/web/package.json", @@ -1304,7 +1304,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 20 + "size_bytes": 19 }, "name": "package.json", "parent_id": "module:dffca0b038d04d1c", @@ -1316,7 +1316,7 @@ "kind": "file", "language": "typescript", "metrics": { - "size_bytes": 122 + "size_bytes": 120 }, "name": "index.ts", "parent_id": "module:dffca0b038d04d1c", @@ -1328,7 +1328,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 65 + "size_bytes": 64 }, "name": "package.json", "parent_id": "module:5a219264bc5d86e9", @@ -1340,7 +1340,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 19 + "size_bytes": 18 }, "name": "package.json", "parent_id": "module:a6a4aa90076b3d1f", @@ -1352,7 +1352,7 @@ "kind": "file", "language": "typescript", "metrics": { - "size_bytes": 75 + "size_bytes": 74 }, "name": "index.ts", "parent_id": "module:a6a4aa90076b3d1f", diff --git a/contracts/mapper-canvas/v1/fixtures/polyglot/golden/visualization-bundle.json b/contracts/mapper-canvas/v1/fixtures/polyglot/golden/visualization-bundle.json index 62122e6..23eb64f 100644 --- a/contracts/mapper-canvas/v1/fixtures/polyglot/golden/visualization-bundle.json +++ b/contracts/mapper-canvas/v1/fixtures/polyglot/golden/visualization-bundle.json @@ -681,7 +681,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "project": { "file_count": 4, "root": "." @@ -1217,7 +1217,7 @@ "rationale": "stable strategy order; renderer chooses coordinates" } ], - "mapper_version": "0.20.0", + "mapper_version": "0.0.0", "nodes": [ { "canonical": "go/main.go", @@ -1225,7 +1225,7 @@ "kind": "file", "language": "go", "metrics": { - "size_bytes": 53 + "size_bytes": 50 }, "name": "main.go", "parent_id": "module:3800de2487085eba", @@ -1237,7 +1237,7 @@ "kind": "file", "language": "json", "metrics": { - "size_bytes": 28 + "size_bytes": 27 }, "name": "package.json", "parent_id": "module:5a219264bc5d86e9", @@ -1249,7 +1249,7 @@ "kind": "file", "language": "python", "metrics": { - "size_bytes": 35 + "size_bytes": 33 }, "name": "app.py", "parent_id": "module:8dbe8155e679c240", @@ -1261,7 +1261,7 @@ "kind": "file", "language": "rust", "metrics": { - "size_bytes": 41 + "size_bytes": 40 }, "name": "lib.rs", "parent_id": "module:42b5302cd11f1a60", diff --git a/simplicio_mapper/query.py b/simplicio_mapper/query.py index b64c63f..3cf7ddd 100644 --- a/simplicio_mapper/query.py +++ b/simplicio_mapper/query.py @@ -422,6 +422,34 @@ def _runtime_ask_query(cwd: str, verb: str, arg: str, limit: int) -> tuple[dict return payload, "delegated" +_NATIVE_BASELINE_SKIP_DIRS = {".git", "node_modules", ".simplicio", "venv", ".venv", "dist", "build", "__pycache__"} +_NATIVE_BASELINE_SOURCE_EXTS = {".py", ".js", ".jsx", ".ts", ".tsx", ".go", ".rs", ".java", ".rb"} + + +def _cheap_repo_baseline_tokens(abs_cwd: str, *, test_files_only: bool) -> int: + """Cheap, artifact-lazy estimate of what the local fallback would have + had to read for `impact` (whole-repo call_graph/symbol_index proxy) or + `tests-for` (every test file's full text). A raw `os.walk` + file-size + sum, deliberately NOT `build_artifacts()` -- a native hit must stay + artifact-lazy (see `test_native_impact_success_stays_artifact_lazy` / + `test_native_tests_for_success_stays_artifact_lazy` in test_query.py), + so this baseline can only ever be a rough, declared-estimate proxy, not + the exact bytes the fallback would have parsed.""" + total_bytes = 0 + for dirpath, dirnames, filenames in os.walk(abs_cwd): + dirnames[:] = [d for d in dirnames if d not in _NATIVE_BASELINE_SKIP_DIRS and not d.startswith(".")] + for name in filenames: + if test_files_only and "test" not in name.lower(): + continue + if os.path.splitext(name)[1] not in _NATIVE_BASELINE_SOURCE_EXTS: + continue + try: + total_bytes += os.path.getsize(os.path.join(dirpath, name)) + except OSError: + continue + return max(1, total_bytes // 4) + + def _record_ask_native_savings(cwd: str, verb: str, baseline_tokens: int, payload: dict, note: str) -> None: """Best-effort savings-ledger receipt for a native `ask` hit. Never raises -- a ledger write failure must never break the query path.""" @@ -544,6 +572,20 @@ def _artifacts() -> dict[str, Any]: method="runtime-native ask impact", ) payload["cache"] = _cache_block(cache, native_key.content_hash(), receipt.to_dict()) + # Savings are recorded on the native HIT, not the local fallback: + # a native hit is precisely the call that *avoided* paying the + # per-verb marginal cost this baseline estimates. Must stay + # artifact-lazy (no `_artifacts()`/`build_artifacts()` call) -- + # see `_cheap_repo_baseline_tokens`. + baseline = _cheap_repo_baseline_tokens(abs_cwd, test_files_only=False) + _record_ask_native_savings( + abs_cwd, + "impact", + baseline, + payload, + note="baseline=call_graph+symbol_index the LLM/loop would otherwise have to read " + "to answer 'what does changing this file affect' manually; method=heuristic:chars-div-4", + ) else: cache_key = _query_cache_key(abs_cwd, out_dir, query) cached_payload, receipt = cache.get_entry(LAYER_CONTEXT_SUMMARY, cache_key) @@ -566,17 +608,6 @@ def _artifacts() -> dict[str, Any]: bytes_avoided=len(json.dumps(payload, sort_keys=True).encode("utf-8")), ) payload["cache"] = _cache_block(cache, cache_key.content_hash(), receipt.to_dict()) - baseline = estimate_tokens( - json.dumps(materialized.get("call_graph"), sort_keys=True) - ) + estimate_tokens(json.dumps(materialized.get("symbol_index"), sort_keys=True)) - _record_ask_native_savings( - abs_cwd, - "impact", - baseline, - payload, - note="baseline=call_graph+symbol_index the LLM/loop would otherwise have to read " - "to answer 'what does changing this file affect' manually; method=heuristic:chars-div-4", - ) payload["delegation"] = { "runtime": "simplicio-runtime", "used": native is not None, @@ -616,6 +647,18 @@ def _artifacts() -> dict[str, Any]: method="runtime-native ask tests-for", ) payload["cache"] = _cache_block(cache, native_key.content_hash(), receipt.to_dict()) + # Savings are recorded on the native HIT (see the matching + # comment in the `impact` branch above for why; must stay + # artifact-lazy here too). + baseline = _cheap_repo_baseline_tokens(abs_cwd, test_files_only=True) + _record_ask_native_savings( + abs_cwd, + "tests-for", + baseline, + payload, + note="baseline=full text of every test file the local fallback would otherwise " + "read in full to find matches; method=heuristic:chars-div-4", + ) else: cache_key = _query_cache_key(abs_cwd, out_dir, query) cached_payload, receipt = cache.get_entry(LAYER_CONTEXT_SUMMARY, cache_key) @@ -634,17 +677,6 @@ def _artifacts() -> dict[str, Any]: bytes_avoided=len(json.dumps(payload, sort_keys=True).encode("utf-8")), ) payload["cache"] = _cache_block(cache, cache_key.content_hash(), receipt.to_dict()) - baseline = sum( - estimate_tokens(_read_text(abs_cwd, f)) for f in project_map.get("test_files") or [] - ) - _record_ask_native_savings( - abs_cwd, - "tests-for", - baseline, - payload, - note="baseline=full text of every test file the local fallback would otherwise " - "read in full to find matches; method=heuristic:chars-div-4", - ) payload["delegation"] = { "runtime": "simplicio-runtime", "used": native is not None, diff --git a/simplicio_mapper/retrieval_index.py b/simplicio_mapper/retrieval_index.py index 9a0fe52..e18f9a7 100644 --- a/simplicio_mapper/retrieval_index.py +++ b/simplicio_mapper/retrieval_index.py @@ -366,6 +366,43 @@ def _symbol_tokens(text: str) -> list[str]: return [t for t in _SYMBOL_TOKEN_RE.findall(text) if len(t) >= 4] +_TASK_INTENT_NOISE_KEYS = {"schema", "fingerprint"} + + +def _task_intent_text(value: Any) -> list[str]: + """Collect only the free-text *values* of a normalized task-intent + object, skipping its own reserved schema key names (`schema`, + `fingerprint`) and skipping key names generally. + + `parse_task_intent()` always returns the full normalized schema shape + (`story`, `acceptance_criteria`, `business_rules`, + `non_functional_requirements`, `additional_information`, ... - see + `task_intent.py`), most of which are empty for any given task. Naively + `json.dumps()`-ing that whole object and tokenizing it (as this used to + do) fed every one of those field *names* into `domain_terms` as if they + were part of the task's own vocabulary, on every single task -- diluting + `coverage_ratio` (matched_count / query_term_count) so badly that a + task with real, present evidence in the repo (e.g. the word "temporal" + verbatim in a source comment) could still fail the minimum-coverage gate + and get an incorrect `needs_broader_context: true`/abstention, because + the denominator was inflated by ~15+ schema-noise tokens that can never + match anything. + """ + texts: list[str] = [] + if isinstance(value, Mapping): + for key, sub_value in value.items(): + if key in _TASK_INTENT_NOISE_KEYS: + continue + texts.extend(_task_intent_text(sub_value)) + elif isinstance(value, (list, tuple)): + for item in value: + texts.extend(_task_intent_text(item)) + elif isinstance(value, str): + if value: + texts.append(value) + return texts + + def build_query_plan( goal: str = "", *, @@ -376,7 +413,7 @@ def build_query_plan( """Normalize a task into weighted query fields (Stage B).""" chunks = [goal] if task_intent: - chunks.append(json.dumps(task_intent, ensure_ascii=False, sort_keys=True)) + chunks.extend(_task_intent_text(task_intent)) text = " ".join(chunks) norm = _normalized_text(text) diff --git a/simplicio_mapper/visualization.py b/simplicio_mapper/visualization.py index 70cc39f..27c0820 100644 --- a/simplicio_mapper/visualization.py +++ b/simplicio_mapper/visualization.py @@ -74,20 +74,36 @@ def _safe_remote(value: str) -> str: def _provenance(root: str) -> dict[str, Any]: - remotes = [] - for line in _git(root, "remote", "-v").splitlines(): - parts = line.split() - if len(parts) >= 2: - url = _safe_remote(parts[1]) - if url and url not in [item["url"] for item in remotes]: - remotes.append({"name": parts[0], "url": url}) - primary = remotes[0]["url"] if remotes else None - match = re.match(r"https?://([^/]+)/([^/]+)/([^/]+?)(?:\.git)?$", primary or "") - head = _git(root, "symbolic-ref", "--short", "HEAD") - default_ref = _git(root, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") - default_branch = default_ref.split("/", 1)[1] if "/" in default_ref else None + # `git -C ...` discovers the nearest ancestor `.git` unless `root` + # itself owns one -- so a fixture/subdirectory with no `.git` of its own + # (e.g. contracts/*/fixtures/*/source nested inside this very repo) would + # otherwise silently inherit the enclosing repo's remote/branch/commit as + # if it were the fixture's own provenance. Only query git when `root` is + # actually a git worktree root, so a genuine plain-folder clone reports + # `remote_url`/`branch`/`commit_sha` as null instead of leaking ambient + # ancestor-repo state. git_dir = os.path.join(root, ".git") is_git = os.path.exists(git_dir) + remotes: list[dict[str, str]] = [] + primary = None + head = "" + default_ref = "" + commit_sha = "" + dirty = False + if is_git: + for line in _git(root, "remote", "-v").splitlines(): + parts = line.split() + if len(parts) >= 2: + url = _safe_remote(parts[1]) + if url and url not in [item["url"] for item in remotes]: + remotes.append({"name": parts[0], "url": url}) + primary = remotes[0]["url"] if remotes else None + head = _git(root, "symbolic-ref", "--short", "HEAD") + default_ref = _git(root, "symbolic-ref", "--short", "refs/remotes/origin/HEAD") + commit_sha = _git(root, "rev-parse", "HEAD") + dirty = bool(_git(root, "status", "--porcelain")) + match = re.match(r"https?://([^/]+)/([^/]+)/([^/]+?)(?:\.git)?$", primary or "") + default_branch = default_ref.split("/", 1)[1] if "/" in default_ref else None submodules = [] gitmodules = os.path.join(root, ".gitmodules") if os.path.isfile(gitmodules): @@ -108,9 +124,9 @@ def _provenance(root: str) -> dict[str, Any]: "owner": match.group(2) if match else None, "repository": match.group(3) if match else None, "branch": head or None, - "commit_sha": _git(root, "rev-parse", "HEAD") or None, + "commit_sha": commit_sha or None, "default_branch": default_branch, - "dirty": bool(_git(root, "status", "--porcelain")), + "dirty": dirty, "scan_root": ".", "clone_type": "git-worktree" if os.path.isfile(git_dir) else ("git-clone" if is_git else "plain-folder"), "submodules": submodules, @@ -172,9 +188,16 @@ def _safe_relative_path(root: str, requested: str) -> tuple[str, str]: raise ValueError("path is required") root_real = os.path.realpath(root) candidate = os.path.abspath(os.path.join(root, requested.replace("/", os.sep))) - if os.path.commonpath([root_real, candidate]) != root_real: + # Compare resolved paths on both sides: on macOS `root` is commonly + # itself a symlink (e.g. tmp dirs under `/var/folders` -> `/private/var/ + # folders`), and `os.path.realpath(root)` resolves that while `candidate` + # (built from the unresolved `root`) does not -- an unresolved `candidate` + # can never share `root_real`'s resolved prefix, so every request was + # incorrectly rejected as "outside mapped root" on those platforms. + candidate_real = os.path.realpath(candidate) + if os.path.commonpath([root_real, candidate_real]) != root_real: raise ValueError("path is outside mapped root") - relative = os.path.relpath(candidate, root_real) + relative = os.path.relpath(candidate_real, root_real) parts = relative.replace(os.sep, "/").split("/") if any(part in {"", ".", ".."} for part in parts) or any(part in _DENIED_PARTS for part in parts[:-1]): raise ValueError("path is denied by preview policy") diff --git a/tests/python/test_behavioral_scorecard.py b/tests/python/test_behavioral_scorecard.py index 14f56ef..06702a4 100644 --- a/tests/python/test_behavioral_scorecard.py +++ b/tests/python/test_behavioral_scorecard.py @@ -9,7 +9,11 @@ class BehavioralScorecardTest(unittest.TestCase): def test_real_cli_scorecard_is_deterministic_and_measured(self) -> None: - command = [sys.executable, "scripts/behavioral_scorecard.py", "--json"] + # NOTE: the script backing this scorecard is scripts/evaluation_scorecard.py + # (schema "simplicio.behavioral-scorecard/v1", see docs/behavioral-scorecard.md). + # scripts/behavioral_scorecard.py never existed in this repo; this test was + # written against a name that was never landed. Point it at the real script. + command = [sys.executable, "scripts/evaluation_scorecard.py", "--json"] first = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=False) second = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=False) self.assertEqual(first.returncode, 0, first.stderr) @@ -21,7 +25,17 @@ def test_real_cli_scorecard_is_deterministic_and_measured(self) -> None: self.assertEqual(left["measurements"]["target_recall_at_k"], 1.0) self.assertEqual(left["measurements"]["test_recall_at_k"], 1.0) self.assertEqual(left["measurements"]["determinism"], 1.0) - self.assertEqual( - [item["first_fingerprint"] for item in left["cases"]], - [item["first_fingerprint"] for item in right["cases"]], - ) + self.assertEqual(right["measurements"]["determinism"], 1.0) + # `context_pack.pack_hash` is intentionally root-path-scoped + # (simplicio_mapper/context_pack.py `root_hash = _sha256_text(abs_root)`, + # by design -- prevents cross-repo cache collisions on identical content). + # Each scorecard run copies fixtures into a fresh random tempdir + # (`scripts/evaluation_scorecard.py::_copy_case`), so the fingerprint + # itself legitimately differs run-to-run; per-run determinism is what + # `first_fingerprint == second_fingerprint` inside a single run + # measures, and that is already asserted via `measurements.determinism` + # above. What must stay stable across independent runs is the case + # identity/count and every per-case "deterministic" verdict. + self.assertEqual([item["id"] for item in left["cases"]], [item["id"] for item in right["cases"]]) + self.assertTrue(all(item["deterministic"] for item in left["cases"])) + self.assertTrue(all(item["deterministic"] for item in right["cases"])) diff --git a/tests/python/test_cli_args_validation.py b/tests/python/test_cli_args_validation.py new file mode 100644 index 0000000..8fa5ef9 --- /dev/null +++ b/tests/python/test_cli_args_validation.py @@ -0,0 +1,112 @@ +"""Coverage for `simplicio_mapper/cli/_args.py`'s argv-validation error +paths (`-- requires a value` / `Invalid -- value`), driven +through the real `main()` CLI entry point. Each of these branches +`sys.exit(2)`s with a specific stderr message; they were previously +untested because every other CLI test only ever passes well-formed argv. +""" + +from __future__ import annotations + +import tempfile +import unittest +from contextlib import redirect_stderr +from io import StringIO +from pathlib import Path + +from simplicio_mapper.cli import main + + +class ArgsValidationCliTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.dir = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _expect_exit_2(self, argv: list[str], expected_message_fragment: str) -> None: + # `_args.py`'s validation branches call `sys.exit(2)` directly + # (argparse-style), which raises `SystemExit` through `main()` + # rather than returning an int -- unlike the rest of the CLI's + # `return ` error paths. + err = StringIO() + with redirect_stderr(err), self.assertRaises(SystemExit) as ctx: + main(argv) + self.assertEqual(ctx.exception.code, 2, err.getvalue()) + self.assertIn(expected_message_fragment, err.getvalue()) + + def test_minimum_query_coverage_out_of_range_is_rejected(self) -> None: + self._expect_exit_2( + ["orient", str(self.dir), "--minimum-query-coverage", "1.5"], + "--minimum-query-coverage requires a number between 0 and 1", + ) + + def test_minimum_query_coverage_non_numeric_is_rejected(self) -> None: + self._expect_exit_2( + ["orient", str(self.dir), "--minimum-query-coverage", "not-a-number"], + "--minimum-query-coverage requires a number between 0 and 1", + ) + + def test_token_budget_zero_is_rejected(self) -> None: + self._expect_exit_2( + ["orient", str(self.dir), "--token-budget", "0"], + "--token-budget requires a positive integer", + ) + + def test_token_budget_non_numeric_is_rejected(self) -> None: + self._expect_exit_2( + ["orient", str(self.dir), "--token-budget", "abc"], + "--token-budget requires a positive integer", + ) + + def test_depth_non_numeric_is_rejected(self) -> None: + self._expect_exit_2(["ask", str(self.dir), "reaches", "x", "--depth", "abc"], "Invalid --depth value") + + def test_limit_non_numeric_is_rejected(self) -> None: + self._expect_exit_2(["ask", str(self.dir), "callers", "x", "--limit", "abc"], "Invalid --limit value") + + def test_drift_threshold_non_numeric_is_rejected(self) -> None: + self._expect_exit_2(["drift", str(self.dir), "--threshold", "abc"], "Invalid --threshold value") + + def test_drift_scope_invalid_value_is_rejected(self) -> None: + self._expect_exit_2( + ["drift", str(self.dir), "--scope", "not-a-real-scope"], + "--scope requires all, product, or template", + ) + + def test_retention_non_numeric_is_rejected(self) -> None: + self._expect_exit_2(["sync", str(self.dir), "--retention", "abc"], "Invalid --retention value") + + def test_background_timeout_non_numeric_is_rejected(self) -> None: + self._expect_exit_2( + ["index", str(self.dir), "--background", "--timeout", "abc"], + "Invalid --timeout value", + ) + + def test_for_llm_unknown_format_is_rejected(self) -> None: + self._expect_exit_2( + ["orient", str(self.dir), "--for-llm", "not-a-real-format"], + "Unknown --for-llm format", + ) + + def test_for_llm_missing_value_is_rejected(self) -> None: + self._expect_exit_2(["orient", str(self.dir), "--for-llm"], "--for-llm requires a value") + + def test_confidence_unknown_tag_is_rejected(self) -> None: + self._expect_exit_2( + ["index", str(self.dir), "--confidence", "not-a-real-tag"], + "Unknown --confidence tag", + ) + + def test_confidence_missing_value_is_rejected(self) -> None: + self._expect_exit_2(["index", str(self.dir), "--confidence"], "--confidence requires a value") + + def test_goal_missing_value_is_rejected(self) -> None: + self._expect_exit_2(["orient", str(self.dir), "--goal"], "--goal requires a value") + + def test_task_file_missing_value_is_rejected(self) -> None: + self._expect_exit_2(["orient", str(self.dir), "--task-file"], "--task-file requires a value") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_cli_repo_commands.py b/tests/python/test_cli_repo_commands.py new file mode 100644 index 0000000..f36e23e --- /dev/null +++ b/tests/python/test_cli_repo_commands.py @@ -0,0 +1,189 @@ +"""End-to-end CLI coverage for the thin subcommand wrappers in +`simplicio_mapper/cli/_repo_commands.py` (`visualize`, `sync`, `history`, +`diff`, `ask`) that were previously exercised only at the library level +(`build_visualization_bundle`, `build_docs_sync`, `run_query`, ...), not +through the real `main()` CLI dispatch -- both the `--json` and +human-readable print branches, and the documented error paths. +""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from contextlib import redirect_stdout, redirect_stderr +from io import StringIO +from pathlib import Path + +from simplicio_mapper.cli import main + + +def _write(base: Path, rel: str, content: str) -> None: + target = base / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +class RepoCommandsCliTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.dir = Path(self._tmp.name) + _write(self.dir, "package.json", json.dumps({"name": "repo-commands-app"})) + _write(self.dir, "src/main.py", "def main():\n return 1\n") + + def tearDown(self) -> None: + self._tmp.cleanup() + + # -- visualize ----------------------------------------------------- + + def test_visualize_json_writes_bundle_and_clustering_metrics(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["visualize", str(self.dir), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["schema"], "simplicio.visualization-bundle/v1") + self.assertTrue((self.dir / ".simplicio" / "visualization-bundle.json").is_file()) + self.assertTrue((self.dir / ".simplicio" / "clustering-metrics.json").is_file()) + + def test_visualize_human_output_summarizes_counts(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["visualize", str(self.dir)]) + self.assertEqual(code, 0) + self.assertIn("nodes=", out.getvalue()) + self.assertIn("edges=", out.getvalue()) + self.assertIn("clusters=", out.getvalue()) + + # -- sync ------------------------------------------------------------ + + def test_sync_json_reports_diff_and_regenerated_docs(self) -> None: + main(["docs", str(self.dir)]) + out = StringIO() + with redirect_stdout(out): + code = main(["sync", str(self.dir), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertIn("diff", payload) + self.assertIn("stale", payload) + + def test_sync_human_output_summarizes_counts(self) -> None: + main(["docs", str(self.dir)]) + out = StringIO() + with redirect_stdout(out): + code = main(["sync", str(self.dir)]) + self.assertEqual(code, 0) + self.assertIn("changed=", out.getvalue()) + self.assertIn("stale=", out.getvalue()) + + def test_sync_check_mode_does_not_snapshot(self) -> None: + main(["docs", str(self.dir)]) + out = StringIO() + with redirect_stdout(out): + code = main(["sync", str(self.dir), "--check", "--json"]) + self.assertIn(code, (0, 1)) + json.loads(out.getvalue()) + + # -- history / diff -------------------------------------------------- + + def test_history_json_reports_no_snapshots_initially(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["history", str(self.dir), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["schema"], "simplicio.doc-history-index/v1") + self.assertEqual(payload["snapshots"], []) + + def test_history_human_output_without_snapshots(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["history", str(self.dir)]) + self.assertEqual(code, 0) + self.assertIn("no snapshots yet", out.getvalue()) + + def test_history_human_output_lists_snapshots_after_docs_run(self) -> None: + main(["docs", str(self.dir)]) + out = StringIO() + with redirect_stdout(out): + code = main(["history", str(self.dir)]) + self.assertEqual(code, 0) + self.assertTrue(out.getvalue().strip()) + self.assertNotIn("no snapshots yet", out.getvalue()) + + def test_diff_without_from_and_to_is_rejected(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["diff", str(self.dir)]) + self.assertEqual(code, 2) + self.assertIn("--from", err.getvalue()) + + def test_diff_with_unknown_snapshot_id_fails_gracefully(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["diff", str(self.dir), "--from", "does-not-exist-1", "--to", "does-not-exist-2", "--json"]) + self.assertEqual(code, 1) + payload = json.loads(out.getvalue()) + self.assertIn("error", payload) + + def test_diff_between_two_real_snapshots(self) -> None: + main(["docs", str(self.dir)]) + first = json.loads( + self._capture(["history", str(self.dir), "--json"]) + )["snapshots"] + _write(self.dir, "src/extra.py", "def extra():\n return 2\n") + main(["docs", str(self.dir)]) + second = json.loads(self._capture(["history", str(self.dir), "--json"]))["snapshots"] + self.assertGreaterEqual(len(second), len(first)) + if len(second) >= 2: + out = StringIO() + with redirect_stdout(out): + code = main( + ["diff", str(self.dir), "--from", second[-1]["id"], "--to", second[0]["id"], "--json"] + ) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertIn("modules", payload) + self.assertIn("symbols", payload) + + def _capture(self, argv: list[str]) -> str: + out = StringIO() + with redirect_stdout(out): + main(argv) + return out.getvalue() + + # -- ask --------------------------------------------------------------- + + def test_ask_without_verb_is_rejected(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["ask", str(self.dir)]) + self.assertEqual(code, 2) + self.assertIn("ask requires a verb", err.getvalue()) + + def test_ask_callers_json(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["ask", str(self.dir), "callers", "main", "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertIn("results", payload) + self.assertIn("total", payload) + + def test_ask_callers_human_output(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["ask", str(self.dir), "callers", "main"]) + self.assertEqual(code, 0) + self.assertIn("verb=callers", out.getvalue()) + + def test_ask_invalid_verb_is_rejected(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["ask", str(self.dir), "not-a-real-verb"]) + self.assertEqual(code, 2) + self.assertIn("ask failed", err.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_cli_snapshot.py b/tests/python/test_cli_snapshot.py new file mode 100644 index 0000000..6058a7b --- /dev/null +++ b/tests/python/test_cli_snapshot.py @@ -0,0 +1,173 @@ +"""End-to-end CLI coverage for `simplicio-mapper snapshot [build|summary|validate|dag]` +(`simplicio_mapper/cli/_snapshot.py`) -- issue #208 Step 1's observer surface. + +Unlike `tests/python/test_context_snapshot.py` (library-level unit tests for +`build_context_snapshot`), this drives the real CLI entry point (`main()`) +end to end against a scratch repo, exactly as a user invoking +`simplicio-mapper snapshot ...` would, closing a full-CLI-dispatch coverage +gap (`cli/_snapshot.py` was previously untested end to end). +""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from contextlib import redirect_stdout, redirect_stderr +from io import StringIO +from pathlib import Path + +from simplicio_mapper.cli import main + + +def _write(base: Path, rel: str, content: str) -> None: + target = base / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +class SnapshotCliTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + _write(self.root, "package.json", json.dumps({"name": "snapshot-cli-app"})) + _write(self.root, "src/main.py", "def main():\n return 1\n") + + def tearDown(self) -> None: + self._tmp.cleanup() + + def test_build_writes_snapshot_and_prints_human_summary(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "build", "--root", str(self.root)]) + self.assertEqual(code, 0) + dest = self.root / ".simplicio" / "context-snapshot.json" + self.assertTrue(dest.is_file()) + snapshot = json.loads(dest.read_text(encoding="utf-8")) + self.assertEqual(snapshot["schema"], "simplicio.context-snapshot/v1") + self.assertIn("snapshot", out.getvalue()) + self.assertIn("wrote", out.getvalue()) + + def test_build_json_emits_full_snapshot_on_stdout(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "build", "--root", str(self.root), "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["schema"], "simplicio.context-snapshot/v1") + + def test_default_argv_with_no_subcommand_builds(self) -> None: + # `run_snapshot_cli([])` (bare `simplicio-mapper snapshot`) defaults + # to `build` against the current working directory. + import os + + cwd = os.getcwd() + try: + os.chdir(self.root) + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot"]) + finally: + os.chdir(cwd) + self.assertEqual(code, 0) + self.assertTrue((self.root / ".simplicio" / "context-snapshot.json").is_file()) + + def test_summary_without_prior_build_fails_with_guidance(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["snapshot", "summary", "--root", str(self.root)]) + self.assertEqual(code, 1) + self.assertIn("snapshot build", err.getvalue()) + + def test_summary_after_build_reprints_same_snapshot_id(self) -> None: + main(["snapshot", "build", "--root", str(self.root)]) + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "summary", "--root", str(self.root)]) + self.assertEqual(code, 0) + dest = self.root / ".simplicio" / "context-snapshot.json" + snapshot = json.loads(dest.read_text(encoding="utf-8")) + self.assertIn(snapshot["snapshot_id"][:16], out.getvalue()) + + def test_validate_reports_ok_for_a_real_snapshot(self) -> None: + main(["snapshot", "build", "--root", str(self.root)]) + dest = self.root / ".simplicio" / "context-snapshot.json" + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "validate", str(dest)]) + self.assertEqual(code, 0) + self.assertIn("[ok]", out.getvalue()) + + def test_validate_reports_fail_for_a_broken_snapshot(self) -> None: + main(["snapshot", "build", "--root", str(self.root)]) + dest = self.root / ".simplicio" / "context-snapshot.json" + snapshot = json.loads(dest.read_text(encoding="utf-8")) + del snapshot["snapshot_id"] + dest.write_text(json.dumps(snapshot), encoding="utf-8") + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "validate", str(dest)]) + self.assertEqual(code, 1) + self.assertIn("[fail]", out.getvalue()) + + def test_validate_without_paths_prints_usage_and_fails(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["snapshot", "validate", "--root", str(self.root)]) + self.assertEqual(code, 2) + self.assertIn("usage", err.getvalue()) + + def test_dag_build_writes_context_dag_and_journal(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "dag", "--root", str(self.root)]) + self.assertEqual(code, 0) + self.assertTrue((self.root / ".simplicio" / "context-dag.json").is_file()) + self.assertTrue((self.root / ".simplicio" / "context-dag-journal.jsonl").is_file()) + + def test_dag_build_json_emits_journal_entry(self) -> None: + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "dag", "--root", str(self.root), "--json"]) + self.assertEqual(code, 0) + entry = json.loads(out.getvalue()) + self.assertEqual(entry["schema"], "simplicio.context-dag-journal/v1") + self.assertIn("dag_id", entry) + + def test_help_flag_prints_usage_without_side_effects(self) -> None: + # A bare `snapshot --help` (dash-prefixed first token) is caught by + # the *top-level* arg parser's generic `-h`/`--help` handling before + # ever reaching `_snapshot.py` (the top-level parser only hands off + # to `run_snapshot_cli` once it sees a non-dash first token -- see + # `_args.py`'s `command == "snapshot" and not arg.startswith("-")` + # break condition). Exercise `_snapshot.py`'s own `--help` branch via + # `snapshot build --help`, which does break through correctly. + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "build", "--help"]) + self.assertEqual(code, 0) + self.assertIn("usage", out.getvalue()) + self.assertFalse((self.root / ".simplicio").exists()) + + def test_unknown_option_is_rejected(self) -> None: + err = StringIO() + with redirect_stderr(err): + code = main(["snapshot", "build", "--root", str(self.root), "--not-a-real-flag"]) + self.assertEqual(code, 2) + self.assertIn("unknown snapshot option", err.getvalue()) + + def test_refresh_forces_a_full_rescan(self) -> None: + main(["snapshot", "build", "--root", str(self.root)]) + _write(self.root, "src/extra.py", "def extra():\n return 2\n") + out = StringIO() + with redirect_stdout(out): + code = main(["snapshot", "build", "--root", str(self.root), "--refresh", "--json"]) + self.assertEqual(code, 0) + payload = json.loads(out.getvalue()) + self.assertEqual(payload["schema"], "simplicio.context-snapshot/v1") + project_map = json.loads((self.root / ".simplicio" / "project-map.json").read_text(encoding="utf-8")) + self.assertTrue(any(f["path"] == "src/extra.py" for f in project_map["files"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_mapper_parse_benchmark.py b/tests/python/test_mapper_parse_benchmark.py new file mode 100644 index 0000000..d0e8e0a --- /dev/null +++ b/tests/python/test_mapper_parse_benchmark.py @@ -0,0 +1,76 @@ +"""Lightweight timeit-based performance benchmark for the mapper's hottest +path: `build_artifacts()` (project-map + symbol-index + call-graph parse and +emit, `simplicio_mapper/mapper.py`) -- the single call every CLI command +(`map`, `index`, `scan`, `ask`, `orient`, ...) pays. + +This is deliberately a stdlib-only `timeit`-style budget check (no +`pytest-benchmark` dependency added), matching the rest of this repo's +scripts/*_benchmark.py convention of producing one real measured number per +run instead of an unverifiable claim. It asserts a generous wall-clock +ceiling (not a strict SLA) so it stays green on slower CI runners while +still catching an actual multi-x regression. +""" + +from __future__ import annotations + +import time +import unittest +from pathlib import Path + +from simplicio_mapper.mapper import build_artifacts + +FIXTURE_ROOT = ( + Path(__file__).resolve().parents[2] + / "contracts" + / "mapper-artifacts" + / "v1" + / "fixtures" + / "mixed-workspace" + / "source" +) + +# Generous ceiling for a small fixture on a slow/shared CI runner. This is a +# regression tripwire (catches an accidental O(n^2) or repeated-I/O +# regression), not a tight performance SLA. +BUDGET_SECONDS_PER_RUN = 2.0 +WARMUP_RUNS = 1 +MEASURED_RUNS = 5 + + +class MapperParseBenchmarkTest(unittest.TestCase): + def test_build_artifacts_stays_within_time_budget(self) -> None: + self.assertTrue(FIXTURE_ROOT.is_dir(), f"missing fixture: {FIXTURE_ROOT}") + + for _ in range(WARMUP_RUNS): + build_artifacts(str(FIXTURE_ROOT)) + + durations = [] + for _ in range(MEASURED_RUNS): + start = time.perf_counter() + artifacts = build_artifacts(str(FIXTURE_ROOT)) + durations.append(time.perf_counter() - start) + + mean_seconds = sum(durations) / len(durations) + max_seconds = max(durations) + + # A real measured number, not a claim: printed so `pytest -s` (or any + # CI log) surfaces it, and asserted against the budget above. + print( + f"[benchmark] build_artifacts({FIXTURE_ROOT.name}): " + f"mean={mean_seconds * 1000:.3f}ms max={max_seconds * 1000:.3f}ms " + f"over {MEASURED_RUNS} runs (+{WARMUP_RUNS} warmup)" + ) + + self.assertLess( + mean_seconds, + BUDGET_SECONDS_PER_RUN, + f"build_artifacts mean duration {mean_seconds:.3f}s exceeded " + f"the {BUDGET_SECONDS_PER_RUN}s/run budget over {MEASURED_RUNS} runs", + ) + # Sanity check that the parse actually did real work, so a future + # short-circuit bug can't "pass" this benchmark by doing nothing. + self.assertGreater(len(artifacts["project_map"]["files"]), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_orient.py b/tests/python/test_orient.py index bafd85f..f9de49c 100644 --- a/tests/python/test_orient.py +++ b/tests/python/test_orient.py @@ -11,6 +11,7 @@ from simplicio_mapper.cli import main from simplicio_mapper.contract import validate_instance from simplicio_mapper.orient import build_orientation +from simplicio_mapper.retrieval_index import build_retrieval_index, write_retrieval_index from simplicio_mapper.toon import decode_toon @@ -27,18 +28,31 @@ def setUp(self) -> None: encoding="utf-8", ) (self.root / "docs.md").write_text("unrelated release notes\n", encoding="utf-8") + project_map = { + "schema": "simplicio.project-map/v1", + "files": [ + {"path": "src/order_lines.py", "importance": 0.4}, + {"path": "docs.md", "importance": 0.9}, + ], + } (self.root / ".simplicio/project-map.json").write_text( - json.dumps( - { - "schema": "simplicio.project-map/v1", - "files": [ - {"path": "src/order_lines.py", "importance": 0.4}, - {"path": "docs.md", "importance": 0.9}, - ], - } - ), + json.dumps(project_map), encoding="utf-8", ) + # `build_orientation` -> `select_context_targets` only does + # content-aware (not just path/metadata) matching against a + # *persisted* retrieval index (the warm path populated by the real + # `scan`/`index` CLI flow) -- a cold, unindexed call is deliberately + # metadata-only so an ad hoc query never has to reopen every + # candidate file body (see retrieval_index.py::select_context_targets + # and test_task_context_selection.py's + # test_selector_does_not_open_irrelevant_candidate_files). Persist + # a real retrieval index here so this test exercises the supported + # warm path instead of asserting content-match behavior that the + # cold path intentionally does not provide. + write_retrieval_index( + str(self.root), ".simplicio", build_retrieval_index(project_map, root=str(self.root)) + ) def tearDown(self) -> None: self.tmp.cleanup() diff --git a/tests/python/test_visualization_contract.py b/tests/python/test_visualization_contract.py index 6b2b4f9..a9500b8 100644 --- a/tests/python/test_visualization_contract.py +++ b/tests/python/test_visualization_contract.py @@ -1,11 +1,16 @@ import os +import subprocess import tempfile import unittest from pathlib import Path from simplicio_mapper.contract import find_contract_root, validate_payload from simplicio_mapper.mapper import build_artifacts -from simplicio_mapper.visualization import build_visualization_bundle, preview_source +from simplicio_mapper.visualization import _provenance, _safe_remote, build_visualization_bundle, preview_source + + +def _run_git(root: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=str(root), check=True, capture_output=True, text=True) class VisualizationContractTests(unittest.TestCase): @@ -67,5 +72,70 @@ def test_symbol_entity_preview_is_read_only_and_full_export_is_explicit(self) -> self.assertTrue(payload["read_only"]) +class ProvenanceGitTests(unittest.TestCase): + """`_provenance()` git-derived fields (issue #185) only exercise their + real branches against an actual `.git` worktree with a remote -- every + other visualization test in this module uses a plain (non-git) fixture, + so these branches were previously untested.""" + + def setUp(self) -> None: + self.tempdir = tempfile.TemporaryDirectory() + self.root = Path(self.tempdir.name) + (self.root / "src").mkdir() + (self.root / "src" / "app.py").write_text("def greet():\n return 'safe'\n", encoding="utf-8") + try: + _run_git(self.root, "init", "-q", "-b", "main") + _run_git(self.root, "config", "user.email", "test@example.com") + _run_git(self.root, "config", "user.name", "Test") + _run_git(self.root, "remote", "add", "origin", "https://github.com/acme/widgets.git") + _run_git(self.root, "add", "-A") + _run_git(self.root, "commit", "-q", "-m", "initial") + except (OSError, subprocess.CalledProcessError): + self.skipTest("git is unavailable in this environment") + + def tearDown(self) -> None: + self.tempdir.cleanup() + + def test_provenance_reports_remote_host_owner_and_repository(self) -> None: + provenance = _provenance(str(self.root)) + self.assertEqual(provenance["remote_url"], "https://github.com/acme/widgets.git") + self.assertEqual(provenance["host"], "github.com") + self.assertEqual(provenance["owner"], "acme") + self.assertEqual(provenance["repository"], "widgets") + self.assertEqual(provenance["branch"], "main") + self.assertTrue(provenance["commit_sha"]) + self.assertEqual(provenance["clone_type"], "git-clone") + self.assertFalse(provenance["dirty"]) + + def test_provenance_marks_dirty_when_worktree_has_uncommitted_changes(self) -> None: + (self.root / "src" / "app.py").write_text("def greet():\n return 'changed'\n", encoding="utf-8") + provenance = _provenance(str(self.root)) + self.assertTrue(provenance["dirty"]) + + def test_provenance_detects_monorepo_roots(self) -> None: + (self.root / "packages" / "core").mkdir(parents=True) + (self.root / "packages" / "core" / "package.json").write_text("{}", encoding="utf-8") + provenance = _provenance(str(self.root)) + self.assertIn("packages/core", provenance["monorepo_roots"]) + + def test_bundle_from_a_git_worktree_uses_remote_derived_repo_identity(self) -> None: + bundle = build_visualization_bundle(str(self.root), generated_at="1970-01-01T00:00:00.000Z") + self.assertEqual(bundle["provenance"]["host"], "github.com") + + +class SafeRemoteNormalizationTests(unittest.TestCase): + def test_scp_style_remote_is_normalized_to_https(self) -> None: + self.assertEqual(_safe_remote("git@github.com:acme/widgets.git"), "https://github.com/acme/widgets.git") + + def test_ssh_scheme_remote_is_normalized_to_https(self) -> None: + self.assertEqual(_safe_remote("ssh://git@github.com/acme/widgets.git"), "https://github.com/acme/widgets.git") + + def test_https_remote_strips_embedded_credentials_and_query(self) -> None: + self.assertEqual( + _safe_remote("https://user:pass@github.com/acme/widgets.git?x=1#frag"), + "https://github.com/acme/widgets.git", + ) + + if __name__ == "__main__": unittest.main()