Skip to content

Commit aed1c19

Browse files
committed
Add artifact-aware documentation generation
CodeWiki only documented what reached the dependency graph, and the graph was built from files whose extension is in CODE_EXTENSIONS. Dockerfiles, CI workflows, Makefiles, package manifests, config, schema and script files were dropped before a single Node existed, so no wiki ever described how a system is built, packaged, shipped, configured or tested. This adds a second analysis pass over the same file tree (dependency_analyzer/analyzers/artifact.py) that emits `artifact` nodes: - one node per artifact file (id `<path>::<file>`, source = capped 16 KB head) plus unit nodes for CI jobs, Dockerfile stages, Makefile targets, package.json scripts and pyproject/setup.cfg entry points; - edges from artifacts to the code they reference (COPY/ENTRYPOINT paths, `run:` lines, `python -m`, `make <target>`, `npm run`, `module:function` entry points, compose build contexts), emitted only for fully resolved ids; - caps: per-file head, 40 files per class, total token budget filled in class priority order; lock files and binaries are never read; README/docs stay out unless --with-prose. Pipeline changes so the new nodes flow through unchanged machinery: - repo_analyzer: an ARTIFACT_WHITELIST lets `.github/workflows`, `*.ini`, `bin/*.sh`, `*.gradle` survive the default ignore list (user excludes and .gitignore still win); artifact names added to DEFAULT_INCLUDE_PATTERNS. - leaf_selection: `artifact` is always a valid leaf type; topo_sort no longer lets artifact->code edges prune code leaves. - cluster prompts mark artifact files `(artifact: <class>)` and tell the LLM they are essential; `ensure_artifact_module` inserts a top-level "Build, Deployment and Configuration" module when clustering placed fewer than 80% of the artifact nodes (both the CLI adapter and run() paths). - prompt_template: fence language falls back to `text` instead of raising KeyError for Dockerfile/Makefile/.yml/.toml; artifact groups inline their capped source instead of re-reading the whole file; a <REPOSITORY_ARTIFACTS> index is appended to module prompts and to the repository overview prompt (USER_PROMPT itself is unchanged for the MCP prompt server). - str_replace_editor `view` on a directory no longer hides `.github` and quotes the path. - Node gains `artifact_class`; the graph builder writes temp/artifact_index.json. Flags: --artifacts/--no-artifacts (default on), --artifact-token-budget (200000), --with-prose (off), --artifact-exclude, and an agent_instructions.artifact_exclude key. Tests: tests/test_artifact_analyzer.py (classifier, whitelist, nodes and units, edge resolution, caps, leaf selection, prompt, fallback module).
1 parent f5dac8c commit aed1c19

21 files changed

Lines changed: 1925 additions & 57 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ tests/*
4444
!tests/test_ruby_analyzer.py
4545
!tests/test_processing_order_update.py
4646
!tests/test_leaf_selection.py
47+
!tests/test_artifact_analyzer.py
4748

4849
# Jupyter
4950
*.ipynb

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ def generate(self) -> DocumentationJob:
150150
agent_instructions=self.config.get('agent_instructions'),
151151
use_gitignore=self.config.get('use_gitignore', True),
152152
prompt_caching=self.config.get('prompt_caching', True),
153+
artifacts_enabled=self.config.get('artifacts_enabled', True),
154+
artifact_token_budget=self.config.get('artifact_token_budget', 200_000),
155+
with_prose=self.config.get('with_prose', False),
153156
)
154157

155158
# Run backend documentation generation
@@ -215,6 +218,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
215218
# Import clustering function
216219
from codewiki.src.be.cluster_modules import (
217220
cluster_modules,
221+
ensure_artifact_module,
218222
get_clustering_input_token_count,
219223
super_group_modules,
220224
)
@@ -273,6 +277,10 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
273277
backend_config,
274278
completer=lambda p: doc_generator.backend.complete(p, model=cluster_model),
275279
)
280+
# Artifact nodes the clustering LLM dropped get a fixed module
281+
# so build/CI/config coverage does not depend on the LLM.
282+
if getattr(backend_config, "artifacts_enabled", True):
283+
module_tree = ensure_artifact_module(module_tree, leaf_nodes, components)
276284
# Only freshly clustered trees are deduped: renaming a cached
277285
# key whose .md already exists would orphan the doc.
278286
from codewiki.src.be.module_naming import dedupe_module_tree_names

‎codewiki/cli/commands/generate.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,30 @@ def _find_affected(tree, parent_names=None):
307307
help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to "
308308
"normal calls if the provider rejects them (default: enabled)",
309309
)
310+
@click.option(
311+
"--artifacts/--no-artifacts",
312+
default=True,
313+
help="Document build, CI, container, packaging, manifest, config, schema and "
314+
"script files as part of the dependency graph (default: enabled)",
315+
)
316+
@click.option(
317+
"--artifact-token-budget",
318+
type=int,
319+
default=200_000,
320+
show_default=True,
321+
help="Total token budget for artifact file contents added to the graph",
322+
)
323+
@click.option(
324+
"--with-prose",
325+
is_flag=True,
326+
help="Also read the root README and docs/ as a `prose` artifact class (off by default)",
327+
)
328+
@click.option(
329+
"--artifact-exclude",
330+
type=str,
331+
default=None,
332+
help="Comma-separated patterns skipped by artifact analysis (e.g. 'docker/data/*,config/generated/*')",
333+
)
310334
@click.option(
311335
"--update",
312336
is_flag=True,
@@ -337,6 +361,10 @@ def generate_command(
337361
max_token_per_leaf_module: Optional[int],
338362
max_depth: Optional[int],
339363
prompt_caching: Optional[bool],
364+
artifacts: bool = True,
365+
artifact_token_budget: int = 200_000,
366+
with_prose: bool = False,
367+
artifact_exclude: Optional[str] = None,
340368
update: bool = False,
341369
compare_to: Optional[str] = None
342370
):
@@ -511,13 +539,14 @@ def generate_command(
511539

512540
# Create runtime agent instructions from CLI options
513541
runtime_instructions = None
514-
if any([include, exclude, focus, doc_type, instructions]):
542+
if any([include, exclude, focus, doc_type, instructions, artifact_exclude]):
515543
runtime_instructions = AgentInstructions(
516544
include_patterns=parse_patterns(include) if include else None,
517545
exclude_patterns=parse_patterns(exclude) if exclude else None,
518546
focus_modules=parse_patterns(focus) if focus else None,
519547
doc_type=doc_type,
520548
custom_instructions=instructions,
549+
artifact_exclude=parse_patterns(artifact_exclude) if artifact_exclude else None,
521550
)
522551

523552
if verbose:
@@ -531,6 +560,8 @@ def generate_command(
531560
logger.debug(f"Doc type: {doc_type}")
532561
if instructions:
533562
logger.debug(f"Custom instructions: {instructions}")
563+
if artifact_exclude:
564+
logger.debug(f"Artifact exclude patterns: {parse_patterns(artifact_exclude)}")
534565

535566
# Log max token settings if verbose
536567
if verbose:
@@ -546,6 +577,7 @@ def generate_command(
546577
logger.debug(f"Max depth: {effective_max_depth}")
547578
logger.debug(f"Use gitignore: {effective_use_gitignore}")
548579
logger.debug(f"Prompt caching: {effective_prompt_caching}")
580+
logger.debug(f"Artifacts: {artifacts} (token budget {artifact_token_budget}, prose {with_prose})")
549581

550582
# Get agent instructions (merge runtime with persistent)
551583
agent_instructions_dict = None
@@ -557,6 +589,7 @@ def generate_command(
557589
focus_modules=runtime_instructions.focus_modules or (config.agent_instructions.focus_modules if config.agent_instructions else None),
558590
doc_type=runtime_instructions.doc_type or (config.agent_instructions.doc_type if config.agent_instructions else None),
559591
custom_instructions=runtime_instructions.custom_instructions or (config.agent_instructions.custom_instructions if config.agent_instructions else None),
592+
artifact_exclude=runtime_instructions.artifact_exclude or (config.agent_instructions.artifact_exclude if config.agent_instructions else None),
560593
)
561594
agent_instructions_dict = merged.to_dict()
562595
elif config.agent_instructions and not config.agent_instructions.is_empty():
@@ -587,6 +620,10 @@ def generate_command(
587620
'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore,
588621
# Prompt caching setting (runtime override takes precedence)
589622
'prompt_caching': prompt_caching if prompt_caching is not None else config.prompt_caching,
623+
# Artifact-aware generation (runtime-only flags)
624+
'artifacts_enabled': artifacts,
625+
'artifact_token_budget': artifact_token_budget,
626+
'with_prose': with_prose,
590627
},
591628
verbose=verbose,
592629
generate_html=github_pages,

‎codewiki/cli/models/config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class AgentInstructions:
4040
focus_modules: Optional[List[str]] = None # e.g., ["src/core", "src/api"]
4141
doc_type: Optional[str] = None # e.g., "api", "architecture", "user-guide"
4242
custom_instructions: Optional[str] = None # Free-form instructions
43+
artifact_exclude: Optional[List[str]] = None # e.g., ["docker/data/*"] skipped by artifact analysis
4344

4445
def to_dict(self) -> dict:
4546
"""Convert to dictionary, excluding None values."""
@@ -48,6 +49,8 @@ def to_dict(self) -> dict:
4849
result['include_patterns'] = self.include_patterns
4950
if self.exclude_patterns:
5051
result['exclude_patterns'] = self.exclude_patterns
52+
if self.artifact_exclude:
53+
result['artifact_exclude'] = self.artifact_exclude
5154
if self.focus_modules:
5255
result['focus_modules'] = self.focus_modules
5356
if self.doc_type:
@@ -65,13 +68,15 @@ def from_dict(cls, data: dict) -> 'AgentInstructions':
6568
focus_modules=data.get('focus_modules'),
6669
doc_type=data.get('doc_type'),
6770
custom_instructions=data.get('custom_instructions'),
71+
artifact_exclude=data.get('artifact_exclude'),
6872
)
6973

7074
def is_empty(self) -> bool:
7175
"""Check if all fields are empty/None."""
7276
return not any([
7377
self.include_patterns,
7478
self.exclude_patterns,
79+
self.artifact_exclude,
7580
self.focus_modules,
7681
self.doc_type,
7782
self.custom_instructions,

‎codewiki/src/be/agent_tools/str_replace_editor.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import json
88
import re
9+
import shlex
910
import subprocess
1011
import sys
1112
from collections import defaultdict
@@ -487,8 +488,11 @@ def view(self, path: Path, view_range: Optional[List[int]] = None):
487488
self.logs.append("The `view_range` parameter is not allowed when `path` points to a directory.")
488489
return
489490

491+
# Hidden entries are skipped except `.github` (CI workflows are
492+
# documentation-relevant artifacts the agent must be able to find).
490493
out = subprocess.run(
491-
rf"find {path} -maxdepth 2 -not -path '*/\.*'",
494+
rf"find {shlex.quote(str(path))} -maxdepth 2 "
495+
r"\( -not -path '*/.*' -o -name .github -o -path '*/.github/*' \)",
492496
shell=True,
493497
stdout=subprocess.PIPE,
494498
stderr=subprocess.PIPE,
@@ -499,7 +503,7 @@ def view(self, path: Path, view_range: Optional[List[int]] = None):
499503

500504
if not stderr:
501505
stdout = stdout.replace(str(path), self._get_display_path(path))
502-
stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items:\n{stdout}\n"
506+
stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items (except .github):\n{stdout}\n"
503507
self.logs.append(stdout)
504508
return
505509

@@ -747,14 +751,14 @@ async def str_replace_editor(
747751
"""
748752
Custom editing tool for viewing, creating and editing files
749753
* State is persistent across command calls and discussions with the user
750-
* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep.
754+
* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep.
751755
* The `create` command cannot be used if the specified `path` already exists as a file
752756
* If a `command` generates a long output, it will be truncated and marked with `<response clipped>`
753757
* The `undo_edit` command will revert the last edit made to the file at `path`
754758
* Only `view` command is allowed when `working_dir` is `repo`.
755759
756760
Args:
757-
working_dir: The working directory to use. Choose `repo` to work with the repository files, or `docs` to work with the generated documentation files.
761+
working_dir: The working directory to use. Choose `repo` to view repository files (source code, and build/CI/container/manifest/config artifacts such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml), or `docs` to work with the generated documentation files.
758762
command: The command to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`.
759763
path: Path to file or directory, e.g. `./chat_core.md` or `./agents/`
760764
file: Alias for `path` parameter (for compatibility with some models)
@@ -805,7 +809,7 @@ async def str_replace_editor(
805809
description="""
806810
Custom editing tool for viewing, creating and editing files
807811
* State is persistent across command calls and discussions with the user
808-
* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep.
812+
* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep.
809813
* The `create` command cannot be used if the specified `path` already exists as a file
810814
* If a `command` generates a long output, it will be truncated and marked with `<response clipped>`
811815
* The `undo_edit` command will revert the last edit made to the file at `path`

‎codewiki/src/be/caw_toolkit.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ async def read_code_components(self, component_ids: list[str]) -> str:
123123
description=(
124124
"Custom editing tool for viewing, creating and editing files.\n"
125125
"* If `path` is a file, `view` displays the result of applying `cat -n`. "
126-
"If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep.\n"
126+
"If `path` is a directory, `view` lists non-hidden files and directories (plus `.github`) up to 2 levels deep.\n"
127127
"* The `create` command cannot be used if the specified `path` already exists as a file.\n"
128128
"* If a `command` generates a long output, it will be truncated and marked with `<response clipped>`.\n"
129129
"* The `undo_edit` command will revert the last edit made to the file at `path`.\n"

‎codewiki/src/be/cluster_modules.py‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,11 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str
4343
potential_core_components = ""
4444
potential_core_components_with_code = ""
4545
for file, leaf_nodes in dict(sorted(leaf_nodes_by_file.items())).items():
46-
potential_core_components += f"# {file}\n"
47-
potential_core_components_with_code += f"# {file}\n"
46+
header = f"# {file}"
47+
if all(components[n].component_type == "artifact" for n in leaf_nodes):
48+
header += f" (artifact: {components[leaf_nodes[0]].artifact_class or 'config'})"
49+
potential_core_components += f"{header}\n"
50+
potential_core_components_with_code += f"{header}\n"
4851
for leaf_node in leaf_nodes:
4952
potential_core_components += f"\t{leaf_node}\n"
5053
potential_core_components_with_code += f"\t{leaf_node}\n"
@@ -603,3 +606,85 @@ def super_group_modules(
603606
len(subsystems),
604607
)
605608
return result
609+
610+
611+
# --------------------------------------------------------------------------- #
612+
# Guaranteed artifact module
613+
# --------------------------------------------------------------------------- #
614+
615+
ARTIFACT_MODULE_NAME = "Build, Deployment and Configuration"
616+
# Insert the fallback module when clustering kept less than this share of the
617+
# artifact leaf nodes.
618+
ARTIFACT_MIN_SHARE = 0.8
619+
620+
621+
def collect_module_tree_component_ids(module_tree: Dict[str, Any]) -> set:
622+
"""Return every component id referenced anywhere in ``module_tree``."""
623+
ids: set = set()
624+
625+
def _walk(tree: Dict[str, Any]) -> None:
626+
for module_info in tree.values():
627+
if not isinstance(module_info, dict):
628+
continue
629+
ids.update(module_info.get("components", []) or [])
630+
children = module_info.get("children", {})
631+
if isinstance(children, dict):
632+
_walk(children)
633+
634+
_walk(module_tree)
635+
return ids
636+
637+
638+
def ensure_artifact_module(
639+
module_tree: Dict[str, Any],
640+
leaf_nodes: List[str],
641+
components: Dict[str, Node],
642+
min_share: float = ARTIFACT_MIN_SHARE,
643+
) -> Dict[str, Any]:
644+
"""Guarantee that artifact leaf nodes are documented.
645+
646+
Clustering is an LLM call and may drop or scatter artifact nodes despite
647+
the prompt. If fewer than ``min_share`` of the artifact leaf nodes landed
648+
in ``module_tree``, add a fixed top-level module holding every unassigned
649+
artifact node. Returns ``module_tree`` unchanged in whole-repository mode
650+
(empty tree: one agent documents all leaf nodes anyway).
651+
"""
652+
if not module_tree:
653+
return module_tree
654+
artifact_leaves = [
655+
n for n in leaf_nodes
656+
if n in components and components[n].component_type == "artifact"
657+
]
658+
if not artifact_leaves:
659+
return module_tree
660+
assigned = collect_module_tree_component_ids(module_tree)
661+
unassigned = [n for n in artifact_leaves if n not in assigned]
662+
share = 1.0 - len(unassigned) / len(artifact_leaves)
663+
logger.info(
664+
"Artifact coverage after clustering: %d/%d artifact leaf nodes assigned (%.0f%%)",
665+
len(artifact_leaves) - len(unassigned), len(artifact_leaves), share * 100,
666+
)
667+
if not unassigned or share >= min_share:
668+
return module_tree
669+
670+
from codewiki.src.be.dependency_analyzer.analyzers.artifact import CLASS_PRIORITY
671+
from codewiki.src.be.module_naming import collect_module_tree_names
672+
673+
def _order(node_id: str):
674+
node = components[node_id]
675+
cls = node.artifact_class or "config"
676+
rank = CLASS_PRIORITY.index(cls) if cls in CLASS_PRIORITY else len(CLASS_PRIORITY)
677+
return (rank, node.relative_path, node_id)
678+
679+
unassigned.sort(key=_order)
680+
name = resolve_unique_name(ARTIFACT_MODULE_NAME, None, collect_module_tree_names(module_tree))
681+
module_tree[name] = {
682+
"path": _common_path_prefix([components[n].relative_path for n in unassigned]) or ".",
683+
"components": unassigned,
684+
"children": {},
685+
}
686+
logger.info(
687+
"Artifact coverage %.0f%% < %.0f%%; inserted top-level module '%s' with %d components",
688+
share * 100, min_share * 100, name, len(unassigned),
689+
)
690+
return module_tree

‎codewiki/src/be/dependency_analyzer/analysis/analysis_service.py‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import logging
1010
import traceback
1111
from pathlib import Path
12-
from typing import Any
12+
from typing import TYPE_CHECKING, Any
1313

1414
from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer
1515
from codewiki.src.be.dependency_analyzer.analysis.cloning import (
@@ -22,6 +22,9 @@
2222
from codewiki.src.be.dependency_analyzer.models.core import Repository
2323
from codewiki.src.be.dependency_analyzer.utils.security import assert_safe_path, safe_open_text
2424

25+
if TYPE_CHECKING: # pragma: no cover
26+
from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions
27+
2528
logger = logging.getLogger(__name__)
2629

2730

@@ -288,14 +291,20 @@ def _read_readme_file(self, repo_dir: str) -> str | None:
288291
logger.debug("No README file found in repository root.")
289292
return None
290293

291-
def _analyze_call_graph(self, file_tree: dict[str, Any], repo_dir: str) -> dict[str, Any]:
294+
def _analyze_call_graph(
295+
self,
296+
file_tree: dict[str, Any],
297+
repo_dir: str,
298+
artifact_options: "ArtifactOptions | None" = None,
299+
) -> dict[str, Any]:
292300
"""
293301
Perform multi-language call graph analysis.
294302
295-
This method will be expanded to handle:
296-
- Python AST analysis (current)
297-
- JavaScript/TypeScript AST analysis (planned)
298-
- Additional language support (future)
303+
When ``artifact_options`` is given and enabled, a second pass turns
304+
build/CI/container/manifest/config files in the same file tree into
305+
``artifact`` nodes (see ``analyzers/artifact.py``) and appends them to
306+
the ``functions``/``relationships`` lists so the rest of the pipeline
307+
treats them like any other component.
299308
"""
300309
logger.debug("Extracting code files from file tree...")
301310
code_files = self.call_graph_analyzer.extract_code_files(file_tree)
@@ -311,6 +320,19 @@ def _analyze_call_graph(self, file_tree: dict[str, Any], repo_dir: str) -> dict[
311320
result["call_graph"]["supported_languages"] = self._get_supported_languages()
312321
result["call_graph"]["unsupported_files"] = len(code_files) - len(supported_files)
313322

323+
if artifact_options is not None and artifact_options.enabled:
324+
from codewiki.src.be.dependency_analyzer.analyzers.artifact import analyze_artifacts
325+
326+
artifacts = analyze_artifacts(
327+
file_tree, repo_dir, result.get("functions", []), artifact_options
328+
)
329+
result.setdefault("functions", []).extend(n.model_dump() for n in artifacts.nodes)
330+
result.setdefault("relationships", []).extend(
331+
r.model_dump() for r in artifacts.relationships
332+
)
333+
result["artifact_index"] = artifacts.index
334+
result["call_graph"]["artifact_nodes"] = len(artifacts.nodes)
335+
314336
return result
315337

316338
def _filter_supported_languages(self, code_files: list[dict]) -> list[dict]:

0 commit comments

Comments
 (0)