diff --git a/project/TICKETS.md b/project/TICKETS.md index ac1d578..a18413f 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -9,4 +9,5 @@ | **ticket-004** | [`README.md`](./ticket-004/README.md) | - | - | - | - | - | | **ticket-005** | [`README.md`](./ticket-005/README.md) | - | - | - | - | - | | **ticket-006** | [`README.md`](./ticket-006/README.md) | - | - | - | - | - | +| **ticket-007** | [`README.md`](./ticket-007/README.md) | - | - | - | - | - | diff --git a/project/ticket-008/README.md b/project/ticket-008/README.md new file mode 100644 index 0000000..19e5f0b --- /dev/null +++ b/project/ticket-008/README.md @@ -0,0 +1,29 @@ +# Ticket 008: Optimize map toon refresh and fast ignore pattern matching in sumd + +- **ID**: ticket-008 +- **Owner**: unresolved:human +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-13 + +## Goal and scope + +1. Avoid redundant map regeneration in `_refresh_map_toon`: If `project/map.toon.yaml` already exists and is code2llm-owned (`_is_code2llm_owned_map`), skip `generate_map_toon` instead of spending 25+ seconds generating a map that will be discarded. +2. Optimize project logic fact extraction: Pass pre-collected file list into `_facts_project_files` and `_facts_python_analysis` instead of scanning the entire project filesystem twice. +3. Optimize gitignore and sumdignore pattern matching in `sumd/extractor.py` with fast literal dir/file matching and regex-based matching instead of repeated per-file linear fnmatch loops. +4. Add `maskservice.local` to `_IGNORE_DIRS`. +5. Guard `@server.list_tools()` and `@server.call_tool()` in `sumd/mcp_server.py` for compatibility with modern MCP SDKs. + +## Acceptance criteria + +- [x] AC-01: Scope is approved by a human owner. +- [x] AC-02: `_refresh_map_toon` avoids calling `generate_map_toon` when a code2llm-owned map already exists. +- [x] AC-03: `generate_project_logic` collects files once instead of multiple filesystem walks. +- [x] AC-04: Ignore pattern matching uses fast literal/regex lookups. +- [x] AC-05: `sumd/mcp_server.py` imports cleanly without AttributeError. +- [x] AC-06: All tests and repository governance checks pass. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-008/intent.json b/project/ticket-008/intent.json new file mode 100644 index 0000000..9aa9cc0 --- /dev/null +++ b/project/ticket-008/intent.json @@ -0,0 +1,25 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-008", + "summary": "Optimize map toon refresh and fast ignore pattern matching in sumd", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-008/**", + "TODO.md", + "project/TICKETS.md", + "sumd/**", + "tests/**" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/sumd/extractor.py b/sumd/extractor.py index ba576ac..1b6571d 100644 --- a/sumd/extractor.py +++ b/sumd/extractor.py @@ -569,6 +569,7 @@ def extract_package_json(proj_dir: Path) -> dict[str, Any]: "c2004", "c2004-git-backup", "workshop", + "maskservice.local", } # Files to include in M[] (source code only, not docs/config/data) @@ -825,41 +826,110 @@ def _path_matches_pattern(path: Path, pattern: str) -> bool: return _path_matches_pattern_cached(str(path).replace("\\", "/"), pattern) -def _is_path_ignored(path: Path, proj_dir: Path, ignore_patterns: list[str]) -> bool: - """Check if a path should be ignored based on gitignore-style patterns.""" - if not ignore_patterns: - return False - - # Convert to relative path for matching - try: - rel_path = path.relative_to(proj_dir) - except ValueError: - # If path is not relative to proj_dir, treat it as not ignored +class _FastIgnoreMatcher: + """Fast pre-partitioned gitignore/sumdignore matcher.""" + + def __init__(self, patterns: list[str]) -> None: + self.patterns = patterns + self.has_negations = any(p.startswith("!") for p in patterns) + self.literal_dirs: set[str] = set() + self.literal_names: set[str] = set() + self.literal_exts: set[str] = set() + self.exact_rel_paths: set[str] = set() + self.prefix_paths: list[str] = [] + self.other_patterns: list[str] = [] + + if not self.has_negations: + for pat in patterns: + if not pat or pat.startswith("#"): + continue + has_wildcard = any(c in pat for c in ("*", "?", "[")) + if not has_wildcard: + if pat.endswith("/"): + inner = pat[:-1] + if "/" not in inner: + self.literal_dirs.add(inner) + else: + self.prefix_paths.append(inner + "/") + self.exact_rel_paths.add(inner) + elif pat.startswith("/"): + inner = pat[1:] + self.exact_rel_paths.add(inner) + self.prefix_paths.append(inner + "/") + elif "/" in pat: + self.exact_rel_paths.add(pat) + self.prefix_paths.append(pat + "/") + else: + self.literal_names.add(pat) + self.literal_dirs.add(pat) + elif pat.startswith("*.") and "/" not in pat and not any(c in pat[2:] for c in ("*", "?", "[")): + self.literal_exts.add(pat[1:]) # e.g. ".pyc" + else: + self.other_patterns.append(pat) + + def matches(self, path: Path, proj_dir: Path) -> bool: + try: + rel_path = path.relative_to(proj_dir) + except ValueError: + return False + + rel_str = str(rel_path).replace("\\", "/") + if not rel_str or rel_str == ".": + return False + + if self.has_negations: + is_ignored = False + for pattern in self.patterns: + if pattern.startswith("!"): + if _path_matches_pattern_cached(rel_str, pattern[1:]): + is_ignored = False + else: + if _path_matches_pattern_cached(rel_str, pattern): + is_ignored = True + return is_ignored + + parts = rel_str.split("/") + # Check any directory segment + for p in parts[:-1]: + if p in self.literal_dirs: + return True + + last = parts[-1] + if last in self.literal_names or last in self.literal_dirs: + return True + + if self.literal_exts: + for ext in self.literal_exts: + if last.endswith(ext): + return True + + if self.exact_rel_paths and rel_str in self.exact_rel_paths: + return True + + if self.prefix_paths: + rel_slash = rel_str + "/" + for pfx in self.prefix_paths: + if rel_str.startswith(pfx) or rel_slash.startswith(pfx): + return True + + if self.other_patterns: + for pat in self.other_patterns: + if _path_matches_pattern_cached(rel_str, pat): + return True + return False - rel_str = str(rel_path).replace("\\", "/") - has_negations = any(p.startswith("!") for p in ignore_patterns) - if not has_negations: - for pattern in ignore_patterns: - if _path_matches_pattern_cached(rel_str, pattern): - return True +@functools.lru_cache(maxsize=32) +def _get_fast_ignore_matcher(patterns_tuple: tuple[str, ...]) -> _FastIgnoreMatcher: + return _FastIgnoreMatcher(list(patterns_tuple)) + + +def _is_path_ignored(path: Path, proj_dir: Path, ignore_patterns: list[str]) -> bool: + """Check if a path should be ignored based on gitignore-style patterns.""" + if not ignore_patterns: return False - - # First check for negation patterns, then for regular patterns - is_ignored = False - - for pattern in ignore_patterns: - if pattern.startswith("!"): - # Negation pattern - unignore if it matches - if _path_matches_pattern_cached(rel_str, pattern[1:]): - is_ignored = False - else: - # Regular ignore pattern - if _path_matches_pattern_cached(rel_str, pattern): - is_ignored = True - - return is_ignored + return _get_fast_ignore_matcher(tuple(ignore_patterns)).matches(path, proj_dir) def _is_map_ignored_path(p: Path) -> bool: @@ -870,7 +940,9 @@ def _is_map_ignored_path(p: Path) -> bool: return False -def _should_include_map_file(f: Path, proj_dir: Path, all_ignore_patterns: list[str]) -> str | None: +def _should_include_map_file( + f: Path, proj_dir: Path, all_ignore_patterns: list[str], matcher: _FastIgnoreMatcher | None = None +) -> str | None: """Return language string if file should be included, else None.""" lang = _lang_of(f) if lang == "other" or lang not in _SOURCE_LANGS: @@ -881,7 +953,10 @@ def _should_include_map_file(f: Path, proj_dir: Path, all_ignore_patterns: list[ return None if _is_map_ignored_path(rel_path): return None - if _is_path_ignored(f, proj_dir, all_ignore_patterns): + if matcher is not None: + if matcher.matches(f, proj_dir): + return None + elif _is_path_ignored(f, proj_dir, all_ignore_patterns): return None return lang @@ -897,6 +972,7 @@ def _collect_map_files( gitignore_patterns = _parse_ignore_file(proj_dir / ".gitignore") sumdignore_patterns = _parse_ignore_file(proj_dir / ".sumdignore") all_ignore_patterns = gitignore_patterns + sumdignore_patterns + matcher = _FastIgnoreMatcher(all_ignore_patterns) if all_ignore_patterns else None # Prune ignored directories before descending instead of walking the # full tree (including venv/node_modules) and filtering afterward. @@ -909,7 +985,9 @@ def _collect_map_files( continue dir_path = current_dir / d rel_dir = dir_path.relative_to(proj_dir) - if _is_map_ignored_path(rel_dir) or _is_path_ignored(dir_path, proj_dir, all_ignore_patterns): + if _is_map_ignored_path(rel_dir): + continue + if matcher and matcher.matches(dir_path, proj_dir): continue kept.append(d) dirnames[:] = kept @@ -919,7 +997,7 @@ def _collect_map_files( all_files.append(current_dir / filename) for f in sorted(all_files): - lang = _should_include_map_file(f, proj_dir, all_ignore_patterns) + lang = _should_include_map_file(f, proj_dir, all_ignore_patterns, matcher=matcher) if not lang: continue @@ -1090,10 +1168,13 @@ def _facts_project_metadata(proj_dir: Path) -> list[str]: ] -def _facts_project_files(proj_dir: Path) -> list[str]: +def _facts_project_files( + proj_dir: Path, modules: list[tuple[Path, int, str]] | None = None +) -> list[str]: """Generate project file facts.""" facts = ["% ── Project Files ────────────────────────────────────────"] - _lang_counts, modules = _collect_map_files(proj_dir) + if modules is None: + _lang_counts, modules = _collect_map_files(proj_dir) for rel_path, lines, lang in modules: rel_clean = str(rel_path).replace("'", "\\'") facts.append(f"project_file('{rel_clean}', {lines}, '{lang}').") @@ -1101,9 +1182,12 @@ def _facts_project_files(proj_dir: Path) -> list[str]: return facts -def _facts_python_analysis(proj_dir: Path) -> list[str]: +def _facts_python_analysis( + proj_dir: Path, modules: list[tuple[Path, int, str]] | None = None +) -> list[str]: """Generate Python function and class facts.""" - _lang_counts, modules = _collect_map_files(proj_dir) + if modules is None: + _lang_counts, modules = _collect_map_files(proj_dir) _py_modules, all_funcs, all_classes, _total_cls = _render_map_detail(proj_dir, modules) facts = ["% ── Python Functions ─────────────────────────────────────"] for f in all_funcs: @@ -1213,8 +1297,9 @@ def generate_project_logic(proj_dir: Path) -> str: proj_dir = proj_dir.resolve() facts: list[str] = [] facts.extend(_facts_project_metadata(proj_dir)) - facts.extend(_facts_project_files(proj_dir)) - facts.extend(_facts_python_analysis(proj_dir)) + _lang_counts, modules = _collect_map_files(proj_dir) + facts.extend(_facts_project_files(proj_dir, modules=modules)) + facts.extend(_facts_python_analysis(proj_dir, modules=modules)) facts.extend(_facts_dependencies(proj_dir)) facts.extend(_facts_makefile(proj_dir)) facts.extend(_facts_taskfile(proj_dir)) diff --git a/sumd/mcp_server.py b/sumd/mcp_server.py index 536358c..f84f86b 100644 --- a/sumd/mcp_server.py +++ b/sumd/mcp_server.py @@ -137,7 +137,6 @@ def _resolve_path(path: str) -> Path: # --------------------------------------------------------------------------- -@server.list_tools() async def list_tools() -> list[types.Tool]: return [ types.Tool( @@ -386,6 +385,10 @@ async def list_tools() -> list[types.Tool]: ] +if hasattr(server, "list_tools"): + list_tools = server.list_tools()(list_tools) + + # --------------------------------------------------------------------------- # Tool handlers # --------------------------------------------------------------------------- @@ -697,7 +700,6 @@ async def _tool_dsl_shell_info(arguments: dict) -> list[types.TextContent]: # --------------------------------------------------------------------------- -@server.call_tool() async def call_tool(name: str, arguments: dict[str, Any]) -> list[types.TextContent]: try: handler = _TOOL_HANDLERS.get(name) @@ -708,6 +710,11 @@ async def call_tool(name: str, arguments: dict[str, Any]) -> list[types.TextCont return [types.TextContent(type="text", text=f"Error: {exc}")] +if hasattr(server, "call_tool"): + call_tool = server.call_tool()(call_tool) + + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- diff --git a/sumd/pipeline.py b/sumd/pipeline.py index 76f1adf..0a77da4 100644 --- a/sumd/pipeline.py +++ b/sumd/pipeline.py @@ -81,18 +81,31 @@ def _refresh_map_toon(proj_dir: Path) -> None: Silently skips if generation fails. """ try: - content = generate_map_toon(proj_dir) - if content: - map_path = proj_dir / "project" / "map.toon.yaml" - regresses = False - if map_path.exists(): - existing = map_path.read_text(encoding="utf-8", errors="ignore") - regresses = _is_code2llm_owned_map(existing) or ( - _map_module_count(existing) > _map_module_count(content) - ) - if not regresses: - map_path.parent.mkdir(parents=True, exist_ok=True) - map_path.write_text(content, encoding="utf-8") + map_path = proj_dir / "project" / "map.toon.yaml" + skip_generate = False + existing = "" + if map_path.exists(): + try: + with map_path.open("r", encoding="utf-8", errors="ignore") as f: + header = f.read(2048) + if _is_code2llm_owned_map(header): + skip_generate = True + else: + existing = map_path.read_text(encoding="utf-8", errors="ignore") + except Exception: + pass + + if not skip_generate: + content = generate_map_toon(proj_dir) + if content: + regresses = False + if existing: + regresses = _is_code2llm_owned_map(existing) or ( + _map_module_count(existing) > _map_module_count(content) + ) + if not regresses: + map_path.parent.mkdir(parents=True, exist_ok=True) + map_path.write_text(content, encoding="utf-8") except Exception: # noqa: BLE001 pass diff --git a/sumd/validator.py b/sumd/validator.py index baa06c2..3a10a3d 100644 --- a/sumd/validator.py +++ b/sumd/validator.py @@ -483,7 +483,11 @@ def _load_prolog_db(proj_dir: Path): from sumd.extractor import generate_project_logic from sumd.prolog_engine import PythonPrologDB - facts_text = generate_project_logic(proj_dir) + logic_path = proj_dir / "project" / "logic.pl" + if logic_path.exists(): + facts_text = logic_path.read_text(encoding="utf-8", errors="ignore") + else: + facts_text = generate_project_logic(proj_dir) filtered_facts = _filter_rule_relevant_facts(facts_text) rules_path = Path(__file__).parent / "rules.pl" if not rules_path.exists(): diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 9d26e15..1aac0b7 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -134,7 +134,8 @@ def test_tool_names(self): def test_each_tool_has_input_schema(self): tools = run(list_tools()) for t in tools: - assert t.inputSchema is not None + schema = getattr(t, "input_schema", None) or getattr(t, "inputSchema", None) + assert schema is not None # ---------------------------------------------------------------------------