feat(search): add tiered BM25 search (briefs/summaries/sources) and taxonomy accessors

- fulltext_index.py: extract shared _BM25Scorer from WikiFullTextIndex
  (no behavior change), add Locator (line/page) on SearchHit, add
  TieredWikiSearch with three independent tiers over summaries/ (briefs +
  full body) and sources/ (whole-file .md + per-page PageIndex .json,
  never the whole long doc as one BM25 unit).
- frontmatter.py: add resolve_description()/body_only() shared helpers
  (kept separate from agent.compiler._resolve_description, which is under
  active unrelated development).
- agent/tools.py: add list_taxonomy_items()/get_taxonomy_item() for
  semantic browsing of persisted concepts/entities (pending candidates
  in PendingTopicsStore are structurally excluded).
- No wiring into CLI/MCP/query-agent yet (follow-up PRs); WikiFullTextIndex
  and agent.tools.search_wiki keep their existing signature/behavior.

Author: Sebastian Braun

openkb/agent/tools.py

@@ -9,10 +9,17 @@
 
 import contextlib
 import json as _json
+from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
+from typing import Literal
 
+from openkb import frontmatter
 from openkb.locks import atomic_write_text
 
+# Maps a taxonomy "kind" to its wiki subdirectory. Single source of truth for
+# list_taxonomy_items/get_taxonomy_item below.
+_TAXONOMY_DIRS: dict[str, str] = {"concept": "concepts", "entity": "entities"}
+
 
 def list_wiki_files(directory: str, wiki_root: str) -> str:
     """List all Markdown files in a wiki subdirectory.
@@ -135,6 +142,103 @@ def get_wiki_page_content(doc_name: str, pages: str, wiki_root: str) -> str:
     return "\n\n".join(parts) + "\n\n"
 
 
+@dataclass(frozen=True)
+class TaxonomyItem:
+    """One persisted concept or entity page (never a pending candidate).
+
+    ``PendingTopicsStore`` (see ``openkb.pending``) buffers not-yet-paged
+    concept/entity candidates separately from the compiled ``.md`` pages
+    under ``concepts/``/``entities/`` — this dataclass, and
+    :func:`list_taxonomy_items`, only ever surface the latter, so a caller
+    never sees an in-progress candidate as if it were a real page.
+    """
+
+    kind: Literal["concept", "entity"]
+    slug: str
+    path: str  # wiki-root-relative, e.g. "concepts/attention.md"
+    brief: str
+    # Entity type (e.g. "person", "organization"); always None for concepts.
+    type: str | None = None
+
+
+def list_taxonomy_items(wiki_root: str, kind: str | None = None) -> list[TaxonomyItem]:
+    """List persisted concept and/or entity pages with their one-line briefs.
+
+    Intended as the first step of the search strategy: browse this compact,
+    semantically-scannable list and let the caller (an LLM) pick the
+    relevant slug(s) by meaning — this is deliberately not a keyword search
+    (see ``search_wiki`` for that, over summaries/sources only).
+
+    Args:
+        wiki_root: Absolute path to the wiki root directory.
+        kind: Restrict to ``"concept"`` or ``"entity"``; ``None`` returns both.
+
+    Returns:
+        Items sorted by kind, then slug. Empty list if the KB has neither
+        directory yet or both are empty.
+
+    Raises:
+        ValueError: *kind* is neither ``None``, ``"concept"``, nor ``"entity"``.
+    """
+    root = Path(wiki_root).resolve()
+    kinds = [kind] if kind else ["concept", "entity"]
+    for k in kinds:
+        if k not in _TAXONOMY_DIRS:
+            raise ValueError(f"Unknown kind {k!r}; expected 'concept' or 'entity'.")
+
+    items: list[TaxonomyItem] = []
+    for k in kinds:
+        directory = root / _TAXONOMY_DIRS[k]
+        if not directory.is_dir():
+            continue
+        for md_file in sorted(directory.glob("*.md")):
+            text = md_file.read_text(encoding="utf-8")
+            fm = frontmatter.parse(text)
+            brief = frontmatter.resolve_description(fm)
+            etype = None
+            if k == "entity":
+                etype = str(fm.get("type") or "").strip().lower() or "other"
+            items.append(
+                TaxonomyItem(
+                    kind=k,  # type: ignore[arg-type]  # validated against _TAXONOMY_DIRS above
+                    slug=md_file.stem,
+                    path=f"{_TAXONOMY_DIRS[k]}/{md_file.name}",
+                    brief=brief,
+                    type=etype,
+                )
+            )
+    return items
+
+
+def get_taxonomy_item(slug: str, wiki_root: str, kind: str | None = None) -> str:
+    """Read a persisted concept or entity page's full Markdown content.
+
+    Args:
+        slug: Page slug (filename without ``.md``), e.g. ``"attention"``.
+        wiki_root: Absolute path to the wiki root directory.
+        kind: ``"concept"`` or ``"entity"`` to disambiguate a same-named
+            slug; ``None`` checks ``concepts/`` first, then ``entities/``.
+
+    Returns:
+        Full file content, or a "not found" message if no match exists in
+        the requested (or either) directory.
+
+    Raises:
+        ValueError: *kind* is neither ``None``, ``"concept"``, nor ``"entity"``.
+    """
+    root = Path(wiki_root).resolve()
+    kinds = [kind] if kind else ["concept", "entity"]
+    for k in kinds:
+        if k not in _TAXONOMY_DIRS:
+            raise ValueError(f"Unknown kind {k!r}; expected 'concept' or 'entity'.")
+
+    for k in kinds:
+        path = (root / _TAXONOMY_DIRS[k] / f"{slug}.md").resolve()
+        if path.is_relative_to(root) and path.exists():
+            return path.read_text(encoding="utf-8")
+    return f"Taxonomy item not found: {slug}"
+
+
 def search_wiki(query: str, wiki_root: str, top_k: int = 5) -> str:
     """Full-text (BM25) search over concepts/entities/summaries wiki pages.
 

openkb/frontmatter.py

@@ -110,3 +110,29 @@ def set_line(fm_block: str, key: str, value: str) -> str:
 def drop_line(fm_block: str, key: str) -> str:
     """Remove any ``key:`` line from a frontmatter block (no-op if absent)."""
     return re.sub(rf"^{re.escape(key)}:.*\n?", "", fm_block, flags=re.MULTILINE)
+
+
+def resolve_description(fm: dict) -> str:
+    """Return a non-empty description string from a parsed frontmatter dict.
+
+    Checks ``description`` first, then the legacy ``brief`` key (pre-migration
+    pages). Returns an empty string when neither key holds a non-blank value.
+    Mirrors ``agent.compiler._resolve_description`` — kept as a separate,
+    dependency-free copy here so callers outside the compiler (search/taxonomy
+    tooling) don't need to import from ``agent.compiler``, which is under
+    active, unrelated development.
+    """
+    for key in ("description", "brief"):
+        v = fm.get(key)
+        if isinstance(v, str) and v.strip():
+            return v.strip()
+    return ""
+
+
+def body_only(text: str) -> str:
+    """Return *text* with any leading YAML frontmatter block removed.
+
+    Returns *text* unchanged when it has no well-formed frontmatter.
+    """
+    parts = split(text)
+    return parts[1] if parts is not None else text