Make excludes an index filter, with an app-wide list and per-source overrides

README.md

@@ -294,13 +294,17 @@ the UI writes the same config file you can edit by hand.
 
 Press `:` to open Settings, move to **Collections**, then:
 
-- **Add a collection**: choose **Add collection** and fill the wizard:
-  **Name**, a **Source path** (a folder; `~/…` is fine), the file types to
-  **Include** and patterns to **Exclude**, an optional markdown
-  **Frontmatter filter**, and a **Follow symlinks** toggle. Press **Ctrl+S** to
-  save; you land on the new collection's page, where **Update index** builds it.
+- **Add a collection**: choose **Add collection** and fill in:
+  - **Collection**: its **Name**.
+  - **Source**, its first folder: a **Path** (`~/…` is fine), **Follow
+    symlinks**, then what gets indexed from it: **Index filters** (file
+    types, excluded paths, tags, size, dates), the **Frontmatter rule**, and
+    **Restrict to these paths**.
+
+  Press **Ctrl+S** to save; you land on the new collection's page, where
+  **Update index** builds it.
 - **Add a source to an existing collection**: open the collection, then
-  **Sources → Add source**, and set the path, includes/excludes, an optional
+  **Sources → Add source**, and set the path, its Index filters, an optional
   per-source app, and (for Obsidian) the vault name. **Ctrl+S** saves and
   returns; **Ctrl+A** saves and adds another.
 - **How editing works**:
@@ -359,9 +363,10 @@ Index filters** edits the defaults; a source's **Index filters** row edits it.
 
 | Filter | What it does |
 | --- | --- |
-| *(always on)* | Hidden files and folders (`.foo`) are skipped, whatever the filters say. Only an `includes` glob naming a dot-prefixed component (`.obsidian/**`) admits one, and then only the paths that glob itself matches. |
+| *(always on)* | Hidden files and folders (`.foo`) are skipped, whatever the filters say. Only an `includes` glob naming a dot-prefixed component (`.obsidian/**`) admits one, and then only the paths that glob itself matches. A file whose own name starts with `.` still meets the default `**/.*` exclude: to index one, give the source its own `excludes` without that glob. |
 | `respect_gitignore` | Honours every `.gitignore` down the tree, with git's rules: negation, directory patterns, nearest file wins. Not `.git/info/exclude` and not `core.excludesFile`: neither is in the tree, so honouring them would make one corpus index differently on two machines. Use `.fndignore` for a rule that is yours alone. **On by default.** |
 | `respect_fndignore` | The same syntax in a `.fndignore`, read only by fnd: how to hide something from search without hiding it from git. **On by default.** |
+| `excludes` | Path globs never indexed; a folder a `name/**` glob covers is pruned unopened. Settings shows them as **Excluded paths** (presets such as Node modules) plus **Excluded globs** you type. Set once in the defaults for every source; a source's own list replaces it. **System files by default**: `**/.*`, `.DS_Store`, `Thumbs.db`, `desktop.ini` and `.git`. |
 | `include_tags` | Index only files carrying one of these tags; the tag rows' `●`. Empty means the tag is not consulted. |
 | `exclude_tags` | Tags that keep a file out; the tag rows' `⊘`. From any source fnd reads: macOS Finder tags and a note's YAML `tags:`. Defaults to `["no_index"]`, so tagging a file `no_index` either way keeps it out. |
 | `kinds` | Restrict to given file types. Empty means every supported type. A source's `includes` folds into this when it names every suffix of a type (`["**/*.md", "**/*.markdown"]`, not `["**/*.md"]` alone). Anything else stays a glob. |
@@ -420,9 +425,11 @@ fuzzy_enabled = true       # auto-fuzzy in the cascade fallback (toggle with Ctr
 [[collections.papers.sources]]
 path     = "~/Documents/Research"
 includes = ["**/*.pdf", "**/*.md"]        # ORed globs; see filters.kinds below
-excludes = ["**/.git/**", "archive/**"]
 follow_symlinks = false
 
+[collections.papers.sources.filters]
+excludes = ["**/.git/**", "archive/**"]   # replaces [defaults.filters] excludes
+
 [[collections.papers.sources]]
 path = "~/Notes"
 
@@ -435,6 +442,7 @@ frontmatter = "Status == 'published' AND NOT ('private' in tags)"  # notes only
 [defaults.filters]
 respect_gitignore = true            # honour .gitignore, with git's own rules
 respect_fndignore = true            # same syntax, read only by fnd
+excludes          = ["**/node_modules/**"]  # replaces the System files default
 exclude_tags      = ["no_index"]    # Finder tags and YAML tags:, see below
 # include_tags    = ["reference"]   # index only files carrying one of these
 # kinds           = ["md", "pdf"]   # omit to index every supported type

fnd/cli.py

@@ -669,7 +669,7 @@ def collection_add(
     exclude: list[str] = typer.Option(
         [],
         "--exclude",
-        help="Glob to exclude. Repeatable.",
+        help="Glob to exclude. Repeatable. Replaces the app-wide excludes (System files by default).",
     ),
     filter: str | None = typer.Option(
         None,
@@ -712,14 +712,14 @@ def collection_add(
     cfg_path.parent.mkdir(parents=True, exist_ok=True)
     from fnd.config import SourceFilters
 
+    # The current shape, not a source-level `excludes` or `frontmatter_filter`:
+    # a new write should not create something the next migration has to move.
+    own = SourceFilters(frontmatter=filter or None, excludes=list(exclude) or None)
     new_source = SourceConfig(
         path=source[0],
         includes=list(include),
-        excludes=list(exclude),
         follow_symlinks=follow_symlinks,
-        # The current shape, not the deprecated `frontmatter_filter`: a new
-        # write should not create something the next migration has to move.
-        filters=SourceFilters(frontmatter=filter) if filter else None,
+        filters=own if filter or exclude else None,
     )
     # Read before the write: the sibling set is what the new source is being
     # compared against, and after the write it contains the new source itself.

fnd/config.py

@@ -55,7 +55,7 @@ def default_config_path() -> Path:
 # any directory anywhere in the tree with one of these names is skipped
 # entirely (no scandir into it). Comprehensive because the typical user
 # is a developer with many code trees on disk and indexing build/cache
-# output is never useful. Per-source ``excludes`` still apply on top;
+# output is never useful. The ``excludes`` filter still applies on top;
 # users can disable this list with ``defaults.skip_junk_dirs = false``
 # or extend it via ``defaults.extra_junk_dirs``.
 DEFAULT_JUNK_DIRS: frozenset[str] = frozenset(
@@ -162,37 +162,31 @@ def default_config_path() -> Path:
     spec.id: f"{spec.label} ({'/'.join(spec.suffixes)})" for spec in KIND_SPECS
 }
 
-# Exclude presets for the Add Collection wizard's Excludes multi-select. Each
-# preset defines a set of globs and a default toggle state. Presets marked
-# default=True are pre-ticked in the UI.
+# The presets Index filters › Excluded paths offers to tick; each is a set of
+# globs a preset counts as on for when all are in a source's excludes.
 EXCLUDES_PRESETS: dict[str, dict[str, Any]] = {
     # Hidden names are pruned by the walk whatever this says, so the label
     # promised something unticking it cannot give back. What it uniquely adds
     # is desktop.ini, plus hidden paths an include glob deliberately admitted.
     "hidden": {
         "label": "System files (hidden are always skipped)",
         "globs": ["**/.*", "**/.DS_Store", "**/Thumbs.db", "**/desktop.ini", "**/.git/**"],
-        "default": True,
     },
     "node_modules": {
         "label": "Node modules",
         "globs": ["**/node_modules/**"],
-        "default": False,
     },
     "python_caches": {
         "label": "Python caches",
         "globs": ["**/__pycache__/**", "**/*.pyc"],
-        "default": False,
     },
     "build_artefacts": {
         "label": "Build artefacts",
         "globs": ["**/dist/**", "**/build/**"],
-        "default": False,
     },
     "obsidian_meta": {
         "label": "Obsidian metadata",
         "globs": ["**/.obsidian/**"],
-        "default": False,
     },
 }
 
@@ -275,6 +269,10 @@ class DefaultFilters(_ConfigModel):
     """Honour .fndignore files. Same syntax as .gitignore, but they apply to fnd
     alone, so they exclude something from search without excluding it from git."""
 
+    excludes: list[str] = Field(default_factory=lambda: list(EXCLUDES_PRESETS["hidden"]["globs"]))
+    """Glob patterns whose files are never indexed; a folder a `name/**` glob
+    covers is pruned unopened. Every source inherits these unless it sets its own."""
+
     include_tags: list[str] | dict[str, list[str]] = Field(default_factory=list)
     """Index only files carrying one of these tags. Empty means no tag is needed."""
 
@@ -342,6 +340,7 @@ class SourceFilters(_ConfigModel):
 
     respect_gitignore: bool | None = None
     respect_fndignore: bool | None = None
+    excludes: list[str] | None = None
     include_tags: list[str] | dict[str, list[str]] | None = None
     exclude_tags: list[str] | dict[str, list[str]] | None = None
     kinds: list[str] | None = None
@@ -430,9 +429,6 @@ class SourceConfig(_ConfigModel):
     """Glob patterns a file must match to be indexed. Empty means every
     supported type. Matched against the path relative to this source root."""
 
-    excludes: list[str] = Field(default_factory=list)
-    """Glob patterns that keep a file out, even when it matched an include."""
-
     follow_symlinks: bool = False
     """Follow symlinks in this source, and allow the root to be one. Off by
     default."""
@@ -461,6 +457,32 @@ class SourceConfig(_ConfigModel):
     """Populated by :class:`Config` once the defaults are known. Private so it
     never reaches the TOML writer, which must emit only what the user set."""
 
+    @model_validator(mode="before")
+    @classmethod
+    def _fold_legacy_excludes(cls, data: Any) -> Any:
+        """A source-level `excludes` list is the filters' `excludes`, said in the old place."""
+        from fnd.config_migrations import legacy_excludes_override
+
+        if not isinstance(data, dict) or "excludes" not in data:
+            return data
+        data = dict(data)
+        legacy = legacy_excludes_override(data.pop("excludes"))
+        if legacy is not None:
+            filters = data.get("filters")
+            filters = (
+                dict(filters)
+                if isinstance(filters, dict)
+                else (filters.model_dump(exclude_none=True) if filters is not None else {})
+            )
+            filters.setdefault("excludes", list(legacy))
+            data["filters"] = filters
+        return data
+
+    @property
+    def excludes(self) -> list[str]:
+        """The excludes this source walks with: its own, or the inherited ones."""
+        return list(self.effective_filters.excludes)
+
     @model_validator(mode="after")
     def _absorb_type_globs(self) -> SourceConfig:
         """``includes = ["**/*.md"]`` is ``filters.kinds = ["md"]`` said twice.
@@ -569,7 +591,7 @@ class CollectionConfig(_ConfigModel):
     A collection can be configured in two equivalent shapes:
 
     * **New (recommended):** ``[[collections.X.sources]]``. One TOML table
-      per source, each with its own includes/excludes/frontmatter_filter.
+      per source, each with its own includes and filters.
     * **Legacy:** flat ``roots = [...]``, ``includes = [...]``,
       ``excludes = [...]`` on the collection. Loader normalises this into
       a single implicit source so downstream code only sees the new shape.
@@ -627,7 +649,7 @@ def _normalise_sources(self) -> CollectionConfig:
                 SourceConfig(
                     path=root,
                     includes=list(self.includes),
-                    excludes=list(self.excludes),
+                    filters=_legacy_filters(self.excludes),
                     follow_symlinks=self.follow_symlinks,
                 )
                 for root in self.roots
@@ -638,6 +660,13 @@ def _normalise_sources(self) -> CollectionConfig:
         return self
 
 
+def _legacy_filters(excludes: list[str]) -> SourceFilters | None:
+    from fnd.config_migrations import legacy_excludes_override
+
+    override = legacy_excludes_override(excludes)
+    return SourceFilters(excludes=override) if override is not None else None
+
+
 class RankingProfileConfig(_ConfigModel):
     """User-facing knobs that map onto :class:`fnd.rerank.RankingProfile`.
 

fnd/config_migrations.py

@@ -25,7 +25,7 @@
 from fnd.globs import names_hidden
 
 #: What this build writes. Bump when adding a migration.
-CONFIG_VERSION: Final = 2
+CONFIG_VERSION: Final = 3
 
 VERSION_KEY: Final = "config_version"
 
@@ -84,6 +84,42 @@ def _to_v2(raw: MutableMapping[str, Any]) -> list[str]:
     ]
 
 
+#: What the pre-v3 Add collection pre-ticked, frozen: the master list's default now.
+V2_WIZARD_EXCLUDES: Final = frozenset(
+    {"**/.*", "**/.DS_Store", "**/Thumbs.db", "**/desktop.ini", "**/.git/**"}
+)
+
+
+def legacy_excludes_override(globs: Any) -> list[str] | None:
+    """A pre-v3 excludes list as a source override; ``None`` inherits the master.
+
+    Empty was never a choice to exclude nothing, and the old wizard's default is
+    the master's now, in either spelling (a released config is respelled by v2)."""
+    listed = [globs] if isinstance(globs, str) else [str(g) for g in globs or ()]
+    wizard = list(V2_WIZARD_EXCLUDES)
+    spellings = {V2_WIZARD_EXCLUDES, frozenset(_respelled(wizard, include=False, narrowed=[]))}
+    if not listed or frozenset(listed) in spellings:
+        return None
+    return listed
+
+
+def _to_v3(raw: MutableMapping[str, Any]) -> None:
+    """Move each source's ``excludes`` into its ``filters``, where a master list
+    in ``[defaults.filters]`` can be inherited and overridden like every filter."""
+    for collection in (raw.get("collections") or {}).values():
+        if not isinstance(collection, dict):
+            continue
+        for source in collection.get("sources") or ():
+            if not isinstance(source, dict) or "excludes" not in source:
+                continue
+            legacy = legacy_excludes_override(source.pop("excludes"))
+            if legacy is None:
+                continue
+            filters = source.setdefault("filters", {})
+            if isinstance(filters, dict):
+                filters.setdefault("excludes", legacy)
+
+
 _MAX_SPELLINGS: Final = 16
 """Path globs one legacy glob may become. Each ``?`` roughly doubles the count."""
 
@@ -322,6 +358,7 @@ def _collapse_globstars(glob: str) -> str:
 MIGRATIONS: Final[tuple[tuple[int, str, Transform], ...]] = (
     (1, "Adopt the canonical layout and record a config version", _to_v1),
     (2, "Respell globs so they still select the files they did", _to_v2),
+    (3, "Move each source's excludes into its index filters", _to_v3),
 )
 
 

fnd/config_render.py

@@ -73,7 +73,6 @@
 _SOURCE_ORDER = (
     "path",
     "includes",
-    "excludes",
     "follow_symlinks",
     "frontmatter_filter",
     "app",

fnd/filters/excludes.py

@@ -0,0 +1,56 @@
+"""Exclude globs as the Index filters screen shows them: presets to tick, globs typed by hand.
+
+A preset counts as on when every glob it ships is present; whatever is left over
+was typed. Edits keep the list in the order written: an inherited list rewritten
+in any other order would save as the source's own.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+
+def split_presets(globs: Iterable[str]) -> tuple[list[str], list[str]]:
+    """``(preset keys on, globs typed by hand)`` for an excludes list."""
+    from fnd.config import EXCLUDES_PRESETS
+
+    remaining = list(dict.fromkeys(globs))
+    on: list[str] = []
+    for key, preset in EXCLUDES_PRESETS.items():
+        shipped = preset["globs"]
+        if all(g in remaining for g in shipped):
+            on.append(key)
+            remaining = [g for g in remaining if g not in shipped]
+    return on, remaining
+
+
+def _globs_of(keys: Iterable[str]) -> list[str]:
+    from fnd.config import EXCLUDES_PRESETS
+
+    picked = set(keys)
+    return [g for k, p in EXCLUDES_PRESETS.items() if k in picked for g in p["globs"]]
+
+
+def retick(excludes: Sequence[str], keys: Iterable[str]) -> tuple[str, ...]:
+    """``excludes`` with exactly the presets ``keys`` on: unticked globs leave, new ones follow."""
+    on, _typed = split_presets(excludes)
+    wanted = set(keys)
+    if set(on) == wanted:
+        return tuple(excludes)
+    dropped = set(_globs_of(set(on) - wanted)) - set(_globs_of(wanted))
+    kept = [g for g in excludes if g not in dropped]
+    return tuple(dict.fromkeys([*kept, *_globs_of(wanted - set(on))]))
+
+
+def retype(excludes: Sequence[str], typed: Sequence[str]) -> tuple[str, ...]:
+    """``excludes`` with its typed globs replaced by ``typed``; the presets stay put."""
+    on, _old = split_presets(excludes)
+    shipped = set(_globs_of(on))
+    return tuple(dict.fromkeys([*(g for g in excludes if g in shipped), *typed]))
+
+
+def invalid_glob(glob: str) -> bool:
+    """Whether ``glob`` cannot compile, so it would exclude nothing."""
+    from fnd.globs import _config_regex  # pyright: ignore[reportPrivateUsage]
+
+    return _config_regex(glob) is None

fnd/filters/model.py

@@ -119,6 +119,7 @@ def spec_from_resolved(resolved: object) -> FilterSpec:
         modified_before=resolved.modified_before,  # type: ignore[attr-defined]
         expression=resolved.expression or "",  # type: ignore[attr-defined]
         frontmatter=resolved.frontmatter or "",  # type: ignore[attr-defined]
+        excludes=tuple(getattr(resolved, "excludes", None) or ()),
     )
 
 
@@ -146,6 +147,8 @@ class FilterSpec:
     modified_before: dt.date | None = None
     frontmatter: str = ""
     expression: str = ""
+    # Pruned by the walk before any file is read, so the text form never shows them.
+    excludes: tuple[str, ...] = ()
     raw: tuple[str, ...] = ()
 
     @property