feat(devtools): add lab schema commit, the real full-corpus persist path

devtools/command_catalog.py

@@ -30,6 +30,7 @@
     "lab projections",
     "lab smoke",
     "lab schema audit",
+    "lab schema commit",
     "lab schema compare",
     "lab schema explain",
     "lab schema generate",
@@ -1567,6 +1568,20 @@ def to_dict(self) -> dict[str, object]:
         use_when="Refresh provider schema package artifacts from archive observations outside the archive CLI.",
         examples=("devtools lab schema generate --provider chatgpt --cluster",),
     ),
+    CommandSpec(
+        "lab schema commit",
+        "verification lab",
+        "Persist a real full-corpus schema generation into committed provider packages.",
+        "devtools.schema_commit",
+        use_when=(
+            "Actually regenerate and write `polylogue/schemas/providers/<provider>/versions/...` from the live "
+            "archive -- 'lab schema generate' only ever previews and never writes committed package files."
+        ),
+        examples=(
+            "devtools lab schema commit --provider chatgpt --full-corpus --dry-run",
+            "devtools lab schema commit --provider chatgpt --full-corpus",
+        ),
+    ),
     CommandSpec(
         "lab schema promote",
         "verification lab",

devtools/schema_commit.py

@@ -0,0 +1,132 @@
+"""Commit a real full-corpus schema generation into committed packages.
+
+``devtools lab schema generate`` only ever produces a preview
+``GenerationResult`` -- it never writes to ``polylogue/schemas/providers/``.
+This command is the actual persisting entry point (polylogue-k45pq):
+it calls ``generate_all_schemas`` for real via
+``polylogue.schemas.operator.commit.commit_provider_schema`` and reports
+which package versions changed, plus whether any previously-committed leaf
+type was lost or narrowed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from polylogue.cli.shared.schema_command_support import build_schema_privacy_config
+from polylogue.config import get_config
+from polylogue.schemas.operator.commit import commit_provider_schema
+from polylogue.schemas.operator.models import SchemaCommitRequest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUTPUT_DIR = REPO_ROOT / "polylogue" / "schemas" / "providers"
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Commit a real full-corpus provider schema generation into committed packages."
+    )
+    parser.add_argument("--provider", required=True, help="Provider to generate and commit schema for.")
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=None,
+        help=f"Committed schema package root to write into (default: {DEFAULT_OUTPUT_DIR}).",
+    )
+    parser.add_argument("--max-samples", type=int, default=None, help="Limit samples for generation.")
+    parser.add_argument(
+        "--full-corpus",
+        action="store_true",
+        default=True,
+        help="Bypass all sample caps for full-corpus schema generation (default: on).",
+    )
+    parser.add_argument(
+        "--no-full-corpus",
+        dest="full_corpus",
+        action="store_false",
+        help="Generate from a capped sample window instead of the full corpus.",
+    )
+    parser.add_argument(
+        "--privacy",
+        choices=("strict", "standard", "permissive"),
+        default=None,
+        help="Privacy preset level. Defaults to standard.",
+    )
+    parser.add_argument("--privacy-config", type=Path, default=None, help="Path to TOML privacy config overrides.")
+    parser.add_argument(
+        "--dry-run",
+        "--check",
+        dest="dry_run",
+        action="store_true",
+        help="Preview what a commit would change without writing to --output-dir.",
+    )
+    parser.add_argument("--json", action="store_true", help="Output as JSON.")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        privacy_config = build_schema_privacy_config(
+            privacy=args.privacy,
+            privacy_config_path=args.privacy_config,
+        )
+    except ValueError as exc:
+        print(f"schema-commit: {exc}", file=sys.stderr)
+        return 1
+    output_dir = args.output_dir if args.output_dir is not None else DEFAULT_OUTPUT_DIR
+
+    result = commit_provider_schema(
+        SchemaCommitRequest(
+            provider=str(args.provider),
+            output_dir=output_dir,
+            db_path=get_config().db_path,
+            max_samples=args.max_samples,
+            privacy_config=privacy_config,
+            full_corpus=bool(args.full_corpus),
+            dry_run=bool(args.dry_run),
+        )
+    )
+
+    if not result.success:
+        error = result.generation.error or "Schema generation failed"
+        if args.json:
+            print(json.dumps({"provider": result.provider, "success": False, "error": error}, sort_keys=True))
+        else:
+            print(f"schema-commit: {error}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(result.to_dict(), sort_keys=True, indent=2))
+    else:
+        mode = "DRY RUN (no files written)" if result.dry_run else f"committed to {output_dir}"
+        print(f"schema-commit: {result.provider} -- {mode}")
+        print(f"  sample_count={result.generation.sample_count}")
+        for version_report in result.versions:
+            flags = []
+            if version_report.narrowed_paths:
+                flags.append(f"NARROWED({len(version_report.narrowed_paths)})")
+            if version_report.added_paths:
+                flags.append(f"added({len(version_report.added_paths)})")
+            suffix = f" [{', '.join(flags)}]" if flags else ""
+            print(
+                f"  {version_report.version}: {version_report.status} "
+                f"sample_count={version_report.sample_count}{suffix}"
+            )
+        if result.narrowed:
+            print(
+                "  WARNING: a previously-committed leaf type was lost or narrowed -- "
+                "review before trusting this commit.",
+                file=sys.stderr,
+            )
+
+    return 1 if result.narrowed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

devtools/schema_generate.py

@@ -1,4 +1,10 @@
-"""Generate provider schema packages from the devtools surface."""
+"""Generate provider schema packages from the devtools surface.
+
+This command previews a schema generation only -- it never writes to
+``polylogue/schemas/providers/``. To actually persist a full-corpus
+generation into the committed package files, use
+``devtools lab schema commit`` (``devtools/schema_commit.py``).
+"""
 
 from __future__ import annotations
 

docs/devtools.md

@@ -65,6 +65,7 @@ They are not a proof ledger or end-user archive workflow.
 | `devtools lab projections` | Inspect the unified projection inventory that feeds runtime coverage, generated docs, and control-plane maps. |
 | `devtools lab smoke` | Run direct archive and reader smoke sets outside the archive CLI. |
 | `devtools lab schema audit` | Check committed schema package quality gates without presenting them as normal archive usage. |
+| `devtools lab schema commit` | Actually regenerate and write `polylogue/schemas/providers/<provider>/versions/...` from the live archive -- 'lab schema generate' only ever previews and never writes committed package files. |
 | `devtools lab schema compare` | Review schema package drift between committed versions in the lab surface. |
 | `devtools lab schema explain` | Inspect schema package annotations, semantic roles, and review evidence from the lab surface. |
 | `devtools lab schema generate` | Refresh provider schema package artifacts from archive observations outside the archive CLI. |
@@ -157,6 +158,7 @@ These are the commands worth remembering during normal repo work:
 | `devtools lab provider completeness` | Report provider/importer package completeness by origin and capture mode. |
 | `devtools lab pytest-witness-repetitions` | Repeat the exact optimize, WAL, and embedding seed-hang witnesses with durable receipts. |
 | `devtools lab schema audit` | Run committed provider schema package quality checks. |
+| `devtools lab schema commit` | Persist a real full-corpus schema generation into committed provider packages. |
 | `devtools lab schema compare` | Compare two committed schema package versions for a provider. |
 | `devtools lab schema explain` | Explain a committed package element schema with evidence and annotations. |
 | `devtools lab schema generate` | Generate provider schema packages and optional evidence clusters. |

docs/internals.md

@@ -636,8 +636,14 @@ Polylogue has two schema-evolution regimes, keyed by tier durability.
   rows move out of authored-user accounting while provider-role counts remain
   available.
 - Provider schemas (the parsing/validation surface, distinct from the
-  storage schema) are still regenerated fresh via
-  `devtools lab schema generate` and promoted via `devtools lab schema promote`.
+  storage schema) are still regenerated fresh. `devtools lab schema generate`
+  only previews a generation (it never writes committed package files);
+  `devtools lab schema commit --full-corpus` is the entry point that actually
+  persists a full-corpus regeneration into
+  `polylogue/schemas/providers/<provider>/versions/...`, and
+  `devtools lab schema promote` promotes a single reviewed evidence cluster
+  (from `generate --cluster`) into a registered package version -- a
+  narrower, single-version operation `commit` does not replace.
 
 For **derived tiers** (`index.db`, `embeddings.db`) this design intentionally
 rejects in-place upgrade-chain complexity (no Alembic, no forward/reverse

docs/plans/classifier-fingerprints.json

@@ -137,11 +137,12 @@
       }
     },
     "polylogue/sources/parsers/claude/ai_parser.py:looks_like_ai": {
-      "fingerprint": "e5886ec01e23439d5a02ce5515e80de2695c483cd0fe2f0e843c52d504debaa9",
+      "fingerprint": "d30a5251e41ab079c8465c6892bd3124bcdd634a352edfaaf3b4c3faf21d080d",
       "covered_by": {
-        "kind": "acknowledged_safe",
-        "reason": "Baseline snapshot at classifier-fingerprint gate introduction (polylogue-gucv).",
-        "ref": "polylogue-gucv"
+        "kind": "semantic_reparse_version",
+        "reason": "PR #3537 tightened looks_like_ai to require positive chat_messages evidence (role/sender+text/content), same shape as PR #3428's looks_like_code fix",
+        "ref": "polylogue-t0ta",
+        "version": 54
       }
     },
     "polylogue/sources/parsers/claude/ai_parser.py:looks_like_claude_design": {

docs/providers/index.md

@@ -88,6 +88,7 @@ pipeline.
 3. **Register the detector** in `polylogue/sources/dispatch.py` at the
    appropriate priority level
 4. **Add a provider schema bundle** under `polylogue/schemas/providers/`
-   (run `devtools lab schema generate` to bootstrap)
+   (run `devtools lab schema generate` to preview, `devtools lab schema commit`
+   to actually write the committed package files)
 5. **Update schema inference** if the new provider introduces novel content
    block types or message structures

polylogue/schemas/generation/workflow.py

@@ -78,6 +78,7 @@ def generate_all_schemas(
     max_samples: int | None = None,
     privacy_config: SchemaPrivacyConfig | None = None,
     include_archive_workload_profile: bool = False,
+    full_corpus: bool = False,
 ) -> list[GenerationResult]:
     """Generate versioned schemas for all providers."""
     if db_path is None:
@@ -93,6 +94,7 @@ def generate_all_schemas(
             db_path=db_path,
             max_samples=max_samples,
             privacy_config=privacy_config,
+            full_corpus=full_corpus,
         )
         results.append(bundle.result)
         persist_generated_provider_bundle(output_dir, provider, bundle)