feat(cli): add 'add-all' command and auto_delete_added_files config option

- New 'openkb add-all' command processes all files in raw/ directory
- New config parameter 'auto_delete_added_files' (default: false)
- When enabled, both 'add' and 'add-all' automatically delete successfully ingested files
- Updated help texts to document the new cleanup behavior
- Config applies to all ingest methods: direct files, directories, and URLs

Author: Sebastian Braun

openkb/cli.py

@@ -452,6 +452,29 @@ def add_single_file(
         return _add_single_file_locked(file_path, kb_dir, stage=stage, bundle=bundle)
 
 
+def _delete_if_auto_cleanup_enabled(
+    file_path: Path, status: Literal["added", "skipped", "failed"], config: dict
+) -> bool:
+    """Delete file if addition succeeded and auto_delete_added_files is enabled.
+
+    Args:
+        file_path: Path to the file to potentially delete.
+        status: Result status from add_single_file ("added", "skipped", or "failed").
+        config: Configuration dict (typically from resolve_effective_config).
+
+    Returns:
+        True if file was deleted, False otherwise.
+    """
+    if status == "added" and config.get("auto_delete_added_files", False):
+        try:
+            file_path.unlink(missing_ok=True)
+            return True
+        except Exception as exc:
+            logger.warning(f"Failed to delete {file_path.name}: {exc}")
+            return False
+    return False
+
+
 def _add_single_file_locked(
     file_path: Path, kb_dir: Path, *, stage: bool = True, bundle=None
 ) -> Literal["added", "skipped", "failed"]:
@@ -1086,6 +1109,9 @@ def add(ctx, path, from_pageindex_cloud):
     Alternatively, pass --from-pageindex-cloud <DOC_ID> to import a document
     that is already indexed in PageIndex Cloud, with no local file. Requires
     the PAGEINDEX_API_KEY environment variable.
+
+    If ``auto_delete_added_files`` is enabled in config.yaml, successfully
+    added files are automatically deleted after ingestion.
     """
     kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
     if kb_dir is None:
@@ -1106,6 +1132,8 @@ def add(ctx, path, from_pageindex_cloud):
         click.echo("Provide a PATH or use --from-pageindex-cloud <DOC_ID>.")
         return
 
+    config = resolve_effective_config(kb_dir)[0]
+
     # URL ingest: download into raw/ first, then call add_single_file explicitly.
     # Keep staged conversion enabled so converted source artifacts do not touch
     # the live KB before the mutation snapshot exists. The tri-state outcome
@@ -1123,6 +1151,8 @@ def add(ctx, path, from_pageindex_cloud):
         # indexing has already succeeded but compilation didn't.
         if outcome == "skipped":
             fetched.unlink(missing_ok=True)
+        else:
+            _delete_if_auto_cleanup_enabled(fetched, outcome, config)
         return
 
     target = Path(path)
@@ -1143,15 +1173,71 @@ def add(ctx, path, from_pageindex_cloud):
         click.echo(f"Found {total} supported file(s) in {path}.")
         for i, f in enumerate(files, 1):
             click.echo(f"\n[{i}/{total}] ", nl=False)
-            add_single_file(f, kb_dir)
+            outcome = add_single_file(f, kb_dir)
+            _delete_if_auto_cleanup_enabled(f, outcome, config)
     else:
         if target.suffix.lower() not in SUPPORTED_EXTENSIONS:
             click.echo(
                 f"Unsupported file type: {target.suffix}. "
                 f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
             )
             return
-        add_single_file(target, kb_dir)
+        outcome = add_single_file(target, kb_dir)
+        _delete_if_auto_cleanup_enabled(target, outcome, config)
+
+
+@cli.command()
+@click.pass_context
+@_with_kb_lock(exclusive=True)
+def add_all(ctx):
+    """Process all files in the ``raw/`` directory and add them to the knowledge base.
+
+    This command walks the ``raw/`` directory recursively for all supported
+    document types and ingests them into the KB. If ``auto_delete_added_files``
+    is enabled in config.yaml, successfully added files are automatically deleted
+    after ingestion.
+
+    Returns a summary of the operation (added, skipped, failed, deleted counts).
+    """
+    kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override"))
+    if kb_dir is None:
+        click.echo("No knowledge base found. Run `openkb init` first.")
+        return
+
+    raw_dir = kb_dir / "raw"
+    if not raw_dir.is_dir():
+        click.echo(f"No raw/ directory found at {raw_dir}")
+        return
+
+    files = [
+        f
+        for f in sorted(raw_dir.rglob("*"))
+        if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS
+    ]
+    if not files:
+        click.echo("No supported files found in raw/ directory.")
+        return
+
+    config = resolve_effective_config(kb_dir)[0]
+    total = len(files)
+    added = skipped = failed = deleted = 0
+
+    click.echo(f"Processing {total} file(s) from raw/ directory...")
+    for i, f in enumerate(files, 1):
+        click.echo(f"\n[{i}/{total}] ", nl=False)
+        outcome = add_single_file(f, kb_dir)
+        if outcome == "added":
+            added += 1
+        elif outcome == "skipped":
+            skipped += 1
+        else:
+            failed += 1
+        if _delete_if_auto_cleanup_enabled(f, outcome, config):
+            deleted += 1
+
+    click.echo(
+        f"\n\nSummary: Added: {added}, Skipped: {skipped}, Failed: {failed}, Deleted: {deleted}"
+    )
 
 
 def _stream_to_tty() -> bool:

openkb/config.py

@@ -36,6 +36,7 @@
     # global/KB list overrides it wholesale; resolve_entity_types cleans the
     # effective value on read.
     "entity_types": list(DEFAULT_ENTITY_TYPES),
+    "auto_delete_added_files": False,
 }
 
 GLOBAL_CONFIG_DIR = Path.home() / ".config" / "openkb"