diff --git a/.plans/planned/project-agent-sessions.md b/.plans/planned/project-agent-sessions.md new file mode 100644 index 0000000..5a9ec30 --- /dev/null +++ b/.plans/planned/project-agent-sessions.md @@ -0,0 +1,56 @@ +# Project agent sessions and agent command installation + +Status: implemented in the working tree. The plan remains here as the requested implementation record. + +## Goal + +Support two entry points into one project-scoped session model: + +1. `dc session start --project materials --agent codex` starts a durable project session, prepares the selected agent's Docling Context MCP connection and `dc` skill, runs the agent in the current terminal, and returns to the caller when it exits. +2. `dc agent install codex` installs the same MCP connection and reusable `dc` skill for an agent started independently. The skill guides project selection, session recording, scoped search, citations, and closure. + +Support Codex, Claude Code, Hermes, and Pi. Use the existing local SQLite store and DCLX session packages. Keep the original caller's working directory and terminal. + +## Current state + +- The CLI has separate user-scoped `session` commands and `project session-*` commands. Project sessions already have DCLX URIs and a catalog link, but no agent identity. +- `docling-context integrate` installs an MCP entry only. The MCP tool set already includes project-aware search, project session start/close, and session append. +- Project refresh jobs generate deterministic abstract/overview inventory documents. They are unrelated to launching agents or session recording. +- Graphify installs agent skills under per-agent user/project paths. Codex's current official skill syntax is `$dc`; Claude, Hermes, and Pi expose skills through their own command menus. + +## Proposed implementation + +### Session model and CLI + +- Store `agent` in `dc_project_sessions`. Keep a local additive schema update for existing SQLite stores. +- Make `dc session start` require `--project` and `--agent`. Generate or accept `--id`. Add `--no-launch` for an agent already running (the skill path). Return the session URI, project, and agent. +- On normal launch, verify the agent executable and install its MCP and skill integration, then start the DCLX session. Pass the full session URI, project ID, agent, store, tenant, and user to the child as environment variables and a short initial prompt. MCP searches default to that project. Wait for the agent process; on exit, close the session, which queues memory compilation. Preserve exit code and terminal control. Do not automatically run a memory worker. +- Show project sessions, including agent, with `dc session list --project PROJECT_ID`; project sessions also appear in `dc project session-list`. +- Keep explicit project session commands and the internal user session API for compatibility; the main human start path is project first. + +### Agent installation + +- Add `dc agent install|status|remove AGENT --scope user|project`. Installation reuses the owned MCP entry in `integration.py` and adds a small `dc` skill in the agent's skill directory. Installation is idempotent; refuse to overwrite unmanaged files and remove only owned files. +- Launcher installs user-scoped integration so it works from any directory. Explicit project scope writes into the current working directory for Codex, Claude, and Pi. Hermes uses user scope for its MCP integration. Skill paths follow the installed agents and the Graphify reference. +- The skill uses the Docling Context MCP tools for project-scoped retrieval and DCLX session events. When independently invoked, it starts a session for an explicit project, or asks for a project if several are available. It does not launch another agent. +- The launcher reuses the installer. No shell interpolation for agent arguments; use an argument list and inherited terminal streams. + +### Documentation and verification + +- Update `docs/index.md`, `docs/agents.md`, `docs/sessions.md`, and `docs/cli.md` with the two workflows, project scoping, agent-specific invocation names, and job distinctions. +- Unit tests cover project/agent requirement, DCLX and catalog association, subprocess argument/environment construction, exit and failure handling, and idempotent installer ownership in temporary homes/project roots. +- Run CLI help and discovery checks against installed `codex`, `claude`, `hermes`, and `pi` binaries. Do not start live model conversations during tests. + +## Acceptance + +- Running `dc session start --project materials --agent codex` enters Codex and returns to the original shell when Codex exits. +- The session is visible under the project with its agent and closed state. +- `dc agent install` makes the agent's Docling Context tools and skill available without starting through `dc`. +- Existing unrelated agent configuration and skills are preserved. +- Docs identify that Codex invokes the skill as `$dc`, while other harnesses may display `/dc`. + +## Verification + +- Full test suite: 111 passed. +- Ruff and mypy passed for the source tree. +- The installed Codex, Claude Code, Hermes, and Pi executables answered their version and help commands. Interactive model sessions were exercised with a subprocess test double rather than live prompts. diff --git a/.plans/planned/refactor-to-have-projects.md b/.plans/planned/refactor-to-have-projects.md new file mode 100644 index 0000000..a90ceab --- /dev/null +++ b/.plans/planned/refactor-to-have-projects.md @@ -0,0 +1,72 @@ +# Shared resources and projects + +## Agreed scope + +Use a fresh local store. SQLite is the single catalog for resource identity, +revisions, project membership, typed relationships, jobs, sessions, search +indexes, and cited structured knowledge. DCLX packages are content-addressed +files beside the SQLite database. No PostgreSQL server or migration is needed. + +```text +docling:// +├── resources/ +│ ├── library/{doc_id} +│ ├── memories/{memory_id} +│ ├── skills/{skill_id} +│ ├── concepts/{concept_id} +│ └── knowledge/{knowledge_id} +├── projects/{project_id}/ +│ ├── .description.dclx +│ ├── .abstract.dclx +│ ├── .overview.dclx +│ └── sessions/{session_id} +└── user/{user_id}/ +``` + +A project is a topic of work. Its membership links to subsets of flat shared +resources without copying packages. Source bytes are hashed before conversion; +a repeat import reuses the existing library document. `add-resource --project` +links the imported document. `dc overview` lists every project with counts of +linked documents, memories, skills, concepts, knowledge, and sessions. + +## Implementation areas + +1. Use `LocalContextStore` and one SQLite catalog for project and resource + tables. Keep tenant-scoped foreign keys and unique constraints. Store raw + source hashes separately from DCLX package hashes and immutable revisions. +2. Route `add-resource` to the shared library; replace `--collection` with + `--project`. Preserve exact URI, revision, and XPath citations. Support + project-filtered lexical and vector retrieval. +3. Store reviewed memories in the flat memory list, compile project sessions + into linked memory proposals, and validate DCLX skill, concept, and + knowledge descriptors. Store typed knowledge facts with source citations. +4. Refresh project abstract and overview DCLX documents after project changes. + Expose project and resource links, overview, and knowledge facts in CLI, + local SDK, HTTP, and MCP. +5. Retire old collection-specific public commands. Keep private user sessions + available until a project is + selected; new project sessions live under the project URI. +6. Rewrite `README.md` and `docs/` around SQLite, a five-minute quick start, + and tested installer recipes for Codex, Claude Code, Hermes, and Pi. + +## Acceptance checks + +- Duplicate imports reuse a document ID, including concurrent imports. +- A resource can link to multiple projects; unlinking preserves it. +- Overview counts are correct for empty and populated projects. +- Project search returns only linked resources and citations remain readable. +- Tenant boundaries and memory owner review checks hold. +- Cited knowledge facts validate field types and source revisions. +- Project sessions compile into shared, project-linked memory proposals. +- CLI, SDK, MCP, docs, and lockfile agree on SQLite; no PostgreSQL setup is + required. +- Run tests, lint, type checking, and smoke-test the quick start. + +## Current implementation + +The SQLite catalog, shared library deduplication, project membership and +search, flat memories, typed descriptors, knowledge facts, project context jobs, +CLI overview, and service/MCP project operations are implemented. Old +collection summary commands are removed from the CLI; internal compatibility +code and private user sessions remain. The documentation describes the SQLite +store and project workflow. diff --git a/.plans/planned/simplify-cli-and-concept-knowledge.md b/.plans/planned/simplify-cli-and-concept-knowledge.md new file mode 100644 index 0000000..1e1d326 --- /dev/null +++ b/.plans/planned/simplify-cli-and-concept-knowledge.md @@ -0,0 +1,76 @@ +# Simplify the CLI and ground knowledge in concepts + +## Agreed behavior + +- Use one canonical command for each operation; remove duplicate CLI routes. +- The top level is `init`, `overview`, `project`, `library`, `session`, `memory`, + `knowledge`, `concept`, `skill`, `index`, and `agent`. The two extra typed groups + keep concepts and skills out of the source-document library. +- `library` owns document import, listing, tree navigation, exact node reading, + outlines, lexical lookup (`find` and `grep`), indexed search, and ingest jobs. +- Projects manage titles, shared resource links, and project context. Group context + refresh, show, set, jobs, and worker under `project context`. +- Remove `project session-*`. `session start --project ID --agent NAME` launches + the agent in the calling terminal, closes the session on exit, and returns to + the caller. `--no-launch` explicitly creates a session without launching. +- Session IDs returned by the CLI work in subsequent commands; reject ambiguous + IDs instead of choosing a project silently. +- Merge store status and pending-work counts into overview. Bare `dc` shows the + overview and examples for creating a project, importing documents, and launching + an agent. Keep the default SQLite store and global `--store` override. +- Memories are reviewed DCLX conclusions, syntheses, and session outcomes, with + citations. Support project-scoped create/list/search and DCLX authoring, retain + review, correction, recall, and compilation commands. +- Concepts are DCLX definitions with a mandatory machine-readable schema for + entity types, typed properties, optional units, and relationship types. +- `knowledge create ID --concept CONCEPT_ID --project PROJECT` creates a DCLX + dataset descriptor and stores cited facts in SQLite. It never takes an + independent `--schema`. Pin the concept URI and immutable revision and copy the + validated schema into the dataset catalog entry. Validate entity types, property + names, value types, and units when adding facts. Later concept edits do not + mutate existing datasets. +- Concepts can be reused by multiple datasets and projects. Dataset-to-concept, + resource-to-resource, and project-to-resource relationships remain globally + queryable. Source citations retain URI, revision, and XPath. +- No CLI compatibility aliases or automatic data migration. Existing documents + and projects remain readable; unbound old datasets require explicit recreation + before accepting new facts. + +## Implementation steps + +1. Refactor argparse command groups and dispatch while reusing existing import, + search, session launcher, job, and inspection implementations. +2. Add typed concept/skill creation and listing, ID-to-URI resolution, concept + schema validation, and concept-bound knowledge creation/show/list/filtering. +3. Resolve project session IDs and add project-scoped memory workflows without + bypassing review or authorization rules. +4. Update CLI/index/agent/storage/conversion/retrieval/session documentation, + README examples, generated `/dc` skill, and command references in source. + Keep MCP tool names stable: recording a session through MCP is separate from + launching an external interactive agent through the CLI. +5. Update existing CLI tests to the canonical routes. Test help, bare invocation, + agent launch/return, resource scoping, memory project links, mandatory concept + schemas, pinned revisions, fact rejection, and exact citations. Run the full + suite and required lint/type checks. + +## Schema contract + +Concept metadata lives in `context/concept.json` inside DCLX. Its `schema` has +nonempty `entity_types` and `properties`; properties map names to definitions +such as `{"type":"number","unit":"%"}`. Supported value types are `text`, +`integer`, `number`, `boolean`, and `json`. Optional `relationship_types` map names +to source/target entity types. Definitions and documentation live in the DCLG +document body. DCLG input requires descriptor metadata with the mandatory schema; +DCLX input may carry the descriptor directly. + +## Verification and completion + +- Implemented the canonical CLI groups and removed duplicate top-level and + project-session routes. Bare `dc` shows the overview and starting examples. +- Added document, skill, and concept relationship commands; project-scoped + memory authoring and lookup; and authored DCLG/DCLX memory preservation. +- Knowledge datasets now require a concept, pin its URI and revision, and + validate typed facts and units against the copied schema. +- Updated the CLI, agent, storage, session, retrieval, conversion, and index + documentation, plus the generated agent skill. +- Verified with Ruff, mypy, and the full pytest suite (119 tests passed). diff --git a/README.md b/README.md index ac65fe7..3567517 100644 --- a/README.md +++ b/README.md @@ -1,120 +1,42 @@ # docling-context -A local context tree that stores documents as immutable DocLang `.dclx` packages. +`docling-context` converts source documents to DCLX and stores them in a shared +library. Projects link to any subset of library documents, memories, skills, +concepts, and structured knowledge. One source can serve several projects. +SQLite stores the catalog, project links, sessions, jobs, and search indexes; +DCLX packages live beside it in the local store. -See the [capability guide](docs/index.md), [retrieval guide](docs/retrieval.md), -[session guide](docs/sessions.md), [memory guide](docs/memory.md), -[agent harness guide](docs/agents.md), -[CLI reference](docs/cli.md), and -[single PDF and folder examples](examples/adding_resources/README.md). +See the [overview and five-minute quick start](docs/index.md), +[CLI guide](docs/cli.md), [storage guide](docs/storage.md), and +[agent installation guide](docs/agents.md). -Ingest native `.dclg`, `.dclg.xml`, and `.dclx` sources with `Ingestor.add_resource`. -Pass bytes with a filename, or a local `Path` under an explicit `allowed_roots` entry. -Each revision stores its full document, a short DocLang summary, and a DocLang TOC -in one DCLX package. Collection summaries are rebuilt by `CollectionWorker`; -`CollectionWorker.status` reports whether a summary is stale. +## Quick start -Document conversion is local and optional: install the `conversion` extra to -enable `LocalDoclingConverter` for Docling's supported formats, including PDF, -Office documents, and images. `DoclingServeConverter` requires an explicit HTTPS -endpoint or loopback HTTP endpoint. -SQLite tracks logical URIs, revisions, tenant ownership, and an append-only change -outbox. Package bytes live under SHA-256 paths. The API is designed so another -storage backend can implement the same contracts later. - -## Development setup - -The project resolves DocLang from commit -`92d299026f2f985328b52424483c711e19b2f306` (Apache-2.0) through uv's Git -source configuration. No local DocLang checkout is required for development. -The source configuration is uv-specific; installers that ignore it resolve the -published `doclang` package, which may not yet have the required native API. - -```bash -uv sync -uv run pytest -q tests -``` - -## CLI quick start +With Python 3.12+ and `uv`: ```bash uv sync --extra conversion -uv run dc --store ./context-data add-resource ./paper.pdf --collection papers --no-ocr -uv run dc --store ./context-data add-resource ./documents -r --from docx --from image -uv run dc --store ./context-data worker --once -uv run dc --store ./context-data tree docling://resources/papers -L 2 -uv run dc --store ./context-data index status -uv run dc --store ./context-data search "invoice" --uri docling://resources/papers -uv run --extra vectors dc --store ./context-data index vector rebuild -uv run --extra vectors dc --store ./context-data search "invoice" --mode hybrid -uv run dc --store ./context-data session start --id chat-1 -uv run dc --store ./context-data session list -uv run dc --store ./context-data memory list --status proposed -uv run docling-context integrate codex --scope project --store ./context-data -uv run docling-context doctor codex --scope project --store ./context-data +uv run dc init +uv run dc project create --project research --title "Research" +uv run dc library add --source ./paper.pdf --project research +uv run dc overview +uv run dc library search --query "key finding" --project research ``` -Use `uv run dc` or activate the project environment before running `dc` by -itself; some systems also provide a calculator named `dc`. See the -[full CLI guide](docs/cli.md) and [resource examples](examples/adding_resources/README.md). -Ingestion builds the lexical index automatically. Vector indexing is optional; -the `vectors` extra provides local embeddings and `dc index vector rebuild`. -`dc index lex rebuild` reconstructs search records from stored DCLX packages. -Search results cite exact source nodes or `@summary:` and `@toc:` sidecars. - -## Local usage - -```python -from doclang import DocLangXDocument -from docling_context import LocalContextStore, NodeAddress, Principal - -document = DocLangXDocument() -assert document.read_xml('Hello') -package = document.write_bytes() -principal = Principal(tenant_id="acme", user_id="alice") - -with LocalContextStore("./context-data") as store: - record = store.put(principal, "docling://resources/manuals/hello", package) - citation = NodeAddress(record.document_id, record.revision_id, "/doclang[1]/text[1]") - print(store.read_node(principal, citation).text) -``` - -For ingestion, allow the source directory explicitly and then process queued -collection summaries. A worker can call `run_forever(stop_event)` in its own -process with its own store connection. - -```python -from pathlib import Path -from docling_context import CollectionWorker, Ingestor, LocalContextStore, Principal - -principal = Principal("acme", "alice") -source_dir = Path("./documents").resolve() -with LocalContextStore("./context-data") as store: - record = Ingestor(store, allowed_roots=(source_dir,)).add_resource( - principal, "docling://resources/manuals/guide", source_dir / "guide.dclx" - ) - worker = CollectionWorker(store) - while worker.run_once() is not None: - pass - print(worker.status(principal, "docling://resources/manuals")) -``` +`dc init` creates the local store; no database server is needed. Set +`DOCLING_CONTEXT_STORE` or pass `--store PATH` to choose its location. A repeated +import of the same source bytes reuses its library URI. Results carry a URI, +revision ID, and XPath for exact citation. Use `--json` for machine-readable +output; the default CLI output is tabular. -Document URIs use `docling://resources/{collection}/{path}` or -`docling://users/{tenant}/{user}/{memories|sessions|skills}/{path}`. Namespace -names are lowercase; path segments keep their case and normalize to Unicode NFC. -Encoded separators, traversal segments, and ambiguous escapes are rejected. -User paths require a matching principal. Resource rows are scoped by tenant in -storage, so the same resource URI can exist independently in two tenants. +To launch an agent in a project session, run +`uv run dc session start --project research --agent codex`. To use Docling +Context from an agent you start yourself, run `uv run dc agent install --agent codex` +and invoke its `dc` skill. Both flows support Claude Code, Hermes, and Pi; see +the [agent guide](docs/agents.md). -`put` creates a document when `expected_revision` is omitted. To replace one, -pass its current revision ID. `delete` also requires the current revision ID. -Each revision keeps its own package hash and remains addressable by -`NodeAddress(document_id, revision_id, xpath)` after replacement or deletion. -Tree and event queries require a principal and have bounded page sizes. +## Development -The package adapter validates the native DCLX report and applies limits to the -compressed package, archive entries, XML node count and depth, and node reads. -Startup reconciliation marks missing or corrupt packages unavailable and removes -orphaned blobs. Reads verify SHA-256 before returning content. A blob can remain -after a failed SQLite transaction until the next reconciliation; no logical -revision or outbox event is exposed. +Run `uv sync --all-extras` and `uv run pytest -q tests`. New stores use the +SQLite project catalog. There is no migration command for PostgreSQL or older +store layouts. diff --git a/docs/agents.md b/docs/agents.md index 1672875..408df5b 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -1,45 +1,144 @@ -# Agent harness installation +# Work with an agent -Install `docling-context` into a Python environment visible to the harness. The -package exposes `dc` for document work and `docling-context` for MCP serving and -harness integration. Local serving uses the same SQLite store and DCLX packages -as `dc`; no cloud account is needed. +Install `docling-context` into a Python environment visible to the agent. The +package exposes `dc` for document work and `docling-context` for MCP serving. +The CLI, workers, and MCP server must use the same local SQLite store. These +examples use the default store at `~/.local/share/docling-context`. ```bash uv sync --extra conversion -uv run dc add-resource ./paper.pdf --collection papers -uv run docling-context integrate codex --scope project -uv run docling-context doctor codex --scope project +uv run dc init +uv run dc project create --project papers --title "Paper review" +uv run dc library add --source ./paper.pdf --project papers ``` -`integrate` launches a temporary MCP server and checks handshake, tool discovery, -backend listing, and one immutable citation when indexed content exists. Then it -adds a named `docling-context` entry to the harness configuration. Restart the -harness to load it. `doctor` repeats the checks; `remove` deletes only an entry -still owned by this installer. Existing entries with the same name are reported -as conflicts. Repeating `integrate` is safe, and changing options upgrades only -the owned entry. The installer writes a small ownership marker next to the -config; keep it alongside the config for future upgrades and removal. +There are two ways to work with an agent: + +| Route | Start with | Session handling | +| --- | --- | --- | +| Launch the agent from `dc` | `dc session start --project papers --agent AGENT` | `dc` creates and closes the project session around the agent process. | +| Start the agent yourself | `dc agent install --agent AGENT`, then invoke its `dc` skill | The skill instructs the agent to create, record, and close a project session through MCP. | + +## Launch an agent from `dc` + +```bash +uv run dc session start --project papers --agent codex +``` + +Replace `codex` with `claude`, `hermes`, or `pi`. The agent executable must be on +`PATH`. The command checks the project, installs the agent's managed MCP +connection and `dc` skill, starts a session under the project, and runs the +agent in your terminal. When the agent exits, you return to the original shell +and the session closes, even if the agent was interrupted. The launcher passes +the project ID and session URI to the agent and asks it to record useful +outcomes. MCP searches without an explicit project default to the launched +project. + +Only events actually sent through `session_append` are stored; the launcher +does not capture the full conversation. Inspect the session and process any +queued memory compilation job with: + +```bash +uv run dc session list --project papers +uv run dc session show --id docling://projects/papers/sessions/SESSION_ID +uv run dc session jobs --id docling://projects/papers/sessions/SESSION_ID +uv run dc memory worker --once +uv run dc memory proposals +``` + +The `SESSION_ID` comes from `session start` or `session list`. Closing a session +queues compilation; it does not itself create an accepted memory. A worker may +propose memories, which need review before acceptance. + +Every project links the shared `doclang-authoring` resource skill. When an +agent writes DCLG or DCLX, it should read that skill first and validate the +result with DocLang's bundled XSD and Schematron rules. The installed `dc` +agent skill includes this instruction. Reinstall it with `dc agent install` +to update an existing agent installation. + +The launch installs user-scoped agent configuration so it works from any +directory. The scope controls where the agent loads its configuration; it is +distinct from the Docling Context project ID. Agent sessions are tagged with +the selected agent in SQLite. + +## Install for an agent you start yourself + +Run the matching command from the environment where `docling-context` is +installed. The installer writes one owned MCP entry and one `dc` skill. Write +tools are enabled because the skill records project sessions. It preserves +unrelated agent configuration and refuses to overwrite an unmanaged `dc` skill. +The default scope is user; use project scope to keep the integration with a +working directory. Project scope is relative to the current directory (or +`--project-root PATH`); it does not choose a Docling Context project. Hermes +currently supports user scope for MCP. + +```bash +uv run dc agent install --agent codex --scope project +uv run dc agent status --agent codex --scope project + +uv run dc agent install --agent claude --scope project +uv run dc agent status --agent claude --scope project + +uv run dc agent install --agent hermes +uv run dc agent status --agent hermes + +uv run dc agent install --agent pi --scope project +uv run dc agent status --agent pi --scope project +``` + +Restart the agent after installation. Invoke the skill and name the Docling +Context project in your request: + +| Agent | Example request inside the agent | +| --- | --- | +| Codex | `$dc Review the papers project and record useful conclusions.` | +| Claude Code | `/dc Review the papers project and record useful conclusions.` | +| Hermes Agent | `/dc Review the papers project and record useful conclusions.` | +| Pi | `/dc Review the papers project and record useful conclusions.` | + +Codex also exposes the skill through `/skills`; the [official Codex skill guide](https://learn.chatgpt.com/docs/build-skills) +documents this syntax. The installed skill instructs the agent to select a +project, start a project session through MCP, search its linked resources, +record useful outcomes, and close the session. Verify what was recorded with +`dc session list --project papers` and `dc session replay --id SESSION_URI`. +`dc session start --project papers --agent codex` launches Codex in the current terminal; +`--no-launch` creates only the project session record. + +If an agent is already running and you want to supply a session yourself, run +`dc session start --project papers --agent codex --no-launch`. This creates an +open session but does not install or launch the agent. Give the returned full +URI to the agent, then run `dc session close --id SESSION_URI` when finished. The +skill leaves launcher-provided sessions open for the launcher to close. + +`dc agent status --agent AGENT` checks that the managed skill and MCP configuration are +present; it does not open an MCP connection. Run `docling-context doctor AGENT` +with the same `--scope` to check the connection. `dc agent remove --agent AGENT` +removes the owned skill and MCP entry. The lower-level `docling-context +integrate`, `doctor`, and `remove` commands remain available for MCP-only or +remote configurations; those commands do not install the skill. ## Harnesses -| Harness | Local version observed | User config | Project config | Local stdio | Remote HTTP | Verification | -| --- | --- | --- | --- | --- | --- | --- | -| Codex | 0.160.0 | `~/.codex/config.toml` | `.codex/config.toml` | Yes | Yes | Config test, MCP handshake, CLI lists entry | -| Claude Code | 2.1.278 | `~/.claude.json` | `.mcp.json` | Yes | Yes | Config test, MCP handshake, CLI sees entry pending project approval | -| Hermes Agent | 0.21.1 | `~/.hermes/config.yaml` | Unsupported by Hermes | Yes | Yes | Config test, MCP handshake, CLI lists entry | -| Pi | Not installed locally | `~/.pi/agent/mcp.json` | `.pi/mcp.json` | Yes | Yes | Config test and MCP SDK handshake; native CLI unverified | - -The config tests check preservation, repeat installs, upgrades, and removal. The -MCP handshake is tested with the Python MCP client; harness binaries are not -required for the test suite. Project configuration is reviewable in Git. -Codex and Pi may require project trust before loading project MCP entries. +| Agent | User MCP config | Project MCP config | User skill | Project skill | +| --- | --- | --- | --- | --- | +| Codex | `~/.codex/config.toml` | `.codex/config.toml` | `~/.agents/skills/dc/SKILL.md` | `.agents/skills/dc/SKILL.md` | +| Claude Code | `~/.claude.json` | `.mcp.json` | `~/.claude/skills/dc/SKILL.md` | `.claude/skills/dc/SKILL.md` | +| Hermes Agent | `~/.hermes/config.yaml` | Unsupported | `~/.hermes/skills/dc/SKILL.md` | Unsupported | +| Pi | `~/.pi/agent/mcp.json` | `.pi/mcp.json` | `~/.pi/agent/skills/dc/SKILL.md` | `.pi/agent/skills/dc/SKILL.md` | + +The installer preserves unrelated configuration and only removes entries and +skill files it owns. Project configuration is reviewable in Git. Codex and Pi +may require project trust before loading project MCP entries. Configuration paths and transports follow the current [Codex](https://developers.openai.com/codex/mcp), [Claude Code](https://code.claude.com/docs/en/mcp), [Hermes](https://hermes-agent.nousresearch.com/docs/reference/mcp-config-reference/), and [Pi](https://pi.dev/docs/latest/mcp) MCP guides. +All four agents support local stdio and remote HTTP MCP connections through +the lower-level integration commands. Those commands manage the connection, +not the `dc` skill: + ```bash uv run docling-context integrate claude --scope user uv run docling-context integrate hermes --scope user @@ -67,19 +166,24 @@ With a configured image-capable vector model, `search` accepts an optional base64 image query (up to 2.5 MB) alongside query text. A text-only model silently skips the image contribution. -Read tools are on by default. Local write tools require `--allow-writes` on -`integrate` or `docling-context mcp`; write tools include `ingest`, +Read tools are on by default. `dc agent install` enables local write tools for +the skill. With the lower-level commands, local write tools require +`--allow-writes` on `integrate` or `docling-context mcp`; write tools include `ingest`, +`project_create`, `project_link`, `project_session_start`, +`project_session_close`, `resource_link`, `knowledge_add_fact`, `session_append`, and `memory_review`. Memory capture remains opt-in. Each operation checks the principal's tenant and user scope. The current local principal comes from the server command, not from tool arguments. -MCP `ingest` accepts base64 source bytes up to 2.5 MB and returns a durable job -ID. Run `docling-context ingest-worker` alongside the agent, or +MCP `ingest` accepts a filename and base64 source bytes up to 2.5 MB, with an +optional `project_id`, and returns a durable job ID. The worker checks the +source hash, stores the DCLX in the flat library, and links it to the project +when requested. Run `docling-context ingest-worker` alongside the agent, or `docling-context ingest-worker --once` for one job. `job_status` reports queued, running, completed, or failed state and the resulting revision. Jobs survive -server and worker restarts. Larger documents can be imported with `dc -add-resource`, which converts synchronously and indexes the result. Use -`dc jobs status JOB_ID` to inspect either job type; add `--json` for a record. +server and worker restarts. Larger documents can be imported with +`dc library add`, which converts synchronously and indexes the result. Use +`dc library jobs status --id JOB_ID` to inspect either job type; add `--json` for a record. Queued Docling conversions use a five-minute document timeout. ## Remote service @@ -112,7 +216,7 @@ from docling_context import ContextService, LocalContextStore, Principal, Search with LocalContextStore("./context-data") as store: service = ContextService(store, Principal("team", "agent")) - hits = service.search(SearchRequest("Koonap", uri="docling://resources/papers")) + hits = service.search(SearchRequest("Koonap", project_id="papers")) for hit in hits.data["hits"]: print(hit["uri"], hit["revision_id"], hit["xpath"]) ``` diff --git a/docs/cli.md b/docs/cli.md index 2cb07cf..bec5847 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,200 +1,240 @@ -# `dc` CLI - -Install the package to expose `dc`. For PDF input, install the `conversion` -extra. The command uses a local store at `~/.local/share/docling-context` by -default; pass `--store PATH` to choose another directory. `--tenant` and -`--user` select the principal; their defaults are `default` and `local`. -Use `uv run dc` or activate the project environment; a system calculator also -uses the name `dc` on some machines. - -```bash -uv run --extra conversion dc --store ./context-data add-resource ./paper.pdf \ - --collection papers --no-ocr --tables --no-charts -uv run dc --store ./context-data task status JOB_ID -uv run dc --store ./context-data jobs status JOB_ID --json -uv run dc --store ./context-data worker --once -uv run dc --store ./context-data status -uv run dc --store ./context-data index status -uv run dc --store ./context-data index lex status -uv run dc --store ./context-data index lex rebuild -uv run --extra vectors dc --store ./context-data index vector status -uv run --extra vectors dc --store ./context-data index vector rebuild -uv run --extra vectors dc --store ./context-data search "invoice" --mode hybrid -uv run dc --store ./context-data ls docling://resources/ -uv run dc --store ./context-data tree docling://resources/papers -L 2 -uv run dc --store ./context-data outline docling://resources/papers/paper -uv run dc --store ./context-data show docling://resources/papers/paper \ - '/doclang[1]/text[1]' --format xml --json -uv run dc --store ./context-data find "invoice" --uri docling://resources/papers -uv run dc --store ./context-data grep "total" --uri docling://resources/papers -uv run dc --store ./context-data search "invoice" --uri docling://resources/papers -uv run dc --store ./context-data session start --id chat-1 -uv run dc --store ./context-data session list --status open -uv run dc --store ./context-data memory list --status proposed -uv run docling-context integrate codex --scope project --store ./context-data -uv run docling-context doctor codex --scope project --store ./context-data +# `dc` command guide + +`dc` stores source documents, reusable conclusions, methods, concepts, and cited facts in one local SQLite store. Projects link selected resources without copying them. The default store is `~/.local/share/docling-context`; pass `--store PATH` before a command to use another one. Run `dc` without arguments for a project overview and three starting commands. The examples below use `uv run dc`. + +| Resource | Purpose | +| --- | --- | +| Library document | An imported source PDF, office document, image, URL, DCLG, or DCLX in `resources/library/`. | +| Memory | A reviewed conclusion or session outcome stored as DCLX in `resources/memories/`. | +| Skill | A reusable method stored as DCLX in `resources/skills/`. | +| Concept | A DCLX definition with a mandatory schema for entity types, properties, units, and relationships. | +| Knowledge dataset | A DCLX descriptor plus cited SQLite facts that follow a pinned concept revision. | +| Project | A topic, its resource links, context documents, and agent sessions. | + +## Five minutes to start + +```sh +uv sync --all-extras +uv run dc init +uv run dc project create --project materials --title 'Materials Research' +uv run dc library add --source ./paper.pdf --project materials +uv run dc library search --query 'porous materials' --project materials +uv run dc session start --project materials --agent codex ``` -`add-resource` accepts one document or a folder of documents. With the -`conversion` extra installed, it accepts Docling's supported formats, including -PDF, DOCX, PPTX, XLSX, images, HTML, and plain text. Native DCLG and DCLX files -also work without that extra. Folder imports scan the top level by default. -Add `-r` or `--recursive` to include subfolders; those paths are retained in -the URI. Use repeatable `--from FORMAT` options to select Docling format names, -such as `--from pdf --from docx --from image` (default: all). Filtering uses -filename extensions; Docling then detects the actual input format. Files with -the same stem and different extensions receive distinct URIs. A single file -can use `--uri`; otherwise `--collection` defaults to `documents`. Use `--config` -for a [JSON conversion config](conversion.md), with command flags overriding it. -Use `--force` to create a new revision for unchanged input. -Table reconstruction always uses accurate mode; there is no CLI mode switch. -Page and picture images are on by default, and there is no default page limit. -`--no-page-images`, `--no-picture-images`, and `--max-pages N` override these -defaults. OCR uses RapidOCR with English PP-OCRv6 `tiny` models when enabled. - -Ingestion is synchronous. Each new resource revision queues a collection-summary -job; `add-resource` prints the revision and its `task_id`. `task status` reports -that job. Run `worker --once` for one due job, or `worker` continuously. A -collection summary may remain stale until its latest job completes. - -`outline URI` reads the TOC sidecar and `show URI XPATH` reads one cited node. -Use `--revision ID` for an immutable revision, `--format xml` for DocLang XML, -and `--max-chars N` to bound output. Both commands support `--json`; JSON -includes the URI, document ID, revision ID, XPath, and truncation marker. - -`docling-context` exposes the existing `dc` commands and adds `mcp`, -`integrate`, `doctor`, `remove`, and `ingest-worker`. See the -[agent harness guide](agents.md) for scopes, remote tokens, and verification. - -Every `status` command prints a table by default. Pass `--json` for structured -output, for example `dc status --json`, `dc task status JOB_ID --json`, or -`dc index vector status --json`. -The main `status` also shows session, memory, and memory compilation job counts -for the selected user. List commands use tables by default and JSON arrays with -`--json`. The session and memory lists support `--limit` and `--offset` pages; -the default page size is 100 and the maximum is 1,000. - -## Sessions and memory - -Start a session before recording events. `start --id` is idempotent for an open -session; omit `--id` to generate one. Session commands use the selected tenant -and user. `append` needs a stable `--key`, so a retried harness hook cannot -duplicate an event. Use `--stdin` for text that should stay out of process -arguments. An optional binary `--attachment` is stored as a typed DCLX part. - -```bash -uv run dc session start --id chat-1 --json -printf 'I prefer concise replies' | uv run dc session append chat-1 \ - --kind feedback --key feedback-1 --stdin --json -uv run dc session replay chat-1 -uv run dc session show chat-1 -uv run dc session list --status open --since 2026-10-01 -uv run dc session close chat-1 +The top level has one route for each kind of work: `init`, `overview`, `project`, `library`, `session`, `memory`, `concept`, `skill`, `knowledge`, `index`, and `agent`. Use `dc COMMAND --help` to inspect options. Agent and script callers can add `--json` to individual commands. + +Every value is passed by name; only command names are positional. Options that recur across commands have one-letter forms with the same meaning everywhere: + +| Short | Long | +| --- | --- | +| `-p` | `--project` | +| `-u` | `--uri` | +| `-d` | `--document` | +| `-i` | `--id` | +| `-x` | `--xpath` | +| `-q` | `--query` | +| `-s` | `--session` | +| `-a` | `--agent` | +| `-j` | `--json` | + +To see every command and nested subcommand at once, run `uv run python scripts/dump_cli_tree.py`. Add `--show-options` to include their arguments. The script reads the CLI parser, so the tree follows command changes. + +## `init` and `overview` + +`init` creates the SQLite catalog and package store. `overview` lists every project with its linked resource counts and also reports store and job counts. Bare `dc` displays the same overview with starting examples. + +```sh +uv run dc init +uv run dc overview +uv run dc overview --json +``` + +## `project` + +Projects define topics and membership. Every project links the shared `doclang-authoring` skill. Project context consists of an optional description and generated abstract and overview documents. + +| Subcommand | Use | +| --- | --- | +| `create -p ID --title TITLE` | Create a project. | +| `show -p ID` | Read its title, context URIs, and counts of linked documents, memories, skills, concepts, knowledge datasets, and sessions. | +| `resources -p ID` | List its linked library documents, memories, skills, concepts, and knowledge datasets. | +| `link -p ID -u URI` / `unlink -p ID -u URI` | Change membership. | +| `context set -p ID --description FILE` | Set a DCLG or DCLX description; `--abstract` and `--overview` also work. | +| `context show -p ID` | Read its context documents. | +| `context refresh -p ID` | Queue regeneration of the abstract and overview inventory. | +| `context jobs -p ID` / `context worker` | Inspect or process regeneration jobs. | + +```sh +uv run dc project create --project oilgas --title 'Oil and Gas' +uv run dc project link --project oilgas --uri docling://resources/library/DOC_ID +uv run dc project resources --project oilgas +uv run dc project context set --project oilgas --description ./description.dclx +uv run dc project context worker +``` + +The generated abstract and overview describe the project's title, description, counts, and linked URIs. They do not summarize the contents of its source documents. + +## `library` + +The library owns original source documents and their discovery. Imported content is deduplicated by binary hash. `--project` links the imported document to a project. A repeat import reuses the library ID; conversion options can still produce a new revision. + +| Subcommand | Use | +| --- | --- | +| `add --source SOURCE` | Import a local file, folder, or HTTP(S) URL. Use `-r` for nested folders. | +| `list [--project ID]` | List source documents. | +| `projects -d ID` | List projects linked to a document. | +| `links -d ID` / `link -d ID --target URI --relation NAME` | Inspect or create document relationships. | +| `tree [-u URI]` | Walk a URI subtree. | +| `outline -u ID` | Read a document's table of contents. | +| `show -u ID -x XPATH` | Read one cited node. Full URIs also work. | +| `find --term TERM` / `grep --term TERM` | Bounded substring lookup. `find` ranks by occurrence count; `grep` keeps scan order. | +| `search -q QUERY` | Indexed lexical, vector, or hybrid search with exact citations. | +| `jobs status -i JOB_ID` | Inspect an asynchronous ingestion job created by an agent or API. | + +```sh +uv run dc library add --source ./papers -r --project oilgas +uv run dc library add --source 'https://example.org/paper.pdf' --project oilgas +uv run dc library list --project oilgas +uv run dc library search --query 'Khurmala diagenesis' --project oilgas --mode hybrid +uv run dc library outline --uri DOC_ID +uv run dc library show --uri DOC_ID --xpath '/doclang[1]/text[1]' +``` + +`library add` converts synchronously. Its terminal progress bar counts PDF pages and advances after each document is reused or converted and committed. Other source types count as one unit. The bar reaches 100% after the whole import finishes; restart the same command after interruption. Conversion settings are in [conversion.md](conversion.md). `--json` keeps import output as JSON lines and hides the progress bar. + +Search covers all indexed nodes in the selected project by default. `--tier 0`, `--tier 1`, and `--tier 2` narrow it to summary, outline, and source nodes. `-k` controls the returned hit count. Search warns when an index is stale and gives its rebuild command. See [retrieval.md](retrieval.md). + +## `session` + +One session command family handles both interactive and manual sessions. `start` installs the agent's managed skill and MCP connection, starts a project session, launches the agent in the current terminal, and returns after the agent exits. `--no-launch` creates a session record for an agent already running. + +| Subcommand | Use | +| --- | --- | +| `start -p ID -a NAME` | Launch Codex, Claude, Hermes, or Pi. | +| `list [--project ID]` | List sessions and their agent and state. | +| `show -i ID` / `replay -i ID` | Inspect session metadata or events. | +| `append -i ID --text TEXT --kind KIND --key KEY` | Save an event with a stable key. | +| `close -i ID` | Close the session and queue memory compilation. | +| `jobs -i ID` | Inspect memory compilation jobs. | +| `trim`, `attachment`, `delete`, `purge` | Manage events and lifecycle. | + +```sh +uv run dc session start --project oilgas --agent claude +uv run dc session start --project oilgas --agent codex --no-launch +uv run dc session list --project oilgas +uv run dc session append --id SESSION_ID --text 'Reviewed section 2' --kind turn --key review-1 +uv run dc session close --id SESSION_ID +``` + +A returned session ID works directly if it uniquely identifies one of your project sessions. Use the full session URI when IDs are ambiguous. Agents record useful outcomes through MCP; the launcher does not automatically copy the agent's complete conversation. See [sessions.md](sessions.md) and [agents.md](agents.md). + +## `memory` + +A memory is a durable conclusion or synthesis. Session compilation can propose one; an accepted memory enters recall and search. Direct creation can use a claim or an authored DCLG/DCLX document. With `--from`, the first `` node is the concise claim and the original document body is retained across review. + +| Subcommand | Use | +| --- | --- | +| `create [KIND] CLAIM [--project ID]` | Create a proposed memory; `--accepted` accepts a user-authored claim immediately. | +| `create --from FILE [-p ID]` | Create from DCLG/DCLX. | +| `list [--project ID]` / `show ID` | Inspect memories and review state. | +| `proposals`, `accept`, `reject`, `correct` | Review or correct a memory. | +| `search -q QUERY [-p ID]` / `recall -q QUERY [-p ID]` | Find accepted conclusions; recall suppresses repeats per session. | +| `worker --once` / `jobs SESSION_ID` | Process or inspect compilation. | +| `delete` / `purge` | Remove a memory or its history. | + +```sh +uv run dc memory create --from ./diagenesis-summary.dclx --project oilgas +uv run dc memory list --project oilgas +uv run dc memory search --query 'Khurmala' --project oilgas +uv run dc memory accept --uri MEMORY_ID uv run dc memory worker --once -uv run dc memory proposals ``` -Set `DOCLING_CONTEXT_SESSION=chat-1` to omit the ID from `session append`, or -use `--text` when sending a short event without stdin. `session close` prevents -new events and queues compilation of the final revision. `session jobs ID` and -`memory jobs ID` show compilation outcomes. `memory worker` polls for jobs; -`--once` handles one due job and exits. Queued jobs for older session revisions -become `superseded` when a newer revision is queued. - -`session list` filters by `--status open|closed` and `--since`/`--until` ISO -dates or timestamps. `session trim ID --keep-last N` creates a shorter current -revision. Use `session attachment ID KEY --output FILE` to save one attached -event payload. - -```bash -uv run dc memory list --kind preference --status proposed -uv run dc memory show MEMORY_URI --json -uv run dc memory accept MEMORY_URI -uv run dc memory search 'concise replies' -uv run dc memory recall 'concise replies' --session chat-1 -uv run dc memory correct MEMORY_URI 'I prefer brief answers' -uv run dc memory reject MEMORY_URI --reason 'Incorrect inference' -uv run dc memory delete MEMORY_URI -uv run dc memory purge MEMORY_URI +See [memory.md](memory.md) for citations and review behavior. + +## `concept` + +A concept defines what facts mean. Create one from a DCLG or DCLX document and a mandatory schema descriptor. Its `schema` defines nonempty `entity_types`, typed `properties`, optional units, and optional `relationship_types` with source and target entity types. + +```json +{ + "name": "Formation properties", + "summary": "Observed properties of geological formations", + "schema": { + "entity_types": ["formation"], + "properties": {"porosity": {"type": "number", "unit": "%"}}, + "relationship_types": {"overlies": {"source": "formation", "target": "formation"}} + } +} +``` + +```sh +uv run dc concept create --id formation-properties --from ./formation-properties.dclg \ + --metadata ./formation-properties.json --project oilgas +uv run dc concept show --id formation-properties +uv run dc concept list --project oilgas +uv run dc concept projects --id formation-properties +``` + +`create` also accepts a DCLX that already contains `context/concept.json`. Both sources are validated before storage. `links` and `link` inspect or create relationships with other resources. + +## `skill` + +Skills are shared, named DCLX methods. Create one from DCLG or DCLX, optionally supplying descriptor JSON with a name and summary. Every project automatically links `doclang-authoring`. + +```sh +uv run dc skill create --id sampling-method --from ./sampling-method.dclx --project oilgas +uv run dc skill show --id sampling-method +uv run dc skill list --project oilgas +``` + +`projects`, `links`, and `link` inspect memberships and resource relationships. + +## `knowledge` + +A knowledge dataset contains individually cited facts in SQLite and a DCLX descriptor in `resources/knowledge/`. Creation requires a concept. The dataset pins that concept's URI and immutable revision; a later concept edit does not change the dataset's schema. One concept can define several datasets in different projects. + +| Subcommand | Use | +| --- | --- | +| `create -i ID --concept CONCEPT_ID [-p ID]` | Create a dataset using a validated concept. | +| `show -u ID` / `list [-p ID]` | Inspect the pinned concept and datasets. | +| `add-fact -u ID --entity-type TYPE --entity-id ID --property NAME --value JSON` | Validate and store one fact with an exact citation. Supply `--unit` when the property defines one. | +| `facts -u ID [--entity ENTITY_ID] [--property NAME]` | List or filter cited facts. | + +```sh +uv run dc knowledge create --id khurmala-properties --concept formation-properties --project oilgas +uv run dc knowledge add-fact --uri khurmala-properties --entity-type formation --entity-id khurmala --property porosity --value 12.5 \ + --unit '%' --source-uri docling://resources/library/DOC_ID \ + --source-revision REVISION_ID --source-xpath '/doclang[1]/table[1]' +uv run dc knowledge facts --uri khurmala-properties --entity khurmala +``` + +Facts must match the pinned entity type, property, value type, and unit. Every source citation identifies a document URI, immutable revision, and XPath. Existing datasets created without a concept remain readable but cannot receive new facts; create a concept-backed dataset to continue writing. + +## `index` + +Inspect and rebuild search storage. Search does not repair stale indexes during a query. + +```sh +uv run dc index status +uv run dc index lex rebuild +uv run dc index vector rebuild +``` + +`index lex status` and `index vector status` show the two indexes separately. Install the vector extra before building embeddings: `uv sync --extra vectors`. + +## `agent` + +Install or inspect the `/dc` skill and MCP connection in an agent. These commands do not launch an agent; `session start` does. + +```sh +uv run dc agent install --agent codex +uv run dc agent status --agent codex +uv run dc agent install --agent claude --scope project +uv run dc agent install --agent hermes +uv run dc agent install --agent pi --scope project +uv run dc agent remove --agent codex ``` -`memory list` includes every non-purged status by default, including rejected -and deleted claims. Filter it by `--status`, `--kind`, `--since`, or `--until`. -`memory proposals` is a shortcut for proposed claims. `memory create KIND CLAIM` -creates a user-authored proposal; `--accepted` accepts it immediately. Optional -source citations require all four flags: `--source-uri`, -`--source-document-id`, `--source-revision-id`, and `--source-xpath`. -`memory search` and `memory recall` return accepted claims only. Recall records -which memory revision was delivered to a session and suppresses repeats. - -### Delete and purge - -`delete` withdraws an item from normal use while retaining its history. `purge` -removes that item's stored revisions and content. Both commands are scoped to -the selected tenant and user. - -| Command | Immediate effect | What remains | -| --- | --- | --- | -| `dc session delete ID` | Removes the current session from `session list`, `show`, and `replay`. | Historical revisions and their attachments remain stored; compilation jobs and the session's recall ledger remain. | -| `dc session purge ID` | Deletes the session if necessary, then removes its revision records, attachments, compilation jobs, and recall ledger. | Derived memories are separate packages. The change outbox retains operation metadata. | -| `dc memory delete URI` | Writes a new revision with status `deleted`. The claim leaves memory search and recall. | The current claim and review history remain visible through `memory show URI` and `memory list --status deleted`; older revisions remain stored. | -| `dc memory purge URI` | Removes a previously deleted claim's current package, revisions, search index entries, provenance links, and recall records. | The source session or document and change outbox metadata remain. | - -`memory purge` requires `memory delete` first. `session purge` can be called on -an active or previously deleted session. Removing a source session causes -dependent accepted memories with no other surviving citation to return to -`proposed` for review; it does not purge those memory packages. Purging a memory -does not erase its wording from a cited session or document. If that source is -later compiled again, it can produce a new proposal. - -Package blobs are content addressed and are removed by purge only when no -remaining revision references them. The outbox retains earlier put/delete -identifiers and operation metadata; purge does not clear it. Purge removes the -item's stored content and revisions, but does not erase every related record. - -`find` and `grep` are case-insensitive substring scans of current documents. -They scan at most 1,000 documents, 2,000 nodes per document, and 50,000 nodes -total, returning at most 100 hits. They accept -an optional URI subtree scope and enforce tenant and user access. They do not -yet provide embedding search or relevance ranking. When scoped to memory -packages, they skip claims that are not accepted. - -`search` uses the SQLite retrieval index and returns exact DCLX citations. -New or changed documents are indexed during ingestion. `index status` reports -both lexical and vector coverage. `index lex status` and `index vector status` -show each index separately. `index lex rebuild` regenerates the SQLite index -from stored DCLX packages without rerunning conversion or OCR. It invalidates -an existing vector snapshot; a configured embedding provider rebuilds it on its -next use. Status and lexical rebuild honor the selected tenant and user scope. - -The CLI defaults to lexical search until a vector model has been built, then -uses hybrid search. `--mode lexical|vector|hybrid` selects a mode explicitly. -Install the `vectors` extra and run `dc index vector rebuild` to generate -embeddings with the local default `BAAI/bge-small-en-v1.5` model. -`--model MODEL` selects another supported FastEmbed text model; -`Qdrant/clip-ViT-B-32` selects paired -text and image encoders. The first build downloads model weights unless -`--offline` is passed. On Apple Silicon, `--backend auto` selects MLX for -supported BGE models and uses ONNX for other models. Use `--backend mlx` or -`--backend onnx` to choose explicitly. MLX uses separate model weights and -cannot be combined with `--offline`; `--backend auto --offline` uses ONNX. -The selected backend is saved with the vector generation, so subsequent -searches use the same encoder. Both lexical and vector rebuilds display a -progress bar on interactive terminals; progress goes to stderr. -After a vector build, document writes and deletes replay from the outbox on -the next vector search. Missing or corrupt snapshots trigger a full rebuild. -`dc index vector status` reports embedding coverage and the -stored model. `search --image PATH --mode vector` accepts an image query with -CLIP; text and image may be supplied together. A text-only model skips the -image input. Vector builds currently require all documents in the store to be -accessible to the selected principal because the snapshot is store-wide. -With an image-capable model, referenced picture assets are indexed alongside -text nodes. Search results include the picture XPath and asset path; image-only -results show an image marker in the table. After upgrading an existing store, -run `dc index lex rebuild` and `dc index vector rebuild` to add picture assets. - -Use `--uri`, `--document`, and `--xpath` to restrict scope; repeat `--tier` -to select context tiers. `-k`, `--max-tokens`, and `--per-result-tokens` -control output size. It prints a compact table by default; pass `--json` for -full citation records. See [scoped retrieval](retrieval.md) for the Python -vector and hybrid API. +See [agents.md](agents.md) for each agent's setup and the MCP workflow. diff --git a/docs/conversion.md b/docs/conversion.md index 00170c2..f390f3b 100644 --- a/docs/conversion.md +++ b/docs/conversion.md @@ -2,7 +2,7 @@ Install local document conversion with `uv sync --extra conversion`. Pass a `PdfConversionConfig` to `LocalDoclingConverter`, or supply a JSON file to -`dc add-resource --config`. Docling runs locally by default; this adapter keeps +`dc library add --config`. Docling runs locally by default; this adapter keeps external services and external plugins disabled. The conversion extra includes RapidOCR and its ONNX Runtime engine for the default OCR configuration. The first OCR run may download model weights; later runs use the local copy. @@ -47,5 +47,7 @@ The opt-in `DoclingServeConverter(endpoint, config=config)` forwards supported OCR, table, chart, code, formula, and text options to the service. Local limits such as `max_pages`, model artifact paths, and image generation settings are not enforced by the service adapter; configure those on the service separately. -Only explicit HTTPS or loopback HTTP endpoints are accepted, and redirects are -rejected. Source URLs are never fetched during ingestion. +Only explicit HTTPS or loopback HTTP conversion-service endpoints are accepted, +and redirects are rejected by that adapter. Separately, `dc library add --source URL` +downloads an explicitly supplied HTTP(S) source into bounded local bytes before +using the local converter. See the [CLI guide](cli.md) for URL limits. diff --git a/docs/index.md b/docs/index.md index bc55350..3628fbc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,33 +1,102 @@ -# Capabilities - -`docling-context` stores document revisions as DocLang DCLX packages and exposes -them through Python, `dc`, MCP, and authenticated HTTP APIs. - -| Capability | Current behavior | -| --- | --- | -| Storage | SQLite metadata and outbox; immutable, SHA-256-addressed DCLX packages | -| Namespaces | Tenant-scoped resources and user-scoped memories, sessions, and skills | -| Ingestion | Native DCLG and DCLX, plus Docling formats through optional local conversion | -| Context tiers | L0 document summary, L1 heading TOC, L2 source document in one package | -| Collection summaries | Durable jobs with retries, leases, and visible stale status | -| Conversion | Typed PDF and image settings for OCR, tables, charts, enrichment, images, and limits | -| Remote conversion | Opt-in Docling Serve adapter; explicit endpoint and bounded response | -| Inspection | `dc status`, `ls`, `tree`, and task status | -| Search | `dc search` provides scoped lexical, vector, and hybrid retrieval, including picture assets with an image-capable model | -| Sessions | `dc session` records, lists, replays, closes, and purges revisioned event packages | -| Durable memory | `dc memory` lists and reviews DCLX claims, runs compilation jobs, and supports scoped recall | -| Agent harnesses | One-command MCP integration for Codex, Claude Code, Hermes, and Pi | -| Public service | Versioned Python and authenticated HTTP operations for inspection, search, ingestion, jobs, sessions, and memory | - -`find` and `grep` retain their bounded substring scans. Indexed retrieval -verifies citations against the current DCLX revision before returning text. +# docling-context +`docling-context` turns source files into citable DCLX packages. Resources are +shared; a project represents a topic of work and links only the resources it +needs. One library document, memory, skill, or concept can support several +projects without being copied. SQLite records resource identity, links, jobs, +sessions, knowledge facts, and search indexes in one local store. + +```text +docling:// +├── resources/ +│ ├── library/{doc_id} original sources converted to DCLX +│ ├── memories/{memory_id} session outcomes and conclusions +│ ├── skills/{skill_id} reusable methods +│ ├── concepts/{concept_id} entity and relationship definitions +│ └── knowledge/{knowledge_id} structured knowledge descriptors +├── projects/{project_id}/ +│ ├── .description.dclx optional short description +│ ├── .abstract.dclx title, description, and linked counts +│ ├── .overview.dclx linked resource inventory +│ └── sessions/{session_id} +└── user/{user_id}/ +``` + +Each resource type is a flat list. Project membership and typed relationships +between resources live in SQLite. `dc overview` shows every project and counts +its linked documents, memories, skills, concepts, knowledge descriptors, and +sessions. Each project starts with the shared `doclang-authoring` skill linked, +so a new project's skill count is one. Project context documents do not count +as linked resources. + +## 5 minutes to fun + +Install Python 3.12+ and `uv`, then run from this repository: + +```bash +uv sync --extra conversion +uv run dc init +uv run dc project create --project materials --title "Materials research" +uv run dc library add --source ./paper.pdf --project materials +uv run dc overview +uv run dc library search --query "material properties" --project materials +``` + +You can import a document directly from an HTTP(S) URL, including PDF URLs +without a `.pdf` suffix: + +```bash +uv run dc library add --source https://arxiv.org/pdf/2608.23083 --project materials +``` + +`library add` prints the canonical library URI and revision. It checks the +source binary hash and reuses the existing library document on a repeat import, +whether the source is a local file or a URL. Search results include URI, +revision ID, and XPath. Read a result with +`uv run dc library outline --uri URI` and `uv run dc library show --uri URI --xpath XPATH --revision REVISION_ID`. +Add `--json` to commands when an agent or script needs structured output. + +If you have no PDF, use a native DocLang source without the conversion extra: + +```bash +printf 'Basalt is an igneous rock.' > basalt.dclg +uv run dc library add --source ./basalt.dclg --project materials +uv run dc library search --query basalt --project materials +``` + +The default store is `~/.local/share/docling-context`. `dc init` creates its +SQLite database and package directory. Use `--store PATH` or set +`DOCLING_CONTEXT_STORE` to select another location, and use the same setting +for the CLI, workers, and MCP server. No PostgreSQL setup is required. + +## Connect an agent + +Start an agent inside a project session. The agent runs in your current +terminal; when it exits, you return to your shell and the session closes: + +```bash +uv run dc session start --project materials --agent codex +``` + +Alternatively, install Docling Context's MCP tools and agent skill, then start +the agent yourself. For Codex, invoke the skill with `$dc`; in Claude Code, +Hermes, and Pi, use `/dc`: + +```bash +uv run dc agent install --agent codex --scope project +codex +``` + +Both paths support Codex, Claude Code, Hermes, and Pi. See +[agent installation](agents.md) for all four recipes, scope and command details, +and remote deployment. + +## Guides + +- [CLI and project workflows](cli.md) +- [Storage, revisions, and jobs](storage.md) +- [Scoped retrieval and citations](retrieval.md) - [Conversion settings](conversion.md) -- [Scoped retrieval](retrieval.md) -- [Retrieval benchmark](../examples/retrieval/benchmark_corpus.py) -- [CLI commands](cli.md) -- [Storage and jobs](storage.md) - [Agent sessions](sessions.md) - [Durable memory](memory.md) -- [Agent harness installation and MCP](agents.md) -- [Single PDF and folder examples](../examples/adding_resources/README.md) +- [Agent installation and MCP](agents.md) diff --git a/docs/memory.md b/docs/memory.md index a7b77ce..a641328 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -1,7 +1,12 @@ # Durable memory +Reviewed memories and compiled project session outcomes are DCLX packages in +the flat `docling://resources/memories/{memory_id}` list. A memory may link to +several projects. `dc memory create --from FILE --project ID` accepts authored DCLG or DCLX. +The first text node is its short claim; the original document body is retained. + `MemoryService` stores each claim as a DCLX package under -`docling://users/{tenant}/{user}/memories/{kind}/{id}`. Supported kinds are +`docling://resources/memories/{memory_id}`. Supported kinds are `preference`, `fact`, `entity`, `concept`, and `procedure`. The DocLang body contains the readable claim and explanation. `context/memory.json` stores confidence, status, author, optional validity dates, source citations, conflict @@ -43,7 +48,7 @@ pass `accepted=True`. Compiler proposals require `accept` before they enter search. `reject`, `correct`, and `delete` write new package revisions. A correction is user authored, accepted, and records the revision it supersedes. `purge` physically removes a deleted claim and its revisions. `search` returns -accepted memories within the current user. `recall(principal, session_id, query)` +accepted memories within the tenant. `recall(principal, session_id, query)` records delivered memory and revision IDs; it suppresses repeats in that session, while a corrected revision can be delivered again. `list` includes all non-purged statuses by default and can filter by status, @@ -52,5 +57,6 @@ search, recall, and purge; see the [CLI guide](cli.md). Deleting a cited session or document checks dependent memories. A claim with no surviving cited source becomes `proposed` and leaves the retrieval index. -All operations check tenant and user ownership. Sharing is not exposed by this -API. +Reads are tenant scoped. Review and purge require the memory's owner. +Closing a project session queues compilation; the worker links each resulting +memory to that project. diff --git a/docs/retrieval.md b/docs/retrieval.md index 427cf25..06bff65 100644 --- a/docs/retrieval.md +++ b/docs/retrieval.md @@ -1,9 +1,9 @@ # Scoped retrieval -`dc search` uses the local SQLite index and works without an embedding model: +`dc library search` uses the local SQLite index and works without an embedding model: ```bash -uv run dc search "glacier" --uri docling://resources/articles \ +uv run dc library search --query "glacier" --project climate \ --xpath '/doclang[1]/text[1]' -k 5 --max-tokens 1000 ``` @@ -11,8 +11,8 @@ Ingestion indexes each document automatically. Run `dc index status` to inspect both indexes, or `dc index lex rebuild` to regenerate lexical records from stored DCLX packages. Rebuilding the SQLite index does not generate embeddings. Run `uv run --extra vectors dc index vector rebuild` to generate them with the default -local model, then `dc search` uses hybrid mode automatically. Use -`dc search --mode lexical` to request lexical search explicitly. +local model, then `dc library search` uses hybrid mode automatically. Use +`dc library search --mode lexical` to request lexical search explicitly. On Apple Silicon, vector rebuilding selects MLX for supported BGE models when installed; other models use FastEmbed's ONNX runtime. The backend is part of the stored model ID, so changing backends regenerates vectors. Both rebuild @@ -25,18 +25,25 @@ the first and last 64 characters separated by ` ... `. Use `--json` for the complete citation records, including URI, revision, page, bounding box, full bounded text, and truncation status. A footer reports how many nodes are shown out of all indexed lexical matches in scope, before result and token limits. +Results are ordered by descending `Score`: lexical score plus nonnegative +vector score and the `Tier`, `Fresh`, and `Near` bonuses. The `Vec` column alone +may therefore rise or fall between adjacent rows. In vector mode, `Lex` is zero. +Search considers every current indexed node in the selected scope. The `-k` +option limits returned citations, not the nodes examined for a match. `--document` restricts a query to one URI; `--tier` can be repeated. `--max-tokens` and `--per-result-tokens` limit the returned text using whitespace-delimited tokens. Multiword lexical searches first look for nodes containing every query word; if none match, they use any-word matching. The footer counts matches for the selected lexical query. -The `@summary:` and `@toc:` XPath prefixes denote sidecars within the cited -DCLX revision. The default summary is the first 600 characters of extracted -heading and text content, so it can include publisher boilerplate. The TOC is +The `@summary:` and `@toc:` XPath prefixes denote the summary and outline +within the cited DCLX revision. Validated outlines are stored as native +`annotations/toc.dclg` companion documents. The default +summary is the first 600 characters of extracted heading and text content, so +it can include publisher boilerplate. The outline is made from headings and may duplicate a heading hit. Source XML is never -rewritten for indexing. Matching summary and outline hits are assembled before -body nodes, with relevance ordering within each tier. +rewritten for indexing. Summary and outline hits receive tier bonuses, while +all hits compete in the same score order. The Python API supports lexical, vector, and hybrid modes: @@ -47,7 +54,7 @@ with LocalContextStore("./context-data") as store: result = Retriever(store).search( Principal("default", "local"), "glacier", - scope=SearchScope(uri="docling://resources/articles"), + scope=SearchScope(project_id="climate"), mode="lexical", ) ``` @@ -58,11 +65,13 @@ and hybrid queries. The provider implements `model_id`, `dimensions`, and stable `uint64` IDs from SQLite, contiguous `float32` arrays, L2 normalization, and `IdMapIndex` with a SQL-derived allowlist. Dimensions must be a multiple of eight, from 8 to 16,384. One model and dimension generation is active per -store. Replacing the provider rebuilds the vector generation. +store. Run `dc index vector rebuild --model MODEL` to change the provider. ```python retriever = Retriever(store, embedder=my_embedding_provider) result = retriever.search(principal, "glacier", mode="hybrid") +for warning in result.warnings: + print(warning) ``` The built-in local adapter supports direct text, image, and combined inputs: @@ -92,9 +101,13 @@ For a picture hit, `store.get_document(principal, hit.uri).get_part_bytes(hit.as reads the cited image from its DCLX package. SQLite stores vector metadata and embedding bytes. The turbovec snapshot is a -rebuildable projection. A missing or corrupt snapshot triggers a full rebuild; -compatible snapshots replay later outbox changes for only the affected URIs. -Changing the embedding model also requires a full rebuild. Search checks each +rebuildable projection. A missing or corrupt snapshot stops vector search with +instructions to run `dc index vector rebuild`. An incomplete snapshot produces +a warning and uses its saved vectors until rebuilt. Search never updates vector +storage implicitly. Changing the embedding model also requires a full rebuild. +Lexical search warns about missing or outdated document records and tells you +to run `dc index lex rebuild`; it does not repair records during search. +Search checks each candidate against the current document revision and reads cited text from DCLX, so stale index records are excluded. The retriever enforces tenant, URI subtree, optional document, XPath subtree, @@ -119,7 +132,7 @@ an existing store, create a JSON list such as: [ { "query": "Koonap formation in the Karoo Basin", - "relevant_uris": ["docling://resources/documents/example-paper"] + "relevant_uris": ["docling://resources/library/example-paper"] } ] ``` diff --git a/docs/sessions.md b/docs/sessions.md index 688e24e..ca8e60e 100644 --- a/docs/sessions.md +++ b/docs/sessions.md @@ -1,5 +1,26 @@ # Agent sessions +Project-owned sessions use +`docling://projects/{project_id}/sessions/{session_id}`. Start one and launch +an agent with `dc session start --project materials --agent codex`. The CLI +installs the agent's MCP connection and `dc` skill, starts the project session, +waits for the agent, then closes the session when it exits. Use +`dc session list --project materials` to see the agent and state. The `--no-launch` flag creates a project session for an agent already running. + +When an agent is already running, its installed `dc` skill uses MCP +`project_session_start`, `session_append`, and `project_session_close`. The +launcher passes `DOCLING_CONTEXT_PROJECT` and the full +`DOCLING_CONTEXT_SESSION` URI to its child. The child should append useful +outcomes; an agent conversation is not automatically copied into DCLX. The +project's session count appears in `dc overview`. + +Closing a project session queues compilation of shared memory proposals; run +`dc memory worker --once` to process the queue. Project abstract/overview +jobs and ingestion jobs are separate queues. + +The Python `SessionStore` still supports user-scoped private sessions. The +human-facing `dc session start` path now requires a project and agent. + `SessionStore` appends agent turns, tool calls, tool results, and feedback to a revisioned DCLX package at `docling://users/{tenant}/{user}/sessions/{session_id}`. Each event is a DocLang diff --git a/docs/storage.md b/docs/storage.md index d4907ee..eb081d7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,46 +1,60 @@ -# Storage, revisions, and jobs - -`LocalContextStore` stores metadata in SQLite and DCLX blobs under SHA-256 -paths. A logical URI points to the latest revision; old revisions remain -addressable by document ID, revision ID, and XPath. Every write also inserts -an outbox event in the same SQLite transaction. Blob reads verify their hash. - -Resource URIs have the form `docling://resources/{collection}/{path}`. User -URIs have the form -`docling://users/{tenant}/{user}/{memories|sessions|skills}/{path}`. Tenant -ownership is checked on reads and writes. Local file ingestion requires an -explicit `allowed_roots` entry; bytes can be ingested without filesystem access. - -Each ingested document is a DCLX package containing `document.xml` (L2), a -DocLang summary (L0), and a DocLang TOC (L1). Source assets and sidecars other -than the generated summary and TOC are kept from DCLX input. `context/manifest.json` records the -summary method and exact source revision. Source hashes, origin, converter -version/options, and warnings are in the revision's provenance. - -Collection summaries are separate DCLX documents at -`docling://resources/{collection}/_context/summary`. Writes and deletes mark -the collection stale and enqueue a durable job. `CollectionWorker` claims jobs -with leases, retries failures, and can resume after a crash without adding a -duplicate summary revision. Its `status` method exposes staleness and the -summary revision ID. - -Schema version 3 adds `retrieval_units`, an FTS table, and `vector_state`. -Schema version 4 adds `vector_members`, which records the stable IDs in each -snapshot so later outbox events can remove replaced or deleted nodes. -Schema version 5 records picture asset paths on retrieval units. Existing -stores can run `dc index lex rebuild` to add picture units from saved DCLX -packages, then `dc index vector rebuild` to embed them with an image-capable -model. -Schema version 6 records each source node's active heading in `section_xpath`. -Rebuild the lexical index after upgrading an existing store to populate that -field for previously indexed nodes. -Document writes replace their retrieval records in the same SQLite transaction -as the revision and outbox event; deletes remove those records. Existing stores -are backfilled from their DCLX packages on upgrade. The FTS table is a -rebuildable lexical projection. Optional vector snapshots are held in -`vectors-{generation}.tvim` files and can be reconstructed from SQLite -embedding bytes. The package and current revision remain authoritative for -all citations. - -The local and in-memory stores implement the same basic `ContextStore` -contract, leaving room for another storage backend later. +# Storage and project catalog + +The local store contains `context.sqlite3` and content-addressed DCLX packages. +SQLite holds projects, resource identities and revisions, project membership, +typed resource links, project sessions, cited knowledge facts, jobs, and search +indexes. `dc init` creates the store at `~/.local/share/docling-context` by +default. Use `DOCLING_CONTEXT_STORE` or `dc --store PATH` to select another +location. Back up the SQLite database and package directory together. + +## Shared resources + +Library documents live at `docling://resources/library/{doc_id}`. The other +flat lists are `memories`, `skills`, `concepts`, and `knowledge`. A package +contains DocLang content (L2), a summary (L0), and a validated DocLang outline +(L1). The summary and outline are native DCLG companion documents at +`annotations/summary.dclg` and `annotations/toc.dclg`. +The source binary SHA-256 is distinct from the DCLX package hash. `library add` checks +the source hash within the tenant before conversion and reuses the library URI +when it exists. `--force` writes a new revision. Revision provenance records +the original filename, origin, converter, options, and warnings. + +Use `dc library add --source SOURCE` for original documents, `dc skill create` for methods, +`dc concept create` for definitions, `dc memory create` for conclusions, and +`dc knowledge create` for a dataset of cited facts. A concept has a mandatory +schema in its DCLX descriptor. Each knowledge dataset pins the concept URI and +revision, so later concept edits cannot silently change fact validation. +`dc knowledge add-fact` checks entity type, property, value type, and unit +against that pinned schema before storing an exact source citation. Resource +links between datasets and concepts are recorded in SQLite. + +## Projects + +`dc project create --project ID --title TITLE` creates a project and links the shared +`docling://resources/skills/doclang-authoring` skill. The skill explains how +to write and validate DCLG XML and DCLX packages. It is required for every +project, including projects already present when a store is opened. `project +link` and `project unlink` change membership; unlinking does not delete a +resource. +`project resources` lists a project's links; `concept projects ID` or `skill projects ID` lists +projects linked to a typed resource. `project context set` attaches +project-owned DCLX documents. Project creation, +membership changes, and description changes queue abstract and overview +refresh jobs. Run `dc project context worker` to process one, `dc project context jobs --project ID` to +inspect them, or `dc project context refresh --project ID` to queue another refresh. The generated +abstract and overview contain the title, optional description, resource counts, +and URI inventory; they do not summarize source document contents. + +`dc overview` includes projects with no links and counts each linked resource +once. `dc library search --query QUERY --project ID` restricts retrieval to linked resources. +Results cite canonical URI, immutable revision ID, and XPath. + +Project sessions live at `docling://projects/{project_id}/sessions/{id}`. +`dc session start --project ID --agent codex` launches an agent and records its +session under that project. `dc session list --project ID` shows the agent and +state. Use `dc session start --no-launch` to create a session without launching an agent. Closing a +project session queues memory compilation; +run `dc memory worker --once` to produce proposed shared memories linked back +to the project. The sessions column in `dc overview` counts project sessions. + +This is a fresh-start layout. There is no migration tool for older stores. diff --git a/pyproject.toml b/pyproject.toml index c070aa6..99ba609 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,10 @@ dependencies = [ ] [project.optional-dependencies] -conversion = ["docling[rapidocr]>=2.134.0,<3"] +conversion = [ + "docling[rapidocr]>=2.138.0,<3", + "docling-parse>=7.22.3,<8", +] vectors = [ "numpy>=1.26", "turbovec>=1,<2", @@ -47,7 +50,7 @@ docling-context = "docling_context.app:main" Repository = "https://github.com/docling-project/docling-context" [tool.uv.sources] -doclang = { git = "https://github.com/doclang-project/doclang", rev = "92d299026f2f985328b52424483c711e19b2f306" } +doclang = { git = "https://github.com/doclang-project/doclang", rev = "24b7ca6cc96e83af25ff9d67903e03c6d500c5e4" } [tool.docling-context.dependencies] doclang-license = "Apache-2.0" diff --git a/scripts/cli/dump_cli_tree.py b/scripts/cli/dump_cli_tree.py new file mode 100644 index 0000000..fdb9814 --- /dev/null +++ b/scripts/cli/dump_cli_tree.py @@ -0,0 +1,94 @@ +"""Print the complete command tree from the live ``dc`` argparse parser. + +Run with ``uv run python scripts/dump_cli_tree.py``. Add ``--show-options`` to +include the arguments accepted by each command. +""" + +from __future__ import annotations + +import argparse + +from docling_context.cli import _parser + + +def _subcommands( + parser: argparse.ArgumentParser, +) -> list[tuple[str, argparse.ArgumentParser, str]]: + commands = [] + for action in parser._actions: + if not isinstance(action, argparse._SubParsersAction): + continue + descriptions = { + choice.dest: choice.help + for choice in action._choices_actions + if choice.help is not argparse.SUPPRESS + } + for name, child in action.choices.items(): + commands.append((name, child, descriptions.get(name, ""))) + return commands + + +def _arguments(parser: argparse.ArgumentParser) -> str: + arguments = [] + for action in parser._actions: + if ( + isinstance(action, argparse._SubParsersAction) + or action.help is argparse.SUPPRESS + ): + continue + if action.dest == "help": + continue + if action.option_strings: + long_options = [ + value for value in action.option_strings if value.startswith("--") + ] + option = ( + "/".join(long_options) + if isinstance(action, argparse.BooleanOptionalAction) + else long_options[0] + if long_options + else action.option_strings[0] + ) + arguments.append(option) + else: + arguments.append(action.dest.upper()) + return ", ".join(arguments) + + +def _print_children( + parser: argparse.ArgumentParser, *, prefix: str = "", show_options: bool = False +) -> None: + children = _subcommands(parser) + for index, (name, child, description) in enumerate(children): + last = index == len(children) - 1 + line = f"{prefix}{'└── ' if last else '├── '}{name}" + if description: + line += f" {description}" + if show_options: + arguments = _arguments(child) + if arguments: + line += f" [{arguments}]" + print(line) + _print_children( + child, + prefix=prefix + (" " if last else "│ "), + show_options=show_options, + ) + + +def main() -> None: + options = argparse.ArgumentParser(description=__doc__) + options.add_argument( + "--show-options", action="store_true", help="Show arguments for each command" + ) + args = options.parse_args() + parser = _parser() + root = f"{parser.prog} {parser.description}" + if args.show_options: + root += f" [{_arguments(parser)}]" + print(root) + _print_children(parser, show_options=args.show_options) + + +if __name__ == "__main__": + main() diff --git a/src/docling_context/__init__.py b/src/docling_context/__init__.py index d713e67..01ba2a7 100644 --- a/src/docling_context/__init__.py +++ b/src/docling_context/__init__.py @@ -1,5 +1,6 @@ """Local-first context storage backed by DocLang packages.""" +from .catalog import Catalog, Project from .compiler import MemoryCompiler, MemoryJob, redact_secrets from .contracts import ContextStore from .conversion_config import PdfConversionConfig @@ -37,6 +38,7 @@ TreeEntry, ) from .package import FilePackageStore, PackageError, PackageMissing +from .project_context import ProjectContextWorker from .remote import RemoteContextService from .retrieval import ContextResult, RetrievalHit, Retriever, SearchScope from .service import ( @@ -44,11 +46,19 @@ ContextService, IngestRequest, JobStatusRequest, + KnowledgeFactRequest, + KnowledgeFactsRequest, ListRequest, MemoryListRequest, MemoryRecallRequest, MemoryReviewRequest, OutlineRequest, + OverviewRequest, + ProjectLinkRequest, + ProjectRequest, + ProjectSessionRequest, + ResourceLinkRequest, + ResourceLinksRequest, SearchRequest, ServiceError, ServiceResult, @@ -65,6 +75,7 @@ "DEFAULT_MODEL", "SCHEMA_VERSION", "AccessDenied", + "Catalog", "ChangeEvent", "CollectionStatus", "CollectionWorker", @@ -86,6 +97,8 @@ "InvalidURI", "Job", "JobStatusRequest", + "KnowledgeFactRequest", + "KnowledgeFactsRequest", "ListRequest", "LocalContextStore", "LocalDoclingConverter", @@ -101,13 +114,21 @@ "NodeAddress", "NodeContent", "OutlineRequest", + "OverviewRequest", "PackageError", "PackageMissing", "PackageRef", "PdfConversionConfig", "Principal", + "Project", + "ProjectContextWorker", + "ProjectLinkRequest", + "ProjectRequest", + "ProjectSessionRequest", "RecordMissing", "RemoteContextService", + "ResourceLinkRequest", + "ResourceLinksRequest", "RetrievalHit", "Retriever", "RevisionConflict", diff --git a/src/docling_context/agent_bridge.py b/src/docling_context/agent_bridge.py new file mode 100644 index 0000000..baede10 --- /dev/null +++ b/src/docling_context/agent_bridge.py @@ -0,0 +1,135 @@ +"""Install a project-aware skill beside the owned MCP connection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from .integration import IntegrationSpec, _atomic_write, inspect, integrate + +AGENTS = ("codex", "claude", "hermes", "pi") +SKILL_PATHS = { + "codex": Path(".agents/skills/dc/SKILL.md"), + "claude": Path(".claude/skills/dc/SKILL.md"), + "hermes": Path(".hermes/skills/dc/SKILL.md"), + "pi": Path(".pi/agent/skills/dc/SKILL.md"), +} + + +def skill_path(spec: IntegrationSpec) -> Path: + if spec.harness not in AGENTS: + raise ValueError(f"unsupported agent: {spec.harness}") + root = spec.project_root if spec.scope == "project" else spec.home + return root / SKILL_PATHS[spec.harness] + + +def skill_text(agent: str) -> str: + if agent not in AGENTS: + raise ValueError(f"unsupported agent: {agent}") + content = f"""--- +name: dc +description: Work with a Docling Context project, search its resources, read exact citations, and record a project session. +--- + +# Docling Context + +Use this skill when the user invokes dc or asks to work with a Docling Context project. This skill is installed for {agent}. + +1. Determine the project. Use §DOCLING_CONTEXT_PROJECT§ when set. Otherwise call the §overview§ MCP tool and ask the user to choose if several projects fit. Never silently search every project. +2. Determine the session. If §DOCLING_CONTEXT_SESSION§ is set, use that full URI and do not start another session. Otherwise call §project_session_start§ with the project ID, a fresh session ID, and §agent="{agent}"§. Keep the returned URI for later events. +3. Call §project_show§ and §project_resources§. Every project links §docling://resources/skills/doclang-authoring§. Before writing DCLG XML or a DCLX package, read it with §show(uri="docling://resources/skills/doclang-authoring", xpath="/doclang[1]/text[1]")§ and follow its validation instructions. Use §search(project_id=...)§ to find relevant nodes, then §show§ with the returned URI, revision ID, and XPath for exact citations. Use §outline§ to navigate a long document. +4. Record useful decisions, conclusions, and corrections with §session_append§ using the full session URI, a unique key, and a suitable event kind. +5. When you started the session yourself, call §project_session_close§ at the end. If §DOCLING_CONTEXT_SESSION§ was supplied by the launcher, leave closure to the launcher. + +For terminal work, use the canonical CLI groups: §dc library add/list/search/show§ for source documents, §dc project resources§ for membership, §dc memory§ for reviewed conclusions, §dc concept§ for schema definitions, and §dc knowledge§ for cited facts defined by a concept. §dc session start --project ID --agent {agent}§ launches an interactive session; do not run it from an already active agent session. + +If Docling Context MCP tools are missing, tell the user to run §dc agent install --agent {agent}§ and restart this agent. Do not invent citations or claim that unrecorded turns were captured. +""" + return content.replace("§", chr(96)) + + +def _marker(path: Path) -> Path: + return path.with_name(".docling-context-managed.json") + + +def _digest(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _owned(path: Path) -> bool: + marker = _marker(path) + if not path.exists() or not marker.exists(): + return False + try: + info = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + return info.get("sha256") == _digest(path.read_text(encoding="utf-8")) + + +def _check_skill(path: Path) -> None: + if path.exists() and not _owned(path): + raise ValueError(f"{path}: existing dc skill is not managed by this installer") + + +def _write_skill(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + previous = path.read_text(encoding="utf-8") if path.exists() else None + marker = _marker(path) + previous_marker = marker.read_text(encoding="utf-8") if marker.exists() else None + try: + _atomic_write(path, content) + _atomic_write( + marker, json.dumps({"sha256": _digest(content)}, sort_keys=True) + "\n" + ) + except OSError: + if previous is None: + path.unlink(missing_ok=True) + else: + _atomic_write(path, previous) + if previous_marker is None: + marker.unlink(missing_ok=True) + else: + _atomic_write(marker, previous_marker) + raise + + +def install_agent(spec: IntegrationSpec) -> dict[str, str]: + path = skill_path(spec) + content = skill_text(spec.harness) + _check_skill(path) + mcp = integrate(spec) + try: + _write_skill(path, content) + except OSError: + if mcp["status"] == "installed": + integrate(spec, remove=True) + raise + return {"status": mcp["status"], "mcp": mcp["path"], "skill": str(path)} + + +def inspect_agent(spec: IntegrationSpec) -> dict[str, str]: + path = skill_path(spec) + mcp = inspect(spec) + return { + "status": "configured" + if mcp["status"] == "configured" and _owned(path) + else "incomplete", + "mcp": mcp["path"], + "skill": str(path), + "skill_status": "managed" + if _owned(path) + else ("unmanaged" if path.exists() else "absent"), + } + + +def remove_agent(spec: IntegrationSpec) -> dict[str, str]: + path = skill_path(spec) + if path.exists() and not _owned(path): + raise ValueError(f"{path}: existing dc skill is not managed by this installer") + mcp = integrate(spec, remove=True) + if path.exists(): + path.unlink() + _marker(path).unlink(missing_ok=True) + return {"status": mcp["status"], "mcp": mcp["path"], "skill": str(path)} diff --git a/src/docling_context/catalog.py b/src/docling_context/catalog.py new file mode 100644 index 0000000..2cb7e32 --- /dev/null +++ b/src/docling_context/catalog.py @@ -0,0 +1,790 @@ +"""Shared resource and project catalog in the local SQLite store.""" + +from __future__ import annotations + +import json +import uuid +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Any, Self +from xml.sax.saxutils import escape + +from .doclang_authoring import ( + DEFAULT_SKILL_NAME, + DEFAULT_SKILL_SUMMARY, + DEFAULT_SKILL_TEXT, + DEFAULT_SKILL_URI, +) +from .models import DocumentRecord, NodeAddress, Principal +from .package import load_package +from .profiles import validate_descriptor +from .uri import parse_uri + +if TYPE_CHECKING: + from .local import LocalContextStore + +RESOURCE_TYPES = ("library", "memories", "skills", "concepts", "knowledge") +CATALOG_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True, slots=True) +class Project: + project_id: str + title: str + description_uri: str | None + abstract_uri: str | None + overview_uri: str | None + + +class Catalog: + def __init__(self, store: LocalContextStore): + self.store = store + self.db = store.db + + def __enter__(self) -> Self: + return self + + def __exit__(self, *_: object) -> None: + pass # The caller owns the LocalContextStore connection. + + def setup(self) -> None: + statements = ( + """CREATE TABLE IF NOT EXISTS dc_schema_version ( + version INTEGER PRIMARY KEY CHECK (version = 1))""", + """CREATE TABLE IF NOT EXISTS dc_projects ( + tenant_id TEXT NOT NULL, project_id TEXT NOT NULL, + title TEXT NOT NULL, description_uri TEXT, abstract_uri TEXT, + overview_uri TEXT, owner_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, project_id))""", + """CREATE TABLE IF NOT EXISTS dc_resources ( + tenant_id TEXT NOT NULL, resource_type TEXT NOT NULL + CHECK (resource_type IN + ('library','memories','skills','concepts','knowledge')), + resource_id TEXT NOT NULL, uri TEXT NOT NULL, + source_sha256 TEXT, current_revision TEXT NOT NULL, + package_hash TEXT NOT NULL, owner_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, resource_type, resource_id), + UNIQUE (tenant_id, uri), + FOREIGN KEY (tenant_id, uri) REFERENCES documents (tenant_id, uri))""", + """CREATE UNIQUE INDEX IF NOT EXISTS dc_library_source_hash + ON dc_resources (tenant_id, source_sha256) + WHERE resource_type='library' AND source_sha256 IS NOT NULL""", + """CREATE TABLE IF NOT EXISTS dc_resource_revisions ( + tenant_id TEXT NOT NULL, resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, revision_id TEXT NOT NULL, + package_hash TEXT NOT NULL, source_sha256 TEXT, + provenance TEXT NOT NULL, created_at TEXT NOT NULL + DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, revision_id), + FOREIGN KEY (tenant_id, resource_type, resource_id) + REFERENCES dc_resources (tenant_id, resource_type, resource_id) + ON DELETE CASCADE, + FOREIGN KEY (revision_id) REFERENCES revisions (revision_id))""", + """CREATE TABLE IF NOT EXISTS dc_project_resource_links ( + tenant_id TEXT NOT NULL, project_id TEXT NOT NULL, + resource_type TEXT NOT NULL, resource_id TEXT NOT NULL, + relation TEXT NOT NULL DEFAULT 'member', + created_by TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, project_id, resource_type, resource_id), + FOREIGN KEY (tenant_id, project_id) + REFERENCES dc_projects (tenant_id, project_id) ON DELETE CASCADE, + FOREIGN KEY (tenant_id, resource_type, resource_id) + REFERENCES dc_resources (tenant_id, resource_type, resource_id) + ON DELETE CASCADE)""", + """CREATE INDEX IF NOT EXISTS dc_links_by_resource + ON dc_project_resource_links + (tenant_id, resource_type, resource_id, project_id)""", + """CREATE TABLE IF NOT EXISTS dc_project_sessions ( + tenant_id TEXT NOT NULL, project_id TEXT NOT NULL, + session_id TEXT NOT NULL, owner_id TEXT NOT NULL, + agent TEXT, status TEXT NOT NULL DEFAULT 'open', revision_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, project_id, session_id), + FOREIGN KEY (tenant_id, project_id) + REFERENCES dc_projects (tenant_id, project_id) ON DELETE CASCADE)""", + """CREATE TABLE IF NOT EXISTS dc_project_documents ( + tenant_id TEXT NOT NULL, project_id TEXT NOT NULL, + document_kind TEXT NOT NULL CHECK + (document_kind IN ('description','abstract','overview')), + uri TEXT NOT NULL, revision_id TEXT NOT NULL, + package_hash TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, project_id, document_kind), + FOREIGN KEY (tenant_id, project_id) + REFERENCES dc_projects (tenant_id, project_id) ON DELETE CASCADE)""", + """CREATE TABLE IF NOT EXISTS dc_resource_links ( + tenant_id TEXT NOT NULL, source_type TEXT NOT NULL, + source_id TEXT NOT NULL, target_type TEXT NOT NULL, + target_id TEXT NOT NULL, relation TEXT NOT NULL, + source_revision TEXT, source_xpath TEXT, + PRIMARY KEY (tenant_id, source_type, source_id, + target_type, target_id, relation), + FOREIGN KEY (tenant_id, source_type, source_id) + REFERENCES dc_resources (tenant_id, resource_type, resource_id) + ON DELETE CASCADE, + FOREIGN KEY (tenant_id, target_type, target_id) + REFERENCES dc_resources (tenant_id, resource_type, resource_id) + ON DELETE CASCADE)""", + """CREATE TABLE IF NOT EXISTS dc_knowledge_datasets ( + tenant_id TEXT NOT NULL, knowledge_id TEXT NOT NULL, + schema_json TEXT NOT NULL, current_version INTEGER NOT NULL, + PRIMARY KEY (tenant_id, knowledge_id))""", + """CREATE TABLE IF NOT EXISTS dc_knowledge_facts ( + tenant_id TEXT NOT NULL, knowledge_id TEXT NOT NULL, + fact_id TEXT NOT NULL, entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, property TEXT NOT NULL, + value_json TEXT NOT NULL, source_uri TEXT NOT NULL, + source_revision TEXT NOT NULL, source_xpath TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, knowledge_id, fact_id), + FOREIGN KEY (tenant_id, knowledge_id) + REFERENCES dc_knowledge_datasets (tenant_id, knowledge_id) + ON DELETE CASCADE)""", + """CREATE INDEX IF NOT EXISTS dc_facts_by_entity + ON dc_knowledge_facts + (tenant_id, knowledge_id, entity_type, entity_id, property)""", + """CREATE TABLE IF NOT EXISTS dc_project_context_jobs ( + tenant_id TEXT NOT NULL, project_id TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('abstract','overview')), + status TEXT NOT NULL DEFAULT 'queued', + generation INTEGER NOT NULL DEFAULT 1, + attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (tenant_id, project_id, kind), + FOREIGN KEY (tenant_id, project_id) + REFERENCES dc_projects (tenant_id, project_id) ON DELETE CASCADE)""", + ) + with self.store.transaction(): + for statement in statements: + self.db.execute(statement) + columns = { + row["name"] + for row in self.db.execute("PRAGMA table_info(dc_project_sessions)") + } + if "agent" not in columns: + self.db.execute("ALTER TABLE dc_project_sessions ADD COLUMN agent TEXT") + self.db.execute( + "INSERT OR IGNORE INTO dc_schema_version (version) VALUES (?)", + (CATALOG_SCHEMA_VERSION,), + ) + self._ensure_existing_project_skills() + + def _ensure_existing_project_skills(self) -> None: + rows = self.db.execute( + """SELECT p.tenant_id,p.project_id,p.owner_id FROM dc_projects p + LEFT JOIN dc_project_resource_links l + ON l.tenant_id=p.tenant_id AND l.project_id=p.project_id + AND l.resource_type='skills' AND l.resource_id=? + WHERE l.project_id IS NULL""", + (DEFAULT_SKILL_URI.rsplit("/", 1)[-1],), + ).fetchall() + for row in rows: + principal = Principal(row["tenant_id"], row["owner_id"]) + with self.store.transaction(): + self._ensure_default_skill(principal) + self.link(principal, row["project_id"], DEFAULT_SKILL_URI) + + @staticmethod + def _project_id(value: str) -> str: + address = parse_uri(f"docling://projects/{value}") + if len(address.segments) != 1: + raise ValueError("project ID must be one URI segment") + return address.segments[0] + + @staticmethod + def _resource_parts(uri: str) -> tuple[str, str]: + address = parse_uri(uri) + if ( + address.namespace != "resources" + or len(address.segments) != 2 + or address.segments[0] not in RESOURCE_TYPES + ): + raise ValueError("expected one flat resource URI") + return address.segments[0], address.segments[1] + + def _queue_context(self, principal: Principal, project_id: str) -> None: + for kind in ("abstract", "overview"): + self.db.execute( + """INSERT INTO dc_project_context_jobs + (tenant_id,project_id,kind,status) VALUES (?,?,?,'queued') + ON CONFLICT (tenant_id,project_id,kind) + DO UPDATE SET status='queued',generation=generation+1, + updated_at=CURRENT_TIMESTAMP""", + (principal.tenant_id, project_id, kind), + ) + + def create_project( + self, principal: Principal, project_id: str, title: str + ) -> Project: + project_id = self._project_id(project_id) + if not title.strip(): + raise ValueError("project title is required") + with self.store.transaction(): + self._ensure_default_skill(principal) + self.db.execute( + """INSERT INTO dc_projects + (tenant_id,project_id,title,owner_id) VALUES (?,?,?,?)""", + (principal.tenant_id, project_id, title.strip(), principal.user_id), + ) + self.link(principal, project_id, DEFAULT_SKILL_URI) + return Project(project_id, title.strip(), None, None, None) + + def _ensure_default_skill(self, principal: Principal) -> None: + row = self.db.execute( + """SELECT 1 FROM dc_resources WHERE tenant_id=? AND uri=?""", + (principal.tenant_id, DEFAULT_SKILL_URI), + ).fetchone() + if row is not None: + return + from .ingestion import Ingestor + + xml = f"{escape(DEFAULT_SKILL_TEXT)}" + record = Ingestor(self.store).add_resource( + principal, + DEFAULT_SKILL_URI, + xml.encode("utf-8"), + filename="doclang-authoring.dclg", + profile_metadata={ + "profile": "skill-v1", + "skill_id": DEFAULT_SKILL_URI.rsplit("/", 1)[-1], + "name": DEFAULT_SKILL_NAME, + "summary": DEFAULT_SKILL_SUMMARY, + }, + ) + self.register_resource(principal, record, "skills") + + def get_project(self, principal: Principal, project_id: str) -> Project: + project_id = self._project_id(project_id) + row = self.db.execute( + """SELECT project_id,title,description_uri,abstract_uri,overview_uri + FROM dc_projects WHERE tenant_id=? AND project_id=?""", + (principal.tenant_id, project_id), + ).fetchone() + if row is None: + raise ValueError(f"project does not exist: {project_id}") + return Project(**dict(row)) + + def set_project_document( + self, + principal: Principal, + project_id: str, + kind: str, + record: DocumentRecord, + *, + generated: bool = False, + ) -> None: + if kind not in {"description", "abstract", "overview"}: + raise ValueError("invalid project document kind") + project_id = self._project_id(project_id) + if record.uri != f"docling://projects/{project_id}/.{kind}.dclx": + raise ValueError("project document URI does not match project and kind") + self.get_project(principal, project_id) + column = { + "description": "description_uri", + "abstract": "abstract_uri", + "overview": "overview_uri", + }[kind] + with self.store.transaction(): + self.db.execute( + """INSERT INTO dc_project_documents + (tenant_id,project_id,document_kind,uri,revision_id,package_hash) + VALUES (?,?,?,?,?,?) + ON CONFLICT (tenant_id,project_id,document_kind) + DO UPDATE SET revision_id=excluded.revision_id, + package_hash=excluded.package_hash,updated_at=CURRENT_TIMESTAMP""", + ( + principal.tenant_id, + project_id, + kind, + record.uri, + record.revision_id, + record.package.sha256, + ), + ) + self.db.execute( + f"UPDATE dc_projects SET {column}=?,updated_at=CURRENT_TIMESTAMP " + "WHERE tenant_id=? AND project_id=?", + (record.uri, principal.tenant_id, project_id), + ) + if kind == "description": + self._queue_context(principal, project_id) + elif not generated: + self.db.execute( + """UPDATE dc_project_context_jobs SET status='completed', + last_error=NULL,updated_at=CURRENT_TIMESTAMP + WHERE tenant_id=? AND project_id=? AND kind=?""", + (principal.tenant_id, project_id, kind), + ) + + def refresh_project(self, principal: Principal, project_id: str) -> None: + project_id = self._project_id(project_id) + self.get_project(principal, project_id) + with self.store.transaction(): + self._queue_context(principal, project_id) + + def context_jobs( + self, principal: Principal, project_id: str + ) -> list[dict[str, Any]]: + project_id = self._project_id(project_id) + self.get_project(principal, project_id) + rows = self.db.execute( + """SELECT kind,status,generation,attempts,last_error,updated_at + FROM dc_project_context_jobs WHERE tenant_id=? AND project_id=? + ORDER BY kind""", + (principal.tenant_id, project_id), + ).fetchall() + return [dict(row) for row in rows] + + def register_session( + self, + principal: Principal, + project_id: str, + session_id: str, + *, + status: str = "open", + revision_id: str | None = None, + agent: str | None = None, + ) -> None: + project_id = self._project_id(project_id) + if status not in {"open", "closed"}: + raise ValueError("invalid session status") + parse_uri(f"docling://projects/{project_id}/sessions/{session_id}") + self.get_project(principal, project_id) + with self.store.transaction(): + self.db.execute( + """INSERT INTO dc_project_sessions + (tenant_id,project_id,session_id,owner_id,agent,status,revision_id) + VALUES (?,?,?,?,?,?,?) + ON CONFLICT (tenant_id,project_id,session_id) + DO UPDATE SET status=excluded.status, + agent=COALESCE(dc_project_sessions.agent,excluded.agent), + revision_id=COALESCE(excluded.revision_id,dc_project_sessions.revision_id)""", + ( + principal.tenant_id, + project_id, + session_id, + principal.user_id, + agent, + status, + revision_id, + ), + ) + + def project_sessions( + self, principal: Principal, project_id: str + ) -> list[dict[str, str]]: + self.get_project(principal, project_id) + rows = self.db.execute( + """SELECT session_id,agent,status,owner_id,created_at FROM dc_project_sessions + WHERE tenant_id=? AND project_id=? ORDER BY session_id""", + (principal.tenant_id, project_id), + ).fetchall() + return [dict(row) for row in rows] + + def find_source(self, principal: Principal, source_sha256: str) -> str | None: + row = self.db.execute( + """SELECT uri FROM dc_resources WHERE tenant_id=? + AND resource_type='library' AND source_sha256=?""", + (principal.tenant_id, source_sha256), + ).fetchone() + return row["uri"] if row else None + + def register_resource( + self, + principal: Principal, + record: DocumentRecord, + resource_type: str = "library", + ) -> None: + actual_type, resource_id = self._resource_parts(record.uri) + if actual_type != resource_type: + raise ValueError("resource URI must be flat and match its type") + source_sha256 = record.source.get("source_sha256") + with self.store.transaction(): + self.db.execute( + """INSERT INTO dc_resources + (tenant_id,resource_type,resource_id,uri,source_sha256, + current_revision,package_hash,owner_id) + VALUES (?,?,?,?,?,?,?,?) + ON CONFLICT (tenant_id,resource_type,resource_id) + DO UPDATE SET current_revision=excluded.current_revision, + package_hash=excluded.package_hash, + updated_at=CURRENT_TIMESTAMP""", + ( + principal.tenant_id, + resource_type, + resource_id, + record.uri, + source_sha256, + record.revision_id, + record.package.sha256, + principal.user_id, + ), + ) + self.db.execute( + """INSERT OR IGNORE INTO dc_resource_revisions + (tenant_id,resource_type,resource_id,revision_id,package_hash, + source_sha256,provenance) VALUES (?,?,?,?,?,?,?)""", + ( + principal.tenant_id, + resource_type, + resource_id, + record.revision_id, + record.package.sha256, + source_sha256, + json.dumps(record.source, sort_keys=True), + ), + ) + + def link(self, principal: Principal, project_id: str, uri: str) -> None: + project_id = self._project_id(project_id) + resource_type, resource_id = self._resource_parts(uri) + self.get_project(principal, project_id) + with self.store.transaction(): + self.db.execute( + """INSERT OR IGNORE INTO dc_project_resource_links + (tenant_id,project_id,resource_type,resource_id,created_by) + VALUES (?,?,?,?,?)""", + ( + principal.tenant_id, + project_id, + resource_type, + resource_id, + principal.user_id, + ), + ) + self._queue_context(principal, project_id) + + def unlink(self, principal: Principal, project_id: str, uri: str) -> None: + project_id = self._project_id(project_id) + resource_type, resource_id = self._resource_parts(uri) + if uri == DEFAULT_SKILL_URI: + raise ValueError( + "the default DocLang authoring skill is required by every project" + ) + self.get_project(principal, project_id) + with self.store.transaction(): + self.db.execute( + """DELETE FROM dc_project_resource_links WHERE tenant_id=? + AND project_id=? AND resource_type=? AND resource_id=?""", + (principal.tenant_id, project_id, resource_type, resource_id), + ) + self._queue_context(principal, project_id) + + def project_resources( + self, principal: Principal, project_id: str + ) -> list[dict[str, str]]: + self.get_project(principal, project_id) + rows = self.db.execute( + """SELECT r.resource_type,r.resource_id,r.uri + FROM dc_project_resource_links l JOIN dc_resources r + USING (tenant_id,resource_type,resource_id) + WHERE l.tenant_id=? AND l.project_id=? + ORDER BY r.resource_type,r.resource_id""", + (principal.tenant_id, project_id), + ).fetchall() + return [dict(row) for row in rows] + + def resource_projects(self, principal: Principal, uri: str) -> list[dict[str, str]]: + resource_type, resource_id = self._resource_parts(uri) + rows = self.db.execute( + """SELECT p.project_id,p.title FROM dc_project_resource_links l + JOIN dc_projects p ON p.tenant_id=l.tenant_id + AND p.project_id=l.project_id + WHERE l.tenant_id=? AND l.resource_type=? AND l.resource_id=? + ORDER BY lower(p.title),p.project_id""", + (principal.tenant_id, resource_type, resource_id), + ).fetchall() + return [dict(row) for row in rows] + + def project_resource_uris( + self, principal: Principal, project_id: str, *, max_items: int = 10_000 + ) -> tuple[str, ...]: + self.get_project(principal, project_id) + rows = self.db.execute( + """SELECT r.uri FROM dc_project_resource_links l + JOIN dc_resources r USING (tenant_id,resource_type,resource_id) + WHERE l.tenant_id=? AND l.project_id=? + ORDER BY r.uri LIMIT ?""", + (principal.tenant_id, project_id, max_items + 1), + ).fetchall() + if len(rows) > max_items: + raise ValueError("project has too many linked resources for one search") + return tuple(row["uri"] for row in rows) + + def link_resources( + self, + principal: Principal, + source_uri: str, + target_uri: str, + relation: str, + *, + source_revision: str | None = None, + source_xpath: str | None = None, + ) -> None: + if not relation or len(relation) > 80: + raise ValueError("relation must be 1 to 80 characters") + if source_xpath is not None and ( + not source_xpath.startswith("/doclang[1]") or len(source_xpath) > 2048 + ): + raise ValueError("invalid source XPath") + source_type, source_id = self._resource_parts(source_uri) + target_type, target_id = self._resource_parts(target_uri) + with self.store.transaction(): + self.db.execute( + """INSERT INTO dc_resource_links + (tenant_id,source_type,source_id,target_type,target_id, + relation,source_revision,source_xpath) + VALUES (?,?,?,?,?,?,?,?) + ON CONFLICT (tenant_id,source_type,source_id, + target_type,target_id,relation) + DO UPDATE SET source_revision=excluded.source_revision, + source_xpath=excluded.source_xpath""", + ( + principal.tenant_id, + source_type, + source_id, + target_type, + target_id, + relation, + source_revision, + source_xpath, + ), + ) + + def resource_links( + self, principal: Principal, uri: str + ) -> list[dict[str, str | None]]: + resource_type, resource_id = self._resource_parts(uri) + rows = self.db.execute( + """SELECT s.uri AS source_uri,t.uri AS target_uri,l.relation, + l.source_revision,l.source_xpath + FROM dc_resource_links l + JOIN dc_resources s ON s.tenant_id=l.tenant_id + AND s.resource_type=l.source_type AND s.resource_id=l.source_id + JOIN dc_resources t ON t.tenant_id=l.tenant_id + AND t.resource_type=l.target_type AND t.resource_id=l.target_id + WHERE l.tenant_id=? AND + ((l.source_type=? AND l.source_id=?) OR + (l.target_type=? AND l.target_id=?)) + ORDER BY s.uri,t.uri,l.relation""", + ( + principal.tenant_id, + resource_type, + resource_id, + resource_type, + resource_id, + ), + ).fetchall() + return [dict(row) for row in rows] + + def overview( + self, principal: Principal, *, limit: int = 100, offset: int = 0 + ) -> list[dict[str, Any]]: + if not 1 <= limit <= 1000 or offset < 0: + raise ValueError("overview limit or offset is out of bounds") + return self._project_counts(principal, None, limit, offset) + + def project_summary(self, principal: Principal, project_id: str) -> dict[str, Any]: + """Return the project record plus counts of its linked resources.""" + project = self.get_project(principal, project_id) + counts = self._project_counts(principal, project.project_id, 1, 0)[0] + counts.pop("project_id") + counts.pop("title") + return {**asdict(project), **counts} + + def _project_counts( + self, principal: Principal, project_id: str | None, limit: int, offset: int + ) -> list[dict[str, Any]]: + rows = self.db.execute( + """SELECT p.project_id,p.title, + count(CASE WHEN l.resource_type='library' THEN 1 END) AS documents, + count(CASE WHEN l.resource_type='memories' THEN 1 END) AS memories, + count(CASE WHEN l.resource_type='skills' THEN 1 END) AS skills, + count(CASE WHEN l.resource_type='concepts' THEN 1 END) AS concepts, + count(CASE WHEN l.resource_type='knowledge' THEN 1 END) AS knowledge, + (SELECT count(*) FROM dc_project_sessions s + WHERE s.tenant_id=p.tenant_id AND s.project_id=p.project_id) AS sessions + FROM (SELECT * FROM dc_projects WHERE tenant_id=? + AND (? IS NULL OR project_id=?) + ORDER BY lower(title),project_id LIMIT ? OFFSET ?) p + LEFT JOIN dc_project_resource_links l + ON l.tenant_id=p.tenant_id AND l.project_id=p.project_id + GROUP BY p.tenant_id,p.project_id,p.title + ORDER BY lower(p.title),p.project_id""", + (principal.tenant_id, project_id, project_id, limit, offset), + ).fetchall() + return [dict(row) for row in rows] + + @staticmethod + def resource_uri(kind: str, value: str) -> str: + uri = ( + value + if value.startswith("docling://") + else f"docling://resources/{kind}/{value}" + ) + address = parse_uri(uri) + if ( + address.namespace != "resources" + or address.segments[0] != kind + or len(address.segments) != 2 + ): + raise ValueError(f"expected a {kind} ID or URI") + return uri + + def resources( + self, principal: Principal, kind: str, project_id: str | None = None + ) -> list[dict[str, Any]]: + if project_id: + return [ + row + for row in self.project_resources(principal, project_id) + if row["resource_type"] == kind + ] + rows = self.db.execute( + "SELECT resource_type,resource_id,uri,current_revision AS revision_id FROM dc_resources WHERE tenant_id=? AND resource_type=? ORDER BY resource_id", + (principal.tenant_id, kind), + ).fetchall() + return [dict(row) for row in rows] + + def knowledge_dataset(self, principal: Principal, uri: str) -> dict[str, Any]: + uri = self.resource_uri("knowledge", uri) + self.store.get_record(principal, uri) + row = self.db.execute( + "SELECT schema_json FROM dc_knowledge_datasets WHERE tenant_id=? AND knowledge_id=?", + (principal.tenant_id, uri.rsplit("/", 1)[-1]), + ).fetchone() + if row is None: + raise ValueError("knowledge dataset is not registered") + data = json.loads(row["schema_json"]) + if "concept_uri" not in data: + raise ValueError( + "knowledge dataset has no pinned concept; recreate it with knowledge create --concept" + ) + return {"uri": uri, **data} + + def register_knowledge_dataset( + self, + principal: Principal, + uri: str, + concept_uri: str, + concept_revision: str | None = None, + ) -> None: + uri = self.resource_uri("knowledge", uri) + concept_uri = self.resource_uri("concepts", concept_uri) + concept = self.store.get_record(principal, concept_uri) + if concept_revision: + concept = self.store.get_revision( + principal, + NodeAddress(concept.document_id, concept_revision, "/doclang[1]"), + ) + metadata = validate_descriptor( + concept_uri, load_package(self.store.packages.get(concept.package.sha256)) + ) + data = { + "concept_uri": concept_uri, + "concept_revision": concept.revision_id, + "schema": metadata["schema"], + } + encoded = json.dumps(data, sort_keys=True) + previous = self.db.execute( + "SELECT schema_json FROM dc_knowledge_datasets WHERE tenant_id=? AND knowledge_id=?", + (principal.tenant_id, uri.rsplit("/", 1)[-1]), + ).fetchone() + if previous and previous[0] != encoded: + raise ValueError("a knowledge dataset's pinned concept cannot be changed") + with self.store.transaction(): + self.db.execute( + "INSERT OR IGNORE INTO dc_knowledge_datasets (tenant_id,knowledge_id,schema_json,current_version) VALUES (?,?,?,1)", + (principal.tenant_id, uri.rsplit("/", 1)[-1], encoded), + ) + self.link_resources(principal, uri, concept_uri, "defined-by") + + def add_fact( + self, + principal: Principal, + knowledge_uri: str, + *, + entity_type: str, + entity_id: str, + property: str, + value: Any, + source_uri: str, + source_revision: str, + source_xpath: str, + unit: str | None = None, + ) -> str: + resource_type, knowledge_id = self._resource_parts(knowledge_uri) + if resource_type != "knowledge": + raise ValueError("expected a knowledge resource") + schema = self.knowledge_dataset(principal, knowledge_uri)["schema"] + definition = schema["properties"].get(property, {}) + kind = definition.get("type") + if entity_type not in schema["entity_types"]: + raise ValueError("entity type does not match the pinned concept schema") + if unit != definition.get("unit"): + raise ValueError("fact unit does not match the pinned concept schema") + valid = { + "text": isinstance(value, str), + "integer": isinstance(value, int) and not isinstance(value, bool), + "number": isinstance(value, (int, float)) and not isinstance(value, bool), + "boolean": isinstance(value, bool), + "json": True, + }.get(kind, False) + if not valid or not entity_type.strip() or not entity_id.strip(): + raise ValueError("fact does not match the knowledge schema") + encoded = json.dumps(value, ensure_ascii=False) + if len(encoded) > 16_384: + raise ValueError("fact value is too large") + source = self.store.get_record(principal, source_uri) + revision = self.store.get_revision( + principal, NodeAddress(source.document_id, source_revision, source_xpath) + ) + if revision.uri != source_uri: + raise ValueError("fact citation URI differs from its revision") + self.store.read_node( + principal, NodeAddress(source.document_id, source_revision, source_xpath) + ) + fact_id = uuid.uuid4().hex + with self.store.transaction(): + self.db.execute( + """INSERT INTO dc_knowledge_facts + (tenant_id,knowledge_id,fact_id,entity_type,entity_id,property, + value_json,source_uri,source_revision,source_xpath) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + ( + principal.tenant_id, + knowledge_id, + fact_id, + entity_type, + entity_id, + property, + encoded, + source_uri, + source_revision, + source_xpath, + ), + ) + return fact_id + + def knowledge_facts( + self, principal: Principal, knowledge_uri: str + ) -> list[dict[str, Any]]: + resource_type, knowledge_id = self._resource_parts(knowledge_uri) + if resource_type != "knowledge": + raise ValueError("expected a knowledge resource") + rows = self.db.execute( + """SELECT fact_id,entity_type,entity_id,property,value_json, + source_uri,source_revision,source_xpath + FROM dc_knowledge_facts WHERE tenant_id=? AND knowledge_id=? + ORDER BY entity_type,entity_id,property,fact_id""", + (principal.tenant_id, knowledge_id), + ).fetchall() + return [{**dict(row), "value": json.loads(row["value_json"])} for row in rows] + + +def new_document_uri() -> str: + return f"docling://resources/library/{uuid.uuid4().hex}" diff --git a/src/docling_context/cli.py b/src/docling_context/cli.py index 8ec871a..6fc501a 100644 --- a/src/docling_context/cli.py +++ b/src/docling_context/cli.py @@ -5,18 +5,27 @@ import argparse import hashlib import json +import logging import os -import re +import shutil +import sqlite3 +import subprocess import sys -from collections import Counter -from collections.abc import Mapping +import threading +import uuid +import warnings +from collections.abc import Mapping, Sequence from dataclasses import asdict, replace from datetime import UTC, date, datetime, time from pathlib import Path +from xml.sax.saxutils import escape +from doclang import DocLangXDocument from tabulate import tabulate # type: ignore[import-untyped] from tqdm import tqdm # type: ignore[import-untyped] +from .agent_bridge import AGENTS, inspect_agent, install_agent, remove_agent +from .catalog import CATALOG_SCHEMA_VERSION, Catalog, new_document_uri from .compiler import MemoryCompiler from .conversion_config import PdfConversionConfig from .converters import LocalDoclingConverter @@ -28,21 +37,26 @@ stored_embedding_provider, ) from .ingestion import Ingestor -from .jobs import CollectionWorker +from .integration import IntegrationSpec from .local import LocalContextStore from .models import Principal -from .package import bounded_nodes +from .package import MAX_PACKAGE_BYTES, bounded_nodes, load_package from .profiles import ( + DESCRIPTOR_NAMES, + DESCRIPTOR_PARTS, MAX_ATTACHMENT_BYTES, MEMORY_KINDS, MEMORY_STATUSES, profile_kind, + validate_descriptor, validate_memory, ) +from .project_context import ProjectContextWorker from .retrieval import RetrievalHit, Retriever, SearchScope from .sessions import SessionInfo, SessionStore +from .source_urls import download_source, is_source_url from .uri import authorize_uri, parse_uri -from .vectors import TurbovecIndex +from .vectors import TurbovecIndex, verified_vector_state def _json(value: object) -> None: @@ -80,7 +94,10 @@ def _status( def _table_or_json( - values: list[dict[str, object]], *, headers: tuple[str, ...], json_output: bool + values: Sequence[Mapping[str, object]], + *, + headers: tuple[str, ...], + json_output: bool, ) -> None: if json_output: _json(values) @@ -100,6 +117,19 @@ def _table_or_json( print("No records.") +def _rows_or_json_lines( + values: Sequence[Mapping[str, object]], + *, + headers: tuple[str, ...], + json_output: bool, +) -> None: + if json_output: + for value in values: + _json(value) + else: + _table_or_json(values, headers=headers, json_output=False) + + def _session_rows(items: tuple[SessionInfo, ...]) -> list[dict[str, object]]: return [asdict(item) for item in items] @@ -111,7 +141,9 @@ def _memory_rows(items: tuple[MemoryRecord, ...]) -> list[dict[str, object]]: def _session_id(value: str | None) -> str: session_id = value or os.environ.get("DOCLING_CONTEXT_SESSION") if not session_id: - raise ValueError("session ID required; pass it or set DOCLING_CONTEXT_SESSION") + raise ValueError( + "session ID required; pass --id (or --session) or set DOCLING_CONTEXT_SESSION" + ) return session_id @@ -136,17 +168,12 @@ def _timestamp_filter(value: str | None, *, upper: bool = False) -> str | None: def _input_text(args: argparse.Namespace, *, name: str = "text") -> str: value = getattr(args, name, None) - option = getattr(args, "text_option", None) - if option is not None: - if value is not None: - raise ValueError("use either positional text or --text") - value = option if args.stdin: if value is not None: - raise ValueError("use either text argument or --stdin") + raise ValueError(f"use either --{name} or --stdin") value = sys.stdin.read(20_001) if value is None: - raise ValueError("text argument or --stdin is required") + raise ValueError(f"--{name} or --stdin is required") return value @@ -157,6 +184,26 @@ def _advance_progress(bar: tqdm, completed: int, total: int) -> None: bar.update(completed - bar.n) +def _source_page_units(source: bytes | Path, filename: str | None) -> int: + name = source.name if isinstance(source, Path) else filename + if not name or not name.lower().endswith(".pdf"): + return 1 + try: + from pypdfium2 import PdfDocument # type: ignore[import-untyped] + + pdf = PdfDocument(source) + try: + return len(pdf) + finally: + pdf.close() + except ImportError as exc: + raise RuntimeError( + "install docling-context[conversion] to count PDF pages" + ) from exc + except Exception as exc: + raise ValueError(f"cannot count pages in {name}: {exc}") from exc + + def _search_excerpt(text: str) -> str: compact = " ".join(text.split()) if len(compact) <= 128: @@ -204,33 +251,6 @@ def _prefix(value: str) -> str: return parse_uri(value.rstrip("/"), prefix=True).value -def _collection(value: str) -> str: - uri = value if value.startswith("docling://") else f"docling://resources/{value}" - address = parse_uri(uri, prefix=True) - if address.namespace != "resources" or len(address.segments) != 1: - raise ValueError("collection must be one resources segment") - return address.value - - -def _slug(value: str) -> str: - result = re.sub(r"[^\w.-]+", "-", value, flags=re.UNICODE).strip(".-") - if not result or result in {".", ".."}: - raise ValueError(f"cannot make a URI segment from {value!r}") - if result != value: - result += f"-{hashlib.sha256(value.encode()).hexdigest()[:8]}" - return result - - -def _uri_for( - path: Path, base: Path, collection: str, *, include_extension: bool = False -) -> str: - relative = path.relative_to(base) - stem = path.name[:-9] if path.name.lower().endswith(".dclg.xml") else path.stem - name = path.name if include_extension else stem - segments = [_slug(part) for part in relative.parts[:-1]] + [_slug(name)] - return f"{collection}/{'/'.join(segments)}" - - def _format_extensions() -> dict[str, tuple[str, ...]]: try: from docling.datamodel.base_models import FormatToExtensions @@ -283,9 +303,8 @@ def _config(args: argparse.Namespace) -> PdfConversionConfig: def _add_resource( args: argparse.Namespace, store: LocalContextStore, principal: Principal -) -> None: - source = args.source.resolve(strict=True) - collection = _collection(args.collection) +) -> int: + from_url = is_source_url(args.source) extensions = _format_extensions() requested = { "xml_doclang" if value == "dclg" else value for value in args.from_formats or () @@ -298,61 +317,192 @@ def selected(path: Path) -> bool: formats = _formats_for(path, extensions) return bool(formats and (not requested or formats & requested)) - if source.is_dir(): - if args.uri: - raise ValueError("--uri can only be used with one file") - candidates = source.rglob("*") if args.recursive else source.iterdir() - paths = sorted(path for path in candidates if path.is_file() and selected(path)) - if not paths: - raise ValueError( - "folder contains no documents matching the selected formats" - ) - base = source - else: + paths: list[Path] = [] + base: Path | None = None + if from_url: if args.recursive: raise ValueError("--recursive requires a folder source") - if not source.is_file(): - raise ValueError("source must be a file or folder") - if not selected(source): - raise ValueError("source does not match a supported selected format") - paths = [source] - base = source.parent - resources = [(path, args.uri or _uri_for(path, base, collection)) for path in paths] - collisions = { - uri for uri, count in Counter(uri for _, uri in resources).items() if count > 1 - } - resources = [ - ( - path, - _uri_for(path, base, collection, include_extension=True) - if uri in collisions - else uri, - ) - for path, uri in resources - ] - if len({uri for _, uri in resources}) != len(resources): - raise ValueError("multiple files map to the same context URI") + else: + source = Path(args.source).resolve(strict=True) + if source.is_dir(): + candidates = source.rglob("*") if args.recursive else source.iterdir() + paths = sorted( + path for path in candidates if path.is_file() and selected(path) + ) + if not paths: + raise ValueError( + "folder contains no documents matching the selected formats" + ) + base = source + else: + if args.recursive: + raise ValueError("--recursive requires a folder source") + if not source.is_file(): + raise ValueError("source must be a file or folder") + if not selected(source): + raise ValueError("source does not match a supported selected format") + paths = [source] + base = source.parent ingestor = Ingestor( store, - allowed_roots=(base,), + allowed_roots=(base,) if base is not None else (), converter=LocalDoclingConverter(_config(args)), ) - worker = CollectionWorker(store) - for path, uri in resources: - record = ingestor.add_resource(principal, uri, path, force=args.force) - address = parse_uri(record.uri) - task_id = None - if address.namespace == "resources": - collection_uri = f"docling://resources/{address.segments[0]}" - matches = [ - job - for job in worker.jobs(principal, collection_uri) - if job.input_revision == record.revision_id - ] - task_id = matches[-1].job_id if matches else None - _json( - {"uri": record.uri, "revision_id": record.revision_id, "task_id": task_id} + with Catalog(store) as catalog: + if args.project: + catalog.get_project(principal, args.project) + sources: list[tuple[bytes | Path, str | None, str | None]] + if from_url: + data, fetched_filename = download_source(args.source) + if not selected(Path(fetched_filename)): + raise ValueError( + "source URL does not identify a supported selected format" + ) + sources = [(data, fetched_filename, args.source)] + else: + sources = [(path, None, None) for path in paths] + # A single unreadable or unconvertible file must not abort a folder + # import: record it, keep going, and report every failure at the end. + keep_going = len(sources) > 1 and not args.fail_fast + failures: list[dict[str, str]] = [] + + def source_label(item: bytes | Path, filename: str | None) -> str: + if isinstance(item, Path): + return str(item.relative_to(base)) if base is not None else str(item) + return filename or "" + + def record_failure(label: str, exc: Exception) -> None: + message = str(exc) or type(exc).__name__ + failures.append({"source": label, "error": message}) + tqdm.write(f"dc: skipped {label}: {message}", file=sys.stderr) + + checked: list[tuple[bytes | Path, str | None, str | None]] = [] + source_digests: list[str] = [] + existing_uris: list[str | None] = [] + page_units: list[int] = [] + progress_enabled = not args.json and sys.stderr.isatty() + if progress_enabled: + print("Counting pages and checking library...", file=sys.stderr) + for item, filename, origin in sources: + try: + if isinstance(item, Path): + if item.stat().st_size > MAX_PACKAGE_BYTES: + raise ValueError(f"source file is too large: {item}") + with item.open("rb") as source_file: + digest = hashlib.file_digest(source_file, "sha256").hexdigest() + else: + digest = hashlib.sha256(item).hexdigest() + existing = catalog.find_source(principal, digest) + units = _source_page_units(item, filename) + except (OSError, ValueError) as exc: + if not keep_going: + raise + record_failure(source_label(item, filename), exc) + continue + checked.append((item, filename, origin)) + source_digests.append(digest) + existing_uris.append(existing) + page_units.append(units) + sources = checked + + already_in_library = sum(uri is not None for uri in existing_uris) + total_page_units = sum(page_units) + imported: list[dict[str, object]] = [] + newly_added: set[str] = set() + with tqdm( + total=total_page_units, + initial=0, + desc="Importing pages", + unit="page", + file=sys.stderr, + disable=not progress_enabled, + ) as bar: + bar.set_postfix_str(f"existing={already_in_library} docs, added=0 docs") + stop_refresh = threading.Event() + + def refresh_progress() -> None: + while not stop_refresh.wait(1): + bar.refresh() + + refresh_thread = threading.Thread(target=refresh_progress, daemon=True) + if progress_enabled: + refresh_thread.start() + try: + for (item, filename, origin_uri), digest, existing_uri, units in zip( + sources, source_digests, existing_uris, page_units, strict=True + ): + bar.set_postfix_str( + f"existing={already_in_library} docs, " + f"added={len(newly_added)} docs, current={units} pages" + ) + try: + with store.transaction(): + uri = ( + catalog.find_source(principal, digest) + or new_document_uri() + ) + record = ingestor.add_resource( + principal, + uri, + item, + filename=filename, + origin_uri=origin_uri, + force=args.force, + ) + catalog.register_resource(principal, record) + if args.project: + catalog.link(principal, args.project, record.uri) + except Exception as exc: + if not keep_going: + raise + record_failure(source_label(item, filename), exc) + bar.update(units) + continue + imported.append( + { + "uri": record.uri, + "revision_id": record.revision_id, + "source_sha256": digest, + "project_id": args.project, + } + ) + if existing_uri is None: + newly_added.add(digest) + bar.update(units) + bar.set_postfix_str( + f"existing={already_in_library} docs, " + f"added={len(newly_added)} docs" + ) + except KeyboardInterrupt: + print( + f"dc: interrupted after processing {bar.n} of {total_page_units} pages; " + f"{len(newly_added)} new documents added. Rerun the command to continue.", + file=sys.stderr, + ) + raise + finally: + stop_refresh.set() + if progress_enabled: + refresh_thread.join() + _rows_or_json_lines( + imported, + headers=("uri", "revision_id", "source_sha256", "project_id"), + json_output=args.json, ) + if failures: + if args.json: + for failure in failures: + _json({"status": "failed", **failure}) + print( + f"dc: {len(failures)} of {len(failures) + len(imported)} documents " + "failed and were skipped; fix them and rerun the command " + "(already imported documents are not converted again):", + file=sys.stderr, + ) + for failure in failures: + print(f" {failure['source']}: {failure['error']}", file=sys.stderr) + return 1 + return 0 def _scan( @@ -363,15 +513,30 @@ def _scan( *, document_limit: int = 1_000, hit_limit: int = 100, + project_id: str | None = None, ) -> list[dict[str, object]]: address = parse_uri(prefix, prefix=True) authorize_uri(principal, address) base = address.value descendant = base + "/" + if project_id: + Catalog(store).get_project(principal, project_id) rows = store.db.execute( """SELECT uri FROM documents WHERE tenant_id=? AND available=1 - AND (uri=? OR substr(uri,1,?)=?) ORDER BY uri LIMIT ?""", - (principal.tenant_id, base, len(descendant), descendant, document_limit), + AND (uri=? OR substr(uri,1,?)=?) + AND (? IS NULL OR uri IN (SELECT r.uri FROM dc_project_resource_links l + JOIN dc_resources r USING (tenant_id,resource_type,resource_id) + WHERE l.tenant_id=? AND l.project_id=?)) ORDER BY uri LIMIT ?""", + ( + principal.tenant_id, + base, + len(descendant), + descendant, + project_id, + principal.tenant_id, + project_id, + document_limit, + ), ).fetchall() needle = term.casefold() if not needle: @@ -419,18 +584,182 @@ def _parser() -> argparse.ArgumentParser: default=Path( os.environ.get("DOCLING_CONTEXT_STORE", "~/.local/share/docling-context") ).expanduser(), + help="Local DCLX and index directory (default: ~/.local/share/docling-context).", ) parser.add_argument( - "--tenant", default=os.environ.get("DOCLING_CONTEXT_TENANT", "default") + "--tenant", + default=os.environ.get("DOCLING_CONTEXT_TENANT", "default"), + help="Tenant scope (default: default).", ) parser.add_argument( - "--user", default=os.environ.get("DOCLING_CONTEXT_USER", "local") - ) - commands = parser.add_subparsers(dest="command", required=True) - status = commands.add_parser("status", help="Show store and job counts") - status.add_argument( - "--json", action="store_true", help="Print JSON instead of a table." - ) + "--user", + default=os.environ.get("DOCLING_CONTEXT_USER", "local"), + help="User scope (default: local).", + ) + commands = parser.add_subparsers(dest="command") + init = commands.add_parser("init", help="Initialize the local SQLite store") + init.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" + ) + overview = commands.add_parser( + "overview", help="List projects and linked resource counts" + ) + overview.add_argument("-j", "--json", action="store_true") + overview.add_argument("--limit", type=int, default=100) + overview.add_argument("--offset", type=int, default=0) + project = commands.add_parser( + "project", help="Create and inspect projects and links" + ) + project_commands = project.add_subparsers(dest="project_command", required=True) + project_create = project_commands.add_parser("create", help="Create a project") + project_create.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + project_create.add_argument("--title", required=True) + project_show = project_commands.add_parser("show", help="Show project details") + project_show.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + for name in ("link", "unlink"): + command = project_commands.add_parser( + name, help=f"{'Link' if name == 'link' else 'Unlink'} a shared resource" + ) + command.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + command.add_argument("-u", "--uri", required=True, help="Resource URI") + project_list = project_commands.add_parser( + "resources", help="List linked resources" + ) + project_list.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + context = project_commands.add_parser( + "context", help="Read and generate project context" + ) + context_commands = context.add_subparsers(dest="context_command", required=True) + for name, help_text in ( + ("refresh", "Queue context regeneration"), + ("show", "Read project context"), + ("jobs", "Inspect context jobs"), + ("worker", "Process a queued context job"), + ): + sub = context_commands.add_parser(name, help=help_text) + sub.set_defaults(project_command="context-show" if name == "show" else name) + if name != "worker": + sub.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + sub.add_argument("-j", "--json", action="store_true") + context_set = context_commands.add_parser( + "set", help="Set a project context document" + ) + context_set.add_argument( + "-p", "--project", dest="project_id", required=True, help="Project ID" + ) + context_set.add_argument("--description", type=Path) + context_set.add_argument("--abstract", type=Path) + context_set.add_argument("--overview", type=Path) + context_set.add_argument("-j", "--json", action="store_true") + context_set.set_defaults(project_command="context-set") + for command in project_commands.choices.values(): + command.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" + ) + library = commands.add_parser( + "library", help="Import, inspect, and search source documents" + ) + library_commands = library.add_subparsers(dest="library_command", required=True) + for group, kind in (("skill", "skills"), ("concept", "concepts")): + typed = commands.add_parser(group, help=f"Create and inspect shared {kind}") + typed_commands = typed.add_subparsers(dest="typed_command", required=True) + create = typed_commands.add_parser( + "create", help=f"Create a named {group} from DCLG or DCLX" + ) + create.add_argument( + "-i", "--id", dest="resource_id", required=True, help=f"{group.title()} ID" + ) + create.add_argument("--from", dest="source", type=Path, required=True) + create.add_argument( + "--metadata", type=Path, help="Descriptor JSON; required for DCLG concepts" + ) + create.add_argument("-p", "--project") + create.set_defaults(command="typed-create", type=kind) + for name in ("list", "show", "projects", "links", "link"): + action_help = { + "list": f"List shared {kind}", + "show": f"Show one {group}", + "projects": f"List projects using a {group}", + "links": f"List a {group}'s resource links", + "link": f"Link a {group} to another resource", + } + sub = typed_commands.add_parser(name, help=action_help[name]) + sub.set_defaults(command="typed-inspect", type=kind) + if name == "list": + sub.add_argument("-p", "--project") + else: + sub.add_argument( + "-i", + "--id", + dest="resource_id", + required=True, + help=f"{group.title()} ID", + ) + if name == "link": + sub.add_argument( + "--target", dest="target_uri", required=True, help="Target URI" + ) + sub.add_argument("--relation", required=True) + for sub in typed_commands.choices.values(): + sub.add_argument("-j", "--json", action="store_true") + knowledge = commands.add_parser( + "knowledge", help="Manage datasets of cited facts defined by concepts" + ) + knowledge_commands = knowledge.add_subparsers( + dest="knowledge_command", required=True + ) + knowledge_create = knowledge_commands.add_parser( + "create", help="Create a dataset from a concept revision" + ) + knowledge_create.add_argument( + "-i", "--id", dest="resource_id", required=True, help="Dataset ID" + ) + knowledge_create.add_argument("--concept", required=True) + knowledge_create.add_argument("-p", "--project") + for name, help_text in ( + ("list", "List knowledge datasets"), + ("show", "Show a dataset's pinned concept and schema"), + ): + sub = knowledge_commands.add_parser(name, help=help_text) + if name == "list": + sub.add_argument("-p", "--project") + else: + sub.add_argument( + "-u", "--uri", dest="knowledge_uri", required=True, help="Dataset URI" + ) + knowledge_add = knowledge_commands.add_parser( + "add-fact", help="Add a schema-checked fact with an exact citation" + ) + knowledge_add.add_argument( + "-u", "--uri", dest="knowledge_uri", required=True, help="Dataset URI" + ) + for field in ("entity-type", "entity-id", "property", "value"): + knowledge_add.add_argument(f"--{field}", required=True) + for field in ("source-uri", "source-revision", "source-xpath"): + knowledge_add.add_argument(f"--{field}", required=True) + knowledge_facts = knowledge_commands.add_parser( + "facts", help="List facts in a knowledge dataset" + ) + knowledge_facts.add_argument( + "-u", "--uri", dest="knowledge_uri", required=True, help="Dataset URI" + ) + knowledge_facts.add_argument("--entity") + knowledge_facts.add_argument("--property") + knowledge_add.add_argument("--unit") + for command in knowledge_commands.choices.values(): + command.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" + ) index = commands.add_parser( "index", help="Inspect or rebuild lexical and vector indexes" ) @@ -439,7 +768,7 @@ def _parser() -> argparse.ArgumentParser: "status", help="Show lexical and vector index status" ) index_status.add_argument( - "--json", action="store_true", help="Print JSON instead of a table." + "-j", "--json", action="store_true", help="Print JSON instead of a table." ) lex = index_commands.add_parser( "lex", help="Inspect or rebuild indexed document nodes" @@ -449,11 +778,14 @@ def _parser() -> argparse.ArgumentParser: "status", help="Show indexed document and node counts" ) lex_status.add_argument( - "--json", action="store_true", help="Print JSON instead of a table." + "-j", "--json", action="store_true", help="Print JSON instead of a table." ) - lex_commands.add_parser( + lex_rebuild = lex_commands.add_parser( "rebuild", help="Rebuild search records from stored DCLX documents" ) + lex_rebuild.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" + ) vector = index_commands.add_parser( "vector", help="Inspect or rebuild vector embeddings" ) @@ -462,7 +794,7 @@ def _parser() -> argparse.ArgumentParser: "status", help="Show vector model and coverage" ) vector_status.add_argument( - "--json", action="store_true", help="Print JSON instead of a table." + "-j", "--json", action="store_true", help="Print JSON instead of a table." ) vector_rebuild = vector_commands.add_parser( "rebuild", help="Embed indexed nodes and rebuild the vector snapshot" @@ -483,16 +815,20 @@ def _parser() -> argparse.ArgumentParser: default="auto", help="Embedding runtime (default: MLX for supported Apple Silicon models, else ONNX).", ) - add = commands.add_parser( - "add-resource", + vector_rebuild.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" + ) + add = library_commands.add_parser( + "add", help="Import one supported document or a folder of documents", - description="Import any supported Docling or DocLang document, or a folder containing them.", + description="Import a supported local document, folder, or HTTP(S) document URL.", epilog="Defaults shown apply without --config. Explicit flags override settings in the config file.", ) + add.set_defaults(command="add-resource") add.add_argument( - "source", - type=Path, - help="A supported document file or folder; folders scan their top level by default.", + "--source", + required=True, + help="A local document file, folder, or HTTP(S) URL; folders scan their top level by default.", ) add.add_argument( "-r", @@ -507,15 +843,7 @@ def _parser() -> argparse.ArgumentParser: metavar="FORMAT", help="Include a source format (repeatable; default: all; e.g. pdf, docx, xlsx, image).", ) - add.add_argument( - "--collection", - default="documents", - help="Collection for generated resource URIs (default: documents).", - ) - add.add_argument( - "--uri", - help="Exact URI for a single file (default: derived from collection and filename).", - ) + add.add_argument("-p", "--project", help="Link imported documents to this project") add.add_argument( "--config", type=Path, @@ -566,47 +894,100 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="Create a new revision for unchanged input (default: off).", ) - task = commands.add_parser("task", help="Inspect collection-summary jobs") - task_commands = task.add_subparsers(dest="task_command", required=True) - task_status = task_commands.add_parser("status") - task_status.add_argument("job_id") - task_status.add_argument( - "--json", action="store_true", help="Print JSON instead of a table." + add.add_argument( + "--fail-fast", + action="store_true", + help="Stop a folder import at the first failing document " + "(default: off; failures are skipped and reported at the end).", ) - jobs = commands.add_parser( - "jobs", help="Inspect durable ingestion or collection jobs" + add.add_argument( + "-j", "--json", action="store_true", help="Print JSON lines instead of a table" ) + jobs = library_commands.add_parser("jobs", help="Inspect durable ingestion jobs") + jobs.set_defaults(command="jobs") jobs_commands = jobs.add_subparsers(dest="jobs_command", required=True) jobs_status = jobs_commands.add_parser("status", help="Show one job by ID") - jobs_status.add_argument("job_id") - jobs_status.add_argument("--json", action="store_true") - worker = commands.add_parser("worker", help="Process queued collection summaries") - worker.add_argument("--once", action="store_true") - listing = commands.add_parser("ls", help="List direct children") - listing.add_argument("uri") - tree = commands.add_parser("tree", help="Walk a context subtree") - tree.add_argument("uri") + jobs_status.add_argument("-i", "--id", dest="job_id", required=True, help="Job ID") + jobs_status.add_argument("-j", "--json", action="store_true") + listing = library_commands.add_parser("list", help="List library documents") + listing.set_defaults(command="library-list") + listing.add_argument("-p", "--project") + for name, help_text in ( + ("projects", "List projects linked to a document"), + ("links", "List a document's resource links"), + ("link", "Link a document to another resource"), + ): + sub = library_commands.add_parser(name, help=help_text) + sub.set_defaults(command="library-relations") + sub.add_argument( + "-d", + "--document", + dest="document_id", + required=True, + help="Document ID or URI", + ) + if name == "link": + sub.add_argument( + "--target", dest="target_uri", required=True, help="Target URI" + ) + sub.add_argument("--relation", required=True) + sub.add_argument("-j", "--json", action="store_true") + listing.add_argument( + "-j", "--json", action="store_true", help="Print JSON lines instead of a table" + ) + tree = library_commands.add_parser("tree", help="Walk a context subtree") + tree.set_defaults(command="tree") + tree.add_argument( + "-u", + "--uri", + default="docling://resources/library/", + help="Subtree URI (default: docling://resources/library/).", + ) tree.add_argument("-L", "--depth", type=int, default=2) - outline = commands.add_parser("outline", help="Read a document TOC sidecar") - outline.add_argument("uri") + tree_output = tree.add_mutually_exclusive_group() + tree_output.add_argument( + "-j", "--json", action="store_true", help="Print JSON lines instead of a table" + ) + tree_output.add_argument( + "--raw", action="store_true", help="Print an indented URI tree" + ) + outline = library_commands.add_parser("outline", help="Read a document TOC sidecar") + outline.set_defaults(command="outline") + outline.add_argument("-u", "--uri", required=True, help="Document URI or ID") outline.add_argument("--revision", help="Immutable document revision") - outline.add_argument("--json", action="store_true") - show = commands.add_parser("show", help="Read one cited DocLang node") - show.add_argument("uri") - show.add_argument("xpath") + outline_output = outline.add_mutually_exclusive_group() + outline_output.add_argument("-j", "--json", action="store_true") + outline_output.add_argument("--raw", action="store_true", help="Print TOC XML only") + show = library_commands.add_parser("show", help="Read one cited DocLang node") + show.set_defaults(command="show") + show.add_argument("-u", "--uri", required=True, help="Document URI or ID") + show.add_argument("-x", "--xpath", required=True, help="DocLang node XPath") show.add_argument("--revision", help="Immutable document revision") show.add_argument("--format", choices=("text", "xml"), default="text") show.add_argument("--max-chars", type=int, default=8_192) - show.add_argument("--json", action="store_true") + show_output = show.add_mutually_exclusive_group() + show_output.add_argument("-j", "--json", action="store_true") + show_output.add_argument( + "--raw", action="store_true", help="Print node content only" + ) for name in ("find", "grep"): - search = commands.add_parser(name, help="Bounded lexical lookup") - search.add_argument("term") - search.add_argument("--uri", default="docling://resources/") - retrieval = commands.add_parser( + search = library_commands.add_parser(name, help="Bounded lexical lookup") + search.set_defaults(command=name) + search.add_argument("-p", "--project") + search.add_argument("--term", required=True, help="Literal search term") + search.add_argument("-u", "--uri", default="docling://resources/") + search.add_argument( + "-j", + "--json", + action="store_true", + help="Print JSON lines instead of a table", + ) + retrieval = library_commands.add_parser( "search", help="Search indexed DocLang nodes with exact citations" ) + retrieval.set_defaults(command="search") retrieval.add_argument( - "query", nargs="?", default="", help="Query text (optional with --image)." + "-q", "--query", default="", help="Query text (optional with --image)." ) retrieval.add_argument( "--image", type=Path, help="Image query file (vector mode only)." @@ -618,15 +999,21 @@ def _parser() -> argparse.ArgumentParser: help="Search mode (default: hybrid if vectors exist, else lexical).", ) retrieval.add_argument( + "-u", "--uri", - default="docling://resources/", - help="Authorized URI subtree (default: all resources).", + default="docling://resources/library/", + help="Authorized URI subtree (default: shared library).", ) retrieval.add_argument( - "--document", help="Restrict to one document URI (default: all in scope)." + "-p", "--project", help="Search only resources linked to this project" ) retrieval.add_argument( - "--xpath", help="Restrict to one source XPath subtree (default: all nodes)." + "-d", "--document", help="Restrict to one document URI (default: all in scope)." + ) + retrieval.add_argument( + "-x", + "--xpath", + help="Restrict to one source XPath subtree (default: all nodes).", ) retrieval.add_argument( "--tier", @@ -651,6 +1038,7 @@ def _parser() -> argparse.ArgumentParser: help="Maximum tokens per citation (default: 500).", ) retrieval.add_argument( + "-j", "--json", action="store_true", help="Print complete JSON records instead of the compact table (default: table).", @@ -658,9 +1046,27 @@ def _parser() -> argparse.ArgumentParser: session = commands.add_parser("session", help="Record and inspect agent sessions") session_commands = session.add_subparsers(dest="session_command", required=True) - session_start = session_commands.add_parser("start", help="Create an open session") - session_start.add_argument("--id", help="Session ID (default: generated UUID)") - session_list = session_commands.add_parser("list", help="List your sessions") + session_start = session_commands.add_parser( + "start", help="Launch an agent in a project session" + ) + session_start.add_argument( + "-i", "--id", help="Session ID (default: generated UUID)" + ) + session_start.add_argument( + "-p", "--project", required=True, help="Owning project ID" + ) + session_start.add_argument("-a", "--agent", choices=AGENTS, required=True) + session_start.add_argument( + "--no-launch", + action="store_true", + help="Record a session without launching an agent", + ) + session_list = session_commands.add_parser( + "list", help="List user sessions or sessions in a project" + ) + session_list.add_argument( + "-p", "--project", help="List sessions owned by this project" + ) session_list.add_argument( "--status", choices=("open", "closed"), @@ -682,22 +1088,26 @@ def _parser() -> argparse.ArgumentParser: for name in ("show", "replay", "close", "delete", "purge", "jobs"): action = "List compilation jobs for" if name == "jobs" else f"{name.title()}" command = session_commands.add_parser(name, help=f"{action} one session") - command.add_argument("session_id", help="Session ID or full URI") command.add_argument( - "--json", action="store_true", help="Print JSON instead of a table" + "-i", + "--id", + dest="session_id", + required=True, + help="Session ID or full URI", + ) + command.add_argument( + "-j", "--json", action="store_true", help="Print JSON instead of a table" ) session_append = session_commands.add_parser( "append", help="Append one session event" ) session_append.add_argument( - "session_id", nargs="?", help="ID or DOCLING_CONTEXT_SESSION" - ) - session_append.add_argument("text", nargs="?", help="Event text, or use --stdin") - session_append.add_argument( - "--text", - dest="text_option", - help="Event text, useful with DOCLING_CONTEXT_SESSION", + "-i", + "--id", + dest="session_id", + help="Session ID or full URI (default: DOCLING_CONTEXT_SESSION)", ) + session_append.add_argument("--text", help="Event text, or use --stdin") session_append.add_argument( "--stdin", action="store_true", help="Read event text from stdin" ) @@ -721,17 +1131,23 @@ def _parser() -> argparse.ArgumentParser: session_trim = session_commands.add_parser( "trim", help="Retain recent session events" ) - session_trim.add_argument("session_id") + session_trim.add_argument( + "-i", "--id", dest="session_id", required=True, help="Session ID or full URI" + ) session_trim.add_argument("--keep-last", type=int, required=True) session_attachment = session_commands.add_parser( "attachment", help="Save one event attachment" ) - session_attachment.add_argument("session_id") - session_attachment.add_argument("key", help="Idempotency key of the attached event") + session_attachment.add_argument( + "-i", "--id", dest="session_id", required=True, help="Session ID or full URI" + ) + session_attachment.add_argument( + "--key", required=True, help="Idempotency key of the attached event" + ) session_attachment.add_argument("--output", type=Path, required=True) - session_attachment.add_argument("--json", action="store_true") + session_attachment.add_argument("-j", "--json", action="store_true") for command in (session_start, session_list, session_append, session_trim): - command.add_argument("--json", action="store_true") + command.add_argument("-j", "--json", action="store_true") memory = commands.add_parser("memory", help="Review and recall durable memories") memory_commands = memory.add_subparsers(dest="memory_command", required=True) @@ -770,27 +1186,33 @@ def _parser() -> argparse.ArgumentParser: ) for command in (memory_list, memory_proposals): command.add_argument( - "--json", action="store_true", help="Print JSON instead of a table" + "-j", "--json", action="store_true", help="Print JSON instead of a table" ) for name in ("show", "accept", "reject", "delete", "purge"): command = memory_commands.add_parser(name, help=f"{name.title()} one memory") - command.add_argument("uri") - command.add_argument("--json", action="store_true") + command.add_argument("-u", "--uri", required=True, help="Memory URI") + command.add_argument("-j", "--json", action="store_true") if name in {"reject", "delete"}: command.add_argument("--reason", default="") memory_correct = memory_commands.add_parser( "correct", help="Write an accepted correction" ) - memory_correct.add_argument("uri") - memory_correct.add_argument("claim", nargs="?") + memory_correct.add_argument("-u", "--uri", required=True, help="Memory URI") + memory_correct.add_argument("--claim", help="Corrected claim, or use --stdin") memory_correct.add_argument("--stdin", action="store_true") memory_correct.add_argument("--explanation") - memory_correct.add_argument("--json", action="store_true") + memory_correct.add_argument("-j", "--json", action="store_true") memory_create = memory_commands.add_parser( "create", help="Create a user-authored claim" ) - memory_create.add_argument("kind", choices=sorted(MEMORY_KINDS)) - memory_create.add_argument("claim", nargs="?") + memory_create.add_argument( + "--kind", + default="fact", + choices=sorted(MEMORY_KINDS), + help="Memory kind (default: fact)", + ) + memory_create.add_argument("--from", dest="source", type=Path) + memory_create.add_argument("--claim", help="Claim text, or use --stdin or --from") memory_create.add_argument("--stdin", action="store_true") memory_create.add_argument("--explanation", default="") memory_create.add_argument("--subject") @@ -802,46 +1224,217 @@ def _parser() -> argparse.ArgumentParser: ) for field in ("uri", "document_id", "revision_id", "xpath"): memory_create.add_argument(f"--source-{field.replace('_', '-')}") - memory_create.add_argument("--json", action="store_true") + memory_create.add_argument("-j", "--json", action="store_true") memory_search = memory_commands.add_parser("search", help="Search accepted claims") - memory_search.add_argument("query", nargs="?", default="") + memory_search.add_argument("-q", "--query", default="", help="Query text") memory_search.add_argument("--limit", type=int, default=10) - memory_search.add_argument("--json", action="store_true") + memory_search.add_argument("-j", "--json", action="store_true") memory_recall = memory_commands.add_parser( "recall", help="Recall claims once per session revision" ) - memory_recall.add_argument("query", nargs="?", default="") - memory_recall.add_argument("--session", help="ID or DOCLING_CONTEXT_SESSION") + memory_recall.add_argument("-q", "--query", default="", help="Query text") + memory_recall.add_argument("-s", "--session", help="ID or DOCLING_CONTEXT_SESSION") memory_recall.add_argument("--limit", type=int, default=10) - memory_recall.add_argument("--json", action="store_true") + memory_recall.add_argument("-j", "--json", action="store_true") memory_worker = memory_commands.add_parser( "worker", help="Process queued memory compilation jobs" ) memory_worker.add_argument( "--once", action="store_true", help="Process one due job then exit" ) - memory_worker.add_argument("--json", action="store_true") + memory_worker.add_argument("-j", "--json", action="store_true") memory_jobs = memory_commands.add_parser( "jobs", help="List compilation jobs for a session" ) - memory_jobs.add_argument("session_id") + memory_jobs.add_argument( + "-s", + "--session", + dest="session_id", + required=True, + help="Session ID or full URI", + ) memory_jobs.add_argument("--limit", type=int, default=100) - memory_jobs.add_argument("--json", action="store_true") + memory_jobs.add_argument("-j", "--json", action="store_true") + for sub in (memory_list, memory_create, memory_search, memory_recall): + sub.add_argument("-p", "--project") + agent = commands.add_parser( + "agent", help="Install the dc skill and MCP tools in an agent" + ) + agent_commands = agent.add_subparsers(dest="agent_command", required=True) + for action, help_text in ( + ("install", "Install the managed skill and MCP connection"), + ("status", "Check the managed skill and MCP connection"), + ("remove", "Remove the managed skill and MCP connection"), + ): + sub = agent_commands.add_parser(action, help=help_text) + sub.add_argument("-a", "--agent", choices=AGENTS, required=True) + sub.add_argument("--scope", choices=("user", "project"), default="user") + sub.add_argument("--project-root", type=Path, default=Path.cwd()) + sub.add_argument( + "--home", type=Path, default=Path.home(), help=argparse.SUPPRESS + ) + sub.add_argument("-j", "--json", action="store_true") return parser +def _source_document(source: Path) -> DocLangXDocument: + if source.name.lower().endswith(".dclx"): + return load_package(source.read_bytes()) + document = DocLangXDocument() + if not document.read_xml(source.read_text()): + raise ValueError(document.last_error() or "invalid DCLG document") + from .doclang_authoring import validate_doclang + + validate_doclang(document, label="resource source") + return document + + +def _typed_create( + args: argparse.Namespace, store: LocalContextStore, principal: Principal +) -> None: + catalog = Catalog(store) + uri = catalog.resource_uri(args.type, args.resource_id) + source = args.source.resolve(strict=True) + document = _source_document(source) + raw = document.get_part_text(DESCRIPTOR_PARTS[args.type]) + supplied = ( + json.loads(args.metadata.read_text()) + if args.metadata + else (json.loads(raw) if raw else {}) + ) + if not isinstance(supplied, dict): + raise TypeError("descriptor metadata must be a JSON object") + singular = DESCRIPTOR_NAMES[args.type] + metadata = { + **supplied, + "profile": f"{singular}-v1", + f"{singular}_id": args.resource_id, + "name": supplied.get("name", args.resource_id), + "summary": supplied.get("summary", ""), + } + document.set_part_text( + DESCRIPTOR_PARTS[args.type], json.dumps(metadata), "application/json" + ) + validate_descriptor(uri, document) + if args.project: + catalog.get_project(principal, args.project) + with store.transaction(): + record = Ingestor(store).add_resource( + principal, + uri, + document.write_bytes(), + filename="resource.dclx", + profile_metadata=metadata, + ) + catalog.register_resource(principal, record, args.type) + if args.project: + catalog.link(principal, args.project, uri) + _status( + {"uri": uri, "revision_id": record.revision_id, "project_id": args.project}, + json_output=args.json, + ) + + +def _knowledge_create( + args: argparse.Namespace, store: LocalContextStore, principal: Principal +) -> None: + catalog = Catalog(store) + uri = catalog.resource_uri("knowledge", args.resource_id) + concept_uri = catalog.resource_uri("concepts", args.concept) + concept = store.get_record(principal, concept_uri) + definition = validate_descriptor( + concept_uri, store.get_document(principal, concept_uri) + ) + metadata = { + "profile": "knowledge-v1", + "knowledge_id": args.resource_id, + "name": args.resource_id, + "summary": definition["summary"], + "concept_uri": concept_uri, + "concept_revision": concept.revision_id, + "fields": { + key: value["type"] + for key, value in definition["schema"]["properties"].items() + }, + } + if args.project: + catalog.get_project(principal, args.project) + data = f"{escape(args.resource_id)}: dataset defined by {escape(concept_uri)} at revision {concept.revision_id}".encode() + with store.transaction(): + record = Ingestor(store).add_resource( + principal, uri, data, filename="dataset.dclg", profile_metadata=metadata + ) + catalog.register_resource(principal, record, "knowledge") + catalog.register_knowledge_dataset( + principal, uri, concept_uri, concept.revision_id + ) + if args.project: + catalog.link(principal, args.project, uri) + catalog.link(principal, args.project, concept_uri) + _status( + { + "uri": uri, + "revision_id": record.revision_id, + "concept_uri": concept_uri, + "concept_revision": concept.revision_id, + }, + json_output=args.json, + ) + + def _session_command( args: argparse.Namespace, store: LocalContextStore, principal: Principal ) -> None: sessions = SessionStore(store) command = args.session_command - if command == "start": - _status(asdict(sessions.start(principal, args.id)), json_output=args.json) - elif command == "list": + if getattr(args, "session_id", None) and not args.session_id.startswith( + "docling://" + ): + rows = store.db.execute( + "SELECT s.project_id,s.session_id FROM dc_project_sessions s JOIN documents d ON d.tenant_id=s.tenant_id AND d.uri='docling://projects/'||s.project_id||'/sessions/'||s.session_id WHERE s.tenant_id=? AND s.owner_id=? AND s.session_id=? AND d.available=1", + (principal.tenant_id, principal.user_id, args.session_id), + ).fetchall() + if len(rows) > 1: + raise ValueError("session ID is ambiguous; use its full URI") + if rows: + args.session_id = ( + f"docling://projects/{rows[0]['project_id']}/sessions/{args.session_id}" + ) + if command == "list": since = _timestamp_filter(args.since) until = _timestamp_filter(args.until, upper=True) if since and until and since > until: raise ValueError("--since is later than --until") + if args.project: + if not 1 <= args.limit <= 1000 or args.offset < 0: + raise ValueError("invalid session list filter or page") + rows = Catalog(store).project_sessions(principal, args.project) + if args.status: + rows = [row for row in rows if row["status"] == args.status] + if since: + rows = [ + row + for row in rows + if datetime.fromisoformat(row["created_at"]) + .replace(tzinfo=UTC) + .isoformat() + >= since + ] + if until: + rows = [ + row + for row in rows + if datetime.fromisoformat(row["created_at"]) + .replace(tzinfo=UTC) + .isoformat() + <= until + ] + _table_or_json( + rows[args.offset : args.offset + args.limit], + headers=("session_id", "agent", "status", "owner_id", "created_at"), + json_output=args.json, + ) + return items = sessions.list( principal, status=args.status, @@ -864,6 +1457,17 @@ def _session_command( elif command == "show": info = sessions.get(principal, args.session_id) values = asdict(info) + address = parse_uri(info.uri) + if address.namespace == "projects": + values["project_id"] = address.segments[0] + matching = [ + row + for row in Catalog(store).project_sessions( + principal, address.segments[0] + ) + if row["session_id"] == address.segments[2] + ] + values["agent"] = matching[0]["agent"] if matching else None jobs = MemoryCompiler(store).jobs(principal, args.session_id) values["compilation"] = jobs[0].status if jobs else "none" _status(values, json_output=args.json) @@ -930,11 +1534,115 @@ def _session_command( ) +def _agent_spec(args: argparse.Namespace, principal: Principal) -> IntegrationSpec: + return IntegrationSpec( + harness=args.agent, + scope=args.scope, + project_root=args.project_root.resolve(), + home=args.home.expanduser().resolve(), + store=args.store.expanduser().resolve(), + tenant=principal.tenant_id, + user=principal.user_id, + allow_writes=True, + ) + + +def _agent_command(agent: str, executable: str, prompt: str) -> list[str]: + if agent == "hermes": + return [executable, "chat", "-q", prompt] + if agent in {"claude", "pi"}: + return [executable, "--append-system-prompt", prompt] + return [executable, prompt] + + +def _start_agent_session(args: argparse.Namespace, principal: Principal) -> int: + session_id = args.id or uuid.uuid4().hex + uri = f"docling://projects/{args.project}/sessions/{session_id}" + with LocalContextStore(args.store) as store: + catalog = Catalog(store) + catalog.get_project(principal, args.project) + existing = [ + row + for row in catalog.project_sessions(principal, args.project) + if row["session_id"] == session_id + ] + if existing and existing[0]["agent"] not in (None, args.agent): + raise ValueError("session ID already belongs to another agent") + if existing and existing[0]["status"] == "closed": + raise ValueError("session ID already closed") + executable = None if args.no_launch else shutil.which(args.agent) + if not args.no_launch and executable is None: + raise ValueError(f"{args.agent} is not installed or is not on PATH") + if not args.no_launch: + spec = IntegrationSpec( + harness=args.agent, + scope="user", + project_root=Path.cwd(), + home=Path.home(), + store=args.store.expanduser().resolve(), + tenant=principal.tenant_id, + user=principal.user_id, + allow_writes=True, + ) + install_agent(spec) + with LocalContextStore(args.store) as store: + sessions = SessionStore(store) + info = sessions.start(principal, uri) + Catalog(store).register_session( + principal, + args.project, + session_id, + agent=args.agent, + revision_id=info.revision_id, + ) + _status( + {**asdict(info), "project_id": args.project, "agent": args.agent}, + json_output=args.json, + ) + if args.no_launch: + return 0 + assert executable is not None + environment = os.environ.copy() + environment.update( + DOCLING_CONTEXT_STORE=str(args.store.expanduser().resolve()), + DOCLING_CONTEXT_TENANT=principal.tenant_id, + DOCLING_CONTEXT_USER=principal.user_id, + DOCLING_CONTEXT_PROJECT=args.project, + DOCLING_CONTEXT_SESSION=uri, + DOCLING_CONTEXT_AGENT=args.agent, + ) + environment["PATH"] = ( + str(Path(sys.executable).parent) + os.pathsep + environment.get("PATH", "") + ) + prompt = ( + f"Work in Docling Context project {args.project}. Your session URI is {uri}. " + "Use the docling-context MCP tools with this project ID for retrieval and exact citations. " + "Record useful conclusions with session_append using the full session URI. " + "The launcher will close the session when you exit." + ) + command = _agent_command(args.agent, executable, prompt) + try: + code = subprocess.run(command, env=environment, check=False).returncode + return 128 - code if code < 0 else code + except KeyboardInterrupt: + return 130 + finally: + with LocalContextStore(args.store) as store: + current = SessionStore(store).get(principal, uri) + if current.status == "open": + SessionStore(store).close(principal, uri) + + def _memory_command( args: argparse.Namespace, store: LocalContextStore, principal: Principal ) -> None: memories = MemoryService(store) command = args.memory_command + if getattr(args, "uri", None): + args.uri = Catalog.resource_uri("memories", args.uri) + project_id = getattr(args, "project", None) + if project_id: + Catalog(store).get_project(principal, project_id) if command in {"list", "proposals"}: since = _timestamp_filter(args.since) if command == "list" else None until = _timestamp_filter(args.until, upper=True) if command == "list" else None @@ -948,6 +1656,7 @@ def _memory_command( until=until, limit=args.limit, offset=args.offset, + project_id=project_id, ) _table_or_json( _memory_rows(items), @@ -973,16 +1682,34 @@ def _memory_command( if any(fields) and not all(fields): raise ValueError("provide all four --source-* citation fields") citations = (SourceCitation(*fields),) if all(fields) else () + if args.source: + if args.claim or args.stdin: + raise ValueError("use --from or a claim, not both") + document = _source_document(args.source.resolve(strict=True)) + text_nodes = [ + str(node["text"]) + for node in bounded_nodes(document, limit=1000, text_bytes=20000) + if node["name"] == "text" and node["text"] + ] + if not text_nodes: + raise ValueError("memory document needs a text node for its claim") + claim = text_nodes[0] + else: + document = None + claim = _input_text(args, name="claim") record = memories.create( principal, args.kind, - _input_text(args, name="claim"), + claim, explanation=args.explanation, subject=args.subject, confidence=args.confidence, citations=citations, accepted=args.accepted, + source_document=document, ) + if project_id: + Catalog(store).link(principal, project_id, record.uri) _status(asdict(record), json_output=args.json) elif command in {"accept", "reject", "correct", "delete", "purge"}: if command == "accept": @@ -1005,10 +1732,16 @@ def _memory_command( _status(asdict(record), json_output=args.json) elif command in {"search", "recall"}: items = ( - memories.search(principal, args.query, limit=args.limit) + memories.search( + principal, args.query, limit=args.limit, project_id=project_id + ) if command == "search" else memories.recall( - principal, _session_id(args.session), args.query, limit=args.limit + principal, + _session_id(args.session), + args.query, + limit=args.limit, + project_id=project_id, ) ) _table_or_json( @@ -1045,39 +1778,305 @@ def _memory_command( pass +def _store_counts(store: LocalContextStore, principal: Principal) -> dict[str, object]: + counts = { + table: store.db.execute( + f"SELECT COUNT(*) FROM {table} WHERE tenant_id=?", + (principal.tenant_id,), + ).fetchone()[0] + for table in ("documents", "revisions", "jobs") + } + user_prefix = f"docling://users/{principal.tenant_id}/{principal.user_id}/" + session_prefix = user_prefix + "sessions/" + counts["sessions"] = store.db.execute( + """SELECT COUNT(*) FROM documents WHERE tenant_id=? AND + ((substr(uri,1,?)=?) OR uri LIKE 'docling://projects/%/sessions/%')""", + (principal.tenant_id, len(session_prefix), session_prefix), + ).fetchone()[0] + counts["memories"] = store.db.execute( + """SELECT COUNT(*) FROM dc_resources WHERE tenant_id=? + AND resource_type='memories'""", + (principal.tenant_id,), + ).fetchone()[0] + counts["memory_jobs"] = store.db.execute( + """SELECT COUNT(*) FROM memory_compile_jobs + WHERE tenant_id=? AND user_id=?""", + (principal.tenant_id, principal.user_id), + ).fetchone()[0] + for label, table in ( + ("pending_ingestion_jobs", "jobs"), + ("pending_project_jobs", "dc_project_context_jobs"), + ("pending_memory_jobs", "memory_compile_jobs"), + ): + user_filter = " AND user_id=?" if label == "pending_memory_jobs" else "" + counts[label] = store.db.execute( + f"SELECT COUNT(*) FROM {table} WHERE tenant_id=? AND status IN ('queued','running'){user_filter}", + (principal.tenant_id, principal.user_id) + if user_filter + else (principal.tenant_id,), + ).fetchone()[0] + return {"store": str(store.root.resolve()), **counts} + + def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) principal = Principal(args.tenant, args.user) + bare = args.command is None + if bare: + args.command, args.json, args.limit, args.offset = "overview", False, 100, 0 try: - with LocalContextStore(args.store) as store: - worker = CollectionWorker(store) - if args.command == "status": - counts = { - table: store.db.execute( - f"SELECT COUNT(*) FROM {table} WHERE tenant_id=?", - (principal.tenant_id,), - ).fetchone()[0] - for table in ("documents", "revisions", "jobs") - } - user_prefix = ( - f"docling://users/{principal.tenant_id}/{principal.user_id}/" + if args.command == "agent": + spec = _agent_spec(args, principal) + agent_result = ( + install_agent(spec) + if args.agent_command == "install" + else inspect_agent(spec) + if args.agent_command == "status" + else remove_agent(spec) + ) + _status(agent_result, json_output=args.json) + return 0 if agent_result["status"] != "incomplete" else 1 + if args.command == "session" and args.session_command == "start": + return _start_agent_session(args, principal) + if args.command == "init": + with LocalContextStore(args.store): + pass + _status( + { + "catalog_schema_version": CATALOG_SCHEMA_VERSION, + "store": str(args.store.resolve()), + }, + json_output=args.json, + ) + return 0 + if args.command == "overview": + with LocalContextStore(args.store) as store, Catalog(store) as catalog: + rows = catalog.overview(principal, limit=args.limit, offset=args.offset) + counts = _store_counts(store, principal) + if args.json: + _json({"projects": rows, "counts": counts}) + else: + _table_or_json( + rows, + headers=( + "project_id", + "title", + "documents", + "memories", + "skills", + "concepts", + "knowledge", + "sessions", + ), + json_output=False, ) - for name in ("sessions", "memories"): - prefix = user_prefix + name + "/" - counts[name] = store.db.execute( - """SELECT COUNT(*) FROM documents WHERE tenant_id=? - AND substr(uri,1,?)=?""", - (principal.tenant_id, len(prefix), prefix), - ).fetchone()[0] - counts["memory_jobs"] = store.db.execute( - """SELECT COUNT(*) FROM memory_compile_jobs - WHERE tenant_id=? AND user_id=?""", - (principal.tenant_id, principal.user_id), - ).fetchone()[0] - _status( - {"store": str(store.root.resolve()), **counts}, - json_output=args.json, + _status(counts, json_output=False) + if bare: + print( + '\nCreate a project: dc project create --project materials --title "Materials"\nAdd documents: dc library add --source ./papers --project materials\nStart an agent: dc session start --project materials --agent codex' + ) + return 0 + with LocalContextStore(args.store) as store: + if args.command == "project": + with Catalog(store) as catalog: + if args.project_command == "create": + _status( + asdict( + catalog.create_project( + principal, args.project_id, args.title + ) + ), + json_output=args.json, + ) + elif args.project_command == "show": + _status( + catalog.project_summary(principal, args.project_id), + json_output=args.json, + ) + elif args.project_command == "resources": + _rows_or_json_lines( + catalog.project_resources(principal, args.project_id), + headers=("resource_type", "resource_id", "uri"), + json_output=args.json, + ) + elif args.project_command == "refresh": + catalog.refresh_project(principal, args.project_id) + _status( + {"project_id": args.project_id, "status": "queued"}, + json_output=args.json, + ) + elif args.project_command == "jobs": + _table_or_json( + catalog.context_jobs(principal, args.project_id), + headers=( + "kind", + "status", + "generation", + "attempts", + "last_error", + "updated_at", + ), + json_output=args.json, + ) + elif args.project_command == "worker": + context_job = ProjectContextWorker(store).run_once() + _status( + asdict(context_job) if context_job else {"job": "none"}, + json_output=args.json, + ) + elif args.project_command == "context-show": + project_info = catalog.get_project(principal, args.project_id) + _status( + { + kind: store.get_document(principal, uri).xml() + if uri + else None + for kind, uri in ( + ("description", project_info.description_uri), + ("abstract", project_info.abstract_uri), + ("overview", project_info.overview_uri), + ) + }, + json_output=args.json, + ) + elif args.project_command == "context-set": + supplied = [ + (kind, getattr(args, kind)) + for kind in ("description", "abstract", "overview") + if getattr(args, kind) + ] + if len(supplied) != 1: + raise ValueError( + "provide exactly one of --description, --abstract, --overview" + ) + kind, args.source = supplied[0] + args.project_command = f"set-{kind}" + catalog.get_project(principal, args.project_id) + source = args.source.resolve(strict=True) + uri = f"docling://projects/{args.project_id}/.{kind}.dclx" + record = Ingestor( + store, allowed_roots=(source.parent,) + ).add_resource(principal, uri, source) + catalog.set_project_document( + principal, args.project_id, kind, record + ) + _status( + {"uri": uri, "revision_id": record.revision_id}, + json_output=args.json, + ) + elif args.project_command == "link": + catalog.link(principal, args.project_id, args.uri) + _status( + {"project_id": args.project_id, "uri": args.uri}, + json_output=args.json, + ) + else: + catalog.unlink(principal, args.project_id, args.uri) + _status( + {"project_id": args.project_id, "uri": args.uri}, + json_output=args.json, + ) + elif args.command == "typed-create": + _typed_create(args, store, principal) + elif args.command == "typed-inspect": + catalog = Catalog(store) + if args.typed_command == "list": + _table_or_json( + catalog.resources(principal, args.type, args.project), + headers=("resource_id", "uri"), + json_output=args.json, + ) + else: + uri = catalog.resource_uri(args.type, args.resource_id) + store.get_record(principal, uri) + if args.typed_command == "show": + _status( + validate_descriptor( + uri, store.get_document(principal, uri) + ), + json_output=args.json, + ) + elif args.typed_command == "projects": + _table_or_json( + catalog.resource_projects(principal, uri), + headers=("project_id", "title"), + json_output=args.json, + ) + elif args.typed_command == "links": + _table_or_json( + catalog.resource_links(principal, uri), + headers=("source_uri", "target_uri", "relation"), + json_output=args.json, + ) + else: + catalog.link_resources( + principal, uri, args.target_uri, args.relation + ) + _status( + { + "source_uri": uri, + "target_uri": args.target_uri, + "relation": args.relation, + }, + json_output=args.json, + ) + elif args.command == "knowledge": + catalog = Catalog(store) + if args.knowledge_command == "create": + _knowledge_create(args, store, principal) + return 0 + if args.knowledge_command == "list": + _table_or_json( + catalog.resources(principal, "knowledge", args.project), + headers=("resource_id", "uri"), + json_output=args.json, + ) + return 0 + args.knowledge_uri = catalog.resource_uri( + "knowledge", args.knowledge_uri ) + if args.knowledge_command == "show": + _status( + catalog.knowledge_dataset(principal, args.knowledge_uri), + json_output=args.json, + ) + return 0 + if args.knowledge_command == "add-fact": + fact_id = catalog.add_fact( + principal, + args.knowledge_uri, + entity_type=args.entity_type, + entity_id=args.entity_id, + property=args.property, + value=json.loads(args.value), + source_uri=args.source_uri, + source_revision=args.source_revision, + source_xpath=args.source_xpath, + unit=args.unit, + ) + _status({"fact_id": fact_id}, json_output=args.json) + else: + _table_or_json( + [ + fact + for fact in catalog.knowledge_facts( + principal, args.knowledge_uri + ) + if (not args.entity or fact["entity_id"] == args.entity) + and (not args.property or fact["property"] == args.property) + ], + headers=( + "fact_id", + "entity_type", + "entity_id", + "property", + "value", + "source_uri", + "source_revision", + "source_xpath", + ), + json_output=args.json, + ) elif args.command == "index": if args.index_command == "status": _status(store.index_status(principal), json_output=args.json) @@ -1104,7 +2103,7 @@ def main(argv: list[str] | None = None) -> int: bar, done, size ), ) - _json(rebuild_result) + _status(rebuild_result, json_output=args.json) else: if args.index_operation == "status": _status( @@ -1139,32 +2138,55 @@ def main(argv: list[str] | None = None) -> int: bar, done, size ), ) - _json(store.index_status(principal)) + _status(store.index_status(principal), json_output=args.json) elif args.command == "add-resource": - _add_resource(args, store, principal) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + previous_logging_disable = logging.root.manager.disable + logging.disable(logging.WARNING) + try: + if _add_resource(args, store, principal): + return 1 + finally: + logging.disable(previous_logging_disable) elif args.command == "session": _session_command(args, store, principal) elif args.command == "memory": _memory_command(args, store, principal) - elif args.command == "task": - _status( - asdict(worker.get_job(principal, args.job_id)), + elif args.command == "library-list": + _table_or_json( + Catalog(store).resources(principal, "library", args.project), + headers=("resource_id", "uri", "revision_id"), json_output=args.json, ) - elif args.command == "worker": - if args.once: - job = worker.run_once() - _json({"job_id": job.job_id if job else None}) + elif args.command == "library-relations": + catalog = Catalog(store) + uri = catalog.resource_uri("library", args.document_id) + store.get_record(principal, uri) + if args.library_command == "projects": + _table_or_json( + catalog.resource_projects(principal, uri), + headers=("project_id", "title"), + json_output=args.json, + ) + elif args.library_command == "links": + _table_or_json( + catalog.resource_links(principal, uri), + headers=("source_uri", "target_uri", "relation"), + json_output=args.json, + ) else: - from threading import Event - - try: - worker.run_forever(Event()) - except KeyboardInterrupt: - pass - elif args.command == "ls": - for entry in store.list_children(principal, _prefix(args.uri)): - _json(asdict(entry)) + catalog.link_resources( + principal, uri, args.target_uri, args.relation + ) + _status( + { + "source_uri": uri, + "target_uri": args.target_uri, + "relation": args.relation, + }, + json_output=args.json, + ) elif args.command == "jobs": from .service import ContextService, JobStatusRequest @@ -1175,11 +2197,21 @@ def main(argv: list[str] | None = None) -> int: ) _status(job_data, json_output=args.json) elif args.command == "tree": - for entry in store.walk( + entries = store.walk( principal, _prefix(args.uri), depth=args.depth, limit=1_000 - ): - print(f"{' ' * (entry.depth - 1)}{entry.uri}") + ) + if args.raw: + for entry in entries: + print(f"{' ' * (entry.depth - 1)}{entry.uri}") + else: + _rows_or_json_lines( + [asdict(entry) for entry in entries], + headers=("depth", "uri", "kind", "document_id", "revision_id"), + json_output=args.json, + ) elif args.command in {"outline", "show"}: + if not args.uri.startswith("docling://"): + args.uri = Catalog.resource_uri("library", args.uri) from .service import ContextService, OutlineRequest, ShowRequest service = ContextService(store, principal) @@ -1187,10 +2219,10 @@ def main(argv: list[str] | None = None) -> int: value = service.outline( OutlineRequest(args.uri, args.revision) ).data - if args.json: - _json(value) - else: + if args.raw: print(value["toc_xml"]) + else: + _status(value, json_output=args.json) else: value = service.show( ShowRequest( @@ -1201,12 +2233,18 @@ def main(argv: list[str] | None = None) -> int: args.format, ) ).data - if args.json: - _json(value) - else: + if args.raw: print(value["content"]) + else: + _status(value, json_output=args.json) elif args.command in {"find", "grep"}: - hits = _scan(store, principal, args.term, _prefix(args.uri)) + hits = _scan( + store, + principal, + args.term, + _prefix(args.uri), + project_id=args.project, + ) if args.command == "find": hits.sort( key=lambda hit: ( @@ -1214,8 +2252,11 @@ def main(argv: list[str] | None = None) -> int: ), reverse=True, ) - for hit in hits: - _json(hit) + _rows_or_json_lines( + hits, + headers=("uri", "revision_id", "xpath", "text", "page"), + json_output=args.json, + ) elif args.command == "search": state = store.db.execute( "SELECT model_id FROM vector_state WHERE singleton=1" @@ -1226,10 +2267,7 @@ def main(argv: list[str] | None = None) -> int: if args.image is not None and mode == "lexical": raise ValueError("image search needs a vector index") if mode in {"vector", "hybrid"}: - if state is None: - raise ValueError( - "run 'dc index vector rebuild' before vector search" - ) + state = verified_vector_state(store) store.require_exclusive_vector_scope(principal) retriever = Retriever( store, embedder=stored_embedding_provider(state["model_id"]) @@ -1243,14 +2281,19 @@ def main(argv: list[str] | None = None) -> int: scope=SearchScope( uri=_prefix(args.uri), xpath=args.xpath, - document_uri=args.document, + document_uri=Catalog.resource_uri("library", args.document) + if args.document + else None, tiers=tuple(args.tier) if args.tier else (0, 1, 2), + project_id=args.project, ), k=args.k, mode=mode, max_tokens=args.max_tokens, per_result_tokens=args.per_result_tokens, ) + for warning in result.warnings: + print(f"dc: warning: {warning}", file=sys.stderr) if args.json: for retrieval_hit in result.hits: _json(asdict(retrieval_hit)) @@ -1262,7 +2305,17 @@ def main(argv: list[str] | None = None) -> int: print( f"Showing {len(result.hits)} of {result.total_hits} matching nodes." ) - except (OSError, ValueError, RuntimeError, KeyError, PermissionError) as exc: + except KeyboardInterrupt: + return 130 + except ( + OSError, + ValueError, + TypeError, + RuntimeError, + KeyError, + PermissionError, + sqlite3.Error, + ) as exc: print(f"dc: {exc}", file=sys.stderr) return 1 return 0 diff --git a/src/docling_context/compiler.py b/src/docling_context/compiler.py index f2b741e..81b832a 100644 --- a/src/docling_context/compiler.py +++ b/src/docling_context/compiler.py @@ -8,11 +8,13 @@ from dataclasses import dataclass from datetime import UTC, datetime, timedelta +from .catalog import Catalog from .durable_memory import MemoryService, SourceCitation from .local import LocalContextStore, RecordMissing from .models import Principal from .profiles import MAX_SESSION_EVENTS from .sessions import SessionStore +from .uri import parse_uri def _now() -> str: @@ -63,17 +65,30 @@ def enqueue_missing(self) -> int: FROM outbox o JOIN documents d ON d.tenant_id=o.tenant_id AND d.uri=o.uri AND d.revision_id=o.revision_id - WHERE o.operation='put' AND o.uri LIKE 'docling://users/%/sessions/%' + WHERE o.operation='put' AND + (o.uri LIKE 'docling://users/%/sessions/%' OR + o.uri LIKE 'docling://projects/%/sessions/%') AND NOT EXISTS (SELECT 1 FROM memory_compile_jobs j WHERE j.tenant_id=o.tenant_id AND j.session_uri=o.uri AND j.session_revision_id=o.revision_id)""" ).fetchall() added = 0 for row in rows: - parts = row["uri"].split("/") - if len(parts) != 7 or parts[5] != "sessions": - continue - principal = Principal(parts[3], parts[4]) + address = parse_uri(row["uri"]) + if address.namespace == "projects": + owner = self.store.db.execute( + """SELECT owner_id FROM dc_project_sessions + WHERE tenant_id=? AND project_id=? AND session_id=?""", + (row["tenant_id"], address.segments[0], address.segments[2]), + ).fetchone() + if owner is None: + continue + principal = Principal(row["tenant_id"], owner["owner_id"]) + else: + parts = row["uri"].split("/") + if len(parts) != 7 or parts[5] != "sessions": + continue + principal = Principal(parts[3], parts[4]) try: self.sessions.uri(principal, row["uri"]) except (ValueError, PermissionError): @@ -187,6 +202,9 @@ def _process(self, row) -> int: generator="deterministic-v1", confidence=0.5, ) + address = parse_uri(current.uri) + if address.namespace == "projects": + Catalog(self.store).link(principal, address.segments[0], before.uri) if before.status == "proposed" and before.citations == (citation,): count += 1 return count diff --git a/src/docling_context/doclang_authoring.py b/src/docling_context/doclang_authoring.py new file mode 100644 index 0000000..38e0ff4 --- /dev/null +++ b/src/docling_context/doclang_authoring.py @@ -0,0 +1,67 @@ +"""Shared DocLang authoring guidance and validated outline helpers.""" + +from __future__ import annotations + +from urllib.parse import quote, unquote +from xml.etree import ElementTree as ET + +from doclang import DoclangDocument, DocLangXDocument, DocLangXDocumentRole + +from .package import PackageError + +DEFAULT_SKILL_ID = "doclang-authoring" +DEFAULT_SKILL_URI = f"docling://resources/skills/{DEFAULT_SKILL_ID}" +DEFAULT_SKILL_NAME = "Write and validate DocLang" +DEFAULT_SKILL_SUMMARY = "Author schema-valid DCLG XML and DCLX packages." +DEFAULT_SKILL_TEXT = """Write DocLang sources as DCLG XML with a root. Use elements allowed by the bundled DocLang XSD, such as Title and Body. Escape XML special characters. A DCLX file is a DocLang package containing a DCLG body and optional companion documents and parts; it is not just XML renamed with a .dclx extension. + +Before saving DCLG, parse it with DocLang and run document.validate(allow_empty_namespace=True). This checks the bundled XSD and Schematron rules. Before saving DCLX, validate the body and every generated DCLG companion document, run document.validate_package(), then write_bytes(); reopen the bytes and validate the package again. Validate summaries and outlines as DocLang documents too. Do not invent XML elements such as or : they are not valid DocLang body elements. Represent an outline with nodes, optionally putting before its title. Use the percent-encoded source XPath in the fragment. Call package.set(DocLangXDocumentRole.TABLE_OF_CONTENTS, outline) to store the native annotations/toc.dclg companion document. Use dc library outline or dc library show to inspect citations. + +For the installed Python API: from doclang import DocLangXDocument; document = DocLangXDocument(); document.read_xml(xml); report = document.validate(allow_empty_namespace=True); assert report.ok(), (report.xsd_errors, report.schematron_errors). For a DCLX package, also assert document.validate_package()["ok"]. The doclang package bundles the authoritative XSD and Schematron rules; follow their current version rather than copying a partial grammar from this skill.""" + + +def validate_doclang(document: DoclangDocument, *, label: str) -> None: + """Reject XML that parses but violates the bundled XSD or Schematron.""" + report = document.validate(allow_empty_namespace=True) + if not report.ok(): + raise PackageError( + f"invalid {label}: {report.xsd_errors or report.schematron_errors}" + ) + + +def get_outline(document: DocLangXDocument) -> DoclangDocument | None: + """Read the native DCLG outline companion document.""" + return document.get(DocLangXDocumentRole.TABLE_OF_CONTENTS) + + +def get_summary(document: DocLangXDocument) -> DoclangDocument | None: + return document.get(DocLangXDocumentRole.SUMMARY) + + +def set_summary(document: DocLangXDocument, xml: str) -> bool: + return document.set(DocLangXDocumentRole.SUMMARY, DoclangDocument(xml)) + + +def set_outline(document: DocLangXDocument, xml: str) -> bool: + return document.set(DocLangXDocumentRole.TABLE_OF_CONTENTS, DoclangDocument(xml)) + + +def outline_xml(entries: list[tuple[str, str, int]]) -> str: + """Build a DocLang TOC; href fragments preserve source XPaths.""" + root = ET.Element("doclang") + for xpath, title, level in entries: + heading = ET.SubElement(root, "heading", {"level": str(max(1, level))}) + ET.SubElement(heading, "href", {"uri": f"#{quote(xpath, safe='')}"}) + heading[-1].tail = title + return ET.tostring(root, encoding="unicode") + + +def outline_links(xml: str) -> list[str]: + links: list[str] = [] + for heading in ET.fromstring(xml).iter("heading"): + href = heading.find("href") + if href is not None and (uri := href.get("uri", "")).startswith("#"): + xpath = unquote(uri[1:]) + if xpath.startswith("/doclang[1]/"): + links.append(xpath) + return links diff --git a/src/docling_context/durable_memory.py b/src/docling_context/durable_memory.py index 22c81d9..3688e7a 100644 --- a/src/docling_context/durable_memory.py +++ b/src/docling_context/durable_memory.py @@ -12,6 +12,7 @@ from doclang import DocLangXDocument +from .catalog import Catalog from .local import LocalContextStore, RecordMissing from .models import NodeAddress, Principal from .package import PackageError, PackageMissing @@ -63,17 +64,16 @@ def __init__(self, store: LocalContextStore): @staticmethod def _prefix(principal: Principal) -> str: - return f"docling://users/{principal.tenant_id}/{principal.user_id}/memories" + return "docling://resources/memories" @classmethod def _uri(cls, principal: Principal, uri: str) -> str: address = parse_uri(uri) authorize_uri(principal, address) if ( - address.namespace != "users" - or len(address.segments) != 5 - or address.segments[2] != "memories" - or address.segments[3] not in MEMORY_KINDS + address.namespace != "resources" + or len(address.segments) != 2 + or address.segments[0] != "memories" ): raise ValueError("expected one memory URI") return address.value @@ -82,10 +82,7 @@ def _record(self, principal: Principal, uri: str) -> MemoryRecord: uri = self._uri(principal, uri) stored = self.store.get_record(principal, uri) metadata = validate_memory(self.store.get_document(principal, uri)) - if ( - metadata["kind"] != parse_uri(uri).segments[3] - or metadata["memory_id"] != parse_uri(uri).segments[4] - ): + if metadata["memory_id"] != parse_uri(uri).segments[1]: raise ValueError("memory metadata differs from URI") return MemoryRecord( uri=uri, @@ -127,6 +124,9 @@ def _sync(self, principal: Principal) -> None: if indexed is not None and indexed["revision_id"] == row["revision_id"]: continue record = self._record(principal, uri) + owner_id = validate_memory(self.store.get_document(principal, uri)).get( + "owner_id", principal.user_id + ) with self.store.transaction(): self.store.db.execute( """INSERT INTO memory_index @@ -140,7 +140,7 @@ def _sync(self, principal: Principal) -> None: updated_at=excluded.updated_at""", ( principal.tenant_id, - principal.user_id, + owner_id, uri, record.memory_id, record.kind, @@ -164,7 +164,7 @@ def _sync(self, principal: Principal) -> None: ( ( principal.tenant_id, - principal.user_id, + owner_id, uri, citation.uri, citation.document_id, @@ -175,8 +175,8 @@ def _sync(self, principal: Principal) -> None: ), ) stale = self.store.db.execute( - "SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=?", - (principal.tenant_id, principal.user_id), + "SELECT uri FROM memory_index WHERE tenant_id=?", + (principal.tenant_id,), ).fetchall() with self.store.transaction(): for row in stale: @@ -194,6 +194,15 @@ def get(self, principal: Principal, uri: str) -> MemoryRecord: self._sync(principal) return self._record(principal, uri) + def _assert_owner(self, principal: Principal, uri: str) -> None: + self._sync(principal) + row = self.store.db.execute( + "SELECT user_id FROM memory_index WHERE tenant_id=? AND uri=?", + (principal.tenant_id, uri), + ).fetchone() + if row is not None and row["user_id"] != principal.user_id: + raise PermissionError("memory owner does not match") + def _validate_citations( self, principal: Principal, citations: tuple[SourceCitation, ...] ) -> None: @@ -222,25 +231,34 @@ def _write( metadata: dict, *, expected_revision: str | None, + source_document: DocLangXDocument | None = None, ) -> MemoryRecord: - document = DocLangXDocument() - xml = ( - f"{escape(metadata['kind'])}" - f"{escape(metadata['claim'])}" - f"{escape(metadata['explanation'])}" - ) - if not document.read_xml(xml): - raise ValueError(document.last_error()) + document = source_document + if document is None and expected_revision and metadata.get("authored_document"): + previous = self.store.get_document(principal, uri) + if validate_memory(previous)["claim"] == metadata["claim"]: + document = previous + if document is None: + document = DocLangXDocument() + xml = ( + f"{escape(metadata['kind'])}" + f"{escape(metadata['claim'])}" + f"{escape(metadata['explanation'])}" + ) + if not document.read_xml(xml): + raise ValueError(document.last_error()) document.set_part_text( MEMORY_PART, json.dumps(metadata, sort_keys=True), "application/json" ) validate_memory(document) - self.store.put( - principal, - uri, - document.write_bytes(), - expected_revision=expected_revision, - ) + with self.store.transaction(): + stored = self.store.put( + principal, + uri, + document.write_bytes(), + expected_revision=expected_revision, + ) + Catalog(self.store).register_resource(principal, stored, "memories") self._sync(principal) return self._record(principal, uri) @@ -259,6 +277,7 @@ def create( accepted: bool = False, valid_from: str | None = None, valid_to: str | None = None, + source_document: DocLangXDocument | None = None, ) -> MemoryRecord: if kind not in MEMORY_KINDS or not isinstance(claim, str) or not claim.strip(): raise ValueError("invalid memory kind or claim") @@ -268,19 +287,19 @@ def create( self._sync(principal) key = _claim_key(claim) existing = self.store.db.execute( - """SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=? + """SELECT uri FROM memory_index WHERE tenant_id=? AND kind=? AND subject IS ? AND claim_key=? ORDER BY updated_at DESC LIMIT 1""", - (principal.tenant_id, principal.user_id, kind, subject, key), + (principal.tenant_id, kind, subject, key), ).fetchone() if existing is not None: return self._record(principal, existing["uri"]) # Corrections retain their old claim keys to prevent replaying an older # session revision from reviving a claim the user already corrected. prior = self.store.db.execute( - """SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=? + """SELECT uri FROM memory_index WHERE tenant_id=? AND kind=? AND subject IS ? AND status!='deleted'""", - (principal.tenant_id, principal.user_id, kind, subject), + (principal.tenant_id, kind, subject), ).fetchall() for row in prior: metadata = validate_memory(self.store.get_document(principal, row["uri"])) @@ -291,13 +310,13 @@ def create( conflicts = [ row["uri"] for row in self.store.db.execute( - """SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=? + """SELECT uri FROM memory_index WHERE tenant_id=? AND kind=? AND subject=? AND status='accepted'""", - (principal.tenant_id, principal.user_id, kind, subject), + (principal.tenant_id, kind, subject), ) ] memory_id = uuid.uuid4().hex - uri = f"{self._prefix(principal)}/{kind}/{memory_id}" + uri = f"{self._prefix(principal)}/{memory_id}" status = "accepted" if accepted else "proposed" metadata = { "profile": "memory-v1", @@ -309,6 +328,7 @@ def create( "confidence": confidence, "subject": subject, "author": author, + "owner_id": principal.user_id, "generator": generator, "valid_from": valid_from, "valid_to": valid_to, @@ -316,7 +336,15 @@ def create( "conflicts": conflicts, "history": [{"status": status, "by": author, "at": _now()}], } - return self._write(principal, uri, metadata, expected_revision=None) + if source_document is not None: + metadata["authored_document"] = True + return self._write( + principal, + uri, + metadata, + expected_revision=None, + source_document=source_document, + ) def transition( self, principal: Principal, uri: str, status: str, *, reason: str = "" @@ -324,6 +352,7 @@ def transition( if status not in MEMORY_STATUSES: raise ValueError("invalid memory status") current = self.get(principal, uri) + self._assert_owner(principal, current.uri) if current.status == status: return current allowed = { @@ -360,6 +389,7 @@ def delete( def purge(self, principal: Principal, uri: str) -> int: """Remove a deleted claim, its revision history, and its recall records.""" memory = self.get(principal, uri) + self._assert_owner(principal, memory.uri) if memory.status != "deleted": raise ValueError("delete the memory before purging it") rows = self.store.db.execute( @@ -405,6 +435,7 @@ def correct( citations: tuple[SourceCitation, ...] | None = None, ) -> MemoryRecord: current = self.get(principal, uri) + self._assert_owner(principal, current.uri) if current.status not in {"accepted", "proposed"} or not claim.strip(): raise ValueError("only active memories can be corrected") sources = citations if citations is not None else current.citations @@ -412,6 +443,7 @@ def correct( metadata = validate_memory(self.store.get_document(principal, current.uri)) aliases = metadata.setdefault("claim_aliases", []) aliases.append(_claim_key(current.claim)) + metadata.pop("authored_document", None) metadata.update( claim=claim.strip(), explanation=current.explanation if explanation is None else explanation, @@ -438,23 +470,24 @@ def reconcile_sources(self, principal: Principal) -> int: """Move accepted claims with no surviving cited source back to review.""" self._sync(principal) rows = self.store.db.execute( - """SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=? + """SELECT uri,user_id FROM memory_index WHERE tenant_id=? AND status='accepted'""", - (principal.tenant_id, principal.user_id), + (principal.tenant_id,), ).fetchall() changed = 0 for row in rows: - memory = self._record(principal, row["uri"]) + owner = Principal(principal.tenant_id, row["user_id"]) + memory = self._record(owner, row["uri"]) if not memory.citations: continue supported = False for citation in memory.citations: try: - current = self.store.get_record(principal, citation.uri) + current = self.store.get_record(owner, citation.uri) if current.document_id != citation.document_id: continue self.store.read_node( - principal, + owner, NodeAddress( citation.document_id, citation.revision_id, citation.xpath ), @@ -465,7 +498,7 @@ def reconcile_sources(self, principal: Principal) -> int: break if not supported: self.transition( - principal, + owner, memory.uri, "proposed", reason="cited sources unavailable", @@ -474,22 +507,35 @@ def reconcile_sources(self, principal: Principal) -> int: return changed def search( - self, principal: Principal, query: str = "", *, limit: int = 10 + self, + principal: Principal, + query: str = "", + *, + limit: int = 10, + project_id: str | None = None, ) -> tuple[MemoryRecord, ...]: if not 1 <= limit <= 100: raise ValueError("memory search limit is out of bounds") self.reconcile_sources(principal) + if project_id: + from .catalog import Catalog + + Catalog(self.store).get_project(principal, project_id) + if not query.strip() and project_id: + return self.list( + principal, status="accepted", limit=limit, project_id=project_id + ) if not query.strip(): rows = self.store.db.execute( - """SELECT uri FROM memory_index WHERE tenant_id=? AND user_id=? + """SELECT uri FROM memory_index WHERE tenant_id=? AND status='accepted' ORDER BY updated_at DESC LIMIT ?""", - (principal.tenant_id, principal.user_id, limit), + (principal.tenant_id, limit), ).fetchall() return tuple(self._record(principal, row["uri"]) for row in rows) result = Retriever(self.store).search( principal, query, - scope=SearchScope(uri=self._prefix(principal)), + scope=SearchScope(uri=self._prefix(principal), project_id=project_id), mode="lexical", k=100, ) @@ -516,6 +562,7 @@ def list( until: str | None = None, limit: int = 100, offset: int = 0, + project_id: str | None = None, ) -> tuple[MemoryRecord, ...]: """List current claims, including review history, for the current user.""" if ( @@ -526,8 +573,16 @@ def list( ): raise ValueError("invalid memory list filter or page") self.reconcile_sources(principal) - conditions = ["tenant_id=?", "user_id=?"] - params: list[str | int] = [principal.tenant_id, principal.user_id] + conditions = ["tenant_id=?"] + params: list[str | int] = [principal.tenant_id] + if project_id: + from .catalog import Catalog + + Catalog(self.store).get_project(principal, project_id) + conditions.append( + "uri IN (SELECT r.uri FROM dc_project_resource_links l JOIN dc_resources r USING (tenant_id,resource_type,resource_id) WHERE l.tenant_id=? AND l.project_id=? AND l.resource_type='memories')" + ) + params.extend((principal.tenant_id, project_id)) for column, value, operator in ( ("status", status, "="), ("kind", kind, "="), @@ -551,12 +606,13 @@ def recall( query: str = "", *, limit: int = 10, + project_id: str | None = None, ) -> tuple[MemoryRecord, ...]: from .sessions import SessionStore session_uri = SessionStore.uri(principal, session_id) self.store.get_record(principal, session_uri) - candidates = self.search(principal, query, limit=100) + candidates = self.search(principal, query, limit=100, project_id=project_id) delivered = [] for memory in candidates: with self.store.transaction(): diff --git a/src/docling_context/ingest_queue.py b/src/docling_context/ingest_queue.py index 1a27e17..f2bb1dc 100644 --- a/src/docling_context/ingest_queue.py +++ b/src/docling_context/ingest_queue.py @@ -2,10 +2,12 @@ from __future__ import annotations +import hashlib import uuid from dataclasses import asdict, dataclass from datetime import UTC, datetime, timedelta +from .catalog import Catalog, new_document_uri from .conversion_config import PdfConversionConfig from .converters import LocalDoclingConverter from .ingestion import Ingestor @@ -29,6 +31,7 @@ class IngestJob: error: str | None created_at: str updated_at: str + project_id: str | None = None def to_dict(self) -> dict: return asdict(self) @@ -46,9 +49,20 @@ def submit( filename: str, *, force: bool = False, + project_id: str | None = None, ) -> IngestJob: + uri = uri or new_document_uri() address = parse_uri(uri) authorize_uri(principal, address) + if project_id and ( + address.namespace != "resources" + or address.segments[0] != "library" + or len(address.segments) != 2 + ): + raise ValueError("project ingestion requires a flat library URI") + if project_id: + with Catalog(self.store) as catalog: + catalog.get_project(principal, project_id) if not 0 < len(source) <= MAX_QUEUED_BYTES or not filename: raise ValueError("queued source must be nonempty and at most 2.5 MB") job_id, stamp = uuid.uuid4().hex, _now() @@ -56,7 +70,8 @@ def submit( self.store.db.execute( """INSERT INTO ingest_jobs (job_id,tenant_id,user_id,uri,filename,source,force,status, - created_at,updated_at) VALUES (?,?,?,?,?,?,?,'queued',?,?)""", + created_at,updated_at,project_id) + VALUES (?,?,?,?,?,?,?,'queued',?,?,?)""", ( job_id, principal.tenant_id, @@ -67,6 +82,7 @@ def submit( int(force), stamp, stamp, + project_id, ), ) return self.get(principal, job_id) @@ -100,31 +116,50 @@ def run_once(self) -> IngestJob | None: ) principal = Principal(row["tenant_id"], row["user_id"]) revision = None + final_uri = row["uri"] error = None try: - record = Ingestor( + ingestor = Ingestor( self.store, converter=LocalDoclingConverter( PdfConversionConfig(document_timeout=300) ), - ).add_resource( - principal, - row["uri"], - row["source"], - filename=row["filename"], - force=bool(row["force"]), ) + address = parse_uri(row["uri"]) + if address.namespace == "resources" and address.segments[0] == "library": + digest = hashlib.sha256(row["source"]).hexdigest() + with Catalog(self.store) as catalog, self.store.transaction(): + final_uri = catalog.find_source(principal, digest) or row["uri"] + record = ingestor.add_resource( + principal, + final_uri, + row["source"], + filename=row["filename"], + force=bool(row["force"]), + ) + catalog.register_resource(principal, record) + if row["project_id"]: + catalog.link(principal, row["project_id"], record.uri) + else: + record = ingestor.add_resource( + principal, + row["uri"], + row["source"], + filename=row["filename"], + force=bool(row["force"]), + ) revision = record.revision_id except Exception as exc: # noqa: BLE001 - converter failures must be recorded error = f"{type(exc).__name__}: {exc}"[:2_000] with self.store.transaction(): self.store.db.execute( - """UPDATE ingest_jobs SET status=?,revision_id=?,error=?, + """UPDATE ingest_jobs SET status=?,revision_id=?,error=?,uri=?, source=x'',lease_until=NULL,updated_at=? WHERE job_id=?""", ( "failed" if error else "completed", revision, error, + final_uri, _now(), row["job_id"], ), diff --git a/src/docling_context/ingestion.py b/src/docling_context/ingestion.py index a73c920..b6152c8 100644 --- a/src/docling_context/ingestion.py +++ b/src/docling_context/ingestion.py @@ -8,16 +8,17 @@ from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import Protocol -from xml.etree import ElementTree as ET from xml.sax.saxutils import escape from doclang import DocLangXDocument from .contracts import ContextStore from .converters import ConversionResult, Converter, LocalDoclingConverter +from .doclang_authoring import outline_xml, set_outline, set_summary, validate_doclang from .local import RecordMissing from .models import DocumentRecord, Principal from .package import MAX_PACKAGE_BYTES, PackageError, bounded_nodes, load_package +from .profiles import DESCRIPTOR_PARTS from .uri import authorize_uri, parse_uri @@ -42,26 +43,21 @@ def _toc(document: DocLangXDocument) -> str: for node in bounded_nodes(document, limit=10_000, text_bytes=512) if node["name"] == "heading" ][:256] - root = ET.Element("doclang") - toc = ET.SubElement(root, "toc") + from xml.etree import ElementTree as ET + xml_headings = [ element for element in ET.fromstring(document.xml()).iter() if element.tag.rsplit("}", 1)[-1] == "heading" ][:256] - stack: list[tuple[int, ET.Element]] = [] + entries: list[tuple[str, str, int]] = [] for node, element in zip(headings, xml_headings, strict=False): try: - level = max(1, min(6, int(element.get("level", "1")))) + level = max(1, int(element.get("level", "1"))) except ValueError: level = 1 - while stack and stack[-1][0] >= level: - stack.pop() - parent = stack[-1][1] if stack else toc - entry = ET.SubElement(parent, "entry", {"xpath": str(node["xpath"])}) - ET.SubElement(entry, "description").text = str(node["text"])[:200] - stack.append((level, entry)) - return ET.tostring(root, encoding="unicode") + entries.append((str(node["xpath"]), str(node["text"])[:200], level)) + return outline_xml(entries) def _fingerprint(source: bytes, converter_id: str, options: dict[str, object]) -> str: @@ -102,6 +98,7 @@ def add_resource( filename: str | None = None, origin_uri: str | None = None, force: bool = False, + profile_metadata: dict[str, object] | None = None, ) -> DocumentRecord: address = parse_uri(uri) authorize_uri(principal, address) @@ -158,6 +155,8 @@ def add_resource( if self.summary_provider is None else type(self.summary_provider).__qualname__ ) + if profile_metadata is not None: + options["profile_metadata"] = profile_metadata fingerprint = _fingerprint(data, converter_id, options) try: current = self.store.get_record(principal, address.value) @@ -177,12 +176,27 @@ def add_resource( document = DocLangXDocument() if not document.read_xml(data.decode("utf-8")): raise PackageError(document.last_error() or "invalid DocLang XML") + validate_doclang(document, label="DCLG source") conversion = ConversionResult(b"", "doclang-native", "1") else: assert converter is not None conversion = converter.convert(data, name) document = load_package(conversion.package) + if profile_metadata is not None: + resource_type = address.segments[0] + if address.namespace != "resources" or resource_type not in { + "skills", + "concepts", + "knowledge", + }: + raise ValueError("profile metadata requires a typed shared resource") + document.set_part_text( + DESCRIPTOR_PARTS[resource_type], + json.dumps(profile_metadata, sort_keys=True), + "application/json", + ) + method = "extractive" summary = _fallback_summary(document) if self.summary_provider is not None: @@ -193,14 +207,25 @@ def add_resource( probe = DocLangXDocument() if not probe.read_xml(proposed): raise ValueError(probe.last_error() or "invalid summary") - if not document.set_document_summary(proposed): + validate_doclang(probe, label="model summary") + if not set_summary(document, proposed): raise ValueError(document.last_error() or "invalid summary sidecar") method = "model" except Exception: # noqa: BLE001 - providers may raise arbitrary exceptions method = "extractive-fallback" - if method != "model" and not document.set_document_summary(summary): - raise PackageError(document.last_error() or "cannot set summary") - if not document.set_toc(_toc(document)): + if method != "model": + summary_probe = DocLangXDocument() + if not summary_probe.read_xml(summary): + raise PackageError(summary_probe.last_error() or "invalid summary") + validate_doclang(summary_probe, label="summary") + if not set_summary(document, summary): + raise PackageError(document.last_error() or "cannot set summary") + toc_xml = _toc(document) + toc_probe = DocLangXDocument() + if not toc_probe.read_xml(toc_xml): + raise PackageError(toc_probe.last_error() or "invalid TOC") + validate_doclang(toc_probe, label="TOC") + if not set_outline(document, toc_xml): raise PackageError(document.last_error() or "cannot set TOC") manifest = { "schema_version": 1, diff --git a/src/docling_context/jobs.py b/src/docling_context/jobs.py index 16b3675..43b8206 100644 --- a/src/docling_context/jobs.py +++ b/src/docling_context/jobs.py @@ -10,6 +10,7 @@ from doclang import DocLangXDocument +from .doclang_authoring import outline_xml, set_outline, set_summary, validate_doclang from .local import LocalContextStore, RecordMissing from .models import Principal from .uri import authorize_uri, parse_uri @@ -211,15 +212,29 @@ def process(self, job: Job, now: datetime | None = None) -> None: document = DocLangXDocument() if not document.read_xml(xml): raise ValueError(document.last_error()) - if not document.set_document_summary( - f"{escape(f'{len(children)} documents in {job.collection_uri}')}" - ): + validate_doclang(document, label="collection summary body") + summary_xml = f"{escape(f'{len(children)} documents in {job.collection_uri}')}" + summary_probe = DocLangXDocument() + if not summary_probe.read_xml(summary_xml): + raise ValueError(summary_probe.last_error()) + validate_doclang(summary_probe, label="collection summary sidecar") + if not set_summary(document, summary_xml): raise ValueError(document.last_error()) - toc_entries = "".join( - f'{escape(child.uri.rsplit("/", 1)[-1])}' - for i, child in enumerate(children[:500], 1) + toc_xml = outline_xml( + [ + ( + f"/doclang[1]/heading[{i}]", + child.uri.rsplit("/", 1)[-1], + 1, + ) + for i, child in enumerate(children[:500], 1) + ] ) - if not document.set_toc(f"{toc_entries}"): + toc_probe = DocLangXDocument() + if not toc_probe.read_xml(toc_xml): + raise ValueError(toc_probe.last_error()) + validate_doclang(toc_probe, label="collection TOC") + if not set_outline(document, toc_xml): raise ValueError(document.last_error()) source = { "kind": "collection-summary", diff --git a/src/docling_context/local.py b/src/docling_context/local.py index e556730..3f5671d 100644 --- a/src/docling_context/local.py +++ b/src/docling_context/local.py @@ -9,6 +9,7 @@ from contextlib import contextmanager from datetime import UTC, datetime from pathlib import Path +from threading import RLock from typing import Self from .models import ( @@ -61,7 +62,11 @@ def __init__(self, root: str | Path, *, reconcile_on_start: bool = True): self.db.row_factory = sqlite3.Row self.db.execute("PRAGMA foreign_keys = ON") self.db.execute("PRAGMA journal_mode = WAL") + self._transaction_lock = RLock() self._migrate() + from .catalog import Catalog + + Catalog(self).setup() if reconcile_on_start: self.reconcile() @@ -76,7 +81,7 @@ def __exit__(self, *_: object) -> None: def _migrate(self) -> None: version = self.db.execute("PRAGMA user_version").fetchone()[0] - if version > 8: + if version > 10: raise RuntimeError("database schema is newer than this library") if version == 0: self.db.executescript( @@ -300,22 +305,57 @@ def _migrate(self) -> None: COMMIT; """ ) - self._backfill_retrieval() + if version < 9: + with self.transaction(): + columns = { + row["name"] + for row in self.db.execute("PRAGMA table_info(ingest_jobs)") + } + if "project_id" not in columns: + self.db.execute( + "ALTER TABLE ingest_jobs ADD COLUMN project_id TEXT" + ) + self.db.execute("PRAGMA user_version = 9") + if version < 10: + with self.transaction(): + self.db.execute( + """CREATE TABLE IF NOT EXISTS lexical_state ( + tenant_id TEXT NOT NULL, uri TEXT NOT NULL, + revision_id TEXT NOT NULL, unit_count INTEGER NOT NULL, + PRIMARY KEY (tenant_id, uri), + FOREIGN KEY (tenant_id, uri) + REFERENCES documents (tenant_id, uri) ON DELETE CASCADE)""" + ) + self.db.execute("PRAGMA user_version = 10") - def _backfill_retrieval(self) -> None: + def lexical_stale_uris(self, principal: Principal) -> tuple[str, ...]: + """Find visible documents whose current lexical projection is incomplete.""" rows = self.db.execute( - """SELECT d.* FROM documents d WHERE d.available=1 AND NOT EXISTS ( - SELECT 1 FROM retrieval_units r WHERE r.tenant_id=d.tenant_id - AND r.uri=d.uri AND r.revision_id=d.revision_id)""" + """SELECT d.uri FROM documents d LEFT JOIN lexical_state s + ON s.tenant_id=d.tenant_id AND s.uri=d.uri + WHERE d.tenant_id=? AND d.available=1 AND ( + s.revision_id IS NULL OR s.revision_id!=d.revision_id + OR s.unit_count!=(SELECT COUNT(*) FROM retrieval_units r + WHERE r.tenant_id=d.tenant_id AND r.uri=d.uri + AND r.revision_id=d.revision_id) + OR EXISTS (SELECT 1 FROM retrieval_units r + WHERE r.tenant_id=d.tenant_id AND r.uri=d.uri + AND r.revision_id=d.revision_id + AND NOT EXISTS (SELECT 1 FROM retrieval_fts + WHERE rowid=r.vector_id)))""", + (principal.tenant_id,), ).fetchall() - for row in rows: - try: - package = self.packages.get(row["package_hash"]) - with self.transaction(): - index_package(self.db, self._record(row), package) - self.db.execute("UPDATE vector_state SET outbox_offset=-1") - except (PackageMissing, PackageError): - continue + return tuple( + row["uri"] for row in rows if self._can_access(principal, row["uri"]) + ) + + def lexical_warning(self, principal: Principal) -> str | None: + if self.lexical_stale_uris(principal): + return ( + "Lexical index is incomplete or stale. " + "Run 'dc index lex rebuild' to rebuild it." + ) + return None def index_status(self, principal: Principal) -> dict[str, int | str | None]: """Report the current, authorized retrieval projection.""" @@ -324,6 +364,7 @@ def index_status(self, principal: Principal) -> dict[str, int | str | None]: (principal.tenant_id,), ).fetchall() visible = [row for row in rows if self._can_access(principal, row["uri"])] + stale = set(self.lexical_stale_uris(principal)) indexed = units = embedded = 0 for row in visible: count, with_embeddings = self.db.execute( @@ -331,7 +372,7 @@ def index_status(self, principal: Principal) -> dict[str, int | str | None]: WHERE tenant_id=? AND uri=? AND revision_id=?""", (principal.tenant_id, row["uri"], row["revision_id"]), ).fetchone() - indexed += count > 0 + indexed += row["uri"] not in stale units += count embedded += with_embeddings state = self.db.execute( @@ -340,13 +381,24 @@ def index_status(self, principal: Principal) -> dict[str, int | str | None]: offset = self.db.execute( "SELECT COALESCE(MAX(sequence),0) FROM outbox" ).fetchone()[0] + snapshot_valid = False + if state is not None: + from .vectors import VectorStorageUnavailable, verified_vector_state + + try: + verified_vector_state(self) + snapshot_valid = True + except VectorStorageUnavailable: + pass return { "documents": len(visible), "indexed_documents": indexed, "indexed_nodes": units, "embedded_nodes": embedded, "vector_model": state["model_id"] if state else None, - "vector_snapshot_current": bool(state and state["outbox_offset"] == offset), + "vector_snapshot_current": bool( + snapshot_valid and state and state["outbox_offset"] == offset + ), } def rebuild_index( @@ -402,6 +454,14 @@ def _queue_collection( self, principal: Principal, uri: str, revision_id: str, event: int, now: str ) -> None: address = parse_uri(uri) + if address.namespace == "resources" and address.segments[0] in { + "library", + "memories", + "skills", + "concepts", + "knowledge", + }: + return if address.namespace != "resources" or address.segments[1:] == ( "_context", "summary", @@ -432,14 +492,21 @@ def _queue_collection( @contextmanager def transaction(self) -> Iterator[sqlite3.Connection]: - self.db.execute("BEGIN IMMEDIATE") - try: - yield self.db - self.db.execute("COMMIT") - except BaseException: - if self.db.in_transaction: - self.db.execute("ROLLBACK") - raise + with self._transaction_lock: + nested = self.db.in_transaction + savepoint = f"dc_{uuid.uuid4().hex}" if nested else "" + self.db.execute(f"SAVEPOINT {savepoint}" if nested else "BEGIN IMMEDIATE") + try: + yield self.db + self.db.execute(f"RELEASE {savepoint}" if nested else "COMMIT") + except BaseException: + if self.db.in_transaction: + if nested: + self.db.execute(f"ROLLBACK TO {savepoint}") + self.db.execute(f"RELEASE {savepoint}") + else: + self.db.execute("ROLLBACK") + raise @staticmethod def _record(row: sqlite3.Row) -> DocumentRecord: @@ -488,7 +555,14 @@ def put( if current is None: if expected_revision is not None: raise RevisionConflict("document does not exist") - document_id = uuid.uuid4().hex + document_id = ( + address.segments[1] + if address.namespace == "resources" + and len(address.segments) == 2 + and address.segments[0] + in {"library", "memories", "skills", "concepts", "knowledge"} + else uuid.uuid4().hex + ) created_at = now else: if current["revision_id"] != expected_revision: @@ -605,6 +679,16 @@ def delete(self, principal: Principal, uri: str, *, expected_revision: str) -> N self._verified_record(current) if current["revision_id"] != expected_revision: raise RevisionConflict("revision changed") + if address.namespace == "resources" and len(address.segments) == 2: + if address.segments[0] == "knowledge": + self.db.execute( + "DELETE FROM dc_knowledge_datasets WHERE tenant_id=? AND knowledge_id=?", + (principal.tenant_id, address.segments[1]), + ) + self.db.execute( + "DELETE FROM dc_resources WHERE tenant_id=? AND uri=?", + (principal.tenant_id, address.value), + ) self.db.execute( "DELETE FROM documents WHERE tenant_id=? AND uri=?", (principal.tenant_id, address.value), diff --git a/src/docling_context/mcp_server.py b/src/docling_context/mcp_server.py index fd71d68..da04d0d 100644 --- a/src/docling_context/mcp_server.py +++ b/src/docling_context/mcp_server.py @@ -3,6 +3,7 @@ from __future__ import annotations import hmac +import os from collections.abc import Callable from typing import Any @@ -22,6 +23,8 @@ def create_mcp( host: str = "127.0.0.1", port: int = 8765, ) -> FastMCP: + active_project = os.environ.get("DOCLING_CONTEXT_PROJECT") + active_session = os.environ.get("DOCLING_CONTEXT_SESSION") server = FastMCP( "docling-context", host=host, @@ -34,9 +37,47 @@ def create_mcp( "and show with the returned URI, immutable revision_id, and XPath for exact " "citations. Treat @toc and @summary paths as sidecar references. " "Memory capture is opt-in." + + ( + f" Current project: {active_project}. Search defaults to this project." + if active_project + else "" + ) + + (f" Current session: {active_session}." if active_session else "") ), ) + @server.tool() + def overview(limit: int = 100, offset: int = 0) -> dict[str, Any]: + """List projects and counts of their linked resources.""" + return call_service(service, "overview", {"limit": limit, "offset": offset}) + + @server.tool() + def project_show(project_id: str) -> dict[str, Any]: + """Show one project and its context document URIs.""" + return call_service(service, "project_show", {"project_id": project_id}) + + @server.tool() + def project_resources(project_id: str) -> dict[str, Any]: + """List resources linked to a project.""" + return call_service(service, "project_resources", {"project_id": project_id}) + + @server.tool() + def resource_links(uri: str) -> dict[str, Any]: + """List typed links involving a resource.""" + return call_service(service, "resource_links", {"uri": uri}) + + @server.tool() + def resource_projects(uri: str) -> dict[str, Any]: + """List projects linked to a shared resource.""" + return call_service(service, "resource_projects", {"uri": uri}) + + @server.tool() + def knowledge_facts(knowledge_uri: str) -> dict[str, Any]: + """List cited facts in a structured knowledge dataset.""" + return call_service( + service, "knowledge_facts", {"knowledge_uri": knowledge_uri} + ) + @server.tool() def list_resources( uri: str = "docling://resources", limit: int = 100 @@ -55,7 +96,7 @@ def tree( @server.tool() def outline(uri: str, revision_id: str | None = None) -> dict[str, Any]: - """Read a document's TOC sidecar and node links.""" + """Read a document's validated outline and source node links.""" return call_service( service, "outline", {"uri": uri, "revision_id": revision_id} ) @@ -94,6 +135,7 @@ def search( per_result_tokens: int = 500, include_context: bool = False, image_base64: str | None = None, + project_id: str | None = None, ) -> dict[str, Any]: """Search indexed DocLang nodes with exact URI, revision, and XPath citations.""" return call_service( @@ -111,21 +153,84 @@ def search( "per_result_tokens": per_result_tokens, "include_context": include_context, "image_base64": image_base64, + "project_id": project_id if project_id is not None else active_project, }, ) @server.tool() def job_status(job_id: str) -> dict[str, Any]: - """Inspect a background collection job.""" + """Inspect a durable ingestion job.""" return call_service(service, "job_status", {"job_id": job_id}) if allow_writes: + @server.tool() + def project_create(project_id: str, title: str) -> dict[str, Any]: + """Create a project in the shared SQLite catalog.""" + return call_service( + service, "project_create", {"project_id": project_id, "title": title} + ) + + @server.tool() + def project_link(project_id: str, uri: str) -> dict[str, Any]: + """Link a shared resource to a project.""" + return call_service( + service, "project_link", {"project_id": project_id, "uri": uri} + ) + + @server.tool() + def project_unlink(project_id: str, uri: str) -> dict[str, Any]: + """Remove a shared resource from a project.""" + return call_service( + service, "project_unlink", {"project_id": project_id, "uri": uri} + ) + + @server.tool() + def project_session_start( + project_id: str, session_id: str, agent: str + ) -> dict[str, Any]: + """Start a session owned by a project.""" + return call_service(service, "project_session_start", locals()) + + @server.tool() + def project_session_close(project_id: str, session_id: str) -> dict[str, Any]: + """Close a project session and queue memory compilation.""" + return call_service(service, "project_session_close", locals()) + + @server.tool() + def resource_link( + source_uri: str, + target_uri: str, + relation: str, + source_revision: str | None = None, + source_xpath: str | None = None, + ) -> dict[str, Any]: + """Record a typed relationship between two shared resources.""" + return call_service(service, "resource_link", locals()) + + @server.tool() + def knowledge_add_fact( + knowledge_uri: str, + entity_type: str, + entity_id: str, + property: str, + value: Any, + source_uri: str, + source_revision: str, + source_xpath: str, + ) -> dict[str, Any]: + """Store a typed fact with an exact source citation.""" + return call_service(service, "knowledge_add_fact", locals()) + @server.tool() def ingest( - uri: str, filename: str, data_base64: str, force: bool = False + filename: str, + data_base64: str, + uri: str = "", + project_id: str | None = None, + force: bool = False, ) -> dict[str, Any]: - """Import bounded base64 document bytes into an authorized resource URI.""" + """Queue bounded source bytes for the shared library, optionally linked to a project.""" return call_service( service, "ingest", @@ -134,6 +239,7 @@ def ingest( "filename": filename, "data_base64": data_base64, "force": force, + "project_id": project_id, }, ) @@ -226,6 +332,13 @@ async def operation(request: Request) -> JSONResponse: try: if not allow_writes and request.path_params["name"] in { "ingest", + "project_create", + "project_link", + "project_unlink", + "project_session_start", + "project_session_close", + "resource_link", + "knowledge_add_fact", "session_append", "memory_list", "memory_review", diff --git a/src/docling_context/profiles.py b/src/docling_context/profiles.py index 3fb6c25..6846029 100644 --- a/src/docling_context/profiles.py +++ b/src/docling_context/profiles.py @@ -15,6 +15,12 @@ SESSION_PART = "context/session.json" MEMORY_PART = "context/memory.json" +DESCRIPTOR_PARTS = { + "skills": "context/skill.json", + "concepts": "context/concept.json", + "knowledge": "context/knowledge.json", +} +DESCRIPTOR_NAMES = {"skills": "skill", "concepts": "concept", "knowledge": "knowledge"} SESSION_KINDS = frozenset({"turn", "tool_call", "tool_result", "feedback"}) SESSION_STATUSES = frozenset({"open", "closed"}) MEMORY_KINDS = frozenset({"preference", "fact", "entity", "concept", "procedure"}) @@ -29,6 +35,14 @@ def profile_kind(uri: str) -> str | None: address = parse_uri(uri) + if ( + address.namespace == "projects" + and len(address.segments) == 3 + and address.segments[1] == "sessions" + ): + return "sessions" + if address.namespace == "resources" and address.segments[0] == "memories": + return "memories" if address.namespace != "users": return None if len(address.segments) == 4 and address.segments[2] == "sessions": @@ -181,6 +195,86 @@ def validate_memory(document: DocLangXDocument) -> dict[str, Any]: return metadata +def validate_concept_schema(schema: Any) -> dict[str, Any]: + if not isinstance(schema, dict): + raise TypeError("concept requires a schema object") + entities = schema.get("entity_types") + properties = schema.get("properties") + if ( + not isinstance(entities, list) + or not entities + or any(not isinstance(item, str) or not item.strip() for item in entities) + or len(set(entities)) != len(entities) + ): + raise ValueError("concept schema requires distinct entity_types") + if not isinstance(properties, dict) or not properties: + raise ValueError("concept schema requires typed properties") + for name, definition in properties.items(): + if ( + not isinstance(name, str) + or not name.strip() + or not isinstance(definition, dict) + or definition.get("type") + not in {"text", "integer", "number", "boolean", "json"} + or ( + "unit" in definition + and ( + not isinstance(definition["unit"], str) + or not definition["unit"].strip() + ) + ) + ): + raise ValueError("invalid concept property definition") + relationships = schema.get("relationship_types", {}) + if not isinstance(relationships, dict): + raise TypeError("invalid concept relationship_types") + for name, definition in relationships.items(): + if ( + not isinstance(name, str) + or not name.strip() + or not isinstance(definition, dict) + or definition.get("source") not in entities + or definition.get("target") not in entities + ): + raise ValueError("invalid concept relationship definition") + return schema + + +def validate_descriptor(uri: str, document: DocLangXDocument) -> dict[str, Any]: + address = parse_uri(uri) + if address.namespace != "resources" or address.segments[0] not in DESCRIPTOR_PARTS: + raise ValueError("expected a skill, concept, or knowledge URI") + resource_type, resource_id = address.segments + metadata = _part(document, DESCRIPTOR_PARTS[resource_type]) + singular = DESCRIPTOR_NAMES[resource_type] + if ( + metadata.get("profile") != f"{singular}-v1" + or metadata.get(f"{singular}_id") != resource_id + or not isinstance(metadata.get("name"), str) + or not 1 <= len(metadata["name"].strip()) <= 200 + or not isinstance(metadata.get("summary"), str) + or len(metadata["summary"]) > 2_000 + ): + raise ValueError(f"invalid {singular} metadata") + if resource_type == "concepts": + validate_concept_schema(metadata.get("schema")) + if resource_type == "knowledge": + if metadata.get("concept_uri") and ( + not metadata.get("concept_revision") + or parse_uri(metadata["concept_uri"]).segments[0] != "concepts" + ): + raise ValueError("knowledge concept_uri must reference a concept") + fields = metadata.get("fields") + if not isinstance(fields, dict) or any( + not isinstance(name, str) + or not name.strip() + or kind not in {"text", "integer", "number", "boolean", "json"} + for name, kind in fields.items() + ): + raise ValueError("invalid knowledge fields") + return metadata + + def validate_profile(uri: str, document: DocLangXDocument) -> dict[str, Any] | None: kind = profile_kind(uri) address = parse_uri(uri) @@ -188,10 +282,16 @@ def validate_profile(uri: str, document: DocLangXDocument) -> dict[str, Any] | N return validate_session(document) if kind == "memories": metadata = validate_memory(document) - if ( - metadata["kind"] != address.segments[3] - or metadata["memory_id"] != address.segments[4] + expected_id = ( + address.segments[1] + if address.namespace == "resources" + else address.segments[4] + ) + if metadata["memory_id"] != expected_id or ( + address.namespace == "users" and metadata["kind"] != address.segments[3] ): raise ValueError("memory package metadata differs from URI") return metadata + if address.namespace == "resources" and address.segments[0] in DESCRIPTOR_PARTS: + return validate_descriptor(uri, document) return None diff --git a/src/docling_context/project_context.py b/src/docling_context/project_context.py new file mode 100644 index 0000000..3a2f40c --- /dev/null +++ b/src/docling_context/project_context.py @@ -0,0 +1,145 @@ +"""Durable, deterministic project abstract and overview generation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from xml.sax.saxutils import escape + +from .catalog import Catalog +from .ingestion import Ingestor +from .local import LocalContextStore +from .models import Principal +from .package import bounded_nodes + + +@dataclass(frozen=True, slots=True) +class ProjectContextJob: + project_id: str + kind: str + status: str + generation: int + attempts: int + last_error: str | None + + +class ProjectContextWorker: + def __init__(self, store: LocalContextStore, *, max_attempts: int = 3): + if max_attempts < 1: + raise ValueError("max_attempts must be positive") + self.store = store + self.catalog = Catalog(store) + self.max_attempts = max_attempts + + def _xml(self, principal: Principal, project_id: str, kind: str) -> bytes: + project = self.catalog.get_project(principal, project_id) + description = "" + if project.description_uri: + document = self.store.get_document(principal, project.description_uri) + description = " ".join( + str(node["text"]) + for node in bounded_nodes(document, limit=100, text_bytes=256) + if node["text"] + )[:1000] + resources = self.catalog.project_resources(principal, project_id) + counts: dict[str, int] = {} + for resource in resources: + counts[resource["resource_type"]] = ( + counts.get(resource["resource_type"], 0) + 1 + ) + count_text = ", ".join( + f"{counts.get(name, 0)} {name}" + for name in ("library", "memories", "skills", "concepts", "knowledge") + ) + if kind == "abstract": + body = f"{project.title}. {description} Linked resources: {count_text}." + return f"{escape(body)}".encode() + parts = [ + f"{escape(project.title)}", + f"{escape(description or 'Project overview')}", + f"Linked resources: {escape(count_text)}.", + ] + for resource in resources[:256]: + label = f"{resource['resource_type']}/{resource['resource_id']}" + parts.append(f"{escape(label)}") + parts.append(f"{escape(resource['uri'])}") + return f"{''.join(parts)}".encode() + + def run_once(self) -> ProjectContextJob | None: + with self.store.transaction(): + row = self.store.db.execute( + """SELECT j.tenant_id,j.project_id,j.kind,j.generation,j.attempts, + p.owner_id + FROM dc_project_context_jobs j JOIN dc_projects p + USING (tenant_id,project_id) + WHERE j.status='queued' + ORDER BY j.updated_at,j.project_id,j.kind LIMIT 1""" + ).fetchone() + if row is None: + return None + self.store.db.execute( + """UPDATE dc_project_context_jobs SET status='running', + attempts=attempts+1,updated_at=CURRENT_TIMESTAMP + WHERE tenant_id=? AND project_id=? AND kind=?""", + (row["tenant_id"], row["project_id"], row["kind"]), + ) + principal = Principal(row["tenant_id"], row["owner_id"]) + try: + uri = f"docling://projects/{row['project_id']}/.{row['kind']}.dclx" + record = Ingestor(self.store).add_resource( + principal, + uri, + self._xml(principal, row["project_id"], row["kind"]), + filename="project-context.dclg", + ) + self.catalog.set_project_document( + principal, row["project_id"], row["kind"], record, generated=True + ) + except Exception as exc: # noqa: BLE001 - durable worker records failures + attempts = row["attempts"] + 1 + status = "failed" if attempts >= self.max_attempts else "queued" + with self.store.transaction(): + self.store.db.execute( + """UPDATE dc_project_context_jobs + SET status=?,last_error=?,updated_at=CURRENT_TIMESTAMP + WHERE tenant_id=? AND project_id=? AND kind=? + AND generation=?""", + ( + status, + str(exc)[:500], + row["tenant_id"], + row["project_id"], + row["kind"], + row["generation"], + ), + ) + else: + with self.store.transaction(): + self.store.db.execute( + """UPDATE dc_project_context_jobs + SET status='completed',last_error=NULL, + updated_at=CURRENT_TIMESTAMP + WHERE tenant_id=? AND project_id=? AND kind=? + AND generation=? AND status='running'""", + ( + row["tenant_id"], + row["project_id"], + row["kind"], + row["generation"], + ), + ) + final = self.store.db.execute( + """SELECT status,generation,attempts,last_error + FROM dc_project_context_jobs + WHERE tenant_id=? AND project_id=? AND kind=?""", + (row["tenant_id"], row["project_id"], row["kind"]), + ).fetchone() + if final is None: + return None + return ProjectContextJob( + row["project_id"], + row["kind"], + final["status"], + final["generation"], + final["attempts"], + final["last_error"], + ) diff --git a/src/docling_context/remote.py b/src/docling_context/remote.py index 9156c37..40db958 100644 --- a/src/docling_context/remote.py +++ b/src/docling_context/remote.py @@ -12,11 +12,19 @@ SCHEMA_VERSION, IngestRequest, JobStatusRequest, + KnowledgeFactRequest, + KnowledgeFactsRequest, ListRequest, MemoryListRequest, MemoryRecallRequest, MemoryReviewRequest, OutlineRequest, + OverviewRequest, + ProjectLinkRequest, + ProjectRequest, + ProjectSessionRequest, + ResourceLinkRequest, + ResourceLinksRequest, SearchRequest, ServiceError, ServiceResult, @@ -81,6 +89,39 @@ def _call(self, operation: str, request: Any) -> ServiceResult[Any]: def list(self, request: ListRequest) -> ServiceResult[Any]: return self._call("list", request) + def overview(self, request: OverviewRequest) -> ServiceResult[Any]: + return self._call("overview", request) + + def project_create(self, request: ProjectRequest) -> ServiceResult[Any]: + return self._call("project_create", request) + + def project_show(self, request: ProjectRequest) -> ServiceResult[Any]: + return self._call("project_show", request) + + def project_resources(self, request: ProjectRequest) -> ServiceResult[Any]: + return self._call("project_resources", request) + + def project_link(self, request: ProjectLinkRequest) -> ServiceResult[Any]: + return self._call("project_link", request) + + def project_unlink(self, request: ProjectLinkRequest) -> ServiceResult[Any]: + return self._call("project_unlink", request) + + def resource_links(self, request: ResourceLinksRequest) -> ServiceResult[Any]: + return self._call("resource_links", request) + + def resource_projects(self, request: ResourceLinksRequest) -> ServiceResult[Any]: + return self._call("resource_projects", request) + + def resource_link(self, request: ResourceLinkRequest) -> ServiceResult[Any]: + return self._call("resource_link", request) + + def knowledge_facts(self, request: KnowledgeFactsRequest) -> ServiceResult[Any]: + return self._call("knowledge_facts", request) + + def knowledge_add_fact(self, request: KnowledgeFactRequest) -> ServiceResult[Any]: + return self._call("knowledge_add_fact", request) + def tree(self, request: TreeRequest) -> ServiceResult[Any]: return self._call("tree", request) @@ -102,6 +143,16 @@ def job_status(self, request: JobStatusRequest) -> ServiceResult[Any]: def session_append(self, request: SessionAppendRequest) -> ServiceResult[Any]: return self._call("session_append", request) + def project_session_start( + self, request: ProjectSessionRequest + ) -> ServiceResult[Any]: + return self._call("project_session_start", request) + + def project_session_close( + self, request: ProjectSessionRequest + ) -> ServiceResult[Any]: + return self._call("project_session_close", request) + def memory_list(self, request: MemoryListRequest) -> ServiceResult[Any]: return self._call("memory_list", request) diff --git a/src/docling_context/retrieval.py b/src/docling_context/retrieval.py index b35561d..01a9d9c 100644 --- a/src/docling_context/retrieval.py +++ b/src/docling_context/retrieval.py @@ -12,6 +12,7 @@ from doclang import DocLangXDocument +from .doclang_authoring import get_outline, get_summary from .models import DocumentRecord, Principal from .package import MAX_NODE_TEXT_CHARS, bounded_nodes, load_package from .profiles import MEMORY_PART, profile_kind @@ -35,7 +36,11 @@ "description", } _MAX_RESULTS = 100 -_MAX_CANDIDATES = 10_000 +_RETRIEVAL_COLUMNS = ( + "r.vector_id,r.tenant_id,r.uri,r.document_id,r.revision_id,r.xpath," + "r.parent_xpath,r.section_xpath,r.tier,r.page,r.bbox_json,r.text," + "r.updated_at,r.asset_path" +) def remove_index_records(db: sqlite3.Connection, tenant_id: str, uri: str) -> None: @@ -62,6 +67,12 @@ def index_package( if profile_kind(record.uri) == "memories": metadata = json.loads(document.get_part_text(MEMORY_PART) or "{}") if metadata.get("status") != "accepted": + db.execute( + """INSERT INTO lexical_state (tenant_id,uri,revision_id,unit_count) + VALUES (?,?,?,0) ON CONFLICT(tenant_id,uri) + DO UPDATE SET revision_id=excluded.revision_id,unit_count=0""", + (record.tenant_id, record.uri, record.revision_id), + ) return units: list[ tuple[str, str | None, str | None, int, int | None, str | None, str, str | None] @@ -82,7 +93,7 @@ def index_package( else None ) picture_index = 0 - summary = document.summary() + summary = get_summary(document) if summary is not None: for node in bounded_nodes(summary, limit=100, text_bytes=4_096): if node["name"] in {"text", "paragraph"} and node["text"].strip(): @@ -99,10 +110,10 @@ def index_package( ) ) break - toc = document.toc() + toc = get_outline(document) if toc is not None: for node in bounded_nodes(toc, limit=512, text_bytes=1_024): - if node["name"] == "description" and node["text"].strip(): + if node["name"] in {"heading", "description"} and node["text"].strip(): units.append( ( f"@toc:{node['xpath']}", @@ -177,6 +188,13 @@ def index_package( "INSERT INTO retrieval_fts(rowid,text) VALUES (?,?)", (cursor.lastrowid, content), ) + db.execute( + """INSERT INTO lexical_state (tenant_id,uri,revision_id,unit_count) + VALUES (?,?,?,?) ON CONFLICT(tenant_id,uri) + DO UPDATE SET revision_id=excluded.revision_id, + unit_count=excluded.unit_count""", + (record.tenant_id, record.uri, record.revision_id, len(units)), + ) @dataclass(frozen=True, slots=True) @@ -185,6 +203,7 @@ class SearchScope: xpath: str | None = None document_uri: str | None = None tiers: tuple[int, ...] = (0, 1, 2) + project_id: str | None = None @dataclass(frozen=True, slots=True) @@ -215,6 +234,7 @@ class ContextResult: tokens_used: int truncated: bool total_hits: int | None = None + warnings: tuple[str, ...] = () def _query_terms(query: str, operator: str = "OR") -> str: @@ -248,7 +268,7 @@ def __init__( if embedder is not None and vector_index is None: from .vectors import TurbovecIndex - vector_index = TurbovecIndex(store, embedder) + vector_index = TurbovecIndex(store, embedder, search_only=True) self.vector_index = vector_index def _scope_conditions( @@ -276,6 +296,16 @@ def _scope_conditions( ] prefix = base + "/" params: list[object] = [principal.tenant_id, base, len(prefix), prefix] + if scope.project_id is not None: + from .catalog import Catalog + + with Catalog(self.store) as catalog: + members = catalog.project_resource_uris(principal, scope.project_id) + if members: + clauses.append(f"r.uri IN ({','.join('?' for _ in members)})") + params.extend(members) + else: + clauses.append("0=1") if scope.document_uri is not None: clauses.append("r.uri=?") params.append(doc.value) @@ -299,12 +329,14 @@ def _allowed( else: source = "retrieval_units r" rows = self.store.db.execute( - "SELECT r.* FROM " + "SELECT " + + _RETRIEVAL_COLUMNS + + " FROM " + source + " JOIN documents d ON d.tenant_id=r.tenant_id AND d.uri=r.uri WHERE " + where - + " ORDER BY r.vector_id LIMIT ?", - (*params, _MAX_CANDIDATES), + + " ORDER BY r.vector_id", + params, ).fetchall() return [ row @@ -313,19 +345,43 @@ def _allowed( or _user_allowed(principal, row["uri"]) ] - def _lexical_count( - self, principal: Principal, scope: SearchScope, terms: str - ) -> int: + def _allowed_ids(self, principal: Principal, scope: SearchScope) -> set[int]: + """Collect every scoped ID without loading text or embeddings.""" where, params = self._scope_conditions(principal, scope) - return self.store.db.execute( - """SELECT COUNT(*) FROM retrieval_units r - JOIN documents d ON d.tenant_id=r.tenant_id AND d.uri=r.uri - WHERE r.vector_id IN ( - SELECT rowid FROM retrieval_fts WHERE retrieval_fts MATCH ? - ) AND """ + cursor = self.store.db.execute( + """SELECT r.vector_id,r.uri FROM retrieval_units r + JOIN documents d ON d.tenant_id=r.tenant_id AND d.uri=r.uri WHERE """ + where, - (terms, *params), - ).fetchone()[0] + params, + ) + allowed: set[int] = set() + while batch := cursor.fetchmany(1_000): + allowed.update( + row["vector_id"] + for row in batch + if parse_uri(row["uri"]).namespace != "users" + or _user_allowed(principal, row["uri"]) + ) + return allowed + + def _rows_for_ids(self, identifiers: list[int]) -> list[sqlite3.Row]: + """Load only vector finalists after the index has searched its allowlist.""" + rows: list[sqlite3.Row] = [] + for start in range(0, len(identifiers), 500): + batch = identifiers[start : start + 500] + marks = ",".join("?" for _ in batch) + rows.extend( + self.store.db.execute( + "SELECT " + + _RETRIEVAL_COLUMNS + + " FROM retrieval_units r JOIN documents d" + " ON d.tenant_id=r.tenant_id AND d.uri=r.uri" + " WHERE d.available=1 AND d.revision_id=r.revision_id" + f" AND r.vector_id IN ({marks})", + batch, + ).fetchall() + ) + return rows def search( self, @@ -350,10 +406,20 @@ def search( raise ValueError("vector search needs an embedding provider and index") if not query.strip() and (image is None or mode != "vector"): raise ValueError("search needs query text or a vector-mode image") + warnings: list[str] = [] + lexical_warning = self.store.lexical_warning(principal) + if lexical_warning: + warnings.append(lexical_warning) if self.vector_index is not None: refresh = getattr(self.vector_index, "refresh", None) if callable(refresh): refresh() + if mode in {"vector", "hybrid"} and self.vector_index is not None: + warning = getattr(self.vector_index, "warning", None) + if callable(warning): + message = warning(principal.tenant_id) + if message: + warnings.append(message) terms = _query_terms(query) if query.strip() else None effective_scope = scope or SearchScope() lexical_only = mode == "lexical" or ( @@ -367,40 +433,21 @@ def search( rows = focused else: rows = self._allowed(principal, effective_scope, terms=terms) + elif mode != "vector": + rows = self._allowed(principal, effective_scope, terms=terms) else: - rows = self._allowed( - principal, effective_scope, terms=terms if lexical_only else None - ) - total_hits = ( - self._lexical_count(principal, effective_scope, terms) - if lexical_only and terms is not None - else None - ) + rows = [] + total_hits = len(rows) if lexical_only and terms is not None else None by_id = {row["vector_id"]: row for row in rows} - if not by_id: - return ContextResult((), 0, False, total_hits) + if not by_id and (mode == "lexical" or self.vector_index is None): + return ContextResult((), 0, False, total_hits, tuple(warnings)) scores: dict[int, list[float]] = {} if mode in {"lexical", "hybrid"} and terms is not None: query_words = re.findall(r"\w+", query.casefold()) - if lexical_only: - for identifier, row in by_id.items(): - content = row["text"].casefold() - frequency = sum(content.count(term) for term in query_words) - scores[identifier] = [min(1.0, frequency / 4), 0.0] - else: - identifiers = list(by_id) - for start in range(0, len(identifiers), 500): - batch = identifiers[start : start + 500] - marks = ",".join("?" for _ in batch) - fts_rows = self.store.db.execute( - "SELECT rowid FROM retrieval_fts WHERE retrieval_fts MATCH ? " - f"AND rowid IN ({marks})", - (terms, *batch), - ).fetchall() - for row in fts_rows: - content = by_id[row["rowid"]]["text"].casefold() - frequency = sum(content.count(term) for term in query_words) - scores[row["rowid"]] = [min(1.0, frequency / 4), 0.0] + for identifier, row in by_id.items(): + content = row["text"].casefold() + frequency = sum(content.count(term) for term in query_words) + scores[identifier] = [min(1.0, frequency / 4), 0.0] if mode in {"vector", "hybrid"} and self.vector_index is not None: assert self.embedder is not None if image is not None: @@ -418,11 +465,19 @@ def search( if callable(query_embed) else self.embedder.embed([query])[0] ) + allowed_ids = self._allowed_ids(principal, effective_scope) found = ( - self.vector_index.search(query_vector, set(by_id), min(k * 4, 100)) + self.vector_index.search(query_vector, allowed_ids, min(k * 4, 100)) if query_vector is not None else [] ) + missing_ids = [ + vector_id + for vector_id, _ in found + if vector_id in allowed_ids and vector_id not in by_id + ] + for row in self._rows_for_ids(missing_ids): + by_id[row["vector_id"]] = row for vector_id, score in found: if vector_id in by_id: scores.setdefault(vector_id, [0.0, 0.0])[1] = score @@ -457,7 +512,6 @@ def components(identifier: int) -> tuple[float, float, float]: ranked = sorted( scores, key=lambda identifier: ( - by_id[identifier]["tier"], -( scores[identifier][0] + max(0.0, scores[identifier][1]) @@ -542,7 +596,7 @@ def components(identifier: int) -> tuple[float, float, float]: ) ) truncated |= shortened or source_truncated - return ContextResult(tuple(hits), used, truncated, total_hits) + return ContextResult(tuple(hits), used, truncated, total_hits, tuple(warnings)) def _exact_text( self, @@ -558,7 +612,9 @@ def _exact_text( if row["xpath"].startswith(("@summary:", "@toc:")): sidecar_name, xpath = row["xpath"].split(":", 1) sidecar = ( - document.summary() if sidecar_name == "@summary" else document.toc() + get_summary(document) + if sidecar_name == "@summary" + else get_outline(document) ) if sidecar is None: raise KeyError(row["xpath"]) diff --git a/src/docling_context/service.py b/src/docling_context/service.py index 53c2400..39fbf3e 100644 --- a/src/docling_context/service.py +++ b/src/docling_context/service.py @@ -7,6 +7,7 @@ import builtins import logging import re +import sqlite3 import time import uuid import xml.etree.ElementTree as ET @@ -14,9 +15,10 @@ from threading import RLock from typing import Any +from .catalog import Catalog +from .doclang_authoring import get_outline, get_summary, outline_links from .durable_memory import MemoryService from .ingest_queue import IngestQueue -from .jobs import CollectionWorker from .local import LocalContextStore, RecordMissing from .models import NodeAddress, Principal from .package import MAX_NODE_TEXT_CHARS, PackageError, PackageMissing @@ -54,6 +56,55 @@ class ListRequest: limit: int = 100 +@dataclass(frozen=True, slots=True) +class OverviewRequest: + limit: int = 100 + offset: int = 0 + + +@dataclass(frozen=True, slots=True) +class ProjectRequest: + project_id: str + title: str | None = None + + +@dataclass(frozen=True, slots=True) +class ProjectLinkRequest: + project_id: str + uri: str + + +@dataclass(frozen=True, slots=True) +class ResourceLinksRequest: + uri: str + + +@dataclass(frozen=True, slots=True) +class ResourceLinkRequest: + source_uri: str + target_uri: str + relation: str + source_revision: str | None = None + source_xpath: str | None = None + + +@dataclass(frozen=True, slots=True) +class KnowledgeFactsRequest: + knowledge_uri: str + + +@dataclass(frozen=True, slots=True) +class KnowledgeFactRequest: + knowledge_uri: str + entity_type: str + entity_id: str + property: str + value: Any + source_uri: str + source_revision: str + source_xpath: str + + @dataclass(frozen=True, slots=True) class TreeRequest: uri: str = "docling://resources" @@ -90,6 +141,7 @@ class SearchRequest: per_result_tokens: int = 500 include_context: bool = False image_base64: str | None = None + project_id: str | None = None @dataclass(frozen=True, slots=True) @@ -98,6 +150,7 @@ class IngestRequest: filename: str data_base64: str force: bool = False + project_id: str | None = None @dataclass(frozen=True, slots=True) @@ -114,6 +167,13 @@ class SessionAppendRequest: turn_id: str = "" +@dataclass(frozen=True, slots=True) +class ProjectSessionRequest: + project_id: str + session_id: str + agent: str | None = None + + @dataclass(frozen=True, slots=True) class MemoryListRequest: status: str | None = None @@ -171,7 +231,11 @@ def _validate_xpath(xpath: str) -> None: class ContextService: """Bind an authenticated principal before accepting operation requests.""" - def __init__(self, store: LocalContextStore, principal: Principal): + def __init__( + self, + store: LocalContextStore, + principal: Principal, + ): self.store = store self.principal = principal self._lock = RLock() @@ -205,6 +269,107 @@ def list( ] ) + def overview( + self, request: OverviewRequest + ) -> ServiceResult[builtins.list[dict[str, Any]]]: + with Catalog(self.store) as catalog: + return self._result( + catalog.overview( + self.principal, limit=request.limit, offset=request.offset + ) + ) + + def project_create(self, request: ProjectRequest) -> ServiceResult[dict[str, Any]]: + if request.title is None: + raise ServiceError("invalid_request", "project title is required") + return self._result( + asdict( + Catalog(self.store).create_project( + self.principal, request.project_id, request.title + ) + ) + ) + + def project_show(self, request: ProjectRequest) -> ServiceResult[dict[str, Any]]: + return self._result( + Catalog(self.store).project_summary(self.principal, request.project_id) + ) + + def project_resources( + self, request: ProjectRequest + ) -> ServiceResult[builtins.list[dict[str, str]]]: + return self._result( + Catalog(self.store).project_resources(self.principal, request.project_id) + ) + + def project_link( + self, request: ProjectLinkRequest + ) -> ServiceResult[dict[str, str]]: + Catalog(self.store).link(self.principal, request.project_id, request.uri) + return self._result({"project_id": request.project_id, "uri": request.uri}) + + def project_unlink( + self, request: ProjectLinkRequest + ) -> ServiceResult[dict[str, str]]: + Catalog(self.store).unlink(self.principal, request.project_id, request.uri) + return self._result({"project_id": request.project_id, "uri": request.uri}) + + def resource_links( + self, request: ResourceLinksRequest + ) -> ServiceResult[builtins.list[dict[str, str | None]]]: + return self._result( + Catalog(self.store).resource_links(self.principal, request.uri) + ) + + def resource_projects( + self, request: ResourceLinksRequest + ) -> ServiceResult[builtins.list[dict[str, str]]]: + return self._result( + Catalog(self.store).resource_projects(self.principal, request.uri) + ) + + def resource_link( + self, request: ResourceLinkRequest + ) -> ServiceResult[dict[str, str]]: + Catalog(self.store).link_resources( + self.principal, + request.source_uri, + request.target_uri, + request.relation, + source_revision=request.source_revision, + source_xpath=request.source_xpath, + ) + return self._result( + { + "source_uri": request.source_uri, + "target_uri": request.target_uri, + "relation": request.relation, + } + ) + + def knowledge_facts( + self, request: KnowledgeFactsRequest + ) -> ServiceResult[builtins.list[dict[str, Any]]]: + return self._result( + Catalog(self.store).knowledge_facts(self.principal, request.knowledge_uri) + ) + + def knowledge_add_fact( + self, request: KnowledgeFactRequest + ) -> ServiceResult[dict[str, str]]: + fact_id = Catalog(self.store).add_fact( + self.principal, + request.knowledge_uri, + entity_type=request.entity_type, + entity_id=request.entity_id, + property=request.property, + value=request.value, + source_uri=request.source_uri, + source_revision=request.source_revision, + source_xpath=request.source_xpath, + ) + return self._result({"fact_id": fact_id}) + def tree( self, request: TreeRequest ) -> ServiceResult[builtins.list[dict[str, Any]]]: @@ -226,13 +391,10 @@ def outline(self, request: OutlineRequest) -> ServiceResult[dict[str, Any]]: from .package import load_package document = load_package(self.store.packages.get(record.package.sha256)) - toc = document.toc() + toc = get_outline(document) links = [] if toc is not None: - for entry in ET.fromstring(toc.xml()).iter("entry"): - target = entry.get("xpath") - if target: - links.append(target) + links.extend(outline_links(toc.xml())) xml = toc.xml() if toc is not None else "" return self._result( { @@ -264,7 +426,9 @@ def show(self, request: ShowRequest) -> ServiceResult[dict[str, Any]]: from .package import load_package document = load_package(self.store.packages.get(record.package.sha256)) - part = document.summary() if sidecar == "summary" else document.toc() + part = ( + get_summary(document) if sidecar == "summary" else get_outline(document) + ) if part is None: raise ServiceError("stale_citation", "sidecar is unavailable") selected_xml = _xml_node(part.xml(), xpath) @@ -321,22 +485,23 @@ def search(self, request: SearchRequest) -> ServiceResult[dict[str, Any]]: retriever = Retriever(self.store) if request.mode in {"vector", "hybrid"}: from .embeddings import stored_embedding_provider + from .vectors import VectorStorageUnavailable, verified_vector_state - state = self.store.db.execute( - "SELECT model_id FROM vector_state WHERE singleton=1" - ).fetchone() - if state is None: - raise ServiceError( - "unsupported_provider", - "vector search needs a configured embedding provider", - ) + try: + state = verified_vector_state(self.store) + except VectorStorageUnavailable as exc: + raise ServiceError("vector_index_unavailable", str(exc)) from exc self.store.require_exclusive_vector_scope(self.principal) retriever = Retriever( self.store, embedder=stored_embedding_provider(state["model_id"]) ) started = time.monotonic() scope = SearchScope( - request.uri, request.xpath, request.document_uri, request.tiers + request.uri, + request.xpath, + request.document_uri, + request.tiers, + request.project_id, ) result = retriever.search( self.principal, @@ -361,6 +526,7 @@ def search(self, request: SearchRequest) -> ServiceResult[dict[str, Any]]: "total_hits": result.total_hits, "tokens_used": result.tokens_used, "truncated": result.truncated, + "warnings": list(result.warnings), "elapsed_ms": round((time.monotonic() - started) * 1_000), } ) @@ -376,6 +542,7 @@ def ingest(self, request: IngestRequest) -> ServiceResult[dict[str, Any]]: data, request.filename, force=request.force, + project_id=request.project_id, ) return self._result(asdict(job)) @@ -387,11 +554,9 @@ def job_status(self, request: JobStatusRequest) -> ServiceResult[dict[str, Any]] data["error_code"] = "conversion_failure" return self._result(data) except KeyError: - return self._result( - asdict( - CollectionWorker(self.store).get_job(self.principal, request.job_id) - ) - ) + raise ServiceError( + "missing_job", f"ingestion job not found: {request.job_id}" + ) from None def session_append( self, request: SessionAppendRequest @@ -409,6 +574,30 @@ def session_append( ) ) + def project_session_start( + self, request: ProjectSessionRequest + ) -> ServiceResult[dict[str, Any]]: + if not request.agent: + raise ValueError("agent is required for a project session") + catalog = Catalog(self.store) + catalog.get_project(self.principal, request.project_id) + uri = f"docling://projects/{request.project_id}/sessions/{request.session_id}" + session = SessionStore(self.store).start(self.principal, uri) + catalog.register_session( + self.principal, + request.project_id, + request.session_id, + agent=request.agent, + ) + return self._result(asdict(session)) + + def project_session_close( + self, request: ProjectSessionRequest + ) -> ServiceResult[dict[str, Any]]: + Catalog(self.store).get_project(self.principal, request.project_id) + uri = f"docling://projects/{request.project_id}/sessions/{request.session_id}" + return self._result(asdict(SessionStore(self.store).close(self.principal, uri))) + def memory_list( self, request: MemoryListRequest ) -> ServiceResult[builtins.list[dict[str, Any]]]: @@ -494,6 +683,17 @@ def _dispatch( """Dispatch one versioned request and return a stable error envelope.""" requests = { "list": ListRequest, + "overview": OverviewRequest, + "project_create": ProjectRequest, + "project_show": ProjectRequest, + "project_resources": ProjectRequest, + "project_link": ProjectLinkRequest, + "project_unlink": ProjectLinkRequest, + "resource_links": ResourceLinksRequest, + "resource_projects": ResourceLinksRequest, + "resource_link": ResourceLinkRequest, + "knowledge_facts": KnowledgeFactsRequest, + "knowledge_add_fact": KnowledgeFactRequest, "tree": TreeRequest, "outline": OutlineRequest, "show": ShowRequest, @@ -501,6 +701,8 @@ def _dispatch( "ingest": IngestRequest, "job_status": JobStatusRequest, "session_append": SessionAppendRequest, + "project_session_start": ProjectSessionRequest, + "project_session_close": ProjectSessionRequest, "memory_list": MemoryListRequest, "memory_review": MemoryReviewRequest, "memory_recall": MemoryRecallRequest, @@ -530,6 +732,8 @@ def _dispatch( error = {"code": "stale_citation", "message": str(exc)} except (TypeError, ValueError) as exc: error = {"code": "invalid_request", "message": str(exc)} + except sqlite3.IntegrityError as exc: + error = {"code": "conflict", "message": str(exc)} return { "schema_version": SCHEMA_VERSION, "operation_id": uuid.uuid4().hex, diff --git a/src/docling_context/sessions.py b/src/docling_context/sessions.py index b9cf35c..af5b8f5 100644 --- a/src/docling_context/sessions.py +++ b/src/docling_context/sessions.py @@ -65,11 +65,17 @@ def uri(principal: Principal, session_id: str) -> str: ) address = parse_uri(value) authorize_uri(principal, address) - if ( - address.namespace != "users" - or len(address.segments) != 4 - or address.segments[2] != "sessions" - ): + is_user_session = ( + address.namespace == "users" + and len(address.segments) == 4 + and address.segments[2] == "sessions" + ) + is_project_session = ( + address.namespace == "projects" + and len(address.segments) == 3 + and address.segments[1] == "sessions" + ) + if not (is_user_session or is_project_session): raise ValueError("expected one session URI") return address.value @@ -163,10 +169,25 @@ def start(self, principal: Principal, session_id: str | None = None) -> SessionI except RecordMissing: pass now = datetime.now(UTC).isoformat() - record = self.store.put(principal, uri, self._package((), {}, created_at=now)) + address = parse_uri(uri) + if address.namespace == "projects": + from .catalog import Catalog + + Catalog(self.store).get_project(principal, address.segments[0]) + with self.store.transaction(): + record = self.store.put( + principal, uri, self._package((), {}, created_at=now) + ) + if address.namespace == "projects": + Catalog(self.store).register_session( + principal, + address.segments[0], + address.segments[2], + revision_id=record.revision_id, + ) return SessionInfo( uri, - parse_uri(uri).segments[3], + parse_uri(uri).segments[-1], record.revision_id, "open", 0, @@ -181,7 +202,7 @@ def get(self, principal: Principal, session_id: str) -> SessionInfo: metadata = validate_session(self.store.get_document(principal, uri)) return SessionInfo( uri, - parse_uri(uri).segments[3], + parse_uri(uri).segments[-1], record.revision_id, metadata.get("status", "open"), len(metadata["events"]), @@ -243,9 +264,21 @@ def close(self, principal: Principal, session_id: str) -> SessionInfo: ), expected_revision=current.revision_id, ) - self._queue_compile( - principal, record.uri, record.document_id, record.revision_id - ) + address = parse_uri(record.uri) + with self.store.transaction(): + if address.namespace == "projects": + from .catalog import Catalog + + Catalog(self.store).register_session( + principal, + address.segments[0], + address.segments[2], + status="closed", + revision_id=record.revision_id, + ) + self._queue_compile( + principal, record.uri, record.document_id, record.revision_id + ) return self.get(principal, current.uri) @staticmethod diff --git a/src/docling_context/source_urls.py b/src/docling_context/source_urls.py new file mode 100644 index 0000000..8c24102 --- /dev/null +++ b/src/docling_context/source_urls.py @@ -0,0 +1,98 @@ +"""Bounded downloads for explicitly supplied document URLs.""" + +from __future__ import annotations + +from email.message import Message +from pathlib import Path +from urllib.parse import unquote, urlsplit + +import httpx + +from .package import MAX_PACKAGE_BYTES + +_CONTENT_EXTENSIONS = { + "application/pdf": ".pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx", + "application/msword": ".doc", + "application/vnd.ms-powerpoint": ".ppt", + "application/vnd.ms-excel": ".xls", + "text/html": ".html", + "text/plain": ".txt", + "image/png": ".png", + "image/jpeg": ".jpg", + "image/tiff": ".tiff", + "image/webp": ".webp", + "image/bmp": ".bmp", +} + + +def is_source_url(value: str) -> bool: + """Distinguish HTTP(S) input from a local path and reject other URL schemes.""" + if "://" not in value: + return False + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("source URL must use HTTP or HTTPS") + if parsed.username or parsed.password: + raise ValueError("source URL must not contain credentials") + return True + + +def _filename(url: str, headers: httpx.Headers, data: bytes) -> str: + disposition = Message() + disposition["Content-Disposition"] = headers.get("content-disposition", "") + supplied = disposition.get_filename() + path_name = Path(unquote(urlsplit(url).path).rstrip("/")).name + name = Path((supplied or path_name or "download").replace("\\", "/")).name + name = "".join(char for char in name if char.isprintable()) or "download" + if data.startswith(b"%PDF-"): + return name if name.lower().endswith(".pdf") else name + ".pdf" + content_type = headers.get("content-type", "").split(";", 1)[0].strip().lower() + extension = _CONTENT_EXTENSIONS.get(content_type) + if extension and not name.lower().endswith(extension): + # URLs such as arxiv.org/pdf/2608.23083 have an ID, not a file suffix. + name += extension + return name + + +def download_source( + url: str, + *, + max_bytes: int = MAX_PACKAGE_BYTES, + timeout: float = 30.0, + transport: httpx.BaseTransport | None = None, +) -> tuple[bytes, str]: + """Return bounded source bytes and an inferred filename for conversion.""" + if not is_source_url(url): + raise ValueError("expected an HTTP or HTTPS source URL") + if max_bytes < 1 or timeout <= 0: + raise ValueError("download limits must be positive") + try: + with ( + httpx.Client( + follow_redirects=True, + max_redirects=5, + timeout=timeout, + transport=transport, + ) as client, + client.stream("GET", url) as response, + ): + response.raise_for_status() + length = response.headers.get("content-length", "") + if length.isdecimal() and int(length) > max_bytes: + raise ValueError("download exceeds the source size limit") + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(chunk_size=65_536): + size += len(chunk) + if size > max_bytes: + raise ValueError("download exceeds the source size limit") + chunks.append(chunk) + data = b"".join(chunks) + if not data: + raise ValueError("source URL returned an empty document") + return data, _filename(str(response.url), response.headers, data) + except httpx.HTTPError as exc: + raise ValueError(f"cannot download source URL: {exc}") from exc diff --git a/src/docling_context/uri.py b/src/docling_context/uri.py index 014fda9..f8be940 100644 --- a/src/docling_context/uri.py +++ b/src/docling_context/uri.py @@ -50,10 +50,10 @@ def parse_uri(value: str, *, prefix: bool = False) -> ContextURI: raise InvalidURI("URI contains a query, fragment, backslash, or invalid escape") rest = value[len("docling://") :] namespace, separator, path = rest.partition("/") - if namespace not in {"resources", "users"} or ( - not separator and not (prefix and namespace == "resources") + if namespace not in {"resources", "users", "projects", "user"} or ( + not separator and not prefix ): - raise InvalidURI("URI namespace must be resources or users") + raise InvalidURI("URI namespace must be resources, projects, user, or users") raw_segments = path.split("/") if separator else [] segments = [] for raw in raw_segments: @@ -73,9 +73,9 @@ def parse_uri(value: str, *, prefix: bool = False) -> ContextURI: raise InvalidURI("URI contains a control character") segments.append(segment) minimum = ( - (0 if namespace == "resources" else 2) + (2 if namespace == "users" else 0) if prefix - else (2 if namespace == "resources" else 4) + else (4 if namespace == "users" else 2 if namespace == "resources" else 1) ) if len(segments) < minimum: raise InvalidURI("URI is too short for its namespace") @@ -85,6 +85,24 @@ def parse_uri(value: str, *, prefix: bool = False) -> ContextURI: and segments[2] not in {"memories", "sessions", "skills"} ): raise InvalidURI("user URI must contain memories, sessions, or skills") + if ( + namespace == "resources" + and segments + and segments[0] in {"library", "memories", "skills", "concepts", "knowledge"} + and not prefix + and len(segments) != 2 + ): + raise InvalidURI("shared resource URI must have one ID") + if ( + namespace == "projects" + and len(segments) > 1 + and (len(segments) != 3 or segments[1] != "sessions") + and not ( + len(segments) == 2 + and segments[1] in {".abstract.dclx", ".overview.dclx", ".description.dclx"} + ) + ): + raise InvalidURI("invalid project path") return ContextURI(namespace, tuple(segments)) @@ -95,3 +113,9 @@ def authorize_uri(principal: Principal, uri: ContextURI) -> None: raise AccessDenied("tenant does not match") if len(uri.segments) < 2 or uri.segments[1] != principal.user_id: raise AccessDenied("user does not match") + if ( + uri.namespace == "user" + and uri.segments + and uri.segments[0] != principal.user_id + ): + raise AccessDenied("user does not match") diff --git a/src/docling_context/vectors.py b/src/docling_context/vectors.py index aaa79af..c94cd97 100644 --- a/src/docling_context/vectors.py +++ b/src/docling_context/vectors.py @@ -32,6 +32,29 @@ def checkpoint(self) -> None: ... def rebuild(self) -> None: ... +class VectorStorageUnavailable(ValueError): + """The saved vector projection cannot be searched.""" + + +_BUILD_HINT = "Run 'dc index vector rebuild' to build vector storage." + + +def verified_vector_state(store: LocalContextStore): + """Check the saved snapshot before loading a potentially costly model.""" + state = store.db.execute("SELECT * FROM vector_state WHERE singleton=1").fetchone() + if state is None: + raise VectorStorageUnavailable(f"Vector storage does not exist. {_BUILD_HINT}") + path = store.root / f"vectors-{state['generation']}.tvim" + try: + if hashlib.sha256(path.read_bytes()).hexdigest() != state["snapshot_hash"]: + raise VectorStorageUnavailable(f"Vector storage is corrupt. {_BUILD_HINT}") + except OSError as exc: + raise VectorStorageUnavailable( + f"Vector storage is missing. {_BUILD_HINT}" + ) from exc + return state + + class TurbovecIndex: """IdMapIndex snapshot; SQLite embeddings and DCLX remain authoritative.""" @@ -41,8 +64,11 @@ def __init__( provider: EmbeddingProvider, *, force_rebuild: bool = False, + search_only: bool = False, progress: Callable[[int, int], None] | None = None, ): + if search_only and not force_rebuild: + verified_vector_state(store) try: import numpy as np from turbovec import IdMapIndex # type: ignore[import-untyped] @@ -63,11 +89,68 @@ def __init__( self.index = IdMapIndex(dim=provider.dimensions, bit_width=4) self.generation = "" self._progress = progress + self.search_only = search_only + self._snapshot_hash = "" if force_rebuild: self.rebuild() + elif search_only: + self._load_existing() else: self._restore_or_rebuild() + def _load_existing(self) -> None: + state = verified_vector_state(self.store) + if (state["model_id"], state["dimensions"], state["normalization"]) != ( + self.provider.model_id, + self.provider.dimensions, + "l2", + ): + raise VectorStorageUnavailable( + f"Vector model does not match storage. {_BUILD_HINT}" + ) + try: + loaded = self.index_type.load(str(self._path(state["generation"]))) + if loaded.dim != self.provider.dimensions: + raise ValueError("vector dimensions differ") + except (OSError, ValueError, RuntimeError) as exc: + raise VectorStorageUnavailable( + f"Vector storage cannot be loaded. {_BUILD_HINT}" + ) from exc + self.index = loaded + self.generation = state["generation"] + self._snapshot_hash = state["snapshot_hash"] + + def warning(self, tenant_id: str) -> str | None: + """Describe a saved projection that omits current searchable units.""" + state = self.store.db.execute( + "SELECT outbox_offset FROM vector_state WHERE singleton=1" + ).fetchone() + incomplete = state is None or state["outbox_offset"] != self._offset() + if not incomplete: + incomplete = ( + self.store.db.execute( + """SELECT 1 FROM retrieval_units r JOIN documents d + ON d.tenant_id=r.tenant_id AND d.uri=r.uri + WHERE r.tenant_id=? AND d.available=1 + AND d.revision_id=r.revision_id + AND (trim(r.text)<>'' OR (? AND r.asset_path IS NOT NULL)) + AND NOT EXISTS (SELECT 1 FROM vector_members m + WHERE m.generation=? AND m.vector_id=r.vector_id) + LIMIT 1""", + ( + tenant_id, + bool(getattr(self.provider, "supports_images", False)), + self.generation, + ), + ).fetchone() + is not None + ) + return ( + f"Vector storage is incomplete or stale. {_BUILD_HINT}" + if incomplete + else None + ) + def _offset(self) -> int: return self.store.db.execute( "SELECT COALESCE(MAX(sequence),0) FROM outbox" @@ -229,6 +312,14 @@ def _replay(self, state, loaded) -> None: self._path(state["generation"]).unlink(missing_ok=True) def refresh(self) -> None: + if self.search_only: + state = verified_vector_state(self.store) + if (state["generation"], state["snapshot_hash"]) != ( + self.generation, + self._snapshot_hash, + ): + self._load_existing() + return state = self.store.db.execute( "SELECT outbox_offset,generation FROM vector_state WHERE singleton=1" ).fetchone() diff --git a/tests/test_agent_sessions.py b/tests/test_agent_sessions.py new file mode 100644 index 0000000..5380ea3 --- /dev/null +++ b/tests/test_agent_sessions.py @@ -0,0 +1,235 @@ +"""Project agent sessions and owned agent skill installation.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from docling_context.agent_bridge import ( + inspect_agent, + install_agent, + remove_agent, + skill_path, +) +from docling_context.catalog import Catalog +from docling_context.cli import main +from docling_context.integration import IntegrationSpec +from docling_context.local import LocalContextStore +from docling_context.models import Principal +from docling_context.sessions import SessionStore + + +@pytest.mark.parametrize("agent", ["codex", "claude", "hermes", "pi"]) +def test_agent_install_is_owned_and_idempotent(tmp_path, agent): + spec = IntegrationSpec( + agent, + scope="user", + home=tmp_path / "home", + project_root=tmp_path / "project", + store=tmp_path / "store", + allow_writes=True, + ) + assert install_agent(spec)["status"] == "installed" + skill = skill_path(spec) + assert skill.exists() and "project_session_start" in skill.read_text() + assert inspect_agent(spec)["status"] == "configured" + assert install_agent(spec)["status"] == "current" + assert remove_agent(spec)["status"] == "removed" + assert not skill.exists() + + +def test_agent_install_preserves_unmanaged_skill(tmp_path): + spec = IntegrationSpec( + "codex", + scope="project", + project_root=tmp_path, + home=tmp_path / "home", + ) + path = skill_path(spec) + path.parent.mkdir(parents=True) + path.write_text("personal skill") + with pytest.raises(ValueError, match="not managed"): + install_agent(spec) + assert path.read_text() == "personal skill" + + +def test_agent_cli_install_status_remove(tmp_path, capsys): + args = [ + "--store", + str(tmp_path / "store"), + "agent", + ] + options = [ + "--agent", + "codex", + "--scope", + "project", + "--project-root", + str(tmp_path), + ] + assert main([*args, "install", *options, "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "installed" + assert main([*args, "status", *options, "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "configured" + assert main([*args, "remove", *options, "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "removed" + + +def test_start_no_launch_requires_project_and_records_agent(tmp_path, capsys): + principal = Principal("default", "local") + with LocalContextStore(tmp_path) as store: + Catalog(store).create_project(principal, "materials", "Materials") + args = [ + "--store", + str(tmp_path), + "session", + "start", + "--project", + "materials", + "--agent", + "codex", + "--no-launch", + "--id", + "review", + "--json", + ] + assert main(args) == 0 + result = json.loads(capsys.readouterr().out) + assert result["uri"] == "docling://projects/materials/sessions/review" + assert result["agent"] == "codex" + with LocalContextStore(tmp_path) as store: + assert SessionStore(store).get(principal, result["uri"]).status == "open" + assert ( + Catalog(store).project_sessions(principal, "materials")[0]["agent"] + == "codex" + ) + assert ( + main( + [ + "--store", + str(tmp_path), + "session", + "list", + "--project", + "materials", + "--json", + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)[0]["agent"] == "codex" + + +@pytest.mark.parametrize( + ("agent", "expected"), + [ + ("codex", ["codex"]), + ("claude", ["claude", "--append-system-prompt"]), + ("hermes", ["hermes", "chat", "-q"]), + ("pi", ["pi", "--append-system-prompt"]), + ], +) +def test_launcher_waits_and_closes(tmp_path, monkeypatch, capsys, agent, expected): + from docling_context import cli + + principal = Principal("default", "local") + with LocalContextStore(tmp_path) as store: + Catalog(store).create_project(principal, "materials", "Materials") + monkeypatch.setattr(cli.shutil, "which", lambda name: name) + installs = [] + + def fake_install(spec): + installs.append(spec) + return {"status": "installed"} + + monkeypatch.setattr(cli, "install_agent", fake_install) + calls = [] + + def fake_run(command, *, env, check): + calls.append((command, env)) + with LocalContextStore(tmp_path) as store: + assert ( + SessionStore(store) + .get(principal, env["DOCLING_CONTEXT_SESSION"]) + .status + == "open" + ) + return SimpleNamespace(returncode=7) + + monkeypatch.setattr(cli.subprocess, "run", fake_run) + assert ( + main( + [ + "--store", + str(tmp_path), + "session", + "start", + "--project", + "materials", + "--agent", + agent, + "--id", + "review", + "--json", + ] + ) + == 7 + ) + capsys.readouterr() + command, env = calls[0] + assert command[: len(expected)] == expected + assert installs[0].scope == "user" + assert env["DOCLING_CONTEXT_PROJECT"] == "materials" + assert ( + env["DOCLING_CONTEXT_SESSION"] == "docling://projects/materials/sessions/review" + ) + with LocalContextStore(tmp_path) as store: + assert ( + SessionStore(store).get(principal, env["DOCLING_CONTEXT_SESSION"]).status + == "closed" + ) + assert ( + Catalog(store).project_sessions(principal, "materials")[0]["agent"] == agent + ) + + +def test_launcher_closes_after_process_error(tmp_path, monkeypatch, capsys): + from docling_context import cli + + principal = Principal("default", "local") + with LocalContextStore(tmp_path) as store: + Catalog(store).create_project(principal, "materials", "Materials") + monkeypatch.setattr(cli.shutil, "which", lambda name: name) + monkeypatch.setattr(cli, "install_agent", lambda spec: {"status": "installed"}) + + def fail(*args, **kwargs): + raise OSError("agent failed to start") + + monkeypatch.setattr(cli.subprocess, "run", fail) + assert ( + main( + [ + "--store", + str(tmp_path), + "session", + "start", + "--project", + "materials", + "--agent", + "codex", + "--id", + "review", + ] + ) + == 1 + ) + assert "agent failed to start" in capsys.readouterr().err + with LocalContextStore(tmp_path) as store: + assert ( + SessionStore(store) + .get(principal, "docling://projects/materials/sessions/review") + .status + == "closed" + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index d701d72..a85967d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import logging +import warnings import pytest from doclang import DocLangXDocument @@ -14,7 +16,16 @@ PdfConversionConfig, Principal, ) -from docling_context.cli import _slug, main +from docling_context import cli as cli_module +from docling_context.catalog import Catalog +from docling_context.cli import main + + +@pytest.fixture +def project_catalog(tmp_path, monkeypatch): + monkeypatch.setenv("DOCLING_CONTEXT_STORE", str(tmp_path / "store")) + with LocalContextStore(tmp_path / "store") as store: + yield Catalog(store) def test_conversion_config_json_is_strict(tmp_path): @@ -35,20 +46,16 @@ def test_conversion_config_json_is_strict(tmp_path): PdfConversionConfig.from_dict({"do_ocr": "false"}) -def test_filename_normalization_does_not_merge_distinct_names(): - assert _slug("a b") != _slug("a-b") - - def test_add_resource_help_describes_defaults(capsys): with pytest.raises(SystemExit) as exc: - main(["add-resource", "--help"]) + main(["library", "add", "--help"]) assert exc.value.code == 0 help_text = " ".join(capsys.readouterr().out.split()) for expected in ( "--recursive", "default: off", "default: on", - "default: documents", + "--project PROJECT", "default: unlimited", "default: none", "--from FORMAT", @@ -58,7 +65,40 @@ def test_add_resource_help_describes_defaults(capsys): assert "--table-mode" not in help_text -def test_cli_folder_scanning_supports_native_documents(tmp_path, capsys): +def test_project_help_explains_subcommands(capsys): + with pytest.raises(SystemExit) as exc: + main(["project", "--help"]) + assert exc.value.code == 0 + help_text = capsys.readouterr().out + assert "Create a project" in help_text + assert "List linked resources" in help_text + assert "Read and generate project context" in help_text + assert "session-start" not in help_text + + +def test_bare_cli_shows_starting_points_and_legacy_route_is_absent(tmp_path, capsys): + assert main(["--store", str(tmp_path / "store")]) == 0 + output = capsys.readouterr().out + assert "Create a project:" in output + assert "dc library add" in output + assert "dc session start" in output + with pytest.raises(SystemExit) as exc: + main(["--store", str(tmp_path / "store"), "add-resource", "paper.pdf"]) + assert exc.value.code == 2 + + +def test_init_creates_local_catalog(tmp_path, capsys): + root = tmp_path / "fresh" + assert main(["--store", str(root), "init", "--json"]) == 0 + assert (root / "context.sqlite3").is_file() + assert json.loads(capsys.readouterr().out)["catalog_schema_version"] == 1 + assert main(["--store", str(root), "overview"]) == 0 + assert capsys.readouterr().out.startswith("No records.\n") + + +def test_cli_folder_scanning_supports_native_documents( + tmp_path, capsys, project_catalog +): folder = tmp_path / "sources" nested = folder / "nested" nested.mkdir(parents=True) @@ -69,27 +109,34 @@ def test_cli_folder_scanning_supports_native_documents(tmp_path, capsys): (folder / "third.dclx").write_bytes(package.write_bytes()) (nested / "fourth.dclg").write_text("Fourth") (folder / "ignored.bin").write_text("Ignored") - base = ["--store", str(tmp_path / "store"), "add-resource", str(folder)] + base = [ + "--store", + str(tmp_path / "store"), + "library", + "add", + "--source", + str(folder), + "--json", + ] assert main(base) == 0 top_level = { json.loads(line)["uri"] for line in capsys.readouterr().out.splitlines() } - assert top_level == { - "docling://resources/documents/first", - "docling://resources/documents/second", - "docling://resources/documents/third", - } + assert len(top_level) == 3 + assert all(uri.startswith("docling://resources/library/") for uri in top_level) assert main(base + ["-r"]) == 0 recursive = { json.loads(line)["uri"] for line in capsys.readouterr().out.splitlines() } - assert recursive == top_level | {"docling://resources/documents/nested/fourth"} + assert top_level < recursive and len(recursive) == 4 assert ( main( [ "--store", str(tmp_path / "store"), - "add-resource", + "library", + "add", + "--source", str(folder / "first.dclg"), "-r", ] @@ -99,7 +146,111 @@ def test_cli_folder_scanning_supports_native_documents(tmp_path, capsys): assert "requires a folder" in capsys.readouterr().err -def test_cli_from_filter_and_filename_collisions(tmp_path, capsys): +def test_add_resource_resume_reports_existing_documents_and_suppresses_warnings( + tmp_path, capsys, monkeypatch, project_catalog +): + folder = tmp_path / "sources" + folder.mkdir() + (folder / "first.dclg").write_text("First") + (folder / "second.dclg").write_text("Second") + command = [ + "--store", + str(tmp_path / "store"), + "library", + "add", + "--source", + str(folder), + ] + + progress: list[object] = [] + + class ProgressSpy: + def __init__(self, *args, total, desc, initial=0, **kwargs): + self.total = total + self.desc = desc + self.n = initial + self.initial = initial + self.postfixes: list[str] = [] + progress.append(self) + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def update(self, count): + self.n += count + + def set_postfix_str(self, value): + self.postfixes.append(value) + + monkeypatch.setattr(cli_module, "tqdm", ProgressSpy) + monkeypatch.setattr( + cli_module, + "_source_page_units", + lambda source, filename: 3 if source.name == "first.dclg" else 2, + ) + original_add_resource = Ingestor.add_resource + calls = 0 + + def interrupt_second(self, *args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise KeyboardInterrupt + warnings.warn("conversion warning", UserWarning, stacklevel=1) + logging.getLogger(__name__).warning("conversion log warning") + return original_add_resource(self, *args, **kwargs) + + monkeypatch.setattr(Ingestor, "add_resource", interrupt_second) + assert main(command) == 130 + interrupted = capsys.readouterr() + assert "3 of 5 pages" in interrupted.err + assert "conversion warning" not in interrupted.err + assert "conversion log warning" not in interrupted.err + + positions_during_ingestion: list[int] = [] + + def observe_progress(self, *args, **kwargs): + positions_during_ingestion.append(progress[-1].n) + return original_add_resource(self, *args, **kwargs) + + monkeypatch.setattr(Ingestor, "add_resource", observe_progress) + assert main(command) == 0 + capsys.readouterr() + resumed = [item for item in progress if item.desc == "Importing pages"][-1] + assert resumed.total == 5 + assert resumed.initial == 0 + assert resumed.n == 5 + assert positions_during_ingestion == [0, 3] + assert resumed.postfixes[0] == "existing=1 docs, added=0 docs" + assert resumed.postfixes[-1] == "existing=1 docs, added=1 docs" + + positions_during_ingestion.clear() + assert main(command + ["--json"]) == 0 + output = capsys.readouterr() + assert len([json.loads(line) for line in output.out.splitlines()]) == 2 + assert "Importing pages" not in output.out + assert positions_during_ingestion == [0, 3] + assert progress[-1].n == 5 + + +def test_pdf_page_count_uses_pdf_pages_and_one_unit_for_other_sources(tmp_path): + pdfium = pytest.importorskip("pypdfium2") + pdf_path = tmp_path / "three-pages.pdf" + pdf = pdfium.PdfDocument.new() + for _ in range(3): + pdf.new_page(width=612, height=792) + pdf.save(pdf_path) + pdf.close() + + assert cli_module._source_page_units(pdf_path, None) == 3 + assert cli_module._source_page_units(pdf_path.read_bytes(), pdf_path.name) == 3 + assert cli_module._source_page_units(tmp_path / "notes.dclg", None) == 1 + + +def test_cli_from_filter_and_filename_collisions(tmp_path, capsys, project_catalog): folder = tmp_path / "sources" folder.mkdir() (folder / "same.dclg").write_text("One") @@ -107,13 +258,19 @@ def test_cli_from_filter_and_filename_collisions(tmp_path, capsys): assert document.read_xml("Two") (folder / "same.dclx").write_bytes(document.write_bytes()) (folder / "other.pdf").write_bytes(b"%PDF") - base = ["--store", str(tmp_path / "store"), "add-resource", str(folder)] + base = [ + "--store", + str(tmp_path / "store"), + "library", + "add", + "--source", + str(folder), + "--json", + ] assert main(base + ["--from", "dclg", "--from", "dclx"]) == 0 uris = {json.loads(line)["uri"] for line in capsys.readouterr().out.splitlines()} - assert uris == { - "docling://resources/documents/same.dclg", - "docling://resources/documents/same.dclx", - } + assert len(uris) == 2 + assert all(uri.startswith("docling://resources/library/") for uri in uris) assert main(base + ["--from", "unknown"]) == 1 assert "unknown source format" in capsys.readouterr().err (folder / "other.pdf").unlink() @@ -121,41 +278,97 @@ def test_cli_from_filter_and_filename_collisions(tmp_path, capsys): assert "no documents matching" in capsys.readouterr().err -def test_cli_import_worker_inspection_and_lexical_lookup(tmp_path, capsys): +def test_cli_import_worker_inspection_and_lexical_lookup( + tmp_path, capsys, project_catalog +): source = tmp_path / "guide.dclg" source.write_text( "GuideBlue glaciers" ) store = tmp_path / "store" base = ["--store", str(store)] - assert main(base + ["add-resource", str(source), "--collection", "manuals"]) == 0 + assert main(base + ["overview"]) == 0 + assert capsys.readouterr().out.startswith("No records.\n") + assert ( + main(base + ["project", "create", "--project", "manuals", "--title", "Manuals"]) + == 0 + ) + project_table = capsys.readouterr().out + assert "Project id" in project_table and "Manuals" in project_table + assert ( + main(base + ["library", "add", "--source", str(source), "--project", "manuals"]) + == 0 + ) + import_table = capsys.readouterr().out + assert "Uri" in import_table and "Revision Id" in import_table + assert ( + main( + base + + [ + "library", + "add", + "--source", + str(source), + "--project", + "manuals", + "--json", + ] + ) + == 0 + ) added = json.loads(capsys.readouterr().out) - assert added["uri"] == "docling://resources/manuals/guide" - assert added["task_id"] - assert main(base + ["task", "status", added["task_id"]]) == 0 - task_table = capsys.readouterr().out - assert "Field" in task_table and "Status" in task_table and "queued" in task_table - assert main(base + ["task", "status", added["task_id"], "--json"]) == 0 - assert json.loads(capsys.readouterr().out)["status"] == "queued" - assert main(base + ["worker", "--once"]) == 0 - capsys.readouterr() - assert main(base + ["task", "status", added["task_id"], "--json"]) == 0 - assert json.loads(capsys.readouterr().out)["status"] == "complete" - assert main(base + ["ls", "docling://resources/"]) == 0 - assert "docling://resources/manuals" in capsys.readouterr().out - assert main(base + ["tree", "docling://resources/manuals", "-L", "2"]) == 0 - assert "guide" in capsys.readouterr().out + assert added["uri"].startswith("docling://resources/library/") + assert added["uri"] in import_table + assert main(base + ["overview"]) == 0 + overview_table = capsys.readouterr().out + assert "Documents" in overview_table and "Manuals" in overview_table + assert main(base + ["overview", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["projects"][0]["documents"] == 1 + assert main(base + ["library", "list"]) == 0 + assert "docling://resources/library" in capsys.readouterr().out assert ( - main(base + ["find", "glaciers", "--uri", "docling://resources/manuals"]) == 0 + main( + base + + ["library", "tree", "--uri", "docling://resources/library", "-L", "2"] + ) + == 0 + ) + assert added["uri"] in capsys.readouterr().out + assert ( + main( + base + + [ + "library", + "find", + "--term", + "glaciers", + "--uri", + "docling://resources/library", + ] + ) + == 0 ) assert "glaciers" in capsys.readouterr().out - assert main(base + ["grep", "blue", "--uri", "docling://resources/manuals"]) == 0 + assert ( + main( + base + + [ + "library", + "grep", + "--term", + "blue", + "--uri", + "docling://resources/library", + ] + ) + == 0 + ) assert "Blue glaciers" in capsys.readouterr().out - assert main(base + ["status"]) == 0 + assert main(base + ["overview"]) == 0 store_table = capsys.readouterr().out assert "Field" in store_table and "Documents" in store_table - assert main(base + ["status", "--json"]) == 0 - assert json.loads(capsys.readouterr().out)["documents"] == 2 + assert main(base + ["overview", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["counts"]["documents"] == 2 def test_pdf_config_changes_ingestion_fingerprint(tmp_path): @@ -185,3 +398,82 @@ def convert(self, _source, _filename): ) assert repeated.revision_id == first.revision_id assert changed.revision_id != first.revision_id + + +def test_folder_import_skips_failing_documents_and_reports_them( + tmp_path, capsys, project_catalog +): + folder = tmp_path / "sources" + folder.mkdir() + (folder / "a_broken.dclg").write_text("unclosed") + (folder / "b_good.dclg").write_text("Good") + base = [ + "--store", + str(tmp_path / "store"), + "library", + "add", + "--source", + str(folder), + ] + + assert main(base + ["--json"]) == 1 + captured = capsys.readouterr() + rows = [json.loads(line) for line in captured.out.splitlines()] + imported = [row for row in rows if "uri" in row] + failed = [row for row in rows if row.get("status") == "failed"] + assert len(imported) == 1 + assert [row["source"] for row in failed] == ["a_broken.dclg"] + assert "1 of 2 documents failed" in captured.err + + assert main(base + ["--json", "--fail-fast"]) == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err.startswith("dc: ") + + +def test_cli_has_no_positional_arguments(): + """Every value is passed by name; only subcommand names are positional.""" + import argparse + + def positionals(parser, path): + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for name, sub in action.choices.items(): + yield from positionals(sub, [*path, name]) + elif not action.option_strings: + yield f"dc {' '.join(path)} <{action.dest}>" + + assert list(positionals(cli_module._parser(), [])) == [] + + +def test_repeated_options_share_consistent_short_flags(): + """A short flag always means the same long option across the whole CLI.""" + import argparse + + expected = { + "-a": "--agent", + "-d": "--document", + "-i": "--id", + "-j": "--json", + "-p": "--project", + "-q": "--query", + "-s": "--session", + "-u": "--uri", + "-x": "--xpath", + } + seen: dict[str, set[str]] = {} + + def walk(parser): + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for sub in action.choices.values(): + walk(sub) + continue + longs = [o for o in action.option_strings if o.startswith("--")] + shorts = [o for o in action.option_strings if len(o) == 2] + for long in longs: + seen.setdefault(long, set()).update(shorts) + + walk(cli_module._parser()) + for short, long in expected.items(): + assert seen[long] == {short}, long diff --git a/tests/test_ingestion_jobs.py b/tests/test_ingestion_jobs.py index abd3950..39126fc 100644 --- a/tests/test_ingestion_jobs.py +++ b/tests/test_ingestion_jobs.py @@ -8,7 +8,7 @@ from zipfile import ZIP_DEFLATED, ZipFile import pytest -from doclang import DocLangXDocument +from doclang import DoclangDocument, DocLangXDocument, DocLangXDocumentRole from docling_context import ( CollectionWorker, @@ -22,6 +22,8 @@ Principal, RecordMissing, ) +from docling_context.doclang_authoring import get_outline, get_summary +from docling_context.package import PackageError PRINCIPAL = Principal("tenant-a", "alice") COLLECTION = "docling://resources/articles" @@ -32,8 +34,9 @@ def dclx() -> bytes: doc = DocLangXDocument() assert doc.read_xml("FirstBody") doc.set_part_bytes("assets/source.bin", b"asset", "application/octet-stream") - assert doc.set_concepts( - "
Topic
" + assert doc.set( + DocLangXDocumentRole.CONCEPTS, + DoclangDocument("Topic"), ) return doc.write_bytes() @@ -46,13 +49,14 @@ def test_native_import_tiers_assets_and_idempotence(tmp_path): assert first.source["origin_uri"].startswith("sha256:") package = store.get_document(PRINCIPAL, URI) assert package.at(xpath="/doclang[1]/text[1]") == "Body" - assert "First" in package.summary().xml() - assert ( - package.toc().at(xpath="/doclang[1]/toc[1]/entry[1]/description[1]") - == "First" - ) + summary = get_summary(package) + assert summary is not None + assert "First" in summary.xml() + assert summary.validate(allow_empty_namespace=True).ok() + assert get_outline(package).at(xpath="/doclang[1]/heading[1]") == "First" assert package.get_part_bytes("assets/source.bin") == b"asset" - assert "Topic" in package.concepts().xml() + concepts = package.get(DocLangXDocumentRole.CONCEPTS) + assert concepts is not None and "Topic" in concepts.xml() manifest = json.loads(package.get_part_text("context/manifest.json")) assert manifest["schema_version"] == 1 assert manifest["source_revision"] == first.revision_id @@ -71,6 +75,19 @@ def test_native_import_tiers_assets_and_idempotence(tmp_path): assert forced.revision_id != first.revision_id +def test_native_dclg_rejects_xml_outside_doclang_schema(tmp_path): + with ( + LocalContextStore(tmp_path) as store, + pytest.raises(PackageError, match="invalid DCLG source"), + ): + Ingestor(store).add_resource( + PRINCIPAL, + URI, + b"Unvalidated", + filename="invalid.dclg", + ) + + def test_ingestor_uses_store_contract_with_in_memory_backend(): store = MemoryContextStore() record = Ingestor(store).add_resource( @@ -153,9 +170,9 @@ def test_pdf_converter_contract_and_model_fallback(tmp_path): ) assert record.source["converter"] == "test-pdf" assert record.source["summary_method"] == "extractive-fallback" - assert ( - store.get_document(PRINCIPAL, URI).summary().at(xpath="/doclang[1]/text[1]") - ) + summary = get_summary(store.get_document(PRINCIPAL, URI)) + assert summary is not None + assert summary.at(xpath="/doclang[1]/text[1]") def test_model_summary_is_tied_to_committed_revision(tmp_path): @@ -166,7 +183,9 @@ def test_model_summary_is_tied_to_committed_revision(tmp_path): ) package = store.get_document(PRINCIPAL, URI) assert provider.seen_revision == record.revision_id - assert package.summary().at(xpath="/doclang[1]/text[1]") == "Model summary" + summary = get_summary(package) + assert summary is not None + assert summary.at(xpath="/doclang[1]/text[1]") == "Model summary" def test_nested_heading_outline(tmp_path): @@ -176,11 +195,9 @@ def test_nested_heading_outline(tmp_path): ) with LocalContextStore(tmp_path) as store: Ingestor(store).add_resource(PRINCIPAL, URI, source, filename="one.dclg") - toc = store.get_document(PRINCIPAL, URI).toc() - assert ( - toc.at(xpath="/doclang[1]/toc[1]/entry[1]/entry[1]/description[1]") - == "Section" - ) + toc = get_outline(store.get_document(PRINCIPAL, URI)) + assert toc.at(xpath="/doclang[1]/heading[2]") == "Section" + assert toc.validate(allow_empty_namespace=True).ok() def test_remote_adapter_uses_zip_target_and_bounded_package(monkeypatch): @@ -292,5 +309,5 @@ def test_existing_database_upgrades_without_losing_documents(tmp_path): ) with LocalContextStore(tmp_path) as store: assert store.get_record(PRINCIPAL, URI).revision_id == original.revision_id - assert store.db.execute("PRAGMA user_version").fetchone()[0] == 8 + assert store.db.execute("PRAGMA user_version").fetchone()[0] == 10 assert len(CollectionWorker(store).jobs(PRINCIPAL, COLLECTION)) == 1 diff --git a/tests/test_pdf_integration.py b/tests/test_pdf_integration.py index 5d62272..355f5ec 100644 --- a/tests/test_pdf_integration.py +++ b/tests/test_pdf_integration.py @@ -16,7 +16,25 @@ PdfConversionConfig, Principal, ) +from docling_context.catalog import Catalog from docling_context.cli import main +from docling_context.doclang_authoring import get_outline, get_summary + + +def _native_conversion_available() -> bool: + if sys.platform != "darwin": + return True + try: + import torch + except ImportError: + return False + return bool(torch.backends.mps.is_available()) + + +native_conversion = pytest.mark.skipif( + not _native_conversion_available(), + reason="native Docling layout conversion requires Metal in this macOS environment", +) def _one_page_pdf() -> bytes: @@ -47,6 +65,7 @@ def _one_page_pdf() -> bytes: return bytes(output) +@native_conversion def test_local_pdf_to_valid_dclx(tmp_path): pytest.importorskip("docling") with LocalContextStore(tmp_path) as store: @@ -58,21 +77,23 @@ def test_local_pdf_to_valid_dclx(tmp_path): ) record = Ingestor(store, converter=LocalDoclingConverter(config)).add_resource( Principal("tenant", "alice"), - "docling://resources/papers/hello", + "docling://resources/library/hello", _one_page_pdf(), filename="hello.pdf", ) package = store.get_document(Principal("tenant", "alice"), record.uri) assert "Hello DocLang" in package.xml() - assert package.summary() is not None - assert package.toc() is not None + assert get_summary(package) is not None + assert get_outline(package) is not None assert any(node["bbox"] is not None for node in package.iter_nodes(limit=1000)) assert record.source["converter_options"]["do_ocr"] is False assert record.source["converter_options"]["do_table_structure"] is False +@native_conversion def test_cli_imports_nested_pdf_folder(tmp_path, capsys): pytest.importorskip("docling") + tenant = "test" folder = tmp_path / "papers" (folder / "year").mkdir(parents=True) (folder / "first.pdf").write_bytes(_one_page_pdf()) @@ -80,29 +101,44 @@ def test_cli_imports_nested_pdf_folder(tmp_path, capsys): config = tmp_path / "config.json" config.write_text('{"do_ocr": false, "do_table_structure": false}') store = tmp_path / "store" - assert ( - main( - [ - "--store", - str(store), - "add-resource", - str(folder), - "--collection", - "papers", - "-r", - "--config", - str(config), - ] + with LocalContextStore(store) as local: + catalog = Catalog(local) + base = ["--store", str(store), "--tenant", tenant] + assert ( + main( + [*base, "project", "create", "--project", "papers", "--title", "Papers"] + ) + == 0 ) - == 0 - ) - uris = {json.loads(line)["uri"] for line in capsys.readouterr().out.splitlines()} - assert uris == { - "docling://resources/papers/first", - "docling://resources/papers/year/second", - } + capsys.readouterr() + assert ( + main( + [ + *base, + "library", + "add", + "--source", + str(folder), + "--project", + "papers", + "-r", + "--config", + str(config), + "--json", + ] + ) + == 0 + ) + uris = { + json.loads(line)["uri"] for line in capsys.readouterr().out.splitlines() + } + # Both PDF files contain identical binary bytes and share one library item. + assert len(uris) == 1 + assert next(iter(uris)).startswith("docling://resources/library/") + assert catalog.overview(Principal(tenant, "local"))[0]["documents"] == 1 +@native_conversion def test_local_office_and_image_formats_to_valid_dclx(tmp_path): pytest.importorskip("docling") word = pytest.importorskip("docx") @@ -138,7 +174,7 @@ def test_local_office_and_image_formats_to_valid_dclx(tmp_path): converter=LocalDoclingConverter(PdfConversionConfig(do_ocr=False)), ) for path in (word_path, slides_path, workbook_path, image_path): - uri = f"docling://resources/documents/{path.stem}" + uri = f"docling://resources/library/{path.stem}" record = ingestor.add_resource(Principal("tenant", "alice"), uri, path) assert record.source["converter"] == "docling-local" package = store.get_document(Principal("tenant", "alice"), uri) diff --git a/tests/test_projects_catalog.py b/tests/test_projects_catalog.py new file mode 100644 index 0000000..c6ab44b --- /dev/null +++ b/tests/test_projects_catalog.py @@ -0,0 +1,566 @@ +"""End-to-end checks for the SQLite project catalog.""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +from docling_context.catalog import Catalog +from docling_context.cli import main +from docling_context.doclang_authoring import DEFAULT_SKILL_URI +from docling_context.ingest_queue import IngestQueue +from docling_context.local import LocalContextStore +from docling_context.models import Principal +from docling_context.service import ( + ContextService, + IngestRequest, + OverviewRequest, + SearchRequest, +) + + +@pytest.fixture +def catalog_scope(tmp_path): + with LocalContextStore(tmp_path / "store") as store: + yield Catalog(store), "test" + + +def test_import_link_overview_and_scoped_search(catalog_scope, tmp_path, capsys): + catalog, tenant = catalog_scope + store = tmp_path / "store" + source = tmp_path / "source.dclg" + source.write_text("Project telescope evidence") + args = ["--store", str(store), "--tenant", tenant] + principal = Principal(tenant, "local") + catalog.create_project(principal, "alpha", "Alpha") + catalog.create_project(principal, "beta", "Beta") + + assert ( + main( + [ + *args, + "library", + "add", + "--source", + str(source), + "--project", + "alpha", + "--json", + ] + ) + == 0 + ) + first = json.loads(capsys.readouterr().out) + assert first["uri"].startswith("docling://resources/library/") + assert ( + main( + [ + *args, + "library", + "add", + "--source", + str(source), + "--project", + "beta", + "--json", + ] + ) + == 0 + ) + second = json.loads(capsys.readouterr().out) + assert second["uri"] == first["uri"] + assert second["revision_id"] == first["revision_id"] + + overview = catalog.overview(principal) + assert [ + (row["project_id"], row["documents"], row["skills"]) for row in overview + ] == [("alpha", 1, 1), ("beta", 1, 1)] + assert main([*args, "project", "show", "-p", "alpha", "-j"]) == 0 + shown = json.loads(capsys.readouterr().out) + assert shown["project_id"] == "alpha" + assert {key: shown[key] for key in ("documents", "concepts", "skills")} == { + "documents": 1, + "concepts": 0, + "skills": 1, + } + assert main([*args, "project", "show", "-p", "alpha"]) == 0 + table = capsys.readouterr().out + assert "Documents" in table and "Concepts" in table and "Skills" in table + assert DEFAULT_SKILL_URI in { + row["uri"] for row in catalog.project_resources(principal, "alpha") + } + assert DEFAULT_SKILL_URI in { + row["uri"] for row in catalog.project_resources(principal, "beta") + } + default_skill = catalog.store.get_document(principal, DEFAULT_SKILL_URI) + assert default_skill.validate(allow_empty_namespace=True).ok() + assert "validate_package" in default_skill.xml() + with pytest.raises(ValueError, match="required by every project"): + catalog.unlink(principal, "alpha", DEFAULT_SKILL_URI) + with LocalContextStore(store) as local: + service = ContextService(local, principal) + assert service.overview(OverviewRequest()).data[0]["documents"] == 1 + found = service.search( + SearchRequest("telescope", mode="lexical", project_id="alpha") + ).data + assert any(hit["uri"] == first["uri"] for hit in found["hits"]) + assert ( + main( + [ + *args, + "library", + "search", + "--query", + "telescope", + "--project", + "alpha", + "--mode", + "lexical", + "--json", + ] + ) + == 0 + ) + assert first["uri"] in capsys.readouterr().out + + skill_source = tmp_path / "skill.dclg" + skill_source.write_text("Method protocol") + assert ( + main( + [ + *args, + "skill", + "create", + "--id", + "method", + "--from", + str(skill_source), + "--project", + "alpha", + "--json", + ] + ) + == 0 + ) + skill = json.loads(capsys.readouterr().out) + catalog.link_resources(principal, first["uri"], skill["uri"], "uses-method") + assert ( + catalog.resource_links(principal, first["uri"])[0]["target_uri"] == skill["uri"] + ) + assert catalog.overview(principal)[0]["skills"] == 2 + + catalog.unlink(principal, "alpha", first["uri"]) + assert catalog.overview(principal)[0]["documents"] == 0 + assert catalog.find_source(principal, first["source_sha256"]) == first["uri"] + assert ( + main( + [ + *args, + "library", + "search", + "--query", + "telescope", + "--project", + "alpha", + "--mode", + "lexical", + "--json", + ] + ) + == 0 + ) + assert capsys.readouterr().out == "" + + assert ( + main( + [ + *args, + "session", + "start", + "--project", + "alpha", + "--no-launch", + "--agent", + "codex", + "--id", + "chat-1", + "--json", + ] + ) + == 0 + ) + session = json.loads(capsys.readouterr().out) + assert session["uri"] == "docling://projects/alpha/sessions/chat-1" + assert ( + main( + [ + *args, + "session", + "append", + "--id", + "chat-1", + "--text", + "Remember: telescope evidence is reusable", + "--kind", + "turn", + "--key", + "turn-1", + ] + ) + == 0 + ) + capsys.readouterr() + assert main([*args, "session", "close", "--id", "chat-1", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "closed" + assert catalog.overview(principal)[0]["sessions"] == 1 + assert main([*args, "memory", "worker", "--once", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "completed" + assert catalog.overview(principal)[0]["memories"] == 1 + linked = catalog.project_resources(principal, "alpha") + assert any(item["resource_type"] == "memories" for item in linked) + + with LocalContextStore(store) as local: + service = ContextService(local, principal) + payload = base64.b64encode( + b"Queued project finding" + ).decode() + queued = service.ingest( + IngestRequest("", "finding.dclg", payload, project_id="alpha") + ).data + assert queued["status"] == "queued" + finished = IngestQueue(local).run_once() + assert finished is not None and finished.status == "completed" + assert finished.uri.startswith("docling://resources/library/") + assert catalog.overview(principal)[0]["documents"] == 1 + + +def test_existing_project_recovers_required_authoring_skill(tmp_path): + root = tmp_path / "store" + principal = Principal("team", "alice") + with LocalContextStore(root) as store: + catalog = Catalog(store) + catalog.create_project(principal, "alpha", "Alpha") + store.db.execute( + """DELETE FROM dc_project_resource_links + WHERE tenant_id=? AND project_id=? AND resource_type='skills' + AND resource_id='doclang-authoring'""", + (principal.tenant_id, "alpha"), + ) + with LocalContextStore(root) as store: + assert DEFAULT_SKILL_URI in { + row["uri"] for row in Catalog(store).project_resources(principal, "alpha") + } + + +def test_typed_descriptors_and_cited_knowledge(tmp_path, capsys): + root = tmp_path / "store" + source = tmp_path / "source.dclg" + source.write_text("Basalt density is 3.0.") + descriptor = tmp_path / "descriptor.dclg" + descriptor.write_text("Material observations") + metadata = tmp_path / "knowledge.json" + metadata.write_text( + json.dumps( + { + "name": "Materials", + "schema": { + "entity_types": ["material"], + "properties": {"density": {"type": "number"}}, + }, + } + ) + ) + base = ["--store", str(root)] + assert ( + main( + [ + *base, + "project", + "create", + "--project", + "materials", + "--title", + "Materials", + ] + ) + == 0 + ) + capsys.readouterr() + assert ( + main( + [ + *base, + "library", + "add", + "--source", + str(source), + "--project", + "materials", + "--json", + ] + ) + == 0 + ) + library = json.loads(capsys.readouterr().out) + assert ( + main( + [ + *base, + "concept", + "create", + "--id", + "material-properties", + "--from", + str(descriptor), + "--metadata", + str(metadata), + "--project", + "materials", + "--json", + ] + ) + == 0 + ) + concept = json.loads(capsys.readouterr().out) + assert ( + main( + [ + *base, + "knowledge", + "create", + "--id", + "materials", + "--concept", + "material-properties", + "--project", + "materials", + "--json", + ] + ) + == 0 + ) + knowledge = json.loads(capsys.readouterr().out) + assert knowledge["concept_revision"] == concept["revision_id"] + assert ( + main( + [ + *base, + "knowledge", + "add-fact", + "--uri", + knowledge["uri"], + "--entity-type", + "material", + "--entity-id", + "basalt", + "--property", + "density", + "--value", + "3.0", + "--source-uri", + library["uri"], + "--source-revision", + library["revision_id"], + "--source-xpath", + "/doclang[1]/text[1]", + "--json", + ] + ) + == 0 + ) + assert json.loads(capsys.readouterr().out)["fact_id"] + assert main([*base, "knowledge", "facts", "--uri", knowledge["uri"], "--json"]) == 0 + facts = json.loads(capsys.readouterr().out) + assert facts[0]["value"] == 3.0 + assert facts[0]["source_uri"] == library["uri"] + with LocalContextStore(root) as store: + catalog = Catalog(store) + principal = Principal("default", "local") + citation = { + "source_uri": library["uri"], + "source_revision": library["revision_id"], + "source_xpath": "/doclang[1]/text[1]", + } + for invalid in ( + {"entity_type": "mineral", "property": "density", "value": 3.0}, + {"entity_type": "material", "property": "porosity", "value": 3.0}, + {"entity_type": "material", "property": "density", "value": "3.0"}, + ): + with pytest.raises(ValueError, match="schema"): + catalog.add_fact( + principal, + knowledge["uri"], + entity_id="basalt", + **citation, + **invalid, + ) + assert main([*base, "overview", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["projects"][0]["knowledge"] == 1 + + +def test_concept_schema_required_and_dataset_revision_is_pinned(tmp_path, capsys): + root = tmp_path / "store" + descriptor = tmp_path / "concept.dclg" + descriptor.write_text("Formation measurements") + metadata = tmp_path / "concept.json" + base = ["--store", str(root)] + + assert ( + main( + [*base, "concept", "create", "--id", "formation", "--from", str(descriptor)] + ) + == 1 + ) + assert "schema" in capsys.readouterr().err + metadata.write_text( + json.dumps( + { + "schema": { + "entity_types": ["formation"], + "properties": {"porosity": {"type": "number", "unit": "%"}}, + } + } + ) + ) + create = [ + *base, + "concept", + "create", + "--id", + "formation", + "--from", + str(descriptor), + "--metadata", + str(metadata), + "--json", + ] + assert main(create) == 0 + first_revision = json.loads(capsys.readouterr().out)["revision_id"] + assert ( + main( + [ + *base, + "knowledge", + "create", + "--id", + "measurements", + "--concept", + "formation", + "--json", + ] + ) + == 0 + ) + dataset = json.loads(capsys.readouterr().out) + assert dataset["concept_revision"] == first_revision + + metadata.write_text( + json.dumps( + { + "schema": { + "entity_types": ["formation"], + "properties": {"porosity": {"type": "integer"}}, + } + } + ) + ) + assert main(create) == 0 + assert json.loads(capsys.readouterr().out)["revision_id"] != first_revision + assert main([*base, "knowledge", "show", "--uri", "measurements", "--json"]) == 0 + pinned = json.loads(capsys.readouterr().out) + assert pinned["concept_revision"] == first_revision + assert pinned["schema"]["properties"]["porosity"] == {"type": "number", "unit": "%"} + + +def test_library_relationship_commands(tmp_path, capsys): + root = tmp_path / "store" + source = tmp_path / "source.dclg" + source.write_text("Shared document") + base = ["--store", str(root)] + assert ( + main([*base, "project", "create", "--project", "alpha", "--title", "Alpha"]) + == 0 + ) + capsys.readouterr() + assert ( + main( + [ + *base, + "library", + "add", + "--source", + str(source), + "--project", + "alpha", + "--json", + ] + ) + == 0 + ) + resource = json.loads(capsys.readouterr().out) + document_id = resource["uri"].rsplit("/", 1)[-1] + assert ( + main([*base, "library", "projects", "--document", document_id, "--json"]) == 0 + ) + assert json.loads(capsys.readouterr().out)[0]["project_id"] == "alpha" + assert ( + main( + [ + *base, + "library", + "link", + "--document", + document_id, + "--target", + resource["uri"], + "--relation", + "related", + "--json", + ] + ) + == 0 + ) + capsys.readouterr() + assert main([*base, "library", "links", "--document", document_id, "--json"]) == 0 + assert json.loads(capsys.readouterr().out)[0]["relation"] == "related" + + +def test_project_context_jobs_refresh_after_membership_change(tmp_path, capsys): + root = tmp_path / "store" + source = tmp_path / "source.dclg" + source.write_text("Basalt field notes") + base = ["--store", str(root)] + assert ( + main([*base, "project", "create", "--project", "geology", "--title", "Geology"]) + == 0 + ) + capsys.readouterr() + for _ in range(2): + assert main([*base, "project", "context", "worker", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "completed" + with LocalContextStore(root) as store: + project = Catalog(store).get_project(Principal("default", "local"), "geology") + assert project.abstract_uri and project.overview_uri + before = store.get_record(Principal("default", "local"), project.overview_uri) + assert ( + main([*base, "library", "add", "--source", str(source), "--project", "geology"]) + == 0 + ) + capsys.readouterr() + assert ( + main([*base, "project", "context", "jobs", "--project", "geology", "--json"]) + == 0 + ) + assert {job["status"] for job in json.loads(capsys.readouterr().out)} == {"queued"} + for _ in range(2): + assert main([*base, "project", "context", "worker", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "completed" + with LocalContextStore(root) as store: + principal = Principal("default", "local") + project = Catalog(store).get_project(principal, "geology") + assert project.overview_uri is not None + after = store.get_record(principal, project.overview_uri) + assert after.revision_id != before.revision_id + assert "library" in store.get_document(principal, project.overview_uri).xml() diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py index 9e379c9..81e2871 100644 --- a/tests/test_retrieval.py +++ b/tests/test_retrieval.py @@ -11,12 +11,15 @@ from docling_context import ( Ingestor, LocalContextStore, + MemoryService, Principal, Retriever, SearchScope, TurbovecIndex, ) from docling_context.cli import _search_excerpt, main +from docling_context.service import ContextService, SearchRequest +from docling_context.vectors import VectorStorageUnavailable def _package(body: str) -> bytes: @@ -52,6 +55,149 @@ def embed(self, texts: list[str]) -> list[list[float]]: return output +def _maintenance_retriever(store, provider): + """Exercise the explicit index maintenance path used by legacy tests.""" + return Retriever( + store, embedder=provider, vector_index=TurbovecIndex(store, provider) + ) + + +def test_vector_search_requires_saved_storage(tmp_path, capsys): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("ice"), + ) + with pytest.raises(VectorStorageUnavailable, match="dc index vector rebuild"): + Retriever(store, embedder=WordVectors()) + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert ( + main([*command, "library", "search", "--query", "ice", "--mode", "vector"]) == 1 + ) + assert "dc index vector rebuild" in capsys.readouterr().err + + +def test_stale_vector_search_warns_without_rebuilding(tmp_path, capsys, monkeypatch): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + store.put( + principal, + "docling://resources/notes/one", + _package("ice"), + ) + TurbovecIndex(store, WordVectors(), force_rebuild=True) + generation = store.db.execute("SELECT generation FROM vector_state").fetchone()[ + 0 + ] + store.put( + principal, + "docling://resources/notes/two", + _package("basalt"), + ) + result = Retriever(store, embedder=WordVectors()).search( + principal, "ice", mode="vector" + ) + assert result.hits + assert result.warnings and "dc index vector rebuild" in result.warnings[0] + monkeypatch.setattr( + "docling_context.cli.stored_embedding_provider", + lambda _model: WordVectors(), + ) + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert ( + main( + [ + *command, + "library", + "search", + "--query", + "ice", + "--mode", + "vector", + "--uri", + "docling://resources/", + "--json", + ] + ) + == 0 + ) + output = capsys.readouterr() + assert "dc index vector rebuild" in output.err + assert json.loads(output.out.splitlines()[0])["uri"].endswith("/one") + monkeypatch.setattr( + "docling_context.embeddings.stored_embedding_provider", + lambda _model: WordVectors(), + ) + service_result = ( + ContextService(store, principal) + .search(SearchRequest("ice", mode="vector")) + .data + ) + assert service_result["warnings"] == list(result.warnings) + assert ( + store.db.execute("SELECT generation FROM vector_state").fetchone()[0] + == generation + ) + assert store.index_status(principal)["vector_snapshot_current"] is False + store.db.execute("DELETE FROM retrieval_fts") + combined = Retriever(store, embedder=WordVectors()).search( + principal, "ice", mode="vector" + ) + assert len(combined.warnings) == 2 + assert "dc index lex rebuild" in combined.warnings[0] + assert "dc index vector rebuild" in combined.warnings[1] + (tmp_path / f"vectors-{generation}.tvim").write_bytes(b"corrupt") + with pytest.raises(VectorStorageUnavailable, match="dc index vector rebuild"): + Retriever(store, embedder=WordVectors()) + + +def test_search_warns_without_repairing_missing_lexical_records(tmp_path, capsys): + principal = Principal("north", "alice") + uri = "docling://resources/notes/one" + with LocalContextStore(tmp_path) as store: + store.put( + principal, uri, _package("amber fossil") + ) + store.db.execute("DELETE FROM retrieval_fts") + store.db.execute("DELETE FROM retrieval_units") + with LocalContextStore(tmp_path) as store: + result = Retriever(store).search(principal, "fossil", mode="lexical") + assert result.hits == () + assert result.warnings and "dc index lex rebuild" in result.warnings[0] + assert store.index_status(principal)["indexed_documents"] == 0 + assert ( + store.db.execute("SELECT COUNT(*) FROM retrieval_units").fetchone()[0] == 0 + ) + service_result = ( + ContextService(store, principal) + .search(SearchRequest("fossil", mode="lexical")) + .data + ) + assert service_result["warnings"] == list(result.warnings) + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert ( + main([*command, "library", "search", "--query", "fossil", "--mode", "lexical"]) + == 0 + ) + assert "dc index lex rebuild" in capsys.readouterr().err + + +def test_proposed_memory_is_not_reindexed_on_each_search(tmp_path, monkeypatch): + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + MemoryService(store).create(principal, "fact", "Unreviewed claim") + assert store.index_status(principal)["indexed_documents"] == 1 + monkeypatch.setattr( + "docling_context.local.index_package", + lambda *_args: pytest.fail("already indexed proposed memory"), + ) + assert Retriever(store).search(principal, "claim", mode="lexical").hits == () + + def test_lexical_scope_exact_citations_and_budget(tmp_path): alice = Principal("north", "alice") bob = Principal("south", "bob") @@ -167,7 +313,7 @@ def test_replacement_delete_and_stale_row_are_never_cited(tmp_path): assert retrieval.search(principal, "basalt", mode="lexical").hits == () -def test_cli_search_and_schema_backfill(tmp_path, capsys): +def test_cli_search_warns_until_lexical_rebuild(tmp_path, capsys): principal = Principal("north", "alice") uri = "docling://resources/notes/one" with LocalContextStore(tmp_path) as store: @@ -177,7 +323,12 @@ def test_cli_search_and_schema_backfill(tmp_path, capsys): store.db.execute("DELETE FROM retrieval_fts") store.db.execute("DELETE FROM retrieval_units") with LocalContextStore(tmp_path) as store: - assert Retriever(store).search(principal, "fossil", mode="lexical").hits + result = Retriever(store).search(principal, "fossil", mode="lexical") + assert result.hits == () + assert result.warnings + command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] + assert main([*command, "index", "lex", "rebuild", "--json"]) == 0 + capsys.readouterr() assert ( main( [ @@ -187,7 +338,9 @@ def test_cli_search_and_schema_backfill(tmp_path, capsys): "north", "--user", "alice", + "library", "search", + "--query", "fossil", "--uri", "docling://resources/notes", @@ -206,7 +359,9 @@ def test_cli_search_and_schema_backfill(tmp_path, capsys): "north", "--user", "alice", + "library", "search", + "--query", "fossil", "--uri", "docling://resources/notes", @@ -258,7 +413,9 @@ def test_search_reports_matching_nodes_before_result_limit(tmp_path, capsys): "north", "--user", "alice", + "library", "search", + "--query", "fossil", "--uri", "docling://resources/notes", @@ -267,7 +424,7 @@ def test_search_reports_matching_nodes_before_result_limit(tmp_path, capsys): ] assert main(command) == 0 assert "Showing 1 of 3 matching nodes." in capsys.readouterr().out - assert main(command[:7] + ["missing"] + command[8:]) == 0 + assert main(command[:9] + ["missing"] + command[10:]) == 0 assert "Showing 0 of 0 matching nodes." in capsys.readouterr().out @@ -298,16 +455,18 @@ def test_index_status_and_rebuild_restore_search_records(tmp_path, capsys): assert main([*command, "index", "status", "--json"]) == 0 status = json.loads(capsys.readouterr().out) assert status["documents"] == 1 - assert status["indexed_documents"] == 1 + assert status["indexed_documents"] == 0 assert status["embedded_nodes"] == 0 assert status["vector_snapshot_current"] is False with LocalContextStore(tmp_path) as store: - assert not Retriever(store).search(alice, "fossil", mode="lexical").hits + result = Retriever(store).search(alice, "fossil", mode="lexical") + assert not result.hits + assert result.warnings and "dc index lex rebuild" in result.warnings[0] assert main([*command, "index", "lex", "status"]) == 0 assert "Indexed documents" in capsys.readouterr().out assert main([*command, "index", "lex", "status", "--json"]) == 0 assert "embedded_nodes" not in json.loads(capsys.readouterr().out) - assert main([*command, "index", "lex", "rebuild"]) == 0 + assert main([*command, "index", "lex", "rebuild", "--json"]) == 0 rebuilt = json.loads(capsys.readouterr().out) assert rebuilt["rebuilt_documents"] == 1 assert rebuilt["indexed_nodes"] >= 1 @@ -321,24 +480,71 @@ def test_index_status_and_rebuild_restore_search_records(tmp_path, capsys): ) -def test_lexical_search_filters_before_candidate_limit(tmp_path, monkeypatch): - monkeypatch.setattr("docling_context.retrieval._MAX_CANDIDATES", 1) +def test_lexical_and_vector_search_reach_nodes_beyond_ten_thousand(tmp_path): principal = Principal("north", "alice") with LocalContextStore(tmp_path) as store: - store.put( + first = store.put( principal, "docling://resources/notes/first", - _package("quartz"), + _package("fossil"), ) - store.put( + first_id = store.db.execute( + "SELECT MAX(vector_id) FROM retrieval_units" + ).fetchone()[0] + filler = [ + ( + first_id + number + 1, + principal.tenant_id, + first.uri, + first.document_id, + first.revision_id, + f"/doclang[1]/text[{number + 2}]", + 2, + "fossil", + first.updated_at, + ) + for number in range(10_001) + ] + with store.transaction(): + store.db.executemany( + """INSERT INTO retrieval_units + (vector_id,tenant_id,uri,document_id,revision_id,xpath,tier,text,updated_at) + VALUES (?,?,?,?,?,?,?,?,?)""", + filler, + ) + store.db.executemany( + "INSERT INTO retrieval_fts(rowid,text) VALUES (?,?)", + ((row[0], "fossil") for row in filler), + ) + store.db.execute( + "UPDATE lexical_state SET unit_count=(SELECT COUNT(*) FROM retrieval_units WHERE uri=?) WHERE uri=?", + (first.uri, first.uri), + ) + target = store.put( principal, "docling://resources/notes/second", - _package("fossil"), + _package("fossil fossil fossil fossil"), ) - result = Retriever(store).search(principal, "fossil", mode="lexical") - assert result.total_hits == 1 - assert len(result.hits) == 1 - assert result.hits[0].uri.endswith("/second") + target_id = store.db.execute( + "SELECT MAX(vector_id) FROM retrieval_units WHERE uri=?", (target.uri,) + ).fetchone()[0] + + class TargetIndex: + def search(self, _query, allowed_ids, _k): + assert target_id in allowed_ids + return [(target_id, 0.9)] + + lexical = Retriever(store).search(principal, "fossil", mode="lexical", k=1) + assert lexical.total_hits > 10_000 + assert lexical.hits[0].uri == target.uri + vector = Retriever( + store, embedder=WordVectors(), vector_index=TargetIndex() + ).search(principal, "fossil", mode="vector", k=1) + assert vector.hits[0].uri == target.uri + hybrid = Retriever( + store, embedder=WordVectors(), vector_index=TargetIndex() + ).search(principal, "fossil", mode="hybrid", k=1) + assert hybrid.hits[0].uri == target.uri def test_vector_snapshot_restart_corruption_and_generation(tmp_path): @@ -357,7 +563,7 @@ def test_vector_snapshot_restart_corruption_and_generation(tmp_path): uri, _package("glacier private south"), ) - retrieval = Retriever(store, embedder=provider) + retrieval = _maintenance_retriever(store, provider) assert isinstance(retrieval.vector_index, TurbovecIndex) result = retrieval.search(principal, "glacier", mode="vector") assert result.hits[0].revision_id == first.revision_id @@ -367,12 +573,12 @@ def test_vector_snapshot_restart_corruption_and_generation(tmp_path): snapshot = tmp_path / f"vectors-{generation}.tvim" assert snapshot.exists() with LocalContextStore(tmp_path) as store: - retrieval = Retriever(store, embedder=provider) + retrieval = _maintenance_retriever(store, provider) assert retrieval.vector_index.generation == generation assert retrieval.search(principal, "glacier", mode="vector").hits snapshot.write_bytes(b"bad snapshot") with LocalContextStore(tmp_path) as store: - retrieval = Retriever(store, embedder=provider) + retrieval = _maintenance_retriever(store, provider) assert retrieval.vector_index.generation != generation assert ( retrieval.search(principal, "glacier", mode="vector").hits[0].text @@ -392,7 +598,7 @@ def test_vector_snapshot_restart_corruption_and_generation(tmp_path): assert retrieval.search(principal, "glacier", mode="vector").hits == () with LocalContextStore(tmp_path) as store: assert ( - Retriever(store, embedder=provider) + _maintenance_retriever(store, provider) .search(principal, "glacier", mode="vector") .hits == () @@ -401,7 +607,7 @@ def test_vector_snapshot_restart_corruption_and_generation(tmp_path): class NewModel(WordVectors): model_id = "word-hash-2" - changed = Retriever(store, embedder=NewModel()) + changed = _maintenance_retriever(store, NewModel()) state = store.db.execute( "SELECT model_id FROM vector_state WHERE singleton=1" ).fetchone() @@ -433,7 +639,7 @@ def embed(self, texts: list[str]) -> list[list[float]]: "docling://resources/notes/two", _package("basalt lava"), ) - retriever = Retriever(store, embedder=provider) + retriever = _maintenance_retriever(store, provider) initial = retriever.vector_index.generation monkeypatch.setattr( retriever.vector_index, @@ -473,12 +679,12 @@ def test_legacy_vector_snapshot_rebuilds_membership_on_upgrade(tmp_path): "docling://resources/notes/one", _package("glacier ice"), ) - old_generation = Retriever( - store, embedder=WordVectors() + old_generation = _maintenance_retriever( + store, WordVectors() ).vector_index.generation store.db.executescript("DROP TABLE vector_members; PRAGMA user_version=3;") with LocalContextStore(tmp_path) as store: - retriever = Retriever(store, embedder=WordVectors()) + retriever = _maintenance_retriever(store, WordVectors()) assert retriever.vector_index.generation != old_generation assert retriever.search(principal, "glacier", mode="vector").hits assert store.db.execute("SELECT COUNT(*) FROM vector_members").fetchone()[0] > 0 @@ -493,15 +699,22 @@ def test_existing_retrieval_index_adds_picture_column_on_upgrade(tmp_path): _package("glacier ice"), ) store.db.executescript( + "DROP TABLE lexical_state; " "ALTER TABLE retrieval_units DROP COLUMN asset_path; PRAGMA user_version=4;" ) with LocalContextStore(tmp_path) as store: - assert store.db.execute("PRAGMA user_version").fetchone()[0] == 8 + assert store.db.execute("PRAGMA user_version").fetchone()[0] == 10 assert "asset_path" in { row["name"] for row in store.db.execute("PRAGMA table_info(retrieval_units)") } - assert Retriever(store).search(principal, "glacier", mode="lexical").hits + stale = Retriever(store).search(principal, "glacier", mode="lexical") + assert stale.hits + assert stale.warnings and "dc index lex rebuild" in stale.warnings[0] + store.rebuild_index(principal) + assert ( + Retriever(store).search(principal, "glacier", mode="lexical").warnings == () + ) def test_section_xpath_tracks_heading_without_changing_xml_parent(tmp_path): @@ -530,7 +743,7 @@ def test_section_xpath_tracks_heading_without_changing_xml_parent(tmp_path): ] -def test_matching_outline_precedes_body_when_budget_allows(tmp_path): +def test_hits_rank_by_total_score_across_tiers(tmp_path): principal = Principal("north", "alice") with LocalContextStore(tmp_path) as store: Ingestor(store).add_resource( @@ -545,7 +758,34 @@ def test_matching_outline_precedes_body_when_budget_allows(tmp_path): ) hits = Retriever(store).search(principal, "glacier", mode="lexical", k=4).hits assert hits - assert [hit.tier for hit in hits] == sorted(hit.tier for hit in hits) + assert [hit.score for hit in hits] == sorted( + (hit.score for hit in hits), reverse=True + ) + + +def test_vector_hits_rank_by_displayed_score(tmp_path): + pytest.importorskip("turbovec") + principal = Principal("north", "alice") + with LocalContextStore(tmp_path) as store: + Ingestor(store).add_resource( + principal, + "docling://resources/notes/vector-order", + _package( + "Glacier overview" + "Glacier glacier glacier glacier" + ), + filename="vector-order.dclx", + ) + TurbovecIndex(store, WordVectors(), force_rebuild=True) + hits = ( + Retriever(store, embedder=WordVectors()) + .search(principal, "glacier", mode="vector", k=10) + .hits + ) + assert len(hits) >= 2 + assert [hit.score for hit in hits] == sorted( + (hit.score for hit in hits), reverse=True + ) def test_search_decodes_each_cited_package_once(tmp_path, monkeypatch): @@ -616,7 +856,7 @@ def embed_text_image(self, text: str, image: bytes): with LocalContextStore(tmp_path) as store: store.put(principal, "docling://resources/images/one", _image_package()) provider = ImageVectors() - retriever = Retriever(store, embedder=provider) + retriever = _maintenance_retriever(store, provider) assert ("image", b"first image") in provider.calls assert ("mixed", "Rock diagram", b"second image") in provider.calls hit = retriever.search(principal, "", image=b"query", mode="vector", k=1).hits[ @@ -625,7 +865,7 @@ def embed_text_image(self, text: str, image: bytes): assert hit.xpath == "/doclang[1]/picture[1]" assert hit.asset_path == "assets/one.png" assert hit.text == "" - text_only = Retriever(store, embedder=WordVectors()) + text_only = _maintenance_retriever(store, WordVectors()) assert text_only.search(principal, "", image=b"query", mode="vector").hits == () assert ( store.db.execute( @@ -645,7 +885,7 @@ def test_rebuild_index_refreshes_existing_vector_retriever(tmp_path): "docling://resources/notes/one", _package("glacier ice"), ) - retriever = Retriever(store, embedder=WordVectors()) + retriever = _maintenance_retriever(store, WordVectors()) first = retriever.search(principal, "glacier", mode="vector") assert first.hits and first.hits[0].vector_score > 0 old_generation = retriever.vector_index.generation @@ -698,7 +938,10 @@ def provider_for_cli(_model, *, backend, offline): lambda *_args, **_kwargs: WordVectors(), ) command = ["--store", str(tmp_path), "--tenant", "north", "--user", "alice"] - assert main([*command, "index", "vector", "rebuild", "--backend", "onnx"]) == 0 + assert ( + main([*command, "index", "vector", "rebuild", "--backend", "onnx", "--json"]) + == 0 + ) assert selected_backends == ["onnx"] assert json.loads(capsys.readouterr().out)["embedded_nodes"] > 0 assert main([*command, "index", "vector", "status"]) == 0 @@ -706,7 +949,7 @@ def provider_for_cli(_model, *, backend, offline): assert "Vector model" in vector_table and "word-hash-1" in vector_table assert main([*command, "index", "vector", "status", "--json"]) == 0 assert json.loads(capsys.readouterr().out)["vector_model"] == "word-hash-1" - assert main([*command, "search", "glacier"]) == 0 + assert main([*command, "library", "search", "--query", "glacier"]) == 0 output = capsys.readouterr().out assert "Showing" in output and "ranked nodes" in output with LocalContextStore(tmp_path) as store: diff --git a/tests/test_service_mcp.py b/tests/test_service_mcp.py index 74ae0e7..a5bbfa6 100644 --- a/tests/test_service_mcp.py +++ b/tests/test_service_mcp.py @@ -62,8 +62,10 @@ def test_service_citations_survive_worker_restart(tmp_path, capsys): PRINCIPAL.tenant_id, "--user", PRINCIPAL.user_id, + "library", "jobs", "status", + "--id", queued["job_id"], "--json", ] @@ -81,9 +83,9 @@ def test_service_citations_survive_worker_restart(tmp_path, capsys): assert search["hits"][0]["revision_id"] == finished.revision_id assert "Koonap Formation" in search["context"] assert ( - service.show( - ShowRequest(URI, "@toc:/doclang[1]/toc[1]/entry[1]/description[1]") - ).data["content"] + service.show(ShowRequest(URI, "@toc:/doclang[1]/heading[1]")).data[ + "content" + ] == "Karoo" ) assert ( @@ -105,8 +107,11 @@ def test_service_citations_survive_worker_restart(tmp_path, capsys): PRINCIPAL.tenant_id, "--user", PRINCIPAL.user_id, + "library", "show", + "--uri", URI, + "--xpath", "/doclang[1]/text[1]", "--revision", finished.revision_id, @@ -123,6 +128,25 @@ def test_service_citations_survive_worker_restart(tmp_path, capsys): key: citation[key] for key in ("uri", "revision_id", "xpath", "content", "truncated") } + show_args = [ + "--store", + str(tmp_path), + "--tenant", + PRINCIPAL.tenant_id, + "--user", + PRINCIPAL.user_id, + "library", + "show", + "--uri", + URI, + "--xpath", + "/doclang[1]/text[1]", + ] + assert dc_main(show_args) == 0 + table = capsys.readouterr().out + assert "Field" in table and "Content" in table and "Koonap Formation" in table + assert dc_main([*show_args, "--raw"]) == 0 + assert capsys.readouterr().out == "Koonap Formation\n" assert ( "Koonap Formation" in service.show(ShowRequest(URI, "/doclang[1]/text[1]", format="xml")).data[ @@ -232,6 +256,110 @@ def test_mcp_tools_and_http_token(tmp_path): assert remote.list(ListRequest()).data == service.list(ListRequest()).data +def test_project_and_resource_operations_share_sqlite_catalog(tmp_path): + with LocalContextStore(tmp_path) as store: + service = ContextService(store, PRINCIPAL) + created = call_service( + service, "project_create", {"project_id": "geology", "title": "Geology"} + ) + assert created["data"]["project_id"] == "geology" + record = Ingestor(store).add_resource( + PRINCIPAL, + "docling://resources/library/karoo", + _package(), + filename="karoo.dclx", + ) + from docling_context.catalog import Catalog + + Catalog(store).register_resource(PRINCIPAL, record, "library") + linked = call_service( + service, + "project_link", + {"project_id": "geology", "uri": record.uri}, + ) + assert linked["data"]["uri"] == record.uri + assert call_service(service, "overview", {})["data"][0]["documents"] == 1 + assert ( + call_service(service, "resource_projects", {"uri": record.uri})["data"][0][ + "project_id" + ] + == "geology" + ) + assert ( + call_service(service, "project_resources", {"project_id": "geology"})[ + "data" + ][0]["uri"] + == record.uri + ) + assert ( + call_service( + service, + "project_unlink", + {"project_id": "geology", "uri": record.uri}, + )["data"]["project_id"] + == "geology" + ) + assert call_service(service, "overview", {})["data"][0]["documents"] == 0 + assert ( + call_service(service, "resource_projects", {"uri": record.uri})["data"] + == [] + ) + assert store.get_record(PRINCIPAL, record.uri).revision_id == record.revision_id + started = call_service( + service, + "project_session_start", + {"project_id": "geology", "session_id": "review", "agent": "codex"}, + ) + assert started["data"]["uri"] == "docling://projects/geology/sessions/review" + assert call_service(service, "overview", {})["data"][0]["sessions"] == 1 + closed = call_service( + service, + "project_session_close", + {"project_id": "geology", "session_id": "review"}, + ) + assert closed["data"]["status"] == "closed" + server = create_mcp(service) + names = {tool.name for tool in asyncio.run(server.list_tools())} + assert { + "project_show", + "project_resources", + "resource_links", + "knowledge_facts", + } <= names + assert "project_link" not in names + writable = create_mcp(service, allow_writes=True) + assert "project_link" in { + tool.name for tool in asyncio.run(writable.list_tools()) + } + + +def test_mcp_search_defaults_to_launched_project(tmp_path, monkeypatch): + from docling_context.catalog import Catalog + + with LocalContextStore(tmp_path) as store: + catalog = Catalog(store) + catalog.create_project(PRINCIPAL, "geology", "Geology") + for name in ("linked", "other"): + document = DocLangXDocument() + assert document.read_xml(f"Koonap {name}") + record = Ingestor(store).add_resource( + PRINCIPAL, + f"docling://resources/library/{name}", + document.write_bytes(), + filename=f"{name}.dclx", + ) + catalog.register_resource(PRINCIPAL, record, "library") + if name == "linked": + catalog.link(PRINCIPAL, "geology", record.uri) + monkeypatch.setenv("DOCLING_CONTEXT_PROJECT", "geology") + server = create_mcp(ContextService(store, PRINCIPAL)) + result = asyncio.run(server.call_tool("search", {"query": "Koonap"})) + assert result[1]["data"]["hits"] + assert {hit["uri"] for hit in result[1]["data"]["hits"]} == { + "docling://resources/library/linked" + } + + def test_failed_ingest_reports_stable_conversion_code(tmp_path): with LocalContextStore(tmp_path) as store: service = ContextService(store, PRINCIPAL) diff --git a/tests/test_session_memory_cli.py b/tests/test_session_memory_cli.py index 659dbe3..e8dd800 100644 --- a/tests/test_session_memory_cli.py +++ b/tests/test_session_memory_cli.py @@ -6,7 +6,11 @@ import json import sys +from docling_context.catalog import Catalog from docling_context.cli import main +from docling_context.local import LocalContextStore +from docling_context.models import Principal +from docling_context.sessions import SessionStore def _run(capsys, root, *args): @@ -15,8 +19,68 @@ def _run(capsys, root, *args): return result, output.out, output.err +def test_authored_memory_preserves_body_and_project_scope(tmp_path, capsys): + root = tmp_path / "store" + source = tmp_path / "conclusion.dclg" + source.write_text( + "ConclusionKhurmala is porous." + "Evidence from the formation study." + ) + principal = Principal("acme", "alice") + with LocalContextStore(root) as store: + Catalog(store).create_project(principal, "oilgas", "Oil and Gas") + Catalog(store).create_project(principal, "materials", "Materials") + code, output, error = _run( + capsys, + root, + "memory", + "create", + "--from", + str(source), + "--project", + "oilgas", + "--json", + ) + assert code == 0, error + memory = json.loads(output) + code, output, error = _run( + capsys, root, "memory", "accept", "--uri", memory["uri"], "--json" + ) + assert code == 0, error + with LocalContextStore(root) as store: + document = store.get_document(principal, memory["uri"]) + assert "Evidence from the formation study." in document.xml() + code, output, error = _run( + capsys, + root, + "memory", + "search", + "--query", + "Khurmala", + "--project", + "oilgas", + "--json", + ) + assert code == 0, error + assert len(json.loads(output)) == 1 + code, output, error = _run( + capsys, + root, + "memory", + "search", + "--query", + "Khurmala", + "--project", + "materials", + "--json", + ) + assert code == 0, error + assert json.loads(output) == [] + + def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypatch): - assert _run(capsys, tmp_path, "session", "start", "--id", "chat", "--json")[0] == 0 + with LocalContextStore(tmp_path) as store: + SessionStore(store).start(Principal("acme", "alice"), "chat") monkeypatch.setenv("DOCLING_CONTEXT_SESSION", "chat") monkeypatch.setattr(sys, "stdin", io.StringIO("Remember: I prefer concise replies")) attachment = tmp_path / "source.bin" @@ -52,7 +116,9 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat capsys, tmp_path, "session", "list", "--offset", "1", "--json" ) assert code == 0 and json.loads(output) == [] - code, output, _ = _run(capsys, tmp_path, "session", "replay", "chat", "--json") + code, output, _ = _run( + capsys, tmp_path, "session", "replay", "--id", "chat", "--json" + ) assert code == 0 assert json.loads(output)[0]["text"] == "Remember: I prefer concise replies" restored = tmp_path / "restored.bin" @@ -62,7 +128,9 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat tmp_path, "session", "attachment", + "--id", "chat", + "--key", "feedback-1", "--output", str(restored), @@ -81,18 +149,21 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat "bob", "session", "show", + "--id", "docling://users/acme/alice/sessions/chat", "--json", ] ) assert code == 1 and "user does not match" in capsys.readouterr().err - assert _run(capsys, tmp_path, "session", "close", "chat", "--json")[0] == 0 + assert _run(capsys, tmp_path, "session", "close", "--id", "chat", "--json")[0] == 0 code, _, error = _run( capsys, tmp_path, "session", "append", + "--id", "chat", + "--text", "too late", "--kind", "turn", @@ -106,7 +177,9 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat assert code == 0 and len(json.loads(output)) == 1 code, output, _ = _run(capsys, tmp_path, "memory", "worker", "--once", "--json") assert code == 0 and json.loads(output)["status"] == "completed" - code, output, _ = _run(capsys, tmp_path, "session", "jobs", "chat", "--json") + code, output, _ = _run( + capsys, tmp_path, "session", "jobs", "--id", "chat", "--json" + ) assert code == 0 and {job["status"] for job in json.loads(output)} == { "completed", "superseded", @@ -115,7 +188,9 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat proposal = json.loads(output)[0] assert code == 0 and proposal["status"] == "proposed" uri = proposal["uri"] - code, output, _ = _run(capsys, tmp_path, "grep", "concise", "--uri", uri) + code, output, _ = _run( + capsys, tmp_path, "library", "grep", "--term", "concise", "--uri", uri, "--json" + ) assert code == 0 and not output code, output, _ = _run( capsys, tmp_path, "memory", "list", "--kind", "preference", "--json" @@ -123,38 +198,54 @@ def test_session_lifecycle_listing_and_memory_review(tmp_path, capsys, monkeypat assert code == 0 and len(json.loads(output)) == 1 code, output, _ = _run(capsys, tmp_path, "memory", "list", "--until", day, "--json") assert code == 0 and len(json.loads(output)) == 1 - assert _run(capsys, tmp_path, "memory", "accept", uri, "--json")[0] == 0 - code, output, _ = _run(capsys, tmp_path, "grep", "concise", "--uri", uri) + assert _run(capsys, tmp_path, "memory", "accept", "--uri", uri, "--json")[0] == 0 + code, output, _ = _run( + capsys, tmp_path, "library", "grep", "--term", "concise", "--uri", uri, "--json" + ) assert code == 0 and output - code, output, _ = _run(capsys, tmp_path, "memory", "search", "concise", "--json") + code, output, _ = _run( + capsys, tmp_path, "memory", "search", "--query", "concise", "--json" + ) assert code == 0 and json.loads(output)[0]["uri"] == uri - code, output, _ = _run(capsys, tmp_path, "memory", "recall", "concise", "--json") + code, output, _ = _run( + capsys, tmp_path, "memory", "recall", "--query", "concise", "--json" + ) assert code == 0 and len(json.loads(output)) == 1 - code, output, _ = _run(capsys, tmp_path, "memory", "recall", "concise", "--json") + code, output, _ = _run( + capsys, tmp_path, "memory", "recall", "--query", "concise", "--json" + ) assert code == 0 and json.loads(output) == [] - code, output, _ = _run(capsys, tmp_path, "status", "--json") + code, output, _ = _run(capsys, tmp_path, "overview", "--json") counts = json.loads(output) - assert code == 0 and counts["sessions"] == 1 and counts["memories"] == 1 - assert counts["memory_jobs"] >= 1 - assert _run(capsys, tmp_path, "memory", "delete", uri, "--json")[0] == 0 + assert ( + code == 0 + and counts["counts"]["sessions"] == 1 + and counts["counts"]["memories"] == 1 + ) + assert counts["counts"]["memory_jobs"] >= 1 + assert _run(capsys, tmp_path, "memory", "delete", "--uri", uri, "--json")[0] == 0 code, output, _ = _run( capsys, tmp_path, "memory", "list", "--status", "deleted", "--json" ) assert code == 0 and len(json.loads(output)) == 1 - code, output, _ = _run(capsys, tmp_path, "grep", "concise", "--uri", uri) + code, output, _ = _run( + capsys, tmp_path, "library", "grep", "--term", "concise", "--uri", uri, "--json" + ) assert code == 0 and not output - assert _run(capsys, tmp_path, "memory", "purge", uri, "--json")[0] == 0 + assert _run(capsys, tmp_path, "memory", "purge", "--uri", uri, "--json")[0] == 0 code, output, _ = _run(capsys, tmp_path, "memory", "list", "--json") assert code == 0 and json.loads(output) == [] -def test_manual_memory_filters_and_user_scope(tmp_path, capsys): +def test_manual_memory_filters_and_shared_tenant_scope(tmp_path, capsys): code, output, _ = _run( capsys, tmp_path, "memory", "create", + "--kind", "concept", + "--claim", "Karoo Basin", "--accepted", "--json", @@ -170,7 +261,15 @@ def test_manual_memory_filters_and_user_scope(tmp_path, capsys): ) assert code == 0 and len(json.loads(output)) == 1 code, output, _ = _run( - capsys, tmp_path, "memory", "correct", uri, "Karoo Supergroup", "--json" + capsys, + tmp_path, + "memory", + "correct", + "--uri", + uri, + "--claim", + "Karoo Supergroup", + "--json", ) assert code == 0 and json.loads(output)["claim"] == "Karoo Supergroup" code = main( @@ -186,7 +285,7 @@ def test_manual_memory_filters_and_user_scope(tmp_path, capsys): "--json", ] ) - assert code == 0 and json.loads(capsys.readouterr().out) == [] + assert code == 0 and json.loads(capsys.readouterr().out)[0]["uri"] == uri code = main( [ "--store", diff --git a/tests/test_sessions_memory.py b/tests/test_sessions_memory.py index ede53a0..b489d59 100644 --- a/tests/test_sessions_memory.py +++ b/tests/test_sessions_memory.py @@ -137,17 +137,19 @@ def test_compilation_review_correction_recall_and_source_cascade(tmp_path): ) assert compiler.run_once().status == "completed" assert store.db.execute("SELECT count(*) FROM memory_index").fetchone()[0] == 1 - for principal in (bob, other): - with pytest.raises(PermissionError): - memories.get(principal, uri) - with pytest.raises(PermissionError): - memories.accept(principal, uri) - with pytest.raises(PermissionError): - memories.correct(principal, uri, "wrong") - assert memories.search(principal, "Koonap") == () - assert memories.list(principal, status="accepted") == () - with pytest.raises(PermissionError): - compiler.jobs(principal, sessions.uri(alice, "chat")) + assert memories.get(bob, uri).uri == uri + assert memories.search(bob, "Koonap")[0].uri == uri + assert memories.list(bob, status="accepted")[0].uri == uri + with pytest.raises(PermissionError): + memories.accept(bob, uri) + with pytest.raises(PermissionError): + memories.correct(bob, uri, "wrong") + with pytest.raises((KeyError, PermissionError)): + memories.get(other, uri) + assert memories.search(other, "Koonap") == () + assert memories.list(other, status="accepted") == () + with pytest.raises(PermissionError): + compiler.jobs(bob, sessions.uri(alice, "chat")) sessions.delete(alice, "chat") assert memories.get(alice, uri).status == "proposed" assert memories.search(alice, "Koonap") == () diff --git a/tests/test_source_urls.py b/tests/test_source_urls.py new file mode 100644 index 0000000..74e7066 --- /dev/null +++ b/tests/test_source_urls.py @@ -0,0 +1,153 @@ +"""Remote document download and shared-library import behavior.""" + +from __future__ import annotations + +import json + +import httpx +import pytest +from doclang import DocLangXDocument + +from docling_context import ConversionResult, LocalContextStore, Principal +from docling_context.catalog import Catalog +from docling_context.cli import main +from docling_context.source_urls import download_source, is_source_url + + +def test_extensionless_pdf_url_downloads_with_a_conversion_filename(): + url = "https://arxiv.org/pdf/2608.23083" + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "application/pdf", "content-length": "8"}, + content=b"%PDF-1.4", + request=request, + ) + ) + data, filename = download_source(url, transport=transport) + assert data == b"%PDF-1.4" + assert filename == "2608.23083.pdf" + + generic_type = httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "application/octet-stream"}, + content=b"%PDF-1.4", + request=request, + ) + ) + assert download_source(url, transport=generic_type)[1] == "2608.23083.pdf" + + +def test_download_rejects_oversized_and_unsupported_sources(): + url = "https://example.test/file.pdf" + advertised = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-length": "100"}, content=b"x", request=request + ) + ) + with pytest.raises(ValueError, match="size limit"): + download_source(url, max_bytes=8, transport=advertised) + streamed = httpx.MockTransport( + lambda request: httpx.Response(200, content=b"x" * 9, request=request) + ) + with pytest.raises(ValueError, match="size limit"): + download_source(url, max_bytes=8, transport=streamed) + with pytest.raises(ValueError, match="HTTP or HTTPS"): + is_source_url("file:///etc/passwd") + with pytest.raises(ValueError, match="credentials"): + is_source_url("https://alice:secret@example.test/file.pdf") + + +def test_download_follows_redirect_and_uses_content_disposition(): + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/paper": + return httpx.Response( + 302, headers={"location": "/files/123"}, request=request + ) + return httpx.Response( + 200, + headers={"content-disposition": 'attachment; filename="paper.pdf"'}, + content=b"%PDF-1.4", + request=request, + ) + + assert download_source( + "https://example.test/paper", transport=httpx.MockTransport(respond) + ) == (b"%PDF-1.4", "paper.pdf") + + +def test_url_import_reuses_library_document_and_links_project( + tmp_path, monkeypatch, capsys +): + url = "https://arxiv.org/pdf/2608.23083" + source = b"%PDF-1.4 mocked source" + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "application/pdf"}, + content=source, + request=request, + ) + ) + monkeypatch.setattr( + "docling_context.cli.download_source", + lambda value: download_source(value, transport=transport), + ) + monkeypatch.setattr( + "docling_context.cli._source_page_units", lambda source, filename: 1 + ) + + class Converter: + def __init__(self, _config): + pass + + def fingerprint_options(self): + return {} + + def convert(self, data, filename): + assert data == source and filename == "2608.23083.pdf" + document = DocLangXDocument() + assert document.read_xml("Material study") + return ConversionResult(document.write_bytes(), "test", "1") + + monkeypatch.setattr("docling_context.cli.LocalDoclingConverter", Converter) + base = ["--store", str(tmp_path / "store")] + assert ( + main( + [ + *base, + "project", + "create", + "--project", + "materials", + "--title", + "Materials", + ] + ) + == 0 + ) + capsys.readouterr() + command = [ + *base, + "library", + "add", + "--source", + url, + "--project", + "materials", + "--from", + "pdf", + "--json", + ] + assert main(command) == 0 + first = json.loads(capsys.readouterr().out) + assert first["uri"].startswith("docling://resources/library/") + assert main(command) == 0 + second = json.loads(capsys.readouterr().out) + assert second["uri"] == first["uri"] + with LocalContextStore(tmp_path / "store") as store: + principal = Principal("default", "local") + record = store.get_record(principal, first["uri"]) + assert record.source["origin_uri"] == url + assert Catalog(store).overview(principal)[0]["documents"] == 1 diff --git a/tests/test_storage.py b/tests/test_storage.py index 5dc7097..1b70424 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -7,7 +7,7 @@ from zipfile import ZipFile import pytest -from doclang import DoclangDocument, DocLangXDocument +from doclang import DoclangDocument, DocLangXDocument, DocLangXDocumentRole from docling_context import ( AccessDenied, @@ -37,8 +37,9 @@ def package(text: str = "Body") -> bytes: assert document.read_xml(f'{text}') document.set_part_bytes("assets/keep.bin", b"opaque", "application/octet-stream") document.set_part_text("context/source.json", '{"id":1}', "application/json") - assert document.set_document_summary( - "Short summary" + assert document.set( + DocLangXDocumentRole.SUMMARY, + DoclangDocument("Short summary"), ) return document.write_bytes() @@ -55,7 +56,8 @@ def test_doclang_native_smoke_and_package_round_trip(): restored = DocLangXDocument() assert restored.read_bytes(rewritten) assert restored.at(xpath=XPATH) == "Body" - assert restored.summary().at(xpath=XPATH) == "Short summary" + summary = restored.get(DocLangXDocumentRole.SUMMARY) + assert summary is not None and summary.at(xpath=XPATH) == "Short summary" with ZipFile(io.BytesIO(rewritten)) as archive: assert archive.read("document.xml").startswith(b"=2.134.0,<3" }, + { name = "doclang", git = "https://github.com/doclang-project/doclang?rev=24b7ca6cc96e83af25ff9d67903e03c6d500c5e4" }, + { name = "docling", extras = ["rapidocr"], marker = "extra == 'conversion'", specifier = ">=2.138.0,<3" }, + { name = "docling-parse", marker = "extra == 'conversion'", specifier = ">=7.22.3,<8" }, { name = "fastembed", marker = "extra == 'vectors'", specifier = ">=0.8.1,<1" }, { name = "httpx", specifier = ">=0.27,<1" }, { name = "mcp", specifier = ">=1.20,<2" }, @@ -736,7 +738,7 @@ dev = [ [[package]] name = "docling-core" -version = "2.99.0" +version = "2.103.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "defusedxml" }, @@ -754,9 +756,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/30/4da99847d85e3a2aa61e751c1e8749a2ba223c2208b0a9a1ad1b174cdb5a/docling_core-2.99.0.tar.gz", hash = "sha256:bff91f3313cc092f0255743f708cb86b4efec41532b3c60aae3b790ef1f2a71a", size = 388534, upload-time = "2026-09-25T15:10:15.996Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/74/70107b222626fa5167a4de9dd24148b2ff657667d9ad57d17d3fba3906ef/docling_core-2.103.1.tar.gz", hash = "sha256:52398d2369fe32fccb7c06527d888b4cfa50226b25f55bbdc5d6e6d6863ee4f4", size = 440557, upload-time = "2026-10-20T14:45:14.447Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/02/61a226c0ba319dc7da463fd137f88a728e4eb0353591d85064181a5016bb/docling_core-2.99.0-py3-none-any.whl", hash = "sha256:2ce0bae815de8139e8c0dc93736ba613609db777c3cbe392d9ec76bfabec6104", size = 309099, upload-time = "2026-09-25T15:10:14.163Z" }, + { url = "https://files.pythonhosted.org/packages/8a/6b/047a02de5828d54462f48f7d12da50861cc36505a600460ae08eeeeca2f5/docling_core-2.103.1-py3-none-any.whl", hash = "sha256:2407bb42bb95468192b69b772ab72477e2a636bd33c5c6a401047b190fc81b96", size = 328165, upload-time = "2026-10-20T14:45:12.455Z" }, ] [package.optional-dependencies] @@ -793,7 +795,7 @@ wheels = [ [[package]] name = "docling-parse" -version = "7.22.2" +version = "7.22.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docling-core" }, @@ -801,28 +803,28 @@ dependencies = [ { name = "pydantic" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bc/19/4036a3e844a6385b720b40f9c44d6ab4b6733707dd698dcfec3beda0a48d/docling_parse-7.22.2.tar.gz", hash = "sha256:368d076d2b887fcb44a90c37c219a190b975f01b257b61bd83ba75dc06861536", size = 7045499, upload-time = "2026-10-05T07:23:55.9Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/ea/f609987314655e8ad0136d68d0961f60a28421d2cd79da5a67d59f4c0c92/docling_parse-7.22.3.tar.gz", hash = "sha256:100e9c320e199343cd7bb03a816956e578109dd53d76cc72ecbd9d9e89954dc6", size = 7050703, upload-time = "2026-10-13T04:24:34.028Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/42/53a5a7d3728bb19ac5d1ae4e511537fa9886be50e73787ad66822a6a28b7/docling_parse-7.22.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:95929dd53516964ebdfc8023d0243a77b78debd9d9b7d51080ee3f66c1bc7167", size = 9878122, upload-time = "2026-10-05T07:23:14.356Z" }, - { url = "https://files.pythonhosted.org/packages/47/53/825d3befbf257312a105848c6517ca7c90c78bc53644d3177a7de79752d2/docling_parse-7.22.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f1fd682ccafc359db82061c4f79fc21fd2b127644e9334ea8c8905322fa4528", size = 10437621, upload-time = "2026-10-05T07:23:17.148Z" }, - { url = "https://files.pythonhosted.org/packages/b0/04/b71e0da1c10af2c3bc9bc19f6894ab6a01ab70d6212c7c0f2c90d654e5a6/docling_parse-7.22.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b5370558fb2c3c4884099f0e9d12bbc0fa87fe879b555fd2442a826d9dc3561", size = 10843432, upload-time = "2026-10-05T07:23:20.473Z" }, - { url = "https://files.pythonhosted.org/packages/a1/3c/5e9849e754bc14d534aeaa858306826a11657e77dbf81c19425243c088a0/docling_parse-7.22.2-cp312-cp312-win_amd64.whl", hash = "sha256:04a38e69c05167c4f1aa8429c051b81c868f019d4a28c81d220f5018ce98d250", size = 11886757, upload-time = "2026-10-05T07:23:22.914Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0a/e7471bc42d9892487f8262526a787c3e8b3cd06710131c1904b0f5a18a85/docling_parse-7.22.2-cp312-cp312-win_arm64.whl", hash = "sha256:a6471c48e678e94584dc45ac95109ba0b5fb1a9eeb74e53c6ff457edd4a3e9c9", size = 9275026, upload-time = "2026-10-05T07:23:25.514Z" }, - { url = "https://files.pythonhosted.org/packages/d2/61/05b5d2fbc5dcb3023aa980daaebfaa03dd2447e21271668cd6b3058552c2/docling_parse-7.22.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:53156ac7c1e389857f23b118f8cd59331d3a215be94f9f9ce9ae3145e4125759", size = 9878343, upload-time = "2026-10-05T07:23:28.238Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d1/538efc18c43a0c6e678e6dea7c14d0599a14a0d6c8fc3b192bd6d6a92f6d/docling_parse-7.22.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8eb209efd837d7ba055bc582ae0aa978e32b6fa9851fb39632a1c303f2b0693", size = 10437835, upload-time = "2026-10-05T07:23:30.77Z" }, - { url = "https://files.pythonhosted.org/packages/5a/3f/c004e1eef07e24059130e9fe97ac5870c11a187162f1f7eb93e9c20c3a75/docling_parse-7.22.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73265a552c3f28bee591a8ca1e0dc38b51ee17f5051a626aef2fb625ee81f305", size = 10842974, upload-time = "2026-10-05T07:23:33.225Z" }, - { url = "https://files.pythonhosted.org/packages/65/a0/8943af36a45726fe5b59c0ed213ef260ed0111309547d5b01b9820dcd5bf/docling_parse-7.22.2-cp313-cp313-win_amd64.whl", hash = "sha256:5a1433aceca0b1be834a79b6a7bddd89681426b8c1b06725aebe7740583d38a3", size = 11886875, upload-time = "2026-10-05T07:23:35.722Z" }, - { url = "https://files.pythonhosted.org/packages/08/44/5d44f7ca3801fbcc5e0f6e51d92cb85a6952873203ecf56227503fb53f6a/docling_parse-7.22.2-cp313-cp313-win_arm64.whl", hash = "sha256:e23c55d856d0124634b42d8b63a879ecb1056769a3b38cb6ea186e03d20c0421", size = 9274980, upload-time = "2026-10-05T07:23:38.454Z" }, - { url = "https://files.pythonhosted.org/packages/1f/71/75b2492ea7de1dea7b23405d8afbbf6be44e194857fad328c400dbcc9ef4/docling_parse-7.22.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9a003356d71578a56718abd343baf5c6d745ea7d7e988ca524b3af5bbd35760d", size = 9878449, upload-time = "2026-10-05T07:23:41.387Z" }, - { url = "https://files.pythonhosted.org/packages/65/e4/82e930b4f534f77dc70871d3b5f7cc4672fa6b89320d14bcad95a6ae3a72/docling_parse-7.22.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7699b216003ef5759ee4fa61bc7a403a6ce63afcbbf316cd17e4562d5b306e8b", size = 10439277, upload-time = "2026-10-05T07:23:44.53Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b2/6161ee6ab8e519177451620976abd1a002dbe1bdc98d4f8cddd7c49f68f4/docling_parse-7.22.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6a5aa4e0c0f62c4e0032c24117410d5773dbc303da5b4a282ff5abf9f491391", size = 10843921, upload-time = "2026-10-05T07:23:47.72Z" }, - { url = "https://files.pythonhosted.org/packages/28/52/3593b031a4319379f9850dd3e8bc76adbcc62d7533f0741e6e3257d77d9c/docling_parse-7.22.2-cp314-cp314-win_amd64.whl", hash = "sha256:4bca9aeefa92896a12eb96e8858adc6b3adbd34c457095e5be68db84c6bb8c56", size = 12324184, upload-time = "2026-10-05T07:23:50.939Z" }, - { url = "https://files.pythonhosted.org/packages/79/ec/c439cced7ed74407b0fd70d48e4de19b7a8e0da88f786b95bfb0af10c77f/docling_parse-7.22.2-cp314-cp314-win_arm64.whl", hash = "sha256:6bfff00d2577f288e037069bbf5087bcf01305f6b78b9432d599c6ffe03557a2", size = 9652594, upload-time = "2026-10-05T07:23:53.627Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2a/7620351e16be14c165b407062ce0cb62f4bfb82c016feaef2e2eee3e2244/docling_parse-7.22.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2df4e5fda5e27392a1dd1a2f0c90e6101ed5c159e9f886ef76c4cd7a7f9fbfdd", size = 9879311, upload-time = "2026-10-13T04:23:59.969Z" }, + { url = "https://files.pythonhosted.org/packages/23/88/8dc36225fea4d3211f60039392986cb15062dbfa2771cc5cdd5ecc747aa5/docling_parse-7.22.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e936ead174ab0b3e21cfbb3812bc0179d136d3acf32bc335374fc94e433247af", size = 10441366, upload-time = "2026-10-13T04:24:02.361Z" }, + { url = "https://files.pythonhosted.org/packages/9d/56/e66b6990912f39e149098860b42ca2d259437f52234eef8e73444145da87/docling_parse-7.22.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72d60fb81817ea08134dfc09eaf34d0dd3242fba4461a668e12a6636018e5da", size = 10848678, upload-time = "2026-10-13T04:24:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/1037162acaeaf1247bfadcdf888b64c90e49d929c5a1a2c702a383052b1a/docling_parse-7.22.3-cp312-cp312-win_amd64.whl", hash = "sha256:f8957f88deec0c8fc010c664df297197bb98fc04ae7e63d66e65704ebcbdaa69", size = 11890707, upload-time = "2026-10-13T04:24:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/08/e7/46be5fa297521f225c32ffc61b61487fff663b881a5aa17585d5d5b908f1/docling_parse-7.22.3-cp312-cp312-win_arm64.whl", hash = "sha256:1e55c0dc5c000ea15b76d67e5151be9010a7100432f6baec439ab36665300b6b", size = 9275488, upload-time = "2026-10-13T04:24:09.007Z" }, + { url = "https://files.pythonhosted.org/packages/67/1e/9f3368114c621db0f0a1eb5caac6a50ba20fd00f041015d91ba850d670b9/docling_parse-7.22.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ac8e20c99bde5ad7c4ff17ed96cdbf6b0f5c3c3f932889c42c2029b3d0ddf04c", size = 9879471, upload-time = "2026-10-13T04:24:10.893Z" }, + { url = "https://files.pythonhosted.org/packages/a3/f2/0cc656170ae7e5069037f7befdd8d46ca428c6315c855455732feeb86e76/docling_parse-7.22.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b1af327b0ec1621d2d78e3ef45a81ddd37c688fdbdcb8c7088d94244db9fa580", size = 10441280, upload-time = "2026-10-13T04:24:13.254Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2b/5fbcdd90bb24b42fa9904d5f2fd5ba09fe7a8dd8a45b3421a4bb7c1dd4ae/docling_parse-7.22.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42cb068b6094f4deadb6e77e0b62cb7c2f64826f937d2affd5ae303603b07988", size = 10848681, upload-time = "2026-10-13T04:24:15.954Z" }, + { url = "https://files.pythonhosted.org/packages/14/9b/84f9e4c329bdc6ee3107e5149c102652ea10089dde899a6d5447150c44b0/docling_parse-7.22.3-cp313-cp313-win_amd64.whl", hash = "sha256:604e1938653e05249217ad6e810f821c18d56b95611040b125d531a2d1c7147d", size = 11890814, upload-time = "2026-10-13T04:24:18.201Z" }, + { url = "https://files.pythonhosted.org/packages/12/b0/2b6ab89e89a3454222a77441b36f3691d64410d293bdf9e06d34e7cc1175/docling_parse-7.22.3-cp313-cp313-win_arm64.whl", hash = "sha256:2003cebdbbb5e22dbb3cdc206cbe7f489d3da2b0ef7fafa0498dc6470aa8f62b", size = 9275555, upload-time = "2026-10-13T04:24:20.533Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c2/3d90b27327b2b9cf575dbf9a7cd3d7be3ee8aaeb802ae29bfeedcb6e302f/docling_parse-7.22.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:ff4944d8122902a938c361e3a860eefaedececd286812b9c3130475c28862268", size = 9879558, upload-time = "2026-10-13T04:24:22.434Z" }, + { url = "https://files.pythonhosted.org/packages/13/45/9a4be8a808f14a30f23c57a19a1f842ac7500a3bed37bdad1457baa3156e/docling_parse-7.22.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b363a03a141b77d0726ed81241a492a65d4daa6c716d883142187b4b9205abf", size = 10443099, upload-time = "2026-10-13T04:24:24.49Z" }, + { url = "https://files.pythonhosted.org/packages/35/cb/e29bfb3ffd9c64970ec24acbc7cb7651d2d9f586782ff22cd88a38748d4d/docling_parse-7.22.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6afe9d92372b2ed60a6965b6494d634ab4929372d43df02fdf8fc21a00eb6ae", size = 10849090, upload-time = "2026-10-13T04:24:27.41Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6d/7796d8a7cf6c9b5ce55cef8cd2e2578635f8bfbe99965dbc56ac3f839cbc/docling_parse-7.22.3-cp314-cp314-win_amd64.whl", hash = "sha256:5baa7bb0a449ad14c46124dcc7beae5c397c682b83e31087833c194283d8b6b2", size = 12329254, upload-time = "2026-10-13T04:24:29.642Z" }, + { url = "https://files.pythonhosted.org/packages/48/d2/9b811d44785b25e2455c26f9158b39bd5bda8ff77fb9800efe8a486d2259/docling_parse-7.22.3-cp314-cp314-win_arm64.whl", hash = "sha256:43aadf6a36e8da863811d39c0c74da81fee11a13a2756d0ff0db1a78074bff9b", size = 9653345, upload-time = "2026-10-13T04:24:31.775Z" }, ] [[package]] name = "docling-slim" -version = "2.134.0" +version = "2.138.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -835,9 +837,9 @@ dependencies = [ { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7b/75/ee898b4b7a310c187d0d7ad89002b000efeead0af177f9e50f71554b194b/docling_slim-2.134.0.tar.gz", hash = "sha256:63f12c79bdd0bbe205f244f97f0a8640c590ea46de4d913d23e963ec57e531dd", size = 795345, upload-time = "2026-10-06T04:57:33.257Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/8d/f1bc0fd27c2d19fa4cba07fcaa9ce67094d51275f84153e6471a91cbc28b/docling_slim-2.138.0.tar.gz", hash = "sha256:1294949bd3ab1c9ef41eebf5c39549e2a845bef360a6dcf7001676644d24bcc6", size = 815599, upload-time = "2026-10-12T06:16:20.226Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/58/526bc8f7c7f46dc22afa6f548c67e0bd685cf5f97faf7ee3d96c26355732/docling_slim-2.134.0-py3-none-any.whl", hash = "sha256:818b350e1e9878c0d54bd4ff972bd1918cc9827f193257162cc6f3a797175c29", size = 974718, upload-time = "2026-10-06T04:57:31.022Z" }, + { url = "https://files.pythonhosted.org/packages/fe/47/237cbeaa6724ac2275a7f5fbd0ed5484e8cda20a5521de9ffea0463e9386/docling_slim-2.138.0-py3-none-any.whl", hash = "sha256:35f709532563f43b004faab02e974ae95c4fc05455cee2e038af9e0abeaaff6e", size = 995749, upload-time = "2026-10-12T06:16:17.925Z" }, ] [package.optional-dependencies]