From 630f6b16f45a058395e648249e5ae952936e1b95 Mon Sep 17 00:00:00 2001 From: mengerj Date: Wed, 24 Jun 2026 07:49:02 +0200 Subject: [PATCH] PR 3: Source-adapter interface + deterministic CELLxGENE adapter Introduce the SourceAdapter / Normalizer contracts and refactor CELLxGENE into a fully deterministic adapter + normalizer, then rip out the LLM/Azure narrative path entirely. New contracts - models/raw_record.py: RawRecord, the source-shaped boundary object between adapters and normalizers (source, study_id, title, free-form payload). - sources/base.py: SourceAdapter Protocol (discover -> [ref], fetch -> RawRecord). - normalize/base.py: Normalizer Protocol (normalize -> KnowledgeGraphOutput). CELLxGENE migrated (no LLM in its path) - tools/cellxgene_fetcher.py -> sources/cellxgene.py (CellxgeneAdapter). Cell-type extraction dropped: the cell_type* Census columns and cell_types summary key are gone (data-inferred -> leakage). - tools/ncbi_fetcher.py -> sources/publication.py (EuropePMC title helper). - graph/builder.py -> normalize/cellxgene.py (CellxgeneNormalizer); reads record.source/title/payload instead of a hardcoded source. The organism-> NCBITaxon map moves here (normalization, future PR 4 resolver). - Vestigial @tool wrappers and the whole tools/ package (incl. GEO stub) removed. Narrative/LLM path removed - Deleted agent/curator.py, agent/prompts.py, models/narrative.py, the NarrativeOutput schema, and main.py's step-2 block. main.py is now a deterministic, synchronous discover -> fetch -> normalize -> write pipeline. Tooling/tests - Removed the parce.tools.* mypy exemption (modules migrated, now type-checked); parce.agent.* exemption stays for PR 5. - Tests: test_builder -> test_normalize, new test_sources (adapter + RawRecord + protocol conformance), rewritten test_orchestration, narrative tests removed. Gates green locally (incl. hermetic run with .env moved aside): ruff check, ruff format --check, mypy (16 files), 47 unit tests. Co-Authored-By: Claude Opus 4.8 --- README.md | 199 ++++++---------- docs/ROADMAP.md | 85 +++++-- pyproject.toml | 11 +- src/parce/agent/__init__.py | 7 + src/parce/agent/curator.py | 59 ----- src/parce/agent/prompts.py | 22 -- src/parce/graph/__init__.py | 7 + src/parce/graph/builder.py | 159 ------------- src/parce/main.py | 217 ++++-------------- src/parce/models/graph_schema.py | 18 -- src/parce/models/narrative.py | 87 ------- src/parce/models/raw_record.py | 41 ++++ src/parce/normalize/__init__.py | 8 + src/parce/normalize/base.py | 29 +++ src/parce/normalize/cellxgene.py | 160 +++++++++++++ src/parce/sources/__init__.py | 6 + src/parce/sources/base.py | 37 +++ .../cellxgene.py} | 107 ++++++--- .../publication.py} | 33 +-- src/parce/tools/__init__.py | 0 src/parce/tools/geo_fetcher.py | 69 ------ tests/test_builder.py | 174 -------------- tests/test_cellxgene_integration.py | 59 +++-- tests/test_graph_schema.py | 16 -- tests/test_integration.py | 12 - tests/test_models.py | 92 -------- tests/test_normalize.py | 186 +++++++++++++++ tests/test_orchestration.py | 110 +++------ tests/test_sources.py | 104 +++++++++ 29 files changed, 911 insertions(+), 1203 deletions(-) delete mode 100644 src/parce/agent/curator.py delete mode 100644 src/parce/agent/prompts.py delete mode 100644 src/parce/graph/builder.py delete mode 100644 src/parce/models/narrative.py create mode 100644 src/parce/models/raw_record.py create mode 100644 src/parce/normalize/__init__.py create mode 100644 src/parce/normalize/base.py create mode 100644 src/parce/normalize/cellxgene.py create mode 100644 src/parce/sources/__init__.py create mode 100644 src/parce/sources/base.py rename src/parce/{tools/cellxgene_fetcher.py => sources/cellxgene.py} (61%) rename src/parce/{tools/ncbi_fetcher.py => sources/publication.py} (52%) delete mode 100644 src/parce/tools/__init__.py delete mode 100644 src/parce/tools/geo_fetcher.py delete mode 100644 tests/test_builder.py delete mode 100644 tests/test_models.py create mode 100644 tests/test_normalize.py create mode 100644 tests/test_sources.py diff --git a/README.md b/README.md index b2d68e7..7427703 100644 --- a/README.md +++ b/README.md @@ -2,161 +2,100 @@ **Programmable Agent for Retrieving Contextualized Experiments** -PARCE is an agentic workflow that fetches public omics data and produces structured JSON narratives describing how the data was obtained. Each narrative interleaves human-readable descriptions with URI references to raw data files, making it suitable for training multimodal autoregressive embedding models. +PARCE harvests **public omics experiments** from heterogeneous repositories and +normalizes them into a single, ontology-grounded **knowledge graph (KG)**. The KG +is the training substrate for a downstream multi-omics autoregressive model: each +study contributes biological *context* (assay, tissue, disease, organism) and +sample-level *covariates* (condition, perturbation, timepoint, subject), plus URIs +to the raw data. + +Two principles drive the design: + +1. **Context is design, not outcome.** Only metadata that describes how an + experiment was *designed* is stored. Data-inferred annotations (e.g. cell type + called from expression) are excluded — they would leak the signal the + downstream model must learn. +2. **One canonical schema, many sources.** Every repository is mapped into the + same Pydantic KG schema and the same ontology IDs. Sources link to each other + *only* through shared ontology entity nodes. + +See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the design and +[docs/ROADMAP.md](docs/ROADMAP.md) for the plan and current status. ## Architecture -``` -User / CLI - │ - ▼ -main.py ──► agent/curator.py ──► AzureAIAgentClient - │ - ┌─────────┴─────────┐ - │ Azure AI Foundry │ - │ (any model) │ - └─────────┬─────────┘ - │ - ┌───────────┼───────────┐ - ▼ │ ▼ - tool_call: │ structured output: - geo_fetcher.py │ ExperimentNarrative - (tools/) │ (models/) - │ │ - ▼ │ - metadata JSON ─────┘ -``` - -The agent uses **Azure AI Foundry** as its model gateway. Any model deployed in your Foundry project -- Mistral, GPT-4o, DeepSeek, Llama, etc. -- can be used by changing a single environment variable (`AZURE_AI_MODEL_DEPLOYMENT_NAME`). Tools, prompts, and Pydantic output schemas remain unchanged. - -### Directory Structure +The pipeline is **deterministic by default**. Each repository has a `SourceAdapter` +(all network IO) that emits source-shaped `RawRecord`s, and a `Normalizer` that +maps a `RawRecord` into canonical KG nodes. An LLM is used for exactly one job — +structured extraction from *unstructured* free-text metadata (e.g. GEO, PRIDE), +constrained to the canonical schema via `response_format`. Structured, +ontology-grounded sources (e.g. CELLxGENE) have **no LLM in their path**. ``` -parce/ -├── pyproject.toml # Dependencies and build config -├── .env.example # Template for required env vars -├── src/ -│ └── parce/ -│ ├── main.py # Entry point -│ ├── agent/ -│ │ ├── curator.py # Agent factory (AzureAIAgentClient) -│ │ └── prompts.py # System prompts -│ ├── tools/ -│ │ └── geo_fetcher.py # GEO metadata fetcher (stub) -│ ├── models/ -│ │ └── narrative.py # Pydantic output schemas -│ └── config/ -│ └── settings.py # Env-based configuration -├── data_pipelines/ # Future: Spark / ADLS scripts -├── data/ # Local test data (gitignored) -└── tests/ - └── test_models.py + per source → SourceAdapter: discover(query) -> [ref] + fetch(ref) -> RawRecord + │ (source-shaped) + per source → Normalizer: RawRecord -> canonical KG nodes + • structured source -> deterministic map + • unstructured -> Azure extraction agent (PR 5+) + │ (canonical nodes w/ free-text terms) + shared → OntologyResolver: text -> UBERON/MONDO/EFO/... (PR 4) + │ (ontology-grounded nodes) + shared → GraphBuilder/Merger: assemble + merge into one KG (PR 6) ``` -**Key design decisions:** - -- **src-layout** prevents accidental imports from the project root. -- **agent/** is decoupled from **tools/** so new data sources (e.g. TCellAtlas, SRA) are added as new tool files without touching orchestration logic. -- **models/** schemas serve double duty: structured output for the agent (`response_format`) and validation for downstream consumers. -- **data_pipelines/** lives at the repo root because Spark jobs are submitted independently from the Python package. - -## Prerequisites - -- Python 3.11+ -- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) installed -- An [Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/what-is-ai-foundry) project with at least one model deployed (e.g. Mistral Large, GPT-4o, DeepSeek-R1) - -## Local Setup +The canonical schema (`models/graph_schema.py`) is the contract: the deterministic +path and the future agent path emit the *same* Pydantic models, so everything +downstream is source-agnostic. -### 1. Authenticate with Azure +### Directory structure -```bash -az login ``` - -This lets `AzureCliCredential` obtain tokens for your Foundry project without managing API keys. - -### 2. Configure environment - -```bash -cp .env.example .env +src/parce/ + models/ # Canonical Pydantic KG schema + RawRecord (boundary model) + sources/ # One adapter per repository: discover() + fetch() -> RawRecord + normalize/ # RawRecord -> canonical nodes (deterministic OR agent-backed) + agent/ # Azure extraction agent (structured output only; PR 5+) + graph/ # Cross-source KG assembly + merge (PR 6) + config/ # pydantic-settings configuration + main.py # CLI entry point / orchestrator +tests/ # Unit tests (offline) + integration tests (marked) +docs/ # ARCHITECTURE.md, ROADMAP.md, AGENT_SESSION_PROMPT.md ``` -Edit `.env` with your Foundry project endpoint and deployment name: - -``` -AZURE_AI_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ -AZURE_AI_MODEL_DEPLOYMENT_NAME=mistral-large -``` +Implemented so far: the canonical schema, the `SourceAdapter`/`Normalizer` +protocols, and the deterministic **CELLxGENE** adapter + normalizer. -The project endpoint is found in your Azure AI Foundry project settings page. +## Setup -### 3. Install +PARCE is [uv](https://docs.astral.sh/uv/)-managed. ```bash -python -m venv .venv -source .venv/bin/activate # Windows: .venv\Scripts\activate -pip install -e ".[dev]" +uv sync --extra dev # install (incl. dev tools) +uv run parce # run the CLI (CELLxGENE path; deterministic) ``` -> **Note:** The `agent-framework` packages are currently in pre-release. If installation fails, run: -> ```bash -> pip install agent-framework agent-framework-azure-ai --pre -> pip install -e ".[dev]" -> ``` +`parce` fetches the datasets for the default collection DOI from CELLxGENE Census, +normalizes them into the canonical KG, and writes `data/graphs/output.json`. This +path needs only network access — **no Azure credentials**. -### 4. Run the agent +Azure AI Foundry credentials are required only for the extraction agent (GEO/PRIDE, +PR 5+) and for the integration test suite. Copy `.env.example` to `.env` and fill +in your Foundry endpoint and deployment name, then `az login`. -```bash -parce -# or -python -m parce.main -``` +## Tests & quality gates -The agent will fetch mock metadata for GSE164378 and return a structured `ExperimentNarrative` JSON. - -### 5. Run tests +CI runs four gates; all four must pass locally before opening a PR: ```bash -pytest +uv run ruff check . +uv run ruff format --check . +uv run mypy src/parce +uv run pytest -m "not integration" # unit tests (offline, hermetic) ``` -## Switching Models - -Because PARCE uses Azure AI Foundry as a model gateway, swapping the underlying LLM is a one-line change in `.env`: +Live/credentialed tests carry the `integration` marker and are excluded from CI: ```bash -# Mistral -AZURE_AI_MODEL_DEPLOYMENT_NAME=mistral-large - -# OpenAI -AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o - -# DeepSeek -AZURE_AI_MODEL_DEPLOYMENT_NAME=DeepSeek-R1 - -# Meta Llama -AZURE_AI_MODEL_DEPLOYMENT_NAME=Meta-Llama-3-70B +uv run pytest -m integration # needs Azure / Census / network ``` - -The deployment name must match a model you have deployed in your Foundry project's model catalog. No code changes are required -- all providers produce a standard `Agent` with the same interface. - -For models **not** in the Azure AI Foundry catalog (e.g. Anthropic Claude), the Microsoft Agent Framework provides dedicated providers (`AnthropicChatClient`) with an identical agent interface. - -## Future Roadmap - -### Data Engineering Pipelines (`data_pipelines/`) - -- **FASTQ-to-Parquet** conversion using PySpark on Azure Databricks -- **Azure Data Lake Storage Gen2** integration for persisting narratives and raw genomic data at scale -- Batch orchestration for processing large T-cell transcriptomics datasets - -### Additional Data Sources - -- **TCellAtlas** fetcher tool -- **SRA** direct metadata fetcher -- **CellxGene** Census integration - -### Embedding Model Training - -The structured narratives produced by PARCE will serve as training data for a multimodal autoregressive embedding model that jointly learns from experimental metadata and raw omics signals. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d8f7ee1..603348a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -8,14 +8,14 @@ protocol and [ARCHITECTURE.md](ARCHITECTURE.md) for the design. ## ▶ Next up -**PR 3 — Source-adapter interface + cheap CELLxGENE adapter.** Define -`SourceAdapter` / `Normalizer` protocols in `sources/` + `normalize/`. Refactor -CELLxGENE into a deterministic adapter. **Remove the LLM/Azure narrative path -entirely** (delete `agent/prompts.py` narrative role, `models/narrative.py`, the -narrative `NarrativeOutput` schema in `models/graph_schema.py`, and step 2 of -`main.py` — `_build_narrative_prompt`, the agent call, and the now-unused -`narrative` variable). Drop cell-type extraction. Remove the `parce.tools.*` -mypy exemption as those modules move under `sources/`. +**PR 4 — Ontology resolver.** Shared `ontology/` stage (see ARCHITECTURE §5). Pin +the **facet → ontology registry** as a constant (EFO, UBERON, MONDO, NCBITaxon, +ChEBI, PSI-MS, EDAM). Implement free-text → term resolution (OLS4 REST + +text2term/Zooma, on-disk cache; LLM fallback) and the **`molecular_layer` +derivation** (walk EFO `is-a` ancestors to anchor classes). Decide the anchor set ++ the no-anchor default. Wire into normalizers — start by replacing the hardcoded +`_ORGANISM_ONTOLOGY` map in `normalize/cellxgene.py`. Resolve IDs at runtime via +OLS — do not hardcode term IDs. --- @@ -29,20 +29,21 @@ Each PR is one branch, one focused scope, green CI, and a roadmap update. - [x] **PR 2 — Canonical KG schema.** Source-agnostic nodes/edges; add `SampleNode` with design covariates; drop `experimental_narrative` and `CellType`. Migrate builder + tests. -- [ ] **PR 3 — Source-adapter interface + cheap CELLxGENE adapter.** Define - `SourceAdapter` / `Normalizer` protocols in `sources/` + `normalize/`. Refactor - CELLxGENE into a deterministic adapter. **Remove the LLM/Azure narrative path - entirely** (delete `agent/prompts.py` narrative role, `models/narrative.py`, - the `NarrativeOutput` schema, and the step-2 block in `main.py`). Drop - cell-type extraction. Remove the `parce.tools.*` mypy exemption as those - modules move under `sources/`. *(Next up.)* +- [x] **PR 3 — Source-adapter interface + cheap CELLxGENE adapter.** Defined + `SourceAdapter` / `Normalizer` protocols in `sources/` + `normalize/`. Refactored + CELLxGENE into a deterministic adapter + normalizer. **Removed the LLM/Azure + narrative path entirely** (deleted `agent/curator.py`, `agent/prompts.py`, + `models/narrative.py`, the `NarrativeOutput` schema, and the step-2 block in + `main.py`). Dropped cell-type extraction. Removed the `parce.tools.*` mypy + exemption as those modules moved under `sources/`. - [ ] **PR 4 — Ontology resolver.** Shared `ontology/` stage (see ARCHITECTURE §5). Pin the **facet → ontology registry** as a constant (EFO, UBERON, MONDO, NCBITaxon, ChEBI, PSI-MS, EDAM). Implement free-text → term resolution (OLS4 REST + text2term/Zooma, on-disk cache; LLM fallback) and the **`molecular_layer` derivation** (walk EFO `is-a` ancestors to anchor classes). - Decide the anchor set + the no-anchor default. Wire into normalizers. Resolve - IDs at runtime via OLS — do not hardcode term IDs. + Decide the anchor set + the no-anchor default. Wire into normalizers (replace + the hardcoded `_ORGANISM_ONTOLOGY` map in `normalize/cellxgene.py`). Resolve + IDs at runtime via OLS — do not hardcode term IDs. *(Next up.)* - [ ] **PR 5 — GEO extraction agent (vertical slice).** GEO adapter (E-utilities/GEOparse) + Azure extraction normalizer emitting the canonical schema via `response_format`; extract sample covariates from @@ -72,6 +73,56 @@ Each PR is one branch, one focused scope, green CI, and a roadmap update. Newest first. One entry per working session: what changed, decisions made, and what the next session should know. Keep entries short and factual. +### 2026-06-24 — PR 3: Source-adapter interface + CELLxGENE adapter + +- Branch `pr3-source-adapter-interface` off `main` (69d1f68). +- **New contracts.** `models/raw_record.py` adds `RawRecord` (source-shaped: + `source`, `study_id`, `title`, free-form `payload`) — the boundary object + between adapters and normalizers. `sources/base.py` defines the `SourceAdapter` + Protocol (`source_name`, `discover(query) -> [ref]`, `fetch(ref) -> RawRecord`); + `normalize/base.py` defines the `Normalizer` Protocol + (`normalize(record) -> KnowledgeGraphOutput`). Both `@runtime_checkable`. +- **CELLxGENE migrated to a deterministic adapter + normalizer:** + - `tools/cellxgene_fetcher.py` → `sources/cellxgene.py` (`CellxgeneAdapter` + + the `fetch_cellxgene_datasets` core fn). **Cell-type extraction dropped**: the + `cell_type*` Census columns and the `cell_types` summary key are gone (reading + a data-inferred annotation is leakage even before it hits the graph). + - `tools/ncbi_fetcher.py` → `sources/publication.py` (`fetch_paper_metadata`, + EuropePMC). The adapter's `fetch` gathers the publication title here because + Census exposes dataset titles + DOI but not the publication title. + - `graph/builder.py` → `normalize/cellxgene.py` (`CellxgeneNormalizer`). It now + reads `record.source`/`title`/`payload` instead of taking a hardcoded source. + - The `@tool`-decorated wrappers (`fetch_cellxgene_data`, `fetch_paper_context`, + `fetch_geo_metadata`) were vestigial from the agent-tool-calling era and are + deleted. The whole `tools/` package and the GEO stub are removed. +- **Narrative/LLM path removed entirely:** deleted `agent/curator.py`, + `agent/prompts.py`, `models/narrative.py` (and `test_models.py`), and the + `NarrativeOutput` schema. `main.py` is now deterministic and **synchronous**: + `discover → fetch → normalize → write`. (PR 5 reintroduces async + the agent.) +- Decisions (rationale): + - **Per-study assembly is the Normalizer's job; `graph/` is reserved for the + PR 6 cross-source merger.** Matches ARCHITECTURE §3's split. `graph/__init__.py` + and `agent/__init__.py` are kept as documented placeholders. + - **Organism→NCBITaxon map (`_ORGANISM_ONTOLOGY`) lives in the normalizer, not + the adapter.** The adapter emits the raw organism string; string→ID mapping is + normalization and becomes the PR 4 OntologyResolver's job. + - **`discover` is the identity on a DOI for CELLxGENE** (a collection = a DOI). + Keyword collection search is backlog. + - **Azure-coupled retry helpers removed from `main.py`** with the LLM call (their + only caller). CELLxGENE/EuropePMC fetches currently have no retry wrapper — + see follow-up below. CLAUDE.md still references "helpers in `main.py`"; left as + is since PR 5 reintroduces retry infra for the agent. +- **mypy:** removed the `parce.tools.*` exemption (modules migrated + now fully + type-checked); `parce.agent.*` exemption stays for PR 5. mypy checks 16 files. +- **Gates green locally, incl. hermetic run with `.env` moved aside:** ruff check, + ruff format --check (24 files), mypy, **47 unit tests**. No dep changes; the + `agent-framework`/`azure-*` deps stay (PR 5 needs them). +- **Follow-up for a later session:** wire bounded-retry/backoff into the source + adapters' network calls (Census, EuropePMC) — resilience regressed when the + Azure-only retry helpers were removed. +- **Next session:** PR 4 (ontology resolver). First integration point: replace + `_ORGANISM_ONTOLOGY` in `normalize/cellxgene.py`. + ### 2026-06-23 — PR 2: Canonical KG schema - Branch `pr2-canonical-kg-schema` off `main`. diff --git a/pyproject.toml b/pyproject.toml index 4e4e219..48930b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,10 +63,11 @@ warn_redundant_casts = true warn_unused_ignores = true check_untyped_defs = true -# The IO-boundary modules (source fetchers, Azure agent glue) are slated for -# rewrite in the source-adapter refactor (see docs/ROADMAP.md). Until then they -# are exempt from type checking so CI tracks the stable core. Remove an entry -# here as each module is migrated to the adapter interface. +# The Azure agent glue is slated for rewrite as an extraction normalizer (PR 5, +# see docs/ROADMAP.md). Until then it is exempt from type checking so CI tracks +# the stable core. Remove this entry when the agent moves to the normalizer +# interface. (The former ``parce.tools.*`` fetchers were migrated to +# ``parce.sources.*`` in PR 3 and are now fully type-checked.) [[tool.mypy.overrides]] -module = ["parce.tools.*", "parce.agent.*"] +module = ["parce.agent.*"] ignore_errors = true diff --git a/src/parce/agent/__init__.py b/src/parce/agent/__init__.py index e69de29..426a4c1 100644 --- a/src/parce/agent/__init__.py +++ b/src/parce/agent/__init__.py @@ -0,0 +1,7 @@ +"""Azure extraction agent (structured output only). + +Reserved for the GEO/PRIDE extraction normalizers (PR 5+): an LLM constrained by +``response_format`` to emit the canonical KG schema from free-text metadata. The +legacy narrative agent that once lived here was removed in PR 3 — the LLM never +writes prose. See docs/ARCHITECTURE.md §3. +""" diff --git a/src/parce/agent/curator.py b/src/parce/agent/curator.py deleted file mode 100644 index 83a8ac6..0000000 --- a/src/parce/agent/curator.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Factory for the PARCE narrative agent. - -Uses ``AzureAIAgentClient`` (Azure AI Foundry provider) so any model -deployed in your Foundry project can be used -- GPT-4o, Mistral, -DeepSeek, Llama, etc. -- by changing a single env var. - -In the hybrid architecture the agent's sole job is to generate an -experimental narrative from provided context. All tool calling and -KG construction happens in Python. -""" - -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager - -from agent_framework import Agent -from agent_framework.azure import AzureAIAgentClient -from azure.identity.aio import AzureCliCredential - -from parce.agent.prompts import NARRATIVE_INSTRUCTIONS -from parce.config.settings import Settings - - -@asynccontextmanager -async def create_narrative_agent( - settings: Settings | None = None, -) -> AsyncIterator[Agent]: - """Build and yield an Agent configured as a narrative writer. - - The agent receives publication abstract + structured ontology context - and returns a ``NarrativeOutput`` via ``response_format``. - - Parameters - ---------- - settings: - Application settings. When *None*, settings are loaded from the - environment / ``.env`` file automatically. - - Yields - ------ - Agent - A ready-to-run agent that produces a narrative string. - """ - if settings is None: - settings = Settings() - - async with ( - AzureCliCredential() as credential, - AzureAIAgentClient( - project_endpoint=settings.azure_ai_project_endpoint, - model_deployment_name=settings.azure_ai_model_deployment_name, - credential=credential, - ).as_agent( - name="PARCE", - instructions=NARRATIVE_INSTRUCTIONS, - ) as agent, - ): - yield agent diff --git a/src/parce/agent/prompts.py b/src/parce/agent/prompts.py deleted file mode 100644 index 7ae7734..0000000 --- a/src/parce/agent/prompts.py +++ /dev/null @@ -1,22 +0,0 @@ -"""System and instruction prompts for the PARCE narrative agent.""" - -NARRATIVE_INSTRUCTIONS = """\ -You are PARCE, a biomedical research narrator specialising in single-cell -genomics experiments. - -You will receive the abstract of a publication together with a structured -summary of the associated CELLxGENE Census datasets (cell types, tissues, -diseases, assays, and organism). - -Your task is to write a concise **experimental narrative** (one paragraph, -3-8 sentences) that: - -1. Explains the biological question the study addresses. -2. Describes the experimental approach (organism, tissue sources, assay - technologies) grounded in the provided ontology data. -3. Summarises the key conditions or disease contexts, if any. - -Be factual. Do not speculate beyond what the abstract and ontology data -support. Do not reproduce the abstract verbatim -- synthesise and add -context from the structured data. -""" diff --git a/src/parce/graph/__init__.py b/src/parce/graph/__init__.py index e69de29..59d12e8 100644 --- a/src/parce/graph/__init__.py +++ b/src/parce/graph/__init__.py @@ -0,0 +1,7 @@ +"""Graph assembly and cross-source merge. + +Reserved for the KG merger (PR 6) that combines per-study +:class:`~parce.models.graph_schema.KnowledgeGraphOutput`s — one per normalized +source — into a single graph deduplicated by ontology ID. Per-study assembly +lives in the source normalizers (``parce.normalize``). +""" diff --git a/src/parce/graph/builder.py b/src/parce/graph/builder.py deleted file mode 100644 index be3b4c7..0000000 --- a/src/parce/graph/builder.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Deterministic Knowledge Graph construction from CELLxGENE + paper metadata. - -This is the CELLxGENE ingestion path: it assembles canonical nodes and edges -programmatically from the structured data returned by the CELLxGENE and -EuropePMC tools. No LLM is involved. (It is slated to become a proper -``SourceAdapter``/``Normalizer`` in PR 3 — see docs/ROADMAP.md.) - -``CellType`` is intentionally not extracted: it is a data-inferred annotation, -not an experiment-design variable. CELLxGENE Census is dataset-level, so no -``SampleNode`` records are emitted here yet (see ARCHITECTURE.md, open question -on sample granularity). -""" - -from __future__ import annotations - -import logging - -from parce.models.graph_schema import ( - BiologicalEntityNode, - DatasetNode, - EntityType, - GraphEdge, - KnowledgeGraphOutput, - StudyNode, -) -from parce.tools.cellxgene_fetcher import _ORGANISM_ONTOLOGY - -logger = logging.getLogger(__name__) - -# Provenance + high-level modality for everything built by this path. -_SOURCE = "CELLxGENE" -_STUDY_MODALITY = "scRNA-seq" - -# Ontology categories from CELLxGENE that become design-context entities. Cell -# types are deliberately omitted (data-inferred → leakage). -_CATEGORY_TO_ENTITY_TYPE: dict[str, EntityType] = { - "tissues": EntityType.TISSUE, - "diseases": EntityType.DISEASE, - "assays": EntityType.ASSAY, -} - -_CATEGORY_TO_RELATION: dict[str, str] = { - "tissues": "HAS_TISSUE", - "diseases": "HAS_CONDITION", - "assays": "MEASURED_WITH", -} - - -def build_knowledge_graph( - paper_data: dict, - cellxgene_data: dict, -) -> KnowledgeGraphOutput: - """Assemble a canonical ``KnowledgeGraphOutput`` from structured data. - - Parameters - ---------- - paper_data: - Dict with keys ``doi``, ``title`` (from ``fetch_paper_metadata``). - cellxgene_data: - Dict with key ``datasets`` containing per-dataset metadata and - ontology summaries (from ``fetch_cellxgene_datasets``). - """ - study_id = paper_data["doi"] - - study = StudyNode( - study_id=study_id, - title=paper_data.get("title", ""), - source=_SOURCE, - modality=_STUDY_MODALITY, - ) - - datasets: list[DatasetNode] = [] - edges: list[GraphEdge] = [] - entity_registry: dict[str, BiologicalEntityNode] = {} - species_seen: set[str] = set() - - for ds in cellxgene_data.get("datasets", []): - dataset_id = ds["dataset_id"] - - datasets.append( - DatasetNode( - dataset_id=dataset_id, - data_uri=ds["h5ad_uri"], - assay=ds.get("modality", "unknown"), - cell_count=ds["cell_count"], - ) - ) - - edges.append( - GraphEdge( - source_id=dataset_id, - target_id=study_id, - relation_type="EXTRACTED_FROM", - ) - ) - - ontology = ds.get("ontology_summary", {}) - - # Register species from the organism field - organism_key = ontology.get("organism", "unknown") - if organism_key in _ORGANISM_ONTOLOGY and organism_key not in species_seen: - ont_id, name = _ORGANISM_ONTOLOGY[organism_key] - species_seen.add(organism_key) - entity_registry[ont_id] = BiologicalEntityNode( - entity_type=EntityType.SPECIES, - ontology_id=ont_id, - name=name, - ) - - # Register entities and create edges per design-context category - for category, entity_type in _CATEGORY_TO_ENTITY_TYPE.items(): - relation = _CATEGORY_TO_RELATION[category] - for term in ontology.get(category, []): - ont_id = term.get("ontology_id", "unknown") - name = term.get("name", "unknown") - - if ont_id not in entity_registry: - entity_registry[ont_id] = BiologicalEntityNode( - entity_type=entity_type, - ontology_id=ont_id, - name=name, - ) - - edges.append( - GraphEdge( - source_id=dataset_id, - target_id=ont_id, - relation_type=relation, - ) - ) - - # Study -> Species edges - for organism_key in species_seen: - species_id = _ORGANISM_ONTOLOGY[organism_key][0] - edges.append( - GraphEdge( - source_id=study_id, - target_id=species_id, - relation_type="STUDIES", - ) - ) - - kg = KnowledgeGraphOutput( - studies=[study], - datasets=datasets, - samples=[], - biological_entities=list(entity_registry.values()), - edges=edges, - ) - - logger.info( - "Built KG: studies=%d datasets=%d samples=%d entities=%d edges=%d", - len(kg.studies), - len(kg.datasets), - len(kg.samples), - len(kg.biological_entities), - len(kg.edges), - ) - return kg diff --git a/src/parce/main.py b/src/parce/main.py index ea46c54..54c891f 100644 --- a/src/parce/main.py +++ b/src/parce/main.py @@ -1,205 +1,81 @@ -"""PARCE entry point -- hybrid orchestrator. +"""PARCE entry point — deterministic CELLxGENE ingestion. -Calls data tools directly from Python, sends a compact context to the LLM -for narrative generation, then assembles the Knowledge Graph programmatically. +Drives one source through the adapter → normalizer pipeline and writes the +canonical knowledge graph to disk. CELLxGENE Census already ships +ontology-grounded metadata, so this path is fully deterministic — there is no +LLM here (the extraction agent enters with GEO in PR 5). + +Run with:: -Run with: python -m parce.main -or, after ``pip install -e .``: + +or, after ``pip install -e .``:: + parce """ from __future__ import annotations -import asyncio import json import logging -import random -import time from pathlib import Path -from parce.agent.curator import create_narrative_agent -from parce.config.settings import Settings -from parce.graph.builder import build_knowledge_graph -from parce.models.graph_schema import NarrativeOutput -from parce.tools.cellxgene_fetcher import fetch_cellxgene_datasets -from parce.tools.ncbi_fetcher import fetch_paper_metadata +from parce.models.graph_schema import KnowledgeGraphOutput +from parce.normalize.cellxgene import CellxgeneNormalizer +from parce.sources.cellxgene import CellxgeneAdapter logger = logging.getLogger(__name__) _OUTPUT_DIR = Path(__file__).resolve().parents[2] / "data" / "graphs" _DEFAULT_DOI = "10.1038/s41586-023-05869-0" -# Resilience constants -_BASE_DELAY = 1.0 -_MAX_DELAY = 30.0 - -# Transient HTTP status codes and exception types that warrant a retry -_TRANSIENT_STATUS_CODES = {429, 500, 502, 503, 504} -_TRANSIENT_EXCEPTIONS = (TimeoutError, ConnectionError, OSError) - - -def _is_transient(exc: BaseException) -> bool: - """Return True if the exception looks transient and worth retrying.""" - if isinstance(exc, _TRANSIENT_EXCEPTIONS): - return True - from azure.core.exceptions import HttpResponseError - - return isinstance(exc, HttpResponseError) and exc.status_code in _TRANSIENT_STATUS_CODES - - -def _backoff_delay(attempt: int) -> float: - """Exponential backoff with full jitter.""" - delay = min(_BASE_DELAY * (2**attempt), _MAX_DELAY) - return random.uniform(0, delay) - -def _build_narrative_prompt(paper_data: dict, cellxgene_data: dict) -> str: - """Build a compact text prompt from tool outputs for the narrative agent.""" - lines = [f"## Publication\nTitle: {paper_data.get('title', 'N/A')}"] - - abstract = paper_data.get("abstract", "") - if abstract: - lines.append(f"Abstract: {abstract}") - - datasets = cellxgene_data.get("datasets", []) - if datasets: - lines.append(f"\n## CELLxGENE Datasets ({len(datasets)} total)") - for ds in datasets: - ontology = ds.get("ontology_summary", {}) - parts = [ - f"- **{ds['dataset_id']}**: {ds.get('modality', '?')}, {ds['cell_count']:,} cells" - ] - for category in ("cell_types", "tissues", "diseases", "assays"): - terms = ontology.get(category, []) - if terms: - names = [t["name"] for t in terms[:10]] - suffix = f" (+{len(terms) - 10} more)" if len(terms) > 10 else "" - parts.append(f" {category}: {', '.join(names)}{suffix}") - organism = ontology.get("organism", "unknown") - parts.append(f" organism: {organism}") - lines.append("\n".join(parts)) - - lines.append("\nWrite the experimental narrative now.") - return "\n".join(lines) - - -async def run(doi: str = _DEFAULT_DOI) -> None: +def run(doi: str = _DEFAULT_DOI) -> None: + """Fetch, normalize and persist the KG for a CELLxGENE collection DOI.""" logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s - %(message)s", ) - settings = Settings() + + adapter = CellxgeneAdapter() + normalizer = CellxgeneNormalizer() # ------------------------------------------------------------------ - # Step 1: Fetch data directly from Python (no agent tool-calling) + # Step 1: Discover study references for the query. # ------------------------------------------------------------------ - logger.info("Step 1/3: Fetching data for DOI=%s", doi) - - t0 = time.perf_counter() - paper_data = fetch_paper_metadata(doi) - paper_elapsed = time.perf_counter() - t0 - logger.info( - "Paper metadata fetched: title=%r chars=%d (%.2fs)", - paper_data.get("title", "")[:60], - len(json.dumps(paper_data, separators=(",", ":"))), - paper_elapsed, - ) - - t0 = time.perf_counter() - cellxgene_data = fetch_cellxgene_datasets(doi) - cellxgene_elapsed = time.perf_counter() - t0 - n_datasets = len(cellxgene_data.get("datasets", [])) - logger.info( - "CELLxGENE data fetched: datasets=%d chars=%d (%.2fs)", - n_datasets, - len(json.dumps(cellxgene_data, separators=(",", ":"))), - cellxgene_elapsed, - ) - - if "error" in cellxgene_data and not cellxgene_data.get("datasets"): - logger.error("No datasets found: %s", cellxgene_data["error"]) + refs = adapter.discover(doi) + logger.info("Step 1/3: Discovered %d reference(s) for query=%s", len(refs), doi) + if not refs: + logger.error("No study references found for query=%s", doi) return # ------------------------------------------------------------------ - # Step 2: Generate narrative via LLM (with resilience) + # Step 2: Fetch + normalize each reference into a canonical subgraph. # ------------------------------------------------------------------ - logger.info("Step 2/3: Generating experimental narrative via LLM") - prompt = _build_narrative_prompt(paper_data, cellxgene_data) - logger.info("Narrative prompt chars=%d", len(prompt)) - - narrative: str | None = None - async with create_narrative_agent(settings) as agent: - for attempt in range(settings.max_retries): - try: - t0 = time.perf_counter() - result = await agent.run( - prompt, - response_format=NarrativeOutput, - options={"temperature": 0}, - ) - llm_elapsed = time.perf_counter() - t0 - - # Log token usage if available - usage = getattr(result, "usage", None) - if usage: - logger.info( - "LLM usage: prompt_tokens=%s completion_tokens=%s total_tokens=%s (%.2fs)", - getattr(usage, "prompt_tokens", "?"), - getattr(usage, "completion_tokens", "?"), - getattr(usage, "total_tokens", "?"), - llm_elapsed, - ) - else: - logger.info("LLM call completed (%.2fs)", llm_elapsed) - - if result.value: - narrative = result.value.experimental_narrative - logger.info("Narrative generated via response_format: %d chars", len(narrative)) - break - - # Fallback: AzureAIAgentClient may not support response_format, - # in which case result.text is the narrative as plain text. - text = (result.text or "").strip() - if not text: - raise ValueError("Agent returned empty response") - - # Try JSON parse first (model might have returned JSON anyway) - try: - parsed = NarrativeOutput.model_validate_json(text) - narrative = parsed.experimental_narrative - except Exception: - narrative = text - logger.info("Narrative generated via text fallback: %d chars", len(narrative)) - break - - except Exception as exc: - if _is_transient(exc) and attempt < settings.max_retries - 1: - delay = _backoff_delay(attempt) - logger.warning( - "Transient error on attempt %d/%d, retrying in %.1fs: %s", - attempt + 1, - settings.max_retries, - delay, - exc, - ) - await asyncio.sleep(delay) - continue - logger.error("LLM narrative generation failed: %s", exc, exc_info=True) - raise - - if narrative is None: - logger.error("Failed to generate narrative after %d attempts", settings.max_retries) + logger.info("Step 2/3: Fetching and normalizing %d reference(s)", len(refs)) + subgraphs: list[KnowledgeGraphOutput] = [] + for ref in refs: + record = adapter.fetch(ref) + if not record.payload.get("datasets"): + logger.warning( + "No datasets for ref=%s (%s); skipping", + ref, + record.payload.get("error", "empty"), + ) + continue + subgraphs.append(normalizer.normalize(record)) + + if not subgraphs: + logger.error("No datasets fetched for query=%s", doi) return # ------------------------------------------------------------------ - # Step 3: Build the canonical Knowledge Graph programmatically + # Step 3: Persist the canonical KG. # ------------------------------------------------------------------ - # NOTE: the canonical KG no longer stores a narrative. The narrative step - # above is retained only until PR 3 removes the LLM/Azure path entirely - # (see docs/ROADMAP.md); its output is intentionally not fed into the KG. - logger.info("Step 3/3: Assembling knowledge graph") - kg = build_knowledge_graph(paper_data, cellxgene_data) + # PR 3 is single-source/single-study, so there is exactly one subgraph here. + # Merging multiple subgraphs into one graph (deduped by ontology ID) is PR 6. + logger.info("Step 3/3: Writing knowledge graph") + kg = subgraphs[0] print("Knowledge graph constructed successfully:") print(f" Studies: {len(kg.studies)}") @@ -210,13 +86,12 @@ async def run(doi: str = _DEFAULT_DOI) -> None: _OUTPUT_DIR.mkdir(parents=True, exist_ok=True) out_path = _OUTPUT_DIR / "output.json" - payload = kg.model_dump(mode="json") - out_path.write_text(json.dumps(payload, indent=2)) + out_path.write_text(json.dumps(kg.model_dump(mode="json"), indent=2)) print(f"\nSaved to {out_path}") def main() -> None: - asyncio.run(run()) + run() if __name__ == "__main__": diff --git a/src/parce/models/graph_schema.py b/src/parce/models/graph_schema.py index 40ccfaa..af0667f 100644 --- a/src/parce/models/graph_schema.py +++ b/src/parce/models/graph_schema.py @@ -159,21 +159,3 @@ class KnowledgeGraphOutput(BaseModel): samples: list[SampleNode] = Field(default_factory=list) biological_entities: list[BiologicalEntityNode] = Field(default_factory=list) edges: list[GraphEdge] = Field(default_factory=list) - - -class NarrativeOutput(BaseModel): - """Transitional schema for the legacy LLM narrative step. - - The canonical KG no longer stores a narrative; this remains only as the - ``response_format`` for the narrative agent until the whole narrative path is - removed in PR 3 (see docs/ROADMAP.md). - """ - - experimental_narrative: str = Field( - ..., - description=( - "A concise paragraph synthesising the publication abstract " - "with the structured CELLxGENE ontology data to explain " - "how the data was obtained." - ), - ) diff --git a/src/parce/models/narrative.py b/src/parce/models/narrative.py deleted file mode 100644 index 122a7c0..0000000 --- a/src/parce/models/narrative.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Pydantic schemas for the structured experiment narrative output. - -These models define the JSON structure the agent produces via structured output -(``response_format``). They also serve as validation schemas for any downstream -consumer of the narrative data. - -All models set ``extra="forbid"`` which is required by the Azure OpenAI -structured-output API for nested Pydantic models. -""" - -from __future__ import annotations - -from pydantic import BaseModel, ConfigDict, Field - - -class DataURI(BaseModel): - """A URI reference to a raw data file (e.g. FASTQ, BAM) in a public repository.""" - - model_config = ConfigDict(extra="forbid") - - uri: str = Field( - ..., - description="Full URI to the data file (e.g. https://sra-pub-run-odp.s3.amazonaws.com/...).", - ) - file_type: str = Field( - ..., - description="File format such as FASTQ, BAM, or H5AD.", - ) - description: str | None = Field( - default=None, - description="Optional human-readable note about this data file.", - ) - - -class SampleRecord(BaseModel): - """Metadata for a single biological sample within an experiment.""" - - model_config = ConfigDict(extra="forbid") - - sample_id: str = Field(..., description="Repository sample accession (e.g. GSM1234567).") - organism: str = Field(..., description="Species name (e.g. Mus musculus).") - strain: str | None = Field(default=None, description="Strain or genetic background.") - cell_type: str | None = Field( - default=None, description="Cell type profiled (e.g. CD8+ T cell)." - ) - tissue: str | None = Field(default=None, description="Tissue of origin (e.g. spleen).") - condition: str | None = Field( - default=None, - description="Experimental condition (e.g. knockout, stimulated, control).", - ) - knockout_gene: str | None = Field( - default=None, - description="Gene knocked out, if applicable.", - ) - data_uris: list[DataURI] = Field( - default_factory=list, - description="Raw data files associated with this sample.", - ) - - -class ExperimentNarrative(BaseModel): - """Top-level structured output: a narrative description of a public experiment - interleaved with sample records and data URIs. - - This is the schema passed as ``response_format`` to the agent so the LLM - returns strictly typed JSON. - """ - - model_config = ConfigDict(extra="forbid") - - accession: str = Field(..., description="Primary accession (e.g. GSE164378).") - title: str = Field(..., description="Experiment title as listed in the repository.") - summary: str = Field( - ..., - description=( - "A narrative paragraph describing how the data was obtained, " - "including organism, experimental design, and sequencing approach." - ), - ) - platform: str | None = Field( - default=None, - description="Sequencing platform (e.g. Illumina NovaSeq 6000).", - ) - samples: list[SampleRecord] = Field( - default_factory=list, - description="Individual sample records with metadata and data URIs.", - ) diff --git a/src/parce/models/raw_record.py b/src/parce/models/raw_record.py new file mode 100644 index 0000000..d48723a --- /dev/null +++ b/src/parce/models/raw_record.py @@ -0,0 +1,41 @@ +"""Source-shaped intermediate record that bridges adapters and normalizers. + +A :class:`SourceAdapter` produces ``RawRecord`` objects; the matching +:class:`~parce.normalize.base.Normalizer` consumes them and emits canonical KG +nodes. The record is *source-shaped*, not canonical: its ``payload`` holds +whatever structure the repository exposes (free text, nested dicts, ontology +summaries). The canonical schema is imposed downstream, in the normalizer. + +The identifying fields ``source``, ``study_id`` and ``title`` are lifted out of +the payload because every source has them and every normalizer needs them; +everything else stays inside ``payload``. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class RawRecord(BaseModel): + """One study's data as returned by a source adapter, before normalization.""" + + model_config = ConfigDict(extra="forbid") + + source: str = Field( + ..., + description="Provenance label written by the adapter (e.g. 'CELLxGENE', 'GEO').", + ) + study_id: str = Field( + ..., + description="Stable study identifier: a DOI or repository accession.", + ) + title: str = Field( + default="", + description="Study title, if the source exposes one (empty otherwise).", + ) + payload: dict[str, Any] = Field( + default_factory=dict, + description="Source-shaped raw data consumed by the matching Normalizer.", + ) diff --git a/src/parce/normalize/__init__.py b/src/parce/normalize/__init__.py new file mode 100644 index 0000000..ca71efc --- /dev/null +++ b/src/parce/normalize/__init__.py @@ -0,0 +1,8 @@ +"""Normalizers: map a source's :class:`~parce.models.raw_record.RawRecord` into +canonical KG nodes. + +A normalizer is deterministic for structured sources (a pure mapping) and +agent-backed for unstructured ones (an LLM constrained to the canonical schema +via ``response_format``). Either way it emits the same +:class:`~parce.models.graph_schema.KnowledgeGraphOutput` contract. +""" diff --git a/src/parce/normalize/base.py b/src/parce/normalize/base.py new file mode 100644 index 0000000..ccc3030 --- /dev/null +++ b/src/parce/normalize/base.py @@ -0,0 +1,29 @@ +"""The :class:`Normalizer` contract. + +A normalizer maps one source's :class:`~parce.models.raw_record.RawRecord` into +canonical KG nodes and edges. The implementation may be: + +* **deterministic** — a pure structural mapping, used when the source already + ships structured, ontology-grounded metadata (e.g. CELLxGENE); or +* **agent-backed** — an LLM constrained to emit the canonical schema via + ``response_format``, used for free-text sources (e.g. GEO, PRIDE). + +Both kinds return the same :class:`~parce.models.graph_schema.KnowledgeGraphOutput`, +so everything downstream (merge, export) is source-agnostic. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from parce.models.graph_schema import KnowledgeGraphOutput +from parce.models.raw_record import RawRecord + + +@runtime_checkable +class Normalizer(Protocol): + """Maps a source-shaped ``RawRecord`` into canonical nodes and edges.""" + + def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: + """Return the canonical single-study subgraph for ``record``.""" + ... diff --git a/src/parce/normalize/cellxgene.py b/src/parce/normalize/cellxgene.py new file mode 100644 index 0000000..8ba1413 --- /dev/null +++ b/src/parce/normalize/cellxgene.py @@ -0,0 +1,160 @@ +"""Deterministic normalizer: a CELLxGENE ``RawRecord`` → canonical KG nodes. + +No LLM is involved: CELLxGENE Census already ships ontology-grounded terms, so +this is a pure structural mapping. Two design rules show up directly here: + +* **Cell type is never consumed** — the adapter does not even read it + (data-inferred → leakage; see docs/ARCHITECTURE.md §1). +* **Census is dataset-level**, not per-sample in the GEO sense, so no + ``SampleNode`` records are emitted yet (open question in ARCHITECTURE §7). + +The free-text → ontology-ID step (here, organism string → NCBITaxon) is a +hardcoded map for now; it becomes the shared OntologyResolver stage in PR 4. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from parce.models.graph_schema import ( + BiologicalEntityNode, + DatasetNode, + EntityType, + GraphEdge, + KnowledgeGraphOutput, + StudyNode, +) +from parce.models.raw_record import RawRecord + +logger = logging.getLogger(__name__) + +# High-level study modality for everything CELLxGENE ingests. (Refined into an +# EFO ``assay`` term + derived ``molecular_layer`` in PR 4.) +_STUDY_MODALITY = "scRNA-seq" + +# Organism free-text (as Census returns it) → (NCBITaxon ID, canonical name). +# Stand-in for the PR 4 OntologyResolver. +_ORGANISM_ONTOLOGY: dict[str, tuple[str, str]] = { + "Homo sapiens": ("NCBITaxon:9606", "Homo sapiens"), + "Mus musculus": ("NCBITaxon:10090", "Mus musculus"), + "homo_sapiens": ("NCBITaxon:9606", "Homo sapiens"), + "mus_musculus": ("NCBITaxon:10090", "Mus musculus"), +} + +# Ontology categories that become design-context entities. Cell types are +# deliberately absent (data-inferred → leakage). +_CATEGORY_TO_ENTITY_TYPE: dict[str, EntityType] = { + "tissues": EntityType.TISSUE, + "diseases": EntityType.DISEASE, + "assays": EntityType.ASSAY, +} + +_CATEGORY_TO_RELATION: dict[str, str] = { + "tissues": "HAS_TISSUE", + "diseases": "HAS_CONDITION", + "assays": "MEASURED_WITH", +} + + +class CellxgeneNormalizer: + """:class:`~parce.normalize.base.Normalizer` for CELLxGENE ``RawRecord``s.""" + + def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: + """Assemble the canonical single-study subgraph for one CELLxGENE study.""" + study_id = record.study_id + + study = StudyNode( + study_id=study_id, + title=record.title, + source=record.source, + modality=_STUDY_MODALITY, + ) + + datasets: list[DatasetNode] = [] + edges: list[GraphEdge] = [] + entity_registry: dict[str, BiologicalEntityNode] = {} + species_seen: set[str] = set() + + for ds in record.payload.get("datasets", []): + dataset_id = ds["dataset_id"] + + datasets.append( + DatasetNode( + dataset_id=dataset_id, + data_uri=ds["h5ad_uri"], + assay=ds.get("modality", "unknown"), + cell_count=ds["cell_count"], + ) + ) + + edges.append( + GraphEdge( + source_id=dataset_id, + target_id=study_id, + relation_type="EXTRACTED_FROM", + ) + ) + + ontology: dict[str, Any] = ds.get("ontology_summary", {}) + + # Register species from the organism field. + organism_key = ontology.get("organism", "unknown") + if organism_key in _ORGANISM_ONTOLOGY and organism_key not in species_seen: + ont_id, name = _ORGANISM_ONTOLOGY[organism_key] + species_seen.add(organism_key) + entity_registry[ont_id] = BiologicalEntityNode( + entity_type=EntityType.SPECIES, + ontology_id=ont_id, + name=name, + ) + + # Register entities and create edges per design-context category. + for category, entity_type in _CATEGORY_TO_ENTITY_TYPE.items(): + relation = _CATEGORY_TO_RELATION[category] + for term in ontology.get(category, []): + ont_id = term.get("ontology_id", "unknown") + name = term.get("name", "unknown") + + if ont_id not in entity_registry: + entity_registry[ont_id] = BiologicalEntityNode( + entity_type=entity_type, + ontology_id=ont_id, + name=name, + ) + + edges.append( + GraphEdge( + source_id=dataset_id, + target_id=ont_id, + relation_type=relation, + ) + ) + + # Study → Species edges. + for organism_key in species_seen: + species_id = _ORGANISM_ONTOLOGY[organism_key][0] + edges.append( + GraphEdge( + source_id=study_id, + target_id=species_id, + relation_type="STUDIES", + ) + ) + + kg = KnowledgeGraphOutput( + studies=[study], + datasets=datasets, + samples=[], + biological_entities=list(entity_registry.values()), + edges=edges, + ) + + logger.info( + "Normalized CELLxGENE study=%s: datasets=%d entities=%d edges=%d", + study_id, + len(kg.datasets), + len(kg.biological_entities), + len(kg.edges), + ) + return kg diff --git a/src/parce/sources/__init__.py b/src/parce/sources/__init__.py new file mode 100644 index 0000000..2745bf8 --- /dev/null +++ b/src/parce/sources/__init__.py @@ -0,0 +1,6 @@ +"""Source adapters: one per repository, isolating all network IO. + +Each adapter implements :class:`~parce.sources.base.SourceAdapter` and turns a +query into source-shaped :class:`~parce.models.raw_record.RawRecord` objects. It +never produces canonical nodes — that is the normalizer's job. +""" diff --git a/src/parce/sources/base.py b/src/parce/sources/base.py new file mode 100644 index 0000000..eb97117 --- /dev/null +++ b/src/parce/sources/base.py @@ -0,0 +1,37 @@ +"""The :class:`SourceAdapter` contract. + +An adapter is the only place network IO for a given repository lives. It does +two things and nothing more: + +* ``discover(query)`` — resolve a query (a DOI, an accession, a search term) into + a list of study references the adapter knows how to fetch. +* ``fetch(ref)`` — pull one reference into a source-shaped + :class:`~parce.models.raw_record.RawRecord`. + +Adapters never produce canonical KG nodes; mapping a ``RawRecord`` into the +canonical schema is the matching :class:`~parce.normalize.base.Normalizer`'s job. +Keeping the two split means a structured source (deterministic normalizer) and an +unstructured source (agent-backed normalizer) share the exact same adapter shape. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from parce.models.raw_record import RawRecord + + +@runtime_checkable +class SourceAdapter(Protocol): + """Per-repository adapter: discover study references, fetch raw records.""" + + #: Provenance label written onto every ``RawRecord`` this adapter emits. + source_name: str + + def discover(self, query: str) -> list[str]: + """Resolve a query into study references this adapter can ``fetch``.""" + ... + + def fetch(self, ref: str) -> RawRecord: + """Fetch one study reference into a source-shaped ``RawRecord``.""" + ... diff --git a/src/parce/tools/cellxgene_fetcher.py b/src/parce/sources/cellxgene.py similarity index 61% rename from src/parce/tools/cellxgene_fetcher.py rename to src/parce/sources/cellxgene.py index 8d99502..3795c9d 100644 --- a/src/parce/tools/cellxgene_fetcher.py +++ b/src/parce/sources/cellxgene.py @@ -1,29 +1,38 @@ -"""Tool for fetching dataset metadata and ontology terms from CELLxGENE Census. +"""CELLxGENE Census source adapter — deterministic, no LLM in the path. -Queries the Census datasets table by collection DOI, retrieves per-dataset -H5AD URIs, and summarises the unique ontology terms (cell types, tissues, -diseases, assays) found in the cell metadata. +Queries the Census datasets table by collection DOI, resolves each dataset's +H5AD URI, and summarises the *design-context* ontology terms (tissue, disease, +assay, organism) found in the cell metadata. CELLxGENE already ships these terms +ontology-grounded, so no extraction agent is needed. + +Cell type is intentionally **not** read: it is a data-inferred annotation (called +from expression), not an experiment-design variable, and carrying it would leak +the very signal the downstream model must learn. See docs/ARCHITECTURE.md §1. + +The Census/EuropePMC calls are the only network IO here; the matching mapper is +:class:`parce.normalize.cellxgene.CellxgeneNormalizer`. """ from __future__ import annotations -import json import logging import time from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Annotated +from typing import Any import cellxgene_census -from agent_framework import tool -from pydantic import Field + +from parce.models.raw_record import RawRecord +from parce.sources.publication import fetch_paper_metadata logger = logging.getLogger(__name__) +SOURCE_NAME = "CELLxGENE" + _MAX_TERMS_PER_CATEGORY = 50 +# Cell-type columns are deliberately omitted (data-inferred → leakage). _ONTOLOGY_COLUMNS = [ - "cell_type", - "cell_type_ontology_term_id", "tissue", "tissue_ontology_term_id", "disease", @@ -32,6 +41,7 @@ "assay_ontology_term_id", ] +# Census splits cell metadata per organism; we try each until one yields rows. _ORGANISM_CANDIDATES = [ "Homo sapiens", "Mus musculus", @@ -39,15 +49,8 @@ "mus_musculus", ] -_ORGANISM_ONTOLOGY: dict[str, tuple[str, str]] = { - "Homo sapiens": ("NCBITaxon:9606", "Homo sapiens"), - "Mus musculus": ("NCBITaxon:10090", "Mus musculus"), - "homo_sapiens": ("NCBITaxon:9606", "Homo sapiens"), - "mus_musculus": ("NCBITaxon:10090", "Mus musculus"), -} - -def _extract_term_pairs(obs, name_col: str, id_col: str) -> list[dict[str, str]]: +def _extract_term_pairs(obs: Any, name_col: str, id_col: str) -> list[dict[str, str]]: """Deduplicate and return structured ontology term pairs, capped.""" unique = obs[[name_col, id_col]].drop_duplicates() terms = sorted( @@ -57,8 +60,8 @@ def _extract_term_pairs(obs, name_col: str, id_col: str) -> list[dict[str, str]] return terms[:_MAX_TERMS_PER_CATEGORY] -def _summarise_ontology_terms(census, dataset_id: str) -> dict: - """Return structured ontology term data for a single dataset.""" +def _summarise_ontology_terms(census: Any, dataset_id: str) -> dict[str, Any]: + """Return structured design-context ontology data for a single dataset.""" last_error: str | None = None for organism in _ORGANISM_CANDIDATES: try: @@ -86,7 +89,6 @@ def _summarise_ontology_terms(census, dataset_id: str) -> dict: return { "organism": organism, "modality": modality, - "cell_types": _extract_term_pairs(obs, "cell_type", "cell_type_ontology_term_id"), "tissues": _extract_term_pairs(obs, "tissue", "tissue_ontology_term_id"), "diseases": _extract_term_pairs(obs, "disease", "disease_ontology_term_id"), "assays": _extract_term_pairs(obs, "assay", "assay_ontology_term_id"), @@ -101,10 +103,9 @@ def _summarise_ontology_terms(census, dataset_id: str) -> dict: ) continue - empty: dict = { + empty: dict[str, Any] = { "organism": "unknown", "modality": "unknown", - "cell_types": [], "tissues": [], "diseases": [], "assays": [], @@ -114,7 +115,7 @@ def _summarise_ontology_terms(census, dataset_id: str) -> dict: return empty -def _process_single_dataset(census, row) -> dict: +def _process_single_dataset(census: Any, row: Any) -> dict[str, Any]: """Resolve URI and ontology terms for one dataset row (thread-safe).""" dataset_id = row["dataset_id"] @@ -142,11 +143,13 @@ def _process_single_dataset(census, row) -> dict: } -def fetch_cellxgene_datasets(doi: str, *, max_workers: int = 4) -> dict: - """Core function: fetch CELLxGENE data and return a Python dict. +def fetch_cellxgene_datasets(doi: str, *, max_workers: int = 4) -> dict[str, Any]: + """Fetch CELLxGENE dataset metadata for a collection DOI as a plain dict. - This is the programmatic entry point used by the orchestrator. - Returns structured ontology terms as lists of ``{"name": ..., "ontology_id": ...}`` dicts. + Returns ``{"doi": doi, "datasets": [...]}`` where each dataset carries its + H5AD URI, cell count and a structured ontology summary (tissues, diseases, + assays — each a list of ``{"name", "ontology_id"}`` — plus organism). When no + dataset matches the DOI, ``datasets`` is empty and an ``error`` key is added. Per-dataset URI resolution and ontology queries run in parallel threads. """ t_all = time.perf_counter() @@ -173,10 +176,11 @@ def fetch_cellxgene_datasets(doi: str, *, max_workers: int = 4) -> dict: rows = [row for _, row in matched.iterrows()] + results: list[dict[str, Any]] if total == 1: results = [_process_single_dataset(census, rows[0])] else: - results = [None] * total + results = [{} for _ in range(total)] with ThreadPoolExecutor(max_workers=min(max_workers, total)) as pool: future_to_idx = { pool.submit(_process_single_dataset, census, row): i @@ -199,14 +203,41 @@ def fetch_cellxgene_datasets(doi: str, *, max_workers: int = 4) -> dict: census.close() -@tool -def fetch_cellxgene_data( - doi: Annotated[str, Field(description="Collection DOI (e.g. '10.1038/s41586-023-05869-0')")], -) -> str: - """Fetch dataset metadata and ontology summaries from CELLxGENE Census for a DOI. +class CellxgeneAdapter: + """:class:`~parce.sources.base.SourceAdapter` for CELLxGENE Census. - Returns a JSON string listing each dataset associated with the DOI, - including the remote H5AD URI, cell count, and unique ontology terms - (cell types, tissues, diseases, assays). + Deterministic: CELLxGENE already emits ontology-grounded terms, so there is + no LLM anywhere in this source's path. """ - return json.dumps(fetch_cellxgene_datasets(doi), separators=(",", ":")) + + source_name = SOURCE_NAME + + def discover(self, query: str) -> list[str]: + """Resolve ``query`` (a collection DOI) to study references. + + For CELLxGENE a study is a Census collection identified by its DOI, so the + DOI *is* the reference and ``discover`` is the identity on it. Keyword + collection search is future work (see docs/ROADMAP.md backlog). + """ + return [query] + + def fetch(self, ref: str, *, max_workers: int = 4) -> RawRecord: + """Fetch one collection DOI into a source-shaped ``RawRecord``. + + Gathers the publication title (EuropePMC) and the per-dataset Census + metadata. The payload carries ``datasets`` (possibly empty) and, when the + DOI matched nothing, an ``error`` string for the caller to act on. + """ + paper = fetch_paper_metadata(ref) + cellxgene = fetch_cellxgene_datasets(ref, max_workers=max_workers) + + payload: dict[str, Any] = {"datasets": cellxgene.get("datasets", [])} + if "error" in cellxgene: + payload["error"] = cellxgene["error"] + + return RawRecord( + source=self.source_name, + study_id=ref, + title=paper.get("title", ""), + payload=payload, + ) diff --git a/src/parce/tools/ncbi_fetcher.py b/src/parce/sources/publication.py similarity index 52% rename from src/parce/tools/ncbi_fetcher.py rename to src/parce/sources/publication.py index f3a8de3..231b42d 100644 --- a/src/parce/tools/ncbi_fetcher.py +++ b/src/parce/sources/publication.py @@ -1,28 +1,29 @@ -"""Tool for fetching publication title and abstract via the EuropePMC REST API. +"""Publication metadata helper: a study's title/abstract for a DOI via EuropePMC. -Uses the free EuropePMC search endpoint (no API key required) to retrieve -core metadata for a given DOI. +Shared by source adapters that need a study's *publication* title — for example +CELLxGENE, whose Census exposes per-dataset titles and the collection DOI but not +the publication title. EuropePMC's search endpoint is free and needs no API key. + +This is a deterministic API call (no LLM), so it lives in ``sources/`` and is +covered by mypy like the rest of the stable core. """ from __future__ import annotations -import json import logging -from typing import Annotated import requests -from agent_framework import tool -from pydantic import Field logger = logging.getLogger(__name__) _EUROPEPMC_SEARCH_URL = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" -def fetch_paper_metadata(doi: str) -> dict: - """Core function: fetch publication metadata and return a Python dict. +def fetch_paper_metadata(doi: str) -> dict[str, str]: + """Fetch publication title and abstract for ``doi`` from EuropePMC. - This is the programmatic entry point used by the orchestrator. + Returns a dict with keys ``doi``, ``title`` and ``abstract``. If the DOI is + not found, ``title``/``abstract`` are empty and an ``error`` key is added. """ logger.info("Fetching paper metadata for DOI=%s", doi) resp = requests.get( @@ -53,15 +54,3 @@ def fetch_paper_metadata(doi: str) -> dict: "title": paper.get("title", ""), "abstract": paper.get("abstractText", ""), } - - -@tool -def fetch_paper_context( - doi: Annotated[str, Field(description="Publication DOI (e.g. '10.1038/s41586-023-05869-0')")], -) -> str: - """Fetch the title and abstract of a publication from EuropePMC. - - Returns a JSON string with keys ``doi``, ``title``, and ``abstract``. - If the DOI is not found, returns an error message. - """ - return json.dumps(fetch_paper_metadata(doi), separators=(",", ":")) diff --git a/src/parce/tools/__init__.py b/src/parce/tools/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/parce/tools/geo_fetcher.py b/src/parce/tools/geo_fetcher.py deleted file mode 100644 index a004f6a..0000000 --- a/src/parce/tools/geo_fetcher.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Stub tool for fetching experiment metadata from NCBI GEO. - -Replace the mock implementation with real GEO E-utilities / Entrez API calls -once the core agent loop is validated. -""" - -from __future__ import annotations - -import json -from typing import Annotated - -from agent_framework import tool -from pydantic import Field - -_MOCK_METADATA = { - "GSE164378": { - "accession": "GSE164378", - "title": "Single-cell multi-omic profiling of human T cells during anti-PD-1 therapy", - "organism": "Homo sapiens", - "platform": "Illumina NovaSeq 6000", - "summary": ( - "Peripheral blood and tumor-infiltrating T cells were profiled " - "using paired scRNA-seq and scATAC-seq before and after anti-PD-1 " - "immunotherapy in melanoma patients." - ), - "samples": [ - { - "sample_id": "GSM5008101", - "organism": "Homo sapiens", - "cell_type": "CD8+ T cell", - "tissue": "peripheral blood", - "condition": "pre-treatment", - "data_uris": [ - { - "uri": "https://sra-pub-run-odp.s3.amazonaws.com/sra/SRR13568101/SRR13568101", - "file_type": "FASTQ", - } - ], - }, - { - "sample_id": "GSM5008102", - "organism": "Homo sapiens", - "cell_type": "CD8+ T cell", - "tissue": "tumor", - "condition": "post-treatment", - "data_uris": [ - { - "uri": "https://sra-pub-run-odp.s3.amazonaws.com/sra/SRR13568102/SRR13568102", - "file_type": "FASTQ", - } - ], - }, - ], - } -} - - -@tool -def fetch_geo_metadata( - accession: Annotated[str, Field(description="GEO series accession ID (e.g. GSE164378)")], -) -> str: - """Fetch experiment metadata from NCBI GEO for a given accession. - - Returns a JSON string containing the experiment title, organism, - platform, summary, and per-sample metadata with data-file URIs. - """ - if accession in _MOCK_METADATA: - return json.dumps(_MOCK_METADATA[accession], indent=2) - return json.dumps({"error": f"Accession {accession} not found (stub data only)."}) diff --git a/tests/test_builder.py b/tests/test_builder.py deleted file mode 100644 index 21d369e..0000000 --- a/tests/test_builder.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Unit tests for the programmatic Knowledge Graph builder.""" - -from __future__ import annotations - -from parce.graph.builder import build_knowledge_graph -from parce.models.graph_schema import EntityType - -_PAPER_DATA = { - "doi": "10.1234/test", - "title": "Test Study", - "abstract": "We profiled T cells.", -} - -# Note: ``cell_types`` are present in the input but must be ignored by the -# builder (CellType is a data-inferred annotation, intentionally excluded). -# ``blood`` (UBERON:0000178) appears in both datasets to exercise dedup. -_CELLXGENE_DATA = { - "doi": "10.1234/test", - "datasets": [ - { - "dataset_id": "ds-001", - "dataset_title": "Dataset One", - "h5ad_uri": "s3://bucket/ds-001.h5ad", - "modality": "10x 3' v3", - "cell_count": 5000, - "ontology_summary": { - "organism": "Homo sapiens", - "modality": "10x 3' v3", - "cell_types": [ - {"name": "T cell", "ontology_id": "CL:0000084"}, - {"name": "B cell", "ontology_id": "CL:0000236"}, - ], - "tissues": [ - {"name": "blood", "ontology_id": "UBERON:0000178"}, - ], - "diseases": [ - {"name": "normal", "ontology_id": "PATO:0000461"}, - ], - "assays": [ - {"name": "10x 3' v3", "ontology_id": "EFO:0009922"}, - ], - }, - }, - { - "dataset_id": "ds-002", - "dataset_title": "Dataset Two", - "h5ad_uri": "s3://bucket/ds-002.h5ad", - "modality": "Smart-seq2", - "cell_count": 1000, - "ontology_summary": { - "organism": "Homo sapiens", - "modality": "Smart-seq2", - "cell_types": [ - {"name": "T cell", "ontology_id": "CL:0000084"}, - ], - "tissues": [ - {"name": "blood", "ontology_id": "UBERON:0000178"}, - {"name": "lung", "ontology_id": "UBERON:0002048"}, - ], - "diseases": [], - "assays": [ - {"name": "Smart-seq2", "ontology_id": "EFO:0008931"}, - ], - }, - }, - ], -} - - -class TestBuildKnowledgeGraph: - def test_basic_structure(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - assert len(kg.studies) == 1 - assert kg.studies[0].study_id == "10.1234/test" - assert kg.studies[0].title == "Test Study" - assert kg.studies[0].source == "CELLxGENE" - assert kg.studies[0].modality == "scRNA-seq" - - assert len(kg.datasets) == 2 - assert kg.datasets[0].dataset_id == "ds-001" - assert kg.datasets[0].data_uri == "s3://bucket/ds-001.h5ad" - assert kg.datasets[0].assay == "10x 3' v3" - assert kg.datasets[1].dataset_id == "ds-002" - - def test_cell_type_excluded(self): - """Cell types in the input must not produce entities or edges.""" - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - names = {e.name for e in kg.biological_entities} - assert "T cell" not in names - assert "B cell" not in names - - ontology_ids = {e.ontology_id for e in kg.biological_entities} - assert "CL:0000084" not in ontology_ids - assert "CL:0000236" not in ontology_ids - - edge_targets = {e.target_id for e in kg.edges} - assert "CL:0000084" not in edge_targets - - def test_tissue_entity_deduplication(self): - """blood (UBERON:0000178) appears in both datasets but is one entity.""" - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - entity_ids = [e.ontology_id for e in kg.biological_entities] - assert entity_ids.count("UBERON:0000178") == 1 - - def test_species_entity_created(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - species = [e for e in kg.biological_entities if e.entity_type == EntityType.SPECIES] - assert len(species) == 1 - assert species[0].ontology_id == "NCBITaxon:9606" - assert species[0].name == "Homo sapiens" - - def test_no_samples_for_cellxgene(self): - """Census is dataset-level; no SampleNode records are emitted (yet).""" - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - assert kg.samples == [] - - def test_extracted_from_edges(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - extracted = [e for e in kg.edges if e.relation_type == "EXTRACTED_FROM"] - assert len(extracted) == 2 - assert {e.source_id for e in extracted} == {"ds-001", "ds-002"} - assert all(e.target_id == "10.1234/test" for e in extracted) - - def test_has_tissue_edges(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - tissue_edges = [e for e in kg.edges if e.relation_type == "HAS_TISSUE"] - pairs = {(e.source_id, e.target_id) for e in tissue_edges} - assert ("ds-001", "UBERON:0000178") in pairs - assert ("ds-002", "UBERON:0000178") in pairs - assert ("ds-002", "UBERON:0002048") in pairs - - def test_has_condition_edges(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - conditions = [e for e in kg.edges if e.relation_type == "HAS_CONDITION"] - assert any(e.target_id == "PATO:0000461" for e in conditions) - - def test_measured_with_edges(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - assay_edges = [e for e in kg.edges if e.relation_type == "MEASURED_WITH"] - assert any(e.source_id == "ds-001" and e.target_id == "EFO:0009922" for e in assay_edges) - assert any(e.source_id == "ds-002" and e.target_id == "EFO:0008931" for e in assay_edges) - - def test_studies_edge(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - - studies = [e for e in kg.edges if e.relation_type == "STUDIES"] - assert len(studies) == 1 - assert studies[0].source_id == "10.1234/test" - assert studies[0].target_id == "NCBITaxon:9606" - - def test_empty_datasets(self): - kg = build_knowledge_graph(_PAPER_DATA, {"doi": "10.1234/test", "datasets": []}) - - assert len(kg.studies) == 1 - assert len(kg.datasets) == 0 - assert len(kg.samples) == 0 - assert len(kg.biological_entities) == 0 - assert len(kg.edges) == 0 - - def test_roundtrip_json(self): - kg = build_knowledge_graph(_PAPER_DATA, _CELLXGENE_DATA) - json_str = kg.model_dump_json() - from parce.models.graph_schema import KnowledgeGraphOutput - - restored = KnowledgeGraphOutput.model_validate_json(json_str) - assert restored == kg diff --git a/tests/test_cellxgene_integration.py b/tests/test_cellxgene_integration.py index a87443a..fb2a0ff 100644 --- a/tests/test_cellxgene_integration.py +++ b/tests/test_cellxgene_integration.py @@ -9,45 +9,19 @@ from __future__ import annotations -import json - import pytest -from parce.tools.cellxgene_fetcher import fetch_cellxgene_data, fetch_cellxgene_datasets +from parce.sources.cellxgene import CellxgeneAdapter, fetch_cellxgene_datasets pytestmark = pytest.mark.integration _TEST_DOI = "10.1038/s41467-025-63202-x" -class TestCellxgeneFetcherTool: - """Test the @tool-decorated JSON-string version.""" - - async def test_fetch_cellxgene_data_returns_datasets(self): - raw = fetch_cellxgene_data(_TEST_DOI) - payload = json.loads(raw) - - assert payload["doi"] == _TEST_DOI - assert "datasets" in payload - assert len(payload["datasets"]) > 0 - assert "ontology_summary" in payload["datasets"][0] - - any_terms = any( - ( - d.get("ontology_summary", {}).get("cell_types") - or d.get("ontology_summary", {}).get("tissues") - or d.get("ontology_summary", {}).get("diseases") - or d.get("ontology_summary", {}).get("assays") - ) - for d in payload["datasets"] - ) - assert any_terms, "No ontology terms extracted from any matched dataset" - - class TestCellxgeneFetcherCore: - """Test the dict-returning core function.""" + """Test the dict-returning core fetch function.""" - async def test_fetch_cellxgene_datasets_structured_terms(self): + def test_fetch_cellxgene_datasets_structured_terms(self): payload = fetch_cellxgene_datasets(_TEST_DOI) assert payload["doi"] == _TEST_DOI @@ -58,16 +32,39 @@ async def test_fetch_cellxgene_datasets_structured_terms(self): assert ds["modality"] != "", "modality should be populated" ontology = ds.get("ontology_summary", {}) - for category in ("cell_types", "tissues", "diseases", "assays"): + # Cell type is deliberately not extracted (data-inferred → leakage). + assert "cell_types" not in ontology + for category in ("tissues", "diseases", "assays"): terms = ontology.get(category, []) if terms: assert isinstance(terms[0], dict), f"{category} terms should be dicts" assert "name" in terms[0] assert "ontology_id" in terms[0] - async def test_modality_populated(self): + def test_modality_populated(self): payload = fetch_cellxgene_datasets(_TEST_DOI) any_modality = any( ds.get("modality") and ds["modality"] != "unknown" for ds in payload["datasets"] ) assert any_modality, "At least one dataset should have a known modality" + + +class TestCellxgeneAdapter: + """Test the SourceAdapter against live Census + EuropePMC.""" + + def test_fetch_returns_raw_record(self): + record = CellxgeneAdapter().fetch(_TEST_DOI) + + assert record.source == "CELLxGENE" + assert record.study_id == _TEST_DOI + assert len(record.payload["datasets"]) > 0 + + any_terms = any( + ( + d.get("ontology_summary", {}).get("tissues") + or d.get("ontology_summary", {}).get("diseases") + or d.get("ontology_summary", {}).get("assays") + ) + for d in record.payload["datasets"] + ) + assert any_terms, "No ontology terms extracted from any matched dataset" diff --git a/tests/test_graph_schema.py b/tests/test_graph_schema.py index 5c0ceea..809e6cc 100644 --- a/tests/test_graph_schema.py +++ b/tests/test_graph_schema.py @@ -11,7 +11,6 @@ EntityType, GraphEdge, KnowledgeGraphOutput, - NarrativeOutput, SampleNode, StudyNode, ) @@ -210,18 +209,3 @@ def test_roundtrip_json(self): def test_extra_forbidden(self): with pytest.raises(ValidationError): KnowledgeGraphOutput(metadata={"bad": True}) - - -class TestNarrativeOutput: - def test_valid(self): - n = NarrativeOutput(experimental_narrative="A narrative.") - assert n.experimental_narrative == "A narrative." - - def test_missing_field(self): - with pytest.raises(ValidationError): - NarrativeOutput() - - def test_roundtrip_json(self): - n = NarrativeOutput(experimental_narrative="Test narrative.") - restored = NarrativeOutput.model_validate_json(n.model_dump_json()) - assert restored == n diff --git a/tests/test_integration.py b/tests/test_integration.py index 8ecdf1d..82b62c7 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -37,15 +37,3 @@ async def test_credential_get_token(self): token = await credential.get_token("https://cognitiveservices.azure.com/.default") assert token.token assert len(token.token) > 0 - - -class TestAgentCreation: - """Verify the agent can be created against the live Foundry project.""" - - async def test_create_narrative_agent(self): - """The narrative agent context manager yields a usable Agent.""" - from parce.agent.curator import create_narrative_agent - - async with create_narrative_agent() as agent: - assert agent is not None - assert agent.name == "PARCE" diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index db28475..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Basic validation tests for the Pydantic narrative models.""" - -from __future__ import annotations - -import pytest -from pydantic import ValidationError - -from parce.models.narrative import DataURI, ExperimentNarrative, SampleRecord - - -class TestDataURI: - def test_valid(self): - uri = DataURI(uri="https://example.com/file.fastq.gz", file_type="FASTQ") - assert uri.uri == "https://example.com/file.fastq.gz" - assert uri.description is None - - def test_extra_fields_forbidden(self): - with pytest.raises(ValidationError): - DataURI(uri="https://example.com/f.bam", file_type="BAM", extra_field="oops") - - -class TestSampleRecord: - def test_minimal(self): - sample = SampleRecord(sample_id="GSM0001", organism="Mus musculus") - assert sample.cell_type is None - assert sample.data_uris == [] - - def test_full(self): - sample = SampleRecord( - sample_id="GSM0002", - organism="Homo sapiens", - strain=None, - cell_type="CD8+ T cell", - tissue="spleen", - condition="knockout", - knockout_gene="Pdcd1", - data_uris=[ - DataURI(uri="https://example.com/SRR001.fastq", file_type="FASTQ"), - ], - ) - assert len(sample.data_uris) == 1 - assert sample.knockout_gene == "Pdcd1" - - def test_extra_fields_forbidden(self): - with pytest.raises(ValidationError): - SampleRecord(sample_id="GSM0003", organism="Mus musculus", bogus=True) - - -class TestExperimentNarrative: - def test_minimal(self): - narrative = ExperimentNarrative( - accession="GSE000001", - title="Test experiment", - summary="A short summary.", - ) - assert narrative.samples == [] - assert narrative.platform is None - - def test_roundtrip_json(self): - narrative = ExperimentNarrative( - accession="GSE164378", - title="T-cell profiling", - summary="Profiled T cells under PD-1 blockade.", - platform="Illumina NovaSeq 6000", - samples=[ - SampleRecord( - sample_id="GSM5008101", - organism="Homo sapiens", - cell_type="CD8+ T cell", - tissue="peripheral blood", - condition="pre-treatment", - data_uris=[ - DataURI( - uri="https://example.com/SRR13568101", - file_type="FASTQ", - ) - ], - ) - ], - ) - json_str = narrative.model_dump_json() - restored = ExperimentNarrative.model_validate_json(json_str) - assert restored == narrative - - def test_extra_fields_forbidden(self): - with pytest.raises(ValidationError): - ExperimentNarrative( - accession="GSE000002", - title="Bad", - summary="Nope", - unexpected="field", - ) diff --git a/tests/test_normalize.py b/tests/test_normalize.py new file mode 100644 index 0000000..aa49a50 --- /dev/null +++ b/tests/test_normalize.py @@ -0,0 +1,186 @@ +"""Unit tests for the deterministic CELLxGENE normalizer.""" + +from __future__ import annotations + +from parce.models.graph_schema import EntityType, KnowledgeGraphOutput +from parce.models.raw_record import RawRecord +from parce.normalize.cellxgene import CellxgeneNormalizer + +# ``cell_types`` are intentionally present in the payload to prove the normalizer +# ignores them (CellType is a data-inferred annotation, deliberately excluded). +# ``blood`` (UBERON:0000178) appears in both datasets to exercise dedup. +_RECORD = RawRecord( + source="CELLxGENE", + study_id="10.1234/test", + title="Test Study", + payload={ + "datasets": [ + { + "dataset_id": "ds-001", + "dataset_title": "Dataset One", + "h5ad_uri": "s3://bucket/ds-001.h5ad", + "modality": "10x 3' v3", + "cell_count": 5000, + "ontology_summary": { + "organism": "Homo sapiens", + "modality": "10x 3' v3", + "cell_types": [ + {"name": "T cell", "ontology_id": "CL:0000084"}, + {"name": "B cell", "ontology_id": "CL:0000236"}, + ], + "tissues": [ + {"name": "blood", "ontology_id": "UBERON:0000178"}, + ], + "diseases": [ + {"name": "normal", "ontology_id": "PATO:0000461"}, + ], + "assays": [ + {"name": "10x 3' v3", "ontology_id": "EFO:0009922"}, + ], + }, + }, + { + "dataset_id": "ds-002", + "dataset_title": "Dataset Two", + "h5ad_uri": "s3://bucket/ds-002.h5ad", + "modality": "Smart-seq2", + "cell_count": 1000, + "ontology_summary": { + "organism": "Homo sapiens", + "modality": "Smart-seq2", + "cell_types": [ + {"name": "T cell", "ontology_id": "CL:0000084"}, + ], + "tissues": [ + {"name": "blood", "ontology_id": "UBERON:0000178"}, + {"name": "lung", "ontology_id": "UBERON:0002048"}, + ], + "diseases": [], + "assays": [ + {"name": "Smart-seq2", "ontology_id": "EFO:0008931"}, + ], + }, + }, + ], + }, +) + + +def _empty_record() -> RawRecord: + return RawRecord( + source="CELLxGENE", + study_id="10.1234/test", + title="Test Study", + payload={"datasets": []}, + ) + + +class TestCellxgeneNormalizer: + def test_basic_structure(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + assert len(kg.studies) == 1 + assert kg.studies[0].study_id == "10.1234/test" + assert kg.studies[0].title == "Test Study" + assert kg.studies[0].source == "CELLxGENE" + assert kg.studies[0].modality == "scRNA-seq" + + assert len(kg.datasets) == 2 + assert kg.datasets[0].dataset_id == "ds-001" + assert kg.datasets[0].data_uri == "s3://bucket/ds-001.h5ad" + assert kg.datasets[0].assay == "10x 3' v3" + assert kg.datasets[1].dataset_id == "ds-002" + + def test_study_source_from_record(self): + """StudyNode.source is taken from the record, not hardcoded.""" + record = _empty_record() + record.source = "SomeOtherSource" + kg = CellxgeneNormalizer().normalize(record) + assert kg.studies[0].source == "SomeOtherSource" + + def test_cell_type_excluded(self): + """Cell types in the payload must not produce entities or edges.""" + kg = CellxgeneNormalizer().normalize(_RECORD) + + names = {e.name for e in kg.biological_entities} + assert "T cell" not in names + assert "B cell" not in names + + ontology_ids = {e.ontology_id for e in kg.biological_entities} + assert "CL:0000084" not in ontology_ids + assert "CL:0000236" not in ontology_ids + + edge_targets = {e.target_id for e in kg.edges} + assert "CL:0000084" not in edge_targets + + def test_tissue_entity_deduplication(self): + """blood (UBERON:0000178) appears in both datasets but is one entity.""" + kg = CellxgeneNormalizer().normalize(_RECORD) + + entity_ids = [e.ontology_id for e in kg.biological_entities] + assert entity_ids.count("UBERON:0000178") == 1 + + def test_species_entity_created(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + species = [e for e in kg.biological_entities if e.entity_type == EntityType.SPECIES] + assert len(species) == 1 + assert species[0].ontology_id == "NCBITaxon:9606" + assert species[0].name == "Homo sapiens" + + def test_no_samples_for_cellxgene(self): + """Census is dataset-level; no SampleNode records are emitted (yet).""" + kg = CellxgeneNormalizer().normalize(_RECORD) + assert kg.samples == [] + + def test_extracted_from_edges(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + extracted = [e for e in kg.edges if e.relation_type == "EXTRACTED_FROM"] + assert len(extracted) == 2 + assert {e.source_id for e in extracted} == {"ds-001", "ds-002"} + assert all(e.target_id == "10.1234/test" for e in extracted) + + def test_has_tissue_edges(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + tissue_edges = [e for e in kg.edges if e.relation_type == "HAS_TISSUE"] + pairs = {(e.source_id, e.target_id) for e in tissue_edges} + assert ("ds-001", "UBERON:0000178") in pairs + assert ("ds-002", "UBERON:0000178") in pairs + assert ("ds-002", "UBERON:0002048") in pairs + + def test_has_condition_edges(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + conditions = [e for e in kg.edges if e.relation_type == "HAS_CONDITION"] + assert any(e.target_id == "PATO:0000461" for e in conditions) + + def test_measured_with_edges(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + assay_edges = [e for e in kg.edges if e.relation_type == "MEASURED_WITH"] + assert any(e.source_id == "ds-001" and e.target_id == "EFO:0009922" for e in assay_edges) + assert any(e.source_id == "ds-002" and e.target_id == "EFO:0008931" for e in assay_edges) + + def test_studies_edge(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + + studies = [e for e in kg.edges if e.relation_type == "STUDIES"] + assert len(studies) == 1 + assert studies[0].source_id == "10.1234/test" + assert studies[0].target_id == "NCBITaxon:9606" + + def test_empty_datasets(self): + kg = CellxgeneNormalizer().normalize(_empty_record()) + + assert len(kg.studies) == 1 + assert len(kg.datasets) == 0 + assert len(kg.samples) == 0 + assert len(kg.biological_entities) == 0 + assert len(kg.edges) == 0 + + def test_roundtrip_json(self): + kg = CellxgeneNormalizer().normalize(_RECORD) + restored = KnowledgeGraphOutput.model_validate_json(kg.model_dump_json()) + assert restored == kg diff --git a/tests/test_orchestration.py b/tests/test_orchestration.py index 39efaa9..aa6eadb 100644 --- a/tests/test_orchestration.py +++ b/tests/test_orchestration.py @@ -1,17 +1,16 @@ """Mock-based tests for the orchestration flow in main.py. -These tests do NOT require Azure credentials or network access. -They mock both data-fetching tools and the LLM agent to verify the -end-to-end assembly pipeline. +These tests do NOT require Azure credentials or network access: the CELLxGENE +adapter's Census/EuropePMC calls are mocked, while the real adapter and +normalizer run end to end. The pipeline is deterministic — there is no LLM. """ from __future__ import annotations -from unittest.mock import AsyncMock, MagicMock, patch +import json +from unittest.mock import patch -import pytest - -from parce.main import _build_narrative_prompt, run +from parce.main import run _MOCK_PAPER = { "doi": "10.1234/mock", @@ -31,7 +30,6 @@ "ontology_summary": { "organism": "Homo sapiens", "modality": "10x 3' v3", - "cell_types": [{"name": "T cell", "ontology_id": "CL:0000084"}], "tissues": [{"name": "blood", "ontology_id": "UBERON:0000178"}], "diseases": [], "assays": [{"name": "10x 3' v3", "ontology_id": "EFO:0009922"}], @@ -41,95 +39,45 @@ } -class TestBuildNarrativePrompt: - def test_includes_title_and_abstract(self): - prompt = _build_narrative_prompt(_MOCK_PAPER, _MOCK_CELLXGENE) - assert "Mock Study" in prompt - assert "We studied cells." in prompt - - def test_includes_dataset_info(self): - prompt = _build_narrative_prompt(_MOCK_PAPER, _MOCK_CELLXGENE) - assert "mock-ds-1" in prompt - assert "T cell" in prompt - assert "100" in prompt - - def test_includes_organism(self): - prompt = _build_narrative_prompt(_MOCK_PAPER, _MOCK_CELLXGENE) - assert "Homo sapiens" in prompt - - def test_empty_datasets(self): - prompt = _build_narrative_prompt(_MOCK_PAPER, {"doi": "x", "datasets": []}) - assert "Mock Study" in prompt - assert "CELLxGENE" not in prompt +def _patch_network(paper=_MOCK_PAPER, cellxgene=_MOCK_CELLXGENE): + return ( + patch("parce.sources.cellxgene.fetch_paper_metadata", return_value=paper), + patch("parce.sources.cellxgene.fetch_cellxgene_datasets", return_value=cellxgene), + ) class TestRunOrchestration: - @pytest.fixture(autouse=True) - def _mock_settings(self): - # The agent is mocked, so real Azure config is irrelevant here. Patch - # Settings so the test never reads the environment / .env file and stays - # hermetic (CI has no .env). max_retries must be a real int for range(). - fake = MagicMock() - fake.max_retries = 3 - with patch("parce.main.Settings", return_value=fake): - yield fake - - @pytest.fixture - def _mock_tools(self): - with ( - patch("parce.main.fetch_paper_metadata", return_value=_MOCK_PAPER) as mock_paper, - patch("parce.main.fetch_cellxgene_datasets", return_value=_MOCK_CELLXGENE) as mock_cx, - ): - yield mock_paper, mock_cx - - @pytest.fixture - def _mock_agent(self): - mock_result = MagicMock() - mock_result.value = MagicMock() - mock_result.value.experimental_narrative = "A mock narrative about T cells." - mock_result.usage = None - - mock_agent = AsyncMock() - mock_agent.run = AsyncMock(return_value=mock_result) - - ctx = AsyncMock() - ctx.__aenter__ = AsyncMock(return_value=mock_agent) - ctx.__aexit__ = AsyncMock(return_value=False) - - with patch("parce.main.create_narrative_agent", return_value=ctx): - yield mock_agent - - async def test_full_pipeline(self, _mock_tools, _mock_agent, tmp_path): - with patch("parce.main._OUTPUT_DIR", tmp_path): - await run(doi="10.1234/mock") + def test_full_pipeline(self, tmp_path): + p_paper, p_cx = _patch_network() + with p_paper, p_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + run(doi="10.1234/mock") out_file = tmp_path / "output.json" assert out_file.exists() - import json - kg = json.loads(out_file.read_text()) assert len(kg["studies"]) == 1 assert kg["studies"][0]["study_id"] == "10.1234/mock" - # The canonical KG no longer stores a narrative. + assert kg["studies"][0]["title"] == "Mock Study" + assert kg["studies"][0]["source"] == "CELLxGENE" + # The canonical KG never stores a narrative. assert "experimental_narrative" not in kg["studies"][0] assert len(kg["datasets"]) == 1 assert len(kg["biological_entities"]) > 0 assert len(kg["edges"]) > 0 - async def test_tools_called_with_doi(self, _mock_tools, _mock_agent, tmp_path): - mock_paper, mock_cx = _mock_tools - with patch("parce.main._OUTPUT_DIR", tmp_path): - await run(doi="10.1234/mock") + def test_fetch_called_with_doi(self, tmp_path): + p_paper, p_cx = _patch_network() + with p_paper as mock_paper, p_cx as mock_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + run(doi="10.1234/mock") mock_paper.assert_called_once_with("10.1234/mock") - mock_cx.assert_called_once_with("10.1234/mock") - - async def test_agent_called_with_response_format(self, _mock_tools, _mock_agent, tmp_path): - from parce.models.graph_schema import NarrativeOutput + mock_cx.assert_called_once_with("10.1234/mock", max_workers=4) - with patch("parce.main._OUTPUT_DIR", tmp_path): - await run(doi="10.1234/mock") + def test_no_datasets_writes_nothing(self, tmp_path): + empty = {"doi": "10.1234/mock", "datasets": [], "error": "No datasets found"} + p_paper, p_cx = _patch_network(cellxgene=empty) + with p_paper, p_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + run(doi="10.1234/mock") - call_kwargs = _mock_agent.run.call_args - assert call_kwargs.kwargs.get("response_format") is NarrativeOutput + assert not (tmp_path / "output.json").exists() diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..14bedc7 --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,104 @@ +"""Unit tests for source adapters, the RawRecord model, and the protocols. + +All Census/EuropePMC IO is mocked — these tests must stay offline. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +from parce.models.raw_record import RawRecord +from parce.normalize.base import Normalizer +from parce.normalize.cellxgene import CellxgeneNormalizer +from parce.sources.base import SourceAdapter +from parce.sources.cellxgene import CellxgeneAdapter + +_MOCK_PAPER = {"doi": "10.1234/mock", "title": "Mock Study", "abstract": "We studied cells."} +_MOCK_CELLXGENE = { + "doi": "10.1234/mock", + "datasets": [ + { + "dataset_id": "mock-ds-1", + "dataset_title": "Mock Dataset", + "h5ad_uri": "s3://bucket/mock-ds-1.h5ad", + "modality": "10x 3' v3", + "cell_count": 100, + "ontology_summary": { + "organism": "Homo sapiens", + "modality": "10x 3' v3", + "tissues": [{"name": "blood", "ontology_id": "UBERON:0000178"}], + "diseases": [], + "assays": [{"name": "10x 3' v3", "ontology_id": "EFO:0009922"}], + }, + } + ], +} + + +class TestRawRecord: + def test_minimal(self): + record = RawRecord(source="CELLxGENE", study_id="10.1234/x") + assert record.title == "" + assert record.payload == {} + + def test_extra_forbidden(self): + with pytest.raises(ValidationError): + RawRecord(source="GEO", study_id="GSE1", bogus=True) + + +class TestCellxgeneAdapter: + def test_source_name(self): + assert CellxgeneAdapter().source_name == "CELLxGENE" + + def test_discover_is_identity_on_doi(self): + assert CellxgeneAdapter().discover("10.1234/mock") == ["10.1234/mock"] + + def test_fetch_builds_raw_record(self): + with ( + patch("parce.sources.cellxgene.fetch_paper_metadata", return_value=_MOCK_PAPER), + patch("parce.sources.cellxgene.fetch_cellxgene_datasets", return_value=_MOCK_CELLXGENE), + ): + record = CellxgeneAdapter().fetch("10.1234/mock") + + assert record.source == "CELLxGENE" + assert record.study_id == "10.1234/mock" + assert record.title == "Mock Study" + assert len(record.payload["datasets"]) == 1 + assert record.payload["datasets"][0]["dataset_id"] == "mock-ds-1" + assert "error" not in record.payload + + def test_fetch_propagates_no_match_error(self): + empty = {"doi": "10.1234/none", "datasets": [], "error": "No datasets found"} + with ( + patch( + "parce.sources.cellxgene.fetch_paper_metadata", + return_value={"doi": "10.1234/none", "title": "", "abstract": ""}, + ), + patch("parce.sources.cellxgene.fetch_cellxgene_datasets", return_value=empty), + ): + record = CellxgeneAdapter().fetch("10.1234/none") + + assert record.payload["datasets"] == [] + assert record.payload["error"] == "No datasets found" + + def test_fetch_passes_max_workers(self): + with ( + patch("parce.sources.cellxgene.fetch_paper_metadata", return_value=_MOCK_PAPER), + patch( + "parce.sources.cellxgene.fetch_cellxgene_datasets", return_value=_MOCK_CELLXGENE + ) as mock_fetch, + ): + CellxgeneAdapter().fetch("10.1234/mock", max_workers=2) + + mock_fetch.assert_called_once_with("10.1234/mock", max_workers=2) + + +class TestProtocolConformance: + def test_adapter_satisfies_source_adapter(self): + assert isinstance(CellxgeneAdapter(), SourceAdapter) + + def test_normalizer_satisfies_normalizer(self): + assert isinstance(CellxgeneNormalizer(), Normalizer)