From a5e684e28f771fff6a8a003e2e806338c4c8939d Mon Sep 17 00:00:00 2001 From: mengerj Date: Thu, 25 Jun 2026 07:52:00 +0200 Subject: [PATCH] PR 4: Ontology resolution stage + CELLxGENE organism wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the shared `parce/ontology/` stage that grounds free-text design fields to ontology term IDs, the precondition for cross-source linking (ARCHITECTURE §5): - registry.py — Facet → ontology constant (EFO/OBI/PSI-MS, UBERON, MONDO, NCBITaxon, ChEBI, EDAM); pins ontologies, never IDs. - ols.py — OlsClient over OLS4 REST (search + is-a ancestors), injectable HTTP getter, retry-wrapped via sources._retry; CURIE→IRI + double-encoding. - cache.py — ResolutionCache: on-disk JSON, atomic, caches negative results. - layers.py — derive_molecular_layer(): EFO ancestor-label anchors, most-specific-first, UNKNOWN default. Anchors PROVISIONAL (see integration test). - resolver.py — OntologyResolver: cache → OLS exact-then-fuzzy → optional LLM fallback hook (default None, keeps ontology free of agent/Azure). Wire into CellxgeneNormalizer: organism string → NCBITaxon at runtime, replacing the hardcoded _ORGANISM_ONTOLOGY map. Unresolved organisms are skipped, not emitted ungrounded. Add MolecularLayer enum to models (consumed by PR 4b). Decisions: OLS4-only for now (text2term/Zooma deferred to GEO/PR 5); LLM fallback pluggable; anchors keyed by label not ID. Split the original PR 4: the schema change (store EFO assay term + molecular_layer) is now PR 4b. Gates: ruff, ruff format, mypy (no new exemptions), 121 unit tests — all green and hermetic (no .env, no network). No dependency changes. Co-Authored-By: Claude Opus 4.8 --- docs/ARCHITECTURE.md | 38 ++++++- docs/ROADMAP.md | 107 +++++++++++++++--- src/parce/models/graph_schema.py | 22 ++++ src/parce/normalize/cellxgene.py | 64 ++++++----- src/parce/ontology/__init__.py | 35 ++++++ src/parce/ontology/base.py | 39 +++++++ src/parce/ontology/cache.py | 72 ++++++++++++ src/parce/ontology/layers.py | 62 +++++++++++ src/parce/ontology/ols.py | 140 ++++++++++++++++++++++++ src/parce/ontology/registry.py | 80 ++++++++++++++ src/parce/ontology/resolver.py | 148 +++++++++++++++++++++++++ tests/test_normalize.py | 67 +++++++++--- tests/test_ontology_cache.py | 59 ++++++++++ tests/test_ontology_integration.py | 61 +++++++++++ tests/test_ontology_layers.py | 42 +++++++ tests/test_ontology_ols.py | 139 +++++++++++++++++++++++ tests/test_ontology_registry.py | 54 +++++++++ tests/test_ontology_resolver.py | 170 +++++++++++++++++++++++++++++ tests/test_orchestration.py | 32 +++++- 19 files changed, 1363 insertions(+), 68 deletions(-) create mode 100644 src/parce/ontology/__init__.py create mode 100644 src/parce/ontology/base.py create mode 100644 src/parce/ontology/cache.py create mode 100644 src/parce/ontology/layers.py create mode 100644 src/parce/ontology/ols.py create mode 100644 src/parce/ontology/registry.py create mode 100644 src/parce/ontology/resolver.py create mode 100644 tests/test_ontology_cache.py create mode 100644 tests/test_ontology_integration.py create mode 100644 tests/test_ontology_layers.py create mode 100644 tests/test_ontology_ols.py create mode 100644 tests/test_ontology_registry.py create mode 100644 tests/test_ontology_resolver.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b9f0e30..179fd89 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -156,6 +156,17 @@ Avoid a free-text `modality`. Instead store, per dataset/study: term's `is-a` ancestors** to a small set of anchor classes. The lineage does the classification; we never re-string it. +> **PR 4 — anchors & default.** `MolecularLayer` lives in `models/graph_schema.py` +> (canonical-vocabulary home); the derivation is `parce.ontology.layers. +> derive_molecular_layer`. The anchor set is keyed by EFO ancestor **label**, not +> term ID — honouring "pin ontologies/anchors, not IDs" (OLS returns canonical +> labels for every ancestor), and matched most-specific-first. The **no-anchor +> default is `MolecularLayer.UNKNOWN`**. The current anchor labels are +> *provisional* (an informed first cut not yet checked against live EFO); the +> marked `tests/test_ontology_integration.py` is the validation harness, and PR 4b +> (which adds the stored field) must tighten them. The derivation logic + default +> are decided; only the exact label strings remain to be confirmed. + The model sees a clean cross-modality categorical (`molecular_layer`) plus a precise term (`assay`) — both controlled, no free text on either. @@ -170,6 +181,18 @@ precise term (`assay`) — both controlled, no free text on either. - **LLM** — fallback only, for strings the deterministic resolvers can't confidently map. +> **PR 4 — what shipped, and why OLS4-only (for now).** The stage is implemented +> in `parce/ontology/` as `OntologyResolver` (cache → OLS4 exact-then-fuzzy → +> optional LLM fallback) over the registry above. **Only OLS4 is wired**; +> text2term/Zooma are deferred to PR 5, where GEO's messy `characteristics_ch1` +> strings actually need fuzzy batch mapping. CELLxGENE already ships ontology IDs +> for tissue/disease/assay, so the only free-text grounding needed today is the +> organism string → NCBITaxon. The **LLM fallback is a pluggable `Callable` +> (default `None`)**, not a hard dependency — this keeps `ontology` free of any +> `parce.agent`/Azure import (dependency direction holds); PR 5 supplies the +> extraction agent as the callback. Resolutions are memoised to an on-disk cache +> (negative results included) so runs are reproducible and don't re-query. + Template to follow rather than reinvent: **SDRF / MAGE-TAB** (and **SDRF-Proteomics**) already specify per-sample, ontology-annotated experiment description and *which ontology per column* — almost exactly the `SampleNode` + @@ -205,12 +228,15 @@ that already ship SDRF. - **Sample granularity for CELLxGENE.** Census is per-cell/dataset, not per-sample in the GEO sense. Defer mapping cxg to `SampleNode` until needed; keep it dataset-level for now. -- **`molecular_layer` anchor set.** The exact EFO ancestor classes that define - each coarse layer need pinning during PR4 (and a default for assays whose - lineage doesn't reach an anchor). -- **Ontology resolver dependency.** OLS4 REST client (needed anyway for the - lineage walk) vs. adding text2term/Zooma — prefer the lightest combination - that covers messy GEO strings; decide in PR4. +- **`molecular_layer` anchor set.** *Partly resolved (PR 4):* the derivation + mechanism and the no-anchor default (`UNKNOWN`) are pinned, and the anchors are + keyed by EFO **label**. **Still open:** the exact label strings are provisional + and unvalidated against live EFO — confirm via the marked integration test and + tighten in PR 4b. +- **Ontology resolver dependency.** *Resolved (PR 4):* **OLS4 REST only** for + now (it covers organism grounding + the lineage walk). text2term/Zooma are + deferred to PR 5, where GEO's messy `characteristics_ch1` strings need fuzzy + batch mapping — add the lightest option that covers them then. - **Graph persistence/export format** for the modeling step (per-study context + sample manifest + URIs). Specified in a later PR. - **Multi-omics is core, not optional.** CELLxGENE alone cannot carry the diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 04290da..0930d70 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -8,14 +8,18 @@ protocol and [ARCHITECTURE.md](ARCHITECTURE.md) for the design. ## ▶ Next up -**PR 4 — Ontology resolver.** Shared `ontology/` stage (see ARCHITECTURE §5). Pin -the **facet → ontology registry** as a constant (EFO, UBERON, MONDO, NCBITaxon, -ChEBI, PSI-MS, EDAM). Implement free-text → term resolution (OLS4 REST + -text2term/Zooma, on-disk cache; LLM fallback) and the **`molecular_layer` -derivation** (walk EFO `is-a` ancestors to anchor classes). Decide the anchor set -+ the no-anchor default. Wire into normalizers — start by replacing the hardcoded -`_ORGANISM_ONTOLOGY` map in `normalize/cellxgene.py`. Resolve IDs at runtime via -OLS — do not hardcode term IDs. +**PR 4b — Schema refinement: EFO `assay` term + stored `molecular_layer`.** PR 4 +shipped the ontology *stage* (resolver, registry, OLS4 client, cache, +`molecular_layer` **derivation**) but did not change the canonical schema. This +PR consumes it: on `StudyNode`/`DatasetNode`, replace the free-text `modality` +string with an EFO `assay` **term ID** (resolved via `OntologyResolver`) plus a +stored `molecular_layer` enum field (derived via `OntologyResolver.molecular_ +layer`). Wire both into `CellxgeneNormalizer` (it already resolves organism; +extend to ground the assay string and derive the layer). Migrate the schema + +all tests. `MolecularLayer` already lives in `models/graph_schema.py`. **Gotcha:** +the provisional anchor labels in `ontology/layers.py` are unvalidated against live +EFO — run `tests/test_ontology_integration.py` and tighten them first, else every +assay derives `UNKNOWN`. --- @@ -36,14 +40,22 @@ Each PR is one branch, one focused scope, green CI, and a roadmap update. `models/narrative.py`, the `NarrativeOutput` schema, and the step-2 block in `main.py`). Dropped cell-type extraction. Removed the `parce.tools.*` mypy exemption as those modules moved under `sources/`. -- [ ] **PR 4 — Ontology resolver.** Shared `ontology/` stage (see ARCHITECTURE - §5). Pin the **facet → ontology registry** as a constant (EFO, UBERON, MONDO, - NCBITaxon, ChEBI, PSI-MS, EDAM). Implement free-text → term resolution - (OLS4 REST + text2term/Zooma, on-disk cache; LLM fallback) and the - **`molecular_layer` derivation** (walk EFO `is-a` ancestors to anchor classes). - Decide the anchor set + the no-anchor default. Wire into normalizers (replace - the hardcoded `_ORGANISM_ONTOLOGY` map in `normalize/cellxgene.py`). Resolve - IDs at runtime via OLS — do not hardcode term IDs. *(Next up.)* +- [x] **PR 4 — Ontology resolver (stage + organism wiring).** Shared `ontology/` + stage (see ARCHITECTURE §5): pinned the **facet → ontology registry** constant + (EFO/OBI/PSI-MS, UBERON, MONDO, NCBITaxon, ChEBI, EDAM); `OlsClient` (OLS4 REST, + retry-wrapped, injectable); on-disk `ResolutionCache` (negative results cached); + `OntologyResolver` (cache → OLS exact-then-fuzzy → pluggable LLM-fallback hook, + default off); and the **`molecular_layer` derivation** (EFO `is-a` ancestor walk + → pinned anchor labels; no-anchor default `UNKNOWN`). Replaced the hardcoded + `_ORGANISM_ONTOLOGY` map in `normalize/cellxgene.py` with runtime NCBITaxon + resolution. **Decided: OLS4-only** for now (text2term/Zooma deferred to GEO/PR 5 + — CELLxGENE ships IDs, only organism free-text needed grounding). IDs resolved + at runtime; none hardcoded. *Split from the original PR 4: the schema change + (store the EFO assay term + `molecular_layer`) became PR 4b.* +- [ ] **PR 4b — Schema refinement.** Replace free-text `modality` with an EFO + `assay` term ID + a stored `molecular_layer` enum on the study/dataset nodes; + wire the resolver's assay grounding + layer derivation into the normalizer; + migrate the schema and tests. *(Next up — see top of file.)* - [ ] **PR 5 — GEO extraction agent (vertical slice).** GEO adapter (E-utilities/GEOparse) + Azure extraction normalizer emitting the canonical schema via `response_format`; extract sample covariates from @@ -73,6 +85,69 @@ Each PR is one branch, one focused scope, green CI, and a roadmap update. Newest first. One entry per working session: what changed, decisions made, and what the next session should know. Keep entries short and factual. +### 2026-06-25 — PR 4: Ontology resolver (stage + organism wiring) + +- Branch `pr4-ontology-resolver` off `main` (64ed4f4). +- **Split decision.** The roadmap's PR 4 bundled the resolver stage *and* the + schema change (free-text `modality` → EFO `assay` term + stored + `molecular_layer`). Shipped the stage + organism wiring here; the schema + migration is now **PR 4b** (new ▶ Next up). Rationale: the stage is a coherent, + fully-tested unit; the schema swap touches the canonical models + every + CELLxGENE test and is cleaner as its own focused PR. +- **New package `src/parce/ontology/`** (stable core, fully mypy-checked): + - `registry.py` — `Facet` enum + `FACET_ONTOLOGY` constant. Pins *ontologies, + not IDs*: EFO (assay; fallbacks OBI then PSI-MS), UBERON, MONDO, NCBITaxon, + ChEBI, EDAM. `FacetBinding.ontologies()` gives primary→fallback order. + - `ols.py` — `OlsClient` for OLS4 REST: `search` (free text → class hits) and + `ancestors` (hierarchical/`is-a`). `http` getter is injectable (offline + tests); requests wrapped in `sources._retry.with_retries`. `obo_id_to_iri` + builds EFO + generic OBO-PURL IRIs and the path is double-URL-encoded. + - `cache.py` — `ResolutionCache`, JSON-on-disk, atomic writes, lock-guarded. + **Caches negative results** (a *miss* vs a cached-`None` are distinguished via + `get`'s first return value) so unresolvable strings aren't re-queried. + - `layers.py` — `derive_molecular_layer(ancestor_labels)`: pure function + matching EFO ancestor **labels** (not IDs) against a pinned anchor set; + most-specific-first; no-anchor default `MolecularLayer.UNKNOWN`. + - `resolver.py` — `OntologyResolver`: `resolve_term(text, facet)` (cache → OLS + exact-then-fuzzy across the facet's ontologies → optional LLM fallback) and + `molecular_layer(assay_id, assay_label=None)`. All collaborators injectable; + default cache is lazy so construction touches neither disk nor network. + - `base.py` — `ResolvedTerm` value type + the narrow `TermResolver` Protocol + normalizers depend on. +- **`MolecularLayer` StrEnum added to `models/graph_schema.py`** (its home, since + `models` depends on nothing and it becomes a node field in PR 4b). Not yet a + stored field anywhere. +- **Normalizer wired:** `CellxgeneNormalizer(resolver=...)` now grounds the bare + organism string to NCBITaxon via the resolver (replacing `_ORGANISM_ONTOLOGY`). + Tissue/disease/assay still use the IDs Census already ships — only organism was + free text. An organism that fails to resolve is **skipped** (no ungrounded node + emitted), logged at WARNING. +- **Decisions / rationale:** + - **OLS4-only** (open question §7 resolved): text2term/Zooma deferred to GEO + (PR 5), where the messy `characteristics_ch1` strings actually appear. + - **LLM fallback is a pluggable `Callable`, default `None`** — keeps `ontology` + free of any `parce.agent`/Azure import (dependency direction preserved); PR 5 + wires the extraction agent in as the callback. + - **Anchor set is keyed by EFO label, not term ID** (honours "pin + ontologies/anchors, not IDs"). **PROVISIONAL** — unvalidated against live EFO; + `tests/test_ontology_integration.py` (marked, live OLS) is the validation + harness. PR 4b must run it and correct labels, else assays derive `UNKNOWN`. + - **`ontology` reuses `sources._retry`** (leaf util, no cycle) per CLAUDE.md. + - Resolver config (OLS base URL, cache dir) is constructor params, **not** + `Settings` — avoids the unit-test `Settings()` hermeticity trap; could move to + Settings later. +- **Tests:** offline unit suites for registry / cache / OLS (fake HTTP getter) / + layers / resolver (fake client + real cache on tmp_path); `test_normalize.py` + and `test_orchestration.py` inject a fake resolver so they stay offline. New + marked integration test for live OLS (organism exact; molecular_layer plumbing). +- **mypy:** no new exemptions; `parce.agent.*` exemption stays (PR 5). 24 source + files checked, clean. +- **Gates green, incl. hermetic run (worktree has no `.env`):** ruff check, + ruff format --check (40 files), mypy, **121 unit tests** (was 47+retry). No dep + changes (stdlib + `requests`), so no `uv.lock` change. +- **Next session:** PR 4b (schema refinement) — and validate the provisional + `molecular_layer` anchors against live EFO first. + ### 2026-06-24 — Generic retry helper (resilience follow-up to PR 3) - Branch `add-generic-retry-helper`. Not a roadmap PR; **closes the retry diff --git a/src/parce/models/graph_schema.py b/src/parce/models/graph_schema.py index af0667f..aab0f41 100644 --- a/src/parce/models/graph_schema.py +++ b/src/parce/models/graph_schema.py @@ -37,6 +37,28 @@ class EntityType(StrEnum): ASSAY = "Assay" +class MolecularLayer(StrEnum): + """Coarse molecular readout an assay measures — a controlled, cross-modality + categorical the downstream model can condition on. + + Derived deterministically by walking an EFO ``assay`` term's ``is-a`` + ancestors to a pinned anchor class (see :mod:`parce.ontology.layers`); never + re-strung from free text. ``UNKNOWN`` is the no-anchor default for assays + whose lineage reaches none of the anchors. + + Defined here, the canonical-vocabulary home, so the ontology stage can target + it now. It becomes a stored field on the study/dataset nodes in the + schema-refinement PR (see docs/ROADMAP.md). + """ + + GENOME = "genome" + EPIGENOME = "epigenome" + TRANSCRIPTOME = "transcriptome" + PROTEOME = "proteome" + METABOLOME = "metabolome" + UNKNOWN = "unknown" + + class StudyNode(BaseModel): """A study (publication or repository accession), source-agnostic. diff --git a/src/parce/normalize/cellxgene.py b/src/parce/normalize/cellxgene.py index 8ba1413..9f151c9 100644 --- a/src/parce/normalize/cellxgene.py +++ b/src/parce/normalize/cellxgene.py @@ -1,15 +1,17 @@ """Deterministic normalizer: a CELLxGENE ``RawRecord`` → canonical KG nodes. No LLM is involved: CELLxGENE Census already ships ontology-grounded terms, so -this is a pure structural mapping. Two design rules show up directly here: +this is a pure structural mapping. Three design rules show up directly here: * **Cell type is never consumed** — the adapter does not even read it (data-inferred → leakage; see docs/ARCHITECTURE.md §1). * **Census is dataset-level**, not per-sample in the GEO sense, so no ``SampleNode`` records are emitted yet (open question in ARCHITECTURE §7). - -The free-text → ontology-ID step (here, organism string → NCBITaxon) is a -hardcoded map for now; it becomes the shared OntologyResolver stage in PR 4. +* **Organism strings are grounded via the shared OntologyResolver**, not a + hardcoded map. Tissue/disease/assay already arrive as ontology IDs from + Census, so only the bare organism string needs runtime resolution (to + NCBITaxon, via OLS). An organism that fails to resolve is skipped rather than + emitted ungrounded. """ from __future__ import annotations @@ -26,22 +28,14 @@ StudyNode, ) from parce.models.raw_record import RawRecord +from parce.ontology import Facet, OntologyResolver, ResolvedTerm, TermResolver logger = logging.getLogger(__name__) # High-level study modality for everything CELLxGENE ingests. (Refined into an -# EFO ``assay`` term + derived ``molecular_layer`` in PR 4.) +# EFO ``assay`` term + derived ``molecular_layer`` in the schema-refinement PR.) _STUDY_MODALITY = "scRNA-seq" -# Organism free-text (as Census returns it) → (NCBITaxon ID, canonical name). -# Stand-in for the PR 4 OntologyResolver. -_ORGANISM_ONTOLOGY: dict[str, tuple[str, str]] = { - "Homo sapiens": ("NCBITaxon:9606", "Homo sapiens"), - "Mus musculus": ("NCBITaxon:10090", "Mus musculus"), - "homo_sapiens": ("NCBITaxon:9606", "Homo sapiens"), - "mus_musculus": ("NCBITaxon:10090", "Mus musculus"), -} - # Ontology categories that become design-context entities. Cell types are # deliberately absent (data-inferred → leakage). _CATEGORY_TO_ENTITY_TYPE: dict[str, EntityType] = { @@ -58,7 +52,15 @@ class CellxgeneNormalizer: - """:class:`~parce.normalize.base.Normalizer` for CELLxGENE ``RawRecord``s.""" + """:class:`~parce.normalize.base.Normalizer` for CELLxGENE ``RawRecord``s. + + Takes a :class:`~parce.ontology.base.TermResolver` (default: a real + :class:`~parce.ontology.resolver.OntologyResolver`) used to ground organism + strings. Inject a deterministic fake to keep unit tests offline. + """ + + def __init__(self, resolver: TermResolver | None = None) -> None: + self._resolver: TermResolver = resolver if resolver is not None else OntologyResolver() def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: """Assemble the canonical single-study subgraph for one CELLxGENE study.""" @@ -74,7 +76,10 @@ def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: datasets: list[DatasetNode] = [] edges: list[GraphEdge] = [] entity_registry: dict[str, BiologicalEntityNode] = {} - species_seen: set[str] = set() + # Resolved species (by NCBITaxon ID), and a per-record memo of organism + # string → resolution so the same string is grounded at most once. + species_ids: set[str] = set() + organism_cache: dict[str, ResolvedTerm | None] = {} for ds in record.payload.get("datasets", []): dataset_id = ds["dataset_id"] @@ -98,15 +103,15 @@ def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: ontology: dict[str, Any] = ds.get("ontology_summary", {}) - # Register species from the organism field. - organism_key = ontology.get("organism", "unknown") - if organism_key in _ORGANISM_ONTOLOGY and organism_key not in species_seen: - ont_id, name = _ORGANISM_ONTOLOGY[organism_key] - species_seen.add(organism_key) - entity_registry[ont_id] = BiologicalEntityNode( + # Ground the organism string to a NCBITaxon term via the resolver. + organism_text = ontology.get("organism", "unknown") + species = self._resolve_organism(organism_text, organism_cache) + if species is not None and species.ontology_id not in species_ids: + species_ids.add(species.ontology_id) + entity_registry[species.ontology_id] = BiologicalEntityNode( entity_type=EntityType.SPECIES, - ontology_id=ont_id, - name=name, + ontology_id=species.ontology_id, + name=species.name, ) # Register entities and create edges per design-context category. @@ -132,8 +137,7 @@ def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: ) # Study → Species edges. - for organism_key in species_seen: - species_id = _ORGANISM_ONTOLOGY[organism_key][0] + for species_id in species_ids: edges.append( GraphEdge( source_id=study_id, @@ -158,3 +162,11 @@ def normalize(self, record: RawRecord) -> KnowledgeGraphOutput: len(kg.edges), ) return kg + + def _resolve_organism( + self, organism_text: str, memo: dict[str, ResolvedTerm | None] + ) -> ResolvedTerm | None: + """Resolve an organism string to a NCBITaxon term, memoised per record.""" + if organism_text not in memo: + memo[organism_text] = self._resolver.resolve_term(organism_text, Facet.ORGANISM) + return memo[organism_text] diff --git a/src/parce/ontology/__init__.py b/src/parce/ontology/__init__.py new file mode 100644 index 0000000..f1f4be7 --- /dev/null +++ b/src/parce/ontology/__init__.py @@ -0,0 +1,35 @@ +"""Shared ontology-resolution stage: free text → ontology term IDs. + +Every source's free-text design fields land here so they resolve to the *same* +IDs, which is what lets studies link across sources (docs/ARCHITECTURE.md §5). +The public surface: + +* :class:`Facet` / :data:`FACET_ONTOLOGY` — which ontology grounds which facet; +* :class:`OntologyResolver` — OLS-first resolver with on-disk cache and an + optional LLM fallback hook; +* :class:`ResolvedTerm` / :class:`TermResolver` — the value type and the narrow + contract normalizers depend on. +""" + +from __future__ import annotations + +from parce.ontology.base import ResolvedTerm, TermResolver +from parce.ontology.cache import ResolutionCache +from parce.ontology.layers import derive_molecular_layer +from parce.ontology.ols import OlsClient, OlsTerm +from parce.ontology.registry import FACET_ONTOLOGY, Facet, FacetBinding, Ontology +from parce.ontology.resolver import OntologyResolver + +__all__ = [ + "FACET_ONTOLOGY", + "Facet", + "FacetBinding", + "OlsClient", + "OlsTerm", + "Ontology", + "OntologyResolver", + "ResolutionCache", + "ResolvedTerm", + "TermResolver", + "derive_molecular_layer", +] diff --git a/src/parce/ontology/base.py b/src/parce/ontology/base.py new file mode 100644 index 0000000..6ebba2e --- /dev/null +++ b/src/parce/ontology/base.py @@ -0,0 +1,39 @@ +"""Value types and the resolver contract for the shared ontology stage. + +Kept dependency-light (only the registry) so both the cache and the normalizers +can import :class:`ResolvedTerm` / :class:`TermResolver` without pulling in the +OLS client or ``requests``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, runtime_checkable + +from parce.ontology.registry import Facet + + +@dataclass(frozen=True, slots=True) +class ResolvedTerm: + """A free-text string grounded to one ontology term. + + ``ontology_id`` is a CURIE (e.g. ``NCBITaxon:9606``); ``name`` is the term's + canonical label as returned by the ontology service. + """ + + ontology_id: str + name: str + + +@runtime_checkable +class TermResolver(Protocol): + """Grounds a free-text term for a given facet, or returns ``None``. + + Normalizers depend on this narrow contract rather than the concrete + :class:`~parce.ontology.resolver.OntologyResolver`, so unit tests can inject a + deterministic fake and stay offline. + """ + + def resolve_term(self, text: str, facet: Facet) -> ResolvedTerm | None: + """Return the grounded term for ``text`` under ``facet``, else ``None``.""" + ... diff --git a/src/parce/ontology/cache.py b/src/parce/ontology/cache.py new file mode 100644 index 0000000..9ec3ddb --- /dev/null +++ b/src/parce/ontology/cache.py @@ -0,0 +1,72 @@ +"""On-disk cache for free-text → ontology-term resolutions. + +Resolution hits a remote service (OLS) and is stable over time, so results are +memoised to a small JSON file: repeat ingests of the same source don't re-query, +and a run is reproducible offline once its terms are cached. Negative results +(text that resolved to nothing) are cached too, so unresolvable strings aren't +retried on every run — delete the cache file to force re-resolution after the +resolver improves. + +The cache is process-local and guarded by a lock; it is *not* an IPC-safe store. +""" + +from __future__ import annotations + +import json +import logging +import threading +from dataclasses import asdict +from pathlib import Path + +from parce.ontology.base import ResolvedTerm + +logger = logging.getLogger(__name__) + + +class ResolutionCache: + """A JSON-backed ``key -> (ResolvedTerm | None)`` cache. + + Loaded eagerly from ``path`` if it exists; written back atomically on each + ``set``. A *missing* key (never resolved) is distinct from a key cached as + ``None`` (resolved to nothing): :meth:`get` reports which via its first + return value. + """ + + def __init__(self, path: Path) -> None: + self._path = path + self._lock = threading.Lock() + self._data: dict[str, ResolvedTerm | None] = {} + self._load() + + def _load(self) -> None: + if not self._path.exists(): + return + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("Ignoring unreadable ontology cache %s: %s", self._path, exc) + return + for key, value in raw.items(): + self._data[key] = None if value is None else ResolvedTerm(**value) + + def get(self, key: str) -> tuple[bool, ResolvedTerm | None]: + """Return ``(present, value)``: ``present`` is False on a cache miss.""" + with self._lock: + if key not in self._data: + return False, None + return True, self._data[key] + + def set(self, key: str, value: ResolvedTerm | None) -> None: + """Cache ``value`` (possibly ``None``) under ``key`` and persist.""" + with self._lock: + self._data[key] = value + self._flush() + + def _flush(self) -> None: + serialisable = { + key: (None if term is None else asdict(term)) for key, term in self._data.items() + } + self._path.parent.mkdir(parents=True, exist_ok=True) + tmp = self._path.with_suffix(self._path.suffix + ".tmp") + tmp.write_text(json.dumps(serialisable, indent=2, sort_keys=True), encoding="utf-8") + tmp.replace(self._path) diff --git a/src/parce/ontology/layers.py b/src/parce/ontology/layers.py new file mode 100644 index 0000000..9123ca0 --- /dev/null +++ b/src/parce/ontology/layers.py @@ -0,0 +1,62 @@ +"""Derive a coarse :class:`~parce.models.graph_schema.MolecularLayer` from an +EFO assay term's ``is-a`` ancestry. + +The classification is done by the *lineage*, not by re-parsing strings: we walk +the assay term's ancestors (via OLS) and match their labels against a small set +of pinned **anchor classes**. The anchors are keyed by their canonical EFO +**label**, not by term ID — the registry pins ontologies/anchors, not IDs +(docs/ARCHITECTURE.md §5), and OLS returns canonical labels for every ancestor. + +PROVISIONAL anchor set: the exact EFO ancestor labels below are an informed +first cut. They are validated against the live EFO assay branch by the marked +integration test ``tests/test_ontology_integration.py``; correct/extend them +there as real lineages are observed. An assay whose ancestry reaches no anchor +falls back to :attr:`MolecularLayer.UNKNOWN` (the pinned no-anchor default). +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from parce.models.graph_schema import MolecularLayer + +# Anchor EFO ancestor label (lower-cased) → molecular layer. Ordered +# most-specific first: when an assay's ancestry hits several anchors, the first +# match in this mapping wins, so narrower readouts beat broader ones. +_ANCHOR_LABELS: dict[str, MolecularLayer] = { + # Transcriptome + "rna assay": MolecularLayer.TRANSCRIPTOME, + "transcription profiling assay": MolecularLayer.TRANSCRIPTOME, + "transcription profiling by high throughput sequencing": MolecularLayer.TRANSCRIPTOME, + # Epigenome (chromatin accessibility, methylation, histone marks) + "atac-seq": MolecularLayer.EPIGENOME, + "dna methylation profiling assay": MolecularLayer.EPIGENOME, + "methylation profiling assay": MolecularLayer.EPIGENOME, + "chromatin immunoprecipitation assay": MolecularLayer.EPIGENOME, + # Proteome + "proteomic profiling assay": MolecularLayer.PROTEOME, + "protein assay": MolecularLayer.PROTEOME, + "mass spectrometry assay": MolecularLayer.PROTEOME, + # Metabolome + "metabolite profiling assay": MolecularLayer.METABOLOME, + "metabolomics assay": MolecularLayer.METABOLOME, + # Genome (sequence/variation, kept last as the broadest DNA readout) + "whole genome sequencing assay": MolecularLayer.GENOME, + "genotyping assay": MolecularLayer.GENOME, + "dna sequencing": MolecularLayer.GENOME, +} + + +def derive_molecular_layer(ancestor_labels: Iterable[str]) -> MolecularLayer: + """Classify an assay from the labels of its ``is-a`` ancestors. + + ``ancestor_labels`` should include the assay term's own label plus its + ancestors' labels. Matching is case-insensitive and exact against the pinned + anchor labels; the first anchor (in :data:`_ANCHOR_LABELS` order) present in + the lineage wins. Returns :attr:`MolecularLayer.UNKNOWN` when none match. + """ + present = {label.strip().lower() for label in ancestor_labels if label} + for anchor, layer in _ANCHOR_LABELS.items(): + if anchor in present: + return layer + return MolecularLayer.UNKNOWN diff --git a/src/parce/ontology/ols.py b/src/parce/ontology/ols.py new file mode 100644 index 0000000..6d14843 --- /dev/null +++ b/src/parce/ontology/ols.py @@ -0,0 +1,140 @@ +"""Thin OLS4 (EBI Ontology Lookup Service) REST client. + +Two operations the resolver needs, and nothing more: + +* :meth:`OlsClient.search` — free text → candidate terms within one ontology + (used to ground organism/tissue/disease/assay strings). +* :meth:`OlsClient.ancestors` — a term's ``is-a`` ancestors (used to walk an EFO + assay term up to a ``molecular_layer`` anchor). + +This is the only module in the ontology stage that performs network IO, so the +rest of the stage stays unit-testable offline. Requests are wrapped in the shared +:func:`parce.sources._retry.with_retries` helper (bounded retries, jittered +backoff on transient 429/5xx/network faults). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Protocol, cast +from urllib.parse import quote + +import requests + +from parce.sources._retry import DEFAULT_MAX_ATTEMPTS, with_retries + +logger = logging.getLogger(__name__) + +DEFAULT_OLS_BASE_URL = "https://www.ebi.ac.uk/ols4/api" +_DEFAULT_TIMEOUT = 30 + + +class _HttpGetter(Protocol): + """The slice of ``requests`` / ``requests.Session`` the client relies on.""" + + def get(self, url: str, **kwargs: Any) -> requests.Response: ... + + +@dataclass(frozen=True, slots=True) +class OlsTerm: + """A single OLS term hit: its CURIE, label, IRI and owning ontology.""" + + obo_id: str + label: str + iri: str + ontology_name: str + + +def obo_id_to_iri(obo_id: str) -> str | None: + """Best-effort CURIE → IRI for the ontologies the resolver walks. + + EFO terms use the ``ebi.ac.uk/efo`` namespace; the other OBO ontologies + (UBERON, MONDO, CHEBI, NCBITaxon, OBI, PSI-MS, …) use the shared OBO PURL. + Returns ``None`` for an unrecognised/non-CURIE id. EDAM has its own scheme + but is never walked for ancestors, so it is intentionally not handled. + """ + if ":" not in obo_id: + return None + prefix, local = obo_id.split(":", 1) + if prefix == "EFO": + return f"http://www.ebi.ac.uk/efo/EFO_{local}" + return f"http://purl.obolibrary.org/obo/{prefix}_{local}" + + +class OlsClient: + """Minimal OLS4 client; ``http`` is injectable so tests stay offline.""" + + def __init__( + self, + base_url: str = DEFAULT_OLS_BASE_URL, + *, + http: _HttpGetter | None = None, + timeout: int = _DEFAULT_TIMEOUT, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + ) -> None: + self._base_url = base_url.rstrip("/") + # The ``requests`` module exposes a matching ``get``; cast for the type. + self._http: _HttpGetter = http if http is not None else cast("_HttpGetter", requests) + self._timeout = timeout + self._max_attempts = max_attempts + + def _get_json(self, url: str, params: dict[str, Any], description: str) -> dict[str, Any]: + def _do() -> requests.Response: + resp = self._http.get(url, params=params, timeout=self._timeout) + resp.raise_for_status() + return resp + + resp = with_retries(_do, max_attempts=self._max_attempts, description=description) + data: dict[str, Any] = resp.json() + return data + + def search( + self, text: str, *, ontology: str, exact: bool = False, rows: int = 5 + ) -> list[OlsTerm]: + """Search ``ontology`` for ``text``; return class hits (best first).""" + params: dict[str, Any] = { + "q": text, + "ontology": ontology, + "type": "class", + "exact": str(exact).lower(), + "rows": rows, + "fieldList": "iri,label,obo_id,ontology_name", + } + data = self._get_json( + f"{self._base_url}/search", + params, + description=f"OLS search q={text!r} ontology={ontology} exact={exact}", + ) + docs = data.get("response", {}).get("docs", []) + return [self._to_term(doc) for doc in docs if doc.get("obo_id")] + + def ancestors(self, obo_id: str, *, ontology: str) -> list[OlsTerm]: + """Return the ``is-a`` (hierarchical) ancestors of ``obo_id``. + + Returns an empty list when the CURIE has no IRI mapping (see + :func:`obo_id_to_iri`). + """ + iri = obo_id_to_iri(obo_id) + if iri is None: + logger.warning("No IRI mapping for %s; cannot fetch ancestors", obo_id) + return [] + # OLS requires the IRI double-URL-encoded in the path segment. + encoded = quote(quote(iri, safe=""), safe="") + url = f"{self._base_url}/ontologies/{ontology}/terms/{encoded}/hierarchicalAncestors" + data = self._get_json( + url, + {"size": 200}, + description=f"OLS ancestors id={obo_id} ontology={ontology}", + ) + terms = data.get("_embedded", {}).get("terms", []) + return [self._to_term(term) for term in terms if term.get("obo_id")] + + @staticmethod + def _to_term(doc: dict[str, Any]) -> OlsTerm: + return OlsTerm( + obo_id=doc.get("obo_id", ""), + label=doc.get("label", ""), + iri=doc.get("iri", ""), + ontology_name=doc.get("ontology_name", ""), + ) diff --git a/src/parce/ontology/registry.py b/src/parce/ontology/registry.py new file mode 100644 index 0000000..03cd74b --- /dev/null +++ b/src/parce/ontology/registry.py @@ -0,0 +1,80 @@ +"""Facet → ontology registry: the single source of truth for *which* controlled +vocabulary annotates *which* experiment-design facet. + +This pins **ontologies, never term IDs**. Specific EFO/MONDO/NCBITaxon/... IDs +are resolved and validated at runtime via OLS (see +:mod:`parce.ontology.resolver`); nothing here is a hardcoded resolution result. +See docs/ARCHITECTURE.md §5. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum + + +class Facet(StrEnum): + """An experiment-*design* facet that binds to one designated ontology. + + Cell type is intentionally absent: it is a data-inferred annotation, not a + design variable (see docs/ARCHITECTURE.md §1). + """ + + ASSAY = "assay" + TISSUE = "tissue" + DISEASE = "disease" + ORGANISM = "organism" + PERTURBATION = "perturbation" + DATA_FORMAT = "data_format" + + +@dataclass(frozen=True, slots=True) +class Ontology: + """A controlled vocabulary used to ground one or more facets. + + ``ols_id`` is the OLS4 ontology slug (lowercase) used in API queries; + ``prefix`` is the CURIE prefix its term IDs carry (e.g. ``NCBITaxon`` in + ``NCBITaxon:9606``). + """ + + ols_id: str + prefix: str + title: str + + +@dataclass(frozen=True, slots=True) +class FacetBinding: + """How a facet binds to ontologies: a primary plus ordered fallbacks. + + Resolution tries ``primary`` first, then each fallback in order — e.g. an + assay missing from EFO is sought in OBI, and MS-specific terms in PSI-MS. + """ + + primary: Ontology + fallbacks: tuple[Ontology, ...] = field(default_factory=tuple) + + def ontologies(self) -> tuple[Ontology, ...]: + """The primary followed by its fallbacks, in resolution order.""" + return (self.primary, *self.fallbacks) + + +# The seven ontologies the registry pins (see docs/ARCHITECTURE.md §5). +EFO = Ontology("efo", "EFO", "Experimental Factor Ontology") +OBI = Ontology("obi", "OBI", "Ontology for Biomedical Investigations") +PSI_MS = Ontology("ms", "MS", "PSI Mass Spectrometry CV") +UBERON = Ontology("uberon", "UBERON", "Uberon anatomy ontology") +MONDO = Ontology("mondo", "MONDO", "Mondo Disease Ontology") +NCBITAXON = Ontology("ncbitaxon", "NCBITaxon", "NCBI organismal taxonomy") +CHEBI = Ontology("chebi", "CHEBI", "Chemical Entities of Biological Interest") +EDAM = Ontology("edam", "EDAM", "EDAM data and format ontology") + +# The registry itself. EFO is the assay anchor; OBI covers upper-level assays it +# lacks and PSI-MS the mass-spectrometry specifics (matching SDRF-Proteomics). +FACET_ONTOLOGY: dict[Facet, FacetBinding] = { + Facet.ASSAY: FacetBinding(EFO, (OBI, PSI_MS)), + Facet.TISSUE: FacetBinding(UBERON), + Facet.DISEASE: FacetBinding(MONDO), + Facet.ORGANISM: FacetBinding(NCBITAXON), + Facet.PERTURBATION: FacetBinding(CHEBI), + Facet.DATA_FORMAT: FacetBinding(EDAM), +} diff --git a/src/parce/ontology/resolver.py b/src/parce/ontology/resolver.py new file mode 100644 index 0000000..8667f90 --- /dev/null +++ b/src/parce/ontology/resolver.py @@ -0,0 +1,148 @@ +"""The shared ontology-resolution stage. + +:class:`OntologyResolver` grounds free-text experiment-design strings to ontology +term IDs and derives the coarse ``molecular_layer`` for an assay. It is the one +place every source's free text lands on the *same* IDs — the precondition for +cross-source linking (docs/ARCHITECTURE.md §3, §5). + +Resolution order, deterministic-first: + +1. on-disk cache (negative results included); +2. OLS4 search across the facet's registered ontologies (primary, then + fallbacks), exact match preferred; +3. an optional **LLM fallback** for strings the deterministic resolvers can't + map — supplied as a plain callable so this package never depends on + ``parce.agent``/Azure. It defaults to ``None`` (no fallback) and is wired up + with the extraction agent in a later PR. + +Network/parse failures degrade gracefully to "unresolved" (logged) rather than +crashing an ingest; the shared retry helper has already exhausted transient +retries by then. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from pathlib import Path + +from parce.models.graph_schema import MolecularLayer +from parce.ontology.base import ResolvedTerm +from parce.ontology.cache import ResolutionCache +from parce.ontology.layers import derive_molecular_layer +from parce.ontology.ols import OlsClient, OlsTerm +from parce.ontology.registry import FACET_ONTOLOGY, Facet, Ontology + +logger = logging.getLogger(__name__) + +# Project-root-relative default cache location. ``data/`` is gitignored. +_DEFAULT_CACHE_DIR = Path(__file__).resolve().parents[3] / "data" / "ontology_cache" + +LlmFallback = Callable[[str, Facet], ResolvedTerm | None] + + +class OntologyResolver: + """Grounds free text to ontology terms; deterministic OLS-first. + + All collaborators are injectable so the resolver can be driven entirely + offline in unit tests. The default :class:`~parce.ontology.cache.ResolutionCache` + is constructed lazily (on first use) so merely instantiating the resolver + touches neither disk nor network. + """ + + def __init__( + self, + *, + client: OlsClient | None = None, + cache: ResolutionCache | None = None, + cache_dir: Path | str = _DEFAULT_CACHE_DIR, + llm_fallback: LlmFallback | None = None, + ) -> None: + self._client = client if client is not None else OlsClient() + self._cache = cache + self._cache_dir = Path(cache_dir) + self._llm_fallback = llm_fallback + + # -- cache (lazy) ---------------------------------------------------- + def _get_cache(self) -> ResolutionCache: + # Built lazily so constructing the resolver touches neither disk nor net. + if self._cache is None: + self._cache = ResolutionCache(self._cache_dir / "resolutions.json") + return self._cache + + @staticmethod + def _cache_key(text: str, facet: Facet) -> str: + return f"{facet.value}|{text.strip().lower()}" + + # -- term resolution ------------------------------------------------- + def resolve_term(self, text: str, facet: Facet) -> ResolvedTerm | None: + """Ground ``text`` to a term for ``facet``; ``None`` if unresolved. + + Caches every outcome (including ``None``) so the same string is queried + at most once per cache lifetime. + """ + clean = text.strip() + if not clean or clean.lower() == "unknown": + return None + + cache = self._get_cache() + key = self._cache_key(clean, facet) + present, cached = cache.get(key) + if present: + return cached + + result = self._resolve_uncached(clean, facet) + cache.set(key, result) + return result + + def _resolve_uncached(self, text: str, facet: Facet) -> ResolvedTerm | None: + for ontology in FACET_ONTOLOGY[facet].ontologies(): + term = self._search_one(text, ontology) + if term is not None: + return term + if self._llm_fallback is not None: + logger.info("Deterministic resolution failed for %r (%s); trying LLM", text, facet) + return self._llm_fallback(text, facet) + logger.warning("Unresolved term %r for facet %s", text, facet) + return None + + def _search_one(self, text: str, ontology: Ontology) -> ResolvedTerm | None: + """Exact match first, then the best fuzzy hit, within one ontology.""" + prefix = f"{ontology.prefix}:" + for exact in (True, False): + try: + hits = self._client.search(text, ontology=ontology.ols_id, exact=exact) + except Exception as exc: # network/HTTP/parse — treat as a miss here + logger.warning( + "OLS search failed for %r in %s (exact=%s): %s", + text, + ontology.ols_id, + exact, + exc, + ) + return None + for hit in hits: + if hit.obo_id.startswith(prefix): + return ResolvedTerm(ontology_id=hit.obo_id, name=hit.label or text) + return None + + # -- molecular layer ------------------------------------------------- + def molecular_layer(self, assay_id: str, *, assay_label: str | None = None) -> MolecularLayer: + """Derive the coarse molecular layer of an EFO ``assay_id``. + + Walks the term's ``is-a`` ancestors via OLS and matches their labels + against the pinned anchor set (see :mod:`parce.ontology.layers`). Passing + the assay's own ``assay_label`` lets a term that is itself an anchor be + classified without relying on the ancestor list including self. Falls + back to :attr:`MolecularLayer.UNKNOWN` on any resolution failure. + """ + try: + ancestors: list[OlsTerm] = self._client.ancestors(assay_id, ontology="efo") + except Exception as exc: + logger.warning("Ancestor walk failed for assay %s: %s", assay_id, exc) + return MolecularLayer.UNKNOWN + + labels = [a.label for a in ancestors] + if assay_label: + labels.append(assay_label) + return derive_molecular_layer(labels) diff --git a/tests/test_normalize.py b/tests/test_normalize.py index aa49a50..cb1ce9f 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1,10 +1,34 @@ -"""Unit tests for the deterministic CELLxGENE normalizer.""" +"""Unit tests for the deterministic CELLxGENE normalizer. + +Offline: organism grounding is driven by a fake resolver, never the live OLS +client, so no network IO occurs. +""" from __future__ import annotations from parce.models.graph_schema import EntityType, KnowledgeGraphOutput from parce.models.raw_record import RawRecord from parce.normalize.cellxgene import CellxgeneNormalizer +from parce.ontology import Facet, ResolvedTerm + +_ORGANISMS = { + "Homo sapiens": ResolvedTerm("NCBITaxon:9606", "Homo sapiens"), + "Mus musculus": ResolvedTerm("NCBITaxon:10090", "Mus musculus"), +} + + +class _FakeResolver: + """Deterministic, offline stand-in for the OLS-backed OntologyResolver.""" + + def resolve_term(self, text: str, facet: Facet) -> ResolvedTerm | None: + if facet is Facet.ORGANISM: + return _ORGANISMS.get(text) + return None + + +def _normalizer() -> CellxgeneNormalizer: + return CellxgeneNormalizer(resolver=_FakeResolver()) + # ``cell_types`` are intentionally present in the payload to prove the normalizer # ignores them (CellType is a data-inferred annotation, deliberately excluded). @@ -77,7 +101,7 @@ def _empty_record() -> RawRecord: class TestCellxgeneNormalizer: def test_basic_structure(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) assert len(kg.studies) == 1 assert kg.studies[0].study_id == "10.1234/test" @@ -95,12 +119,12 @@ def test_study_source_from_record(self): """StudyNode.source is taken from the record, not hardcoded.""" record = _empty_record() record.source = "SomeOtherSource" - kg = CellxgeneNormalizer().normalize(record) + kg = _normalizer().normalize(record) assert kg.studies[0].source == "SomeOtherSource" def test_cell_type_excluded(self): """Cell types in the payload must not produce entities or edges.""" - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) names = {e.name for e in kg.biological_entities} assert "T cell" not in names @@ -115,13 +139,13 @@ def test_cell_type_excluded(self): def test_tissue_entity_deduplication(self): """blood (UBERON:0000178) appears in both datasets but is one entity.""" - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) entity_ids = [e.ontology_id for e in kg.biological_entities] assert entity_ids.count("UBERON:0000178") == 1 def test_species_entity_created(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) species = [e for e in kg.biological_entities if e.entity_type == EntityType.SPECIES] assert len(species) == 1 @@ -130,11 +154,11 @@ def test_species_entity_created(self): def test_no_samples_for_cellxgene(self): """Census is dataset-level; no SampleNode records are emitted (yet).""" - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) assert kg.samples == [] def test_extracted_from_edges(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) extracted = [e for e in kg.edges if e.relation_type == "EXTRACTED_FROM"] assert len(extracted) == 2 @@ -142,7 +166,7 @@ def test_extracted_from_edges(self): assert all(e.target_id == "10.1234/test" for e in extracted) def test_has_tissue_edges(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) tissue_edges = [e for e in kg.edges if e.relation_type == "HAS_TISSUE"] pairs = {(e.source_id, e.target_id) for e in tissue_edges} @@ -151,20 +175,20 @@ def test_has_tissue_edges(self): assert ("ds-002", "UBERON:0002048") in pairs def test_has_condition_edges(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) conditions = [e for e in kg.edges if e.relation_type == "HAS_CONDITION"] assert any(e.target_id == "PATO:0000461" for e in conditions) def test_measured_with_edges(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) assay_edges = [e for e in kg.edges if e.relation_type == "MEASURED_WITH"] assert any(e.source_id == "ds-001" and e.target_id == "EFO:0009922" for e in assay_edges) assert any(e.source_id == "ds-002" and e.target_id == "EFO:0008931" for e in assay_edges) def test_studies_edge(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) studies = [e for e in kg.edges if e.relation_type == "STUDIES"] assert len(studies) == 1 @@ -172,7 +196,7 @@ def test_studies_edge(self): assert studies[0].target_id == "NCBITaxon:9606" def test_empty_datasets(self): - kg = CellxgeneNormalizer().normalize(_empty_record()) + kg = _normalizer().normalize(_empty_record()) assert len(kg.studies) == 1 assert len(kg.datasets) == 0 @@ -181,6 +205,21 @@ def test_empty_datasets(self): assert len(kg.edges) == 0 def test_roundtrip_json(self): - kg = CellxgeneNormalizer().normalize(_RECORD) + kg = _normalizer().normalize(_RECORD) restored = KnowledgeGraphOutput.model_validate_json(kg.model_dump_json()) assert restored == kg + + def test_unresolved_organism_skipped(self): + """An organism the resolver can't ground yields no species node/edge.""" + + class _NoOpResolver: + def resolve_term(self, text: str, facet: Facet) -> ResolvedTerm | None: + return None + + kg = CellxgeneNormalizer(resolver=_NoOpResolver()).normalize(_RECORD) + + species = [e for e in kg.biological_entities if e.entity_type == EntityType.SPECIES] + assert species == [] + assert [e for e in kg.edges if e.relation_type == "STUDIES"] == [] + # Non-organism entities are unaffected (they arrive pre-grounded). + assert any(e.ontology_id == "UBERON:0000178" for e in kg.biological_entities) diff --git a/tests/test_ontology_cache.py b/tests/test_ontology_cache.py new file mode 100644 index 0000000..7cce8cc --- /dev/null +++ b/tests/test_ontology_cache.py @@ -0,0 +1,59 @@ +"""Unit tests for the on-disk resolution cache (uses tmp_path, no network).""" + +from __future__ import annotations + +from parce.ontology.base import ResolvedTerm +from parce.ontology.cache import ResolutionCache + + +class TestResolutionCache: + def test_miss_then_set_then_hit(self, tmp_path): + cache = ResolutionCache(tmp_path / "c.json") + present, value = cache.get("organism|homo sapiens") + assert present is False + assert value is None + + term = ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + cache.set("organism|homo sapiens", term) + + present, value = cache.get("organism|homo sapiens") + assert present is True + assert value == term + + def test_negative_result_is_distinct_from_miss(self, tmp_path): + cache = ResolutionCache(tmp_path / "c.json") + cache.set("organism|martian", None) + + present, value = cache.get("organism|martian") + assert present is True # cached... + assert value is None # ...as a negative result + + def test_persists_across_instances(self, tmp_path): + path = tmp_path / "c.json" + term = ResolvedTerm("MONDO:0008903", "lung cancer") + ResolutionCache(path).set("disease|lung cancer", term) + + reloaded = ResolutionCache(path) + present, value = reloaded.get("disease|lung cancer") + assert present is True + assert value == term + + def test_negative_persists_across_instances(self, tmp_path): + path = tmp_path / "c.json" + ResolutionCache(path).set("organism|martian", None) + + present, value = ResolutionCache(path).get("organism|martian") + assert present is True + assert value is None + + def test_creates_parent_dir_on_write(self, tmp_path): + path = tmp_path / "nested" / "dir" / "c.json" + ResolutionCache(path).set("k", None) + assert path.exists() + + def test_corrupt_file_is_ignored(self, tmp_path): + path = tmp_path / "c.json" + path.write_text("{not valid json", encoding="utf-8") + cache = ResolutionCache(path) # must not raise + present, _ = cache.get("anything") + assert present is False diff --git a/tests/test_ontology_integration.py b/tests/test_ontology_integration.py new file mode 100644 index 0000000..53fc8b0 --- /dev/null +++ b/tests/test_ontology_integration.py @@ -0,0 +1,61 @@ +"""Live OLS4 integration tests for the ontology resolver. + +These hit the public EBI OLS4 API (no credentials needed) and are excluded from +CI. Run with:: + + uv run pytest -m integration tests/test_ontology_integration.py + +They validate the network plumbing the unit tests mock: CURIE→IRI construction, +double-encoding, search field parsing, and the ancestor walk. The organism +assertions are exact and reliable. The molecular_layer assertions are softer +because the anchor labels in ``parce.ontology.layers`` are provisional — see the +note there; tighten these to exact ``==`` once a real lineage is observed. +""" + +from __future__ import annotations + +import logging + +import pytest + +from parce.models.graph_schema import MolecularLayer +from parce.ontology.registry import Facet +from parce.ontology.resolver import OntologyResolver + +pytestmark = pytest.mark.integration + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def resolver(tmp_path): + # Real OLS client; cache in a throwaway dir so the run is self-contained. + return OntologyResolver(cache_dir=tmp_path) + + +class TestLiveResolution: + def test_resolves_homo_sapiens(self, resolver): + term = resolver.resolve_term("Homo sapiens", Facet.ORGANISM) + assert term is not None + assert term.ontology_id == "NCBITaxon:9606" + + def test_resolves_mus_musculus(self, resolver): + term = resolver.resolve_term("Mus musculus", Facet.ORGANISM) + assert term is not None + assert term.ontology_id == "NCBITaxon:10090" + + def test_disease_resolves_to_mondo(self, resolver): + term = resolver.resolve_term("lung cancer", Facet.DISEASE) + assert term is not None + assert term.ontology_id.startswith("MONDO:") + + +class TestLiveMolecularLayer: + def test_scrna_seq_ancestor_walk_runs(self, resolver): + # EFO:0008913 = "single cell RNA sequencing". Assert the walk produces a + # valid layer (plumbing works); log it so anchor labels can be confirmed. + layer = resolver.molecular_layer("EFO:0008913", assay_label="single cell RNA sequencing") + logger.info("Derived molecular_layer for scRNA-seq: %s", layer) + assert isinstance(layer, MolecularLayer) + # Target once anchors are validated: this should be TRANSCRIPTOME. + assert layer in {MolecularLayer.TRANSCRIPTOME, MolecularLayer.UNKNOWN} diff --git a/tests/test_ontology_layers.py b/tests/test_ontology_layers.py new file mode 100644 index 0000000..f1a9fa1 --- /dev/null +++ b/tests/test_ontology_layers.py @@ -0,0 +1,42 @@ +"""Unit tests for the molecular_layer derivation (pure function, no IO).""" + +from __future__ import annotations + +from parce.models.graph_schema import MolecularLayer +from parce.ontology.layers import derive_molecular_layer + + +class TestDeriveMolecularLayer: + def test_transcriptome(self): + labels = ["single-cell RNA sequencing", "RNA assay", "assay by molecule"] + assert derive_molecular_layer(labels) is MolecularLayer.TRANSCRIPTOME + + def test_proteome(self): + assert derive_molecular_layer(["proteomic profiling assay"]) is MolecularLayer.PROTEOME + + def test_epigenome(self): + assert derive_molecular_layer(["ATAC-seq"]) is MolecularLayer.EPIGENOME + + def test_metabolome(self): + assert derive_molecular_layer(["metabolite profiling assay"]) is MolecularLayer.METABOLOME + + def test_genome(self): + assert derive_molecular_layer(["whole genome sequencing assay"]) is MolecularLayer.GENOME + + def test_case_insensitive(self): + assert derive_molecular_layer(["RNA ASSAY"]) is MolecularLayer.TRANSCRIPTOME + + def test_no_anchor_defaults_to_unknown(self): + assert derive_molecular_layer(["some bespoke assay", "thing"]) is MolecularLayer.UNKNOWN + + def test_empty_defaults_to_unknown(self): + assert derive_molecular_layer([]) is MolecularLayer.UNKNOWN + + def test_blank_labels_ignored(self): + assert derive_molecular_layer(["", " "]) is MolecularLayer.UNKNOWN + + def test_specific_anchor_wins_over_broad(self): + """When transcriptome and genome anchors co-occur, the earlier-listed + (more specific) anchor wins deterministically.""" + labels = ["RNA assay", "whole genome sequencing assay"] + assert derive_molecular_layer(labels) is MolecularLayer.TRANSCRIPTOME diff --git a/tests/test_ontology_ols.py b/tests/test_ontology_ols.py new file mode 100644 index 0000000..9fcf9bf --- /dev/null +++ b/tests/test_ontology_ols.py @@ -0,0 +1,139 @@ +"""Offline unit tests for the OLS4 REST client. + +No network: a fake HTTP getter records calls and returns canned JSON. The double +URL-encoding and CURIE→IRI conventions are asserted against the recorded URL. +""" + +from __future__ import annotations + +import pytest +import requests + +from parce.ontology.ols import OlsClient, obo_id_to_iri + + +class _FakeResponse: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + err = requests.HTTPError(f"HTTP {self.status_code}") + resp = requests.Response() + resp.status_code = self.status_code + err.response = resp + raise err + + +class _FakeHttp: + """Records the last GET and replays a queued response.""" + + def __init__(self, response): + self._response = response + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._response + + +class TestOboIdToIri: + def test_efo_namespace(self): + assert obo_id_to_iri("EFO:0009922") == "http://www.ebi.ac.uk/efo/EFO_0009922" + + def test_generic_obo_purl(self): + assert obo_id_to_iri("UBERON:0000178") == "http://purl.obolibrary.org/obo/UBERON_0000178" + assert obo_id_to_iri("NCBITaxon:9606") == "http://purl.obolibrary.org/obo/NCBITaxon_9606" + + def test_non_curie_returns_none(self): + assert obo_id_to_iri("not-a-curie") is None + + +class TestSearch: + def test_parses_docs_and_filters_missing_obo_id(self): + payload = { + "response": { + "docs": [ + { + "obo_id": "NCBITaxon:9606", + "label": "Homo sapiens", + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "ontology_name": "ncbitaxon", + }, + {"label": "no obo id here"}, # dropped + ] + } + } + http = _FakeHttp(_FakeResponse(payload)) + client = OlsClient(http=http) + + terms = client.search("Homo sapiens", ontology="ncbitaxon", exact=True) + + assert len(terms) == 1 + assert terms[0].obo_id == "NCBITaxon:9606" + assert terms[0].label == "Homo sapiens" + + url, kwargs = http.calls[0] + assert url.endswith("/search") + params = kwargs["params"] + assert params["q"] == "Homo sapiens" + assert params["ontology"] == "ncbitaxon" + assert params["exact"] == "true" + assert params["type"] == "class" + + def test_empty_response(self): + http = _FakeHttp(_FakeResponse({"response": {"docs": []}})) + assert OlsClient(http=http).search("nonsense", ontology="efo") == [] + + +class TestAncestors: + def test_double_encodes_iri_and_parses_terms(self): + payload = { + "_embedded": { + "terms": [ + {"obo_id": "EFO:0002772", "label": "assay by molecule"}, + {"obo_id": "EFO:0001457", "label": "RNA assay"}, + {"label": "skipped, no obo_id"}, + ] + } + } + http = _FakeHttp(_FakeResponse(payload)) + client = OlsClient(http=http) + + terms = client.ancestors("EFO:0009922", ontology="efo") + + labels = [t.label for t in terms] + assert labels == ["assay by molecule", "RNA assay"] + + url, _ = http.calls[0] + # IRI is http://www.ebi.ac.uk/efo/EFO_0009922, double-URL-encoded: + # ':' -> %3A -> %253A and '/' -> %2F -> %252F. + assert "%253A" in url + assert "%252F" in url + assert url.endswith("/hierarchicalAncestors") + + def test_unmappable_curie_returns_empty_without_network(self): + http = _FakeHttp(_FakeResponse({})) + terms = OlsClient(http=http).ancestors("not-a-curie", ontology="efo") + assert terms == [] + assert http.calls == [] # never hit the network + + +class TestErrorHandling: + def test_non_transient_http_error_propagates(self): + http = _FakeHttp(_FakeResponse({}, status_code=404)) + with pytest.raises(requests.HTTPError): + OlsClient(http=http).search("x", ontology="efo") + + def test_transient_error_is_retried_then_raises(self, monkeypatch): + # 503 is transient: with_retries retries (sleep patched) then re-raises. + monkeypatch.setattr("parce.sources._retry.time.sleep", lambda _s: None) + http = _FakeHttp(_FakeResponse({}, status_code=503)) + client = OlsClient(http=http, max_attempts=3) + with pytest.raises(requests.HTTPError): + client.search("x", ontology="efo") + assert len(http.calls) == 3 # all attempts made diff --git a/tests/test_ontology_registry.py b/tests/test_ontology_registry.py new file mode 100644 index 0000000..2605fca --- /dev/null +++ b/tests/test_ontology_registry.py @@ -0,0 +1,54 @@ +"""Unit tests for the facet → ontology registry (pure data, no IO).""" + +from __future__ import annotations + +from parce.ontology.registry import ( + CHEBI, + EDAM, + EFO, + FACET_ONTOLOGY, + MONDO, + NCBITAXON, + OBI, + PSI_MS, + UBERON, + Facet, +) + + +class TestRegistry: + def test_every_facet_is_bound(self): + assert set(FACET_ONTOLOGY) == set(Facet) + + def test_primary_bindings(self): + primaries = {facet: binding.primary for facet, binding in FACET_ONTOLOGY.items()} + assert primaries[Facet.ASSAY] is EFO + assert primaries[Facet.TISSUE] is UBERON + assert primaries[Facet.DISEASE] is MONDO + assert primaries[Facet.ORGANISM] is NCBITAXON + assert primaries[Facet.PERTURBATION] is CHEBI + assert primaries[Facet.DATA_FORMAT] is EDAM + + def test_all_seven_pinned_ontologies_present(self): + """The seven ontologies named in ARCHITECTURE §5 all appear.""" + used = set() + for binding in FACET_ONTOLOGY.values(): + used.update(binding.ontologies()) + assert {EFO, OBI, PSI_MS, UBERON, MONDO, NCBITAXON, CHEBI, EDAM} <= used + + def test_assay_falls_back_to_obi_then_psi_ms(self): + binding = FACET_ONTOLOGY[Facet.ASSAY] + assert binding.ontologies() == (EFO, OBI, PSI_MS) + + def test_organism_has_no_fallback(self): + assert FACET_ONTOLOGY[Facet.ORGANISM].ontologies() == (NCBITAXON,) + + def test_curie_prefixes(self): + assert NCBITAXON.prefix == "NCBITaxon" + assert EFO.prefix == "EFO" + assert UBERON.prefix == "UBERON" + + def test_ols_slugs_are_lowercase(self): + for binding in FACET_ONTOLOGY.values(): + for onto in binding.ontologies(): + assert onto.ols_id == onto.ols_id.lower() diff --git a/tests/test_ontology_resolver.py b/tests/test_ontology_resolver.py new file mode 100644 index 0000000..6ad0d44 --- /dev/null +++ b/tests/test_ontology_resolver.py @@ -0,0 +1,170 @@ +"""Offline unit tests for the OntologyResolver. + +The OLS client is a deterministic fake; the cache is a real ResolutionCache on +tmp_path. No network IO. +""" + +from __future__ import annotations + +import pytest +import requests + +from parce.models.graph_schema import MolecularLayer +from parce.ontology.base import ResolvedTerm +from parce.ontology.cache import ResolutionCache +from parce.ontology.ols import OlsTerm +from parce.ontology.registry import Facet +from parce.ontology.resolver import OntologyResolver + + +def _term(obo_id: str, label: str) -> OlsTerm: + return OlsTerm(obo_id=obo_id, label=label, iri="", ontology_name="") + + +class _FakeClient: + """Records calls; returns queued hits keyed by ontology (and optionally exact).""" + + def __init__(self, by_ontology=None, ancestors_terms=None): + self._by_ontology = by_ontology or {} + self._ancestors_terms = ancestors_terms or [] + self.search_calls: list[tuple[str, str, bool]] = [] + self.ancestors_calls: list[tuple[str, str]] = [] + + def search(self, text, *, ontology, exact=False, rows=5): + self.search_calls.append((text, ontology, exact)) + results = self._by_ontology.get(ontology, []) + if isinstance(results, dict): + return list(results.get(exact, [])) + return list(results) + + def ancestors(self, obo_id, *, ontology): + self.ancestors_calls.append((obo_id, ontology)) + return list(self._ancestors_terms) + + +class _RaisingClient: + def search(self, text, *, ontology, exact=False, rows=5): + raise requests.ConnectionError("boom") + + def ancestors(self, obo_id, *, ontology): + raise requests.ConnectionError("boom") + + +def _resolver(tmp_path, client, **kw) -> OntologyResolver: + cache = ResolutionCache(tmp_path / "resolutions.json") + return OntologyResolver(client=client, cache=cache, **kw) + + +_HUMAN = _term("NCBITaxon:9606", "Homo sapiens") + + +class TestResolveTerm: + def test_resolves_and_caches(self, tmp_path): + client = _FakeClient(by_ontology={"ncbitaxon": [_HUMAN]}) + resolver = _resolver(tmp_path, client) + + result = resolver.resolve_term("Homo sapiens", Facet.ORGANISM) + assert result == ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + + # Second call is served from cache: no further client search. + again = resolver.resolve_term("Homo sapiens", Facet.ORGANISM) + assert again == result + assert len(client.search_calls) == 1 + + def test_cache_hit_short_circuits_client(self, tmp_path): + cache = ResolutionCache(tmp_path / "resolutions.json") + cache.set("organism|homo sapiens", ResolvedTerm("NCBITaxon:9606", "Homo sapiens")) + client = _FakeClient() # would return nothing if consulted + resolver = OntologyResolver(client=client, cache=cache) + + result = resolver.resolve_term("Homo sapiens", Facet.ORGANISM) + assert result == ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + assert client.search_calls == [] + + def test_exact_match_preferred(self, tmp_path): + client = _FakeClient(by_ontology={"ncbitaxon": {True: [_HUMAN], False: []}}) + result = _resolver(tmp_path, client).resolve_term("Homo sapiens", Facet.ORGANISM) + assert result == ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + # Stops after the exact hit — only one search. + assert client.search_calls == [("Homo sapiens", "ncbitaxon", True)] + + def test_falls_back_to_fuzzy_when_no_exact(self, tmp_path): + client = _FakeClient(by_ontology={"ncbitaxon": {True: [], False: [_HUMAN]}}) + result = _resolver(tmp_path, client).resolve_term("homo sapien", Facet.ORGANISM) + assert result == ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + assert [c[2] for c in client.search_calls] == [True, False] + + def test_facet_fallback_ontology(self, tmp_path): + # EFO has nothing; OBI provides the assay term. PSI-MS is never reached. + obi_term = _term("OBI:0001271", "RNA-seq assay") + client = _FakeClient(by_ontology={"efo": [], "obi": [obi_term]}) + result = _resolver(tmp_path, client).resolve_term("RNA-seq", Facet.ASSAY) + assert result == ResolvedTerm("OBI:0001271", "RNA-seq assay") + searched = {c[1] for c in client.search_calls} + assert "efo" in searched and "obi" in searched + assert "ms" not in searched + + def test_prefix_mismatch_is_ignored(self, tmp_path): + # A hit whose CURIE is from the wrong ontology must not be accepted. + wrong = _term("CL:0000084", "T cell") + client = _FakeClient(by_ontology={"ncbitaxon": [wrong]}) + result = _resolver(tmp_path, client).resolve_term("Homo sapiens", Facet.ORGANISM) + assert result is None + + def test_llm_fallback_used_when_deterministic_fails(self, tmp_path): + calls: list[tuple[str, Facet]] = [] + + def fallback(text, facet): + calls.append((text, facet)) + return ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + + client = _FakeClient(by_ontology={"ncbitaxon": []}) + resolver = _resolver(tmp_path, client, llm_fallback=fallback) + + result = resolver.resolve_term("hooman", Facet.ORGANISM) + assert result == ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + assert calls == [("hooman", Facet.ORGANISM)] + + def test_no_fallback_returns_none_and_caches_negative(self, tmp_path): + client = _FakeClient(by_ontology={"ncbitaxon": []}) + resolver = _resolver(tmp_path, client) + + assert resolver.resolve_term("hooman", Facet.ORGANISM) is None + # Negative result cached: a second call does not re-query. + assert resolver.resolve_term("hooman", Facet.ORGANISM) is None + assert [c[2] for c in client.search_calls] == [True, False] + + @pytest.mark.parametrize("text", ["", " ", "unknown", "UNKNOWN"]) + def test_blank_or_unknown_text_skips_client(self, tmp_path, text): + client = _FakeClient(by_ontology={"ncbitaxon": [_HUMAN]}) + resolver = _resolver(tmp_path, client) + assert resolver.resolve_term(text, Facet.ORGANISM) is None + assert client.search_calls == [] + + def test_search_error_degrades_to_none(self, tmp_path): + resolver = _resolver(tmp_path, _RaisingClient()) + assert resolver.resolve_term("Homo sapiens", Facet.ORGANISM) is None + + +class TestMolecularLayer: + def test_derives_from_ancestors(self, tmp_path): + ancestors = [_term("EFO:0001457", "RNA assay")] + client = _FakeClient(ancestors_terms=ancestors) + layer = _resolver(tmp_path, client).molecular_layer("EFO:0009922") + assert layer is MolecularLayer.TRANSCRIPTOME + assert client.ancestors_calls == [("EFO:0009922", "efo")] + + def test_uses_assay_label_when_term_is_its_own_anchor(self, tmp_path): + client = _FakeClient(ancestors_terms=[]) # no ancestors returned + layer = _resolver(tmp_path, client).molecular_layer( + "EFO:0008931", assay_label="mass spectrometry assay" + ) + assert layer is MolecularLayer.PROTEOME + + def test_unmatched_lineage_is_unknown(self, tmp_path): + client = _FakeClient(ancestors_terms=[_term("EFO:0000001", "experimental factor")]) + assert _resolver(tmp_path, client).molecular_layer("EFO:0009922") is MolecularLayer.UNKNOWN + + def test_ancestor_error_degrades_to_unknown(self, tmp_path): + resolver = _resolver(tmp_path, _RaisingClient()) + assert resolver.molecular_layer("EFO:0009922") is MolecularLayer.UNKNOWN diff --git a/tests/test_orchestration.py b/tests/test_orchestration.py index aa6eadb..2538abd 100644 --- a/tests/test_orchestration.py +++ b/tests/test_orchestration.py @@ -11,6 +11,17 @@ from unittest.mock import patch from parce.main import run +from parce.ontology import Facet, ResolvedTerm + + +class _FakeResolver: + """Offline organism resolver so the real normalizer never calls OLS.""" + + def resolve_term(self, text: str, facet: Facet) -> ResolvedTerm | None: + if facet is Facet.ORGANISM and text == "Homo sapiens": + return ResolvedTerm("NCBITaxon:9606", "Homo sapiens") + return None + _MOCK_PAPER = { "doi": "10.1234/mock", @@ -40,16 +51,20 @@ def _patch_network(paper=_MOCK_PAPER, cellxgene=_MOCK_CELLXGENE): + # The real adapter + normalizer run end to end; only the network seams are + # mocked. ``OntologyResolver`` is swapped for the offline fake so organism + # grounding (the normalizer's only outbound call) never touches OLS. return ( patch("parce.sources.cellxgene.fetch_paper_metadata", return_value=paper), patch("parce.sources.cellxgene.fetch_cellxgene_datasets", return_value=cellxgene), + patch("parce.normalize.cellxgene.OntologyResolver", _FakeResolver), ) class TestRunOrchestration: def test_full_pipeline(self, tmp_path): - p_paper, p_cx = _patch_network() - with p_paper, p_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + p_paper, p_cx, p_res = _patch_network() + with p_paper, p_cx, p_res, patch("parce.main._OUTPUT_DIR", tmp_path): run(doi="10.1234/mock") out_file = tmp_path / "output.json" @@ -67,8 +82,13 @@ def test_full_pipeline(self, tmp_path): assert len(kg["edges"]) > 0 def test_fetch_called_with_doi(self, tmp_path): - p_paper, p_cx = _patch_network() - with p_paper as mock_paper, p_cx as mock_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + p_paper, p_cx, p_res = _patch_network() + with ( + p_paper as mock_paper, + p_cx as mock_cx, + p_res, + patch("parce.main._OUTPUT_DIR", tmp_path), + ): run(doi="10.1234/mock") mock_paper.assert_called_once_with("10.1234/mock") @@ -76,8 +96,8 @@ def test_fetch_called_with_doi(self, tmp_path): def test_no_datasets_writes_nothing(self, tmp_path): empty = {"doi": "10.1234/mock", "datasets": [], "error": "No datasets found"} - p_paper, p_cx = _patch_network(cellxgene=empty) - with p_paper, p_cx, patch("parce.main._OUTPUT_DIR", tmp_path): + p_paper, p_cx, p_res = _patch_network(cellxgene=empty) + with p_paper, p_cx, p_res, patch("parce.main._OUTPUT_DIR", tmp_path): run(doi="10.1234/mock") assert not (tmp_path / "output.json").exists()