From cf5c223a8800ad55d6ec99d2b8655c1d53cf964e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 12:33:43 +0000 Subject: [PATCH 01/19] feat(samples): add `samples import` command for public-repository accessions Adds `client.samples.import_samples`/`get_import` (POST/GET `/v2/sample-imports`) and the `flowbio samples import` CLI verb, mirroring `upload-batch`'s validate-up-front / report structure for a CSV accession sheet (an `accession` column plus optional name/organism and metadata columns, in place of reads1/reads2/project). Unlike upload-batch, the server tracks every accession submitted together as a single job (one status/error, accessions/sample_ids returned positionally) rather than a per-row upload. The CLI submits all valid rows in one call and polls that one job to completion (--poll-interval/--timeout), per the ticket amendment to poll per job rather than per accession. A completed job reports every row imported; a failed or timed-out job reports every row failed with the job's message, since the job has one outcome for the whole batch. _sheet.py's per-row metadata validation (required/closed-option/annotation rules) is extracted into a shared `metadata_errors` helper so the new accession-sheet validation in _accession_sheet.py reuses it instead of duplicating it; behaviour of validate_row is unchanged. Verified with respx-mocked unit tests (client methods, sheet parsing, and the CLI command including polling/timeout/failure paths) and manual CLI --help smoke checks; full suite passes (343 tests). Refs FLOW-689. Acceptance-test driver's import_by_accession is not routed through flowbio in this PR - see PR description for why. --- flowbio/cli/_accession_sheet.py | 158 +++++++++++++ flowbio/cli/_samples.py | 232 ++++++++++++++++++- flowbio/cli/_sheet.py | 32 ++- flowbio/v2/samples.py | 102 ++++++++- source/cli.rst | 63 +++++ source/v2/samples.rst | 45 +++- tests/unit/cli/test_accession_sheet.py | 227 ++++++++++++++++++ tests/unit/cli/test_samples.py | 305 +++++++++++++++++++++++++ tests/unit/v2/test_samples.py | 186 +++++++++++++++ 9 files changed, 1338 insertions(+), 12 deletions(-) create mode 100644 flowbio/cli/_accession_sheet.py create mode 100644 tests/unit/cli/test_accession_sheet.py diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py new file mode 100644 index 0000000..8446923 --- /dev/null +++ b/flowbio/cli/_accession_sheet.py @@ -0,0 +1,158 @@ +"""CSV accession-sheet parsing and pre-flight validation for ``samples import``. + +An accession sheet is a CSV with one row per accession to import: an +``accession`` column (a public-repository run or experiment accession) plus +optional ``name``/``organism`` and per-accession metadata columns. This +mirrors ``_sheet.py``'s reads-based sample sheet, but the reserved columns +differ — there is nothing to upload (no ``reads1``/``reads2``) and the import +API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, +``organism`` instead. +""" +from __future__ import annotations + +import csv +import re +from dataclasses import dataclass +from pathlib import Path + +from flowbio.cli._exit_codes import CliUsageError +from flowbio.cli._files import existing_file +from flowbio.cli._sheet import metadata_errors +from flowbio.v2.samples import MetadataAttribute, SampleTypeId + +RESERVED_COLUMNS = ("accession", "name", "organism") + +# Mirrors the API's own accession format rule (one run or experiment accession +# per entry), so a malformed accession is reported up front like every other +# validation problem instead of failing the whole batch request server-side. +_SUPPORTED_ACCESSION = re.compile(r"^[SED]R[RX]\d+$") + + +@dataclass(frozen=True) +class AccessionSheetRow: + """One data row of an accession sheet. + + ``accession`` may be empty (empty cell) — that is reported by + :func:`validate_accession_row` rather than rejected here, so all errors + surface together. + """ + + row_number: int + accession: str + name: str | None + organism: str | None + metadata: dict[str, str] + + +@dataclass(frozen=True) +class AccessionSheet: + """A parsed accession sheet.""" + + path: Path + metadata_columns: list[str] + rows: list[AccessionSheetRow] + + +def parse_accession_sheet(path: Path) -> AccessionSheet: + """Parse a CSV accession sheet into an :class:`AccessionSheet`. + + :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or + ``.tsv`` is a usage error directing the user to export to CSV. + :returns: The parsed sheet with reserved/metadata columns separated, + accessions normalised to upper case, and empty cells dropped. + :raises CliUsageError: If the file is not a readable ``.csv``. + """ + if path.suffix.lower() != ".csv": + raise CliUsageError( + f"Accession sheet must be a .csv file: {path}. " + f"Export your spreadsheet to CSV first.", + ) + existing_file(path) + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + headers = reader.fieldnames or [] + metadata_columns = [ + header for header in headers if header not in RESERVED_COLUMNS + ] + rows = [ + _build_row(record, row_number, metadata_columns) + for row_number, record in enumerate(reader, start=1) + ] + return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) + + +def validate_accession_row( + row: AccessionSheetRow, + attributes: list[MetadataAttribute], + sample_type: SampleTypeId, +) -> list[str]: + """Return every validation problem on ``row`` (empty when the row is valid). + + :param row: The parsed row to validate. + :param attributes: The server's metadata attributes, deciding required and + closed-option columns. + :param sample_type: The sample type applied to the whole import; an + attribute required for it must be present. + :returns: One human-readable message per problem, collected so the caller + can report them all at once. + """ + errors: list[str] = [] + if not row.accession: + errors.append("missing required value: accession") + elif not _SUPPORTED_ACCESSION.match(row.accession): + errors.append( + f"'{row.accession}' is not a supported accession — import one run " + f"(SRR/ERR/DRR) or experiment (SRX/ERX/DRX) accession per row", + ) + errors.extend(metadata_errors(row.metadata, attributes, sample_type)) + return errors + + +def duplicate_accession_errors(rows: list[AccessionSheetRow]) -> dict[int, list[str]]: + """Return extra errors for rows whose accession repeats an earlier row. + + Mirrors the API's own duplicate-accession rejection so a sheet with + repeats is reported up front, in the same per-row shape as every other + validation problem, rather than surfacing as one opaque batch failure. + + :param rows: Every row in the sheet, including ones already found invalid. + :returns: A mapping of ``row_number`` to the duplicate-accession messages + for rows after the first occurrence of a repeated accession. Rows with + a blank accession (already reported by :func:`validate_accession_row`) + are never flagged as duplicates of each other. + """ + first_seen: dict[str, int] = {} + errors: dict[int, list[str]] = {} + for row in rows: + if not row.accession: + continue + if row.accession in first_seen: + errors.setdefault(row.row_number, []).append( + f"duplicate accession '{row.accession}' " + f"(first seen at row {first_seen[row.accession]})", + ) + else: + first_seen[row.accession] = row.row_number + return errors + + +def _build_row( + record: dict[str, str], row_number: int, metadata_columns: list[str], +) -> AccessionSheetRow: + def cell(column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + metadata = { + column: value + for column in metadata_columns + if (value := (record.get(column) or "").strip()) + } + accession = cell("accession") + return AccessionSheetRow( + row_number=row_number, + accession=accession.upper() if accession else "", + name=cell("name"), + organism=cell("organism"), + metadata=metadata, + ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index abd1896..afd1f3e 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,10 +10,18 @@ import argparse import json +import math +import time from dataclasses import dataclass from pathlib import Path from typing import Literal +from flowbio.cli._accession_sheet import ( + AccessionSheetRow, + duplicate_accession_errors, + parse_accession_sheet, + validate_accession_row, +) from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -26,7 +34,12 @@ from flowbio.cli._types import JsonValue from flowbio.v2.client import Client from flowbio.v2.exceptions import FlowApiError -from flowbio.v2.samples import MetadataAttribute, SampleTypeId +from flowbio.v2.samples import ( + MetadataAttribute, + SampleImportJob, + SampleImportSpec, + SampleTypeId, +) def register( @@ -76,6 +89,16 @@ def register( "valid rows sequentially, reporting each row's outcome." ), )) + _configure_import(verbs.add_parser( + "import", + parents=[global_parent], + help="Import samples from public-repository accessions.", + description=( + "Validate every row of a CSV accession sheet up front, kick off a " + "single import job for the valid rows, poll it to completion, and " + "report each accession's outcome." + ), + )) def _configure_upload(upload: argparse.ArgumentParser) -> None: @@ -233,6 +256,47 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: ) +_DEFAULT_POLL_INTERVAL = 5.0 +_DEFAULT_TIMEOUT = 1800.0 + + +def _configure_import(import_parser: argparse.ArgumentParser) -> None: + import_parser.set_defaults(command_parser=import_parser, handler=_import_command) + import_parser.add_argument( + "--sheet", + required=True, + metavar="PATH", + type=Path, + help="CSV accession sheet (accession, optional name/organism, plus metadata columns).", + ) + import_parser.add_argument( + "--sample-type", + required=True, + metavar="TYPE", + type=SampleTypeId, + help="Sample type applied to every accession (sent as-is; validated server-side).", + ) + import_parser.add_argument( + "--skip-invalid", + action="store_true", + help="Skip invalid rows (reporting why) instead of aborting the import.", + ) + import_parser.add_argument( + "--poll-interval", + type=float, + default=_DEFAULT_POLL_INTERVAL, + metavar="SECONDS", + help=f"Seconds to wait between checks of the import job's status (default: {_DEFAULT_POLL_INTERVAL:g}).", + ) + import_parser.add_argument( + "--timeout", + type=float, + default=_DEFAULT_TIMEOUT, + metavar="SECONDS", + help=f"Maximum seconds to wait for the import job to finish (default: {_DEFAULT_TIMEOUT:g}).", + ) + + def _upload_command(args: argparse.Namespace, client: Client, output: Output) -> ExitCode: """Upload a single sample and report its identifier. @@ -558,6 +622,172 @@ def _invalid_line(row: SheetRow, reasons: list[str]) -> str: return f"Row {row.row_number} ({row.name}): {'; '.join(reasons)}" +@dataclass(frozen=True) +class _ImportResult: + """The outcome of an ``import`` run, rendered to text or JSON. + + Unlike :class:`_BatchResult`, every accession shares one server-side job: + all rows land in ``imported`` together on a completed job, or all land in + ``failed`` together (carrying the job's one error message) otherwise. + """ + + imported: list[dict[str, JsonValue]] + failed: list[dict[str, JsonValue]] + skipped: list[dict[str, JsonValue]] + + @property + def counts(self) -> dict[str, int]: + return { + "imported": len(self.imported), + "failed": len(self.failed), + "skipped": len(self.skipped), + } + + @property + def document(self) -> dict[str, JsonValue]: + return { + "imported": self.imported, + "failed": self.failed, + "skipped": self.skipped, + "counts": self.counts, + } + + @property + def summary(self) -> str: + counts = self.counts + return ( + f"Imported {counts['imported']}, failed {counts['failed']}, " + f"skipped {counts['skipped']}." + ) + + @property + def exit_code(self) -> ExitCode: + return ExitCode.RUNTIME if self.failed else ExitCode.SUCCESS + + +def _import_command( + args: argparse.Namespace, client: Client, output: Output, +) -> ExitCode: + """Validate an accession sheet up front, then run one import job for it. + + :param args: Parsed command-line arguments. + :param client: The authenticated Flow client. + :param output: The result/error renderer. + :returns: :attr:`ExitCode.SUCCESS` when the import job completes, + :attr:`ExitCode.USAGE` on a pre-flight validation failure without + ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job + fails or does not finish within ``--timeout``. + """ + sheet = parse_accession_sheet(args.sheet) + attributes = client.samples.get_metadata_attributes() + duplicates = duplicate_accession_errors(sheet.rows) + classified = [ + ( + row, + validate_accession_row(row, attributes, args.sample_type) + + duplicates.get(row.row_number, []), + ) + for row in sheet.rows + ] + invalid = [(row, reasons) for row, reasons in classified if reasons] + valid = [row for row, reasons in classified if not reasons] + + if invalid and not args.skip_invalid: + output.emit_error( + "Accession sheet has invalid rows; nothing was imported.", + details=[_invalid_accession_line(row, reasons) for row, reasons in invalid], + ) + return ExitCode.USAGE + + for row, reasons in invalid: + output.emit_advisory(f"Skipped {_invalid_accession_line(row, reasons)}") + skipped = [ + {"row_number": row.row_number, "accession": row.accession, "reasons": reasons} + for row, reasons in invalid + ] + + result = _run_import(valid, args, client, output, skipped) + output.emit_result(result.summary, result.document) + return result.exit_code + + +def _run_import( + rows: list[AccessionSheetRow], + args: argparse.Namespace, + client: Client, + output: Output, + skipped: list[dict[str, JsonValue]], +) -> _ImportResult: + if not rows: + return _ImportResult(imported=[], failed=[], skipped=skipped) + + specs = [ + SampleImportSpec( + accession=row.accession, + sample_type=args.sample_type, + name=row.name, + organism_id=row.organism, + metadata=row.metadata or None, + ) + for row in rows + ] + job = client.samples.import_samples(specs) + output.emit_advisory(f"Import job {job.id} started for {len(rows)} accession(s); polling...") + job = _poll_job(client, job, args.poll_interval, args.timeout) + return _import_result(job, rows, skipped, output) + + +def _poll_job( + client: Client, job: SampleImportJob, poll_interval: float, timeout: float, +) -> SampleImportJob: + max_attempts = max(1, math.ceil(timeout / poll_interval)) + for _ in range(max_attempts): + if job.status != "RUNNING": + break + time.sleep(poll_interval) + job = client.samples.get_import(job.id) + return job + + +def _import_result( + job: SampleImportJob, + rows: list[AccessionSheetRow], + skipped: list[dict[str, JsonValue]], + output: Output, +) -> _ImportResult: + if job.status == "COMPLETED": + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) + imported = [ + { + "row_number": row.row_number, + "accession": row.accession, + "sample_id": accession_to_sample_id.get(row.accession), + } + for row in rows + ] + for entry in imported: + output.emit_advisory( + f"Row {entry['row_number']} ({entry['accession']}): " + f"imported sample {entry['sample_id']}", + ) + return _ImportResult(imported=imported, failed=[], skipped=skipped) + + message = job.error or f"import job {job.id} ended with status {job.status}" + failed = [ + {"row_number": row.row_number, "accession": row.accession, "message": message} + for row in rows + ] + for entry in failed: + output.emit_advisory( + f"Row {entry['row_number']} ({entry['accession']}): import failed — {message}", + ) + return _ImportResult(imported=[], failed=failed, skipped=skipped) + + +def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: + return f"Row {row.row_number} ({row.accession or ''}): {'; '.join(reasons)}" + + def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index e956ccb..2e70a4f 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -96,7 +96,6 @@ def validate_row( :returns: One human-readable message per problem, collected so the caller can report them all at once. """ - by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] if not row.name: errors.append("missing required value: name") @@ -107,7 +106,7 @@ def validate_row( for label, reads in (("reads1", row.reads1), ("reads2", row.reads2)): if reads is not None and not reads.is_file(): errors.append(f"{label} file not found: {reads}") - errors.extend(_metadata_errors(row, by_identifier, attributes, sample_type)) + errors.extend(metadata_errors(row.metadata, attributes, sample_type)) return errors @@ -144,20 +143,35 @@ def _resolve(value: str | None, base_dir: Path) -> Path | None: return path if path.is_absolute() else base_dir / path -def _metadata_errors( - row: SheetRow, - by_identifier: dict[str, MetadataAttribute], +def metadata_errors( + metadata: dict[str, str], attributes: list[MetadataAttribute], sample_type: SampleTypeId, ) -> list[str]: + """Return every metadata validation problem for ``metadata`` (empty when valid). + + Shared by :func:`validate_row` and the accession-sheet equivalent in + :mod:`flowbio.cli._accession_sheet`, since required/closed-option/annotation + rules apply identically regardless of what the rest of the row looks like. + + :param metadata: The row's metadata, keyed by attribute identifier (or + ``__annotation`` for a free-text companion). + :param attributes: The server's metadata attributes, deciding required and + closed-option columns. + :param sample_type: The sample type applied to the whole batch; an attribute + required for it must be present. + :returns: One human-readable message per problem, collected so the caller can + report them all at once. + """ + by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] for attribute in attributes: required = ( attribute.required or sample_type in attribute.required_for_sample_types ) - if required and not row.metadata.get(attribute.identifier): + if required and not metadata.get(attribute.identifier): errors.append(f"missing required metadata: {attribute.identifier}") - for key, value in row.metadata.items(): + for key, value in metadata.items(): if key.endswith(ANNOTATION_SUFFIX): continue attribute = by_identifier.get(key) @@ -170,11 +184,11 @@ def _metadata_errors( f"value '{value}' for {key} is not one of: " f"{', '.join(attribute.options)}", ) - for key in row.metadata: + for key in metadata: if not key.endswith(ANNOTATION_SUFFIX): continue base = key[: -len(ANNOTATION_SUFFIX)] - if not row.metadata.get(base): + if not metadata.get(base): errors.append(f"{key} set without a value for {base}") continue attribute = by_identifier.get(base) diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index fd888ed..8dada4f 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -27,8 +27,9 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, NewType +from typing import TYPE_CHECKING, Literal, NewType from pydantic import BaseModel, Field @@ -139,6 +140,58 @@ class MultiplexedUpload(BaseModel, frozen=True): ) +SampleImportJobId = NewType("SampleImportJobId", int) +"""The identifier of a sample-import job, as returned by +:meth:`SampleResource.import_samples`.""" + + +SampleImportStatus = Literal["RUNNING", "COMPLETED", "FAILED"] +"""The lifecycle state of a :class:`SampleImportJob`.""" + + +@dataclass(frozen=True) +class SampleImportSpec: + """One accession to import, with its per-accession identity and metadata. + + Example:: + + specs = [ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + SampleImportSpec( + accession="ERR10677146", + sample_type="RNA-Seq", + organism_id="Hs", + metadata={"strandedness": "reverse"}, + ), + ] + """ + + accession: str + sample_type: SampleTypeId + name: str | None = None + organism_id: str | None = None + metadata: dict[str, str] | None = None + + +class SampleImportJob(BaseModel, frozen=True): + """A batch job that imports one or more accessions into samples. + + All accessions submitted in one :meth:`SampleResource.import_samples` call + share a single job: ``status`` and ``error`` describe the whole batch, and + ``accessions``/``sample_ids`` correspond positionally once ``status`` is + ``"COMPLETED"``. + """ + + id: SampleImportJobId = Field(description="Unique identifier for this import job.") + status: SampleImportStatus = Field(description="The job's current lifecycle state.") + accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") + sample_ids: list[int] = Field( + description="The created samples' ids, corresponding to `accessions` once the job has completed.", + ) + execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") + error: str | None = Field(description='The failure reason, set only when status is "FAILED".') + + class SampleResource: """Provides access to sample-related API endpoints. @@ -399,6 +452,53 @@ def get_metadata_attributes(self) -> list[MetadataAttribute]: """ return [self._create_metadata_attribute(item) for item in (self._transport.get("/samples/metadata"))] + def import_samples(self, imports: Sequence[SampleImportSpec]) -> SampleImportJob: + """Kick off a batch import of samples from public-repository accessions. + + Every accession is submitted together and tracked as a single job — poll + it with :meth:`get_import` until its status leaves ``"RUNNING"``. + + Requires authentication. + + Example:: + + job = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + ]) + print(f"Import job: {job.id}") + + :param imports: The accessions to import, one :class:`SampleImportSpec` each. + :raises FlowApiError: If any entry is invalid, e.g. an unsupported + accession format, unknown sample type, or missing required metadata. + """ + payload = {"imports": [self._import_spec_fields(spec) for spec in imports]} + return SampleImportJob(**self._transport.post("/v2/sample-imports", json=payload)) + + def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: + """Fetch the current state of an import job. + + Example:: + + job = client.samples.get_import(job.id) + if job.status == "COMPLETED": + print(f"Imported samples: {job.sample_ids}") + + :param job_id: The job id returned by :meth:`import_samples`. + :raises NotFoundError: If no import job with that id exists. + """ + return SampleImportJob(**self._transport.get(f"/v2/sample-imports/{job_id}")) + + @staticmethod + def _import_spec_fields(spec: SampleImportSpec) -> dict: + fields: dict = {"accession": spec.accession, "sample_type": spec.sample_type} + if spec.name is not None: + fields["name"] = spec.name + if spec.organism_id is not None: + fields["organism"] = spec.organism_id + if spec.metadata: + fields["metadata"] = spec.metadata + return fields + def _create_metadata_attribute(self, item: dict) -> MetadataAttribute: item["required_for_sample_types"] = [ SampleTypeId(link["sample_type_identifier"]) diff --git a/source/cli.rst b/source/cli.rst index 029c86a..e26fcf1 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -367,6 +367,69 @@ authentication failure; otherwise the standard mapping above. $ flowbio samples upload-batch --sheet ./samples.csv --sample-type RNA-Seq --json {"uploaded": [{"row_number": 1, "name": "liver_r1", "sample_id": "samp_1"}], "failed": [], "skipped": [], "counts": {"uploaded": 1, "failed": 0, "skipped": 0}} +``samples import`` +~~~~~~~~~~~~~~~~~~~ + +Import samples from public-repository accessions (SRR/ERR/DRR run or +SRX/ERX/DRX experiment accessions), applying one sample type to every row — +no files to upload yourself. + +:: + + flowbio samples import --sheet PATH --sample-type TYPE + [--skip-invalid] [--poll-interval SECONDS] [--timeout SECONDS] + +Run ``flowbio samples import --help`` for the full option list. The sheet is +a CSV with an ``accession`` column plus optional ``name``/``organism`` and +metadata columns (there is no ``batch-template`` equivalent for it, since it +has no reads files or project field). ``name`` defaults to the accession when +omitted. The sample type is sent as-is and validated server-side. + +**Validation is up front**, mirroring ``upload-batch``: a missing or +malformed accession, a duplicate accession within the sheet, a value outside +a closed-option attribute's allowed values, or missing metadata required for +the chosen type — every problem is collected before anything is submitted. +By default any invalid row aborts the whole run (exit ``2``); +``--skip-invalid`` skips them (reporting why) and imports the rest. + +Unlike ``upload-batch``, every valid row is submitted **together as one +server-side job** — there is no per-row upload loop. The command polls that +job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` +seconds, default 1800) until it leaves ``"RUNNING"``. Once the job completes, +every submitted row is reported as imported; if it fails or times out, every +row is reported as failed with the same message, since the job has one +outcome for the whole batch. + +**Output** — human: each row's outcome on stderr, then a final counts summary +on stdout. ``--json``: a single document on stdout with ``imported``, +``failed``, and ``skipped`` lists plus a ``counts`` summary: + +.. code-block:: json + + { + "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], + "failed": [], + "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], + "counts": {"imported": 1, "failed": 0, "skipped": 1} + } + +**Exit codes** — ``0`` the import job completed; ``2`` a pre-flight +validation failure (without ``--skip-invalid``) or a non-CSV sheet; ``1`` the +import job failed or did not finish within ``--timeout``; ``3`` +authentication failure; otherwise the standard mapping above. + +**Example** + +.. code-block:: bash + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + Row 1 (ERR1160845): imported sample 101 + Row 2 (ERR10677146): imported sample 102 + Imported 2, failed 0, skipped 0. + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}} + ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 6fb5dd2..d78855b 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -166,6 +166,45 @@ To reject the upload when warnings are present, set This raises :class:`~flowbio.v2.exceptions.AnnotationValidationError` if the annotation has any warnings. +.. _sample-imports: + +Importing samples from public repositories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create +samples directly from public-repository run or experiment accessions +(SRR/ERR/DRR or SRX/ERX/DRX), instead of uploading files yourself. Every +accession submitted together is tracked as a single job:: + + from flowbio.v2.samples import SampleImportSpec + + job = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + SampleImportSpec( + accession="ERR10677146", + sample_type="RNA-Seq", + metadata={"strandedness": "reverse"}, + ), + ]) + +The job starts out ``"RUNNING"``. Poll it with +:meth:`~flowbio.v2.samples.SampleResource.get_import` until its status +leaves ``"RUNNING"``:: + + import time + + while job.status == "RUNNING": + time.sleep(5) + job = client.samples.get_import(job.id) + + if job.status == "COMPLETED": + print(f"Imported samples: {job.sample_ids}") + else: + print(f"Import failed: {job.error}") + +``job.accessions`` and ``job.sample_ids`` correspond positionally once the +job has completed. + API Reference ------------- @@ -185,4 +224,8 @@ Models .. autopydantic_model:: flowbio.v2.samples.Organism -.. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload \ No newline at end of file +.. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload + +.. autoclass:: flowbio.v2.samples.SampleImportSpec + +.. autopydantic_model:: flowbio.v2.samples.SampleImportJob \ No newline at end of file diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py new file mode 100644 index 0000000..f7edca8 --- /dev/null +++ b/tests/unit/cli/test_accession_sheet.py @@ -0,0 +1,227 @@ +import csv +from pathlib import Path + +import pytest + +from flowbio.cli._accession_sheet import ( + duplicate_accession_errors, + parse_accession_sheet, + validate_accession_row, +) +from flowbio.cli._exit_codes import CliUsageError +from flowbio.v2.samples import MetadataAttribute, SampleTypeId + +SAMPLE_TYPE = SampleTypeId("rna_seq") +HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + +def _attributes() -> list[MetadataAttribute]: + return [ + MetadataAttribute( + identifier="cell_type", + name="Cell Type", + description="The cell type", + required=False, + required_for_sample_types=[SAMPLE_TYPE], + options=["Neuron", "Fibroblast"], + allow_annotation=False, + ), + MetadataAttribute( + identifier="source", + name="Source", + description="Sample source", + required=False, + required_for_sample_types=[], + options=None, + allow_annotation=True, + ), + ] + + +def _write_sheet( + directory: Path, *records: dict[str, str], headers: list[str] = HEADERS, +) -> Path: + path = directory / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=headers) + writer.writeheader() + writer.writerows(records) + return path + + +class TestParseAccessionSheet: + + def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] + + def test_accession_is_normalised_to_upper_case(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "err1160845"}), + ) + + assert sheet.rows[0].accession == "ERR1160845" + + def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845", "cell_type": "", "source": "blood"}, + )) + + assert sheet.rows[0].metadata == {"source": "blood"} + + def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.rows[0].name is None + assert sheet.rows[0].organism is None + + def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845", "name": "liver_r1", "organism": "Hs"}, + )) + + assert sheet.rows[0].name == "liver_r1" + assert sheet.rows[0].organism == "Hs" + + def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="", encoding="utf-8-sig") as handle: + writer = csv.DictWriter(handle, fieldnames=HEADERS) + writer.writeheader() + writer.writerow({"accession": "ERR1160845"}) + + sheet = parse_accession_sheet(path) + + assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] + assert sheet.rows[0].accession == "ERR1160845" + + def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845"}, + {"accession": "ERR10677146"}, + )) + + assert [row.row_number for row in sheet.rows] == [1, 2] + + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK") + + with pytest.raises(CliUsageError, match="CSV"): + parse_accession_sheet(xlsx) + + def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: + tsv = tmp_path / "sheet.tsv" + tsv.write_text("accession\nERR1160845\n") + + with pytest.raises(CliUsageError, match="CSV"): + parse_accession_sheet(tsv) + + def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError): + parse_accession_sheet(tmp_path / "absent.csv") + + +class TestValidateAccessionRow: + + def _row(self, directory: Path, **overrides: str): + record = {"accession": "ERR1160845"} + record.update(overrides) + sheet = parse_accession_sheet(_write_sheet(directory, record)) + return sheet.rows[0] + + def test_valid_row_has_no_errors(self, tmp_path: Path) -> None: + row = self._row(tmp_path, cell_type="Neuron") + + assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] + + def test_missing_accession_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, accession="", cell_type="Neuron") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("accession" in error for error in errors) + + def test_unsupported_accession_format_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, accession="GSE12345", cell_type="Neuron") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("GSE12345" in error for error in errors) + + def test_run_and_experiment_accessions_are_supported(self, tmp_path: Path) -> None: + for accession in ("ERR1160845", "SRR1234567", "DRR7654321", "SRX111", "ERX222", "DRX333"): + row = self._row(tmp_path, accession=accession, cell_type="Neuron") + + assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] + + def test_value_outside_options_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, cell_type="Alien") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("Alien" in error for error in errors) + + def test_required_for_type_metadata_missing_reports_error( + self, tmp_path: Path, + ) -> None: + row = self._row(tmp_path) + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("cell_type" in error for error in errors) + + def test_annotation_set_without_its_value_reports_error( + self, tmp_path: Path, + ) -> None: + row = self._row( + tmp_path, cell_type="Neuron", **{"source__annotation": "qPCR"}, + ) + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("source__annotation" in error for error in errors) + + def test_all_per_row_errors_collected(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "", "cell_type": "Alien"}, + )) + + errors = validate_accession_row(sheet.rows[0], _attributes(), SAMPLE_TYPE) + + assert len(errors) >= 2 + + +class TestDuplicateAccessionErrors: + + def test_no_errors_when_all_unique(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1"}, {"accession": "ERR2"}, + )) + + assert duplicate_accession_errors(sheet.rows) == {} + + def test_repeated_accession_reports_error_on_later_row(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1"}, {"accession": "err1"}, + )) + + errors = duplicate_accession_errors(sheet.rows) + + assert 1 not in errors + assert any("ERR1" in message for message in errors[2]) + + def test_blank_accessions_are_not_treated_as_duplicates(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": ""}, {"accession": ""}, + )) + + assert duplicate_accession_errors(sheet.rows) == {} diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5c461c1..3d2bf6a 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -2,6 +2,7 @@ import json from http import HTTPStatus from pathlib import Path +from unittest.mock import patch import httpx import respx @@ -13,6 +14,7 @@ ANNOTATION_TEMPLATE_URL = f"{DEFAULT_BASE_URL}/annotation" ANNOTATION_UPLOAD_URL = f"{DEFAULT_BASE_URL}/upload/annotation" MULTIPLEXED_UPLOAD_URL = f"{DEFAULT_BASE_URL}/upload/multiplexed" +SAMPLE_IMPORTS_URL = f"{DEFAULT_BASE_URL}/v2/sample-imports" TOKEN = "test.token" @@ -923,3 +925,306 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert result.exit_code == 2 assert upload.call_count == 0 assert "CSV" in result.stderr + + +IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + +def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: + path = directory / "accessions.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=IMPORT_HEADERS) + writer.writeheader() + writer.writerows(records) + return path + + +def _job_json( + job_id: int, + status: str, + accessions: list[str], + sample_ids: list[int] | None = None, + error: str | None = None, +) -> dict: + return { + "id": job_id, + "status": status, + "accessions": accessions, + "sample_ids": sample_ids or [], + "execution_id": 7, + "error": error, + } + + +class TestSamplesImport: + + @respx.mock + def test_completed_job_reports_imported_samples( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "ERR2"], [101, 102], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Fibroblast"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert "101" in result.stderr + assert "102" in result.stderr + assert "Imported 2, failed 0, skipped 0." in result.stdout + + @respx.mock + def test_json_document_reports_outcomes_and_counts( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert result.stdout.count("\n") == 1 + assert document["counts"] == {"imported": 1, "failed": 0, "skipped": 0} + assert document["imported"][0] == { + "row_number": 1, "accession": "ERR1", "sample_id": 101, + } + + @respx.mock + def test_sends_sample_type_and_metadata_in_kickoff_payload( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, {"accession": "err1", "cell_type": "Neuron", "organism": "Hs"}, + ) + + run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{ + "accession": "ERR1", + "sample_type": "rna_seq", + "organism": "Hs", + "metadata": {"cell_type": "Neuron"}, + }], + } + + @respx.mock + @patch("time.sleep") + def test_polls_until_job_completes( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1") + poll_route.side_effect = [ + httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), + ] + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 0 + assert poll_route.call_count == 2 + assert "101" in result.stderr + + @respx.mock + @patch("time.sleep") + def test_failed_job_reports_error_message_per_row( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + error_message = "download failed: connection reset" + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], error=error_message, + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 1 + assert error_message in result.stderr + assert "Row 1 (ERR1)" in result.stderr + assert "Row 2 (ERR2)" in result.stderr + + @respx.mock + @patch("time.sleep") + def test_timeout_reports_failure_without_hanging( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--timeout", "2", + ) + + assert result.exit_code == 1 + assert poll_route.call_count == 2 + assert "Row 1 (ERR1)" in result.stderr + + @respx.mock + def test_invalid_row_aborts_and_imports_nothing( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "not-an-accession", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "Row 2" in result.stderr + assert "NOT-AN-ACCESSION" in result.stderr + + @respx.mock + def test_skip_invalid_imports_valid_rows( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "bogus", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--skip-invalid", + ) + + assert result.exit_code == 0 + assert route.call_count == 1 + payload = json.loads(route.calls[0].request.content) + assert len(payload["imports"]) == 1 + assert "BOGUS" in result.stderr + + @respx.mock + def test_duplicate_accession_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "err1", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "duplicate" in result.stderr.lower() + + @respx.mock + def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK\x03\x04") + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(xlsx), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "CSV" in result.stderr + + def test_missing_sheet_is_usage_error(self, run_cli) -> None: + result = run_cli( + "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + + def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index cc42ad7..1f62a46 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1,3 +1,4 @@ +import json from http import HTTPStatus from pathlib import Path from unittest.mock import ANY, patch @@ -10,6 +11,7 @@ from flowbio.v2.exceptions import ( AnnotationValidationError, BadRequestError, + FlowApiError, NotFoundError, ) from flowbio.v2.samples import ( @@ -17,6 +19,9 @@ MultiplexedUpload, Organism, Project, + SampleImportJob, + SampleImportJobId, + SampleImportSpec, SampleResource, SampleType, Sample, @@ -1049,3 +1054,184 @@ def test_chunked_multiplexed_upload(self, tmp_path: Path) -> None: assert mux_route.call_count == 3 assert result.data_ids == ["mux_1"] + +class TestImportSamples: + + @respx.mock + def test_posts_imports_and_parses_job(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 42, + "status": "RUNNING", + "created": 1700000000, + "started": None, + "finished": None, + "accessions": ["ERR1160845"], + "sample_ids": [], + "execution_id": None, + "error": None, + }), + ) + + client = Client() + result = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="rna_seq"), + ]) + + assert result == SampleImportJob( + id=SampleImportJobId(42), + status="RUNNING", + accessions=["ERR1160845"], + sample_ids=[], + execution_id=None, + error=None, + ) + assert route.call_count == 1 + + @respx.mock + def test_sends_accession_and_sample_type(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec(accession="ERR1", sample_type="rna_seq"), + ]) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{"accession": "ERR1", "sample_type": "rna_seq"}], + } + + @respx.mock + def test_sends_optional_fields_when_present(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec( + accession="ERR1", + sample_type="rna_seq", + name="my_sample", + organism_id="Hs", + metadata={"strandedness": "reverse"}, + ), + ]) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{ + "accession": "ERR1", + "sample_type": "rna_seq", + "name": "my_sample", + "organism": "Hs", + "metadata": {"strandedness": "reverse"}, + }], + } + + @respx.mock + def test_sends_multiple_imports_in_one_request(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1", "ERR2"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec(accession="ERR1", sample_type="rna_seq"), + SampleImportSpec(accession="ERR2", sample_type="rna_seq"), + ]) + + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == ["ERR1", "ERR2"] + assert route.call_count == 1 + + @respx.mock + def test_raises_flow_api_error_on_validation_failure(self) -> None: + respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response( + HTTPStatus.UNPROCESSABLE_ENTITY, + json={"error": "at least one accession is required"}, + ), + ) + + client = Client() + + with pytest.raises(FlowApiError) as exc_info: + client.samples.import_samples([]) + + assert exc_info.value.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + +class TestGetImport: + + @respx.mock + def test_parses_completed_job(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "COMPLETED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, + "accessions": ["ERR1160845", "ERR10677146"], + "sample_ids": [101, 102], + "execution_id": 7, + "error": None, + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result == SampleImportJob( + id=SampleImportJobId(42), + status="COMPLETED", + accessions=["ERR1160845", "ERR10677146"], + sample_ids=[101, 102], + execution_id=7, + error=None, + ) + + @respx.mock + def test_parses_failed_job_with_error(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "FAILED", + "accessions": ["ERR1160845"], + "sample_ids": [], + "execution_id": 7, + "error": "download failed: connection reset", + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.status == "FAILED" + assert result.error == "download failed: connection reset" + + @respx.mock + def test_raises_not_found_for_unknown_job(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/999").mock( + return_value=httpx.Response( + HTTPStatus.NOT_FOUND, json={"error": "sample import 999 does not exist"}, + ), + ) + + client = Client() + + with pytest.raises(NotFoundError): + client.samples.get_import(SampleImportJobId(999)) From aec9c961647043396dac71f09470a066e40f1a1b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:58:53 +0000 Subject: [PATCH 02/19] fix(samples): address import-command review findings - Surface job_id/job_status/execution_id in the JSON document (null when no job was created), so a timed-out or interrupted import can be resumed with client.samples.get_import instead of leaving an untrackable server-side job. - Reject non-positive --poll-interval/--timeout as a usage error (exit 2) instead of crashing with ZeroDivisionError/ValueError. - Switch _poll_job from an attempt-count budget to a time.monotonic() deadline, so --timeout is actually a wall-clock bound rather than poll_interval * attempts (which silently added unbounded request latency and ignored a --timeout smaller than --poll-interval). - A COMPLETED job whose sample_ids don't cover every submitted accession no longer silently reports those rows as imported with sample_id: null; they are now reported failed with an explanatory message. - A failed/timed-out job now still reports the sample id for any accession the job did manage to create before failing, so a retry doesn't have to guess what to skip. - The timeout message no longer reads as "ended with status RUNNING"; it now says the job didn't finish in time and points at get_import to resume. - Reordered _sheet.py so the public metadata_errors sits above the private _build_row/_resolve helpers. Addresses Claude's review on flowbio#18; see PR comments for the findings left as deliberate, explained trade-offs (SampleImportSpec as a dataclass, sample_ids staying int, the accession-format regex, and not adding poll-error retry tolerance). Refs FLOW-689. --- flowbio/cli/_samples.py | 121 +++++++++++++++++------- flowbio/cli/_sheet.py | 66 ++++++------- source/cli.rst | 27 ++++-- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 164 ++++++++++++++++++++++++++++++++- 5 files changed, 302 insertions(+), 78 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index afd1f3e..2bfb371 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,7 +10,6 @@ import argparse import json -import math import time from dataclasses import dataclass from pathlib import Path @@ -286,14 +285,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: type=float, default=_DEFAULT_POLL_INTERVAL, metavar="SECONDS", - help=f"Seconds to wait between checks of the import job's status (default: {_DEFAULT_POLL_INTERVAL:g}).", + help=( + "Seconds to wait between checks of the import job's status; must be " + f"positive (default: {_DEFAULT_POLL_INTERVAL:g})." + ), ) import_parser.add_argument( "--timeout", type=float, default=_DEFAULT_TIMEOUT, metavar="SECONDS", - help=f"Maximum seconds to wait for the import job to finish (default: {_DEFAULT_TIMEOUT:g}).", + help=( + "Maximum seconds to wait for the import job to finish; must be " + f"positive (default: {_DEFAULT_TIMEOUT:g})." + ), ) @@ -629,11 +634,17 @@ class _ImportResult: Unlike :class:`_BatchResult`, every accession shares one server-side job: all rows land in ``imported`` together on a completed job, or all land in ``failed`` together (carrying the job's one error message) otherwise. + ``job_id``/``job_status``/``execution_id`` are ``None`` only when no job + was ever created (every row was invalid or skipped), so a timed-out or + interrupted run can still be resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] failed: list[dict[str, JsonValue]] skipped: list[dict[str, JsonValue]] + job_id: int | None = None + job_status: str | None = None + execution_id: int | None = None @property def counts(self) -> dict[str, int]: @@ -650,6 +661,9 @@ def document(self) -> dict[str, JsonValue]: "failed": self.failed, "skipped": self.skipped, "counts": self.counts, + "job_id": self.job_id, + "job_status": self.job_status, + "execution_id": self.execution_id, } @property @@ -675,9 +689,11 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` when the import job completes, :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job - fails or does not finish within ``--timeout``. + ``--skip-invalid`` or a non-positive ``--poll-interval``/``--timeout``, + or :attr:`ExitCode.RUNTIME` if the import job fails or does not + finish within ``--timeout``. """ + _validate_poll_options(args.poll_interval, args.timeout) sheet = parse_accession_sheet(args.sheet) attributes = client.samples.get_metadata_attributes() duplicates = duplicate_accession_errors(sheet.rows) @@ -711,6 +727,13 @@ def _import_command( return result.exit_code +def _validate_poll_options(poll_interval: float, timeout: float) -> None: + if poll_interval <= 0: + raise CliUsageError(f"--poll-interval must be positive, got {poll_interval!r}.") + if timeout <= 0: + raise CliUsageError(f"--timeout must be positive, got {timeout!r}.") + + def _run_import( rows: list[AccessionSheetRow], args: argparse.Namespace, @@ -740,10 +763,8 @@ def _run_import( def _poll_job( client: Client, job: SampleImportJob, poll_interval: float, timeout: float, ) -> SampleImportJob: - max_attempts = max(1, math.ceil(timeout / poll_interval)) - for _ in range(max_attempts): - if job.status != "RUNNING": - break + deadline = time.monotonic() + timeout + while job.status == "RUNNING" and time.monotonic() < deadline: time.sleep(poll_interval) job = client.samples.get_import(job.id) return job @@ -755,33 +776,67 @@ def _import_result( skipped: list[dict[str, JsonValue]], output: Output, ) -> _ImportResult: + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) if job.status == "COMPLETED": - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - imported = [ - { - "row_number": row.row_number, - "accession": row.accession, - "sample_id": accession_to_sample_id.get(row.accession), - } - for row in rows - ] - for entry in imported: - output.emit_advisory( - f"Row {entry['row_number']} ({entry['accession']}): " - f"imported sample {entry['sample_id']}", - ) - return _ImportResult(imported=imported, failed=[], skipped=skipped) + imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + else: + imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) + return _ImportResult( + imported=imported, + failed=failed, + skipped=skipped, + job_id=job.id, + job_status=job.status, + execution_id=job.execution_id, + ) - message = job.error or f"import job {job.id} ended with status {job.status}" - failed = [ - {"row_number": row.row_number, "accession": row.accession, "message": message} - for row in rows - ] - for entry in failed: - output.emit_advisory( - f"Row {entry['row_number']} ({entry['accession']}): import failed — {message}", + +def _completed_outcomes( + job: SampleImportJob, + rows: list[AccessionSheetRow], + accession_to_sample_id: dict[str, int], + output: Output, +) -> tuple[list[dict[str, JsonValue]], list[dict[str, JsonValue]]]: + imported: list[dict[str, JsonValue]] = [] + failed: list[dict[str, JsonValue]] = [] + for row in rows: + sample_id = accession_to_sample_id.get(row.accession) + if sample_id is None: + message = f"import job {job.id} completed but returned no sample id for this accession" + failed.append({"row_number": row.row_number, "accession": row.accession, "message": message}) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + continue + imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): imported sample {sample_id}") + return imported, failed + + +def _failed_outcomes( + job: SampleImportJob, + rows: list[AccessionSheetRow], + accession_to_sample_id: dict[str, int], + output: Output, +) -> list[dict[str, JsonValue]]: + if job.status == "RUNNING": + message = ( + f"import job {job.id} did not finish within the configured timeout " + f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) - return _ImportResult(imported=[], failed=failed, skipped=skipped) + else: + message = job.error or f"import job {job.id} ended with status {job.status}" + failed: list[dict[str, JsonValue]] = [] + for row in rows: + entry: dict[str, JsonValue] = { + "row_number": row.row_number, "accession": row.accession, "message": message, + } + sample_id = accession_to_sample_id.get(row.accession) + advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" + if sample_id is not None: + entry["sample_id"] = sample_id + advisory += f" (sample {sample_id} was created)" + failed.append(entry) + output.emit_advisory(advisory) + return failed def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index 2e70a4f..a5effa1 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -110,39 +110,6 @@ def validate_row( return errors -def _build_row( - record: dict[str, str], - row_number: int, - base_dir: Path, - metadata_columns: list[str], -) -> SheetRow: - def cell(column: str) -> str | None: - value = (record.get(column) or "").strip() - return value or None - - metadata = { - column: value - for column in metadata_columns - if (value := (record.get(column) or "").strip()) - } - return SheetRow( - row_number=row_number, - name=cell("name") or "", - reads1=_resolve(cell("reads1"), base_dir), - reads2=_resolve(cell("reads2"), base_dir), - project=cell("project"), - organism=cell("organism"), - metadata=metadata, - ) - - -def _resolve(value: str | None, base_dir: Path) -> Path | None: - if value is None: - return None - path = Path(value) - return path if path.is_absolute() else base_dir / path - - def metadata_errors( metadata: dict[str, str], attributes: list[MetadataAttribute], @@ -195,3 +162,36 @@ def metadata_errors( if attribute is not None and not attribute.allow_annotation: errors.append(f"{base} does not allow an annotation") return errors + + +def _build_row( + record: dict[str, str], + row_number: int, + base_dir: Path, + metadata_columns: list[str], +) -> SheetRow: + def cell(column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + metadata = { + column: value + for column in metadata_columns + if (value := (record.get(column) or "").strip()) + } + return SheetRow( + row_number=row_number, + name=cell("name") or "", + reads1=_resolve(cell("reads1"), base_dir), + reads2=_resolve(cell("reads2"), base_dir), + project=cell("project"), + organism=cell("organism"), + metadata=metadata, + ) + + +def _resolve(value: str | None, base_dir: Path) -> Path | None: + if value is None: + return None + path = Path(value) + return path if path.is_absolute() else base_dir / path diff --git a/source/cli.rst b/source/cli.rst index e26fcf1..5d3762f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -395,14 +395,19 @@ By default any invalid row aborts the whole run (exit ``2``); Unlike ``upload-batch``, every valid row is submitted **together as one server-side job** — there is no per-row upload loop. The command polls that job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800) until it leaves ``"RUNNING"``. Once the job completes, -every submitted row is reported as imported; if it fails or times out, every -row is reported as failed with the same message, since the job has one -outcome for the whole batch. +seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. +Once the job completes, every submitted row is reported as imported; if it +fails or times out, every row is reported as failed with the same message +(and the sample id, if the job did partially create one before failing), +since the job has one outcome for the whole batch. Either way, the job's +``job_id`` is included in the output so a timed-out or interrupted run can be +resumed with ``client.samples.get_import(job_id)`` from the library. **Output** — human: each row's outcome on stderr, then a final counts summary on stdout. ``--json``: a single document on stdout with ``imported``, -``failed``, and ``skipped`` lists plus a ``counts`` summary: +``failed``, and ``skipped`` lists, a ``counts`` summary, and the job's +``job_id``/``job_status``/``execution_id`` (``null`` when no job was created, +e.g. every row was invalid or skipped): .. code-block:: json @@ -410,13 +415,15 @@ on stdout. ``--json``: a single document on stdout with ``imported``, "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], - "counts": {"imported": 1, "failed": 0, "skipped": 1} + "counts": {"imported": 1, "failed": 0, "skipped": 1}, + "job_id": 42, "job_status": "COMPLETED", "execution_id": 7 } **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight -validation failure (without ``--skip-invalid``) or a non-CSV sheet; ``1`` the -import job failed or did not finish within ``--timeout``; ``3`` -authentication failure; otherwise the standard mapping above. +validation failure (without ``--skip-invalid``), a non-CSV sheet, or a +non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed +or did not finish within ``--timeout``; ``3`` authentication failure; +otherwise the standard mapping above. **Example** @@ -428,7 +435,7 @@ authentication failure; otherwise the standard mapping above. Imported 2, failed 0, skipped 0. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}} + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7} ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index d78855b..db5726d 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -228,4 +228,4 @@ Models .. autoclass:: flowbio.v2.samples.SampleImportSpec -.. autopydantic_model:: flowbio.v2.samples.SampleImportJob \ No newline at end of file +.. autopydantic_model:: flowbio.v2.samples.SampleImportJob diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d2bf6a..3d4471b 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1008,6 +1008,50 @@ def test_json_document_reports_outcomes_and_counts( assert document["imported"][0] == { "row_number": 1, "accession": "ERR1", "sample_id": 101, } + assert document["job_id"] == 1 + assert document["job_status"] == "COMPLETED" + assert document["execution_id"] == 7 + + @respx.mock + def test_json_document_has_null_job_fields_when_nothing_submitted( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, {"accession": "bogus", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--skip-invalid", "--json", + ) + + assert result.exit_code == 0 + assert route.call_count == 0 + document = json.loads(result.stdout) + assert document["counts"] == {"imported": 0, "failed": 0, "skipped": 1} + assert document["job_id"] is None + assert document["job_status"] is None + assert document["execution_id"] is None + + @respx.mock + def test_header_only_sheet_imports_nothing_without_calling_api( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert route.call_count == 0 + assert "Imported 0, failed 0, skipped 0." in result.stdout @respx.mock def test_sends_sample_type_and_metadata_in_kickoff_payload( @@ -1101,11 +1145,13 @@ def test_failed_job_reports_error_message_per_row( assert "Row 2 (ERR2)" in result.stderr @respx.mock + @patch("time.monotonic") @patch("time.sleep") def test_timeout_reports_failure_without_hanging( - self, _mock_sleep, run_cli, tmp_path: Path, + self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() + mock_monotonic.side_effect = [0.0, 0.5, 1.5, 2.5] respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1125,6 +1171,122 @@ def test_timeout_reports_failure_without_hanging( assert result.exit_code == 1 assert poll_route.call_count == 2 assert "Row 1 (ERR1)" in result.stderr + assert "did not finish within" in result.stderr + assert "still RUNNING" in result.stderr + assert "ended with status RUNNING" not in result.stderr + + @respx.mock + def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "0", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--poll-interval" in result.stderr + + @respx.mock + def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--timeout", "-1", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--timeout" in result.stderr + + @respx.mock + def test_completed_job_missing_sample_id_reports_row_as_failed( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert document["failed"][0]["accession"] == "ERR1" + assert "no sample id" in document["failed"][0]["message"] + + @respx.mock + @patch("time.sleep") + def test_failed_job_reports_partial_sample_ids( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} + assert failed_by_accession["ERR1"]["sample_id"] == 101 + assert "sample_id" not in failed_by_accession["ERR2"] + + @respx.mock + @patch("time.sleep") + def test_transient_poll_error_propagates( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.NOT_FOUND, json={"error": "gone"}), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 4 @respx.mock def test_invalid_row_aborts_and_imports_nothing( From 9688f9f464dbd24361c6100854bc98b685b5f653 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:12:53 +0000 Subject: [PATCH 03/19] fix(samples): address round-2 import-command review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Export SampleImportSpec/SampleImportJob/SampleImportJobId from flowbio.v2, matching every other public sample model; update the samples.rst example to import from the package root instead of the submodule. - Clamp _poll_job's sleep to the time remaining before the deadline, so --poll-interval can no longer make a single sleep overshoot --timeout (e.g. --timeout 10 --poll-interval 300 previously blocked for 300s). - Stop pairing job.accessions/job.sample_ids positionally via zip when their lengths disagree — that silently truncated (reporting a row failed even though the job created its sample) or mis-attributed (handing a row another row's sample id) instead of raising. Both directions are now reported as an explicit, unmatchable failure instead. - Give each failed-row entry a status: "failed" (safe to retry), "running" (timed out but the job may still finish — don't re-run) or "unknown" (accessions/sample_ids couldn't be matched), so an automated caller doesn't treat a timeout the same as a genuine failure. - Move --poll-interval/--timeout validation into an argparse type= callable, so a non-positive value fails at parse time (exit 2) before _dispatch resolves credentials, instead of after an interactive login. - Narrow _ImportResult.job_id/job_status to SampleImportJobId/ SampleImportStatus instead of bare int/str. - Replace the timeout test's hand-counted time.monotonic() side_effect list with itertools.count(...), an infinite fake clock that doesn't need re-counting whenever _poll_job's loop shape changes. Addresses Claude's round-2 review on flowbio#18. Finding 8 (sheet with no accession column degrading confusingly) is left for the same follow-up as round 1's unknown-column-warning deferral, as suggested. Refs FLOW-689. --- flowbio/cli/_samples.py | 100 +++++++++++++++------ flowbio/v2/__init__.py | 6 ++ source/cli.rst | 24 +++-- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 123 ++++++++++++++++++++++++-- tests/unit/v2/test_package_exports.py | 12 +++ tests/unit/v2/test_samples.py | 1 + 7 files changed, 227 insertions(+), 41 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 2bfb371..fe859a5 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -36,7 +36,9 @@ from flowbio.v2.samples import ( MetadataAttribute, SampleImportJob, + SampleImportJobId, SampleImportSpec, + SampleImportStatus, SampleTypeId, ) @@ -259,6 +261,13 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: _DEFAULT_TIMEOUT = 1800.0 +def _positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError(f"must be positive, got {value!r}") + return parsed + + def _configure_import(import_parser: argparse.ArgumentParser) -> None: import_parser.set_defaults(command_parser=import_parser, handler=_import_command) import_parser.add_argument( @@ -282,7 +291,7 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--poll-interval", - type=float, + type=_positive_float, default=_DEFAULT_POLL_INTERVAL, metavar="SECONDS", help=( @@ -292,7 +301,7 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--timeout", - type=float, + type=_positive_float, default=_DEFAULT_TIMEOUT, metavar="SECONDS", help=( @@ -633,17 +642,21 @@ class _ImportResult: Unlike :class:`_BatchResult`, every accession shares one server-side job: all rows land in ``imported`` together on a completed job, or all land in - ``failed`` together (carrying the job's one error message) otherwise. - ``job_id``/``job_status``/``execution_id`` are ``None`` only when no job - was ever created (every row was invalid or skipped), so a timed-out or - interrupted run can still be resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. + ``failed`` together (carrying the job's one error message) otherwise. Each + ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely + failed), ``"running"`` (the job timed out but may still complete), or + ``"unknown"`` (the job's accessions/sample_ids couldn't be matched) — only + ``"failed"`` is safe to blindly retry. ``job_id``/``job_status``/ + ``execution_id`` are ``None`` only when no job was ever created (every row + was invalid or skipped), so a timed-out or interrupted run can still be + resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] failed: list[dict[str, JsonValue]] skipped: list[dict[str, JsonValue]] - job_id: int | None = None - job_status: str | None = None + job_id: SampleImportJobId | None = None + job_status: SampleImportStatus | None = None execution_id: int | None = None @property @@ -689,11 +702,9 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` when the import job completes, :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid`` or a non-positive ``--poll-interval``/``--timeout``, - or :attr:`ExitCode.RUNTIME` if the import job fails or does not - finish within ``--timeout``. + ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job + fails or does not finish within ``--timeout``. """ - _validate_poll_options(args.poll_interval, args.timeout) sheet = parse_accession_sheet(args.sheet) attributes = client.samples.get_metadata_attributes() duplicates = duplicate_accession_errors(sheet.rows) @@ -727,13 +738,6 @@ def _import_command( return result.exit_code -def _validate_poll_options(poll_interval: float, timeout: float) -> None: - if poll_interval <= 0: - raise CliUsageError(f"--poll-interval must be positive, got {poll_interval!r}.") - if timeout <= 0: - raise CliUsageError(f"--timeout must be positive, got {timeout!r}.") - - def _run_import( rows: list[AccessionSheetRow], args: argparse.Namespace, @@ -764,8 +768,11 @@ def _poll_job( client: Client, job: SampleImportJob, poll_interval: float, timeout: float, ) -> SampleImportJob: deadline = time.monotonic() + timeout - while job.status == "RUNNING" and time.monotonic() < deadline: - time.sleep(poll_interval) + while job.status == "RUNNING": + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(poll_interval, remaining)) job = client.samples.get_import(job.id) return job @@ -776,11 +783,15 @@ def _import_result( skipped: list[dict[str, JsonValue]], output: Output, ) -> _ImportResult: - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - if job.status == "COMPLETED": - imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + if _sample_ids_unmatchable(job): + imported: list[dict[str, JsonValue]] = [] + failed = _mismatched_outcome(job, rows, output) else: - imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) + if job.status == "COMPLETED": + imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + else: + imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) return _ImportResult( imported=imported, failed=failed, @@ -791,6 +802,33 @@ def _import_result( ) +def _sample_ids_unmatchable(job: SampleImportJob) -> bool: + # accessions/sample_ids are only guaranteed to correspond positionally when + # their lengths agree (an empty sample_ids on a RUNNING/FAILED job is + # expected, not a mismatch) — anything else means the job's response + # doesn't support the positional pairing the outcome mapping relies on. + return len(job.sample_ids) not in (0, len(job.accessions)) + + +def _mismatched_outcome( + job: SampleImportJob, rows: list[AccessionSheetRow], output: Output, +) -> list[dict[str, JsonValue]]: + message = ( + f"import job {job.id} returned {len(job.sample_ids)} sample id(s) for " + f"{len(job.accessions)} accession(s); cannot match them to rows" + ) + failed: list[dict[str, JsonValue]] = [] + for row in rows: + failed.append({ + "row_number": row.row_number, + "accession": row.accession, + "message": message, + "status": "unknown", + }) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + return failed + + def _completed_outcomes( job: SampleImportJob, rows: list[AccessionSheetRow], @@ -803,7 +841,10 @@ def _completed_outcomes( sample_id = accession_to_sample_id.get(row.accession) if sample_id is None: message = f"import job {job.id} completed but returned no sample id for this accession" - failed.append({"row_number": row.row_number, "accession": row.accession, "message": message}) + failed.append({ + "row_number": row.row_number, "accession": row.accession, + "message": message, "status": "failed", + }) output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") continue imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) @@ -817,17 +858,20 @@ def _failed_outcomes( accession_to_sample_id: dict[str, int], output: Output, ) -> list[dict[str, JsonValue]]: - if job.status == "RUNNING": + timed_out = job.status == "RUNNING" + if timed_out: message = ( f"import job {job.id} did not finish within the configured timeout " f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) else: message = job.error or f"import job {job.id} ended with status {job.status}" + entry_status = "running" if timed_out else "failed" failed: list[dict[str, JsonValue]] = [] for row in rows: entry: dict[str, JsonValue] = { - "row_number": row.row_number, "accession": row.accession, "message": message, + "row_number": row.row_number, "accession": row.accession, + "message": message, "status": entry_status, } sample_id = accession_to_sample_id.get(row.accession) advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" diff --git a/flowbio/v2/__init__.py b/flowbio/v2/__init__.py index 1b239ee..99f2621 100644 --- a/flowbio/v2/__init__.py +++ b/flowbio/v2/__init__.py @@ -45,6 +45,9 @@ Organism, Project, Sample, + SampleImportJob, + SampleImportJobId, + SampleImportSpec, SampleType, SampleTypeId, ) @@ -59,6 +62,9 @@ "Organism", "Project", "Sample", + "SampleImportJob", + "SampleImportJobId", + "SampleImportSpec", "SampleType", "SampleTypeId", "TokenCredentials", diff --git a/source/cli.rst b/source/cli.rst index 5d3762f..a78e301 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -398,10 +398,18 @@ job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. Once the job completes, every submitted row is reported as imported; if it fails or times out, every row is reported as failed with the same message -(and the sample id, if the job did partially create one before failing), -since the job has one outcome for the whole batch. Either way, the job's -``job_id`` is included in the output so a timed-out or interrupted run can be -resumed with ``client.samples.get_import(job_id)`` from the library. +(and the sample id, if the job did create one before failing), since the job +has one outcome for the whole batch. Either way, the job's ``job_id`` is +included in the output so a timed-out or interrupted run can be resumed with +``client.samples.get_import(job_id)`` from the library. + +Each ``failed`` entry carries a ``status``: ``"failed"`` means the job +genuinely failed (or completed without creating that row's sample) — safe to +re-run once fixed. ``"running"`` means ``--timeout`` was reached while the job +was still in progress: it may yet succeed, so check ``job_status``/ +``get_import(job_id)`` rather than re-submitting, which would risk duplicate +samples. ``"unknown"`` means the job's ``accessions``/``sample_ids`` couldn't +be matched to rows at all (an unexpected shape from the server). **Output** — human: each row's outcome on stderr, then a final counts summary on stdout. ``--json``: a single document on stdout with ``imported``, @@ -421,9 +429,11 @@ e.g. every row was invalid or skipped): **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed -or did not finish within ``--timeout``; ``3`` authentication failure; -otherwise the standard mapping above. +non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed, +did not finish within ``--timeout`` (check ``job_status``/each row's +``status`` before re-running — see above), or returned a shape the rows +couldn't be matched against; ``3`` authentication failure; otherwise the +standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index db5726d..eec9091 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -176,7 +176,7 @@ samples directly from public-repository run or experiment accessions (SRR/ERR/DRR or SRX/ERX/DRX), instead of uploading files yourself. Every accession submitted together is tracked as a single job:: - from flowbio.v2.samples import SampleImportSpec + from flowbio.v2 import SampleImportSpec job = client.samples.import_samples([ SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d4471b..97b0144 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1,4 +1,5 @@ import csv +import itertools import json from http import HTTPStatus from pathlib import Path @@ -1151,7 +1152,7 @@ def test_timeout_reports_failure_without_hanging( self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() - mock_monotonic.side_effect = [0.0, 0.5, 1.5, 2.5] + mock_monotonic.side_effect = itertools.count(0.0, 1.0) respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1165,7 +1166,7 @@ def test_timeout_reports_failure_without_hanging( result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "2", + "--poll-interval", "1", "--timeout", "3", ) assert result.exit_code == 1 @@ -1175,6 +1176,62 @@ def test_timeout_reports_failure_without_hanging( assert "still RUNNING" in result.stderr assert "ended with status RUNNING" not in result.stderr + @respx.mock + @patch("time.monotonic") + @patch("time.sleep") + def test_timeout_failed_rows_have_running_status( + self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + mock_monotonic.side_effect = itertools.count(0.0, 1.0) + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--timeout", "3", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["failed"][0]["status"] == "running" + assert document["job_status"] == "RUNNING" + + @respx.mock + @patch("time.monotonic") + @patch("time.sleep") + def test_poll_interval_exceeding_timeout_is_clamped( + self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + mock_monotonic.side_effect = [0.0, 0.0] + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "300", "--timeout", "10", + ) + + assert result.exit_code == 0 + mock_sleep.assert_called_once_with(10.0) + @respx.mock def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: _mock_metadata() @@ -1228,11 +1285,66 @@ def test_completed_job_missing_sample_id_reports_row_as_failed( document = json.loads(result.stdout) assert document["imported"] == [] assert document["failed"][0]["accession"] == "ERR1" + assert document["failed"][0]["status"] == "failed" assert "no sample id" in document["failed"][0]["message"] + @respx.mock + def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "ERR2"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert len(document["failed"]) == 2 + for entry in document["failed"]: + assert entry["status"] == "unknown" + assert "sample_id" not in entry + assert "cannot match" in entry["message"] + + @respx.mock + def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERX1"], [101, 102], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERX1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert document["failed"][0]["status"] == "unknown" + assert "sample_id" not in document["failed"][0] + assert "cannot match" in document["failed"][0]["message"] + @respx.mock @patch("time.sleep") - def test_failed_job_reports_partial_sample_ids( + def test_failed_job_with_all_sample_ids_created_reports_them_per_row( self, _mock_sleep, run_cli, tmp_path: Path, ) -> None: _mock_metadata() @@ -1243,7 +1355,7 @@ def test_failed_job_reports_partial_sample_ids( ) respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + 1, "FAILED", ["ERR1", "ERR2"], [101, 102], error="post-processing failed", )), ) sheet = _write_import_sheet( @@ -1262,7 +1374,8 @@ def test_failed_job_reports_partial_sample_ids( document = json.loads(result.stdout) failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} assert failed_by_accession["ERR1"]["sample_id"] == 101 - assert "sample_id" not in failed_by_accession["ERR2"] + assert failed_by_accession["ERR2"]["sample_id"] == 102 + assert failed_by_accession["ERR1"]["status"] == "failed" @respx.mock @patch("time.sleep") diff --git a/tests/unit/v2/test_package_exports.py b/tests/unit/v2/test_package_exports.py index 2bb0a12..d1bebbd 100644 --- a/tests/unit/v2/test_package_exports.py +++ b/tests/unit/v2/test_package_exports.py @@ -20,6 +20,18 @@ def test_username_password_credentials_is_exported(self) -> None: assert UsernamePasswordCredentials is DirectCredentials + def test_sample_import_types_are_exported(self) -> None: + from flowbio.v2 import SampleImportJob, SampleImportJobId, SampleImportSpec + from flowbio.v2.samples import ( + SampleImportJob as DirectJob, + SampleImportJobId as DirectJobId, + SampleImportSpec as DirectSpec, + ) + + assert SampleImportSpec is DirectSpec + assert SampleImportJob is DirectJob + assert SampleImportJobId is DirectJobId + class TestSubmoduleExports: diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 1f62a46..c254c3e 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1055,6 +1055,7 @@ def test_chunked_multiplexed_upload(self, tmp_path: Path) -> None: assert mux_route.call_count == 3 assert result.data_ids == ["mux_1"] + class TestImportSamples: @respx.mock From d7ed277e82362a9145eabc83d3d356d33d2add1f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:25:03 +0000 Subject: [PATCH 04/19] fix(samples): address round-3 import-command review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stop discarding a job's sample_ids just because they can't be paired positionally: a length mismatch (the normal partial-failure shape, e.g. a FAILED job that created one of two samples before failing) now surfaces those ids verbatim as job_sample_ids in the document instead of silently dropping them, while still refusing to attribute them to specific rows. - Treat a COMPLETED job that returns zero sample ids for its accessions as unmatchable (status "unknown", don't re-run) instead of "failed" (safe to re-run) — a completed job reporting nothing created is exactly as broken an invariant as any other length mismatch, and re-running it risks duplicating whatever it did create. An empty sample_ids on a RUNNING/FAILED job is still the normal, expected shape. - Reject non-finite --poll-interval/--timeout values (nan, inf): nan previously disabled the poll deadline entirely (every comparison against nan is False) and reached time.sleep(nan), an uncaught ValueError. - Export SampleImportStatus from flowbio.v2 alongside the other three new names, so a caller never has to reach into flowbio.v2.samples. - Distinguish "import failed" (genuine failure) from "import did not finish" (timeout) in the human-readable advisory, matching the JSON status field's distinction. - Add a test asserting the sheet's name column reaches the kickoff payload. - Reword source/cli.rst's failed/running/unknown status guidance, document job_sample_ids, and make the human/--json worked examples agree (both import the same two accessions) instead of silently disagreeing. - Replace the last hand-counted time.monotonic() side_effect list with itertools.count(...), matching the round-2 fixture fix. Addresses Claude's round-3 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 45 +++++---- flowbio/v2/__init__.py | 2 + source/cli.rst | 56 ++++++----- tests/unit/cli/test_samples.py | 133 +++++++++++++++++++++++++- tests/unit/v2/test_package_exports.py | 9 +- 5 files changed, 198 insertions(+), 47 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index fe859a5..ddb5974 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,8 +10,9 @@ import argparse import json +import math import time -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -263,8 +264,8 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: def _positive_float(value: str) -> float: parsed = float(value) - if parsed <= 0: - raise argparse.ArgumentTypeError(f"must be positive, got {value!r}") + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError(f"must be a positive, finite number, got {value!r}") return parsed @@ -644,12 +645,16 @@ class _ImportResult: all rows land in ``imported`` together on a completed job, or all land in ``failed`` together (carrying the job's one error message) otherwise. Each ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely - failed), ``"running"`` (the job timed out but may still complete), or - ``"unknown"`` (the job's accessions/sample_ids couldn't be matched) — only - ``"failed"`` is safe to blindly retry. ``job_id``/``job_status``/ - ``execution_id`` are ``None`` only when no job was ever created (every row - was invalid or skipped), so a timed-out or interrupted run can still be - resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. + failed, or completed without creating that row's sample), ``"running"`` + (the job timed out but may still complete), or ``"unknown"`` (the job's + accessions/sample_ids couldn't be matched to rows) — only ``"failed"`` is + safe to blindly retry. ``job_sample_ids`` carries whatever sample ids the + job did return, verbatim and unattributed to any row, so a ``"running"`` + or ``"unknown"`` outcome doesn't hide that the job may already have + created something. ``job_id``/``job_status``/``execution_id`` are + ``None`` only when no job was ever created (every row was invalid or + skipped), so a timed-out or interrupted run can still be resumed with + :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] @@ -658,6 +663,7 @@ class _ImportResult: job_id: SampleImportJobId | None = None job_status: SampleImportStatus | None = None execution_id: int | None = None + job_sample_ids: list[int] = field(default_factory=list) @property def counts(self) -> dict[str, int]: @@ -677,6 +683,7 @@ def document(self) -> dict[str, JsonValue]: "job_id": self.job_id, "job_status": self.job_status, "execution_id": self.execution_id, + "job_sample_ids": self.job_sample_ids, } @property @@ -799,15 +806,19 @@ def _import_result( job_id=job.id, job_status=job.status, execution_id=job.execution_id, + job_sample_ids=list(job.sample_ids), ) def _sample_ids_unmatchable(job: SampleImportJob) -> bool: - # accessions/sample_ids are only guaranteed to correspond positionally when - # their lengths agree (an empty sample_ids on a RUNNING/FAILED job is - # expected, not a mismatch) — anything else means the job's response - # doesn't support the positional pairing the outcome mapping relies on. - return len(job.sample_ids) not in (0, len(job.accessions)) + # An empty sample_ids is the normal, expected shape before anything has + # been created (RUNNING) or when nothing survived to be created (FAILED). + # A COMPLETED job is different: it has told us the batch is done, so + # returning zero sample ids for one or more accessions is exactly as + # broken an invariant as any other length mismatch, not a quiet success. + if not job.sample_ids: + return job.status == "COMPLETED" + return len(job.sample_ids) != len(job.accessions) def _mismatched_outcome( @@ -825,7 +836,7 @@ def _mismatched_outcome( "message": message, "status": "unknown", }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import outcome unknown — {message}") return failed @@ -864,8 +875,10 @@ def _failed_outcomes( f"import job {job.id} did not finish within the configured timeout " f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) + verb = "did not finish" else: message = job.error or f"import job {job.id} ended with status {job.status}" + verb = "failed" entry_status = "running" if timed_out else "failed" failed: list[dict[str, JsonValue]] = [] for row in rows: @@ -874,7 +887,7 @@ def _failed_outcomes( "message": message, "status": entry_status, } sample_id = accession_to_sample_id.get(row.accession) - advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" + advisory = f"Row {row.row_number} ({row.accession}): import {verb} — {message}" if sample_id is not None: entry["sample_id"] = sample_id advisory += f" (sample {sample_id} was created)" diff --git a/flowbio/v2/__init__.py b/flowbio/v2/__init__.py index 99f2621..d0d5f1f 100644 --- a/flowbio/v2/__init__.py +++ b/flowbio/v2/__init__.py @@ -48,6 +48,7 @@ SampleImportJob, SampleImportJobId, SampleImportSpec, + SampleImportStatus, SampleType, SampleTypeId, ) @@ -65,6 +66,7 @@ "SampleImportJob", "SampleImportJobId", "SampleImportSpec", + "SampleImportStatus", "SampleType", "SampleTypeId", "TokenCredentials", diff --git a/source/cli.rst b/source/cli.rst index a78e301..1fee17e 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -395,43 +395,49 @@ By default any invalid row aborts the whole run (exit ``2``); Unlike ``upload-batch``, every valid row is submitted **together as one server-side job** — there is no per-row upload loop. The command polls that job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. -Once the job completes, every submitted row is reported as imported; if it -fails or times out, every row is reported as failed with the same message -(and the sample id, if the job did create one before failing), since the job -has one outcome for the whole batch. Either way, the job's ``job_id`` is -included in the output so a timed-out or interrupted run can be resumed with -``client.samples.get_import(job_id)`` from the library. +seconds, default 1800; both must be positive, finite numbers) until it leaves +``"RUNNING"``. Once the job completes, every submitted row is reported as +imported; if it fails or times out, every row is reported as failed, since +the job has one outcome for the whole batch. Either way, the job's ``job_id`` +is included in the output so a timed-out or interrupted run can be resumed +with ``client.samples.get_import(job_id)`` from the library. Each ``failed`` entry carries a ``status``: ``"failed"`` means the job -genuinely failed (or completed without creating that row's sample) — safe to -re-run once fixed. ``"running"`` means ``--timeout`` was reached while the job -was still in progress: it may yet succeed, so check ``job_status``/ -``get_import(job_id)`` rather than re-submitting, which would risk duplicate -samples. ``"unknown"`` means the job's ``accessions``/``sample_ids`` couldn't -be matched to rows at all (an unexpected shape from the server). - -**Output** — human: each row's outcome on stderr, then a final counts summary -on stdout. ``--json``: a single document on stdout with ``imported``, -``failed``, and ``skipped`` lists, a ``counts`` summary, and the job's -``job_id``/``job_status``/``execution_id`` (``null`` when no job was created, -e.g. every row was invalid or skipped): +genuinely failed, or completed but didn't create that specific row's sample — +safe to re-run once fixed. ``"running"`` means ``--timeout`` was reached +while the job was still in progress: it may yet succeed, so check +``job_status``/``get_import(job_id)`` rather than re-submitting, which would +risk duplicate samples. ``"unknown"`` means the job's ``accessions``/ +``sample_ids`` couldn't be matched to rows at all (e.g. a completed job that +returned no sample ids, or fewer/more than the accessions submitted) — an +unexpected shape from the server, so treat it like ``"running"`` and don't +blindly re-run. Whatever sample ids the job did return are always available, +unattributed, as ``job_sample_ids`` — useful to check a ``"running"``/ +``"unknown"`` outcome didn't already create something. + +**Output** — human: each row's outcome on stderr (``import failed`` for a +genuine failure, ``import did not finish`` for a timeout, ``import outcome +unknown`` for an unmatchable job), then a final counts summary on stdout. +``--json``: a single document on stdout with ``imported``, ``failed``, and +``skipped`` lists, a ``counts`` summary, and the job's ``job_id``/ +``job_status``/``execution_id``/``job_sample_ids`` (``null``/``[]`` when no +job was created, e.g. every row was invalid or skipped): .. code-block:: json { "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], - "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], + "skipped": [{"row_number": 2, "accession": "BOGUS", "reasons": ["..."]}], "counts": {"imported": 1, "failed": 0, "skipped": 1}, - "job_id": 42, "job_status": "COMPLETED", "execution_id": 7 + "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101] } **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed, -did not finish within ``--timeout`` (check ``job_status``/each row's -``status`` before re-running — see above), or returned a shape the rows +non-positive/non-finite ``--poll-interval``/``--timeout``; ``1`` the import +job failed, did not finish within ``--timeout`` (check ``job_status``/each +row's ``status`` before re-running — see above), or returned a shape the rows couldn't be matched against; ``3`` authentication failure; otherwise the standard mapping above. @@ -445,7 +451,7 @@ standard mapping above. Imported 2, failed 0, skipped 0. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7} + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}, {"row_number": 2, "accession": "ERR10677146", "sample_id": 102}], "failed": [], "skipped": [], "counts": {"imported": 2, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101, 102]} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 97b0144..e2eebf3 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1083,6 +1083,26 @@ def test_sends_sample_type_and_metadata_in_kickoff_payload( }], } + @respx.mock + def test_sends_name_in_kickoff_payload(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, {"accession": "ERR1", "name": "liver_r1", "cell_type": "Neuron"}, + ) + + run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + payload = json.loads(route.calls[0].request.content) + assert payload["imports"][0]["name"] == "liver_r1" + @respx.mock @patch("time.sleep") def test_polls_until_job_completes( @@ -1212,7 +1232,7 @@ def test_poll_interval_exceeding_timeout_is_clamped( self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() - mock_monotonic.side_effect = [0.0, 0.0] + mock_monotonic.side_effect = itertools.count(0.0, 0.5) respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1230,7 +1250,9 @@ def test_poll_interval_exceeding_timeout_is_clamped( ) assert result.exit_code == 0 - mock_sleep.assert_called_once_with(10.0) + mock_sleep.assert_called_once() + slept_for = mock_sleep.call_args[0][0] + assert 0 < slept_for < 300 @respx.mock def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1265,7 +1287,39 @@ def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "--timeout" in result.stderr @respx.mock - def test_completed_job_missing_sample_id_reports_row_as_failed( + def test_nan_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "nan", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--poll-interval" in result.stderr + + @respx.mock + def test_infinite_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--timeout", "inf", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--timeout" in result.stderr + + @respx.mock + def test_completed_job_with_no_sample_ids_is_reported_as_unmatchable( self, run_cli, tmp_path: Path, ) -> None: _mock_metadata() @@ -1285,8 +1339,37 @@ def test_completed_job_missing_sample_id_reports_row_as_failed( document = json.loads(result.stdout) assert document["imported"] == [] assert document["failed"][0]["accession"] == "ERR1" - assert document["failed"][0]["status"] == "failed" - assert "no sample id" in document["failed"][0]["message"] + assert document["failed"][0]["status"] == "unknown" + assert "cannot match" in document["failed"][0]["message"] + assert document["job_sample_ids"] == [] + + @respx.mock + def test_completed_job_with_matching_sample_id_missing_reports_row_as_failed( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "err-other"], [101, 102], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} + assert document["imported"][0]["accession"] == "ERR1" + assert failed_by_accession["ERR2"]["status"] == "failed" + assert "no sample id" in failed_by_accession["ERR2"]["message"] @respx.mock def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( @@ -1317,6 +1400,7 @@ def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( assert entry["status"] == "unknown" assert "sample_id" not in entry assert "cannot match" in entry["message"] + assert document["job_sample_ids"] == [101] @respx.mock def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( @@ -1341,6 +1425,45 @@ def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( assert document["failed"][0]["status"] == "unknown" assert "sample_id" not in document["failed"][0] assert "cannot match" in document["failed"][0]["message"] + assert document["job_sample_ids"] == [101, 102] + + @respx.mock + @patch("time.sleep") + def test_failed_job_with_partial_sample_ids_still_surfaces_them( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + for entry in document["failed"]: + assert entry["status"] == "unknown" + assert "sample_id" not in entry + # The job did create one sample before failing; that isn't attributable + # to a specific row, but it must not be silently dropped. + assert document["job_sample_ids"] == [101] @respx.mock @patch("time.sleep") diff --git a/tests/unit/v2/test_package_exports.py b/tests/unit/v2/test_package_exports.py index d1bebbd..06bff5a 100644 --- a/tests/unit/v2/test_package_exports.py +++ b/tests/unit/v2/test_package_exports.py @@ -21,16 +21,23 @@ def test_username_password_credentials_is_exported(self) -> None: assert UsernamePasswordCredentials is DirectCredentials def test_sample_import_types_are_exported(self) -> None: - from flowbio.v2 import SampleImportJob, SampleImportJobId, SampleImportSpec + from flowbio.v2 import ( + SampleImportJob, + SampleImportJobId, + SampleImportSpec, + SampleImportStatus, + ) from flowbio.v2.samples import ( SampleImportJob as DirectJob, SampleImportJobId as DirectJobId, SampleImportSpec as DirectSpec, + SampleImportStatus as DirectStatus, ) assert SampleImportSpec is DirectSpec assert SampleImportJob is DirectJob assert SampleImportJobId is DirectJobId + assert SampleImportStatus is DirectStatus class TestSubmoduleExports: From 98d130614ca2a18da990cb155d1ed31723693002 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:26:31 +0000 Subject: [PATCH 05/19] refactor(samples): make 'import' non-blocking and let the API validate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction on the PR: - Drop all client-side pre-validation of accession sheets (accession format, duplicates, required/closed-option metadata). The API already validates every one of these server-side; duplicating the rules client-side was a second, driftable copy of the same logic for no benefit now that the command doesn't need a local reject/skip decision. A malformed sheet surfaces as a normal FlowApiError instead of a local pre-flight rejection — 'samples import' makes exactly one API call. - Make 'samples import' non-blocking: it submits the whole sheet as one batch, reports the created job's id/status, and returns immediately instead of polling to completion. Add a new 'samples import-status --job-id ID' verb (a thin, one-shot wrapper over client.samples.get_import) so a user (or script) can check on a job whenever they choose; there is no built-in polling. This removes _poll_job, the failed/running/unknown outcome taxonomy, job_sample_ids, the positional accessions/sample_ids matching and its mismatch handling, --skip-invalid/--poll-interval/--timeout, and the accession-format regex and duplicate-detection helpers — all of which existed to support blocking-with-local-validation behaviour that no longer applies. flowbio/v2/samples.py (import_samples/get_import) is unchanged; only the CLI orchestration on top of it is simplified. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 77 +-- flowbio/cli/_samples.py | 361 +++----------- source/cli.rst | 123 +++-- tests/unit/cli/test_accession_sheet.py | 128 +---- tests/unit/cli/test_samples.py | 622 +++++-------------------- 5 files changed, 251 insertions(+), 1060 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8446923..b55659c 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,4 +1,4 @@ -"""CSV accession-sheet parsing and pre-flight validation for ``samples import``. +"""CSV accession-sheet parsing for ``samples import``. An accession sheet is a CSV with one row per accession to import: an ``accession`` column (a public-repository run or experiment accession) plus @@ -7,35 +7,27 @@ differ — there is nothing to upload (no ``reads1``/``reads2``) and the import API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, ``organism`` instead. + +Rows are not validated here: the accession format, duplicates, sample type, +organism, and metadata rules are all checked server-side when the sheet is +submitted — duplicating that locally would just be a second, driftable copy +of the same rules. """ from __future__ import annotations import csv -import re from dataclasses import dataclass from pathlib import Path from flowbio.cli._exit_codes import CliUsageError from flowbio.cli._files import existing_file -from flowbio.cli._sheet import metadata_errors -from flowbio.v2.samples import MetadataAttribute, SampleTypeId RESERVED_COLUMNS = ("accession", "name", "organism") -# Mirrors the API's own accession format rule (one run or experiment accession -# per entry), so a malformed accession is reported up front like every other -# validation problem instead of failing the whole batch request server-side. -_SUPPORTED_ACCESSION = re.compile(r"^[SED]R[RX]\d+$") - @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet. - - ``accession`` may be empty (empty cell) — that is reported by - :func:`validate_accession_row` rather than rejected here, so all errors - surface together. - """ + """One data row of an accession sheet.""" row_number: int accession: str @@ -81,61 +73,6 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) -def validate_accession_row( - row: AccessionSheetRow, - attributes: list[MetadataAttribute], - sample_type: SampleTypeId, -) -> list[str]: - """Return every validation problem on ``row`` (empty when the row is valid). - - :param row: The parsed row to validate. - :param attributes: The server's metadata attributes, deciding required and - closed-option columns. - :param sample_type: The sample type applied to the whole import; an - attribute required for it must be present. - :returns: One human-readable message per problem, collected so the caller - can report them all at once. - """ - errors: list[str] = [] - if not row.accession: - errors.append("missing required value: accession") - elif not _SUPPORTED_ACCESSION.match(row.accession): - errors.append( - f"'{row.accession}' is not a supported accession — import one run " - f"(SRR/ERR/DRR) or experiment (SRX/ERX/DRX) accession per row", - ) - errors.extend(metadata_errors(row.metadata, attributes, sample_type)) - return errors - - -def duplicate_accession_errors(rows: list[AccessionSheetRow]) -> dict[int, list[str]]: - """Return extra errors for rows whose accession repeats an earlier row. - - Mirrors the API's own duplicate-accession rejection so a sheet with - repeats is reported up front, in the same per-row shape as every other - validation problem, rather than surfacing as one opaque batch failure. - - :param rows: Every row in the sheet, including ones already found invalid. - :returns: A mapping of ``row_number`` to the duplicate-accession messages - for rows after the first occurrence of a repeated accession. Rows with - a blank accession (already reported by :func:`validate_accession_row`) - are never flagged as duplicates of each other. - """ - first_seen: dict[str, int] = {} - errors: dict[int, list[str]] = {} - for row in rows: - if not row.accession: - continue - if row.accession in first_seen: - errors.setdefault(row.row_number, []).append( - f"duplicate accession '{row.accession}' " - f"(first seen at row {first_seen[row.accession]})", - ) - else: - first_seen[row.accession] = row.row_number - return errors - - def _build_row( record: dict[str, str], row_number: int, metadata_columns: list[str], ) -> AccessionSheetRow: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ddb5974..c13ae01 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,18 +10,11 @@ import argparse import json -import math -import time -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import ( - AccessionSheetRow, - duplicate_accession_errors, - parse_accession_sheet, - validate_accession_row, -) +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -39,7 +32,6 @@ SampleImportJob, SampleImportJobId, SampleImportSpec, - SampleImportStatus, SampleTypeId, ) @@ -96,11 +88,17 @@ def register( parents=[global_parent], help="Import samples from public-repository accessions.", description=( - "Validate every row of a CSV accession sheet up front, kick off a " - "single import job for the valid rows, poll it to completion, and " - "report each accession's outcome." + "Kick off a batch import job from a CSV accession sheet and report " + "its id. Does not wait for the job to finish — check its progress " + "with 'samples import-status'." ), )) + _configure_import_status(verbs.add_parser( + "import-status", + parents=[global_parent], + help="Check the status of an import job.", + description="Fetch and report the current state of a 'samples import' job.", + )) def _configure_upload(upload: argparse.ArgumentParser) -> None: @@ -258,17 +256,6 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: ) -_DEFAULT_POLL_INTERVAL = 5.0 -_DEFAULT_TIMEOUT = 1800.0 - - -def _positive_float(value: str) -> float: - parsed = float(value) - if not math.isfinite(parsed) or parsed <= 0: - raise argparse.ArgumentTypeError(f"must be a positive, finite number, got {value!r}") - return parsed - - def _configure_import(import_parser: argparse.ArgumentParser) -> None: import_parser.set_defaults(command_parser=import_parser, handler=_import_command) import_parser.add_argument( @@ -285,30 +272,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: type=SampleTypeId, help="Sample type applied to every accession (sent as-is; validated server-side).", ) - import_parser.add_argument( - "--skip-invalid", - action="store_true", - help="Skip invalid rows (reporting why) instead of aborting the import.", - ) - import_parser.add_argument( - "--poll-interval", - type=_positive_float, - default=_DEFAULT_POLL_INTERVAL, - metavar="SECONDS", - help=( - "Seconds to wait between checks of the import job's status; must be " - f"positive (default: {_DEFAULT_POLL_INTERVAL:g})." - ), - ) - import_parser.add_argument( - "--timeout", - type=_positive_float, - default=_DEFAULT_TIMEOUT, - metavar="SECONDS", - help=( - "Maximum seconds to wait for the import job to finish; must be " - f"positive (default: {_DEFAULT_TIMEOUT:g})." - ), + + +def _job_id(value: str) -> SampleImportJobId: + return SampleImportJobId(int(value)) + + +def _configure_import_status(import_status: argparse.ArgumentParser) -> None: + import_status.set_defaults(command_parser=import_status, handler=_import_status_command) + import_status.add_argument( + "--job-id", + required=True, + metavar="ID", + type=_job_id, + help="Import job id, as reported by 'samples import'.", ) @@ -637,124 +614,23 @@ def _invalid_line(row: SheetRow, reasons: list[str]) -> str: return f"Row {row.row_number} ({row.name}): {'; '.join(reasons)}" -@dataclass(frozen=True) -class _ImportResult: - """The outcome of an ``import`` run, rendered to text or JSON. - - Unlike :class:`_BatchResult`, every accession shares one server-side job: - all rows land in ``imported`` together on a completed job, or all land in - ``failed`` together (carrying the job's one error message) otherwise. Each - ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely - failed, or completed without creating that row's sample), ``"running"`` - (the job timed out but may still complete), or ``"unknown"`` (the job's - accessions/sample_ids couldn't be matched to rows) — only ``"failed"`` is - safe to blindly retry. ``job_sample_ids`` carries whatever sample ids the - job did return, verbatim and unattributed to any row, so a ``"running"`` - or ``"unknown"`` outcome doesn't hide that the job may already have - created something. ``job_id``/``job_status``/``execution_id`` are - ``None`` only when no job was ever created (every row was invalid or - skipped), so a timed-out or interrupted run can still be resumed with - :meth:`~flowbio.v2.samples.SampleResource.get_import`. - """ - - imported: list[dict[str, JsonValue]] - failed: list[dict[str, JsonValue]] - skipped: list[dict[str, JsonValue]] - job_id: SampleImportJobId | None = None - job_status: SampleImportStatus | None = None - execution_id: int | None = None - job_sample_ids: list[int] = field(default_factory=list) - - @property - def counts(self) -> dict[str, int]: - return { - "imported": len(self.imported), - "failed": len(self.failed), - "skipped": len(self.skipped), - } - - @property - def document(self) -> dict[str, JsonValue]: - return { - "imported": self.imported, - "failed": self.failed, - "skipped": self.skipped, - "counts": self.counts, - "job_id": self.job_id, - "job_status": self.job_status, - "execution_id": self.execution_id, - "job_sample_ids": self.job_sample_ids, - } - - @property - def summary(self) -> str: - counts = self.counts - return ( - f"Imported {counts['imported']}, failed {counts['failed']}, " - f"skipped {counts['skipped']}." - ) - - @property - def exit_code(self) -> ExitCode: - return ExitCode.RUNTIME if self.failed else ExitCode.SUCCESS - - def _import_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: - """Validate an accession sheet up front, then run one import job for it. + """Kick off a batch import job from an accession sheet and report its id. + + Every row is submitted as-is: the accession format, duplicates, sample + type, organism, and metadata rules are all validated server-side, so a + malformed sheet surfaces as a normal :class:`FlowApiError` rather than a + local pre-flight rejection. This command does not wait for the job to + finish — poll it yourself with ``samples import-status``. :param args: Parsed command-line arguments. :param client: The authenticated Flow client. :param output: The result/error renderer. - :returns: :attr:`ExitCode.SUCCESS` when the import job completes, - :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job - fails or does not finish within ``--timeout``. + :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. """ sheet = parse_accession_sheet(args.sheet) - attributes = client.samples.get_metadata_attributes() - duplicates = duplicate_accession_errors(sheet.rows) - classified = [ - ( - row, - validate_accession_row(row, attributes, args.sample_type) - + duplicates.get(row.row_number, []), - ) - for row in sheet.rows - ] - invalid = [(row, reasons) for row, reasons in classified if reasons] - valid = [row for row, reasons in classified if not reasons] - - if invalid and not args.skip_invalid: - output.emit_error( - "Accession sheet has invalid rows; nothing was imported.", - details=[_invalid_accession_line(row, reasons) for row, reasons in invalid], - ) - return ExitCode.USAGE - - for row, reasons in invalid: - output.emit_advisory(f"Skipped {_invalid_accession_line(row, reasons)}") - skipped = [ - {"row_number": row.row_number, "accession": row.accession, "reasons": reasons} - for row, reasons in invalid - ] - - result = _run_import(valid, args, client, output, skipped) - output.emit_result(result.summary, result.document) - return result.exit_code - - -def _run_import( - rows: list[AccessionSheetRow], - args: argparse.Namespace, - client: Client, - output: Output, - skipped: list[dict[str, JsonValue]], -) -> _ImportResult: - if not rows: - return _ImportResult(imported=[], failed=[], skipped=skipped) - specs = [ SampleImportSpec( accession=row.accession, @@ -763,141 +639,56 @@ def _run_import( organism_id=row.organism, metadata=row.metadata or None, ) - for row in rows + for row in sheet.rows ] job = client.samples.import_samples(specs) - output.emit_advisory(f"Import job {job.id} started for {len(rows)} accession(s); polling...") - job = _poll_job(client, job, args.poll_interval, args.timeout) - return _import_result(job, rows, skipped, output) - - -def _poll_job( - client: Client, job: SampleImportJob, poll_interval: float, timeout: float, -) -> SampleImportJob: - deadline = time.monotonic() + timeout - while job.status == "RUNNING": - remaining = deadline - time.monotonic() - if remaining <= 0: - break - time.sleep(min(poll_interval, remaining)) - job = client.samples.get_import(job.id) - return job - - -def _import_result( - job: SampleImportJob, - rows: list[AccessionSheetRow], - skipped: list[dict[str, JsonValue]], - output: Output, -) -> _ImportResult: - if _sample_ids_unmatchable(job): - imported: list[dict[str, JsonValue]] = [] - failed = _mismatched_outcome(job, rows, output) - else: - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - if job.status == "COMPLETED": - imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) - else: - imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) - return _ImportResult( - imported=imported, - failed=failed, - skipped=skipped, - job_id=job.id, - job_status=job.status, - execution_id=job.execution_id, - job_sample_ids=list(job.sample_ids), - ) - - -def _sample_ids_unmatchable(job: SampleImportJob) -> bool: - # An empty sample_ids is the normal, expected shape before anything has - # been created (RUNNING) or when nothing survived to be created (FAILED). - # A COMPLETED job is different: it has told us the batch is done, so - # returning zero sample ids for one or more accessions is exactly as - # broken an invariant as any other length mismatch, not a quiet success. - if not job.sample_ids: - return job.status == "COMPLETED" - return len(job.sample_ids) != len(job.accessions) - - -def _mismatched_outcome( - job: SampleImportJob, rows: list[AccessionSheetRow], output: Output, -) -> list[dict[str, JsonValue]]: - message = ( - f"import job {job.id} returned {len(job.sample_ids)} sample id(s) for " - f"{len(job.accessions)} accession(s); cannot match them to rows" + output.emit_result( + f"Started import job {job.id} for {len(job.accessions)} accession(s) " + f"(status: {job.status}). Check progress with " + f"'flowbio samples import-status --job-id {job.id}'.", + _job_document(job), ) - failed: list[dict[str, JsonValue]] = [] - for row in rows: - failed.append({ - "row_number": row.row_number, - "accession": row.accession, - "message": message, - "status": "unknown", - }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import outcome unknown — {message}") - return failed + return ExitCode.SUCCESS -def _completed_outcomes( - job: SampleImportJob, - rows: list[AccessionSheetRow], - accession_to_sample_id: dict[str, int], - output: Output, -) -> tuple[list[dict[str, JsonValue]], list[dict[str, JsonValue]]]: - imported: list[dict[str, JsonValue]] = [] - failed: list[dict[str, JsonValue]] = [] - for row in rows: - sample_id = accession_to_sample_id.get(row.accession) - if sample_id is None: - message = f"import job {job.id} completed but returned no sample id for this accession" - failed.append({ - "row_number": row.row_number, "accession": row.accession, - "message": message, "status": "failed", - }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") - continue - imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): imported sample {sample_id}") - return imported, failed +def _import_status_command( + args: argparse.Namespace, client: Client, output: Output, +) -> ExitCode: + """Fetch and report the current state of an import job. + + :param args: Parsed command-line arguments. + :param client: The authenticated Flow client. + :param output: The result/error renderer. + :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched + (the job's own ``status`` — not this command's exit code — reflects + whether the import itself succeeded). + """ + job = client.samples.get_import(args.job_id) + output.emit_result(_job_summary(job), _job_document(job)) + return ExitCode.SUCCESS + + +def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: + return { + "id": job.id, + "status": job.status, + "accessions": job.accessions, + "sample_ids": job.sample_ids, + "execution_id": job.execution_id, + "error": job.error, + } + + +def _job_summary(job: SampleImportJob) -> str: + if job.status == "COMPLETED": + ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) + return f"Job {job.id}: COMPLETED. Sample ids: {ids}." + if job.status == "FAILED": + detail = f" {job.error}" if job.error else "" + return f"Job {job.id}: FAILED.{detail}" + return f"Job {job.id}: {job.status}." -def _failed_outcomes( - job: SampleImportJob, - rows: list[AccessionSheetRow], - accession_to_sample_id: dict[str, int], - output: Output, -) -> list[dict[str, JsonValue]]: - timed_out = job.status == "RUNNING" - if timed_out: - message = ( - f"import job {job.id} did not finish within the configured timeout " - f"(still RUNNING) — check it later with client.samples.get_import({job.id})" - ) - verb = "did not finish" - else: - message = job.error or f"import job {job.id} ended with status {job.status}" - verb = "failed" - entry_status = "running" if timed_out else "failed" - failed: list[dict[str, JsonValue]] = [] - for row in rows: - entry: dict[str, JsonValue] = { - "row_number": row.row_number, "accession": row.accession, - "message": message, "status": entry_status, - } - sample_id = accession_to_sample_id.get(row.accession) - advisory = f"Row {row.row_number} ({row.accession}): import {verb} — {message}" - if sample_id is not None: - entry["sample_id"] = sample_id - advisory += f" (sample {sample_id} was created)" - failed.append(entry) - output.emit_advisory(advisory) - return failed - - -def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: - return f"Row {row.row_number} ({row.accession or ''}): {'; '.join(reasons)}" def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index 1fee17e..32cc406 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -370,88 +370,83 @@ authentication failure; otherwise the standard mapping above. ``samples import`` ~~~~~~~~~~~~~~~~~~~ -Import samples from public-repository accessions (SRR/ERR/DRR run or -SRX/ERX/DRX experiment accessions), applying one sample type to every row — -no files to upload yourself. +Kick off a batch import of samples from public-repository accessions (SRR/ +ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type +to every row — no files to upload yourself. :: flowbio samples import --sheet PATH --sample-type TYPE - [--skip-invalid] [--poll-interval SECONDS] [--timeout SECONDS] Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with an ``accession`` column plus optional ``name``/``organism`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when -omitted. The sample type is sent as-is and validated server-side. - -**Validation is up front**, mirroring ``upload-batch``: a missing or -malformed accession, a duplicate accession within the sheet, a value outside -a closed-option attribute's allowed values, or missing metadata required for -the chosen type — every problem is collected before anything is submitted. -By default any invalid row aborts the whole run (exit ``2``); -``--skip-invalid`` skips them (reporting why) and imports the rest. - -Unlike ``upload-batch``, every valid row is submitted **together as one -server-side job** — there is no per-row upload loop. The command polls that -job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800; both must be positive, finite numbers) until it leaves -``"RUNNING"``. Once the job completes, every submitted row is reported as -imported; if it fails or times out, every row is reported as failed, since -the job has one outcome for the whole batch. Either way, the job's ``job_id`` -is included in the output so a timed-out or interrupted run can be resumed -with ``client.samples.get_import(job_id)`` from the library. - -Each ``failed`` entry carries a ``status``: ``"failed"`` means the job -genuinely failed, or completed but didn't create that specific row's sample — -safe to re-run once fixed. ``"running"`` means ``--timeout`` was reached -while the job was still in progress: it may yet succeed, so check -``job_status``/``get_import(job_id)`` rather than re-submitting, which would -risk duplicate samples. ``"unknown"`` means the job's ``accessions``/ -``sample_ids`` couldn't be matched to rows at all (e.g. a completed job that -returned no sample ids, or fewer/more than the accessions submitted) — an -unexpected shape from the server, so treat it like ``"running"`` and don't -blindly re-run. Whatever sample ids the job did return are always available, -unattributed, as ``job_sample_ids`` — useful to check a ``"running"``/ -``"unknown"`` outcome didn't already create something. - -**Output** — human: each row's outcome on stderr (``import failed`` for a -genuine failure, ``import did not finish`` for a timeout, ``import outcome -unknown`` for an unmatchable job), then a final counts summary on stdout. -``--json``: a single document on stdout with ``imported``, ``failed``, and -``skipped`` lists, a ``counts`` summary, and the job's ``job_id``/ -``job_status``/``execution_id``/``job_sample_ids`` (``null``/``[]`` when no -job was created, e.g. every row was invalid or skipped): +omitted. The sample type, accession format, duplicates, and metadata rules +are all sent as-is and validated **server-side** — this command does not +pre-validate rows itself; an invalid sheet surfaces as a normal API error +(exit ``5`` or ``1``), not a local rejection. + +Every row is submitted **together as one server-side job**. This command +does **not wait for it to finish** — it reports the job's id and initial +status (almost always ``"RUNNING"``) and returns immediately. Check on it +with ``samples import-status --job-id ID``; polling (if you want it) is up +to you, e.g. in a shell loop. + +**Output** — human: a confirmation line with the job id and a pointer to +``import-status``. ``--json``: the created job as a single document — +``id``, ``status``, ``accessions``, ``sample_ids`` (empty until the job +completes), ``execution_id``, ``error``. + +**Exit codes** — ``0`` the job was created (regardless of its eventual +outcome — check that with ``import-status``); ``2`` a non-CSV sheet; ``1`` +the API rejected the batch (e.g. unknown sample type, missing required +metadata, an unsupported accession format — the server returns these as a +``422``, which isn't one of the exit codes with its own mapping below); ``3`` +authentication failure; otherwise the standard mapping above. -.. code-block:: json +**Example** - { - "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], - "failed": [], - "skipped": [{"row_number": 2, "accession": "BOGUS", "reasons": ["..."]}], - "counts": {"imported": 1, "failed": 0, "skipped": 1}, - "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101] - } +.. code-block:: bash -**Exit codes** — ``0`` the import job completed; ``2`` a pre-flight -validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive/non-finite ``--poll-interval``/``--timeout``; ``1`` the import -job failed, did not finish within ``--timeout`` (check ``job_status``/each -row's ``status`` before re-running — see above), or returned a shape the rows -couldn't be matched against; ``3`` authentication failure; otherwise the -standard mapping above. + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + +``samples import-status`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Fetch and report the current state of a ``samples import`` job. + +:: + + flowbio samples import-status --job-id ID + +Read-only — checking a job's status never changes it. There is no built-in +polling; run this again (or wrap it in your own loop, e.g. ``watch``) until +``status`` leaves ``"RUNNING"``. + +**Output** — human: a one-line summary including the sample ids on +``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single +document — ``id``, ``status``, ``accessions``, ``sample_ids``, +``execution_id``, ``error``. + +**Exit codes** — ``0`` the job's state was fetched (the job's own ``status``, +not this command's exit code, reflects whether the import itself succeeded +or failed); ``4`` no job with that id exists; ``3`` authentication failure; +otherwise the standard mapping above. **Example** .. code-block:: bash - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq - Row 1 (ERR1160845): imported sample 101 - Row 2 (ERR10677146): imported sample 102 - Imported 2, failed 0, skipped 0. + $ flowbio samples import-status --job-id 42 + Job 42: COMPLETED. Sample ids: 101, 102. - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}, {"row_number": 2, "accession": "ERR10677146", "sample_id": 102}], "failed": [], "skipped": [], "counts": {"imported": 2, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101, 102]} + $ flowbio samples import-status --job-id 42 --json + {"id": 42, "status": "COMPLETED", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index f7edca8..57d0a81 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -3,41 +3,12 @@ import pytest -from flowbio.cli._accession_sheet import ( - duplicate_accession_errors, - parse_accession_sheet, - validate_accession_row, -) +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError -from flowbio.v2.samples import MetadataAttribute, SampleTypeId -SAMPLE_TYPE = SampleTypeId("rna_seq") HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] -def _attributes() -> list[MetadataAttribute]: - return [ - MetadataAttribute( - identifier="cell_type", - name="Cell Type", - description="The cell type", - required=False, - required_for_sample_types=[SAMPLE_TYPE], - options=["Neuron", "Fibroblast"], - allow_annotation=False, - ), - MetadataAttribute( - identifier="source", - name="Source", - description="Sample source", - required=False, - required_for_sample_types=[], - options=None, - allow_annotation=True, - ), - ] - - def _write_sheet( directory: Path, *records: dict[str, str], headers: list[str] = HEADERS, ) -> Path: @@ -128,100 +99,3 @@ def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError): parse_accession_sheet(tmp_path / "absent.csv") - - -class TestValidateAccessionRow: - - def _row(self, directory: Path, **overrides: str): - record = {"accession": "ERR1160845"} - record.update(overrides) - sheet = parse_accession_sheet(_write_sheet(directory, record)) - return sheet.rows[0] - - def test_valid_row_has_no_errors(self, tmp_path: Path) -> None: - row = self._row(tmp_path, cell_type="Neuron") - - assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] - - def test_missing_accession_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, accession="", cell_type="Neuron") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("accession" in error for error in errors) - - def test_unsupported_accession_format_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, accession="GSE12345", cell_type="Neuron") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("GSE12345" in error for error in errors) - - def test_run_and_experiment_accessions_are_supported(self, tmp_path: Path) -> None: - for accession in ("ERR1160845", "SRR1234567", "DRR7654321", "SRX111", "ERX222", "DRX333"): - row = self._row(tmp_path, accession=accession, cell_type="Neuron") - - assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] - - def test_value_outside_options_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, cell_type="Alien") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("Alien" in error for error in errors) - - def test_required_for_type_metadata_missing_reports_error( - self, tmp_path: Path, - ) -> None: - row = self._row(tmp_path) - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("cell_type" in error for error in errors) - - def test_annotation_set_without_its_value_reports_error( - self, tmp_path: Path, - ) -> None: - row = self._row( - tmp_path, cell_type="Neuron", **{"source__annotation": "qPCR"}, - ) - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("source__annotation" in error for error in errors) - - def test_all_per_row_errors_collected(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "", "cell_type": "Alien"}, - )) - - errors = validate_accession_row(sheet.rows[0], _attributes(), SAMPLE_TYPE) - - assert len(errors) >= 2 - - -class TestDuplicateAccessionErrors: - - def test_no_errors_when_all_unique(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1"}, {"accession": "ERR2"}, - )) - - assert duplicate_accession_errors(sheet.rows) == {} - - def test_repeated_accession_reports_error_on_later_row(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1"}, {"accession": "err1"}, - )) - - errors = duplicate_accession_errors(sheet.rows) - - assert 1 not in errors - assert any("ERR1" in message for message in errors[2]) - - def test_blank_accessions_are_not_treated_as_duplicates(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": ""}, {"accession": ""}, - )) - - assert duplicate_accession_errors(sheet.rows) == {} diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index e2eebf3..b7ca63b 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1,9 +1,7 @@ import csv -import itertools import json from http import HTTPStatus from pathlib import Path -from unittest.mock import patch import httpx import respx @@ -929,8 +927,6 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] - - def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: path = directory / "accessions.csv" with path.open("w", newline="") as handle: @@ -960,13 +956,12 @@ def _job_json( class TestSamplesImport: @respx.mock - def test_completed_job_reports_imported_samples( + def test_kicks_off_job_and_reports_id_without_waiting( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( + route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "ERR2"], [101, 102], + 42, "RUNNING", ["ERR1", "ERR2"], )), ) sheet = _write_import_sheet( @@ -981,18 +976,17 @@ def test_completed_job_reports_imported_samples( ) assert result.exit_code == 0 - assert "101" in result.stderr - assert "102" in result.stderr - assert "Imported 2, failed 0, skipped 0." in result.stdout + assert route.call_count == 1 + assert "42" in result.stdout + assert "import-status" in result.stdout @respx.mock - def test_json_document_reports_outcomes_and_counts( + def test_json_document_reports_job_fields( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], + 42, "RUNNING", ["ERR1"], )), ) sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) @@ -1005,68 +999,58 @@ def test_json_document_reports_outcomes_and_counts( assert result.exit_code == 0 document = json.loads(result.stdout) assert result.stdout.count("\n") == 1 - assert document["counts"] == {"imported": 1, "failed": 0, "skipped": 0} - assert document["imported"][0] == { - "row_number": 1, "accession": "ERR1", "sample_id": 101, + assert document == { + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + "execution_id": 7, + "error": None, } - assert document["job_id"] == 1 - assert document["job_status"] == "COMPLETED" - assert document["execution_id"] == 7 @respx.mock - def test_json_document_has_null_job_fields_when_nothing_submitted( + def test_sends_every_row_without_local_validation( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) + # No metadata/sample-type endpoints are mocked: if the command called + # client.samples.get_metadata_attributes() or otherwise validated + # locally, respx would fail the test for an unmocked request. + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["NOT-AN-ACCESSION", "ERR1", "ERR1"], + )), + ) sheet = _write_import_sheet( - tmp_path, {"accession": "bogus", "cell_type": "Neuron"}, + tmp_path, + {"accession": "not-an-accession"}, + {"accession": "ERR1"}, + {"accession": "ERR1"}, ) result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", - "--skip-invalid", "--json", ) assert result.exit_code == 0 - assert route.call_count == 0 - document = json.loads(result.stdout) - assert document["counts"] == {"imported": 0, "failed": 0, "skipped": 1} - assert document["job_id"] is None - assert document["job_status"] is None - assert document["execution_id"] is None - - @respx.mock - def test_header_only_sheet_imports_nothing_without_calling_api( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 0 - assert route.call_count == 0 - assert "Imported 0, failed 0, skipped 0." in result.stdout + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == [ + "NOT-AN-ACCESSION", "ERR1", "ERR1", + ] @respx.mock - def test_sends_sample_type_and_metadata_in_kickoff_payload( + def test_sends_name_organism_and_metadata_in_payload( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], + 1, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet( - tmp_path, {"accession": "err1", "cell_type": "Neuron", "organism": "Hs"}, - ) + sheet = _write_import_sheet(tmp_path, { + "accession": "err1", "name": "liver_r1", "organism": "Hs", + "cell_type": "Neuron", + }) run_cli( "--token", TOKEN, "samples", "import", @@ -1078,551 +1062,161 @@ def test_sends_sample_type_and_metadata_in_kickoff_payload( "imports": [{ "accession": "ERR1", "sample_type": "rna_seq", + "name": "liver_r1", "organism": "Hs", "metadata": {"cell_type": "Neuron"}, }], } @respx.mock - def test_sends_name_in_kickoff_payload(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], - )), - ) - sheet = _write_import_sheet( - tmp_path, {"accession": "ERR1", "name": "liver_r1", "cell_type": "Neuron"}, - ) - - run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - payload = json.loads(route.calls[0].request.content) - assert payload["imports"][0]["name"] == "liver_r1" - - @respx.mock - @patch("time.sleep") - def test_polls_until_job_completes( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1") - poll_route.side_effect = [ - httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), - httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), - ] - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", - ) - - assert result.exit_code == 0 - assert poll_route.call_count == 2 - assert "101" in result.stderr - - @respx.mock - @patch("time.sleep") - def test_failed_job_reports_error_message_per_row( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - error_message = "download failed: connection reset" - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], error=error_message, - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", - ) - - assert result.exit_code == 1 - assert error_message in result.stderr - assert "Row 1 (ERR1)" in result.stderr - assert "Row 2 (ERR2)" in result.stderr - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_timeout_reports_failure_without_hanging( - self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 1.0) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "3", - ) - - assert result.exit_code == 1 - assert poll_route.call_count == 2 - assert "Row 1 (ERR1)" in result.stderr - assert "did not finish within" in result.stderr - assert "still RUNNING" in result.stderr - assert "ended with status RUNNING" not in result.stderr - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_timeout_failed_rows_have_running_status( - self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + def test_api_rejection_propagates_as_error( + self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 1.0) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response( + HTTPStatus.UNPROCESSABLE_ENTITY, + json={"error": "sample type 'bogus' does not exist"}, + ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "3", "--json", + "--sheet", str(sheet), "--sample-type", "bogus", ) assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["failed"][0]["status"] == "running" - assert document["job_status"] == "RUNNING" - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_poll_interval_exceeding_timeout_is_clamped( - self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 0.5) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "300", "--timeout", "10", - ) - - assert result.exit_code == 0 - mock_sleep.assert_called_once() - slept_for = mock_sleep.call_args[0][0] - assert 0 < slept_for < 300 + assert route.call_count == 1 + assert "bogus" in result.stderr @respx.mock - def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() + def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK\x03\x04") result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "0", + "--sheet", str(xlsx), "--sample-type", "rna_seq", ) assert result.exit_code == 2 assert route.call_count == 0 - assert "--poll-interval" in result.stderr - - @respx.mock - def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + assert "CSV" in result.stderr + def test_missing_sheet_is_usage_error(self, run_cli) -> None: result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--timeout", "-1", + "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", ) assert result.exit_code == 2 - assert route.call_count == 0 - assert "--timeout" in result.stderr - @respx.mock - def test_nan_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "nan", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 - assert route.call_count == 0 - assert "--poll-interval" in result.stderr - @respx.mock - def test_infinite_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--timeout", "inf", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "--timeout" in result.stderr +class TestSamplesImportStatus: @respx.mock - def test_completed_job_with_no_sample_ids_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [], - )), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", - ) - - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert document["failed"][0]["accession"] == "ERR1" - assert document["failed"][0]["status"] == "unknown" - assert "cannot match" in document["failed"][0]["message"] - assert document["job_sample_ids"] == [] - - @respx.mock - def test_completed_job_with_matching_sample_id_missing_reports_row_as_failed( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "err-other"], [101, 102], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", - ) - - assert result.exit_code == 1 - document = json.loads(result.stdout) - failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} - assert document["imported"][0]["accession"] == "ERR1" - assert failed_by_accession["ERR2"]["status"] == "failed" - assert "no sample id" in failed_by_accession["ERR2"]["message"] - - @respx.mock - def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "ERR2"], [101], + def test_reports_running_job(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "RUNNING", ["ERR1", "ERR2"], )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert len(document["failed"]) == 2 - for entry in document["failed"]: - assert entry["status"] == "unknown" - assert "sample_id" not in entry - assert "cannot match" in entry["message"] - assert document["job_sample_ids"] == [101] + assert result.exit_code == 0 + assert "42" in result.stdout + assert "RUNNING" in result.stdout @respx.mock - def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERX1"], [101, 102], + def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", ["ERR1", "ERR2"], [101, 102], )), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERX1", "cell_type": "Neuron"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert document["failed"][0]["status"] == "unknown" - assert "sample_id" not in document["failed"][0] - assert "cannot match" in document["failed"][0]["message"] - assert document["job_sample_ids"] == [101, 102] + assert result.exit_code == 0 + assert "101" in result.stdout + assert "102" in result.stdout @respx.mock - @patch("time.sleep") - def test_failed_job_with_partial_sample_ids_still_surfaces_them( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + def test_reports_failed_job_with_error(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + 42, "FAILED", ["ERR1"], error="download failed", )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - for entry in document["failed"]: - assert entry["status"] == "unknown" - assert "sample_id" not in entry - # The job did create one sample before failing; that isn't attributable - # to a specific row, but it must not be silently dropped. - assert document["job_sample_ids"] == [101] + assert result.exit_code == 0 + assert "FAILED" in result.stdout + assert "download failed" in result.stdout @respx.mock - @patch("time.sleep") - def test_failed_job_with_all_sample_ids_created_reports_them_per_row( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + def test_json_document_matches_job_shape(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101, 102], error="post-processing failed", + 42, "COMPLETED", ["ERR1"], [101], )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", ) - assert result.exit_code == 1 + assert result.exit_code == 0 document = json.loads(result.stdout) - failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} - assert failed_by_accession["ERR1"]["sample_id"] == 101 - assert failed_by_accession["ERR2"]["sample_id"] == 102 - assert failed_by_accession["ERR1"]["status"] == "failed" + assert result.stdout.count("\n") == 1 + assert document == { + "id": 42, + "status": "COMPLETED", + "accessions": ["ERR1"], + "sample_ids": [101], + "execution_id": 7, + "error": None, + } @respx.mock - @patch("time.sleep") - def test_transient_poll_error_propagates( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.NOT_FOUND, json={"error": "gone"}), + def test_unknown_job_id_is_not_found(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/999").mock( + return_value=httpx.Response( + HTTPStatus.NOT_FOUND, json={"error": "sample import 999 does not exist"}, + ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", + "--token", TOKEN, "samples", "import-status", "--job-id", "999", ) assert result.exit_code == 4 - @respx.mock - def test_invalid_row_aborts_and_imports_nothing( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "not-an-accession", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) + def test_missing_job_id_is_usage_error(self, run_cli) -> None: + result = run_cli("--token", TOKEN, "samples", "import-status") assert result.exit_code == 2 - assert route.call_count == 0 - assert "Row 2" in result.stderr - assert "NOT-AN-ACCESSION" in result.stderr - - @respx.mock - def test_skip_invalid_imports_valid_rows( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "bogus", "cell_type": "Neuron"}, - ) + def test_non_numeric_job_id_is_usage_error(self, run_cli) -> None: result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--skip-invalid", - ) - - assert result.exit_code == 0 - assert route.call_count == 1 - payload = json.loads(route.calls[0].request.content) - assert len(payload["imports"]) == 1 - assert "BOGUS" in result.stderr - - @respx.mock - def test_duplicate_accession_is_usage_error( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "err1", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "duplicate" in result.stderr.lower() - - @respx.mock - def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - xlsx = tmp_path / "sheet.xlsx" - xlsx.write_bytes(b"PK\x03\x04") - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(xlsx), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "CSV" in result.stderr - - def test_missing_sheet_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - - def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", ) assert result.exit_code == 2 From 694198fa34a79d3b120c762e024c96b8f6e8d316 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:41:33 +0000 Subject: [PATCH 06/19] fix(samples): address round-5 findings on the non-blocking design - Revert flowbio/cli/_sheet.py to master: the metadata_errors extraction existed to be shared with accession-sheet validation, which the non-blocking rework removed entirely. Its only caller is validate_row again, so the helper goes back to being private (_metadata_errors), and the stale docstring claiming a shared caller that no longer exists is gone with it. - samples import-status now exits RUNTIME for a FAILED job (was always SUCCESS), so a caller polling it can branch on the exit code alone without parsing --json output. - Drop the client-side accession.upper() call, the only remaining rewrite of user input left over from the pre-rework design. The server already normalises case itself; letting it do so is more consistent with letting the API validate than silently uppercasing client-side first. - samples import now rejects a header-only sheet locally (CliUsageError, exit 2) instead of spending a round trip to be told by the API that at least one accession is required -- this is a structural check (does the sheet have anything to submit), not a validation-rule duplication. - Documented the utf-8-sig BOM-handling rationale in _accession_sheet.py (previously only in _sheet.py, now that the two parsers aren't sharing code). - Docs: replaced the ambiguous "exit 5 or 1" with the one code this endpoint's 422 responses actually map to; added a bounded polling example to the library docs; added API Reference entries for SampleImportStatus and SampleImportJobId; gave SampleImportSpec's fields rendered descriptions (:members: plus :param: docstring, matching ClientConfig's pattern); fixed a stray single/double-backtick inconsistency; documented import-status's new exit-code contract with a --json/jq polling example. Verified the Sphinx build renders all of this without new warnings (a pre-existing, unrelated legacy.rst directive error aside). - Polish: extra blank lines, and _job_summary no longer renders an empty 'Sample ids: .' for a completed job with none. Addresses Claude's round-5 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 11 +-- flowbio/cli/_samples.py | 17 +++-- flowbio/cli/_sheet.py | 96 +++++++++++--------------- flowbio/v2/samples.py | 13 +++- source/cli.rst | 35 ++++++---- source/v2/samples.rst | 12 +++- tests/unit/cli/test_accession_sheet.py | 4 +- tests/unit/cli/test_samples.py | 39 +++++++++-- 8 files changed, 138 insertions(+), 89 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index b55659c..89923c8 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -50,8 +50,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet with reserved/metadata columns separated, - accessions normalised to upper case, and empty cells dropped. + :returns: The parsed sheet with reserved/metadata columns separated and + empty cells dropped. Values are otherwise passed through unchanged — + including ``accession``, sent to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``. """ if path.suffix.lower() != ".csv": @@ -60,6 +61,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: f"Export your spreadsheet to CSV first.", ) existing_file(path) + # utf-8-sig transparently strips a leading BOM, which spreadsheet tools + # (notably Excel's "CSV UTF-8" export) prepend — otherwise the first header + # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) headers = reader.fieldnames or [] @@ -85,10 +89,9 @@ def cell(column: str) -> str | None: for column in metadata_columns if (value := (record.get(column) or "").strip()) } - accession = cell("accession") return AccessionSheetRow( row_number=row_number, - accession=accession.upper() if accession else "", + accession=cell("accession") or "", name=cell("name"), organism=cell("organism"), metadata=metadata, diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index c13ae01..1e05e86 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -629,8 +629,12 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. + :raises CliUsageError: If the sheet has no rows — there is nothing an + API call could tell us about an empty sheet that we can't see already. """ sheet = parse_accession_sheet(args.sheet) + if not sheet.rows: + raise CliUsageError(f"Accession sheet has no rows: {args.sheet}") specs = [ SampleImportSpec( accession=row.accession, @@ -659,13 +663,14 @@ def _import_status_command( :param args: Parsed command-line arguments. :param client: The authenticated Flow client. :param output: The result/error renderer. - :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched - (the job's own ``status`` — not this command's exit code — reflects - whether the import itself succeeded). + :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched, + unless the job itself is ``"FAILED"``, in which case + :attr:`ExitCode.RUNTIME` — so a caller polling this command can + branch on its exit code alone, without parsing ``--json`` output. """ job = client.samples.get_import(args.job_id) output.emit_result(_job_summary(job), _job_document(job)) - return ExitCode.SUCCESS + return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: @@ -681,7 +686,7 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": - ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) + ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" return f"Job {job.id}: COMPLETED. Sample ids: {ids}." if job.status == "FAILED": detail = f" {job.error}" if job.error else "" @@ -689,8 +694,6 @@ def _job_summary(job: SampleImportJob) -> str: return f"Job {job.id}: {job.status}." - - def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index a5effa1..e956ccb 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -96,6 +96,7 @@ def validate_row( :returns: One human-readable message per problem, collected so the caller can report them all at once. """ + by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] if not row.name: errors.append("missing required value: name") @@ -106,61 +107,7 @@ def validate_row( for label, reads in (("reads1", row.reads1), ("reads2", row.reads2)): if reads is not None and not reads.is_file(): errors.append(f"{label} file not found: {reads}") - errors.extend(metadata_errors(row.metadata, attributes, sample_type)) - return errors - - -def metadata_errors( - metadata: dict[str, str], - attributes: list[MetadataAttribute], - sample_type: SampleTypeId, -) -> list[str]: - """Return every metadata validation problem for ``metadata`` (empty when valid). - - Shared by :func:`validate_row` and the accession-sheet equivalent in - :mod:`flowbio.cli._accession_sheet`, since required/closed-option/annotation - rules apply identically regardless of what the rest of the row looks like. - - :param metadata: The row's metadata, keyed by attribute identifier (or - ``__annotation`` for a free-text companion). - :param attributes: The server's metadata attributes, deciding required and - closed-option columns. - :param sample_type: The sample type applied to the whole batch; an attribute - required for it must be present. - :returns: One human-readable message per problem, collected so the caller can - report them all at once. - """ - by_identifier = {attribute.identifier: attribute for attribute in attributes} - errors: list[str] = [] - for attribute in attributes: - required = ( - attribute.required or sample_type in attribute.required_for_sample_types - ) - if required and not metadata.get(attribute.identifier): - errors.append(f"missing required metadata: {attribute.identifier}") - for key, value in metadata.items(): - if key.endswith(ANNOTATION_SUFFIX): - continue - attribute = by_identifier.get(key) - if ( - attribute is not None - and attribute.options is not None - and value not in attribute.options - ): - errors.append( - f"value '{value}' for {key} is not one of: " - f"{', '.join(attribute.options)}", - ) - for key in metadata: - if not key.endswith(ANNOTATION_SUFFIX): - continue - base = key[: -len(ANNOTATION_SUFFIX)] - if not metadata.get(base): - errors.append(f"{key} set without a value for {base}") - continue - attribute = by_identifier.get(base) - if attribute is not None and not attribute.allow_annotation: - errors.append(f"{base} does not allow an annotation") + errors.extend(_metadata_errors(row, by_identifier, attributes, sample_type)) return errors @@ -195,3 +142,42 @@ def _resolve(value: str | None, base_dir: Path) -> Path | None: return None path = Path(value) return path if path.is_absolute() else base_dir / path + + +def _metadata_errors( + row: SheetRow, + by_identifier: dict[str, MetadataAttribute], + attributes: list[MetadataAttribute], + sample_type: SampleTypeId, +) -> list[str]: + errors: list[str] = [] + for attribute in attributes: + required = ( + attribute.required or sample_type in attribute.required_for_sample_types + ) + if required and not row.metadata.get(attribute.identifier): + errors.append(f"missing required metadata: {attribute.identifier}") + for key, value in row.metadata.items(): + if key.endswith(ANNOTATION_SUFFIX): + continue + attribute = by_identifier.get(key) + if ( + attribute is not None + and attribute.options is not None + and value not in attribute.options + ): + errors.append( + f"value '{value}' for {key} is not one of: " + f"{', '.join(attribute.options)}", + ) + for key in row.metadata: + if not key.endswith(ANNOTATION_SUFFIX): + continue + base = key[: -len(ANNOTATION_SUFFIX)] + if not row.metadata.get(base): + errors.append(f"{key} set without a value for {base}") + continue + attribute = by_identifier.get(base) + if attribute is not None and not attribute.allow_annotation: + errors.append(f"{base} does not allow an annotation") + return errors diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 8dada4f..398baf4 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -164,6 +164,17 @@ class SampleImportSpec: metadata={"strandedness": "reverse"}, ), ] + + :param accession: The public-repository run or experiment accession + (e.g. ``"ERR1160845"``), validated server-side. + :param sample_type: The sample type identifier (e.g. ``"RNA-Seq"``), + validated server-side. See :meth:`SampleResource.get_types`. + :param name: Optional sample name. Defaults to ``accession`` server-side + when omitted. + :param organism_id: Optional organism id (e.g. ``"Hs"``) to associate + with the sample, sent as ``organism``. + :param metadata: Optional metadata key-value pairs. See + :ref:`metadata-attributes` for details on required attributes. """ accession: str @@ -186,7 +197,7 @@ class SampleImportJob(BaseModel, frozen=True): status: SampleImportStatus = Field(description="The job's current lifecycle state.") accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( - description="The created samples' ids, corresponding to `accessions` once the job has completed.", + description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") error: str | None = Field(description='The failure reason, set only when status is "FAILED".') diff --git a/source/cli.rst b/source/cli.rst index 32cc406..265120c 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -384,8 +384,10 @@ metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules are all sent as-is and validated **server-side** — this command does not -pre-validate rows itself; an invalid sheet surfaces as a normal API error -(exit ``5`` or ``1``), not a local rejection. +pre-validate rows itself (beyond checking the sheet has at least one row); an +invalid sheet surfaces as a normal API error (exit ``1`` — the API returns +these as an HTTP ``422``, which isn't one of the codes with its own mapping +below), not a local rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -399,11 +401,10 @@ to you, e.g. in a shell loop. completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual -outcome — check that with ``import-status``); ``2`` a non-CSV sheet; ``1`` -the API rejected the batch (e.g. unknown sample type, missing required -metadata, an unsupported accession format — the server returns these as a -``422``, which isn't one of the exit codes with its own mapping below); ``3`` -authentication failure; otherwise the standard mapping above. +outcome — check that with ``import-status``); ``2`` a non-CSV or empty sheet; +``1`` the API rejected the batch (e.g. unknown sample type, missing required +metadata, an unsupported accession format); ``3`` authentication failure; +otherwise the standard mapping above. **Example** @@ -425,18 +426,26 @@ Fetch and report the current state of a ``samples import`` job. flowbio samples import-status --job-id ID Read-only — checking a job's status never changes it. There is no built-in -polling; run this again (or wrap it in your own loop, e.g. ``watch``) until -``status`` leaves ``"RUNNING"``. +polling; run this again (or wrap it in your own loop) until ``status`` leaves +``"RUNNING"``: + +.. code-block:: bash + + until [ "$(flowbio samples import-status --job-id 42 --json | jq -r .status)" != "RUNNING" ]; do + sleep 30 + done **Output** — human: a one-line summary including the sample ids on ``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, ``error``. -**Exit codes** — ``0`` the job's state was fetched (the job's own ``status``, -not this command's exit code, reflects whether the import itself succeeded -or failed); ``4`` no job with that id exists; ``3`` authentication failure; -otherwise the standard mapping above. +**Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or +``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller +can branch on the exit code alone, without parsing ``--json`` output — the +loop above still needs ``--json``/``jq`` to tell ``"RUNNING"`` from +``"COMPLETED"``, since both exit ``0``); ``4`` no job with that id exists; +``3`` authentication failure; otherwise the standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index eec9091..8098515 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -189,18 +189,19 @@ accession submitted together is tracked as a single job:: The job starts out ``"RUNNING"``. Poll it with :meth:`~flowbio.v2.samples.SampleResource.get_import` until its status -leaves ``"RUNNING"``:: +leaves ``"RUNNING"`` — bound the wait so a stuck job doesn't loop forever:: import time - while job.status == "RUNNING": + deadline = time.monotonic() + 1800 # 30 minutes + while job.status == "RUNNING" and time.monotonic() < deadline: time.sleep(5) job = client.samples.get_import(job.id) if job.status == "COMPLETED": print(f"Imported samples: {job.sample_ids}") else: - print(f"Import failed: {job.error}") + print(f"Import failed or still running: {job.error}") ``job.accessions`` and ``job.sample_ids`` correspond positionally once the job has completed. @@ -227,5 +228,10 @@ Models .. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload .. autoclass:: flowbio.v2.samples.SampleImportSpec + :members: .. autopydantic_model:: flowbio.v2.samples.SampleImportJob + +.. autodata:: flowbio.v2.samples.SampleImportJobId + +.. autodata:: flowbio.v2.samples.SampleImportStatus diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 57d0a81..2ccd8ec 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -29,12 +29,12 @@ def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] - def test_accession_is_normalised_to_upper_case(self, tmp_path: Path) -> None: + def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( _write_sheet(tmp_path, {"accession": "err1160845"}), ) - assert sheet.rows[0].accession == "ERR1160845" + assert sheet.rows[0].accession == "err1160845" def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index b7ca63b..4ebae57 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -927,6 +927,8 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: path = directory / "accessions.csv" with path.open("w", newline="") as handle: @@ -1017,7 +1019,7 @@ def test_sends_every_row_without_local_validation( # locally, respx would fail the test for an unmocked request. route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["NOT-AN-ACCESSION", "ERR1", "ERR1"], + 1, "RUNNING", ["not-an-accession", "ERR1", "ERR1"], )), ) sheet = _write_import_sheet( @@ -1035,7 +1037,7 @@ def test_sends_every_row_without_local_validation( assert result.exit_code == 0 payload = json.loads(route.calls[0].request.content) assert [entry["accession"] for entry in payload["imports"]] == [ - "NOT-AN-ACCESSION", "ERR1", "ERR1", + "not-an-accession", "ERR1", "ERR1", ] @respx.mock @@ -1048,7 +1050,7 @@ def test_sends_name_organism_and_metadata_in_payload( )), ) sheet = _write_import_sheet(tmp_path, { - "accession": "err1", "name": "liver_r1", "organism": "Hs", + "accession": "ERR1", "name": "liver_r1", "organism": "Hs", "cell_type": "Neuron", }) @@ -1120,6 +1122,20 @@ def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> No assert result.exit_code == 2 + @respx.mock + def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + class TestSamplesImportStatus: @@ -1167,10 +1183,25 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 0 + assert result.exit_code == 1 assert "FAILED" in result.stdout assert "download failed" in result.stdout + @respx.mock + def test_completed_job_with_no_sample_ids_reports_none(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", [], + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "none" in result.stdout + @respx.mock def test_json_document_matches_job_shape(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 8f829b7b02a68ff918e34cbec99e5979bd137456 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:51:44 +0000 Subject: [PATCH 07/19] fix(samples): address round-6 findings on the non-blocking design - Filter out rows with no accession (blank cell, comma-only trailing row, or a sheet with no 'accession' column at all) before submitting, and raise CliUsageError naming the sheet if nothing is left. Previously an empty accession was shipped as accession: "", which the API can never accept, turning a locally-detectable structural issue into an opaque all-or-nothing rejection of the whole batch. - samples import-status now emits a stderr advisory naming the job and its error when the job is FAILED, matching every other non-zero exit path in the CLI (which always has something on stderr, not just stdout/--json). - --job-id's type callback now raises a clear ArgumentTypeError instead of letting argparse report the private helper's own name ("invalid _job_id value"). - samples import's confirmation line now reports len(specs) (what was actually submitted) rather than echoing job.accessions back. - Fixed a test that didn't exercise the branch its name claimed (an empty job vs. a completed job with accessions but no sample ids). - Docs: hedged the exit-code claim for a rejected batch (1 for the 422s this endpoint returns in practice, 5 if it ever answers 400 instead), updated the import-status polling example to check the command's own exit code so a transient failure isn't read as "still running", and documented the FAILED stderr advisory. Rebuilt the docs locally to confirm they still render cleanly. Addresses Claude's round-6 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 24 ++++++--- source/cli.rst | 31 ++++++----- tests/unit/cli/test_samples.py | 95 +++++++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 28 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 1e05e86..4309452 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -275,7 +275,10 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: def _job_id(value: str) -> SampleImportJobId: - return SampleImportJobId(int(value)) + try: + return SampleImportJobId(int(value)) + except ValueError: + raise argparse.ArgumentTypeError(f"job id must be an integer, got {value!r}") from None def _configure_import_status(import_status: argparse.ArgumentParser) -> None: @@ -629,12 +632,17 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet has no rows — there is nothing an - API call could tell us about an empty sheet that we can't see already. + :raises CliUsageError: If the sheet is not a readable ``.csv``, or if it + has no row with an accession — there is nothing an API call could + tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - if not sheet.rows: - raise CliUsageError(f"Accession sheet has no rows: {args.sheet}") + rows = [row for row in sheet.rows if row.accession] + if not rows: + raise CliUsageError( + f"Accession sheet has no row with an accession: {args.sheet}. " + f"Check it has an 'accession' column and at least one filled-in row.", + ) specs = [ SampleImportSpec( accession=row.accession, @@ -643,11 +651,11 @@ def _import_command( organism_id=row.organism, metadata=row.metadata or None, ) - for row in sheet.rows + for row in rows ] job = client.samples.import_samples(specs) output.emit_result( - f"Started import job {job.id} for {len(job.accessions)} accession(s) " + f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", _job_document(job), @@ -669,6 +677,8 @@ def _import_status_command( branch on its exit code alone, without parsing ``--json`` output. """ job = client.samples.get_import(args.job_id) + if job.status == "FAILED": + output.emit_advisory(f"Job {job.id} failed: {job.error or 'no error message returned'}") output.emit_result(_job_summary(job), _job_document(job)) return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS diff --git a/source/cli.rst b/source/cli.rst index 265120c..3b2a8e2 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -383,11 +383,11 @@ a CSV with an ``accession`` column plus optional ``name``/``organism`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules -are all sent as-is and validated **server-side** — this command does not -pre-validate rows itself (beyond checking the sheet has at least one row); an -invalid sheet surfaces as a normal API error (exit ``1`` — the API returns -these as an HTTP ``422``, which isn't one of the codes with its own mapping -below), not a local rejection. +are all sent as-is and validated **server-side** — this command only checks +that the sheet is a readable ``.csv`` with at least one row that has an +accession (rows without one are dropped rather than submitted as an empty +string); anything else invalid surfaces as a normal API error, not a local +rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -401,9 +401,11 @@ to you, e.g. in a shell loop. completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual -outcome — check that with ``import-status``); ``2`` a non-CSV or empty sheet; -``1`` the API rejected the batch (e.g. unknown sample type, missing required -metadata, an unsupported accession format); ``3`` authentication failure; +outcome — check that with ``import-status``); ``2`` the sheet isn't a +readable ``.csv``, or has no row with an accession; ``1`` the API rejected +the batch (e.g. unknown sample type, missing required metadata, an +unsupported accession format — these come back as an HTTP ``422``; ``5`` in +the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the standard mapping above. **Example** @@ -427,18 +429,21 @@ Fetch and report the current state of a ``samples import`` job. Read-only — checking a job's status never changes it. There is no built-in polling; run this again (or wrap it in your own loop) until ``status`` leaves -``"RUNNING"``: +``"RUNNING"``. Check the command's own exit code alongside ``jq`` so a +transient failure (auth, network) doesn't get read as ``"RUNNING"``: .. code-block:: bash - until [ "$(flowbio samples import-status --job-id 42 --json | jq -r .status)" != "RUNNING" ]; do + while status=$(flowbio samples import-status --job-id 42 --json | jq -r .status); do + [ "$status" = "RUNNING" ] || break sleep 30 done **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single -document — ``id``, ``status``, ``accessions``, ``sample_ids``, -``execution_id``, ``error``. +``"COMPLETED"`` or the error on ``"FAILED"``, which is also reported as an +advisory on stderr in that case. ``--json``: the job as a single document — +``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, +``error``. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4ebae57..4ba66f2 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1136,6 +1136,65 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert route.call_count == 0 assert str(sheet) in result.stderr + @respx.mock + def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": ""}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + assert len(payload["imports"]) == 1 + assert payload["imports"][0]["accession"] == "ERR1" + + @respx.mock + def test_sheet_with_only_blank_accessions_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + + @respx.mock + def test_sheet_with_no_accession_column_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + with sheet.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer.writeheader() + writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + class TestSamplesImportStatus: @@ -1186,12 +1245,39 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: assert result.exit_code == 1 assert "FAILED" in result.stdout assert "download failed" in result.stdout + assert "download failed" in result.stderr + + @respx.mock + def test_failed_job_json_mode_has_error_on_stdout_only(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "FAILED", ["ERR1"], error="download failed", + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 1 + assert result.stderr == "" + document = json.loads(result.stdout) + assert document["error"] == "download failed" + + def test_non_numeric_job_id_reports_clear_message(self, run_cli) -> None: + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", + ) + + assert result.exit_code == 2 + assert "_job_id" not in result.stderr + assert "job id" in result.stderr.lower() @respx.mock def test_completed_job_with_no_sample_ids_reports_none(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 42, "COMPLETED", [], + 42, "COMPLETED", ["ERR1"], [], )), ) @@ -1244,10 +1330,3 @@ def test_missing_job_id_is_usage_error(self, run_cli) -> None: result = run_cli("--token", TOKEN, "samples", "import-status") assert result.exit_code == 2 - - def test_non_numeric_job_id_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", - ) - - assert result.exit_code == 2 From 54d6279294ccd2fbc43f93f3d48bb67217736640 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:00:56 +0000 Subject: [PATCH 08/19] fix(samples): address round-7 findings on the non-blocking design - samples import now reports each skipped (accession-less) row on stderr before submitting, instead of only a lower count in the confirmation line -- a 200-row sheet missing one accession previously looked like a clean success for 199. - AccessionSheetRow.accession is now str | None instead of using "" as an absent-value sentinel, matching name/organism/metadata (all of which already drop empty cells to None). The filter in _import_command reads as an explicit "is None" check instead of a bare truthiness test. - _job_summary no longer repeats a FAILED job's error text on stdout now that _import_status_command also puts it on stderr as an advisory -- a human running the command was seeing the same sentence twice. - Docs: rewrote the import-status polling example to gate the while loop on flowbio's own exit code (assigning the JSON document, not a value piped through jq, in the condition) -- the previous version's condition captured jq's exit status, not flowbio's, so the "checks the command's own exit code" claim it made was false. Documented that exit 1 covers both "the job failed" and "the request itself failed", and how to tell them apart in each output mode. Fixed two RST heading underlines that overshot their titles by one character. Rebuilt the docs locally to confirm everything still renders. Addresses Claude's round-7 review on flowbio#18. The remaining deferred item (no import-template verb for the accession sheet, so a batch-template sheet's reads1/reads2/project columns ship as bogus metadata) is unchanged from prior rounds' decision -- worth a follow-up ticket rather than folding into this one, as the review itself suggests. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 11 +++++-- flowbio/cli/_samples.py | 15 ++++++---- source/cli.rst | 51 +++++++++++++++++++-------------- tests/unit/cli/test_samples.py | 3 +- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 89923c8..af16bfd 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -27,10 +27,15 @@ @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet.""" + """One data row of an accession sheet. + + ``accession`` may be ``None`` (empty cell, or no ``accession`` column at + all) — reported by the caller rather than rejected here, so a missing + accession is one visible problem instead of an opaque server rejection. + """ row_number: int - accession: str + accession: str | None name: str | None organism: str | None metadata: dict[str, str] @@ -91,7 +96,7 @@ def cell(column: str) -> str | None: } return AccessionSheetRow( row_number=row_number, - accession=cell("accession") or "", + accession=cell("accession"), name=cell("name"), organism=cell("organism"), metadata=metadata, diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 4309452..11d8913 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -637,12 +637,14 @@ def _import_command( tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - rows = [row for row in sheet.rows if row.accession] - if not rows: + skipped = [row for row in sheet.rows if row.accession is None] + if len(skipped) == len(sheet.rows): raise CliUsageError( f"Accession sheet has no row with an accession: {args.sheet}. " f"Check it has an 'accession' column and at least one filled-in row.", ) + for row in skipped: + output.emit_advisory(f"Skipped row {row.row_number}: no accession") specs = [ SampleImportSpec( accession=row.accession, @@ -651,7 +653,8 @@ def _import_command( organism_id=row.organism, metadata=row.metadata or None, ) - for row in rows + for row in sheet.rows + if row.accession is not None ] job = client.samples.import_samples(specs) output.emit_result( @@ -699,8 +702,10 @@ def _job_summary(job: SampleImportJob) -> str: ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" return f"Job {job.id}: COMPLETED. Sample ids: {ids}." if job.status == "FAILED": - detail = f" {job.error}" if job.error else "" - return f"Job {job.id}: FAILED.{detail}" + # The error, if any, is on stderr as an advisory (see + # _import_status_command) rather than repeated here, so a human + # running this doesn't see the same sentence twice. + return f"Job {job.id}: FAILED." return f"Job {job.id}: {job.status}." diff --git a/source/cli.rst b/source/cli.rst index 3b2a8e2..e53e39b 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -368,7 +368,7 @@ authentication failure; otherwise the standard mapping above. {"uploaded": [{"row_number": 1, "name": "liver_r1", "sample_id": "samp_1"}], "failed": [], "skipped": [], "counts": {"uploaded": 1, "failed": 0, "skipped": 0}} ``samples import`` -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type @@ -385,9 +385,9 @@ has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules are all sent as-is and validated **server-side** — this command only checks that the sheet is a readable ``.csv`` with at least one row that has an -accession (rows without one are dropped rather than submitted as an empty -string); anything else invalid surfaces as a normal API error, not a local -rejection. +accession; rows without one are dropped (reported by row number on stderr) +rather than submitted as an empty string. Anything else invalid surfaces as +a normal API error, not a local rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -396,9 +396,9 @@ with ``samples import-status --job-id ID``; polling (if you want it) is up to you, e.g. in a shell loop. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``. ``--json``: the created job as a single document — -``id``, ``status``, ``accessions``, ``sample_ids`` (empty until the job -completes), ``execution_id``, ``error``. +``import-status``, plus one advisory per skipped row. ``--json``: the created +job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids`` +(empty until the job completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -419,7 +419,7 @@ otherwise the standard mapping above. {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} ``samples import-status`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~ Fetch and report the current state of a ``samples import`` job. @@ -429,28 +429,37 @@ Fetch and report the current state of a ``samples import`` job. Read-only — checking a job's status never changes it. There is no built-in polling; run this again (or wrap it in your own loop) until ``status`` leaves -``"RUNNING"``. Check the command's own exit code alongside ``jq`` so a -transient failure (auth, network) doesn't get read as ``"RUNNING"``: +``"RUNNING"``. Gate the loop on ``flowbio``'s own exit code (assigning the +document, not a value piped through ``jq``, in the ``while`` condition) so a +transient failure (auth, network) breaks the loop instead of being read as +``"RUNNING"``: .. code-block:: bash - while status=$(flowbio samples import-status --job-id 42 --json | jq -r .status); do - [ "$status" = "RUNNING" ] || break + while out=$(flowbio samples import-status --job-id 42 --json); do + [ "$(printf '%s' "$out" | jq -r .status)" = RUNNING ] || break sleep 30 done + printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"`` or the error on ``"FAILED"``, which is also reported as an -advisory on stderr in that case. ``--json``: the job as a single document — -``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, -``error``. +``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the +error as a separate advisory on stderr (so it isn't printed twice). ``--json`` +never prints prose to stderr (or anywhere but the one stdout document); the +failure reason there is the document's ``error`` field. ``--json``: the job +as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, +``execution_id``, ``error``. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or -``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller -can branch on the exit code alone, without parsing ``--json`` output — the -loop above still needs ``--json``/``jq`` to tell ``"RUNNING"`` from -``"COMPLETED"``, since both exit ``0``); ``4`` no job with that id exists; -``3`` authentication failure; otherwise the standard mapping above. +``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the +request itself failed (e.g. a transient server error) — human mode +distinguishes them (an ``Error:`` line means the request failed; a +``Job N: FAILED.`` line means the job did), and under ``--json`` a request +failure's document is on stderr while a ``FAILED`` job's is on stdout like +any other successful fetch. The loop above still needs ``--json``/``jq`` to +tell ``"RUNNING"`` from ``"COMPLETED"``, since both exit ``0``; ``4`` no job +with that id exists; ``3`` authentication failure; otherwise the standard +mapping above. **Example** diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4ba66f2..ea9497d 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1158,6 +1158,7 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: payload = json.loads(route.calls[0].request.content) assert len(payload["imports"]) == 1 assert payload["imports"][0]["accession"] == "ERR1" + assert "Skipped row 2" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1244,7 +1245,7 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: assert result.exit_code == 1 assert "FAILED" in result.stdout - assert "download failed" in result.stdout + assert "download failed" not in result.stdout assert "download failed" in result.stderr @respx.mock From 154e9bcdc0d0b2031cfceff3389a2e6bc1e24a56 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:21:27 +0000 Subject: [PATCH 09/19] fix(samples): address round-8 findings (counter reset per operator) - SampleImportJob now declares created/started/finished (the API already sends them, per the existing mocked test payloads -- pydantic's default extra="ignore" was silently dropping them). created is required (the server always sends it); started/finished default to None. Surfaced in both samples import's and samples import-status's JSON documents, so a caller resuming a poll from elsewhere can tell how long a job has been running without tracking wall-clock time itself. - samples import's JSON document now includes a skipped list (row numbers of any accession-less rows). Previously this was only visible as a stderr advisory in human mode, so an automated --json caller had no signal a row had been dropped at all. - _import_command now computes specs/skipped in a single pass over sheet.rows instead of two separate list comprehensions restating the same "does this row have an accession" predicate. Addresses Claude's round-8 review on flowbio#18. Findings left as-is per prior rounds' reasoning (a job's execution_id/error having no default=None, _job_document's list-in-JsonValue shape, sample_ids as list[int]) are noted on the PR rather than repeated here. Refs FLOW-689. --- flowbio/cli/_samples.py | 36 +++++++++++++---------- flowbio/v2/samples.py | 5 ++++ source/cli.rst | 21 ++++++++----- tests/unit/cli/test_samples.py | 54 +++++++++++++++++++++++++++++++++- tests/unit/v2/test_samples.py | 15 ++++++++-- 5 files changed, 104 insertions(+), 27 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 11d8913..ce3c8ca 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import parse_accession_sheet +from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -637,31 +637,34 @@ def _import_command( tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - skipped = [row for row in sheet.rows if row.accession is None] - if len(skipped) == len(sheet.rows): - raise CliUsageError( - f"Accession sheet has no row with an accession: {args.sheet}. " - f"Check it has an 'accession' column and at least one filled-in row.", - ) - for row in skipped: - output.emit_advisory(f"Skipped row {row.row_number}: no accession") - specs = [ - SampleImportSpec( + specs: list[SampleImportSpec] = [] + skipped: list[AccessionSheetRow] = [] + for row in sheet.rows: + if row.accession is None: + skipped.append(row) + continue + specs.append(SampleImportSpec( accession=row.accession, sample_type=args.sample_type, name=row.name, organism_id=row.organism, metadata=row.metadata or None, + )) + if not specs: + raise CliUsageError( + f"Accession sheet has no row with an accession: {args.sheet}. " + f"Check it has an 'accession' column and at least one filled-in row.", ) - for row in sheet.rows - if row.accession is not None - ] + for row in skipped: + output.emit_advisory(f"Skipped row {row.row_number}: no accession") job = client.samples.import_samples(specs) + document = _job_document(job) + document["skipped"] = [{"row_number": row.row_number} for row in skipped] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", - _job_document(job), + document, ) return ExitCode.SUCCESS @@ -690,6 +693,9 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: return { "id": job.id, "status": job.status, + "created": job.created, + "started": job.started, + "finished": job.finished, "accessions": job.accessions, "sample_ids": job.sample_ids, "execution_id": job.execution_id, diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 398baf4..ef49f4c 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -195,6 +195,11 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") + created: int = Field(description="Unix timestamp when the job was created.") + started: int | None = Field(default=None, description="Unix timestamp when the job started running, if it has.") + finished: int | None = Field( + default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", + ) accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", diff --git a/source/cli.rst b/source/cli.rst index e53e39b..0e68950 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -397,8 +397,11 @@ to you, e.g. in a shell loop. **Output** — human: a confirmation line with the job id and a pointer to ``import-status``, plus one advisory per skipped row. ``--json``: the created -job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids`` -(empty until the job completes), ``execution_id``, ``error``. +job as a single document — ``id``, ``status``, ``created``/``started``/ +``finished`` (Unix timestamps; the latter two ``null`` until the job reaches +those stages), ``accessions``, ``sample_ids`` (empty until the job +completes), ``execution_id``, ``error``, and ``skipped`` (the row numbers of +any accession-less rows, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -416,7 +419,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + {"id": 42, "status": "RUNNING", "created": 1712345678, "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null, "skipped": []} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -444,11 +447,13 @@ transient failure (auth, network) breaks the loop instead of being read as **Output** — human: a one-line summary including the sample ids on ``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the -error as a separate advisory on stderr (so it isn't printed twice). ``--json`` -never prints prose to stderr (or anywhere but the one stdout document); the -failure reason there is the document's ``error`` field. ``--json``: the job -as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, -``execution_id``, ``error``. +error as a separate advisory on stderr (so it isn't printed twice). ``--json``: +the job as a single document — ``id``, ``status``, ``created``/``started``/ +``finished`` (Unix timestamps, useful for judging how long a job has been +running when you've resumed polling one from elsewhere), ``accessions``, +``sample_ids``, ``execution_id``, ``error``. ``--json`` never prints prose to +stderr (or anywhere but that one stdout document); the failure reason on a +``"FAILED"`` job is the document's ``error`` field, not a separate message. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index ea9497d..3362b47 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -944,13 +944,17 @@ def _job_json( accessions: list[str], sample_ids: list[int] | None = None, error: str | None = None, + execution_id: int | None = 7, ) -> dict: return { "id": job_id, "status": status, + "created": 1700000000, + "started": 1700000001 if status != "RUNNING" else None, + "finished": 1700000002 if status in ("COMPLETED", "FAILED") else None, "accessions": accessions, "sample_ids": sample_ids or [], - "execution_id": 7, + "execution_id": execution_id, "error": error, } @@ -1004,10 +1008,14 @@ def test_json_document_reports_job_fields( assert document == { "id": 42, "status": "RUNNING", + "created": 1700000000, + "started": None, + "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": 7, "error": None, + "skipped": [], } @respx.mock @@ -1160,6 +1168,31 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: assert payload["imports"][0]["accession"] == "ERR1" assert "Skipped row 2" in result.stderr + @respx.mock + def test_blank_accession_row_is_reported_in_json_document( + self, run_cli, tmp_path: Path, + ) -> None: + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": ""}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 0 + assert result.stderr == "" + document = json.loads(result.stdout) + assert document["skipped"] == [{"row_number": 2}] + @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, @@ -1307,12 +1340,31 @@ def test_json_document_matches_job_shape(self, run_cli) -> None: assert document == { "id": 42, "status": "COMPLETED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, "accessions": ["ERR1"], "sample_ids": [101], "execution_id": 7, "error": None, } + @respx.mock + def test_reports_job_with_no_execution_yet(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "RUNNING", ["ERR1"], execution_id=None, + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["execution_id"] is None + @respx.mock def test_unknown_job_id_is_not_found(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/999").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index c254c3e..9b49029 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1082,6 +1082,9 @@ def test_posts_imports_and_parses_job(self) -> None: assert result == SampleImportJob( id=SampleImportJobId(42), status="RUNNING", + created=1700000000, + started=None, + finished=None, accessions=["ERR1160845"], sample_ids=[], execution_id=None, @@ -1093,7 +1096,7 @@ def test_posts_imports_and_parses_job(self) -> None: def test_sends_accession_and_sample_type(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1112,7 +1115,7 @@ def test_sends_accession_and_sample_type(self) -> None: def test_sends_optional_fields_when_present(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1143,7 +1146,7 @@ def test_sends_optional_fields_when_present(self) -> None: def test_sends_multiple_imports_in_one_request(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1", "ERR2"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1", "ERR2"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1199,6 +1202,9 @@ def test_parses_completed_job(self) -> None: assert result == SampleImportJob( id=SampleImportJobId(42), status="COMPLETED", + created=1700000000, + started=1700000001, + finished=1700000002, accessions=["ERR1160845", "ERR10677146"], sample_ids=[101, 102], execution_id=7, @@ -1211,6 +1217,9 @@ def test_parses_failed_job_with_error(self) -> None: return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "FAILED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, "accessions": ["ERR1160845"], "sample_ids": [], "execution_id": 7, From b3de2ff3a2c9865d18aef132ae1640d261475702 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:30:38 +0000 Subject: [PATCH 10/19] fix(samples): address round-9 findings - created is now int | None (default None), matching started/finished -- round 8 gave two of the three timestamps a default and left the third required on identical evidence, so a response omitting created would have raised an uncatchable ValidationError in both samples import and samples import-status where the previous model simply ignored the field. Added a test parsing a job payload with no timestamp keys at all. - samples import's JSON "skipped" entries now carry a reason ("no accession"), matching upload-batch's skipped-row shape (row_number/name/ reasons) instead of a bare row number with no indication why. - samples import-status now surfaces started in the human summary too ("Job 42: RUNNING (started 2024-...UTC).") -- round 8 added the timestamps to --json only, leaving the human mode (arguably the more likely place to ask "how long has this been running") unchanged. - Docs: updated the import-status --json example to include created/ started/finished (it had been left out while the import example got them), gave the previously-unreferenced sample-imports doc label a purpose via a :ref: from the CLI section instead of leaving it dangling, and fixed a heading underline overshoot in samples.rst. Rebuilt the docs locally to confirm everything still renders and the new cross-reference resolves. Addresses Claude's round-9 review on flowbio#18 (counter reset per operator request). Standing deferred items (execution_id/error with no default, _job_document's list-in-JsonValue shape, metadata_columns' one test-only consumer) are unchanged from prior rounds' reasoning. Refs FLOW-689. --- flowbio/cli/_samples.py | 11 ++++++++++- flowbio/v2/samples.py | 6 ++++-- source/cli.rst | 27 +++++++++++++++------------ source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 19 ++++++++++++++++++- tests/unit/v2/test_samples.py | 20 ++++++++++++++++++++ 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ce3c8ca..832fd57 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,6 +11,7 @@ import argparse import json from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from typing import Literal @@ -659,7 +660,9 @@ def _import_command( output.emit_advisory(f"Skipped row {row.row_number}: no accession") job = client.samples.import_samples(specs) document = _job_document(job) - document["skipped"] = [{"row_number": row.row_number} for row in skipped] + document["skipped"] = [ + {"row_number": row.row_number, "reason": "no accession"} for row in skipped + ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " @@ -712,9 +715,15 @@ def _job_summary(job: SampleImportJob) -> str: # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. return f"Job {job.id}: FAILED." + if job.started is not None: + return f"Job {job.id}: {job.status} (started {_format_timestamp(job.started)})." return f"Job {job.id}: {job.status}." +def _format_timestamp(timestamp: int) -> str: + return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index ef49f4c..f354d7c 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -195,8 +195,10 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") - created: int = Field(description="Unix timestamp when the job was created.") - started: int | None = Field(default=None, description="Unix timestamp when the job started running, if it has.") + created: int | None = Field(default=None, description="Unix timestamp when the job was created.") + started: int | None = Field( + default=None, description="Unix timestamp when the job started running, if it has.", + ) finished: int | None = Field( default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", ) diff --git a/source/cli.rst b/source/cli.rst index 0e68950..e41ae18 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -393,15 +393,17 @@ Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial status (almost always ``"RUNNING"``) and returns immediately. Check on it with ``samples import-status --job-id ID``; polling (if you want it) is up -to you, e.g. in a shell loop. +to you, e.g. in a shell loop. Building the same thing directly against the +library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to ``import-status``, plus one advisory per skipped row. ``--json``: the created job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps; the latter two ``null`` until the job reaches -those stages), ``accessions``, ``sample_ids`` (empty until the job -completes), ``execution_id``, ``error``, and ``skipped`` (the row numbers of -any accession-less rows, empty if none). +``finished`` (Unix timestamps, ``null`` if not yet reached — or if the server +response omits one, which the client tolerates), ``accessions``, +``sample_ids`` (empty until the job completes), ``execution_id``, ``error``, +and ``skipped`` (``{"row_number": ..., "reason": "no accession"}`` for each +dropped row, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -446,12 +448,13 @@ transient failure (auth, network) breaks the loop instead of being read as printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the -error as a separate advisory on stderr (so it isn't printed twice). ``--json``: -the job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps, useful for judging how long a job has been -running when you've resumed polling one from elsewhere), ``accessions``, -``sample_ids``, ``execution_id``, ``error``. ``--json`` never prints prose to +``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or — on +``"FAILED"`` — a plain ``FAILED.`` summary plus the error as a separate +advisory on stderr (so it isn't printed twice). ``--json``: the job as a +single document — ``id``, ``status``, ``created``/``started``/``finished`` +(Unix timestamps, useful for judging how long a job has been running when +you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, +``execution_id``, ``error``. ``--json`` never prints prose to stderr (or anywhere but that one stdout document); the failure reason on a ``"FAILED"`` job is the document's ``error`` field, not a separate message. @@ -474,7 +477,7 @@ mapping above. Job 42: COMPLETED. Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json - {"id": 42, "status": "COMPLETED", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} + {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 8098515..c055350 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -169,7 +169,7 @@ if the annotation has any warnings. .. _sample-imports: Importing samples from public repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create samples directly from public-repository run or experiment accessions diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3362b47..6f540c8 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1191,7 +1191,7 @@ def test_blank_accession_row_is_reported_in_json_document( assert result.exit_code == 0 assert result.stderr == "" document = json.loads(result.stdout) - assert document["skipped"] == [{"row_number": 2}] + assert document["skipped"] == [{"row_number": 2, "reason": "no accession"}] @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1248,6 +1248,23 @@ def test_reports_running_job(self, run_cli) -> None: assert "42" in result.stdout assert "RUNNING" in result.stdout + @respx.mock + def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": 1700000001, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 2023-11-14" in result.stdout + @respx.mock def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 9b49029..7d8a99b 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1245,3 +1245,23 @@ def test_raises_not_found_for_unknown_job(self) -> None: with pytest.raises(NotFoundError): client.samples.get_import(SampleImportJobId(999)) + + @respx.mock + def test_parses_job_missing_timestamps(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + "execution_id": None, + "error": None, + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.created is None + assert result.started is None + assert result.finished is None From 02189c9a658e17322cee9cb14aefbcaa1137cbe6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:39:59 +0000 Subject: [PATCH 11/19] fix(samples): address round-10 findings - _format_timestamp now falls back to the raw number on ValueError/ OverflowError/OSError instead of letting a display-only computation crash a read-only status check. Added a test for an out-of-range (millisecond-precision) timestamp. - samples import's "skipped" JSON entries now use a reasons list and carry name, matching upload-batch's skipped-row shape exactly instead of a one-off {row_number, reason} pair a consumer of both commands would need a second code path for. - The stderr advisory and JSON entry for a skipped row now include its name when the sheet provided one, since an accession-less row often has no other identifying detail and "Skipped row 2" alone sends the user to the wrong place if they count from the header. - import-status's human summary now shows when a COMPLETED/FAILED job finished, not just when a RUNNING job started -- the previous round fixed half of the human/machine asymmetry the finding was about. - Fixed the heading underline in samples.rst that the previous round's attempt undershot by one character. - Docs: updated the skipped-entry description and the import/import-status examples to match. Rebuilt the docs locally to confirm they still render. Addresses Claude's round-10 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 32 +++++++++++++----- source/cli.rst | 25 +++++++------- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 59 +++++++++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 22 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 832fd57..9aca30d 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -657,11 +657,12 @@ def _import_command( f"Check it has an 'accession' column and at least one filled-in row.", ) for row in skipped: - output.emit_advisory(f"Skipped row {row.row_number}: no accession") + output.emit_advisory(f"Skipped {_skipped_row_label(row)}: no accession") job = client.samples.import_samples(specs) document = _job_document(job) document["skipped"] = [ - {"row_number": row.row_number, "reason": "no accession"} for row in skipped + {"row_number": row.row_number, "name": row.name, "reasons": ["no accession"]} + for row in skipped ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " @@ -672,6 +673,11 @@ def _import_command( return ExitCode.SUCCESS +def _skipped_row_label(row: AccessionSheetRow) -> str: + name_suffix = f" ({row.name})" if row.name else "" + return f"row {row.row_number}{name_suffix}" + + def _import_status_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: @@ -709,19 +715,29 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" - return f"Job {job.id}: COMPLETED. Sample ids: {ids}." + return f"Job {job.id}: COMPLETED{_timestamp_suffix('finished', job.finished)}. Sample ids: {ids}." if job.status == "FAILED": # The error, if any, is on stderr as an advisory (see # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. - return f"Job {job.id}: FAILED." - if job.started is not None: - return f"Job {job.id}: {job.status} (started {_format_timestamp(job.started)})." - return f"Job {job.id}: {job.status}." + return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." + return f"Job {job.id}: {job.status}{_timestamp_suffix('started', job.started)}." + + +def _timestamp_suffix(label: str, timestamp: int | None) -> str: + if timestamp is None: + return "" + return f" ({label} {_format_timestamp(timestamp)})" def _format_timestamp(timestamp: int) -> str: - return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + # The API is only known to send Unix-seconds timestamps, but this only + # renders a display string — falling back to the raw value on anything + # unexpected keeps a formatting surprise from failing a status check. + try: + return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + except (ValueError, OverflowError, OSError): + return str(timestamp) def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index e41ae18..5a01bb1 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -397,13 +397,14 @@ to you, e.g. in a shell loop. Building the same thing directly against the library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``, plus one advisory per skipped row. ``--json``: the created -job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps, ``null`` if not yet reached — or if the server -response omits one, which the client tolerates), ``accessions``, -``sample_ids`` (empty until the job completes), ``execution_id``, ``error``, -and ``skipped`` (``{"row_number": ..., "reason": "no accession"}`` for each -dropped row, empty if none). +``import-status``, plus one advisory per skipped row (identified by name if +it has one). ``--json``: the created job as a single document — ``id``, +``status``, ``created``/``started``/``finished`` (Unix timestamps, ``null`` +if not yet reached — or if the server response omits one, which the client +tolerates), ``accessions``, ``sample_ids`` (empty until the job completes), +``execution_id``, ``error``, and ``skipped`` (``{"row_number": ..., "name": +..., "reasons": [...]}`` for each dropped row, matching ``upload-batch``'s +shape, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -447,10 +448,10 @@ transient failure (auth, network) breaks the loop instead of being read as done printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored -**Output** — human: a one-line summary including the sample ids on -``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or — on -``"FAILED"`` — a plain ``FAILED.`` summary plus the error as a separate -advisory on stderr (so it isn't printed twice). ``--json``: the job as a +**Output** — human: a one-line summary including the sample ids and when it +finished on ``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or +— on ``"FAILED"`` — when it finished plus the error as a separate advisory on +stderr (so it isn't printed twice). ``--json``: the job as a single document — ``id``, ``status``, ``created``/``started``/``finished`` (Unix timestamps, useful for judging how long a job has been running when you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, @@ -474,7 +475,7 @@ mapping above. .. code-block:: bash $ flowbio samples import-status --job-id 42 - Job 42: COMPLETED. Sample ids: 101, 102. + Job 42: COMPLETED (finished 2024-04-05 19:38:20 UTC). Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} diff --git a/source/v2/samples.rst b/source/v2/samples.rst index c055350..5cfee62 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -169,7 +169,7 @@ if the annotation has any warnings. .. _sample-imports: Importing samples from public repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create samples directly from public-repository run or experiment accessions diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 6f540c8..5fe07d8 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1191,7 +1191,32 @@ def test_blank_accession_row_is_reported_in_json_document( assert result.exit_code == 0 assert result.stderr == "" document = json.loads(result.stdout) - assert document["skipped"] == [{"row_number": 2, "reason": "no accession"}] + assert document["skipped"] == [ + {"row_number": 2, "name": None, "reasons": ["no accession"]}, + ] + + @respx.mock + def test_blank_accession_row_with_name_is_identified_by_name( + self, run_cli, tmp_path: Path, + ) -> None: + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": "", "name": "liver_r2"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert "Skipped row 2 (liver_r2)" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1265,6 +1290,38 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert result.exit_code == 0 assert "started 2023-11-14" in result.stdout + @respx.mock + def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": 1700000000000, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 1700000000000" in result.stdout + + @respx.mock + def test_completed_job_reports_when_it_finished(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", ["ERR1"], [101], + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "finished 2023-11-14" in result.stdout + @respx.mock def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 78115d4921ad43a0ec5bf2224871f5dc70e8efc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:19:06 +0000 Subject: [PATCH 12/19] refactor(samples): require accession, add per-row sample_type, simplify Per operator review comments on the PR: - AccessionSheetRow.accession is now a required str, not str | None. parse_accession_sheet rejects the whole sheet up front (CliUsageError, naming every offending row) if any row has no accession, or if the sheet has no rows at all, instead of silently skipping accession-less rows as the last several rounds had it doing. This removes the entire skip-tracking/"skipped" JSON-document machinery that existed only to support that behaviour: _skipped_row_label, _job_document's "skipped" key, and the per-row stderr advisories are all gone with it. - Added an optional sample_type column to the accession sheet: a row's own value overrides --sample-type for that row only, so a mixed-type sheet no longer has to be split into separate --sample-type invocations. AccessionSheetRow.to_spec(default_sample_type) builds the SampleImportSpec for a row, replacing the inline construction that used to live in _import_command. - Removed AccessionSheet.metadata_columns -- each row already carries its own metadata dict, so the sheet-level list was redundant and had no production caller. - _build_row's metadata-column comprehension now goes through the same cell() helper as accession/name/organism instead of duplicating its "strip, treat blank as absent" logic inline. - SampleImportJob.created/started/finished are now datetime | None instead of int | None -- pydantic parses the server's Unix-timestamp ints (and would parse ISO strings too) directly into proper datetime objects, so the CLI's display code no longer needs its own out-of-range fallback; formatting a valid datetime can't fail the way formatting an arbitrary int could. - _job_document is gone: both samples import and samples import-status now pass job.model_dump(mode="json") straight to Output.emit_result instead of a hand-maintained dict that had to be kept in sync with the model's fields by hand. Test suite and docs (source/cli.rst) updated throughout to match: the skip-related tests are replaced with tests asserting a blank accession (alone or mixed with valid rows) rejects the whole sheet, and a new TestAccessionSheetRowToSpec class covers the default/per-row sample_type behaviour. JSON-document assertions now expect ISO 8601 timestamp strings. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 75 +++++++++++++------ flowbio/cli/_samples.py | 90 ++++++---------------- flowbio/v2/samples.py | 11 ++- source/cli.rst | 59 +++++++-------- tests/unit/cli/test_accession_sheet.py | 92 ++++++++++++++++++++--- tests/unit/cli/test_samples.py | 100 +++++++++++-------------- 6 files changed, 233 insertions(+), 194 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index af16bfd..021a6d7 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,17 +1,18 @@ """CSV accession-sheet parsing for ``samples import``. -An accession sheet is a CSV with one row per accession to import: an +An accession sheet is a CSV with one row per accession to import: a required ``accession`` column (a public-repository run or experiment accession) plus -optional ``name``/``organism`` and per-accession metadata columns. This -mirrors ``_sheet.py``'s reads-based sample sheet, but the reserved columns -differ — there is nothing to upload (no ``reads1``/``reads2``) and the import -API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, -``organism`` instead. - -Rows are not validated here: the accession format, duplicates, sample type, -organism, and metadata rules are all checked server-side when the sheet is -submitted — duplicating that locally would just be a second, driftable copy -of the same rules. +optional ``name``/``organism``/``sample_type`` and per-accession metadata +columns. This mirrors ``_sheet.py``'s reads-based sample sheet, but the +reserved columns differ — there is nothing to upload (no ``reads1``/ +``reads2``) and the import API has no project field. + +Domain rules (accession format, duplicates, sample type, organism, metadata) +are all checked server-side when the sheet is submitted — duplicating that +locally would just be a second, driftable copy of the same rules. An +accession is different: it is the one column every row must have to mean +anything at all, so a missing one is rejected here rather than silently +skipped or shipped as an empty string the server would just reject anyway. """ from __future__ import annotations @@ -21,32 +22,47 @@ from flowbio.cli._exit_codes import CliUsageError from flowbio.cli._files import existing_file +from flowbio.v2.samples import SampleImportSpec, SampleTypeId -RESERVED_COLUMNS = ("accession", "name", "organism") +RESERVED_COLUMNS = ("accession", "name", "organism", "sample_type") @dataclass(frozen=True) class AccessionSheetRow: """One data row of an accession sheet. - ``accession`` may be ``None`` (empty cell, or no ``accession`` column at - all) — reported by the caller rather than rejected here, so a missing - accession is one visible problem instead of an opaque server rejection. + ``sample_type`` is only set when the sheet has its own ``sample_type`` + column for this row; :meth:`to_spec` falls back to the batch's + ``--sample-type`` when it's absent. """ row_number: int - accession: str | None + accession: str name: str | None organism: str | None + sample_type: SampleTypeId | None metadata: dict[str, str] + def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: + """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. + + :param default_sample_type: The sample type to use when this row has + no ``sample_type`` of its own. + """ + return SampleImportSpec( + accession=self.accession, + sample_type=self.sample_type or default_sample_type, + name=self.name, + organism_id=self.organism, + metadata=self.metadata or None, + ) + @dataclass(frozen=True) class AccessionSheet: """A parsed accession sheet.""" path: Path - metadata_columns: list[str] rows: list[AccessionSheetRow] @@ -55,10 +71,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet with reserved/metadata columns separated and - empty cells dropped. Values are otherwise passed through unchanged — - including ``accession``, sent to the server exactly as entered. - :raises CliUsageError: If the file is not a readable ``.csv``. + :returns: The parsed sheet, with empty cells dropped. Values are otherwise + passed through unchanged — including ``accession``, sent to the + server exactly as entered. + :raises CliUsageError: If the file is not a readable ``.csv``, has no + rows, or has a row with no accession. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -79,7 +96,15 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: _build_row(record, row_number, metadata_columns) for row_number, record in enumerate(reader, start=1) ] - return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) + if not rows: + raise CliUsageError(f"Accession sheet has no rows: {path}.") + missing = [row.row_number for row in rows if not row.accession] + if missing: + raise CliUsageError( + f"Accession sheet row(s) {', '.join(str(number) for number in missing)} " + f"have no accession: {path}.", + ) + return AccessionSheet(path=path, rows=rows) def _build_row( @@ -92,12 +117,14 @@ def cell(column: str) -> str | None: metadata = { column: value for column in metadata_columns - if (value := (record.get(column) or "").strip()) + if (value := cell(column)) is not None } + sample_type = cell("sample_type") return AccessionSheetRow( row_number=row_number, - accession=cell("accession"), + accession=cell("accession") or "", name=cell("name"), organism=cell("organism"), + sample_type=SampleTypeId(sample_type) if sample_type is not None else None, metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 9aca30d..86c97ed 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,11 +11,11 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -32,7 +32,6 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, - SampleImportSpec, SampleTypeId, ) @@ -264,14 +263,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: required=True, metavar="PATH", type=Path, - help="CSV accession sheet (accession, optional name/organism, plus metadata columns).", + help=( + "CSV accession sheet (required accession column, optional " + "name/organism/sample_type, plus metadata columns)." + ), ) import_parser.add_argument( "--sample-type", required=True, metavar="TYPE", type=SampleTypeId, - help="Sample type applied to every accession (sent as-is; validated server-side).", + help=( + "Default sample type (sent as-is; validated server-side), used for any " + "row without its own sample_type column." + ), ) @@ -623,8 +628,8 @@ def _import_command( ) -> ExitCode: """Kick off a batch import job from an accession sheet and report its id. - Every row is submitted as-is: the accession format, duplicates, sample - type, organism, and metadata rules are all validated server-side, so a + Every row is submitted as-is: the accession format, sample type, + organism, and metadata rules are all validated server-side, so a malformed sheet surfaces as a normal :class:`FlowApiError` rather than a local pre-flight rejection. This command does not wait for the job to finish — poll it yourself with ``samples import-status``. @@ -633,51 +638,21 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet is not a readable ``.csv``, or if it - has no row with an accession — there is nothing an API call could - tell us about either that we can't see already. + :raises CliUsageError: If the sheet is not a readable ``.csv``, has no + rows, or has a row with no accession. """ sheet = parse_accession_sheet(args.sheet) - specs: list[SampleImportSpec] = [] - skipped: list[AccessionSheetRow] = [] - for row in sheet.rows: - if row.accession is None: - skipped.append(row) - continue - specs.append(SampleImportSpec( - accession=row.accession, - sample_type=args.sample_type, - name=row.name, - organism_id=row.organism, - metadata=row.metadata or None, - )) - if not specs: - raise CliUsageError( - f"Accession sheet has no row with an accession: {args.sheet}. " - f"Check it has an 'accession' column and at least one filled-in row.", - ) - for row in skipped: - output.emit_advisory(f"Skipped {_skipped_row_label(row)}: no accession") + specs = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) - document = _job_document(job) - document["skipped"] = [ - {"row_number": row.row_number, "name": row.name, "reasons": ["no accession"]} - for row in skipped - ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", - document, + job.model_dump(mode="json"), ) return ExitCode.SUCCESS -def _skipped_row_label(row: AccessionSheetRow) -> str: - name_suffix = f" ({row.name})" if row.name else "" - return f"row {row.row_number}{name_suffix}" - - def _import_status_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: @@ -694,24 +669,10 @@ def _import_status_command( job = client.samples.get_import(args.job_id) if job.status == "FAILED": output.emit_advisory(f"Job {job.id} failed: {job.error or 'no error message returned'}") - output.emit_result(_job_summary(job), _job_document(job)) + output.emit_result(_job_summary(job), job.model_dump(mode="json")) return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS -def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: - return { - "id": job.id, - "status": job.status, - "created": job.created, - "started": job.started, - "finished": job.finished, - "accessions": job.accessions, - "sample_ids": job.sample_ids, - "execution_id": job.execution_id, - "error": job.error, - } - - def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" @@ -721,23 +682,14 @@ def _job_summary(job: SampleImportJob) -> str: # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." - return f"Job {job.id}: {job.status}{_timestamp_suffix('started', job.started)}." + label = "started" if job.started else "created" + return f"Job {job.id}: {job.status}{_timestamp_suffix(label, job.started or job.created)}." -def _timestamp_suffix(label: str, timestamp: int | None) -> str: +def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - return f" ({label} {_format_timestamp(timestamp)})" - - -def _format_timestamp(timestamp: int) -> str: - # The API is only known to send Unix-seconds timestamps, but this only - # renders a display string — falling back to the raw value on anything - # unexpected keeps a formatting surprise from failing a status check. - try: - return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - except (ValueError, OverflowError, OSError): - return str(timestamp) + return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index f354d7c..27fb79e 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -28,6 +28,7 @@ from collections.abc import Sequence from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Literal, NewType @@ -195,12 +196,10 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") - created: int | None = Field(default=None, description="Unix timestamp when the job was created.") - started: int | None = Field( - default=None, description="Unix timestamp when the job started running, if it has.", - ) - finished: int | None = Field( - default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", + created: datetime | None = Field(default=None, description="When the job was created.") + started: datetime | None = Field(default=None, description="When the job started running, if it has.") + finished: datetime | None = Field( + default=None, description="When the job finished (completed or failed), if it has.", ) accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( diff --git a/source/cli.rst b/source/cli.rst index 5a01bb1..4b5172f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -379,15 +379,17 @@ to every row — no files to upload yourself. flowbio samples import --sheet PATH --sample-type TYPE Run ``flowbio samples import --help`` for the full option list. The sheet is -a CSV with an ``accession`` column plus optional ``name``/``organism`` and -metadata columns (there is no ``batch-template`` equivalent for it, since it -has no reads files or project field). ``name`` defaults to the accession when -omitted. The sample type, accession format, duplicates, and metadata rules -are all sent as-is and validated **server-side** — this command only checks -that the sheet is a readable ``.csv`` with at least one row that has an -accession; rows without one are dropped (reported by row number on stderr) -rather than submitted as an empty string. Anything else invalid surfaces as -a normal API error, not a local rejection. +a CSV with a required ``accession`` column, plus optional ``name``/ +``organism``/``sample_type`` and metadata columns (there is no +``batch-template`` equivalent for it, since it has no reads files or project +field). ``name`` defaults to the accession when omitted. A row's own +``sample_type`` column, if present, overrides ``--sample-type`` for that row +only — useful for a mixed-type sheet. The sample type, accession format, and +metadata rules are all sent as-is and validated **server-side**; this command +only checks that the sheet is a readable ``.csv`` and that every row has an +accession — that column is the one thing every row must have to mean +anything, so a blank cell rejects the whole sheet up front rather than +shipping an empty string the server would just reject anyway. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -397,19 +399,15 @@ to you, e.g. in a shell loop. Building the same thing directly against the library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``, plus one advisory per skipped row (identified by name if -it has one). ``--json``: the created job as a single document — ``id``, -``status``, ``created``/``started``/``finished`` (Unix timestamps, ``null`` -if not yet reached — or if the server response omits one, which the client -tolerates), ``accessions``, ``sample_ids`` (empty until the job completes), -``execution_id``, ``error``, and ``skipped`` (``{"row_number": ..., "name": -..., "reasons": [...]}`` for each dropped row, matching ``upload-batch``'s -shape, empty if none). +``import-status``. ``--json``: the created job as a single document — +``id``, ``status``, ``created``/``started``/``finished`` (ISO 8601 +timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` +(empty until the job completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, or has no row with an accession; ``1`` the API rejected -the batch (e.g. unknown sample type, missing required metadata, an +readable ``.csv``, has no rows, or has a row with no accession; ``1`` the API +rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the standard mapping above. @@ -422,7 +420,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "created": 1712345678, "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null, "skipped": []} + {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -449,15 +447,16 @@ transient failure (auth, network) breaks the loop instead of being read as printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids and when it -finished on ``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or -— on ``"FAILED"`` — when it finished plus the error as a separate advisory on -stderr (so it isn't printed twice). ``--json``: the job as a -single document — ``id``, ``status``, ``created``/``started``/``finished`` -(Unix timestamps, useful for judging how long a job has been running when -you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, -``execution_id``, ``error``. ``--json`` never prints prose to -stderr (or anywhere but that one stdout document); the failure reason on a -``"FAILED"`` job is the document's ``error`` field, not a separate message. +finished on ``"COMPLETED"``, when it started — or, if it hasn't yet, when it +was created — on ``"RUNNING"``, or — on ``"FAILED"`` — when it finished plus +the error as a separate advisory on stderr (so it isn't printed twice). +``--json``: the job as a single document — ``id``, ``status``, +``created``/``started``/``finished`` (ISO 8601 timestamps, useful for judging +how long a job has been running when you've resumed polling one from +elsewhere), ``accessions``, ``sample_ids``, ``execution_id``, ``error``. +``--json`` never prints prose to stderr (or anywhere but that one stdout +document); the failure reason on a ``"FAILED"`` job is the document's +``error`` field, not a separate message. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the @@ -478,7 +477,7 @@ mapping above. Job 42: COMPLETED (finished 2024-04-05 19:38:20 UTC). Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json - {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} + {"id": 42, "status": "COMPLETED", "created": "2024-04-05T19:34:38Z", "started": "2024-04-05T19:34:40Z", "finished": "2024-04-05T19:38:20Z", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 2ccd8ec..193b1be 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -5,8 +5,9 @@ from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError +from flowbio.v2.samples import SampleImportSpec, SampleTypeId -HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] +HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] def _write_sheet( @@ -22,13 +23,6 @@ def _write_sheet( class TestParseAccessionSheet: - def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), - ) - - assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] - def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( _write_sheet(tmp_path, {"accession": "err1160845"}), @@ -61,6 +55,20 @@ def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None assert sheet.rows[0].name == "liver_r1" assert sheet.rows[0].organism == "Hs" + def test_sample_type_column_is_optional(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.rows[0].sample_type is None + + def test_sample_type_column_is_parsed_when_present(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1160845", "sample_type": "chip_seq"}, + )) + + assert sheet.rows[0].sample_type == "chip_seq" + def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: path = tmp_path / "sheet.csv" with path.open("w", newline="", encoding="utf-8-sig") as handle: @@ -70,7 +78,6 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(path) - assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] assert sheet.rows[0].accession == "ERR1160845" def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: @@ -99,3 +106,70 @@ def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError): parse_accession_sheet(tmp_path / "absent.csv") + + def test_header_only_sheet_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="no rows"): + parse_accession_sheet(_write_sheet(tmp_path)) + + def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="row"): + parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845"}, + {"accession": ""}, + )) + + def test_row_with_no_accession_column_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer.writeheader() + writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + + with pytest.raises(CliUsageError): + parse_accession_sheet(path) + + def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="1.*3"): + parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": ""}, + {"accession": "ERR1"}, + {"accession": ""}, + )) + + +class TestAccessionSheetRowToSpec: + + def _row(self, tmp_path: Path, **overrides: str): + record = {"accession": "ERR1160845"} + record.update(overrides) + sheet = parse_accession_sheet(_write_sheet(tmp_path, record)) + return sheet.rows[0] + + def test_uses_default_sample_type_when_row_has_none(self, tmp_path: Path) -> None: + row = self._row(tmp_path) + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec == SampleImportSpec(accession="ERR1160845", sample_type="rna_seq") + + def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: + row = self._row(tmp_path, sample_type="chip_seq") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec.sample_type == "chip_seq" + + def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: + row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec == SampleImportSpec( + accession="ERR1160845", + sample_type="rna_seq", + name="liver_r1", + organism_id="Hs", + metadata={"cell_type": "Neuron"}, + ) diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5fe07d8..4b26f71 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -926,7 +926,7 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "CSV" in result.stderr -IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] +IMPORT_HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: @@ -1008,14 +1008,13 @@ def test_json_document_reports_job_fields( assert document == { "id": 42, "status": "RUNNING", - "created": 1700000000, + "created": "2023-11-14T22:13:20Z", "started": None, "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": 7, "error": None, - "skipped": [], } @respx.mock @@ -1145,16 +1144,18 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert str(sheet) in result.stderr @respx.mock - def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: + def test_row_sample_type_overrides_the_default( + self, run_cli, tmp_path: Path, + ) -> None: route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], + 1, "RUNNING", ["ERR1", "ERR2"], )), ) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2"}, ) result = run_cli( @@ -1164,67 +1165,38 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: assert result.exit_code == 0 payload = json.loads(route.calls[0].request.content) - assert len(payload["imports"]) == 1 - assert payload["imports"][0]["accession"] == "ERR1" - assert "Skipped row 2" in result.stderr + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "rna_seq"] @respx.mock - def test_blank_accession_row_is_reported_in_json_document( + def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, - ) + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--sheet", str(sheet), "--sample-type", "rna_seq", ) - assert result.exit_code == 0 - assert result.stderr == "" - document = json.loads(result.stdout) - assert document["skipped"] == [ - {"row_number": 2, "name": None, "reasons": ["no accession"]}, - ] + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr @respx.mock - def test_blank_accession_row_with_name_is_identified_by_name( + def test_mixed_valid_and_blank_accession_rows_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) + # A blank accession is rejected outright rather than silently + # skipped, even when the rest of the sheet is otherwise fine. + route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, {"accession": "ERR1"}, - {"accession": "", "name": "liver_r2"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + {"accession": ""}, ) - assert result.exit_code == 0 - assert "Skipped row 2 (liver_r2)" in result.stderr - - @respx.mock - def test_sheet_with_only_blank_accessions_is_usage_error( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) - result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", @@ -1232,7 +1204,6 @@ def test_sheet_with_only_blank_accessions_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr @respx.mock def test_sheet_with_no_accession_column_is_usage_error( @@ -1291,7 +1262,7 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert "started 2023-11-14" in result.stdout @respx.mock - def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: + def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", "created": 1700000000, @@ -1305,7 +1276,24 @@ def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: ) assert result.exit_code == 0 - assert "started 1700000000000" in result.stdout + assert "started 2023-11-14" in result.stdout + + @respx.mock + def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": None, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "created 2023-11-14" in result.stdout @respx.mock def test_completed_job_reports_when_it_finished(self, run_cli) -> None: @@ -1414,9 +1402,9 @@ def test_json_document_matches_job_shape(self, run_cli) -> None: assert document == { "id": 42, "status": "COMPLETED", - "created": 1700000000, - "started": 1700000001, - "finished": 1700000002, + "created": "2023-11-14T22:13:20Z", + "started": "2023-11-14T22:13:21Z", + "finished": "2023-11-14T22:13:22Z", "accessions": ["ERR1"], "sample_ids": [101], "execution_id": 7, From e1fde614e4091ccdc27b4d2e109d7c75858e198a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:30:52 +0000 Subject: [PATCH 13/19] fix(samples): address round-12 Claude review findings - _timestamp_suffix now converts aware datetimes to UTC and treats naive ones as UTC explicitly, instead of labelling wall-clock fields "UTC" unconditionally (a real bug once created/started/finished became datetime instead of epoch int). - AccessionSheetRow now enforces non-empty accession by construction (__post_init__), and parse_accession_sheet no longer needs an "or ''" sentinel to satisfy typing before its own precondition check. - Missing-accession error message says "data row(s)" and gets its grammar right for the singular case; cli.rst documents the 1-based data-row numbering convention. - SampleImportJob.execution_id/error now default to None, matching the three timestamp fields, so a response that omits them doesn't raise an uncatchable ValidationError. - Removed a test that only pinned pydantic's own millisecond-timestamp auto-detection rather than any behaviour of this client. - cli.rst's samples import --json example shows execution_id as null (a freshly-kicked-off job has no pipeline execution yet), and notes sample_type is a reserved column name. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 51 +++++++++++++++++++++------------ flowbio/cli/_samples.py | 15 ++++++++-- flowbio/v2/samples.py | 8 ++++-- source/cli.rst | 18 +++++++----- tests/unit/cli/test_samples.py | 23 +++++++++++++-- tests/unit/v2/test_samples.py | 17 +++++++++++ 6 files changed, 99 insertions(+), 33 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 021a6d7..f29ec44 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -43,6 +43,10 @@ class AccessionSheetRow: sample_type: SampleTypeId | None metadata: dict[str, str] + def __post_init__(self) -> None: + if not self.accession: + raise ValueError("accession must not be empty") + def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. @@ -92,39 +96,50 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] - rows = [ - _build_row(record, row_number, metadata_columns) - for row_number, record in enumerate(reader, start=1) - ] - if not rows: + records = list(enumerate(reader, start=1)) + if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") - missing = [row.row_number for row in rows if not row.accession] + missing = [ + row_number for row_number, record in records if not _cell(record, "accession") + ] if missing: + numbers = ", ".join(str(number) for number in missing) + # Data row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + verb = "has" if len(missing) == 1 else "have" raise CliUsageError( - f"Accession sheet row(s) {', '.join(str(number) for number in missing)} " - f"have no accession: {path}.", + f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", ) + # The walrus filter is a no-op here: parse_accession_sheet already raised + # above if any record lacked an accession. Filtering on it (rather than + # asserting) is what gives the accession its narrowed str type below. + rows = [ + _build_row(record, row_number, metadata_columns, accession) + for row_number, record in records + if (accession := _cell(record, "accession")) is not None + ] return AccessionSheet(path=path, rows=rows) +def _cell(record: dict[str, str], column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + def _build_row( - record: dict[str, str], row_number: int, metadata_columns: list[str], + record: dict[str, str], row_number: int, metadata_columns: list[str], accession: str, ) -> AccessionSheetRow: - def cell(column: str) -> str | None: - value = (record.get(column) or "").strip() - return value or None - metadata = { column: value for column in metadata_columns - if (value := cell(column)) is not None + if (value := _cell(record, column)) is not None } - sample_type = cell("sample_type") + sample_type = _cell(record, "sample_type") return AccessionSheetRow( row_number=row_number, - accession=cell("accession") or "", - name=cell("name"), - organism=cell("organism"), + accession=accession, + name=_cell(record, "name"), + organism=_cell(record, "organism"), sample_type=SampleTypeId(sample_type) if sample_type is not None else None, metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 86c97ed..ebe72c0 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,7 +11,7 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Literal @@ -676,7 +676,8 @@ def _import_status_command( def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" - return f"Job {job.id}: COMPLETED{_timestamp_suffix('finished', job.finished)}. Sample ids: {ids}." + suffix = _timestamp_suffix("finished", job.finished) + return f"Job {job.id}: COMPLETED{suffix}. Sample ids: {ids}." if job.status == "FAILED": # The error, if any, is on stderr as an advisory (see # _import_status_command) rather than repeated here, so a human @@ -689,7 +690,15 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" + # A naive value (no tzinfo) is treated as already UTC rather than + # converted with astimezone(), which would assume the *local* system + # timezone instead. + utc_timestamp = ( + timestamp.astimezone(timezone.utc) + if timestamp.tzinfo is not None + else timestamp.replace(tzinfo=timezone.utc) + ) + return f" ({label} {utc_timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 27fb79e..be217c9 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -205,8 +205,12 @@ class SampleImportJob(BaseModel, frozen=True): sample_ids: list[int] = Field( description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) - execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") - error: str | None = Field(description='The failure reason, set only when status is "FAILED".') + execution_id: int | None = Field( + default=None, description="The pipeline execution backing this job, if one was created.", + ) + error: str | None = Field( + default=None, description='The failure reason, set only when status is "FAILED".', + ) class SampleResource: diff --git a/source/cli.rst b/source/cli.rst index 4b5172f..97d8bb0 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -382,14 +382,18 @@ Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with a required ``accession`` column, plus optional ``name``/ ``organism``/``sample_type`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project -field). ``name`` defaults to the accession when omitted. A row's own -``sample_type`` column, if present, overrides ``--sample-type`` for that row -only — useful for a mixed-type sheet. The sample type, accession format, and -metadata rules are all sent as-is and validated **server-side**; this command -only checks that the sheet is a readable ``.csv`` and that every row has an +field; note ``sample_type`` is reserved for the per-row override, so a +metadata attribute of that exact name can't be sent through the sheet). +``name`` defaults to the accession when omitted. A row's own ``sample_type`` +column, if present, overrides ``--sample-type`` for that row only — useful +for a mixed-type sheet. The sample type, accession format, and metadata +rules are all sent as-is and validated **server-side**; this command only +checks that the sheet is a readable ``.csv`` and that every row has an accession — that column is the one thing every row must have to mean anything, so a blank cell rejects the whole sheet up front rather than -shipping an empty string the server would just reject anyway. +shipping an empty string the server would just reject anyway. Rows are +counted from ``1`` for the first data row, after the header (the same +convention as ``upload-batch``'s ``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -420,7 +424,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": null, "error": null} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4b26f71..5a7cfa6 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1262,11 +1262,11 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert "started 2023-11-14" in result.stdout @respx.mock - def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: + def test_started_with_non_utc_offset_is_reported_in_utc(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", "created": 1700000000, - "started": 1700000000000, "finished": None, + "started": "2024-04-05T19:34:38+02:00", "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1276,7 +1276,24 @@ def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: ) assert result.exit_code == 0 - assert "started 2023-11-14" in result.stdout + assert "started 2024-04-05 17:34:38 UTC" in result.stdout + + @respx.mock + def test_started_with_no_offset_is_treated_as_utc(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 2024-04-05 19:34:38 UTC" in result.stdout @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 7d8a99b..3f30458 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1265,3 +1265,20 @@ def test_parses_job_missing_timestamps(self) -> None: assert result.created is None assert result.started is None assert result.finished is None + + @respx.mock + def test_parses_job_missing_execution_id_and_error(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.execution_id is None + assert result.error is None From 3c34f87d054ad2b93844301819234d6668668b6d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:41:55 +0000 Subject: [PATCH 14/19] fix(samples): address round-13 Claude review findings - SampleImportJob.accessions/sample_ids now default_factory=list, joining the timestamps/execution_id/error group as optional, since a POST kickoff response for a job that has created nothing yet plausibly omits them. Added a test parsing a payload of just {id, status} that covers all seven defaulted fields at once. - parse_accession_sheet is now a single pass collecting missing-accession row numbers and building rows together, replacing the two-pass walrus-filter version and the comment explaining why its filter could never exclude anything (a CLAUDE.md-flagged implementation-flow comment). - --sample-type is now optional: it's only required when some row has no sample_type column of its own. A sheet where every row specifies its own type no longer needs a --sample-type value that's silently discarded. AccessionSheetRow.to_spec documents and raises ValueError for the (CLI-unreachable) case of neither being available. - Added a test constructing AccessionSheetRow(accession="") directly, pinning the __post_init__ invariant independently of the parser. - Added a test for a blank sample_type cell (column present, empty) falling back to the default, distinct from the column being absent. - The library polling example in samples.rst now distinguishes FAILED from a timed-out RUNNING job instead of printing job.error (None) for both. - Fixed parse_accession_sheet's :returns: docstring, which claimed empty cells are dropped for accession too. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 37 ++++++++++++---------- flowbio/cli/_samples.py | 20 +++++++++--- flowbio/v2/samples.py | 5 ++- source/cli.rst | 31 ++++++++++--------- source/v2/samples.rst | 4 ++- tests/unit/cli/test_accession_sheet.py | 27 +++++++++++++++- tests/unit/cli/test_samples.py | 43 ++++++++++++++++++++++++++ tests/unit/v2/test_samples.py | 23 ++------------ 8 files changed, 133 insertions(+), 57 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index f29ec44..135e6e6 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -47,15 +47,22 @@ def __post_init__(self) -> None: if not self.accession: raise ValueError("accession must not be empty") - def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: + def to_spec(self, default_sample_type: SampleTypeId | None) -> SampleImportSpec: """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. :param default_sample_type: The sample type to use when this row has no ``sample_type`` of its own. + :raises ValueError: If this row has no ``sample_type`` of its own and + ``default_sample_type`` is ``None``. """ + sample_type = self.sample_type or default_sample_type + if sample_type is None: + raise ValueError( + f"row {self.row_number} has no sample_type and no default was given", + ) return SampleImportSpec( accession=self.accession, - sample_type=self.sample_type or default_sample_type, + sample_type=sample_type, name=self.name, organism_id=self.organism, metadata=self.metadata or None, @@ -75,9 +82,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped. Values are otherwise - passed through unchanged — including ``accession``, sent to the - server exactly as entered. + :returns: The parsed sheet, with empty cells dropped (other than + ``accession``, which rejects the sheet instead — see below). Values + are otherwise passed through unchanged, including ``accession``, sent + to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession. """ @@ -99,9 +107,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: records = list(enumerate(reader, start=1)) if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") - missing = [ - row_number for row_number, record in records if not _cell(record, "accession") - ] + rows: list[AccessionSheetRow] = [] + missing: list[int] = [] + for row_number, record in records: + accession = _cell(record, "accession") + if accession is None: + missing.append(row_number) + else: + rows.append(_build_row(record, row_number, metadata_columns, accession)) if missing: numbers = ", ".join(str(number) for number in missing) # Data row 1 is the first row after the header, matching _sheet.py's @@ -110,14 +123,6 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: raise CliUsageError( f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", ) - # The walrus filter is a no-op here: parse_accession_sheet already raised - # above if any record lacked an accession. Filtering on it (rather than - # asserting) is what gives the accession its narrowed str type below. - rows = [ - _build_row(record, row_number, metadata_columns, accession) - for row_number, record in records - if (accession := _cell(record, "accession")) is not None - ] return AccessionSheet(path=path, rows=rows) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ebe72c0..27af796 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -32,6 +32,7 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, + SampleImportSpec, SampleTypeId, ) @@ -270,12 +271,11 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--sample-type", - required=True, metavar="TYPE", type=SampleTypeId, help=( "Default sample type (sent as-is; validated server-side), used for any " - "row without its own sample_type column." + "row without its own sample_type column. Required unless every row has one." ), ) @@ -639,10 +639,22 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has no - rows, or has a row with no accession. + rows, has a row with no accession, or has a row with no + ``sample_type`` of its own and no ``--sample-type`` given. """ sheet = parse_accession_sheet(args.sheet) - specs = [row.to_spec(args.sample_type) for row in sheet.rows] + missing_type = [ + row.row_number for row in sheet.rows + if row.sample_type is None and args.sample_type is None + ] + if missing_type: + numbers = ", ".join(str(number) for number in missing_type) + verb = "has" if len(missing_type) == 1 else "have" + raise CliUsageError( + f"Accession sheet data row(s) {numbers} {verb} no sample_type and " + f"--sample-type was not given: {args.sheet}.", + ) + specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index be217c9..77a0044 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -201,8 +201,11 @@ class SampleImportJob(BaseModel, frozen=True): finished: datetime | None = Field( default=None, description="When the job finished (completed or failed), if it has.", ) - accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") + accessions: list[str] = Field( + default_factory=list, description="The accessions submitted with this job, in submission order.", + ) sample_ids: list[int] = Field( + default_factory=list, description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) execution_id: int | None = Field( diff --git a/source/cli.rst b/source/cli.rst index 97d8bb0..4f276e6 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -376,7 +376,7 @@ to every row — no files to upload yourself. :: - flowbio samples import --sheet PATH --sample-type TYPE + flowbio samples import --sheet PATH [--sample-type TYPE] Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with a required ``accession`` column, plus optional ``name``/ @@ -386,14 +386,16 @@ field; note ``sample_type`` is reserved for the per-row override, so a metadata attribute of that exact name can't be sent through the sheet). ``name`` defaults to the accession when omitted. A row's own ``sample_type`` column, if present, overrides ``--sample-type`` for that row only — useful -for a mixed-type sheet. The sample type, accession format, and metadata -rules are all sent as-is and validated **server-side**; this command only -checks that the sheet is a readable ``.csv`` and that every row has an -accession — that column is the one thing every row must have to mean -anything, so a blank cell rejects the whole sheet up front rather than -shipping an empty string the server would just reject anyway. Rows are -counted from ``1`` for the first data row, after the header (the same -convention as ``upload-batch``'s ``row_number``). +for a mixed-type sheet; ``--sample-type`` itself is only required if some +row has no ``sample_type`` column of its own. The sample type, accession +format, and metadata rules are all sent as-is and validated **server-side**; +this command only checks that the sheet is a readable ``.csv``, that every +row has an accession, and that every row can resolve a sample type (its own +column, or ``--sample-type``) — a blank ``accession`` cell or a row with +neither rejects the whole sheet up front rather than shipping something the +server would just reject anyway. Rows are counted from ``1`` for the first +data row, after the header (the same convention as ``upload-batch``'s +``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -410,11 +412,12 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has no rows, or has a row with no accession; ``1`` the API -rejected the batch (e.g. unknown sample type, missing required metadata, an -unsupported accession format — these come back as an HTTP ``422``; ``5`` in -the unlikely case it answers ``400`` instead); ``3`` authentication failure; -otherwise the standard mapping above. +readable ``.csv``, has no rows, has a row with no accession, or has a row +with no resolvable sample type; ``1`` the API rejected the batch (e.g. +unknown sample type, missing required metadata, an unsupported accession +format — these come back as an HTTP ``422``; ``5`` in the unlikely case it +answers ``400`` instead); ``3`` authentication failure; otherwise the +standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 5cfee62..3ebeafd 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -200,8 +200,10 @@ leaves ``"RUNNING"`` — bound the wait so a stuck job doesn't loop forever:: if job.status == "COMPLETED": print(f"Imported samples: {job.sample_ids}") + elif job.status == "FAILED": + print(f"Import failed: {job.error}") else: - print(f"Import failed or still running: {job.error}") + print(f"Import did not finish within the deadline (status: {job.status})") ``job.accessions`` and ``job.sample_ids`` correspond positionally once the job has completed. diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 193b1be..aa8f9fc 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -3,7 +3,7 @@ import pytest -from flowbio.cli._accession_sheet import parse_accession_sheet +from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError from flowbio.v2.samples import SampleImportSpec, SampleTypeId @@ -139,6 +139,18 @@ def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) )) +def test_row_rejects_empty_accession_by_construction() -> None: + with pytest.raises(ValueError, match="accession"): + AccessionSheetRow( + row_number=1, + accession="", + name=None, + organism=None, + sample_type=None, + metadata={}, + ) + + class TestAccessionSheetRowToSpec: def _row(self, tmp_path: Path, **overrides: str): @@ -161,6 +173,19 @@ def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: assert spec.sample_type == "chip_seq" + def test_blank_sample_type_cell_falls_back_to_default(self, tmp_path: Path) -> None: + row = self._row(tmp_path, sample_type="") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec.sample_type == "rna_seq" + + def test_raises_when_no_sample_type_or_default_available(self, tmp_path: Path) -> None: + row = self._row(tmp_path) + + with pytest.raises(ValueError, match="sample_type"): + row.to_spec(None) + def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5a7cfa6..3d646d7 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1168,6 +1168,49 @@ def test_row_sample_type_overrides_the_default( sample_types = [entry["sample_type"] for entry in payload["imports"]] assert sample_types == ["chip_seq", "rna_seq"] + @respx.mock + def test_sample_type_flag_is_optional_when_every_row_has_its_own( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2", "sample_type": "atac_seq"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "atac_seq"] + + @respx.mock + def test_row_missing_sample_type_without_default_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "2" in result.stderr + @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 3f30458..65fb582 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1247,15 +1247,11 @@ def test_raises_not_found_for_unknown_job(self) -> None: client.samples.get_import(SampleImportJobId(999)) @respx.mock - def test_parses_job_missing_timestamps(self) -> None: + def test_parses_job_with_only_id_and_status(self) -> None: respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", - "accessions": ["ERR1"], - "sample_ids": [], - "execution_id": None, - "error": None, }), ) @@ -1265,20 +1261,7 @@ def test_parses_job_missing_timestamps(self) -> None: assert result.created is None assert result.started is None assert result.finished is None - - @respx.mock - def test_parses_job_missing_execution_id_and_error(self) -> None: - respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( - return_value=httpx.Response(HTTPStatus.OK, json={ - "id": 42, - "status": "RUNNING", - "accessions": ["ERR1"], - "sample_ids": [], - }), - ) - - client = Client() - result = client.samples.get_import(SampleImportJobId(42)) - + assert result.accessions == [] + assert result.sample_ids == [] assert result.execution_id is None assert result.error is None From 84465685c95de78688ea7864798ea7a63dce03a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:50:18 +0000 Subject: [PATCH 15/19] fix(samples): address round-14 Claude review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SampleImportJob's created/started/finished now go through an after-validator that assumes UTC for a naive value, so the rule holds for --json and library callers, not just the CLI's human renderer. _timestamp_suffix simplifies to astimezone(timezone.utc) since every timestamp reaching it is now guaranteed aware. - Fixed cli.rst's samples import summary sentence, which still said "applying one sample type to every row" after the per-row override landed two rounds ago. - Hoisted the args.sample_type is None check out of the missing_type comprehension in _import_command, so it's stated once instead of re-evaluated (and short-circuited) per row. - Fixed two tests: the incidental-digit assertion in test_row_missing_sample_type_without_default_is_usage_error now checks the actual message phrase, and test_missing_sample_type_is_usage_error gained the @respx.mock/ call-count assertion its siblings have. - Documented in cli.rst that sheet-level usage errors for 'samples import' are only detected after authentication, since whether --sample-type is required depends on the sheet's contents. Left as documented trade-offs (per the review's own framing): the SampleImportSpec-frozen-dataclass-with-a-dict-field hashability wrinkle, and the RNA-Seq/rna_seq casing convention mismatch between docs and tests — both cosmetic/informational rather than defects. Linear: FLOW-689 --- flowbio/cli/_samples.py | 32 ++++++++++++-------------------- flowbio/v2/samples.py | 15 +++++++++++++-- source/cli.rst | 9 ++++++--- tests/unit/cli/test_samples.py | 23 ++++++++++++++++++++++- tests/unit/v2/test_samples.py | 16 ++++++++++++++++ 5 files changed, 69 insertions(+), 26 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 27af796..f64224b 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -643,17 +643,15 @@ def _import_command( ``sample_type`` of its own and no ``--sample-type`` given. """ sheet = parse_accession_sheet(args.sheet) - missing_type = [ - row.row_number for row in sheet.rows - if row.sample_type is None and args.sample_type is None - ] - if missing_type: - numbers = ", ".join(str(number) for number in missing_type) - verb = "has" if len(missing_type) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no sample_type and " - f"--sample-type was not given: {args.sheet}.", - ) + if args.sample_type is None: + missing_type = [row.row_number for row in sheet.rows if row.sample_type is None] + if missing_type: + numbers = ", ".join(str(number) for number in missing_type) + verb = "has" if len(missing_type) == 1 else "have" + raise CliUsageError( + f"Accession sheet data row(s) {numbers} {verb} no sample_type and " + f"--sample-type was not given: {args.sheet}.", + ) specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( @@ -702,15 +700,9 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - # A naive value (no tzinfo) is treated as already UTC rather than - # converted with astimezone(), which would assume the *local* system - # timezone instead. - utc_timestamp = ( - timestamp.astimezone(timezone.utc) - if timestamp.tzinfo is not None - else timestamp.replace(tzinfo=timezone.utc) - ) - return f" ({label} {utc_timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" + # SampleImportJob normalises a naive value to UTC, so every timestamp + # reaching here is already aware. + return f" ({label} {timestamp.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 77a0044..4f4dcfc 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -28,11 +28,11 @@ from collections.abc import Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Literal, NewType -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from flowbio.v2._pagination import PageIterator from flowbio.v2.exceptions import ( @@ -215,6 +215,17 @@ class SampleImportJob(BaseModel, frozen=True): default=None, description='The failure reason, set only when status is "FAILED".', ) + @field_validator("created", "started", "finished", mode="after") + @classmethod + def _assume_utc_if_naive(cls, value: datetime | None) -> datetime | None: + # A naive value (no tzinfo) is treated as already UTC, matching what + # the server always means by these timestamps, rather than left + # ambiguous for every consumer (CLI, --json, library) to decide on + # its own. + if value is not None and value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + class SampleResource: """Provides access to sample-related API endpoints. diff --git a/source/cli.rst b/source/cli.rst index 4f276e6..1a3db40 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -371,8 +371,8 @@ authentication failure; otherwise the standard mapping above. ~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ -ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type -to every row — no files to upload yourself. +ERR/DRR run or SRX/ERX/DRX experiment accessions), applying a default sample +type that each row can override — no files to upload yourself. :: @@ -417,7 +417,10 @@ with no resolvable sample type; ``1`` the API rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the -standard mapping above. +standard mapping above. Because whether ``--sample-type`` is required +depends on the sheet's contents, these sheet-level usage errors (``2``) are +only detected after authentication succeeds, unlike a malformed flag value +(which fails before it). **Example** diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d646d7..451accf 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1120,7 +1120,9 @@ def test_missing_sheet_is_usage_error(self, run_cli) -> None: assert result.exit_code == 2 + @respx.mock def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( @@ -1128,6 +1130,7 @@ def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> No ) assert result.exit_code == 2 + assert route.call_count == 0 @respx.mock def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1209,7 +1212,7 @@ def test_row_missing_sample_type_without_default_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "2" in result.stderr + assert "data row(s) 2" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1338,6 +1341,24 @@ def test_started_with_no_offset_is_treated_as_utc(self, run_cli) -> None: assert result.exit_code == 0 assert "started 2024-04-05 19:34:38 UTC" in result.stdout + @respx.mock + def test_naive_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["started"] == "2024-04-05T19:34:38Z" + @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 65fb582..59cade8 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timezone from http import HTTPStatus from pathlib import Path from unittest.mock import ANY, patch @@ -1265,3 +1266,18 @@ def test_parses_job_with_only_id_and_status(self) -> None: assert result.sample_ids == [] assert result.execution_id is None assert result.error is None + + @respx.mock + def test_naive_timestamp_is_treated_as_utc(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "started": "2024-04-05T19:34:38", + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.started == datetime(2024, 4, 5, 19, 34, 38, tzinfo=timezone.utc) From 5e1c82b4e7c553b857398f236988e3d0325419c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:09:33 +0000 Subject: [PATCH 16/19] feat(samples): make sample_type an accession-sheet-only concept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction: eliminated the possibility of supplying a sample type anywhere but the accession sheet's own column for 'samples import'. - Removed the --sample-type flag from 'samples import' entirely. - AccessionSheetRow.sample_type is now a required SampleTypeId (was optional with a CLI-supplied default). parse_accession_sheet rejects a sheet with any row missing accession or sample_type, naming every offending row for each column independently, the same way it already did for accession alone. - AccessionSheetRow.to_spec() takes no arguments now that there is nothing to resolve a default against. - This fully resolves round-15 finding 1 (an empty --sample-type value slipping past the required check) by removing the flag it depended on, rather than patching the identity/truthiness mismatch. Also fixed two more round-15 findings while in here: - The naive-timestamp UTC validator on SampleImportJob now also converts an already-aware non-UTC offset to UTC (astimezone), not just naive values — so --json and library callers get a uniformly UTC-normalised value, not merely a tz-aware one. Added a --json test for a +02:00 server timestamp. - Tightened two sheet-parsing tests whose match= patterns were satisfiable by digits/words in pytest's tmp_path rather than the actual error message. Left round-15 finding 2 (the missing-column message enumerating every row) as-is per operator direction — it errors, which is sufficient. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 98 +++++++------ flowbio/cli/_samples.py | 27 +--- flowbio/v2/samples.py | 8 +- source/cli.rst | 56 ++++---- tests/unit/cli/test_accession_sheet.py | 111 +++++++-------- tests/unit/cli/test_samples.py | 184 +++++++++++-------------- 6 files changed, 219 insertions(+), 265 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 135e6e6..0ed3e6a 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,18 +1,19 @@ """CSV accession-sheet parsing for ``samples import``. -An accession sheet is a CSV with one row per accession to import: a required -``accession`` column (a public-repository run or experiment accession) plus -optional ``name``/``organism``/``sample_type`` and per-accession metadata -columns. This mirrors ``_sheet.py``'s reads-based sample sheet, but the -reserved columns differ — there is nothing to upload (no ``reads1``/ -``reads2``) and the import API has no project field. +An accession sheet is a CSV with one row per accession to import: required +``accession`` and ``sample_type`` columns, plus optional ``name``/ +``organism`` and per-accession metadata columns. This mirrors ``_sheet.py``'s +reads-based sample sheet, but the reserved columns differ — there is nothing +to upload (no ``reads1``/``reads2``) and the import API has no project field. Domain rules (accession format, duplicates, sample type, organism, metadata) are all checked server-side when the sheet is submitted — duplicating that -locally would just be a second, driftable copy of the same rules. An -accession is different: it is the one column every row must have to mean -anything at all, so a missing one is rejected here rather than silently -skipped or shipped as an empty string the server would just reject anyway. +locally would just be a second, driftable copy of the same rules. +``accession`` and ``sample_type`` are different: they are the two columns +every row must have to mean anything at all, so a missing one is rejected +here rather than silently skipped or shipped as an empty string the server +would just reject anyway. There is deliberately no other way to supply a +sample type for ``samples import`` — the sheet is the single source of it. """ from __future__ import annotations @@ -29,40 +30,26 @@ @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet. - - ``sample_type`` is only set when the sheet has its own ``sample_type`` - column for this row; :meth:`to_spec` falls back to the batch's - ``--sample-type`` when it's absent. - """ + """One data row of an accession sheet.""" row_number: int accession: str name: str | None organism: str | None - sample_type: SampleTypeId | None + sample_type: SampleTypeId metadata: dict[str, str] def __post_init__(self) -> None: if not self.accession: raise ValueError("accession must not be empty") + if not self.sample_type: + raise ValueError("sample_type must not be empty") - def to_spec(self, default_sample_type: SampleTypeId | None) -> SampleImportSpec: - """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. - - :param default_sample_type: The sample type to use when this row has - no ``sample_type`` of its own. - :raises ValueError: If this row has no ``sample_type`` of its own and - ``default_sample_type`` is ``None``. - """ - sample_type = self.sample_type or default_sample_type - if sample_type is None: - raise ValueError( - f"row {self.row_number} has no sample_type and no default was given", - ) + def to_spec(self) -> SampleImportSpec: + """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row.""" return SampleImportSpec( accession=self.accession, - sample_type=sample_type, + sample_type=self.sample_type, name=self.name, organism_id=self.organism, metadata=self.metadata or None, @@ -83,11 +70,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. :returns: The parsed sheet, with empty cells dropped (other than - ``accession``, which rejects the sheet instead — see below). Values - are otherwise passed through unchanged, including ``accession``, sent - to the server exactly as entered. + ``accession``/``sample_type``, which reject the sheet instead — see + below). Values are otherwise passed through unchanged, including + ``accession``, sent to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no - rows, or has a row with no accession. + rows, or has a row with no accession or no sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -108,43 +95,54 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") rows: list[AccessionSheetRow] = [] - missing: list[int] = [] + missing_accession: list[int] = [] + missing_sample_type: list[int] = [] for row_number, record in records: accession = _cell(record, "accession") + sample_type = _cell(record, "sample_type") if accession is None: - missing.append(row_number) - else: - rows.append(_build_row(record, row_number, metadata_columns, accession)) - if missing: - numbers = ", ".join(str(number) for number in missing) - # Data row 1 is the first row after the header, matching _sheet.py's - # convention (and upload-batch's documented "1-based row number"). - verb = "has" if len(missing) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", - ) + missing_accession.append(row_number) + if sample_type is None: + missing_sample_type.append(row_number) + if accession is not None and sample_type is not None: + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if missing_accession: + raise _missing_column_error("accession", missing_accession, path) + if missing_sample_type: + raise _missing_column_error("sample_type", missing_sample_type, path) return AccessionSheet(path=path, rows=rows) +def _missing_column_error(column: str, missing: list[int], path: Path) -> CliUsageError: + numbers = ", ".join(str(number) for number in missing) + # Data row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + verb = "has" if len(missing) == 1 else "have" + return CliUsageError(f"Accession sheet data row(s) {numbers} {verb} no {column}: {path}.") + + def _cell(record: dict[str, str], column: str) -> str | None: value = (record.get(column) or "").strip() return value or None def _build_row( - record: dict[str, str], row_number: int, metadata_columns: list[str], accession: str, + record: dict[str, str], + row_number: int, + metadata_columns: list[str], + accession: str, + sample_type: str, ) -> AccessionSheetRow: metadata = { column: value for column in metadata_columns if (value := _cell(record, column)) is not None } - sample_type = _cell(record, "sample_type") return AccessionSheetRow( row_number=row_number, accession=accession, name=_cell(record, "name"), organism=_cell(record, "organism"), - sample_type=SampleTypeId(sample_type) if sample_type is not None else None, + sample_type=SampleTypeId(sample_type), metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index f64224b..5904509 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -265,17 +265,8 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: metavar="PATH", type=Path, help=( - "CSV accession sheet (required accession column, optional " - "name/organism/sample_type, plus metadata columns)." - ), - ) - import_parser.add_argument( - "--sample-type", - metavar="TYPE", - type=SampleTypeId, - help=( - "Default sample type (sent as-is; validated server-side), used for any " - "row without its own sample_type column. Required unless every row has one." + "CSV accession sheet (required accession/sample_type columns, " + "optional name/organism, plus metadata columns)." ), ) @@ -639,20 +630,10 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has no - rows, has a row with no accession, or has a row with no - ``sample_type`` of its own and no ``--sample-type`` given. + rows, or has a row with no accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) - if args.sample_type is None: - missing_type = [row.row_number for row in sheet.rows if row.sample_type is None] - if missing_type: - numbers = ", ".join(str(number) for number in missing_type) - verb = "has" if len(missing_type) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no sample_type and " - f"--sample-type was not given: {args.sheet}.", - ) - specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] + specs: list[SampleImportSpec] = [row.to_spec() for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 4f4dcfc..26b48ea 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -217,14 +217,16 @@ class SampleImportJob(BaseModel, frozen=True): @field_validator("created", "started", "finished", mode="after") @classmethod - def _assume_utc_if_naive(cls, value: datetime | None) -> datetime | None: + def _normalize_to_utc(cls, value: datetime | None) -> datetime | None: + if value is None: + return None # A naive value (no tzinfo) is treated as already UTC, matching what # the server always means by these timestamps, rather than left # ambiguous for every consumer (CLI, --json, library) to decide on # its own. - if value is not None and value.tzinfo is None: + if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) - return value + return value.astimezone(timezone.utc) class SampleResource: diff --git a/source/cli.rst b/source/cli.rst index 1a3db40..b7bf4b5 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -371,31 +371,28 @@ authentication failure; otherwise the standard mapping above. ~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ -ERR/DRR run or SRX/ERX/DRX experiment accessions), applying a default sample -type that each row can override — no files to upload yourself. +ERR/DRR run or SRX/ERX/DRX experiment accessions) — no files to upload +yourself. :: - flowbio samples import --sheet PATH [--sample-type TYPE] + flowbio samples import --sheet PATH Run ``flowbio samples import --help`` for the full option list. The sheet is -a CSV with a required ``accession`` column, plus optional ``name``/ -``organism``/``sample_type`` and metadata columns (there is no -``batch-template`` equivalent for it, since it has no reads files or project -field; note ``sample_type`` is reserved for the per-row override, so a -metadata attribute of that exact name can't be sent through the sheet). -``name`` defaults to the accession when omitted. A row's own ``sample_type`` -column, if present, overrides ``--sample-type`` for that row only — useful -for a mixed-type sheet; ``--sample-type`` itself is only required if some -row has no ``sample_type`` column of its own. The sample type, accession -format, and metadata rules are all sent as-is and validated **server-side**; -this command only checks that the sheet is a readable ``.csv``, that every -row has an accession, and that every row can resolve a sample type (its own -column, or ``--sample-type``) — a blank ``accession`` cell or a row with -neither rejects the whole sheet up front rather than shipping something the -server would just reject anyway. Rows are counted from ``1`` for the first -data row, after the header (the same convention as ``upload-batch``'s -``row_number``). +a CSV with required ``accession``/``sample_type`` columns, plus optional +``name``/``organism`` and metadata columns (there is no ``batch-template`` +equivalent for it, since it has no reads files or project field). ``name`` +defaults to the accession when omitted. There is deliberately no +``--sample-type`` flag: the sheet's own column is the only way to supply a +sample type, so a mixed-type sheet needs no special handling and a +single-type sheet just repeats the same value down the column. The sample +type, accession format, and metadata rules are all sent as-is and validated +**server-side**; this command only checks that the sheet is a readable +``.csv`` and that every row has an accession and a sample type — a blank +cell in either column rejects the whole sheet up front rather than shipping +something the server would just reject anyway. Rows are counted from ``1`` +for the first data row, after the header (the same convention as +``upload-batch``'s ``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -412,24 +409,21 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has no rows, has a row with no accession, or has a row -with no resolvable sample type; ``1`` the API rejected the batch (e.g. -unknown sample type, missing required metadata, an unsupported accession -format — these come back as an HTTP ``422``; ``5`` in the unlikely case it -answers ``400`` instead); ``3`` authentication failure; otherwise the -standard mapping above. Because whether ``--sample-type`` is required -depends on the sheet's contents, these sheet-level usage errors (``2``) are -only detected after authentication succeeds, unlike a malformed flag value -(which fails before it). +readable ``.csv``, has no rows, or has a row with no accession or no +sample type; ``1`` the API rejected the batch (e.g. unknown sample type, +missing required metadata, an unsupported accession format — these come +back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` +instead); ``3`` authentication failure; otherwise the standard mapping +above. **Example** .. code-block:: bash - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + $ flowbio samples import --sheet ./accessions.csv Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + $ flowbio samples import --sheet ./accessions.csv --json {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": null, "error": null} ``samples import-status`` diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index aa8f9fc..0403c25 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -5,7 +5,7 @@ from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError -from flowbio.v2.samples import SampleImportSpec, SampleTypeId +from flowbio.v2.samples import SampleImportSpec HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] @@ -21,11 +21,17 @@ def _write_sheet( return path +def _record(**overrides: str) -> dict[str, str]: + record = {"accession": "ERR1160845", "sample_type": "rna_seq"} + record.update(overrides) + return record + + class TestParseAccessionSheet: def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "err1160845"}), + _write_sheet(tmp_path, _record(accession="err1160845")), ) assert sheet.rows[0].accession == "err1160845" @@ -33,14 +39,14 @@ def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845", "cell_type": "", "source": "blood"}, + _record(cell_type="", source="blood"), )) assert sheet.rows[0].metadata == {"source": "blood"} def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), + _write_sheet(tmp_path, _record()), ) assert sheet.rows[0].name is None @@ -49,22 +55,15 @@ def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845", "name": "liver_r1", "organism": "Hs"}, + _record(name="liver_r1", organism="Hs"), )) assert sheet.rows[0].name == "liver_r1" assert sheet.rows[0].organism == "Hs" - def test_sample_type_column_is_optional(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), - ) - - assert sheet.rows[0].sample_type is None - - def test_sample_type_column_is_parsed_when_present(self, tmp_path: Path) -> None: + def test_sample_type_is_parsed(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1160845", "sample_type": "chip_seq"}, + tmp_path, _record(sample_type="chip_seq"), )) assert sheet.rows[0].sample_type == "chip_seq" @@ -74,7 +73,7 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: with path.open("w", newline="", encoding="utf-8-sig") as handle: writer = csv.DictWriter(handle, fieldnames=HEADERS) writer.writeheader() - writer.writerow({"accession": "ERR1160845"}) + writer.writerow(_record()) sheet = parse_accession_sheet(path) @@ -83,8 +82,8 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845"}, - {"accession": "ERR10677146"}, + _record(accession="ERR1160845"), + _record(accession="ERR10677146"), )) assert [row.row_number for row in sheet.rows] == [1, 2] @@ -112,32 +111,46 @@ def test_header_only_sheet_is_usage_error(self, tmp_path: Path) -> None: parse_accession_sheet(_write_sheet(tmp_path)) def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: - with pytest.raises(CliUsageError, match="row"): + with pytest.raises(CliUsageError, match=r"data row\(s\) 2 has no accession"): parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845"}, - {"accession": ""}, + _record(accession="ERR1160845"), + _record(accession=""), )) def test_row_with_no_accession_column_is_usage_error(self, tmp_path: Path) -> None: path = tmp_path / "sheet.csv" with path.open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer = csv.DictWriter(handle, fieldnames=["run", "name", "sample_type"]) writer.writeheader() - writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + writer.writerow({"run": "ERR1160845", "name": "liver_r1", "sample_type": "rna_seq"}) with pytest.raises(CliUsageError): parse_accession_sheet(path) def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) -> None: - with pytest.raises(CliUsageError, match="1.*3"): + with pytest.raises(CliUsageError, match=r"data row\(s\) 1, 3 have no accession"): parse_accession_sheet(_write_sheet( tmp_path, - {"accession": ""}, - {"accession": "ERR1"}, - {"accession": ""}, + _record(accession=""), + _record(accession="ERR1"), + _record(accession=""), )) + def test_row_with_blank_sample_type_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has no sample_type"): + parse_accession_sheet(_write_sheet(tmp_path, _record(sample_type=""))) + + def test_row_with_no_sample_type_column_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["accession", "name"]) + writer.writeheader() + writer.writerow({"accession": "ERR1160845", "name": "liver_r1"}) + + with pytest.raises(CliUsageError): + parse_accession_sheet(path) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): @@ -146,7 +159,19 @@ def test_row_rejects_empty_accession_by_construction() -> None: accession="", name=None, organism=None, - sample_type=None, + sample_type="rna_seq", + metadata={}, + ) + + +def test_row_rejects_empty_sample_type_by_construction() -> None: + with pytest.raises(ValueError, match="sample_type"): + AccessionSheetRow( + row_number=1, + accession="ERR1160845", + name=None, + organism=None, + sample_type="", metadata={}, ) @@ -154,42 +179,20 @@ def test_row_rejects_empty_accession_by_construction() -> None: class TestAccessionSheetRowToSpec: def _row(self, tmp_path: Path, **overrides: str): - record = {"accession": "ERR1160845"} - record.update(overrides) - sheet = parse_accession_sheet(_write_sheet(tmp_path, record)) + sheet = parse_accession_sheet(_write_sheet(tmp_path, _record(**overrides))) return sheet.rows[0] - def test_uses_default_sample_type_when_row_has_none(self, tmp_path: Path) -> None: - row = self._row(tmp_path) - - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec == SampleImportSpec(accession="ERR1160845", sample_type="rna_seq") - - def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: + def test_uses_the_row_sample_type(self, tmp_path: Path) -> None: row = self._row(tmp_path, sample_type="chip_seq") - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec.sample_type == "chip_seq" - - def test_blank_sample_type_cell_falls_back_to_default(self, tmp_path: Path) -> None: - row = self._row(tmp_path, sample_type="") - - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec.sample_type == "rna_seq" - - def test_raises_when_no_sample_type_or_default_available(self, tmp_path: Path) -> None: - row = self._row(tmp_path) + spec = row.to_spec() - with pytest.raises(ValueError, match="sample_type"): - row.to_spec(None) + assert spec == SampleImportSpec(accession="ERR1160845", sample_type="chip_seq") def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") - spec = row.to_spec(SampleTypeId("rna_seq")) + spec = row.to_spec() assert spec == SampleImportSpec( accession="ERR1160845", diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 451accf..3d2df5f 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -938,6 +938,12 @@ def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: return path +def _import_record(**overrides: str) -> dict[str, str]: + record = {"accession": "ERR1", "sample_type": "rna_seq"} + record.update(overrides) + return record + + def _job_json( job_id: int, status: str, @@ -972,13 +978,12 @@ def test_kicks_off_job_and_reports_id_without_waiting( ) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Fibroblast"}, + _import_record(accession="ERR1", cell_type="Neuron"), + _import_record(accession="ERR2", cell_type="Fibroblast"), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 0 @@ -995,11 +1000,10 @@ def test_json_document_reports_job_fields( 42, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + sheet = _write_import_sheet(tmp_path, _import_record(cell_type="Neuron")) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--json", ) assert result.exit_code == 0 @@ -1031,14 +1035,13 @@ def test_sends_every_row_without_local_validation( ) sheet = _write_import_sheet( tmp_path, - {"accession": "not-an-accession"}, - {"accession": "ERR1"}, - {"accession": "ERR1"}, + _import_record(accession="not-an-accession", sample_type="bogus"), + _import_record(accession="ERR1"), + _import_record(accession="ERR1"), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 0 @@ -1056,14 +1059,12 @@ def test_sends_name_organism_and_metadata_in_payload( 1, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet(tmp_path, { - "accession": "ERR1", "name": "liver_r1", "organism": "Hs", - "cell_type": "Neuron", - }) + sheet = _write_import_sheet(tmp_path, _import_record( + name="liver_r1", organism="Hs", cell_type="Neuron", + )) run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) payload = json.loads(route.calls[0].request.content) @@ -1077,6 +1078,30 @@ def test_sends_name_organism_and_metadata_in_payload( }], } + @respx.mock + def test_each_row_uses_its_own_sample_type( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + _import_record(accession="ERR1", sample_type="chip_seq"), + _import_record(accession="ERR2", sample_type="atac_seq"), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "atac_seq"] + @respx.mock def test_api_rejection_propagates_as_error( self, run_cli, tmp_path: Path, @@ -1087,11 +1112,10 @@ def test_api_rejection_propagates_as_error( json={"error": "sample type 'bogus' does not exist"}, ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) + sheet = _write_import_sheet(tmp_path, _import_record(sample_type="bogus")) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "bogus", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 1 @@ -1105,8 +1129,7 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: xlsx.write_bytes(b"PK\x03\x04") result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(xlsx), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(xlsx), ) assert result.exit_code == 2 @@ -1114,23 +1137,9 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "CSV" in result.stderr def test_missing_sheet_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - - @respx.mock - def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), - ) + result = run_cli("--token", TOKEN, "samples", "import") assert result.exit_code == 2 - assert route.call_count == 0 @respx.mock def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1138,8 +1147,7 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None sheet = _write_import_sheet(tmp_path) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1147,63 +1155,14 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert str(sheet) in result.stderr @respx.mock - def test_row_sample_type_overrides_the_default( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 0 - payload = json.loads(route.calls[0].request.content) - sample_types = [entry["sample_type"] for entry in payload["imports"]] - assert sample_types == ["chip_seq", "rna_seq"] - - @respx.mock - def test_sample_type_flag_is_optional_when_every_row_has_its_own( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2", "sample_type": "atac_seq"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), - ) - - assert result.exit_code == 0 - payload = json.loads(route.calls[0].request.content) - sample_types = [entry["sample_type"] for entry in payload["imports"]] - assert sample_types == ["chip_seq", "atac_seq"] - - @respx.mock - def test_row_missing_sample_type_without_default_is_usage_error( + def test_row_missing_sample_type_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2"}, + _import_record(accession="ERR1"), + _import_record(accession="ERR2", sample_type=""), ) result = run_cli( @@ -1212,18 +1171,19 @@ def test_row_missing_sample_type_without_default_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "data row(s) 2" in result.stderr + assert "data row(s) 2 has no sample_type" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) + sheet = _write_import_sheet( + tmp_path, _import_record(accession=""), _import_record(accession=""), + ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1239,13 +1199,12 @@ def test_mixed_valid_and_blank_accession_rows_is_usage_error( route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, + _import_record(accession="ERR1"), + _import_record(accession=""), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1258,13 +1217,12 @@ def test_sheet_with_no_accession_column_is_usage_error( route = respx.post(SAMPLE_IMPORTS_URL) sheet = tmp_path / "accessions.csv" with sheet.open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer = csv.DictWriter(handle, fieldnames=["run", "name", "sample_type"]) writer.writeheader() - writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + writer.writerow({"run": "ERR1160845", "name": "liver_r1", "sample_type": "rna_seq"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1359,6 +1317,24 @@ def test_naive_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: document = json.loads(result.stdout) assert document["started"] == "2024-04-05T19:34:38Z" + @respx.mock + def test_non_utc_offset_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38+02:00", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["started"] == "2024-04-05T17:34:38Z" + @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 60aff5b632f03bbdc8aad5e71d8fb78ff01f5703 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:18:25 +0000 Subject: [PATCH 17/19] fix(samples): address round-16 Claude review findings - parse_accession_sheet now raises a single CliUsageError naming both missing accessions and missing sample types when a sheet is broken in both columns, instead of only ever reporting the accession problem and requiring a second run to learn about the sample_type one. This restores the 'collect every problem up front' property this slice has kept through every redesign. - Renamed the error-building helper (_missing_column_error -> _missing_value_clause) to describe what it actually reports (blank cells across a set of rows, not a missing column), and it now builds one clause per column rather than a whole exception. - Tightened four CLI-level sheet-rejection tests that could previously pass for any locally-rejected sheet (they asserted only that the sheet path appeared in stderr, which every parse_accession_sheet error satisfies) to assert the distinguishing phrase instead. Added a test for the both-columns-broken case at both the CLI and parser level. - Added a copy-pasteable accession-sheet example to cli.rst, now that there's no --sample-type flag or template verb to get the header from and both required columns reject a blank cell. - Dropped the now-unnecessary list[SampleImportSpec] annotation (and the import it required) on _import_command's list comprehension. Left as informational/deferred, per the review's own framing: the two ValueError invariants in AccessionSheetRow.__post_init__, the frozen- dataclass-with-dict hashability wrinkle, and import_samples([])'s round-trip to the server for an empty list (consistent with 'let the API validate'). Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 18 +++++++++++------ flowbio/cli/_samples.py | 3 +-- source/cli.rst | 8 ++++++++ tests/unit/cli/test_accession_sheet.py | 13 +++++++++++++ tests/unit/cli/test_samples.py | 27 +++++++++++++++++++++++--- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 0ed3e6a..4752eeb 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -106,19 +106,25 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: missing_sample_type.append(row_number) if accession is not None and sample_type is not None: rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - if missing_accession: - raise _missing_column_error("accession", missing_accession, path) - if missing_sample_type: - raise _missing_column_error("sample_type", missing_sample_type, path) + if missing_accession or missing_sample_type: + clauses = [ + clause for clause in ( + _missing_value_clause("accession", missing_accession), + _missing_value_clause("sample_type", missing_sample_type), + ) if clause is not None + ] + raise CliUsageError(f"Accession sheet {'; '.join(clauses)}: {path}.") return AccessionSheet(path=path, rows=rows) -def _missing_column_error(column: str, missing: list[int], path: Path) -> CliUsageError: +def _missing_value_clause(column: str, missing: list[int]) -> str | None: + if not missing: + return None numbers = ", ".join(str(number) for number in missing) # Data row 1 is the first row after the header, matching _sheet.py's # convention (and upload-batch's documented "1-based row number"). verb = "has" if len(missing) == 1 else "have" - return CliUsageError(f"Accession sheet data row(s) {numbers} {verb} no {column}: {path}.") + return f"data row(s) {numbers} {verb} no {column}" def _cell(record: dict[str, str], column: str) -> str | None: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 5904509..00a90b4 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -32,7 +32,6 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, - SampleImportSpec, SampleTypeId, ) @@ -633,7 +632,7 @@ def _import_command( rows, or has a row with no accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) - specs: list[SampleImportSpec] = [row.to_spec() for row in sheet.rows] + specs = [row.to_spec() for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/source/cli.rst b/source/cli.rst index b7bf4b5..5ff4d6f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -418,6 +418,14 @@ above. **Example** +``accessions.csv``: + +.. code-block:: text + + accession,sample_type,name,organism + ERR1160845,RNA-Seq,liver_r1,Hs + ERR10677146,RNA-Seq,, + .. code-block:: bash $ flowbio samples import --sheet ./accessions.csv diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 0403c25..7cbcbfd 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -151,6 +151,19 @@ def test_row_with_no_sample_type_column_is_usage_error(self, tmp_path: Path) -> with pytest.raises(CliUsageError): parse_accession_sheet(path) + def test_sheet_broken_in_both_columns_reports_both_in_one_error( + self, tmp_path: Path, + ) -> None: + with pytest.raises(CliUsageError) as excinfo: + parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession=""), + _record(accession="ERR2", sample_type=""), + )) + + assert "data row(s) 1 has no accession" in str(excinfo.value) + assert "data row(s) 2 has no sample_type" in str(excinfo.value) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d2df5f..77b57b9 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1152,7 +1152,7 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "no rows" in result.stderr @respx.mock def test_row_missing_sample_type_is_usage_error( @@ -1188,7 +1188,7 @@ def test_sheet_with_only_blank_accessions_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "data row(s) 1, 2 have no accession" in result.stderr @respx.mock def test_mixed_valid_and_blank_accession_rows_is_usage_error( @@ -1209,6 +1209,27 @@ def test_mixed_valid_and_blank_accession_rows_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 + assert "data row(s) 2 has no accession" in result.stderr + + @respx.mock + def test_sheet_broken_in_both_columns_reports_both_in_one_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + _import_record(accession=""), + _import_record(accession="ERR2", sample_type=""), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "data row(s) 1 has no accession" in result.stderr + assert "data row(s) 2 has no sample_type" in result.stderr @respx.mock def test_sheet_with_no_accession_column_is_usage_error( @@ -1227,7 +1248,7 @@ def test_sheet_with_no_accession_column_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "data row(s) 1 has no accession" in result.stderr class TestSamplesImportStatus: From 433528aea21bfbfd09574c396addec12a5fa02c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:29:33 +0000 Subject: [PATCH 18/19] fix(samples): address round-17 Claude review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - parse_accession_sheet now skips a row whose every cell is blank (the trailing comma-only line a spreadsheet export leaves below the data) instead of treating it as a row missing accession and sample_type — that's spreadsheet noise, not a row with data but no accession, and rejecting a whole sheet on it was a bad first-run experience. A sheet of nothing but blank rows still reports 'no rows', same as header-only. - Documented, based on reading flow-api's _assign_metadata/ SampleType.validate_metadata directly: a __annotation metadata column is not given upload-batch's special handling for samples import — it's forwarded as an ordinary key that matches no real MetadataAttribute, so it's silently ignored rather than attached as an annotation. Added a test pinning the client-side forwarding behaviour (the ignoring itself is server-side and can't be tested from here). - Fixed parse_accession_sheet's docstring and cli.rst, which promised values are sent 'exactly as entered'/'as-is' when cells are in fact whitespace-stripped. - parse_accession_sheet no longer materialises every record into a list before processing (streams the reader directly, matching _sheet.py's style), and states the accession/sample_type presence check once instead of re-testing an already-decided condition. - Moved the row-numbering convention comment to the enumerate() call it actually explains, and dropped a '— see below' cross-reference the following :raises: line already covered. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 48 +++++++++++++++----------- source/cli.rst | 21 +++++++---- tests/unit/cli/test_accession_sheet.py | 27 +++++++++++++++ 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 4752eeb..8825267 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -14,10 +14,14 @@ here rather than silently skipped or shipped as an empty string the server would just reject anyway. There is deliberately no other way to supply a sample type for ``samples import`` — the sheet is the single source of it. +A row with every cell blank (e.g. a trailing comma-only line some spreadsheet +exports append below the data) is skipped rather than treated as a row +missing values, since there is nothing there to be missing. """ from __future__ import annotations import csv +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @@ -69,10 +73,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped (other than - ``accession``/``sample_type``, which reject the sheet instead — see - below). Values are otherwise passed through unchanged, including - ``accession``, sent to the server exactly as entered. + :returns: The parsed sheet, with empty cells dropped and surrounding + whitespace trimmed. Values are otherwise passed through unchanged, + including ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession or no sample_type. """ @@ -82,6 +85,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: f"Export your spreadsheet to CSV first.", ) existing_file(path) + rows: list[AccessionSheetRow] = [] + missing_accession: list[int] = [] + missing_sample_type: list[int] = [] # utf-8-sig transparently strips a leading BOM, which spreadsheet tools # (notably Excel's "CSV UTF-8" export) prepend — otherwise the first header # parses as "accession" and every row reports a missing accession. @@ -91,21 +97,21 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] - records = list(enumerate(reader, start=1)) - if not records: + # Row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + for row_number, record in enumerate(reader, start=1): + if _is_blank_row(record, headers): + continue + accession = _cell(record, "accession") + sample_type = _cell(record, "sample_type") + if accession is None: + missing_accession.append(row_number) + if sample_type is None: + missing_sample_type.append(row_number) + if accession is not None and sample_type is not None: + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if not rows and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") - rows: list[AccessionSheetRow] = [] - missing_accession: list[int] = [] - missing_sample_type: list[int] = [] - for row_number, record in records: - accession = _cell(record, "accession") - sample_type = _cell(record, "sample_type") - if accession is None: - missing_accession.append(row_number) - if sample_type is None: - missing_sample_type.append(row_number) - if accession is not None and sample_type is not None: - rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) if missing_accession or missing_sample_type: clauses = [ clause for clause in ( @@ -117,12 +123,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, rows=rows) +def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: + return not any((record.get(header) or "").strip() for header in headers) + + def _missing_value_clause(column: str, missing: list[int]) -> str | None: if not missing: return None numbers = ", ".join(str(number) for number in missing) - # Data row 1 is the first row after the header, matching _sheet.py's - # convention (and upload-batch's documented "1-based row number"). verb = "has" if len(missing) == 1 else "have" return f"data row(s) {numbers} {verb} no {column}" diff --git a/source/cli.rst b/source/cli.rst index 5ff4d6f..2c66631 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -386,13 +386,20 @@ defaults to the accession when omitted. There is deliberately no ``--sample-type`` flag: the sheet's own column is the only way to supply a sample type, so a mixed-type sheet needs no special handling and a single-type sheet just repeats the same value down the column. The sample -type, accession format, and metadata rules are all sent as-is and validated -**server-side**; this command only checks that the sheet is a readable -``.csv`` and that every row has an accession and a sample type — a blank -cell in either column rejects the whole sheet up front rather than shipping -something the server would just reject anyway. Rows are counted from ``1`` -for the first data row, after the header (the same convention as -``upload-batch``'s ``row_number``). +type, accession format, and metadata rules are all sent as-is (surrounding +whitespace trimmed) and validated **server-side**; this command only checks +that the sheet is a readable ``.csv`` and that every row has an accession +and a sample type — a blank cell in either column rejects the whole sheet +up front rather than shipping something the server would just reject +anyway. A row with every cell blank (e.g. a trailing comma-only line some +spreadsheet exports leave below the data) is skipped rather than treated as +a row missing values. Rows are counted from ``1`` for the first data row, +after the header (the same convention as ``upload-batch``'s ``row_number``). + +Unlike ``upload-batch``, a metadata column named ``__annotation`` +is **not** given any special handling here — it is forwarded as an ordinary +metadata key, which the server does not recognise, so it is silently +ignored rather than attached as an annotation or rejected. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 7cbcbfd..54ee957 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -164,6 +164,21 @@ def test_sheet_broken_in_both_columns_reports_both_in_one_error( assert "data row(s) 1 has no accession" in str(excinfo.value) assert "data row(s) 2 has no sample_type" in str(excinfo.value) + def test_trailing_blank_row_is_skipped(self, tmp_path: Path) -> None: + # A comma-only line, e.g. one a spreadsheet export leaves below the + # data — not a row with data but no accession, so it isn't an error. + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession="ERR1"), + {}, + )) + + assert [row.accession for row in sheet.rows] == ["ERR1"] + + def test_sheet_of_only_blank_rows_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="no rows"): + parse_accession_sheet(_write_sheet(tmp_path, {}, {})) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): @@ -214,3 +229,15 @@ def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: organism_id="Hs", metadata={"cell_type": "Neuron"}, ) + + def test_annotation_suffixed_column_is_forwarded_as_a_plain_metadata_key( + self, tmp_path: Path, + ) -> None: + # Unlike upload-batch, there's no special handling of `__annotation` + # here — it's just another metadata column, and the server does not + # recognise it as one (see cli.rst). + row = self._row(tmp_path, source="blood", source__annotation="left lobe") + + spec = row.to_spec() + + assert spec.metadata == {"source": "blood", "source__annotation": "left lobe"} From d7a4bc2e7cd9862d757a34a7cde000882a05a44a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:38:11 +0000 Subject: [PATCH 19/19] fix(samples): address round-18 Claude review findings - Accession-sheet header names are now trimmed too, not just cell values: reassigning DictReader.fieldnames to the stripped list means a hand-authored header like 'accession, sample_type, name' (a space after each comma) is recognised correctly instead of treating ' sample_type'/' name' as bogus metadata columns and rejecting every row for a missing sample_type it visibly has. - Removed the now-redundant astimezone(timezone.utc) call in _timestamp_suffix (SampleImportJob's validator already guarantees a UTC-normalised value, aware or not, so the call was a no-op) along with its comment, and trimmed _job_summary's FAILED-branch comment to drop its cross-reference to _import_status_command. - Fixed a cli.rst sentence that said sample-type/accession-format/ metadata *rules* are sent as-is, when it's values that are sent and rules that are validated server-side. - parse_accession_sheet's own :returns: now mentions the blank-row skip as an observable property of what it returns, and states the accession/sample_type presence check once (an early continue) rather than a third redundant combined check. - Added tests: a header with spaces after commas, a blank row in the middle of a sheet (proving later rows keep their file-relative row numbers), and a CLI-level test that a trailing blank row is skipped and the remaining rows are still submitted. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 18 ++++++++++++------ flowbio/cli/_samples.py | 11 ++++------- source/cli.rst | 7 ++++--- tests/unit/cli/test_accession_sheet.py | 22 ++++++++++++++++++++++ tests/unit/cli/test_samples.py | 21 +++++++++++++++++++++ 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8825267..cace45a 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -73,9 +73,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped and surrounding - whitespace trimmed. Values are otherwise passed through unchanged, - including ``accession``, sent to the server as-entered. + :returns: The parsed sheet, with wholly-blank rows skipped, empty cells + dropped, and surrounding whitespace trimmed (including in header + names). Values are otherwise passed through unchanged, including + ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession or no sample_type. """ @@ -93,7 +94,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - headers = reader.fieldnames or [] + # A hand-authored header (unlike upload-batch's template-generated + # one) routinely has a stray space after a comma; reassigning + # fieldnames makes every row dict keyed by the trimmed name too. + headers = [header.strip() for header in reader.fieldnames or []] + reader.fieldnames = headers metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] @@ -108,8 +113,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: missing_accession.append(row_number) if sample_type is None: missing_sample_type.append(row_number) - if accession is not None and sample_type is not None: - rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if accession is None or sample_type is None: + continue + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) if not rows and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") if missing_accession or missing_sample_type: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 00a90b4..9633221 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,7 +11,7 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Literal @@ -669,9 +669,8 @@ def _job_summary(job: SampleImportJob) -> str: suffix = _timestamp_suffix("finished", job.finished) return f"Job {job.id}: COMPLETED{suffix}. Sample ids: {ids}." if job.status == "FAILED": - # The error, if any, is on stderr as an advisory (see - # _import_status_command) rather than repeated here, so a human - # running this doesn't see the same sentence twice. + # A human sees the error as a separate advisory on stderr, so it + # isn't repeated here. return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." label = "started" if job.started else "created" return f"Job {job.id}: {job.status}{_timestamp_suffix(label, job.started or job.created)}." @@ -680,9 +679,7 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - # SampleImportJob normalises a naive value to UTC, so every timestamp - # reaching here is already aware. - return f" ({label} {timestamp.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')})" + return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index 2c66631..153d227 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -385,9 +385,10 @@ equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. There is deliberately no ``--sample-type`` flag: the sheet's own column is the only way to supply a sample type, so a mixed-type sheet needs no special handling and a -single-type sheet just repeats the same value down the column. The sample -type, accession format, and metadata rules are all sent as-is (surrounding -whitespace trimmed) and validated **server-side**; this command only checks +single-type sheet just repeats the same value down the column. Every value +is sent as-is (surrounding whitespace trimmed, including in header names); +the accession format, sample type, and metadata rules are validated +**server-side**. This command only checks that the sheet is a readable ``.csv`` and that every row has an accession and a sample type — a blank cell in either column rejects the whole sheet up front rather than shipping something the server would just reject diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 54ee957..64fac0f 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -88,6 +88,17 @@ def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: assert [row.row_number for row in sheet.rows] == [1, 2] + def test_header_with_spaces_after_commas_is_still_recognised(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession, sample_type, name\nERR1160845, rna_seq, liver_r1\n") + + sheet = parse_accession_sheet(path) + + assert sheet.rows[0].accession == "ERR1160845" + assert sheet.rows[0].sample_type == "rna_seq" + assert sheet.rows[0].name == "liver_r1" + assert sheet.rows[0].metadata == {} + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") @@ -179,6 +190,17 @@ def test_sheet_of_only_blank_rows_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match="no rows"): parse_accession_sheet(_write_sheet(tmp_path, {}, {})) + def test_blank_row_in_the_middle_does_not_shift_later_row_numbers( + self, tmp_path: Path, + ) -> None: + with pytest.raises(CliUsageError, match=r"data row\(s\) 3 has no accession"): + parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession="ERR1"), + {}, + _record(accession=""), + )) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 77b57b9..3f04081 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1102,6 +1102,27 @@ def test_each_row_uses_its_own_sample_type( sample_types = [entry["sample_type"] for entry in payload["imports"]] assert sample_types == ["chip_seq", "atac_seq"] + @respx.mock + def test_trailing_blank_row_is_skipped( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, _import_record(accession="ERR1"), {}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == ["ERR1"] + @respx.mock def test_api_rejection_propagates_as_error( self, run_cli, tmp_path: Path,