diff --git a/README.md b/README.md index 2e10db879..d103575c7 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,24 @@ provenance token. Orcapod supports Python 3.11, 3.12 and 3.13 and is published on PyPI as `orcapod`: +The recommended way to install it is [uv](https://docs.astral.sh/uv/): `uv add` adds +Orcapod to a uv project (start one with `uv init` in a new directory), and `uv run` runs +your code in that project's environment: + ```bash -pip install orcapod -# or, in a uv project uv add orcapod ``` +In an environment not managed by uv, use `pip install orcapod` instead. A plain +`uv add orcapod` installs the latest stable release; to try a release candidate, name it, +for example `uv add "orcapod==0.4.0rc1"`. + 0.3.0 is the first release on PyPI since 0.1.4; 0.2.0 was never published to PyPI. Tables written by 0.2.0 or earlier are not read by 0.3.0: see [Upgrading from 0.2.0](https://github.com/metamorphic-labs/orcapod-python/blob/main/CHANGELOG.md#upgrading-from-020) (it also covers 0.1.4). -Optional features are installed as extras: +Optional features are installed as extras, for example `uv add "orcapod[spiraldb]"`: | Extra | Adds | |-------|------| diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index 7332f0b41..e5a5c4012 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -23,7 +23,7 @@ source = DictSource( tag_columns=["trial"], source_id="trials", ) -for tag, data in scale.pod(source).iter_data(): +for tag, data in scale.pod(source): print(tag["trial"], data["scaled"].dtype, data["scaled"].shape) # t1 float32 (2, 3) ``` @@ -246,7 +246,7 @@ big = np.asfortranarray(np.arange(6, dtype=">f4").reshape(2, 3)) kept = DictSource( data=[{"trial": "t1", "signal": big}], tag_columns=["trial"], source_id="kept" ) -for tag, data in kept.iter_data(): +for tag, data in kept: print(data["signal"].dtype.str, data["signal"].flags.f_contiguous) # >f4 True ``` @@ -269,9 +269,9 @@ values have different hashes. ## PyTorch tensors A `torch.Tensor` can be a data value too. PyTorch is an optional dependency. Tensor support -works whenever PyTorch can be imported, however it was installed: `pip install "orcapod[torch]"` -installs it from PyPI, or install the build that suits your hardware yourself (see -pytorch.org; on Linux the PyPI build includes CUDA, and pytorch.org also offers smaller +works whenever PyTorch can be imported, however it was installed: `uv add "orcapod[torch]"` +(or `pip install "orcapod[torch]"`) installs it from PyPI, or install the build that suits +your hardware yourself (see pytorch.org; on Linux the PyPI build includes CUDA, and pytorch.org also offers smaller CPU-only builds). Orcapod needs PyTorch 2.6 or later, the first version whose restricted loader (below) reads `uint16`, `uint32` and `uint64` tensors, and is tested with PyTorch 2.15. Orcapod hashes a tensor's elements through NumPy (`Tensor.numpy()`), so your PyTorch build must @@ -299,7 +299,7 @@ tensors = DictSource( tag_columns=["trial"], source_id="tensor_trials", ) -for tag, data in scale_tensor.pod(tensors).iter_data(): +for tag, data in scale_tensor.pod(tensors): print(tag["trial"], data["scaled"].dtype, tuple(data["scaled"].shape)) # t1 torch.float32 (2, 3) ``` diff --git a/docs/concepts/data-tags.md b/docs/concepts/data-tags.md index 1131a1cd3..21dd4e5aa 100644 --- a/docs/concepts/data-tags.md +++ b/docs/concepts/data-tags.md @@ -227,7 +227,7 @@ column: batched = spikes.batch() print(batched.keys()) # (('unit', 'time'), ('unit', 'time', 'amp')) -for tag, data in batched.iter_data(): +for tag, data in batched: print(tag["time"], data["time"]) # [10, 20] [10, 20] ``` diff --git a/docs/concepts/file-hash-caching.md b/docs/concepts/file-hash-caching.md index cc8096c11..b738a7d2b 100644 --- a/docs/concepts/file-hash-caching.md +++ b/docs/concepts/file-hash-caching.md @@ -265,8 +265,8 @@ enable_file_hash_caching(db_path=Path("shared/nfs/cache.db")) ### Using the Postgres backend -Requires `psycopg` (install with `pip install 'orcapod[postgresql]'`) and a reachable -Postgres server: +Requires `psycopg` (install with `uv add "orcapod[postgresql]"`, or +`pip install "orcapod[postgresql]"`) and a reachable Postgres server: ```python diff --git a/docs/concepts/file-types.md b/docs/concepts/file-types.md index a69c97191..7b435d582 100644 --- a/docs/concepts/file-types.md +++ b/docs/concepts/file-types.md @@ -139,7 +139,7 @@ def summarize(recording: op.File) -> Path: result = summarize.pod(files) print(result.output_schema()[1]) # Schema({'summary': Path}) -for tag, data in result.iter_data(): +for tag, data in result: print(tag.as_dict(), data.as_dict()["summary"].read_text().strip()) # {'name': 'a'} 3 lines # {'name': 'b'} 2 lines @@ -180,7 +180,7 @@ def total_lines(recording: list[op.File]) -> int: everything = files.batch() -for _, data in total_lines.pod(everything).iter_data(): +for _, data in total_lines.pod(everything): print(data.as_dict()) # {'total_lines': 5} ``` diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index f43ec43ae..b6e1f475b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -82,7 +82,7 @@ print("Data schema:", dict(data_schema)) # Data schema: {'bmi': } # Iterate over results -for tag, data in result.iter_data(): +for tag, data in result: print(f" {tag.as_dict()} -> {data.as_dict()}") # {'subject_id': 'mouse_01'} -> {'bmi': 1756.9444444444446} # {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} @@ -114,7 +114,7 @@ wide = DictSource( ) result = compute_bmi.pod(wide) # 'age' is not a parameter: it is dropped -for tag, data in result.iter_data(): +for tag, data in result: print(f"{tag.as_dict()} -> {data.as_dict()}") # {'subject_id': 'mouse_01'} -> {'bmi': 1756.9444444444446} # {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} @@ -289,7 +289,7 @@ explicit: def twice(weight: float) -> list[float]: return list((weight, weight * 2)) -for tag, data in twice.pod(weights).iter_data(): +for tag, data in twice.pod(weights): print(data.as_dict()) # {'weights': [25.3, 50.6]} ``` @@ -1131,12 +1131,12 @@ recordings = DictSource( tag_columns=["session"], tag_only_columns=True, ) readings = load_readings.pod(recordings) -for tag, data in readings.iter_data(): +for tag, data in readings: print(tag["session"], dict(data["readings"].schema)) # s1 {'sample': UInt16, 'voltage': Float32} # s2 {'sample': UInt16, 'voltage': Float32} -for tag, data in count_rows.pod(readings).iter_data(): +for tag, data in count_rows.pod(readings): print(tag["session"], data["n_rows"]) # s1 2 # s2 3 @@ -1177,7 +1177,7 @@ labels = DictSource( tag_columns=["sample"], tag_only_columns=True, data_schema={"sample": str, "counts": list[int | None]}, ) -for tag, data in labels.iter_data(): +for tag, data in labels: print(data["counts"]) # [3, None, 1] ``` @@ -1218,7 +1218,7 @@ heights = DictSource( ) result = compute_bmi.pod(weights, heights) -for tag, data in result.iter_data(): +for tag, data in result: print(f"{tag.as_dict()} -> {data.as_dict()}") # {'subject_id': 'mouse_01'} -> {'bmi': 1756.9444444444446} # {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} @@ -1242,7 +1242,7 @@ chosen = DictSource( ) result = compute_bmi.pod(weights, heights, chosen) -for tag, data in result.iter_data(): +for tag, data in result: print(f"{tag.as_dict()} -> {data.as_dict()}") # {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} ``` diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 096e60609..2a3ca180e 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -149,7 +149,7 @@ scale_b = DictSource( ) merged = MergeJoin()(scale_a, scale_b) -for tag, data in merged.iter_data(): +for tag, data in merged: print(tag.as_dict(), "->", data.as_dict()) # {'subject_id': 'mouse_01'} -> {'weight': [24.9, 25.3]} # {'subject_id': 'mouse_02'} -> {'weight': [22.1, 22.6]} @@ -197,7 +197,7 @@ weighed = DictSource( ) only_weighed = SemiJoin()(all_subjects, weighed) -for tag, data in only_weighed.iter_data(): +for tag, data in only_weighed: print(tag.as_dict(), "->", data.as_dict()) # {'subject_id': 'mouse_01'} -> {'age': 12} # {'subject_id': 'mouse_02'} -> {'age': 8} @@ -222,7 +222,7 @@ source = DictSource( batch = Batch() batched = batch.process(source) -for tag, data in batched.iter_data(): +for tag, data in batched: print("Tags:", tag.as_dict()) # Tags: {'subject_id': ['mouse_01', 'mouse_02']} print("Data:", data.as_dict()) @@ -263,7 +263,7 @@ source = DictSource( grouped = GroupBy(by=["subject_id"])(source) print(grouped.keys()) # (('subject_id',), ('spike_count',)) -for tag, data in grouped.iter_data(): +for tag, data in grouped: print("Tags:", tag.as_dict(), "Data:", data.as_dict()) # Tags: {'subject_id': 'mouse_01'} Data: {'spike_count': [1200, 980]} # Tags: {'subject_id': 'mouse_02'} Data: {'spike_count': [1450]} @@ -293,7 +293,7 @@ with_probe = DerivedSource(source, tags_as_data={"probe": "probe_name"}) by_subject = GroupBy(by=["subject_id"])(with_probe) print(by_subject.keys()) # (('subject_id',), ('spike_count', 'probe_name')) -for tag, data in by_subject.iter_data(): +for tag, data in by_subject: print(tag.as_dict(), data.as_dict()) # {'subject_id': 'mouse_01'} {'spike_count': [1200, 980], 'probe_name': ['imec0', 'imec1']} # {'subject_id': 'mouse_02'} {'spike_count': [1450], 'probe_name': ['imec0']} @@ -455,7 +455,7 @@ from orcapod.operators import PolarsFilter filt = PolarsFilter(predicates=[pl.col("age") > 10]) filtered = filt.process(source) -for tag, pkt in filtered.iter_data(): +for tag, pkt in filtered: print(f"{tag.as_dict()} -> {pkt.as_dict()}") # {'subject_id': 'mouse_01'} -> {'age': 12, 'weight': 25.3} # {'subject_id': 'mouse_03'} -> {'age': 15, 'weight': 27.8} @@ -659,7 +659,7 @@ keyword arguments or as one dict (the only positional argument), as `map_tags` t ```python labelled = sessions.add_tags(cohort="A", run=3) print(labelled.keys()) # (('subject_id', 'session', 'cohort', 'run'), ('age', 'weight')) -for tag, data in labelled.iter_data(): +for tag, data in labelled: print(tag.as_dict(), "->", data.as_dict()) # {'subject_id': 'mouse_01', 'session': 1, 'cohort': 'A', 'run': 3} -> {'age': 12, 'weight': 25.3} # {'subject_id': 'mouse_02', 'session': 2, 'cohort': 'A', 'run': 3} -> {'age': 8, 'weight': 22.1} @@ -803,7 +803,7 @@ Every stream also exposes the operators as methods, which reads well when chaini import polars as pl adults = subjects.join(measurements).polars_filter(pl.col("age") > 10) -for tag, data in adults.iter_data(): +for tag, data in adults: print(tag.as_dict(), "->", data.as_dict()) # {'subject_id': 'mouse_01'} -> {'weight': 25.3, 'age': 12} ``` diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index ce9c1b0ca..a695bd1b4 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -94,8 +94,8 @@ pyspiral keeps the login in its per-user directory (`~/Library/Application Suppo on macOS, `~/.config/pyspiral/` on Linux): ```bash -pip install "orcapod[spiraldb]" -spiral login +uv add "orcapod[spiraldb]" # or, without uv: pip install "orcapod[spiraldb]" +uv run spiral login ``` Log in before you build a source. Without a saved login, pyspiral starts a sign-in in the @@ -634,7 +634,7 @@ print("Data keys:", data_keys) # Data keys: ('age', 'weight') # Iterate over (Tag, Data) pairs -for tag, data in source.iter_data(): +for tag, data in source: print(f" Tag: {tag.as_dict()}, Data: {data.as_dict()}") # Tag: {'subject_id': 'mouse_01'}, Data: {'age': 12, 'weight': 25.3} # Tag: {'subject_id': 'mouse_02'}, Data: {'age': 8, 'weight': 22.1} diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index cbbfa1840..7487ae670 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -78,11 +78,13 @@ tag_keys, data_keys = stream.keys() ### `iter_data()` -Iterates over (Tag, Data) pairs. Each Tag and Data is an immutable datagram that you can -inspect with `.as_dict()`: +Iterates over (Tag, Data) pairs. A stream is itself iterable, and iterating it is the same as +calling `iter_data()`, so a loop is written `for tag, data in stream:`; the method is there +for when the iterator is passed on as a value (`list(stream.iter_data())`). Each Tag and Data +is an immutable datagram that you can inspect with `.as_dict()`: ```python -for tag, data in stream.iter_data(): +for tag, data in stream: print(tag.as_dict(), data.as_dict()) # {'subject_id': 'mouse_01'} {'age': 12, 'weight': 25.3} # {'subject_id': 'mouse_02'} {'age': 8, 'weight': 22.1} @@ -358,7 +360,7 @@ print("Data schema:", dict(data_schema)) # Data schema: {'age': , 'weight': } # Iterate over (Tag, Data) pairs -for tag, data in source.iter_data(): +for tag, data in source: print(f" {tag.as_dict()} -> {data.as_dict()}") # {'subject_id': 'mouse_01'} -> {'age': 12, 'weight': 25.3} # {'subject_id': 'mouse_02'} -> {'age': 8, 'weight': 22.1} diff --git a/docs/getting-started.md b/docs/getting-started.md index a48403905..86bdb185e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -4,6 +4,19 @@ This guide walks you through the core workflow in Orcapod: creating a data sourc inspecting its stream, applying a transformation with a function pod, and examining the results. +## Installing + +Install Orcapod with [uv](https://docs.astral.sh/uv/), the recommended way: start a project +with `uv init` (in a new directory), add Orcapod to it, and run your scripts with `uv run`: + +```bash +uv init +uv add orcapod +uv run python my_pipeline.py +``` + +In an environment not managed by uv, `pip install orcapod` installs it instead. + ## Creating a source A **source** is the entry point for data in an Orcapod pipeline. The simplest way to @@ -74,10 +87,11 @@ print(data_keys) ### Iterating over rows -Use `iter_data()` to walk through each (Tag, Data) pair: +A stream is iterable: a `for` loop walks through each (Tag, Data) pair. Iterating a stream +is the same as calling its `iter_data()` method: ```python -for tag, data in source.iter_data(): +for tag, data in source: print(f"Tag: {tag.as_dict()}, Data: {data.as_dict()}") # Tag: {'experiment': 'exp_001'}, Data: {'temperature': 20.5, 'pressure': 1.01} # Tag: {'experiment': 'exp_002'}, Data: {'temperature': 22.3, 'pressure': 0.98} @@ -160,7 +174,7 @@ now reflects the function's output types. Iterate over the results: ```python -for tag, data in result.iter_data(): +for tag, data in result: print(f"Tag: {tag.as_dict()}, Data: {data.as_dict()}") # Tag: {'experiment': 'exp_001'}, Data: {'temp_fahrenheit': 68.9, 'is_high_pressure': True} # Tag: {'experiment': 'exp_002'}, Data: {'temp_fahrenheit': 72.14, 'is_high_pressure': False} diff --git a/docs/index.md b/docs/index.md index a7f23ec6b..a160250ef 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,12 +10,18 @@ content-addressable, and reproducible by design. Orcapod supports Python 3.11, 3.12 and 3.13 and is published on PyPI as `orcapod`, under the MIT License: +The recommended way to install it is [uv](https://docs.astral.sh/uv/): `uv add` adds +Orcapod to a uv project (start one with `uv init` in a new directory), and `uv run` runs +your code in that project's environment: + ```bash -pip install orcapod -# or, in a uv project uv add orcapod ``` +In an environment not managed by uv, use `pip install orcapod` instead. A plain +`uv add orcapod` installs the latest stable release; to try a release candidate, name it, +for example `uv add "orcapod==0.4.0rc1"`. + `0.3.0` is the first release on PyPI since `0.1.4`; `0.2.0` was never published to PyPI. Tables written by `0.2.0` or earlier are not read by `0.3.0`: see `CHANGELOG.md` in the repository, *Upgrading from 0.2.0* @@ -47,7 +53,7 @@ def compute_birth_year(age: int) -> int: # 3. Apply the function pod and inspect the output result = compute_birth_year.pod(source) -for tag, data in result.iter_data(): +for tag, data in result: print(f"{tag.as_dict()} -> {data.as_dict()}") # {'name': 'Alice'} -> {'birth_year': 1996} # {'name': 'Bob'} -> {'birth_year': 2001} diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f09cec2af..1693c4b67 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -314,3 +314,6 @@ Reading a Delta table yourself with `deltalake.DeltaTable(...).to_pyarrow_table( are installing a version of Orcapod older than 0.2, which required the Graphviz development headers. Install Orcapod 0.2 or newer. - **Python version.** Orcapod needs Python 3.11 or newer. +- **A release candidate is not installed.** `uv add orcapod` (like `pip install orcapod`) + takes the latest stable release and skips pre-releases. Name the release candidate: + `uv add "orcapod==0.4.0rc1"`. diff --git a/docs/tutorial/00-setup.md b/docs/tutorial/00-setup.md index 876d61ca3..4106a2263 100644 --- a/docs/tutorial/00-setup.md +++ b/docs/tutorial/00-setup.md @@ -8,7 +8,8 @@ workshop dataset on disk. Orcapod needs Python 3.11 or newer. Create a fresh environment for the workshop so that nothing else on your machine is affected. -With [uv](https://docs.astral.sh/uv/): +With [uv](https://docs.astral.sh/uv/), the recommended way (`uv init` starts a project in the +new directory, and `uv add` adds Orcapod to it): ```bash mkdir orcapod-workshop && cd orcapod-workshop @@ -16,7 +17,7 @@ uv init uv add orcapod ``` -With `pip`: +Without uv, with `pip` in a virtual environment: ```bash mkdir orcapod-workshop && cd orcapod-workshop @@ -25,8 +26,10 @@ source .venv/bin/activate # on Windows: .venv\Scripts\activate pip install orcapod ``` -If your workshop organiser asked you to install the development version instead, replace -`orcapod` with `git+https://github.com/metamorphic-labs/orcapod-python.git` in either command. +If your workshop organiser asked you to install a release candidate, name it: +`uv add "orcapod==0.4.0rc1"` (a plain `uv add orcapod` takes the latest stable release). For +the development version, replace `orcapod` with +`git+https://github.com/metamorphic-labs/orcapod-python.git` in either command. The tutorial needs nothing beyond this. Orcapod already depends on `pandas`, `polars` and `pyarrow`, which the samples use. @@ -38,8 +41,8 @@ The tutorial needs nothing beyond this. Orcapod already depends on `pandas`, `po ## Check your install -Save this as `check_install.py` and run it with `python check_install.py` (or -`uv run python check_install.py`): +Save this as `check_install.py` and run it with `uv run python check_install.py` (with +`pip`, in the activated environment: `python check_install.py`): ```python from orcapod import PipelineJob, function_pod diff --git a/docs/tutorial/01-first-stream.md b/docs/tutorial/01-first-stream.md index 1c0a72093..a3b50d493 100644 --- a/docs/tutorial/01-first-stream.md +++ b/docs/tutorial/01-first-stream.md @@ -103,11 +103,11 @@ What a source gives you is a **stream**: an ordered, read-only sequence of rows, pair of (tag, data). Every step you add later takes streams in and gives a stream back, so the four ways of looking at a stream shown in this part work at every stage of a pipeline. -You have seen `output_schema()` and `keys()`. To walk through the rows, use `iter_data()`. -Each row comes as two objects; call `.as_dict()` on either to get a plain dictionary: +You have seen `output_schema()` and `keys()`. To walk through the rows, loop over the stream +(the same as calling its `iter_data()` method). Each row comes as two objects; call `.as_dict()` on either to get a plain dictionary: ```python -for tag, data in sessions.iter_data(): +for tag, data in sessions: print(tag.as_dict(), "->", data.as_dict()) # {'subject_id': 'mouse_01', 'session': 's1'} -> {'date': datetime.date(2026, 10, 5), 'duration_s': 60.0} # {'subject_id': 'mouse_01', 'session': 's2'} -> {'date': datetime.date(2026, 10, 12), 'duration_s': 120.0} diff --git a/docs/tutorial/04-many-rows-into-one.md b/docs/tutorial/04-many-rows-into-one.md index c27bca04e..6468e7738 100644 --- a/docs/tutorial/04-many-rows-into-one.md +++ b/docs/tutorial/04-many-rows-into-one.md @@ -162,7 +162,7 @@ becomes a single row: ```python everyone = subjects.batch() -for tag, data in everyone.iter_data(): +for tag, data in everyone: print(tag.as_dict()) print(data.as_dict()) # {'subject_id': ['mouse_01', 'mouse_02', 'mouse_03', 'mouse_04']} @@ -174,7 +174,7 @@ Here the tags became lists too, since no column identifies the single row any mo `drop_partial_batch=True`: ```python -for tag, data in subjects.batch(batch_size=3).iter_data(): +for tag, data in subjects.batch(batch_size=3): print(tag.as_dict()) # {'subject_id': ['mouse_01', 'mouse_02', 'mouse_03']} # {'subject_id': ['mouse_04']} @@ -256,7 +256,7 @@ filter, then narrow, then `batch()`.) .select_data_columns(["mean_rate_hz"]) .batch() ) - for _, data in mean_of_means.pod(one_genotype).iter_data(): + for _, data in mean_of_means.pod(one_genotype): print(genotype, round(data.as_dict()["genotype_mean_hz"], 3)) # wildtype 6.545 # knockout 5.058 diff --git a/docs/tutorial/07-files.md b/docs/tutorial/07-files.md index e6e3fc8dd..55e307622 100644 --- a/docs/tutorial/07-files.md +++ b/docs/tutorial/07-files.md @@ -213,7 +213,7 @@ summaries = job.nodes["summaries"] print(summaries.output_schema()[1]) # Schema({'summary': Path}) wanted = {"subject_id": "mouse_01", "session": "s1", "probe": "imec0"} -data = next(data for tag, data in summaries.iter_data() if tag.as_dict() == wanted) +data = next(data for tag, data in summaries if tag.as_dict() == wanted) print(data.as_dict()["summary"].read_text()) # n_spikes: 1737 # last_spike_s: 59.9999 @@ -377,7 +377,7 @@ containing the total number of spikes over both probes. (Hint: Part 4. After # Schema({'spikes': list[orcapod.logical_types.file_type.File]}) totals = write_session_total.pod(per_session) - for tag, data in totals.iter_data(): + for tag, data in totals: path = data.as_dict()["total_file"] print(path.name, path.read_text().strip()) # mouse_01_s1_total.txt 2897