diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..71a307647 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +name: Lint and type check + +# The public repository (nauticalab/orcapod-python) has no organization-level required checks, so +# the checks CLAUDE.md lists under "Lint, format and type checks" run here. pytest is covered by +# run-tests.yml. The job does not run on the development repository, whose required CI runs the +# same commands (ITL-1007). + +on: + push: + branches: [main] + pull_request: + +jobs: + lint: + name: Ruff and basedpyright + if: github.repository == 'nauticalab/orcapod-python' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 # required: hatch-vcs needs full tag history + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + - name: Set up Python 3.11 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Install dependencies + run: uv sync --locked --all-extras --dev --python "3.11" + + - name: Ruff check + run: uv run --with ruff ruff check . + + - name: Ruff format check + run: uv run --with ruff ruff format --check . + + - name: Basedpyright + run: uv run --with basedpyright basedpyright diff --git a/.github/workflows/release-sync.yml b/.github/workflows/release-sync.yml index 8aa903405..1cd5bab9a 100644 --- a/.github/workflows/release-sync.yml +++ b/.github/workflows/release-sync.yml @@ -7,6 +7,8 @@ on: jobs: sync: + # Linear is wired to the development repository only (ITL-1007). + if: github.repository == 'metamorphic-labs/orcapod-python' runs-on: ubuntu-latest timeout-minutes: 5 permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 485243489..5501bfba8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -174,6 +174,7 @@ jobs: linear-sync: name: Sync Linear release needs: build + if: github.repository == 'metamorphic-labs/orcapod-python' runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -193,6 +194,7 @@ jobs: linear-complete: name: Complete Linear release needs: [build, publish-pypi, linear-sync] + if: github.repository == 'metamorphic-labs/orcapod-python' runs-on: ubuntu-latest timeout-minutes: 5 permissions: diff --git a/.github/workflows/run-objective-tests.yml b/.github/workflows/run-objective-tests.yml index 672c1a4fa..3df72ceff 100644 --- a/.github/workflows/run-objective-tests.yml +++ b/.github/workflows/run-objective-tests.yml @@ -34,6 +34,7 @@ jobs: run: uv run --python ${{ matrix.python-version }} pytest test-objective/ -v --cov=src --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov + if: github.repository == 'metamorphic-labs/orcapod-python' uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run-postgres-tests.yml b/.github/workflows/run-postgres-tests.yml index 5e9552405..83acccdb6 100644 --- a/.github/workflows/run-postgres-tests.yml +++ b/.github/workflows/run-postgres-tests.yml @@ -55,6 +55,7 @@ jobs: PGDATABASE: testdb - name: Upload coverage reports to Codecov + if: github.repository == 'metamorphic-labs/orcapod-python' uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0bf782c86..130ca4618 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -37,6 +37,7 @@ jobs: run: uv run --python ${{ matrix.python-version }} pytest -v -m "not postgres" --cov=src --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov + if: github.repository == 'metamorphic-labs/orcapod-python' uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -46,8 +47,11 @@ jobs: # Only run on push/manual trigger — not on PRs — to avoid consuming SpiralDB # resources on every review cycle. # Requires SPIRAL_WORKLOAD_ID secret; fails loudly if absent so missing - # credentials don't silently drop coverage. - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + # credentials don't silently drop coverage. Runs only on the development repository: + # the public repository has no SpiralDB credentials, so the job is skipped there (ITL-1007). + if: >- + github.repository == 'metamorphic-labs/orcapod-python' + && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') needs: test permissions: id-token: write # required to request a GitHub OIDC token for SpiralDB auth diff --git a/.zed/rules b/.zed/rules index a9740f662..41af43df0 100644 --- a/.zed/rules +++ b/.zed/rules @@ -218,6 +218,19 @@ resolves several issues uses its primary issue's id). (see RELEASING.md). Entries already in CHANGELOG.md on main stay there. - changelog.d/README.md is the full statement of the convention. +## Releasing and the public repository + +Developer decisions, November 12, 2026 (ITL-1007); the full procedure is RELEASING.md. + +- All development, review and CI happen on metamorphic-labs/orcapod-python. The public + nauticalab/orcapod-python receives main only at release points, by **fast-forward push** + (never a squash, never a pull request), and runs the Release workflow. +- An agent never pushes to nauticalab, adds a remote for it, creates or pushes a tag, runs the + Release workflow, or touches a setting, secret or environment there. The push happens only on + the developer's explicit go. The v0.2.0 tag is never pushed. +- Linear, Codecov and SpiralDB jobs run only when + github.repository == 'metamorphic-labs/orcapod-python'; lint.yml runs only on nauticalab. + ## Lint, format and type checks Every pull request must pass the metamorphic-labs required CI, which runs exactly these diff --git a/CLAUDE.md b/CLAUDE.md index c3dc23676..57f3b08c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,6 +227,19 @@ resolves several issues uses its primary issue's id). (see `RELEASING.md`). Entries already in `CHANGELOG.md` on `main` stay there. - `changelog.d/README.md` is the full statement of the convention. +## Releasing and the public repository + +Developer decisions, November 12, 2026 (ITL-1007); the full procedure is `RELEASING.md`. + +- All development, review and CI happen on `metamorphic-labs/orcapod-python`. The public + `nauticalab/orcapod-python` receives `main` only at release points, by **fast-forward push** + (never a squash, never a pull request), and runs the Release workflow. +- An agent never pushes to `nauticalab`, adds a remote for it, creates or pushes a tag, runs the + Release workflow, or touches a setting, secret or environment there. The push happens only on + the developer's explicit go. The `v0.2.0` tag is never pushed. +- Linear, Codecov and SpiralDB jobs run only when + `github.repository == 'metamorphic-labs/orcapod-python'`; `lint.yml` runs only on `nauticalab`. + ## Lint, format and type checks Every pull request must pass the metamorphic-labs required CI, which runs exactly these diff --git a/README.md b/README.md index c2790ef5e..43c4168d4 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ SpikeInterface support lives in a separate package, [`orcapod-extension-spikeinterface`](https://github.com/nauticalab/orcapod-extension-spikeinterface). It is not published on PyPI; install it from its repository. -See [CHANGELOG.md](https://github.com/metamorphic-labs/orcapod-python/blob/main/CHANGELOG.md) +See [CHANGELOG.md](https://github.com/nauticalab/orcapod-python/blob/main/CHANGELOG.md) for release notes, including the migration steps for databases written by an earlier release. @@ -45,13 +45,13 @@ Releases are cut by running the GitHub Actions **Release** workflow by hand. It tests and the license check, creates and pushes the version tag, builds the wheel and sdist, and publishes them to TestPyPI and then PyPI through trusted publishing. The version comes from the tag via `hatch-vcs`, so there is no version number to bump in the source -tree. See [RELEASING.md](https://github.com/metamorphic-labs/orcapod-python/blob/main/RELEASING.md) +tree. See [RELEASING.md](https://github.com/nauticalab/orcapod-python/blob/main/RELEASING.md) for the full workflow. ## License Orcapod is released under the -[MIT License](https://github.com/metamorphic-labs/orcapod-python/blob/main/LICENSE). +[MIT License](https://github.com/nauticalab/orcapod-python/blob/main/LICENSE). ## Continuous Integration diff --git a/RELEASING.md b/RELEASING.md index d565f497f..3cad656d7 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -2,7 +2,7 @@ This document describes how to cut a release of `orcapod`. -## Publish target +## Publish target and where things run Orcapod is published on public PyPI as [`orcapod`](https://pypi.org/project/orcapod/), under the MIT License (developer decision, 2026-11-10; the license is applied from `0.2.0`). Users @@ -10,13 +10,19 @@ install it with `pip install orcapod` or `uv add orcapod`. `0.2.0` was cut as a (developer decision, 2026-11-10); `0.3.0` is the first release published to PyPI from this repository. +Two repositories take part (developer decision, 2026-11-12, ITL-1007): + +| Repository | Role | +|---|---| +| `metamorphic-labs/orcapod-python` | **All development**: branches, pull requests, review, CI, Linear integration, the design log. | +| `nauticalab/orcapod-python` (public) | **Releases only**: receives `main` by fast-forward push at a release point, and runs the Release workflow. | + The Release workflow (`.github/workflows/release.yml`) is the only publishing route. It runs -only when dispatched by hand, uploads to TestPyPI first and then to PyPI, and authenticates -to both through OIDC trusted publishing: the workflow uses no API token. That -requires, on each index, a trusted publisher for the `orcapod` project naming owner -`metamorphic-labs`, repository `orcapod-python`, workflow `release.yml` and environment -`testpypi` (TestPyPI) or `pypi` (PyPI); see *Prerequisites* below. Only an owner of the -`orcapod` project on each index can add one. +only when dispatched by hand **on `nauticalab/orcapod-python`**, uploads to TestPyPI first and then +to PyPI, and authenticates to both through OIDC trusted publishing: the workflow uses no API +token. That requires, on each index, a trusted publisher for the `orcapod` project naming owner +`nauticalab`, repository `orcapod-python`, workflow `release.yml` and environment `testpypi` +(TestPyPI) or `pypi` (PyPI); see *One-time setup* below. ## Branching Model @@ -26,7 +32,7 @@ requires, on each index, a trusted publisher for the `orcapod` project naming ow ## Before You Cut -Do these on `main` before triggering the workflow: +Do these on `metamorphic-labs` `main` before anything is pushed: 1. **Update `CHANGELOG.md`** in a release pull request, the only kind of pull request that edits it. Feature pull requests add `changelog.d/.md` fragments instead (see @@ -58,6 +64,19 @@ Do these on `main` before triggering the workflow: Per-release checklists live in `superpowers/plans/` (for example `2026-11-09-v0.2-release-checklist.md`). +## Pre-push checklist + +All of these hold before the developer is asked for the go: + +1. **The full suite is green**: `uv run pytest tests/` passes on the commit to be pushed. +2. **The release pull request is merged** on `metamorphic-labs` (`CHANGELOG.md` folded, section + dated, `changelog.d/` fragments deleted). +3. **The data context freeze is done**: every released manifest is pinned in + `contexts/released_manifests.json` and the golden hash tests pass unchanged. +4. **`main` is green** on `metamorphic-labs`, and `git status` is clean. +5. **The version is free**: the tag does not exist on `nauticalab`, and the version is on + neither PyPI nor TestPyPI. + ## Cutting a Release Releases are triggered from the GitHub Actions UI — no manual `git tag` step required. @@ -65,14 +84,26 @@ Releases are triggered from the GitHub Actions UI — no manual `git tag` step r on origin, so a hand-made `vX.Y.Z` tag means that version can no longer be published by the workflow until the tag is deleted. -1. **Merge your branch into `main`** — open a PR, get it reviewed, merge it. +1. **Push `main` to `nauticalab` as a fast-forward, on the developer's explicit go only.** + A push to a public repository cannot be undone, so no agent does this on its own. -2. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/metamorphic-labs/orcapod-python/actions/workflows/release.yml)** - in the GitHub UI, enter the version (e.g. `0.1.0`), optionally enter a branch name - (defaults to `main`), and click **Run workflow**. + ``` + git remote add nauticalab https://github.com/nauticalab/orcapod-python.git # once + git fetch nauticalab + git merge-base --is-ancestor nauticalab/main main && echo "fast-forward" # must print it + git push nauticalab main:main # no --force + ``` - A leading `v` is stripped automatically — `v0.1.0` and `0.1.0` both work. + Never a squash, never a pull request: commit ids must stay identical in both repositories, + because the version comes from the tag (`hatch-vcs`). Never push the `v0.2.0` tag (or any + tag): the Release workflow creates the release tag on `nauticalab`. The push also replaces + the workflows `nauticalab` carries (its old `tests.yml` is deleted by it, as `main` removed + that file). + +2. **Run Release on `nauticalab`** — go to + **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** + in the GitHub UI, enter the version (e.g. `0.3.0`), optionally enter a branch name + (defaults to `main`), and click **Run workflow**. A leading `v` is stripped automatically. 3. **CI takes over** — the workflow runs the following jobs automatically: @@ -88,6 +119,32 @@ workflow until the tag is deleted. - PyPI: in the `pypi` environment, after TestPyPI succeeds, `uv publish dist/*` with an OIDC token, then creates the GitHub Release for the tag with generated release notes and the dist files attached - Linear: `linear-sync` starts immediately after build (parallel to publish); `linear-complete` runs after both PyPI publish and linear-sync succeed +4. **Fetch the tag back into `metamorphic-labs`** once the run has succeeded: + + ``` + git fetch nauticalab --tags + git push origin vX.Y.Z + ``` + + `metamorphic-labs` then holds the same tag on the same commit. + +## One-time setup (only the developer can do this) + +On `nauticalab/orcapod-python` and the package indexes; no agent changes settings, secrets or +environments: + +1. Create the environments `testpypi` and `pypi` (`Settings → Environments`). +2. On pypi.org and test.pypi.org, an owner of the `orcapod` project opens + *Manage → Publishing* and adds a GitHub trusted publisher with owner `nauticalab`, + repository `orcapod-python`, workflow `release.yml` and environment `pypi` (PyPI) or + `testpypi` (TestPyPI). An upload from a repository that no trusted publisher names is + rejected, which fails `publish-testpypi` after the tag has been pushed. Remove the entries + that name `metamorphic-labs`, if any. +3. Protect `main` on `nauticalab` (no force push, no deletion). It is written only by the + fast-forward push above. +4. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on + `metamorphic-labs` and are skipped on `nauticalab`. + ## Pre-releases Release candidates (e.g. `0.1.0rc1`) follow exactly the same path — enter the version in the @@ -104,8 +161,12 @@ workflow. **Example:** cutting `v0.1.1` from `hotfix/0.1.x`: +0. **Make sure `hotfix/0.1.x` on `nauticalab` points at the commit to release.** The workflow + resolves the **Branch** input only there and tags its tip. If the branch is missing or behind, + fast-forward it (on the developer's explicit go only, never with `--force`): + `git push nauticalab hotfix/0.1.x:hotfix/0.1.x` 1. **Trigger the release workflow** — go to - **[Actions → Release → Run workflow](https://github.com/metamorphic-labs/orcapod-python/actions/workflows/release.yml)** + **[Actions → Release → Run workflow](https://github.com/nauticalab/orcapod-python/actions/workflows/release.yml)** 2. In the **Use workflow from** dropdown, keep `main` selected (so the workflow version includes the `Branch` field — do not select `hotfix/0.1.x` here) 3. Set **Version** to `0.1.1` @@ -155,51 +216,20 @@ manual version bump is needed. ## Linear Release System -When a release is triggered, the CI automatically transitions all Linear issues in -**"Merged"** status (whose PRs were included in the release) to **"Done"**. This is -handled by two jobs in `release.yml` that call `linear/linear-release-action`, plus a -separate `release-sync.yml` that runs on every push to `main`. - -### How it works +Linear is wired to the development repository only. `release-sync.yml` and the `linear-sync` and +`linear-complete` jobs of `release.yml` run only when +`github.repository == 'metamorphic-labs/orcapod-python'`; on `nauticalab` they are skipped, which +does not block or fail the release (no publish job depends on them). They call +`linear/linear-release-action` and transition the issues in **"Merged"** status to **"Done"**. | Trigger | Workflow / Job | Action | Effect | |---------|---------------|--------|--------| -| Push to `main` | `release-sync.yml / sync` | `sync` (no version) | Associates the merged PR with the open Linear release draft | -| Release workflow `build` succeeds | `release.yml / linear-sync` | `sync --version ` | Finalises the commit set for this release version in Linear | -| After `publish-pypi` succeeds | `release.yml / linear-complete` | `complete --version ` | Marks the release done in Linear; triggers Merged → Done | - -### Prerequisites - -Before cutting the first release with this system active (items 1 and 4 need a repository -admin, item 2 a Linear workspace admin, item 3 an owner of the `orcapod` project on PyPI and -TestPyPI): - -1. **Set the `LINEAR_ACCESS_KEY` repo secret** in `metamorphic-labs/orcapod-python` GitHub - settings (`Settings → Secrets and variables → Actions → New repository secret`). - Use the same Linear API key as `metamorphic-brain/axon`. - -2. **Configure a Linear release** in the workspace for orcapod-python via the Linear UI: - set the release name, which statuses count as "included" (at minimum: Merged), and - the target state for the transition (Done). - -3. **Point the PyPI and TestPyPI trusted publishers at this repository.** The repository - moved from `nauticalab/orcapod-python` to `metamorphic-labs/orcapod-python`. On - pypi.org and test.pypi.org, an owner of the `orcapod` project opens *Manage → Publishing* - and adds a GitHub trusted publisher with owner `metamorphic-labs`, repository - `orcapod-python`, workflow `release.yml` and environment `pypi` (PyPI) or `testpypi` - (TestPyPI). An upload from a repository that no trusted publisher names is rejected, which fails `publish-testpypi` after the tag has been pushed. -4. **Create the `testpypi` and `pypi` environments** in the repository settings - (`Settings → Environments`), if they do not exist. - -If `LINEAR_ACCESS_KEY` is not set: the `release-sync` workflow runs but the Linear action -may fail; the `linear-sync` and `linear-complete` jobs in `release.yml` fail but do not -block publishing (they have no dependents in the publish chain). - -### Replicating for other orcapod repos - -1. Copy `.github/workflows/release-sync.yml` verbatim. -2. Add `linear-sync` and `linear-complete` jobs to the repo's release workflow, - adjusting `needs:` in `linear-complete` to point at whichever job creates the GitHub - Release. -3. Set the `LINEAR_ACCESS_KEY` repo secret. -4. Configure a Linear release in the workspace for the new repo. +| Push to `main` (`metamorphic-labs`) | `release-sync.yml / sync` | `sync` (no version) | Associates the merged PR with the open Linear release draft | +| Release workflow `build` succeeds (`metamorphic-labs` only) | `release.yml / linear-sync` | `sync --version ` | Finalises the commit set for the version | +| After `publish-pypi` succeeds (`metamorphic-labs` only) | `release.yml / linear-complete` | `complete --version ` | Marks the release done in Linear | + +Because the Release workflow runs on `nauticalab`, these jobs do not run for a release at all +today; completing the Linear release is a manual step until the developer decides otherwise +(the issue ITL-1007 notes this "needs more thought"). Setup on `metamorphic-labs`: the +`LINEAR_ACCESS_KEY` repo secret (the key `metamorphic-brain/axon` uses) and a Linear release +configured in the workspace. diff --git a/changelog.d/ITL-1007.md b/changelog.d/ITL-1007.md new file mode 100644 index 000000000..d8b23db77 --- /dev/null +++ b/changelog.d/ITL-1007.md @@ -0,0 +1,11 @@ +### Changed + +- The repository is prepared for release from the public `nauticalab/orcapod-python` + repository (ITL-1007). The project URLs in `pyproject.toml` and the links in `README.md` point + there. `RELEASING.md` describes the flow: development stays on `metamorphic-labs`, `main` is + pushed to `nauticalab` by fast-forward at a release point, and the Release workflow runs there. +- The Linear jobs, the Codecov uploads and the SpiralDB integration job run only on + `metamorphic-labs`; a new `lint.yml` workflow runs ruff, the format check and basedpyright on + `nauticalab`. +- Removed: the `workshop/` tutorial-dataset tool and its tests (`tests/test_workshop/`), and its + implementation plan. They stay in git history. diff --git a/pyproject.toml b/pyproject.toml index c41a5b765..69b47d483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,10 @@ classifiers = [ ] [project.urls] -Homepage = "https://github.com/metamorphic-labs/orcapod-python" -Repository = "https://github.com/metamorphic-labs/orcapod-python" -Changelog = "https://github.com/metamorphic-labs/orcapod-python/blob/main/CHANGELOG.md" -Issues = "https://github.com/metamorphic-labs/orcapod-python/issues" +Homepage = "https://github.com/nauticalab/orcapod-python" +Repository = "https://github.com/nauticalab/orcapod-python" +Changelog = "https://github.com/nauticalab/orcapod-python/blob/main/CHANGELOG.md" +Issues = "https://github.com/nauticalab/orcapod-python/issues" [project.optional-dependencies] redis = ["redis>=6.2.0"] diff --git a/superpowers/plans/2026-11-11-itl-933-edi-tutorial-dataset.md b/superpowers/plans/2026-11-11-itl-933-edi-tutorial-dataset.md deleted file mode 100644 index ac3945d0a..000000000 --- a/superpowers/plans/2026-11-11-itl-933-edi-tutorial-dataset.md +++ /dev/null @@ -1,144 +0,0 @@ -# ITL-933: build the workshop Spiral project `edi-tutorial-v1` - -**Linear:** ITL-933 (Urgent; the workshop is November 17, 2026). -**Branch:** `eywalker/itl-933-build-the-workshop-spiral-project-edi` (from `origin/main`). -**Brief:** the coordinator's dispatch `task_58826b467f46`. Phase 1 is design plus a dry-run-tested -tool, with no writes to Spiral. Phase 2 (after approval): the coordinator runs create, copy and -verify; I fix what the live run shows and open the PR (body: `Part of ITL-933`). - -## Goal - -A reproducible tool under `workshop/edi_tutorial_dataset/` that copies a small subset of the -V1 EDI sample (`edi-sample-v1-855632`, dataset `default`) into a new Spiral project. The copy -declares columns non-nullable where the EDI schema specification (v0.3.0) requires them. It is -tested against a fake of the pyspiral client and documented in a short README. - -## Facts measured in Phase 1 (read-only, November 11, 2026, pyspiral 0.16.33) - -Scratch scripts and outputs are in the session scratchpad -(`/private/tmp/claude-502/-Users-eywalker-orca-workspaces-orcapod-python-itl-933-edi-tutorial-dataset/f447d959-4b29-40ec-b9bd-ec08e6d758b2/scratchpad/`): -`measure.py` → `measure.json`, `window_probe.py` → `windowA.json`, `schemas.py` → `schemas.json`, -`parse_spec.py` → `spec.json`, `compare.py` → `compare.txt`, `storage_probe.py`. - -* **Schema vs. spec.** The 24 sample tables the spec defines match it exactly: same columns, - same Arrow types, same key columns in the same order, no renames. Not in the spec: - `sample_info`, `sample_unit` (sample bookkeeping) and `experiment_screen_capture_frame_270p` - (the spec has `experiment_screen_capture`, one MP4 blob per experiment, instead). - Spec tables absent from the sample: the five `_run` tables, `experiment_sync_event`, - `experiment_screen_capture`, `atlas_info`, `atlas_preference_history`, `mri_dataset`. -* **Nulls.** Every column the spec marks required has zero nulls in the whole table, key - columns included. `lfp_voltage_uv` was checked inside the window only (0 row nulls; exactly - one null item per row, which the spec allows: `fixed_size_list(f32?)[384]`). -* **One clock.** Every `timestamp` is µs relative to `session_start_datetime`: spikes and LFP - start at `recording_start_timestamp` (1,896.000 s), gaze at `experiment_start_timestamp`, and - the 270p frames carry the same instants as `experiment_screen_capture_event` (as `int64`). - One half-open window `[t0, t1)` therefore applies to every time-keyed table unchanged. -* **Rates.** LFP 600 Hz; gaze 200 Hz; screen-capture events 120 Hz; 270p frames about 30 Hz - (about 41 KB each); free-viewing events about 26/s; spikes about 707/s over 100 units. -* **Storage.** All 52 `edi-sample-*` projects use Volume `vol_01m6s4jsjeedya88dc8t2ns96q` - (selected). The Volume's URI is visible only to org owners; the infra README says the - `edi-sample-datasets` Volume is `s3://edi-sample-datasets/` (whole bucket). A project created - with `--volume` cannot choose a sub-prefix; only a separately registered Volume can. - -## Subset proposal (to confirm with the developer) - -* Window A: start of trial 215 to end of trial 225, `[5226928854, 5370814965)` µs, 143.9 s, - 11 consecutive valid trials. -* Units: the 32 units with the most spikes in the window among units whose `bombcell_label` is - not `noise` (63,349 spikes). -* Trials kept: those overlapping the window, plus any trial referenced by a kept row (the - window is trial-aligned, so 215–225). -* Everything else is copied whole. No value is ever edited. - -## Design - -Files: - -* `workshop/edi_tutorial_dataset/edi_tutorial_dataset.py`: one self-contained module and CLI - (loaded by path in tests, the layout PR #274 uses for `docs/tutorial/workshop_data.py`). - Subcommands: `derive-map` (spec + live source schemas → map), `plan` (read-only dry run of - the whole copy: rows, bytes, null checks, foreign keys), `create`, `probe`, `copy` - (`--dry-run`), `verify`, `render-map`. -* `nullability_map.json`: per table and column: spec type, target nullability, key flag, - foreign keys, and `pending` entries for what the developer must decide. `copy` refuses to run - while any entry is pending. -* `subset.json`: window, unit IDs, and how each table is selected. -* `README.md`: what the dataset is, the commands, the nullability table. -* Tests: `tests/test_workshop/test_edi_tutorial_dataset.py` and - `tests/test_workshop/fake_spiral.py` (an in-memory fake of the pyspiral surface the tool - uses, including filter expressions and the live type rule for comparison literals). - -Safety: - -* Reads only from `--source-project`; the source is held behind a read-only facade. -* Writes only to `--target-project`, which must equal `--allow-writes`, differ from the - source, not start with `edi-sample-v`, and carry the tool's marker table - (`meta.edi_tutorial_build`), which only `create` writes, into a project it has just created. - Every table it writes carries the marker in its metadata; a target holding any table - without the marker is refused. -* Before any write, each batch is checked: column names and types equal the source's exactly - (type strict), and every non-nullable column (top level and declared struct children) has no - nulls. A null raises; nothing is coerced. -* Re-runs are idempotent: a table whose status metadata says complete for the same subset - digest is skipped; an incomplete table is rewritten (Spiral upserts by key); a complete - table from a different subset is refused. - -`verify`: row counts per table against the plan; target types equal source types exactly; -nullability equals the map; key schema equals the source's key columns, non-null; no nulls in -required columns; every foreign key and trial reference resolves; values equal the source -selection row by row (streamed, both sides key-sorted). - -## Steps - -1. [x] Read the brief, the issue, the spec, the inputs, PR #274's layout, the ITL-926 findings - and fake, and the pyspiral 0.16.33 API. -2. [x] Live read-only measurements (above). -3. [x] Write the fake and failing tests for: the target guards, the marker, null rejection, - type strictness, idempotent re-run, dry run writes nothing, verify catches each defect. -4. [x] Implement the tool until the tests pass; ruff and basedpyright clean. -5. [x] Write `nullability_map.json` (with `derive-map`) and `subset.json`; run `plan` live - (read-only) and record the per-table numbers. -6. [x] README (draft) with the human nullability table and the commands. -7. [ ] WIP commit and push; send the coordinator ONE `ask` with the mismatches, the subset - numbers, the project/volume commands and the live-step commands. -8. [ ] Phase 2 (after approval): the coordinator runs `create`, `probe`, `copy`, `verify`; fix - what the live run shows; record the project ID in the README; full suite; PR. - -## Notes for a successor - -* Live calls work from this terminal with `uv run --with pyspiral==0.16.33 ...`; the repo lock - (0.14.9) does not use the developer's login. -* A scan must project at least one non-key column (`Reading key columns without including at - least one column from any column group is currently not supported`); use `scan_keys` for keys - only. -* Filter literals must carry the column's exact Arrow type (`pa.scalar(x, pa.uint64())`); an - `int64` literal against a `u64` column fails with `Cannot compare different DTypes`. - -## Phase 1 results (November 11, 2026) - -* `plan` live, read-only: `superpowers/plans/2026-11-11-itl-933-plan-live.txt`. 204,948 rows, - 361 MB uncompressed, 103 s, peak RSS 715 MB. -* Source integrity break found by `plan`: 58 of 383 `lfp_location` rows reference - `('AFNI_D99v2', 'White Matter')`, which `atlas_label` does not hold. -* Tests: 34 pass; mutation checks (null check, marker check, typed literals, trial references, - prefix guard, abort) each fail at least one test. -* basedpyright: 0 errors, about 1,160 warnings (mostly `reportAny` from the untyped client and - unannotated fixtures). To reduce in Phase 2 with Protocol types for the client surface. - -## Phase 2 notes (November 11, 2026) - -* Developer decisions applied (decisions.json): see the README. Project created by the - coordinator: `edi-tutorial-v1-841226` (marker table only so far). -* Probe on the server: SpiralDB stores every top-level column nullable (keys included) and - every struct field nullable; only list items keep a non-null flag. pyspiral's - `Schema.from_arrow` already makes top-level columns nullable (my local check suggested - struct fields kept the flag; the server does not). A top-level null under a non-null - declaration is refused by Arrow's batch validation, not by a stored constraint. -* Consequence: copy enforces required columns before writing; verify expects the storable - schema and checks required columns from the data. Orcapod sources still show `T | None`; - the developer is choosing between O1 (nullability record in table metadata + Orcapod - support), O2 (accept `T | None`) and O3 (source parameter). `annotate --key` can add the - record later without touching data. -* `copy --tables X --replace` refreshes a table after a source fix (the EDI owner is adding - the `White Matter` row to `atlas_label`). -* `copy --dry-run` ran live against `edi-tutorial-v1-841226`: exit 0, 204,947 rows. diff --git a/superpowers/plans/2026-11-12-itl-1007-public-release.md b/superpowers/plans/2026-11-12-itl-1007-public-release.md new file mode 100644 index 000000000..f3146a9fc --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1007-public-release.md @@ -0,0 +1,13 @@ +# ITL-1007: make the repository ready for public release from nauticalab + +Scope after the developer's decisions of 2026-11-12: Spiral project ids stay as they are (they +name only the sample dataset and sit behind credentials), and anything workshop related moves +to the ediop repository. So: no id scrub, no guard test; the workshop tool, its tests and its +plan are removed (they stay in git history); URLs, workflow repository guards, `lint.yml`, +`RELEASING.md` and the agent instructions stay. + +Facts checked: nauticalab main is 61d0f18e, an ancestor of our main; nauticalab has `tests.yml`, +deleted on our main by 07e5a2ea (#267), so a fast-forward push deletes it. + +Order: undo scrub -> drop guard -> remove workshop plan -> instructions and fragment -> full +suite on 3.11/3.12/3.13 -> push once -> CodeRabbit. diff --git a/tests/test_workshop/__init__.py b/tests/test_workshop/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_workshop/fake_spiral.py b/tests/test_workshop/fake_spiral.py deleted file mode 100644 index 6d1ba87a0..000000000 --- a/tests/test_workshop/fake_spiral.py +++ /dev/null @@ -1,497 +0,0 @@ -"""An in-memory fake of the pyspiral surface ``edi_tutorial_dataset.py`` uses. - -Modelled on pyspiral 0.16.33 and on what was seen live (ITL-926, ITL-933): - -* ``Spiral.project``, ``list_projects``, ``create_project(id_prefix=, description=, - volume_id=)``, ``scan(table_or_select, where=)``. -* ``Project.list_tables`` (``.dataset``, ``.table``), ``table`` (``ValueError`` when - missing), ``create_table(identifier, key_schema=)``, ``drop_table``. -* ``Table[...]`` and ``Table.select(...)`` build expressions; comparisons and ``&`` - combine them. A comparison literal must carry the column's exact Arrow type, as live - SpiralDB refuses ``u64`` against ``i64`` ("Cannot compare different DTypes"). -* ``Table.key_schema.to_arrow()``, ``Table.schema().to_arrow()`` (the key schema until - data is written), ``get_metadata`` / ``set_metadata`` (upsert), ``txn()`` with - ``write``, ``commit``, ``abort`` and the context-manager protocol. -* Writes upsert by key and keep rows sorted by key. The first write fixes the table's - schema; a later write whose schema differs beyond nullability raises. A struct-level - null raises, as SpiralDB refuses one. -* Nullability as the server keeps it (live probe on edi-tutorial-v1-841226, November 11, - 2026): every top-level column, keys included, and every struct field is stored - nullable; only list and fixed-size-list items keep a non-null flag. A top-level null - under a non-null declaration is refused (Arrow validates the incoming batch); a null - in a struct field declared non-null is accepted. - -Every mutating call is appended to ``Spiral.log`` as ``(project_id, operation, name)``. -""" - -from __future__ import annotations - -import dataclasses -import itertools -from collections.abc import Iterator -from typing import Any - -import pyarrow as pa -import pyarrow.compute as pc - - -class Expr: - """A filter expression over one table's columns.""" - - def evaluate(self, table: pa.Table) -> Any: # pragma: no cover - """Evaluate the expression against ``table``.""" - raise NotImplementedError - - def _compare(self, other: Any, op: str) -> Expr: - """Return a comparison of this expression with ``other`` using ``op``.""" - return Compare(self, other, op) - - def __ge__(self, other: Any) -> Expr: - """Build ``self >= other``.""" - return self._compare(other, "greater_equal") - - def __gt__(self, other: Any) -> Expr: - """Build ``self > other``.""" - return self._compare(other, "greater") - - def __le__(self, other: Any) -> Expr: - """Build ``self <= other``.""" - return self._compare(other, "less_equal") - - def __lt__(self, other: Any) -> Expr: - """Build ``self < other``.""" - return self._compare(other, "less") - - def __and__(self, other: Expr) -> Expr: - """Combine two expressions with a logical and.""" - return And(self, other) - - -@dataclasses.dataclass(eq=False) -class Column(Expr): - """A column reference, as ``Table[name]`` returns it.""" - - name: str - - def evaluate(self, table: pa.Table) -> Any: - """Return the named column of ``table``.""" - return table.column(self.name) - - -@dataclasses.dataclass(eq=False) -class Compare(Expr): - """A comparison of an expression with a literal. - - The literal must be a ``pa.Scalar`` of the column's exact Arrow type, as live - SpiralDB refuses ``u64`` against ``i64`` ("Cannot compare different DTypes"). - """ - - left: Expr - right: Any - op: str - - def evaluate(self, table: pa.Table) -> Any: - """Return the comparison's boolean mask over ``table``. - - Raises: - ValueError: if the literal's Arrow type differs from the column's. - """ - col = self.left.evaluate(table) - if not isinstance(self.right, pa.Scalar) or self.right.type != col.type: - got = ( - self.right.type - if isinstance(self.right, pa.Scalar) - else type(self.right) - ) - raise ValueError(f"Cannot compare different DTypes {col.type} and {got}") - return getattr(pc, self.op)(col, self.right) - - -@dataclasses.dataclass(eq=False) -class And(Expr): - """Two expressions combined with ``&``.""" - - left: Expr - right: Expr - - def evaluate(self, table: pa.Table) -> Any: - """Return the logical and of both sides' masks over ``table``.""" - return pc.and_(self.left.evaluate(table), self.right.evaluate(table)) - - -@dataclasses.dataclass(eq=False) -class Select: - """A column projection of a table, as ``Table.select(...)`` returns it.""" - - table: FakeTable - columns: tuple[str, ...] - - -class _Schema: - """A Spiral schema; ``to_arrow()`` gives the Arrow schema.""" - - def __init__(self, schema: pa.Schema) -> None: - """Wrap ``schema``.""" - self._schema = schema - - def to_arrow(self) -> pa.Schema: - """Return the Arrow schema.""" - return self._schema - - @property - def names(self) -> list[str]: - """Return the column names.""" - return self._schema.names - - -@dataclasses.dataclass(frozen=True) -class TableResource: - """An entry of ``Project.list_tables`` (``.dataset``, ``.table``).""" - - project_id: str - dataset: str - table: str - - -@dataclasses.dataclass -class _Stored: - """A stored table: key schema, rows (``None`` until written) and metadata.""" - - key_schema: pa.Schema - data: pa.Table | None = None - metadata: dict[str, bytes] = dataclasses.field(default_factory=dict) - - -def stored_type(t: pa.DataType) -> pa.DataType: - """Return ``t`` as the server stores it: struct fields nullable, list items kept.""" - if pa.types.is_struct(t): - return pa.struct([pa.field(f.name, stored_type(f.type), True) for f in t]) - if pa.types.is_fixed_size_list(t): - item = t.value_field - return pa.list_( - pa.field(item.name, stored_type(item.type), item.nullable), t.list_size - ) - if pa.types.is_list(t): - item = t.value_field - return pa.list_(pa.field(item.name, stored_type(item.type), item.nullable)) - if pa.types.is_large_list(t): - item = t.value_field - return pa.large_list(pa.field(item.name, stored_type(item.type), item.nullable)) - return t - - -def stored_schema(schema: pa.Schema) -> pa.Schema: - """Return ``schema`` as the server stores it: every top-level column nullable.""" - return pa.schema([pa.field(f.name, stored_type(f.type), True) for f in schema]) - - -class FakeTxn: - """A write transaction, as ``Table.txn()`` returns it. - - Batches are checked on ``write`` and applied on ``commit``; as a context manager it - commits on a clean exit and aborts on an exception. - """ - - def __init__(self, table: FakeTable) -> None: - """Open a transaction on ``table``.""" - self._table = table - self._batches: list[pa.RecordBatch] = [] - self._done = False - - def __enter__(self) -> FakeTxn: - """Return the transaction.""" - return self - - def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: - """Commit on a clean exit; abort on an exception.""" - if exc_type is None: - self.commit() - else: - self.abort() - - def write(self, batch: pa.RecordBatch | pa.Table) -> None: - """Check ``batch`` against the table and stage it. - - Raises: - ValueError: if the batch breaks the table's key or stored schema, holds a - struct-level null, or a null under a non-null top-level declaration. - """ - self._table._check_write(batch) - if isinstance(batch, pa.Table): - self._batches.extend(batch.to_batches()) - else: - self._batches.append(batch) - - def commit(self) -> None: - """Apply the staged batches once; later calls do nothing.""" - if not self._done: - self._done = True - if self._batches: - self._table._commit(pa.Table.from_batches(self._batches)) - - def abort(self) -> None: - """Discard the staged batches.""" - self._done = True - self._batches = [] - - -class FakeTable: - """A handle on a stored table, as ``Project.table`` returns it.""" - - def __init__(self, spiral: FakeSpiral, project_id: str, ident: str) -> None: - """Refer to table ``ident`` of project ``project_id``.""" - self._spiral = spiral - self._project_id = project_id - self._ident = ident - - @property - def _stored(self) -> _Stored: - """Return the table's stored state.""" - return self._spiral._projects[self._project_id][self._ident] - - def __getitem__(self, name: str) -> Column: - """Return a column expression for ``name``.""" - return Column(name) - - def select(self, *columns: str) -> Select: - """Return a projection of ``columns`` for ``Spiral.scan``.""" - return Select(self, columns) - - @property - def key_schema(self) -> _Schema: - """Return the key schema, as the server stores it (every column nullable).""" - return _Schema(self._stored.key_schema) - - def schema(self) -> _Schema: - """Return the stored schema: the key schema until data is written.""" - data = self._stored.data - return _Schema(self._stored.key_schema if data is None else data.schema) - - def get_metadata(self) -> dict[str, bytes]: - """Return a copy of the table's metadata.""" - return dict(self._stored.metadata) - - def set_metadata(self, metadata: dict[str, bytes]) -> None: - """Upsert ``metadata`` into the table's metadata (logged).""" - self._spiral.log.append((self._project_id, "set_metadata", self._ident)) - self._stored.metadata.update(metadata) - - def txn(self) -> FakeTxn: - """Open a write transaction.""" - return FakeTxn(self) - - def _check_write(self, batch: pa.RecordBatch | pa.Table) -> None: - """Log the write and check ``batch`` as the server would. - - Raises: - ValueError: if a key column is missing or differs from the key schema, the - schema differs from the stored one beyond nullability, a struct column - holds a struct-level null, or a non-nullable top-level column holds - nulls. - """ - self._spiral.log.append((self._project_id, "write", self._ident)) - self._spiral.written_schemas.setdefault( - (self._project_id, self._ident), [] - ).append(batch.schema) - keys = self._stored.key_schema - for k in keys: - if k.name not in batch.schema.names: - raise ValueError(f"write is missing key column {k.name!r}") - if stored_schema(pa.schema([batch.schema.field(k.name)])).field(0) != k: - raise ValueError( - f"key column {batch.schema.field(k.name)} differs from key schema {k}" - ) - data = self._stored.data - if data is not None and stored_schema(batch.schema) != data.schema: - raise ValueError(f"write schema {batch.schema} differs from {data.schema}") - for i, field in enumerate(batch.schema): - col = batch.column(i) - if pa.types.is_struct(field.type) and col.null_count: - raise ValueError("SpiralDB does not allow struct-level nulls") - if not field.nullable and col.null_count: - raise ValueError( - f"Column '{field.name}' is declared as non-nullable but contains " - "null values" - ) - - def _commit(self, incoming: pa.Table) -> None: - """Upsert ``incoming`` by key, stored nullable, keeping rows sorted by key.""" - stored = self._stored - keys = stored.key_schema.names - incoming = pa.Table.from_arrays( - incoming.columns, schema=stored_schema(incoming.schema) - ) - if stored.data is not None: - new_keys = set( - zip(*(incoming.column(k).to_pylist() for k in keys), strict=True) - ) - keep = pa.array( - [ - k not in new_keys - for k in zip( - *(stored.data.column(k).to_pylist() for k in keys), strict=True - ) - ], - pa.bool_(), - ) - incoming = pa.concat_tables([stored.data.filter(keep), incoming]) - stored.data = incoming.sort_by( - [(k, "ascending") for k in keys] - ).combine_chunks() - - -class FakeScan: - """A scan result, as ``Spiral.scan`` returns it.""" - - def __init__(self, table: pa.Table, batch_rows: int) -> None: - """Hold ``table``, read back in batches of ``batch_rows`` rows.""" - self._table = table - self._batch_rows = batch_rows - - def to_record_batches(self) -> pa.RecordBatchReader: - """Return a reader over the rows in batches of ``batch_rows`` rows.""" - batches = self._table.to_batches(max_chunksize=self._batch_rows) - return pa.RecordBatchReader.from_batches(self._table.schema, batches) - - def to_table(self) -> pa.Table: - """Return every row as one table.""" - return self._table - - -@dataclasses.dataclass -class ProjectInfo: - """An entry of ``Spiral.list_projects``, as ``create_project`` also returns.""" - - id: str - description: str = "" - - -class FakeProject: - """A project, as ``Spiral.project`` returns it.""" - - def __init__(self, spiral: FakeSpiral, project_id: str) -> None: - """Refer to project ``project_id``.""" - self._spiral = spiral - self.id = project_id - - @property - def _tables(self) -> dict[str, _Stored]: - """Return the project's stored tables by identifier.""" - return self._spiral._projects.setdefault(self.id, {}) - - def list_tables(self) -> list[TableResource]: - """Return the project's tables, sorted by identifier.""" - return [ - TableResource(self.id, *ident.split(".", 1)) - for ident in sorted(self._tables) - ] - - def table(self, identifier: str) -> FakeTable: - """Return the table ``identifier``. - - Raises: - ValueError: if the table does not exist. - """ - if identifier not in self._tables: - raise ValueError(f"Table not found: {self.id}.{identifier}") - return FakeTable(self._spiral, self.id, identifier) - - def create_table(self, identifier: str, *, key_schema: pa.Schema) -> FakeTable: - """Create an empty table keyed by ``key_schema`` (stored nullable; logged). - - Raises: - ValueError: if the table already exists. - """ - self._spiral.log.append((self.id, "create_table", identifier)) - if identifier in self._tables: - raise ValueError(f"Table already exists: {identifier}") - self._tables[identifier] = _Stored(stored_schema(key_schema)) - return FakeTable(self._spiral, self.id, identifier) - - def drop_table(self, identifier: str) -> None: - """Drop the table ``identifier`` (logged).""" - self._spiral.log.append((self.id, "drop_table", identifier)) - del self._tables[identifier] - - -class FakeSpiral: - """The client. ``batch_rows`` sets how many rows each scanned batch holds.""" - - def __init__(self, *, batch_rows: int = 3) -> None: - """Start with no projects; scans return batches of ``batch_rows`` rows.""" - self._projects: dict[str, dict[str, _Stored]] = {} - self._descriptions: dict[str, str] = {} - self._suffix = itertools.count(100001) - self.batch_rows = batch_rows - self.log: list[tuple[str, str, str]] = [] - self.written_schemas: dict[tuple[str, str], list[pa.Schema]] = {} - - # -- setup helpers for tests (not part of pyspiral) -------------------------- - def add_table( - self, project_id: str, identifier: str, table: pa.Table, keys: list[str] - ) -> None: - """Store ``table`` as-is (setup only; not logged).""" - projects = self._projects.setdefault(project_id, {}) - key_schema = pa.schema([table.schema.field(k) for k in keys]) - projects[identifier] = _Stored( - key_schema, table.sort_by([(k, "ascending") for k in keys]) - ) - self._descriptions.setdefault(project_id, "") - - def rows(self, project_id: str, identifier: str) -> pa.Table: - """Return the stored rows of a table (test helper, not part of pyspiral).""" - data = self._projects[project_id][identifier].data - assert data is not None - return data - - # -- pyspiral surface --------------------------------------------------------- - def project(self, project_id: str) -> FakeProject: - """Return the project ``project_id``.""" - return FakeProject(self, project_id) - - def list_projects(self) -> list[ProjectInfo]: - """Return every project with its description, sorted by ID.""" - return [ - ProjectInfo(p, self._descriptions.get(p, "")) - for p in sorted(self._projects) - ] - - def create_project( - self, *, id_prefix: str, description: str, volume_id: str | None = None - ) -> ProjectInfo: - """Create a project named ``id_prefix`` plus a numeric suffix (logged).""" - pid = f"{id_prefix}-{next(self._suffix)}" - self.log.append((pid, "create_project", volume_id or "")) - self._projects[pid] = {} - self._descriptions[pid] = description - return ProjectInfo(pid, description) - - def scan( - self, projection: FakeTable | Select, *, where: Expr | None = None - ) -> FakeScan: - """Scan a table or a projection, optionally filtered by ``where``. - - Raises: - ValueError: if the projection selects key columns only, which SpiralDB does - not support, or a comparison literal's type differs from its column's. - """ - table = projection.table if isinstance(projection, Select) else projection - data = table._stored.data - if data is None: - data = table._stored.key_schema.empty_table() - if isinstance(projection, Select): - keys = set(table._stored.key_schema.names) - if set(projection.columns) <= keys: - raise ValueError( - "Reading key columns without including at least one column from " - "any column group is currently not supported" - ) - data = data.select(list(projection.columns)) - if where is not None: - data = data.filter(where.evaluate(data)) - return FakeScan(data, self.batch_rows) - - -def iter_rows(table: pa.Table) -> Iterator[dict[str, Any]]: - """Yield the rows of ``table`` as dictionaries.""" - yield from table.to_pylist() diff --git a/tests/test_workshop/test_edi_tutorial_dataset.py b/tests/test_workshop/test_edi_tutorial_dataset.py deleted file mode 100644 index 8b51ece53..000000000 --- a/tests/test_workshop/test_edi_tutorial_dataset.py +++ /dev/null @@ -1,1550 +0,0 @@ -"""The workshop EDI dataset tool, run against an in-memory fake of pyspiral. - -``workshop/edi_tutorial_dataset/edi_tutorial_dataset.py`` builds a Spiral project the -workshop depends on and may write to Spiral, so its guards and its copy are tested like -library code. The source here is a small synthetic project with the EDI sample's shapes: -composite keys, ``u64`` and ``i64`` timestamps, a column group, a ``fixed_size_list`` -column, and tables in and outside the specification. -""" - -from __future__ import annotations - -import copy as copy_module -import datetime -import importlib.util -import json -import sys -from pathlib import Path -from types import ModuleType -from typing import Any - -import numpy as np -import pyarrow as pa -import pyarrow.compute as pc -import pytest - -from tests.test_workshop.fake_spiral import FakeSpiral - -REPO_ROOT = Path(__file__).resolve().parents[2] -TOOL_DIR = REPO_ROOT / "workshop" / "edi_tutorial_dataset" -TOOL = TOOL_DIR / "edi_tutorial_dataset.py" - -SOURCE = "edi-sample-v1-855632" - - -def _load_tool() -> ModuleType: - """Import the tool script as the ``edi_tutorial_dataset`` module.""" - spec = importlib.util.spec_from_file_location("edi_tutorial_dataset", TOOL) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules["edi_tutorial_dataset"] = module # dataclasses resolve their module - spec.loader.exec_module(module) - return module - - -tool = _load_tool() - -# A cut-down specification in the format of edi_schema_spec.md. -MINI_SPEC = """ -#### session - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| subject_id | utf8 | PK | Subject | -| session_start_datetime | datetime | PK | Start | -| booth | utf8 | | Booth | -| session_notes | utf8? | | Notes | - -#### experiment - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| experiment_preprocessing_hash | utf8 | PK, FK | hash | -| subject_id | utf8 | PK, FK | Subject (FK → `session`) | -| session_start_datetime | datetime | PK, FK | Start (FK → `session`) | -| experiment_index | u32 | PK | index | -| task_id | utf8 | FK | Task | - -#### experiment_freeviewing_trial - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| experiment_preprocessing_hash | utf8 | PK, FK | (FK → `experiment`) | -| subject_id | utf8 | PK, FK | (FK → `experiment`) | -| session_start_datetime | datetime | PK, FK | (FK → `experiment`) | -| experiment_index | u32 | PK, FK | (FK → `experiment`) | -| trial_index | u32 | PK | trial | -| trial_start_timestamp | u64 | | start | -| trial_end_timestamp | u64? | | end | - -#### experiment_freeviewing_event - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| experiment_preprocessing_hash | utf8 | PK, FK | (FK → `experiment`) | -| subject_id | utf8 | PK, FK | (FK → `experiment`) | -| session_start_datetime | datetime | PK, FK | (FK → `experiment`) | -| experiment_index | u32 | PK, FK | (FK → `experiment`) | -| timestamp | u64 | PK | time | -| name | utf8 | PK | name | -| trial_index | u32 | | trial | - -#### unit - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| subject_id | utf8 | PK, FK | (FK → `session`) | -| session_start_datetime | datetime | PK, FK | (FK → `session`) | -| unit_id | u32 | PK | unit | -| **Classification labels (`labels` column group)** | | | | -| kilosort_label | bool? | | label | -| bombcell_label | utf8? | | label | - -#### spike_event - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| subject_id | utf8 | PK, FK | (FK → `unit`) | -| session_start_datetime | datetime | PK, FK | (FK → `unit`) | -| timestamp | u64 | PK | time | -| unit_id | u32 | PK, FK | (FK → `unit`) | -| spike_amplitude_uv | f32 | | amplitude | - -#### lfp_timeseries - -| Column | Type | Key | Description | -| --- | --- | --- | --- | -| subject_id | utf8 | PK, FK | (FK → `session`) | -| session_start_datetime | datetime | PK, FK | (FK → `session`) | -| timestamp | u64 | PK | time | -| lfp_voltage_uv | fixed_size_list(f32?)[4] | | voltages | -""" - -SST = datetime.datetime(2026, 3, 24, 17, 54, 17) -HASH = "semantic_v0.1:abc" -EXP_KEYS = { - "experiment_preprocessing_hash": pa.string(), - "subject_id": pa.string(), - "session_start_datetime": pa.timestamp("us"), - "experiment_index": pa.uint32(), -} -EXP_ROW = { - "experiment_preprocessing_hash": HASH, - "subject_id": "Goliath", - "session_start_datetime": SST, - "experiment_index": 1, -} -SESSION_ROW = {"subject_id": "Goliath", "session_start_datetime": SST} - -WINDOW = (100, 200) -# trial 0 ends before the window, 1 lies inside it, 2 straddles its end, 3 starts -# after it but an event inside the window references it, 4 is never referenced. -TRIALS = [(0, 50, 99), (1, 100, 150), (2, 160, 260), (3, 300, 400), (4, 500, None)] -EVENTS = [(90, "a", 0), (120, "b", 1), (170, "c", 2), (199, "d", 3), (250, "e", 2)] -UNITS = [1, 2, 3] -KEPT_UNITS = [1, 3] - - -def _nullable(types: dict[str, pa.DataType]) -> pa.Schema: - """The live sample declares every column nullable, keys included.""" - return pa.schema([pa.field(n, t, True) for n, t in types.items()]) - - -def _table(types: dict[str, pa.DataType], rows: list[dict[str, Any]]) -> pa.Table: - """Return an Arrow table of ``rows`` with every column nullable.""" - return pa.Table.from_pylist(rows, schema=_nullable(types)) - - -def build_source(client: FakeSpiral) -> None: - """Store the synthetic source project in ``client``.""" - add = client.add_table - add( - SOURCE, - "default.session", - _table( - { - "subject_id": pa.string(), - "session_start_datetime": pa.timestamp("us"), - "booth": pa.string(), - "session_notes": pa.string(), - }, - [{**SESSION_ROW, "booth": "2", "session_notes": None}], - ), - ["subject_id", "session_start_datetime"], - ) - add( - SOURCE, - "default.experiment", - _table( - {**EXP_KEYS, "task_id": pa.string()}, - [{**EXP_ROW, "task_id": "FreeViewing"}], - ), - list(EXP_KEYS), - ) - add( - SOURCE, - "default.experiment_freeviewing_trial", - _table( - { - **EXP_KEYS, - "trial_index": pa.uint32(), - "trial_start_timestamp": pa.uint64(), - "trial_end_timestamp": pa.uint64(), - }, - [ - { - **EXP_ROW, - "trial_index": i, - "trial_start_timestamp": s, - "trial_end_timestamp": e, - } - for i, s, e in TRIALS - ], - ), - [*EXP_KEYS, "trial_index"], - ) - add( - SOURCE, - "default.experiment_freeviewing_event", - _table( - { - **EXP_KEYS, - "timestamp": pa.uint64(), - "name": pa.string(), - "trial_index": pa.uint32(), - }, - [ - {**EXP_ROW, "timestamp": t, "name": n, "trial_index": i} - for t, n, i in EVENTS - ], - ), - [*EXP_KEYS, "timestamp", "name"], - ) - labels = pa.struct( - [("kilosort_label", pa.bool_()), ("bombcell_label", pa.string())] - ) - add( - SOURCE, - "default.unit", - _table( - { - "subject_id": pa.string(), - "session_start_datetime": pa.timestamp("us"), - "unit_id": pa.uint32(), - "labels": labels, - }, - [ - { - **SESSION_ROW, - "unit_id": u, - "labels": {"kilosort_label": u == 1, "bombcell_label": None}, - } - for u in UNITS - ], - ), - ["subject_id", "session_start_datetime", "unit_id"], - ) - add( - SOURCE, - "default.spike_event", - _table( - { - "subject_id": pa.string(), - "session_start_datetime": pa.timestamp("us"), - "timestamp": pa.uint64(), - "unit_id": pa.uint32(), - "spike_amplitude_uv": pa.float32(), - }, - [ - { - **SESSION_ROW, - "timestamp": t, - "unit_id": u, - "spike_amplitude_uv": float(t + u), - } - for t in range(90, 261, 10) - for u in UNITS - ], - ), - ["subject_id", "session_start_datetime", "timestamp", "unit_id"], - ) - add( - SOURCE, - "default.lfp_timeseries", - _table( - { - "subject_id": pa.string(), - "session_start_datetime": pa.timestamp("us"), - "timestamp": pa.uint64(), - "lfp_voltage_uv": pa.list_(pa.float32(), 4), - }, - [ - { - **SESSION_ROW, - "timestamp": t, - "lfp_voltage_uv": [1.0, None, float(t), 0.5], - } - for t in range(95, 206, 5) - ], - ), - ["subject_id", "session_start_datetime", "timestamp"], - ) - add( - SOURCE, - "default.experiment_screen_capture_frame_270p", - _table( - { - **EXP_KEYS, - "screen_capture_import_version_id": pa.int64(), - "timestamp": pa.int64(), - "frame_index": pa.uint32(), - "frame_jpeg": pa.large_binary(), - }, - [ - { - **EXP_ROW, - "screen_capture_import_version_id": 0, - "timestamp": t, - "frame_index": i, - "frame_jpeg": b"\xff\xd8" + bytes([i]), - } - for i, t in enumerate(range(70, 251, 30)) - ], - ), - [*EXP_KEYS, "screen_capture_import_version_id", "timestamp", "frame_index"], - ) - add( - SOURCE, - "default.sample_info", - _table( - {"region": pa.string(), "n_units": pa.int64()}, - [{"region": "V1", "n_units": 3}], - ), - ["region"], - ) - - -SUBSET_JSON = { - "dataset": "default", - "window_us": list(WINDOW), - "unit_ids": KEPT_UNITS, - "trial_table": "experiment_freeviewing_trial", - "tables": { - "session": {"mode": "whole"}, - "experiment": {"mode": "whole"}, - "experiment_freeviewing_trial": {"mode": "trials"}, - "experiment_freeviewing_event": {"mode": "window"}, - "unit": {"mode": "units"}, - "spike_event": {"mode": "window_units"}, - "lfp_timeseries": {"mode": "window"}, - "experiment_screen_capture_frame_270p": {"mode": "window"}, - "sample_info": {"mode": "whole"}, - }, -} - - -def _source_schemas(client: FakeSpiral) -> dict[str, tuple[pa.Schema, list[str]]]: - """Return each source table's schema and key columns, for ``derive_map``.""" - project = client.project(SOURCE) - return { - r.table: ( - project.table(f"default.{r.table}").schema().to_arrow(), - project.table(f"default.{r.table}").key_schema.to_arrow().names, - ) - for r in project.list_tables() - } - - -def derived_map(client: FakeSpiral) -> dict[str, Any]: - """The map ``derive-map`` builds for the synthetic source (decisions pending).""" - return tool.derive_map(MINI_SPEC, "0.3.0", SOURCE, _source_schemas(client)) - - -def decide(raw: dict[str, Any]) -> dict[str, Any]: - """Resolve every pending decision the way a developer might.""" - raw = copy_module.deepcopy(raw) - for t in raw["tables"].values(): - for c in t["columns"].values(): - if c["nullable"] is None: - c["nullable"] = not c["key"] - for g in t["groups"].values(): - if g["nullable"] is None: - g["nullable"] = False - return raw - - -@pytest.fixture -def client() -> FakeSpiral: - """Return a fake Spiral client holding the synthetic source project.""" - c = FakeSpiral() - build_source(c) - return c - - -@pytest.fixture -def nmap(client: FakeSpiral): - """Return the derived map with every pending decision resolved.""" - return tool.parse_map(decide(derived_map(client))) - - -@pytest.fixture -def subset(): - """Return the test subset.""" - return tool.parse_subset(SUBSET_JSON) - - -def make_target(client: FakeSpiral) -> str: - """Create a marked target project and return its ID.""" - return tool.create_project( - client, - id_prefix="edi-sample-tutorial", - allow_create="edi-sample-tutorial", - description="edi-tutorial-v1", - volume_id="vol_test", - source_project=SOURCE, - ) - - -def run_copy(client: FakeSpiral, target: str, nmap: Any, subset: Any, **kw: Any): - """Copy ``subset`` from the source into ``target``, allowing writes to it.""" - return tool.copy( - client, - source_project=SOURCE, - target_project=target, - allow_writes=target, - nmap=nmap, - subset=subset, - **kw, - ) - - -def writes_to(client: FakeSpiral, project: str) -> list[tuple[str, str, str]]: - """Return the logged mutating calls on ``project``.""" - return [entry for entry in client.log if entry[0] == project] - - -# ------------------------------------------------------------------------ derive-map - - -class TestDeriveMap: - """``derive_map``, ``apply_decisions`` and ``render_map``.""" - - def test_spec_nullability_and_keys(self, client): - raw = derived_map(client) - session = raw["tables"]["session"]["columns"] - assert ( - session["subject_id"]["nullable"] is False and session["subject_id"]["key"] - ) - assert session["booth"]["nullable"] is False - assert session["session_notes"]["nullable"] is True - assert raw["mismatches"] == [] - - def test_tables_outside_the_spec_and_groups_are_pending(self, client): - nm = tool.parse_map(derived_map(client)) - pending = nm.pending() - assert set(pending) == { - "experiment_screen_capture_frame_270p", - "sample_info", - "unit", - } - assert pending["unit"] == ["labels"] - - def test_type_difference_is_a_mismatch(self, client): - schemas = _source_schemas(client) - schema, keys = schemas["spike_event"] - i = schema.get_field_index("spike_amplitude_uv") - schemas["spike_event"] = ( - schema.set(i, pa.field("spike_amplitude_uv", pa.float64())), - keys, - ) - raw = tool.derive_map(MINI_SPEC, "0.3.0", SOURCE, schemas) - assert any("spike_event.spike_amplitude_uv" in m for m in raw["mismatches"]) - - def test_foreign_keys_use_the_parents_key_columns(self, client): - raw = derived_map(client) - fks = { - fk["parent"]: fk["columns"] - for fk in raw["tables"]["spike_event"]["foreign_keys"] - } - assert fks == {"unit": ["subject_id", "session_start_datetime", "unit_id"]} - event_fks = { - fk["parent"] - for fk in raw["tables"]["experiment_freeviewing_event"]["foreign_keys"] - } - assert event_fks == {"experiment", "experiment_freeviewing_trial"} - - def test_decisions_resolve_every_pending_entry(self, client): - decisions = { - "decided_by": "test", - "tables_not_in_spec": {"keys": "required", "values": "nullable"}, - "column_group_structs": "required", - "accepted_unresolved": [ - {"child": "spike_event", "parent": "unit", "keys": [["x", "y", 1]]} - ], - } - raw = tool.derive_map( - MINI_SPEC, "0.3.0", SOURCE, _source_schemas(client), decisions - ) - nm = tool.parse_map(raw) - assert nm.pending() == {} - frames = nm.tables["experiment_screen_capture_frame_270p"].columns - assert frames["timestamp"].nullable is False - assert frames["frame_jpeg"].nullable is True - assert nm.tables["unit"].groups["labels"].nullable is False - assert nm.tables["spike_event"].foreign_keys[0].accepted_unresolved == { - ("x", "y", 1) - } - - def test_a_decision_for_a_missing_foreign_key_is_refused(self, client): - decisions = { - "accepted_unresolved": [{"child": "session", "parent": "unit", "keys": []}] - } - with pytest.raises(tool.ToolError, match="no foreign key"): - tool.derive_map( - MINI_SPEC, "0.3.0", SOURCE, _source_schemas(client), decisions - ) - - def test_render_map_lists_every_column(self, client): - text = tool.render_map(tool.parse_map(derived_map(client))) - assert "| `spike_amplitude_uv` | `float` | `f32` | | **required** |" in text - assert "| `labels` (column group) | struct | — | | pending |" in text - - -# ---------------------------------------------------------------------------- guards - - -class TestTargetGuards: - """The guards that refuse to create or write an unsafe target project.""" - - def test_allow_writes_must_repeat_the_target(self, client, nmap, subset): - target = make_target(client) - with pytest.raises(tool.TargetRefusedError, match="allow-writes"): - tool.copy( - client, - source_project=SOURCE, - target_project=target, - allow_writes=target + "x", - nmap=nmap, - subset=subset, - ) - - def test_source_is_refused_as_target(self, client, nmap, subset): - with pytest.raises(tool.TargetRefusedError): - run_copy(client, SOURCE, nmap, subset) - - @pytest.mark.parametrize( - "target", ["edi-sample-v2-303737", "edi-sample-v1-tutorial"] - ) - def test_edi_sample_v_prefix_is_refused(self, client, nmap, subset, target): - with pytest.raises(tool.TargetRefusedError, match="edi-sample-v"): - run_copy(client, target, nmap, subset) - - def test_project_without_marker_is_refused(self, client, nmap, subset): - client.add_table( - "edi-sample-46d-924415", - "default.subject", - _table({"subject_id": pa.string()}, [{"subject_id": "x"}]), - ["subject_id"], - ) - with pytest.raises(tool.TargetRefusedError, match="create step"): - run_copy(client, "edi-sample-46d-924415", nmap, subset) - assert writes_to(client, "edi-sample-46d-924415") == [] - - def test_project_with_a_foreign_table_is_refused(self, client, nmap, subset): - target = make_target(client) - client.add_table( - target, "default.other", _table({"k": pa.string()}, [{"k": "x"}]), ["k"] - ) - with pytest.raises(tool.TargetRefusedError, match="did not write"): - run_copy(client, target, nmap, subset) - - def test_create_refuses_a_forbidden_prefix(self, client): - with pytest.raises(tool.TargetRefusedError): - tool.create_project( - client, - id_prefix="edi-sample-v1", - allow_create="edi-sample-v1", - description="x", - volume_id=None, - source_project=SOURCE, - ) - - def test_create_requires_the_prefix_twice(self, client): - with pytest.raises(tool.TargetRefusedError): - tool.create_project( - client, - id_prefix="edi-sample-tutorial", - allow_create=None, - description="x", - volume_id=None, - source_project=SOURCE, - ) - - def test_create_makes_a_marked_project_on_the_volume(self, client): - target = make_target(client) - assert target.startswith("edi-sample-tutorial-") - assert (target, "create_project", "vol_test") in client.log - marker = tool.read_marker(client.project(target)) - assert marker["source_project"] == SOURCE - - def test_pending_decisions_block_the_copy(self, client, subset): - target = make_target(client) - pending = tool.parse_map(derived_map(client)) - with pytest.raises(tool.PendingDecisionError): - run_copy(client, target, pending, subset) - assert [op for _, op, _ in writes_to(client, target)] == [ - "create_project", - "create_table", - "set_metadata", - ] - - -# ------------------------------------------------------------------------------ copy - - -class TestCopy: - """``copy``: what it writes, how it enforces the map, and how it re-runs.""" - - def test_copies_the_subset_and_never_writes_the_source(self, client, nmap, subset): - target = make_target(client) - report = run_copy(client, target, nmap, subset) - assert writes_to(client, SOURCE) == [] - rows = {t.name: t.rows for t in report.tables} - # spikes at 100..190 for units 1 and 3 - assert rows["spike_event"] == 20 - assert ( - sorted( - set( - client.rows(target, "default.spike_event") - .column("unit_id") - .to_pylist() - ) - ) - == KEPT_UNITS - ) - assert rows["lfp_timeseries"] == 20 # 100, 105, ..., 195 - assert client.rows( - target, "default.experiment_screen_capture_frame_270p" - ).column("timestamp").to_pylist() == [100, 130, 160, 190] - assert rows["experiment_freeviewing_event"] == 3 # 120, 170, 199 - assert report.kept_trials == [1, 2, 3] - assert rows["unit"] == 2 and rows["session"] == 1 and rows["sample_info"] == 1 - assert report.problems == [] - - def test_writes_the_maps_nullability_and_spiral_stores_it_nullable( - self, client, nmap, subset - ): - target = make_target(client) - run_copy(client, target, nmap, subset) - written = client.written_schemas[(target, "default.spike_event")][0] - assert all(not f.nullable for f in written) - session = client.written_schemas[(target, "default.session")][0] - assert session.field("session_notes").nullable - assert not session.field("booth").nullable - # SpiralDB keeps every top-level column nullable, keys included ... - stored = client.project(target).table("default.spike_event").schema().to_arrow() - assert all(f.nullable for f in stored) - assert all( - f.nullable - for f in client.project(target) - .table("default.spike_event") - .key_schema.to_arrow() - ) - # ... and the types stay exactly the source's, list item flags included - lfp = client.project(target).table("default.lfp_timeseries").schema().to_arrow() - assert lfp.field("lfp_voltage_uv").type == pa.list_(pa.float32(), 4) - - def test_column_group_is_written_required_with_its_fields( - self, client, nmap, subset - ): - target = make_target(client) - run_copy(client, target, nmap, subset) - labels = client.written_schemas[(target, "default.unit")][0].field("labels") - assert not labels.nullable - assert [f.name for f in labels.type] == ["kilosort_label", "bombcell_label"] - assert all(f.nullable for f in labels.type) - - def test_a_null_in_a_required_column_fails_loudly(self, client, nmap, subset): - data = client.rows(SOURCE, "default.spike_event") - i = data.schema.get_field_index("spike_amplitude_uv") - values = data.column(i).to_pylist() - values[data.column("timestamp").to_pylist().index(150)] = None - client.add_table( - SOURCE, - "default.spike_event", - data.set_column(i, data.schema.field(i), pa.array(values, pa.float32())), - ["subject_id", "session_start_datetime", "timestamp", "unit_id"], - ) - target = make_target(client) - with pytest.raises(tool.NullInRequiredColumnError) as info: - run_copy(client, target, nmap, subset) - assert info.value.path == ("spike_amplitude_uv",) - assert any("spike_event" in note for note in info.value.__notes__) - # the aborted transaction left no rows behind - assert client.project(target).table( - "default.spike_event" - ).schema().to_arrow().names == [ - "subject_id", - "session_start_datetime", - "timestamp", - "unit_id", - ] - - def test_a_null_in_a_required_group_field_fails_loudly(self, client, subset): - raw = decide(derived_map(client)) - raw["tables"]["unit"]["groups"]["labels"]["fields"]["bombcell_label"][ - "nullable" - ] = False - target = make_target(client) - with pytest.raises(tool.NullInRequiredColumnError) as info: - run_copy(client, target, tool.parse_map(raw), subset) - assert info.value.path == ("labels", "bombcell_label") - - def test_source_type_drift_is_refused(self, client, nmap, subset): - data = client.rows(SOURCE, "default.session") - i = data.schema.get_field_index("booth") - client.add_table( - SOURCE, - "default.session", - data.set_column( - i, - pa.field("booth", pa.large_string()), - data.column(i).cast(pa.large_string()), - ), - ["subject_id", "session_start_datetime"], - ) - target = make_target(client) - with pytest.raises(tool.SchemaMismatchError, match="booth"): - run_copy(client, target, nmap, subset) - - def test_records_provenance_on_the_marker_table(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - meta = client.project(target).table(tool.MARKER_TABLE).get_metadata() - record = json.loads(meta[tool.PROVENANCE_KEY]) - assert record["source_project"] == SOURCE - assert record["window_us"] == list(WINDOW) - assert record["unit_ids"] == KEPT_UNITS - assert record["kept_trials"] == [1, 2, 3] - assert record["source_sample_info"] == [{"region": "V1", "n_units": 3}] - assert record["rows"]["spike_event"] == 20 - - def test_verify_requires_the_provenance(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target][tool.MARKER_TABLE] - del stored.metadata[tool.PROVENANCE_KEY] - with pytest.raises(tool.VerificationError, match="provenance"): - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - def test_dry_run_writes_nothing(self, client, nmap, subset): - target = make_target(client) - before = list(client.log) - report = run_copy(client, target, nmap, subset, dry_run=True) - assert client.log == before - assert {t.action for t in report.tables} == {"would write"} - - def test_plan_reads_only(self, client, subset): - before = list(client.log) - report = tool.plan(client, SOURCE, tool.parse_map(derived_map(client)), subset) - assert client.log == before - assert set(report.pending) == { - "experiment_screen_capture_frame_270p", - "sample_info", - "unit", - } - assert sum(t.rows for t in report.tables) > 0 - - def test_rerun_skips_complete_tables(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - before = { - name: client.rows(target, f"default.{name}") - for name in SUBSET_JSON["tables"] - } - log_size = len(client.log) - report = run_copy(client, target, nmap, subset) - assert {t.action for t in report.tables} == {"skip"} - assert len(client.log) == log_size - for name, table in before.items(): - assert client.rows(target, f"default.{name}").equals(table) - - def test_rerun_after_a_failure_completes(self, client, nmap, subset): - target = make_target(client) - good = client.rows(SOURCE, "default.spike_event") - i = good.schema.get_field_index("spike_amplitude_uv") - bad = good.set_column( - i, good.schema.field(i), pa.array([None] * good.num_rows, pa.float32()) - ) - keys = ["subject_id", "session_start_datetime", "timestamp", "unit_id"] - client.add_table(SOURCE, "default.spike_event", bad, keys) - with pytest.raises(tool.NullInRequiredColumnError): - run_copy(client, target, nmap, subset) - client.add_table(SOURCE, "default.spike_event", good, keys) - report = run_copy(client, target, nmap, subset) - actions = {t.name: t.action for t in report.tables} - assert actions["spike_event"] == "rewrite" - assert actions["session"] == "skip" - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - def test_a_changed_subset_is_refused_for_complete_tables( - self, client, nmap, subset - ): - target = make_target(client) - run_copy(client, target, nmap, subset) - changed = tool.parse_subset({**SUBSET_JSON, "unit_ids": [1]}) - with pytest.raises(tool.StaleTableError): - run_copy(client, target, nmap, changed) - - def test_a_foreign_key_annotation_does_not_make_tables_stale( - self, client, nmap, subset - ): - # removing an accepted exception changes no copied value, so a re-run skips - target = make_target(client) - run_copy(client, target, nmap, subset) - raw = copy_module.deepcopy(dict(nmap.raw)) - raw["tables"]["spike_event"]["foreign_keys"][0]["accepted_unresolved"] = [ - ["Goliath", SST.isoformat(), 9] - ] - report = run_copy(client, target, tool.parse_map(raw), subset) - assert {t.action for t in report.tables} == {"skip"} - - def test_a_nullability_change_makes_the_table_stale(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - raw = copy_module.deepcopy(dict(nmap.raw)) - raw["tables"]["session"]["columns"]["session_notes"]["nullable"] = False - raw["tables"]["session"]["columns"]["session_notes"]["key"] = False - with pytest.raises(tool.StaleTableError): - run_copy(client, target, tool.parse_map(raw), subset) - - def test_window_literals_carry_the_column_type(self, client, nmap, subset): - # the fake, like SpiralDB, refuses a u64 column compared with an i64 literal; - # the frames table keys time as int64 and the others as uint64 - target = make_target(client) - report = run_copy(client, target, nmap, subset) - assert {t.name: t.rows for t in report.tables}[ - "experiment_screen_capture_frame_270p" - ] == 4 - - -# ---------------------------------------------------------------------------- verify - - -class TestVerify: - """``verify``: a clean copy passes; changed data, schemas and references fail.""" - - def test_passes_after_copy(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - report = tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - assert report.problems == [] - - def test_detects_a_changed_value(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.spike_event"] - assert stored.data is not None - i = stored.data.schema.get_field_index("spike_amplitude_uv") - values = stored.data.column(i).to_pylist() - values[0] += 1 - stored.data = stored.data.set_column( - i, stored.data.schema.field(i), pa.array(values, pa.float32()) - ) - with pytest.raises(tool.VerificationError, match="spike_amplitude_uv"): - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - def test_detects_a_missing_row_and_a_dangling_reference(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.experiment_freeviewing_trial"] - assert stored.data is not None - stored.data = stored.data.filter( - pc.not_equal(stored.data.column("trial_index"), pa.scalar(3, pa.uint32())) - ) - with pytest.raises(tool.VerificationError) as info: - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - text = "\n".join(info.value.problems) - assert "row count" in text - assert "experiment_freeviewing_event -> experiment_freeviewing_trial" in text - - def test_an_accepted_unresolved_reference_is_reported_not_failed( - self, client, subset - ): - # the source's unit table loses unit 3, whose spikes are kept: a broken - # reference in the source itself, like lfp_location -> atlas_label live - units = client.rows(SOURCE, "default.unit") - client.add_table( - SOURCE, - "default.unit", - units.filter( - pc.not_equal(units.column("unit_id"), pa.scalar(3, pa.uint32())) - ), - ["subject_id", "session_start_datetime", "unit_id"], - ) - raw = decide(derived_map(client)) - report = tool.plan(client, SOURCE, tool.parse_map(raw), subset) - assert any(p.startswith("spike_event -> unit") for p in report.problems) - fk = raw["tables"]["spike_event"]["foreign_keys"][0] - fk["accepted_unresolved"] = [["Goliath", SST.isoformat(), 3]] - report = tool.plan(client, SOURCE, tool.parse_map(raw), subset) - assert report.problems == [] - assert len(report.accepted) == 1 and "spike_event -> unit" in report.accepted[0] - - def test_an_accepted_reference_that_now_resolves_is_not_reported( - self, client, nmap, subset - ): - # the source was fixed after the exception was recorded: nothing to report, - # and nothing fails because the accepted key is no longer missing - raw = decide(derived_map(client)) - raw["tables"]["spike_event"]["foreign_keys"][0]["accepted_unresolved"] = [ - ["Goliath", SST.isoformat(), 3] - ] - accepted_map = tool.parse_map(raw) - report = tool.plan(client, SOURCE, accepted_map, subset) - assert report.problems == [] and report.accepted == [] - target = make_target(client) - run_copy(client, target, accepted_map, subset) - verified = tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=accepted_map, - subset=subset, - ) - assert verified.problems == [] and verified.accepted == [] - - def test_detects_a_null_in_a_required_column(self, client, nmap, subset): - # SpiralDB stores the column nullable, so only the data can show the breach - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.session"] - assert stored.data is not None - i = stored.data.schema.get_field_index("booth") - stored.data = stored.data.set_column( - i, stored.data.schema.field(i), pa.array([None], pa.string()) - ) - with pytest.raises( - tool.VerificationError, match="'booth' is declared non-nullable" - ): - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - def test_detects_a_schema_that_differs_from_the_source(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.session"] - assert stored.data is not None - stored.data = stored.data.cast( - pa.schema( - [ - f.with_type(pa.large_string()) if f.name == "booth" else f - for f in stored.data.schema - ] - ) - ) - with pytest.raises(tool.VerificationError, match="session: schema"): - tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - -class TestCommittedFiles: - """The committed map and subset, and the command-line entry point.""" - - def test_map_and_subset_load_and_agree(self): - nm = tool.load_map() - sub = tool.load_subset() - assert nm.source_project == SOURCE - assert set(sub.tables) <= set(nm.tables) - start, stop = sub.window - assert 0 < stop - start <= 200_000_000 # at most 200 s - - def test_cli_render_map(self, capsys): - assert tool.main(["render-map"]) == 0 - assert "### `spike_event`" in capsys.readouterr().out - - def test_cli_plan_uses_the_given_client(self, client, capsys, tmp_path): - map_path = tmp_path / "map.json" - map_path.write_text(json.dumps(derived_map(client))) - subset_path = tmp_path / "subset.json" - subset_path.write_text(json.dumps(SUBSET_JSON)) - code = tool.main( - ["--map", str(map_path), "--subset", str(subset_path), "plan"], - client_factory=lambda: client, - ) - assert code == 0 - assert "pending decisions" in capsys.readouterr().out - - -class _StubSource: - """Stands in for an Orcapod source: only ``output_schema`` is used.""" - - def __init__(self, tags: dict[str, Any], data: dict[str, Any]) -> None: - """Hold the tag and data schemas to return.""" - self._schemas = (tags, data) - - def output_schema(self) -> tuple[dict[str, Any], dict[str, Any]]: - """Return the ``(tag schema, data schema)`` pair.""" - return self._schemas - - -class TestOrcapodCheck: - """``orcapod_check``: column types against the map's nullability.""" - - def test_plain_types_for_required_and_optional_for_nullable(self, nmap, subset): - def make(project: str, table: str, dataset: str) -> _StubSource: - """Return a stub source with plain types for required columns.""" - if table == "session": - return _StubSource( - {"subject_id": str, "session_start_datetime": datetime.datetime}, - {"booth": str, "session_notes": str | None}, - ) - return _StubSource({}, {}) - - lines, problems = tool.orcapod_check("p", nmap, subset, make) - assert problems == [] - assert "lfp_timeseries: skipped (fixed_size_list, ITL-927)" in lines - - def test_a_required_column_seen_as_optional_is_a_problem(self, nmap, subset): - def make(project: str, table: str, dataset: str) -> _StubSource: - """Return a stub source that types a required key column as optional.""" - if table == "session": - return _StubSource( - { - "subject_id": str | None, - "session_start_datetime": datetime.datetime, - }, - {"booth": str, "session_notes": str | None}, - ) - return _StubSource({}, {}) - - _, problems = tool.orcapod_check("p", nmap, subset, make) - assert problems == [ - "session: nullability differs from the map: ['subject_id: str | None']" - ] - - -class TestProbe: - """``probe_nullability``: writes only a scratch table, on a marked target.""" - - def test_probe_writes_only_a_scratch_table_and_drops_it(self, client): - target = make_target(client) - result = tool.probe_nullability( - client, target_project=target, allow_writes=target, source_project=SOURCE - ) - assert {r.table for r in client.project(target).list_tables()} == { - "edi_tutorial_build" - } - assert { - e[2] for e in writes_to(client, target) if e[1] != "create_project" - } <= { - tool.PROBE_TABLE, - tool.MARKER_TABLE, - } - assert ( - result["null_v_declared_non_null"]["raised"] is None - or "v" in (result["null_v_declared_non_null"]["raised"]) - ) - assert "after_non_null_write" in result - - def test_probe_refuses_a_target_without_marker(self, client): - with pytest.raises(tool.TargetRefusedError): - tool.probe_nullability( - client, - target_project="test-orcapod-885785", - allow_writes="test-orcapod-885785", - source_project=SOURCE, - ) - - -UNIT_KEYS = ["subject_id", "session_start_datetime", "unit_id"] - - -class TestReplace: - """``copy`` with ``replace``: dropping and copying chosen tables again.""" - - def test_a_source_fix_is_picked_up_by_replacing_one_table(self, client, subset): - # the source's unit table first lacks unit 3 (an accepted exception), then the - # owner adds it back, as with atlas_label 'White Matter' live - units = client.rows(SOURCE, "default.unit") - missing = units.filter( - pc.not_equal(units.column("unit_id"), pa.scalar(3, pa.uint32())) - ) - client.add_table(SOURCE, "default.unit", missing, UNIT_KEYS) - raw = decide(derived_map(client)) - raw["tables"]["spike_event"]["foreign_keys"][0]["accepted_unresolved"] = [ - ["Goliath", SST.isoformat(), 3] - ] - nm = tool.parse_map(raw) - target = make_target(client) - first = run_copy(client, target, nm, subset) - assert len(first.accepted) == 1 - - client.add_table(SOURCE, "default.unit", units, UNIT_KEYS) - before = len(client.log) - report = run_copy(client, target, nm, subset, tables=["unit"], replace=True) - assert [(r.name, r.action, r.rows) for r in report.tables] == [ - ("unit", "replace", 2) - ] - touched = {e[2] for e in client.log[before:] if e[1] != "set_metadata"} - assert touched == {"default.unit"} - verified = tool.verify( - client, source_project=SOURCE, target_project=target, nmap=nm, subset=subset - ) - assert verified.problems == [] and verified.accepted == [] - record = json.loads( - client.project(target) - .table(tool.MARKER_TABLE) - .get_metadata()[tool.PROVENANCE_KEY] - ) - assert record["rows"]["unit"] == 2 and record["rows"]["spike_event"] == 20 - - def test_replacing_twice_gives_the_same_table(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - run_copy(client, target, nmap, subset, tables=["session"], replace=True) - once = client.rows(target, "default.session") - run_copy(client, target, nmap, subset, tables=["session"], replace=True) - assert client.rows(target, "default.session").equals(once) - - def test_replace_needs_tables(self, client, nmap, subset): - target = make_target(client) - with pytest.raises(tool.ToolError, match="--replace needs --tables"): - run_copy(client, target, nmap, subset, replace=True) - - def test_unknown_tables_are_refused(self, client, nmap, subset): - target = make_target(client) - with pytest.raises(tool.ToolError, match="not in the subset"): - run_copy(client, target, nmap, subset, tables=["atlas"], replace=True) - - def test_dry_run_replace_writes_nothing(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - before = list(client.log) - report = run_copy( - client, target, nmap, subset, tables=["session"], replace=True, dry_run=True - ) - assert client.log == before - assert [r.action for r in report.tables] == ["would replace"] - - -class _RecordWriter: - """Stands in for Orcapod's ``write_nullability_record`` (ITL-936, not on main).""" - - def __init__(self) -> None: - """Start with no calls and no recorded schemas.""" - self.calls: list[tuple[Any, pa.Schema]] = [] - self.current: dict[int, pa.Schema] = {} - - def __call__(self, table: Any, schema: pa.Schema) -> bool: - """Record ``schema`` on ``table``; return whether the record changed.""" - self.calls.append((table, schema)) - key = id(table._stored) - changed = self.current.get(key) != schema - if changed: - table.set_metadata({"__record__": str(schema).encode()}) - self.current[key] = schema - return changed - - -class TestAnnotate: - """``annotate``: recording the written nullability in table metadata.""" - - def _annotate(self, client, target, nmap, subset, writer): - """Run ``annotate`` on ``target`` with ``writer`` as the record writer.""" - return tool.annotate( - client, - source_project=SOURCE, - target_project=target, - allow_writes=target, - nmap=nmap, - subset=subset, - write_record=writer, - ) - - def test_records_the_schema_the_copy_wrote_metadata_only( - self, client, nmap, subset - ): - target = make_target(client) - run_copy(client, target, nmap, subset) - writer = _RecordWriter() - before = len(client.log) - lines = self._annotate(client, target, nmap, subset, writer) - assert all(line.endswith("recorded") for line in lines) - assert {e[1] for e in client.log[before:]} == {"set_metadata"} - declared = {tbl._ident: schema for tbl, schema in writer.calls} - for name in SUBSET_JSON["tables"]: - assert ( - declared[f"default.{name}"] - == client.written_schemas[(target, f"default.{name}")][0] - ) - again = self._annotate(client, target, nmap, subset, writer) - assert all(line.endswith("unchanged") for line in again) - - def test_a_null_in_the_target_blocks_the_record(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.session"] - assert stored.data is not None - i = stored.data.schema.get_field_index("booth") - stored.data = stored.data.set_column( - i, stored.data.schema.field(i), pa.array([None], pa.string()) - ) - writer = _RecordWriter() - with pytest.raises(tool.NullInRequiredColumnError): - self._annotate(client, target, nmap, subset, writer) - assert all(tbl._ident != "default.session" for tbl, _ in writer.calls) - - def test_orcapod_without_the_record_helper_is_a_clear_error(self, monkeypatch): - import orcapod.databases - - monkeypatch.delattr( - orcapod.databases, "write_nullability_record", raising=False - ) - with pytest.raises(tool.ToolError, match="ITL-936"): - tool.load_record_writer() - - -class TestFloatComparison: - """verify compares floats bit for bit: NaN equals itself, -0.0 differs from 0.0. - - Live, eye_x_deg (gaze) and unit quality metrics hold NaN, and Arrow's - ``equals`` treats NaN as unequal to itself (ITL-933 verify run, November 11, 2026). - """ - - def _with_nans(self, client: FakeSpiral) -> None: - """Put NaN (and -0.0) values in the source's float and float-list columns.""" - data = client.rows(SOURCE, "default.spike_event") - i = data.schema.get_field_index("spike_amplitude_uv") - values = [ - float("nan") if n % 2 else v - for n, v in enumerate(data.column(i).to_pylist()) - ] - client.add_table( - SOURCE, - "default.spike_event", - data.set_column(i, data.schema.field(i), pa.array(values, pa.float32())), - ["subject_id", "session_start_datetime", "timestamp", "unit_id"], - ) - lfp = client.rows(SOURCE, "default.lfp_timeseries") - j = lfp.schema.get_field_index("lfp_voltage_uv") - rows = [[float("nan"), None, -0.0, 1.0] for _ in range(lfp.num_rows)] - client.add_table( - SOURCE, - "default.lfp_timeseries", - lfp.set_column( - j, lfp.schema.field(j), pa.array(rows, lfp.schema.field(j).type) - ), - ["subject_id", "session_start_datetime", "timestamp"], - ) - - def _verify(self, client: FakeSpiral, target: str, nmap: Any, subset: Any): - """Run ``verify`` on ``target``.""" - return tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - ) - - def test_nan_values_copied_unchanged_verify(self, client, nmap, subset): - self._with_nans(client) - target = make_target(client) - run_copy(client, target, nmap, subset) - assert self._verify(client, target, nmap, subset).problems == [] - - def test_a_different_nan_payload_is_detected(self, client, nmap, subset): - self._with_nans(client) - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.spike_event"] - assert stored.data is not None - i = stored.data.schema.get_field_index("spike_amplitude_uv") - col = stored.data.column(i).combine_chunks() - buffer = col.buffers()[1] - assert buffer is not None - raw = np.frombuffer(buffer.to_pybytes(), dtype=np.uint32)[: len(col)].copy() - raw[np.isnan(col.to_numpy(zero_copy_only=False))] = ( - 0x7FC00001 # another quiet NaN - ) - stored.data = stored.data.set_column( - i, stored.data.schema.field(i), pa.array(raw.view(np.float32), pa.float32()) - ) - with pytest.raises(tool.VerificationError, match="spike_amplitude_uv"): - self._verify(client, target, nmap, subset) - - def test_negative_zero_is_not_zero(self, client, nmap, subset): - self._with_nans(client) - target = make_target(client) - run_copy(client, target, nmap, subset) - stored = client._projects[target]["default.lfp_timeseries"] - assert stored.data is not None - j = stored.data.schema.get_field_index("lfp_voltage_uv") - field = stored.data.schema.field(j) - rows = stored.data.column(j).to_pylist() - rows[0] = [rows[0][0], None, 0.0, rows[0][3]] - stored.data = stored.data.set_column(j, field, pa.array(rows, field.type)) - with pytest.raises(tool.VerificationError, match="lfp_voltage_uv"): - self._verify(client, target, nmap, subset) - - -class TestNullabilityRecordChecks: - """annotate --dry-run, the list-item null check, and verify against the record.""" - - def test_a_null_list_item_declared_non_null_is_refused(self): - item = pa.field("item", pa.float32(), False) - field = pa.field("l", pa.list_(item, 2), False) - # pyarrow builds this array without checking the item's non-null flag - arr = pa.FixedSizeListArray.from_arrays( - pa.array([1.0, None], pa.float32()), - type=field.type, # pyright: ignore[reportArgumentType] - ) - batch = pa.RecordBatch.from_arrays([arr], names=["l"]) - with pytest.raises(tool.NullInRequiredColumnError) as info: - tool.conform_batch(batch, pa.schema([field])) - assert info.value.path == ("l", "[]") - - def test_a_null_nested_list_item_declared_non_null_is_refused(self): - inner = pa.list_(pa.field("item", pa.int32(), False)) - field = pa.field("l", pa.list_(pa.field("item", inner, False)), False) - values = pa.ListArray.from_arrays( - pa.array([0, 2], pa.int32()), - pa.array([1, None], pa.int32()), - type=inner, # pyright: ignore[reportArgumentType] - ) - arr = pa.ListArray.from_arrays( - pa.array([0, 1], pa.int32()), - values, - type=field.type, # pyright: ignore[reportArgumentType] - ) - batch = pa.RecordBatch.from_arrays([arr], names=["l"]) - with pytest.raises(tool.NullInRequiredColumnError) as info: - tool.conform_batch(batch, pa.schema([field])) - assert info.value.path == ("l", "[]", "[]") - - def test_a_null_below_a_struct_between_two_lists_is_refused(self): - xs_type = pa.list_(pa.field("item", pa.int32(), False)) - struct_type = pa.struct([pa.field("xs", xs_type, True)]) - field = pa.field("l", pa.list_(pa.field("item", struct_type, True)), True) - xs = pa.ListArray.from_arrays( - pa.array([0, 1], pa.int32()), - pa.array([None], pa.int32()), - type=xs_type, # pyright: ignore[reportArgumentType] - ) - items = pa.StructArray.from_arrays([xs], fields=list(struct_type)) - arr = pa.ListArray.from_arrays( - pa.array([0, 1], pa.int32()), - items, - type=field.type, # pyright: ignore[reportArgumentType] - ) - batch = pa.RecordBatch.from_arrays([arr], names=["l"]) - with pytest.raises(tool.NullInRequiredColumnError) as info: - tool.conform_batch(batch, pa.schema([field])) - assert info.value.path == ("l", "[]", "xs", "[]") - - def test_a_null_in_a_nullable_nested_item_is_accepted(self): - inner = pa.list_(pa.field("item", pa.int32(), True)) - field = pa.field("l", pa.list_(pa.field("item", inner, False)), False) - arr = pa.array([[[1, None]]], field.type) - batch = pa.RecordBatch.from_arrays([arr], names=["l"]) - assert tool.conform_batch(batch, pa.schema([field])).num_rows == 1 - - def test_non_null_paths_follow_the_record_layout(self): - schema = pa.schema( - [ - pa.field("k", pa.int64(), False), - pa.field("w", pa.int32(), True), - pa.field( - "g", - pa.struct( - [pa.field("x", pa.int32(), False), pa.field("y", pa.int32())] - ), - False, - ), - pa.field("l", pa.list_(pa.field("item", pa.float32(), False), 2), True), - ] - ) - assert tool.non_null_paths(schema) == [("g",), ("g", "x"), ("k",), ("l", "[]")] - - def test_annotate_dry_run_lists_paths_and_writes_nothing( - self, client, nmap, subset - ): - target = make_target(client) - run_copy(client, target, nmap, subset) - before = list(client.log) - lines = tool.annotate( - client, - source_project=SOURCE, - target_project=target, - allow_writes=target, - nmap=nmap, - subset=subset, - write_record=None, - dry_run=True, - ) - assert client.log == before - session = next(line for line in lines if line.startswith("session:")) - assert "booth" in session and "session_notes" not in session - - def _verify(self, client, target, nmap, subset, reader): - """Run ``verify`` on ``target``, reading nullability records with ``reader``.""" - return tool.verify( - client, - source_project=SOURCE, - target_project=target, - nmap=nmap, - subset=subset, - read_record=reader, - ) - - def test_verify_checks_each_record_against_the_map(self, client, nmap, subset): - target = make_target(client) - run_copy(client, target, nmap, subset) - written = client.written_schemas - - def no_record(metadata: Any) -> Any: - """Report that no table carries a nullability record.""" - return None - - report = self._verify(client, target, nmap, subset, no_record) - assert len(report.notes) == len(SUBSET_JSON["tables"]) - - for (pid, ident), schemas in written.items(): - if pid == target: - paths = sorted(tool.non_null_paths(schemas[0])) - client._projects[target][ident].metadata["rec"] = json.dumps( - paths - ).encode() - - def reader(metadata: Any) -> Any: - """Read the record the test stores under ``rec``.""" - raw = metadata.get("rec") - return None if raw is None else frozenset(map(tuple, json.loads(raw))) - - report = self._verify(client, target, nmap, subset, reader) - assert report.problems == [] and report.notes == [] - - client._projects[target]["default.session"].metadata["rec"] = ( - b'[["subject_id"]]' - ) - with pytest.raises(tool.VerificationError, match="session: nullability record"): - self._verify(client, target, nmap, subset, reader) - - -class TestReviewRound1: - """Fixes from the first CodeRabbit round on PR #307.""" - - def test_an_incomplete_table_is_rebuilt_without_old_rows( - self, client, nmap, subset - ): - # data committed but the status did not; a retry with another subset must not - # keep the earlier attempt's rows - target = make_target(client) - run_copy(client, target, nmap, subset) - del client._projects[target]["default.spike_event"].metadata[tool.STATUS_KEY] - narrower = tool.parse_subset({**SUBSET_JSON, "unit_ids": [1]}) - report = run_copy(client, target, nmap, narrower, tables=["spike_event"]) - assert [r.action for r in report.tables] == ["rewrite"] - units = client.rows(target, "default.spike_event").column("unit_id").to_pylist() - assert set(units) == {1} - - def test_an_empty_selection_fails_and_is_not_marked_complete(self, client, nmap): - target = make_target(client) - empty = tool.parse_subset({**SUBSET_JSON, "unit_ids": [99]}) - with pytest.raises(tool.ToolError, match="no rows"): - run_copy(client, target, nmap, empty) - meta = client.project(target).table("default.spike_event").get_metadata() - assert tool.STATUS_KEY not in meta - - def test_source_key_columns_must_match_the_map(self, client, nmap, subset): - data = client.rows(SOURCE, "default.session") - client.add_table( - SOURCE, "default.session", data, ["session_start_datetime", "subject_id"] - ) - target = make_target(client) - with pytest.raises(tool.SchemaMismatchError, match="source key columns"): - run_copy(client, target, nmap, subset) - - @pytest.mark.parametrize( - ("left", "right"), - [ - (pa.array([-0.0], pa.float32()), pa.array([0.0], pa.float32())), - ( - pa.array([[1.0, -0.0]], pa.list_(pa.float64())), - pa.array([[1.0, 0.0]], pa.list_(pa.float64())), - ), - ( - pa.array([{"x": -0.0}], pa.struct([("x", pa.float32())])), - pa.array([{"x": 0.0}], pa.struct([("x", pa.float32())])), - ), - ], - ids=["top-level", "list", "struct"], - ) - def test_negative_zero_differs_without_nan(self, left, right): - assert left.equals(right) # Arrow's own equality would accept it - assert not tool.identical(left, right) - assert tool.identical(left, left) - - def test_a_leftover_probe_table_is_accepted_and_dropped(self, client): - # a probe killed before its cleanup leaves its table, marked like the others - target = make_target(client) - project = client.project(target) - marker = project.table(tool.MARKER_TABLE).get_metadata()[tool.MARKER_KEY] - leftover = project.create_table( - tool.PROBE_TABLE, key_schema=pa.schema([pa.field("k", pa.int64(), False)]) - ) - leftover.set_metadata({tool.MARKER_KEY: marker}) - tool.probe_nullability( - client, target_project=target, allow_writes=target, source_project=SOURCE - ) - assert {f"{r.dataset}.{r.table}" for r in project.list_tables()} == { - tool.MARKER_TABLE - } - - def test_the_probe_table_carries_the_marker(self, client): - target = make_target(client) - tool.probe_nullability( - client, target_project=target, allow_writes=target, source_project=SOURCE - ) - assert (target, "set_metadata", tool.PROBE_TABLE) in client.log diff --git a/workshop/edi_tutorial_dataset/README.md b/workshop/edi_tutorial_dataset/README.md deleted file mode 100644 index 89bd5229b..000000000 --- a/workshop/edi_tutorial_dataset/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# EDI tutorial dataset (`edi-tutorial-v1`) - -The Orcapod workshop reads EDI tables from SpiralDB. This directory holds the tool that built -its Spiral project: a small subset of the V1 EDI sample (`edi-sample-v1-855632`, dataset -`default`). Each column is declared non-nullable where the -[EDI schema specification](https://github.com/enigma-brain/enigma-data-interface) -(`edi_schema_spec.md`, version 0.3.0) requires a value. - -**Project: `edi-tutorial-v1-841226`** (dataset `default`), built and verified on November 11, -2026 (ITL-933). It uses the Volume every `edi-sample-*` project uses -(`vol_01m6s4jsjeedya88dc8t2ns96q`, the `edi-sample-datasets` bucket); its description names -it `edi-tutorial-v1 · edi-sample-tutorial`. - -## What is in it - -One subject, one session, one recording, one probe and one FreeViewing experiment, as in the -sample, cut down to: - -* **one window of 143.9 s**: µs `[5226928854, 5370814965)` relative to - `session_start_datetime`, from the start of trial 215 to the end of trial 225 (11 - consecutive valid trials). Every `timestamp` in EDI uses that one clock, so the same window - applies to every time-keyed table; -* **32 units**: the units with the most spikes in the window, among those whose - `labels.bombcell_label` is not `noise`; -* the trials that overlap the window, plus any trial a kept row references; -* every other table whole, except `sample_info`, which describes the whole sample - (`n_units` 100, its unit filter and sources). `copy` records the source's `sample_info` - row in the provenance metadata of the project's `meta.edi_tutorial_build` table instead, - with the source project, window, units, trials and row counts. - -Values are copied unchanged (developer, November 11, 2026). Columns that summarize the -whole sample therefore still describe the whole sample, not the subset: - -* `experiment`: `total_trials` (873), `correct_trials` (619), `incorrect_trials`, - `experiment_start_timestamp`, `experiment_stop_timestamp`; -* `recording`: `recording_start_timestamp`, `recording_end_timestamp`; -* `session`: `session_end_datetime`; -* `lfp_acquisition`: `lfp_n_samples` (6,866,378), `recording_start_timestamp`; -* `unit`: every `quality_metrics` field (for example `num_spikes`, `firing_rate_hz`, - `presence_ratio`), the `rolling_quality_metrics` series over the whole session, and the - `waveform_metrics`; -* `sample_unit`: `firing_rate_hz`, `presence_ratio`, `fano_factor_detrended`. - -204,948 rows, 361 MB uncompressed. The largest tables are `lfp_timeseries` (86,331 rows, -154 MB) and `experiment_screen_capture_frame_270p` (4,312 JPEG frames, 177 MB). -`lfp_timeseries`, `experiment_display_params`, `experiment_freeviewing_params` and -`experiment_screen_capture_event` hold `fixed_size_list` columns, which Orcapod sources -cannot hash until starfix supports the type (ITL-927). Some float columns hold NaN (gaze -positions during blinks, some unit quality metrics); they are copied bit for bit. - -| Table | Selection | Rows | Key columns | Required values | Nullable values | -| --- | --- | ---: | ---: | --- | ---: | -| `atlas` | whole | 1 | 1 | `atlas_name` | 3 | -| `atlas_label` | whole | 199 | 2 | — | 3 | -| `chamber` | whole | 2 | 2 | — | 2 | -| `experiment` | whole | 1 | 4 | `task_id`, `task_version` | 5 | -| `experiment_display_params` | whole | 1 | 4 | — | 9 | -| `experiment_freeviewing_event` | window | 4,202 | 6 | `trial_index` | 0 | -| `experiment_freeviewing_params` | whole | 1 | 4 | — | 15 | -| `experiment_freeviewing_trial` | trials | 11 | 5 | `trial_start_timestamp` | 37 | -| `experiment_gaze_online_timeseries` | window | 28,780 | 5 | — | 8 | -| `experiment_screen_capture_event` | window | 17,249 | 6 | — | 15 | -| `experiment_screen_capture_frame_270p` | window | 4,312 | 7 | — | 1 (`frame_jpeg`) | -| `lfp_acquisition` | whole | 1 | 5 | `lfp_channel_ids`, `recording_start_timestamp` | 3 | -| `lfp_location` | whole | 383 | 9 | — | 5 | -| `lfp_timeseries` | window | 86,331 | 6 | `lfp_voltage_uv` | 0 | -| `penetration` | whole | 1 | 4 | `probe_id` | 11 | -| `probe` | whole | 22 | 1 | — | 3 | -| `probe_location` | whole | 1 | 6 | — | 4 | -| `probe_map` | whole | 1 | 4 | — | 6 | -| `recording` | whole | 1 | 3 | — | 5 | -| `sample_unit` | units | 32 | 6 | — | 4 | -| `session` | whole | 1 | 2 | `booth`, `session_end_datetime` | 3 | -| `spike_event` | window + units | 63,349 | 7 | `spike_amplitude_uv` | 0 | -| `subject` | whole | 1 | 1 | — | 3 | -| `task` | whole | 1 | 2 | — | 3 | -| `unit` | units | 32 | 6 | 4 column groups (structs) | 56 fields in the groups | -| `unit_location` | units | 32 | 9 | — | 5 | - -## Nullability - -`nullability_map.json` records, for every column, the source Arrow type, the -specification's type, the nullability to declare and the foreign keys. Key columns are -always required. Types are the sample's, exactly; only nullability changes. The sample's 24 -specification tables match the specification exactly (columns, types, keys and key order). -Every required column holds no nulls in the source. Print the full per-column table with: - -```bash -uv run python workshop/edi_tutorial_dataset/edi_tutorial_dataset.py render-map -``` - -The developer's decisions of November 11, 2026 cover what the specification leaves open. -They live in `decisions.json`, which `derive-map` applies: - -* Tables the specification does not define (`experiment_screen_capture_frame_270p`, - `sample_info`, `sample_unit`): key columns required, other columns nullable, as in the - source. -* The four column groups of `unit` (`labels`, `quality_metrics`, `rolling_quality_metrics`, - `waveform_metrics`) are required structs; their fields stay nullable, as the - specification says. -* `accepted_unresolved` lists references known not to resolve in the source; `plan` and - `verify` report them instead of failing. It held one entry until the source was fixed: - 58 of the 383 `lfp_location` rows referenced region `White Matter` of atlas - `AFNI_D99v2`, which `atlas_label` lacked. This was reported to the EDI owners on November 11, 2026 - and fixed at the source the same day (`atlas_label` gained the row, 199 rows); the - project's `atlas_label` was then copied again. - -### What SpiralDB stores - -SpiralDB cannot store a non-null column. Probed on the project with pyspiral 0.16.33 on -November 11, 2026, it keeps every top-level column nullable, key columns included, and every -struct field nullable; only list items keep a non-null flag. pyspiral's own -`Schema.from_arrow` already turns a non-null key column into a nullable one, and a null -written into a struct field declared non-null is accepted. So: - -* the copy writes each batch with the map's nullability and refuses a null in a required - column before writing (that table's transaction is aborted). Nothing is coerced; -* `verify` expects the schema SpiralDB can store (the source's types, every column - nullable except list items) and checks, from the data, that every required column holds - no nulls; -* Orcapod sources on these tables still show every column as `T | None`. Showing the - declared nullability needs a record of it in each table's metadata, which Orcapod's - Spiral connector reads and enforces: ITL-936. `annotate` gives each table that record - by calling Orcapod's `write_nullability_record` with the exact schema `copy` wrote, after - checking the table's data for nulls in the declared non-nullable columns. It writes table - metadata only, never data, and needs an Orcapod that has ITL-936; without it, `annotate` - stops with an error naming ITL-936. - -## Commands - -Every command needs `pyspiral` 0.16.33 or newer (older versions do not use the `spiral login` -credentials) and a `spiral login`. Run them from the repository root. `plan`, `verify` and -`orcapod-check` only read; `create`, `probe`, `copy` and `annotate` write, and only to a -project `create` made. - -```bash -TOOL="uv run --with pyspiral==0.16.33 python workshop/edi_tutorial_dataset/edi_tutorial_dataset.py" - -$TOOL plan # read-only dry run: rows, bytes, nulls, foreign keys -$TOOL create --id-prefix edi-tutorial-v1 --allow-create edi-tutorial-v1 \ - --volume vol_01m6s4jsjeedya88dc8t2ns96q \ - --description "edi-tutorial-v1 · edi-sample-tutorial · ..." # prints the new project ID -$TOOL probe --target-project ID --allow-writes ID # how SpiralDB treats non-null columns -$TOOL copy --target-project ID --allow-writes ID --dry-run -$TOOL copy --target-project ID --allow-writes ID -$TOOL verify --target-project ID -$TOOL copy --target-project ID --allow-writes ID --tables atlas_label --replace - # copy one table again after a source fix -$TOOL annotate --target-project ID --allow-writes ID # needs Orcapod with ITL-936 -$TOOL orcapod-check --target-project ID # after ITL-936: required columns show as T -``` - -Safety rules: - -* The tool reads only from `--source-project` (default `edi-sample-v1-855632`). -* It writes only to `--target-project`, which must be repeated as `--allow-writes`, must - differ from the source and must not start with `edi-sample-v`; `create` refuses such an - ID prefix too. -* The target must hold the marker table `meta.edi_tutorial_build`, which only `create` - writes, into the project it has just made. Every table the tool writes carries the same - marker in its metadata, and a target holding any table without it is refused. -* `copy` refuses to run while the map has pending decisions. -* Re-running `copy` skips tables already complete for the same subset and the same - column declarations, and rewrites an incomplete one. A complete table built from a - different subset or declarations is refused. (The project's tables were stamped before - this rule excluded foreign keys from the stamp, so a plain re-run there reports them - stale; use `--tables ... --replace`.) -* `copy --tables T1,T2 --replace` drops those tables (they must carry the marker) and - copies them again from the source; running it twice gives the same tables. Between the - drop and the end of the copy, the table is missing. - -`verify` checks that the project holds exactly the subset's tables, that each table has the -schema SpiralDB can store for the source's types, that its key columns are the source's (in -order), that required columns hold no nulls, that every foreign key resolves, that the -provenance record matches the subset, and that every row equals the source bit for bit. - -To rebuild the map from the specification (reads the source's schemas): - -```bash -$TOOL derive-map --spec path/to/edi_schema_spec.md --spec-version 0.3.0 \ - --output workshop/edi_tutorial_dataset/nullability_map.json -``` - -The tests (`tests/test_workshop/`) run the tool against an in-memory fake of pyspiral that -models the storage behaviour seen live. diff --git a/workshop/edi_tutorial_dataset/decisions.json b/workshop/edi_tutorial_dataset/decisions.json deleted file mode 100644 index 830b4ad50..000000000 --- a/workshop/edi_tutorial_dataset/decisions.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "decided_by": "developer, November 11, 2026 (ITL-933)", - "tables_not_in_spec": { - "keys": "required", - "values": "nullable" - }, - "column_group_structs": "required", - "accepted_unresolved": [] -} diff --git a/workshop/edi_tutorial_dataset/edi_tutorial_dataset.py b/workshop/edi_tutorial_dataset/edi_tutorial_dataset.py deleted file mode 100644 index 849175156..000000000 --- a/workshop/edi_tutorial_dataset/edi_tutorial_dataset.py +++ /dev/null @@ -1,2350 +0,0 @@ -"""Build the workshop Spiral project: a small subset of the V1 EDI sample. - -The workshop reads EDI tables from SpiralDB through Orcapod sources. The V1 EDI sample -project (``edi-sample-v1-855632``) is too large for a laptop, and every column in it, -key columns included, is declared nullable. This tool copies a subset of it (a set of -units and one time window; small tables whole) into a new project, and declares each -column non-nullable where the EDI schema specification (v0.3.0) requires a value. - -Two files next to this module drive it: - -* ``nullability_map.json``: for every table and column, the specification's type, the - nullability to declare, the key columns and the foreign keys. Entries whose - nullability is ``null`` are decisions still pending; ``copy`` refuses to run while - any remain. -* ``subset.json``: the time window, the unit IDs, and how each table is selected. - -Every command needs ``pyspiral>=0.16.33`` and a ``spiral login``. Run from the -repository root, for example:: - - uv run --with pyspiral==0.16.33 python \\ - workshop/edi_tutorial_dataset/edi_tutorial_dataset.py plan - -Commands (see ``--help`` on each): - -* ``plan``: read-only dry run of the whole copy against the source. -* ``create``: create the target project and its marker table. -* ``probe``: check, in the target project, how SpiralDB treats non-nullable columns. -* ``copy``: copy the subset (``--dry-run`` writes nothing). -* ``verify``: check the target against the source and the map. -* ``derive-map`` / ``render-map``: build the map from the specification, print it as - Markdown. - -The tool reads only from ``--source-project``. It writes only to ``--target-project``, -which must be repeated as ``--allow-writes``, must not start with ``edi-sample-v``, and -must be a project that ``create`` made: ``create`` writes a marker table -(``meta.edi_tutorial_build``) into the new project, and every table the tool writes -carries the same marker in its metadata. A target holding any table without the marker -is refused. -""" - -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import json -import re -import sys -import typing -import uuid -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from pathlib import Path -from typing import Any - -import numpy as np -import pyarrow as pa -import pyarrow.compute as pc - -HERE = Path(__file__).resolve().parent -MAP_PATH = HERE / "nullability_map.json" -SUBSET_PATH = HERE / "subset.json" -DECISIONS_PATH = HERE / "decisions.json" - -MIN_PYSPIRAL = (0, 16, 33) -DEFAULT_SOURCE = "edi-sample-v1-855632" -MARKER_TABLE = "meta.edi_tutorial_build" -PROBE_TABLE = "meta.nullability_probe" -MARKER_KEY = "edi_tutorial_dataset.marker" -STATUS_KEY = "edi_tutorial_dataset.status" -PROVENANCE_KEY = "edi_tutorial_dataset.provenance" -# SpiralDB limits each table-metadata key and value pair to 8 KiB. -METADATA_LIMIT = 8 * 1024 -FORBIDDEN_TARGET_PREFIXES = ("edi-sample-v",) -TOOL_NAME = "orcapod-python workshop/edi_tutorial_dataset" - -# Rows per slice when two key-sorted streams are compared value by value. -COMPARE_SLICE_ROWS = 4096 - - -# --------------------------------------------------------------------------- errors - - -class ToolError(Exception): - """Base class for every error this tool raises on purpose.""" - - -class TargetRefusedError(ToolError): - """The target project is not one this tool may write to.""" - - -class PendingDecisionError(ToolError): - """The nullability map still holds decisions for the developer.""" - - -class SchemaMismatchError(ToolError): - """A table's columns or Arrow types differ from what the map records.""" - - -class NullInRequiredColumnError(ToolError): - """A column declared non-nullable holds nulls. - - Attributes: - path: Column path, outermost first (a struct child is ``(group, field)``). - null_count: Number of nulls found. - """ - - def __init__(self, path: tuple[str, ...], null_count: int) -> None: - """Record the column path and null count and build the message.""" - self.path = path - self.null_count = null_count - super().__init__( - f"column {'.'.join(path)!r} is declared non-nullable but holds " - f"{null_count} null(s); nothing is coerced" - ) - - def within(self, parent: str) -> NullInRequiredColumnError: - """Return the same error with ``parent`` prepended to the column path.""" - return NullInRequiredColumnError((parent, *self.path), self.null_count) - - -class StaleTableError(ToolError): - """A complete target table was built from a different subset or map.""" - - -class VerificationError(ToolError): - """``verify`` found problems; ``problems`` lists them.""" - - def __init__(self, problems: list[str]) -> None: - """Keep ``problems`` and list them in the message.""" - self.problems = problems - super().__init__(f"{len(problems)} problem(s):\n " + "\n ".join(problems)) - - -# ------------------------------------------------------------------- map and subset - - -@dataclasses.dataclass(frozen=True) -class ColumnSpec: - """One column (or one field of a column group) in the nullability map. - - Attributes: - arrow_type: The source column's Arrow type, as ``str(pa.DataType)``. - nullable: Nullability to declare; ``None`` while the decision is pending. - spec_type: The specification's type (``None`` if the spec does not define it). - key: Whether the column is part of the table's key. - """ - - arrow_type: str - nullable: bool | None - spec_type: str | None = None - key: bool = False - - -@dataclasses.dataclass(frozen=True) -class GroupSpec: - """A Spiral column group (a top-level struct column) in the nullability map.""" - - arrow_type: str - nullable: bool | None - fields: Mapping[str, ColumnSpec] - - -@dataclasses.dataclass(frozen=True) -class ForeignKey: - """Child columns that must resolve to the key of rows in ``parent``. - - The child columns carry the same names as the parent's key columns. - - Attributes: - parent: The referenced table. - columns: The parent's key columns, present under the same names in the child. - note: Why the reference is checked, when the specification does not say. - accepted_unresolved: Key tuples known not to resolve in the source, accepted - by the developer; they are reported, not counted as problems. Values are - JSON scalars; a timestamp or date is written in ISO format. - """ - - parent: str - columns: tuple[str, ...] - note: str = "" - accepted_unresolved: frozenset[tuple[Any, ...]] = frozenset() - - -@dataclasses.dataclass(frozen=True) -class TableSpec: - """One table of the nullability map.""" - - name: str - in_spec: bool - keys: tuple[str, ...] - columns: Mapping[str, ColumnSpec] - groups: Mapping[str, GroupSpec] - foreign_keys: tuple[ForeignKey, ...] - - def pending(self) -> list[str]: - """Return the column paths whose nullability is still undecided.""" - out = [n for n, c in self.columns.items() if c.nullable is None] - for g, group in self.groups.items(): - if group.nullable is None: - out.append(g) - out.extend( - f"{g}.{n}" for n, c in group.fields.items() if c.nullable is None - ) - return out - - -@dataclasses.dataclass(frozen=True) -class NullabilityMap: - """The whole nullability map: tables by name, plus where it came from.""" - - spec_version: str - source_project: str - tables: Mapping[str, TableSpec] - raw: Mapping[str, Any] - - def pending(self) -> dict[str, list[str]]: - """Return pending decisions by table (tables without any are left out).""" - return {n: p for n, t in self.tables.items() if (p := t.pending())} - - -def _column_from_json(d: Mapping[str, Any]) -> ColumnSpec: - """Build a ``ColumnSpec`` from its JSON form.""" - return ColumnSpec( - arrow_type=d["arrow_type"], - nullable=d["nullable"], - spec_type=d.get("spec_type"), - key=bool(d.get("key", False)), - ) - - -def parse_map(raw: Mapping[str, Any]) -> NullabilityMap: - """Build a ``NullabilityMap`` from its JSON form.""" - tables: dict[str, TableSpec] = {} - for name, t in raw["tables"].items(): - groups = { - g: GroupSpec( - arrow_type=gd["arrow_type"], - nullable=gd["nullable"], - fields={f: _column_from_json(fd) for f, fd in gd["fields"].items()}, - ) - for g, gd in t.get("groups", {}).items() - } - tables[name] = TableSpec( - name=name, - in_spec=bool(t["in_spec"]), - keys=tuple(t["keys"]), - columns={c: _column_from_json(cd) for c, cd in t["columns"].items()}, - groups=groups, - foreign_keys=tuple( - ForeignKey( - fk["parent"], - tuple(fk["columns"]), - fk.get("note", ""), - frozenset(tuple(k) for k in fk.get("accepted_unresolved", [])), - ) - for fk in t.get("foreign_keys", []) - ), - ) - return NullabilityMap(raw["spec_version"], raw["source_project"], tables, raw) - - -def load_map(path: Path = MAP_PATH) -> NullabilityMap: - """Read the nullability map from ``path``.""" - return parse_map(json.loads(Path(path).read_text(encoding="utf-8"))) - - -@dataclasses.dataclass(frozen=True) -class Subset: - """Which rows of the source make up the tutorial dataset. - - Attributes: - dataset: Spiral dataset of every table, in source and target. - window: Half-open ``[start, stop)`` in µs relative to ``session_start_datetime``. - unit_ids: The kept units. - tables: Selection mode per table: ``whole``, ``window``, ``window_units``, - ``units`` or ``trials``. - trial_table: The table selected by ``trials`` mode. - raw: The JSON form, hashed into each table's status digest. - """ - - dataset: str - window: tuple[int, int] - unit_ids: tuple[int, ...] - tables: Mapping[str, str] - trial_table: str - raw: Mapping[str, Any] - - -SELECTION_MODES = ("whole", "window", "window_units", "units", "trials") - - -def parse_subset(raw: Mapping[str, Any]) -> Subset: - """Build a ``Subset`` from its JSON form. - - Raises: - ToolError: if the window is empty or a selection mode is unknown. - """ - start, stop = (int(x) for x in raw["window_us"]) - if not 0 <= start < stop: - raise ToolError( - f"window_us must satisfy 0 <= start < stop, got {[start, stop]}" - ) - tables = {n: t["mode"] for n, t in raw["tables"].items()} - unknown = {n: m for n, m in tables.items() if m not in SELECTION_MODES} - if unknown: - raise ToolError(f"unknown selection modes {unknown}; known: {SELECTION_MODES}") - return Subset( - dataset=raw.get("dataset", "default"), - window=(start, stop), - unit_ids=tuple(sorted(int(u) for u in raw["unit_ids"])), - tables=tables, - trial_table=raw.get("trial_table", "experiment_freeviewing_trial"), - raw=raw, - ) - - -def load_subset(path: Path = SUBSET_PATH) -> Subset: - """Read the subset definition from ``path``.""" - return parse_subset(json.loads(Path(path).read_text(encoding="utf-8"))) - - -# The parts of a table's map entry that decide what is written. Foreign keys and -# their accepted exceptions only decide what plan and verify report. -_CONTENT_KEYS = ("in_spec", "keys", "columns", "groups") - - -def table_digest(nmap: NullabilityMap, subset: Subset, table: str) -> str: - """Return a digest of everything that decides a table's copied content.""" - entry = nmap.raw["tables"][table] - payload = { - "source_project": nmap.source_project, - "map": {k: entry[k] for k in _CONTENT_KEYS if k in entry}, - "subset": {k: v for k, v in subset.raw.items() if k != "notes"}, - } - return hashlib.sha256( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() - ).hexdigest() - - -# ------------------------------------------------------------------ schema handling - - -def _same_type_up_to_nullability(a: pa.DataType, b: pa.DataType) -> bool: - """Return whether two types are equal, ignoring struct children's nullability.""" - if pa.types.is_struct(a) and pa.types.is_struct(b): - if a.num_fields != b.num_fields: - return False - return all( - a.field(i).name == b.field(i).name - and _same_type_up_to_nullability(a.field(i).type, b.field(i).type) - for i in range(a.num_fields) - ) - return a == b - - -def target_schema( - source: pa.Schema, spec: TableSpec, *, allow_pending: bool = False -) -> pa.Schema: - """Return the schema the target table is written with. - - Types are the source's, exactly. Nullability comes from the map: for every - top-level column, and for a column group and its fields. - - Args: - source: The source table's schema. - spec: The table's entry in the nullability map. - allow_pending: Keep the source's nullability where a decision is pending - (used by ``plan``); otherwise a pending decision raises. - - Raises: - SchemaMismatchError: if the source's columns or types differ from the map. - PendingDecisionError: if a decision is pending and ``allow_pending`` is false. - """ - expected = list(spec.columns) + list(spec.groups) - if sorted(source.names) != sorted(expected): - missing = sorted(set(expected) - set(source.names)) - extra = sorted(set(source.names) - set(expected)) - raise SchemaMismatchError( - f"source columns differ from the map: missing {missing}, unexpected {extra}" - ) - - def nullability(decided: bool | None, field: pa.Field[Any], path: str) -> bool: - """Return the decided nullability, or the source's while pending if allowed. - - Raises: - PendingDecisionError: if ``decided`` is pending and ``allow_pending`` is - false. - """ - if decided is not None: - return decided - if allow_pending: - return field.nullable - raise PendingDecisionError(f"nullability of {path!r} is pending in the map") - - fields: list[pa.Field[Any]] = [] - for field in source: - if field.name in spec.groups: - group = spec.groups[field.name] - if str(field.type) != group.arrow_type: - raise SchemaMismatchError( - f"column group {field.name!r} has type {field.type}, " - f"the map records {group.arrow_type}" - ) - children = [] - for child in field.type: - cs = group.fields.get(child.name) - if cs is None: - raise SchemaMismatchError( - f"field {field.name}.{child.name} is not in the map" - ) - children.append( - child.with_nullable( - nullability(cs.nullable, child, f"{field.name}.{child.name}") - ) - ) - fields.append( - pa.field( - field.name, - pa.struct(children), - nullability(group.nullable, field, field.name), - ) - ) - else: - cs = spec.columns[field.name] - if str(field.type) != cs.arrow_type: - raise SchemaMismatchError( - f"column {field.name!r} has type {field.type}, " - f"the map records {cs.arrow_type}" - ) - fields.append( - field.with_nullable(nullability(cs.nullable, field, field.name)) - ) - return pa.schema(fields) - - -def key_schema(schema: pa.Schema, keys: Sequence[str]) -> pa.Schema: - """Return the key columns of ``schema``, in key order.""" - return pa.schema([schema.field(k) for k in keys]) - - -def _stored_type(t: pa.DataType) -> pa.DataType: - """Return ``t`` as SpiralDB stores it: struct fields nullable, list items kept.""" - if pa.types.is_struct(t): - return pa.struct([pa.field(f.name, _stored_type(f.type), True) for f in t]) - if pa.types.is_fixed_size_list(t): - item = t.value_field - return pa.list_(item.with_type(_stored_type(item.type)), t.list_size) - if pa.types.is_list(t): - return pa.list_(t.value_field.with_type(_stored_type(t.value_field.type))) - if pa.types.is_large_list(t): - return pa.large_list(t.value_field.with_type(_stored_type(t.value_field.type))) - return t - - -def stored_schema(schema: pa.Schema) -> pa.Schema: - """Return ``schema`` as SpiralDB stores it. - - SpiralDB (pyspiral 0.16.33, probed live on November 11, 2026) stores every - top-level column nullable, key columns included, and every struct field nullable; - only list and fixed-size-list items keep a non-null flag. The nullability the map - declares is therefore enforced by the copy (a null in a required column stops it) - and checked by ``verify`` from the data, not from the stored schema. - """ - return pa.schema([pa.field(f.name, _stored_type(f.type), True) for f in schema]) - - -# The path segment for a list item, as in Orcapod's nullability record (ITL-936). -ITEM = "[]" - - -def _is_list(t: pa.DataType) -> bool: - """Return whether ``t`` is a list, large list or fixed-size list type.""" - return ( - pa.types.is_list(t) - or pa.types.is_large_list(t) - or pa.types.is_fixed_size_list(t) - ) - - -def non_null_paths(schema: pa.Schema) -> list[tuple[str, ...]]: - """Return the paths ``schema`` declares non-nullable, sorted. - - A path starts with a top-level column; each further segment is a struct field - name, or ``"[]"`` for a list item, as Orcapod's nullability record (ITL-936) - lists them. Used to show what ``annotate`` records and to check it in ``verify``. - """ - out: list[tuple[str, ...]] = [] - - def walk(field: pa.Field[Any], path: tuple[str, ...]) -> None: - """Record ``path`` if ``field`` is non-nullable; descend into its children.""" - if not field.nullable: - out.append(path) - t = field.type - if pa.types.is_struct(t): - for child in t: - walk(child, (*path, child.name)) - elif _is_list(t): - walk(t.value_field, (*path, ITEM)) - - for f in schema: - walk(f, (f.name,)) - return sorted(out) - - -def _conform_array(arr: pa.Array[Any], field: pa.Field[Any]) -> pa.Array[Any]: - """Return ``arr`` typed as ``field`` declares, refusing nulls it does not allow. - - Raises: - SchemaMismatchError: if the types differ beyond nullability. - NullInRequiredColumnError: if ``field`` (or a declared struct child, or a - list item declared non-nullable) holds nulls. - """ - if not _same_type_up_to_nullability(arr.type, field.type): - raise SchemaMismatchError( - f"column {field.name!r} has type {arr.type}, expected {field.type}" - ) - if not field.nullable and arr.null_count: - raise NullInRequiredColumnError((field.name,), arr.null_count) - if _is_list(field.type): - # Check the items against the item field, recursively, so a required item - # of a nested list or a required field of a struct item is checked too. - try: - _conform_array(pc.list_flatten(arr), field.type.value_field) - except NullInRequiredColumnError as err: - raise NullInRequiredColumnError( - (field.name, ITEM, *err.path[1:]), err.null_count - ) from None - if not pa.types.is_struct(field.type): - return arr - struct = arr if isinstance(arr, pa.StructArray) else None - if struct is None: - raise SchemaMismatchError(f"column {field.name!r} is not a struct array") - # Check every child even when the types already match: Arrow builds arrays - # without enforcing child nullability, so equal types prove nothing about values. - children = [] - for child_field, child in zip(field.type, struct.flatten(), strict=True): - try: - children.append(_conform_array(child, child_field)) - except NullInRequiredColumnError as err: - raise err.within(field.name) from None - if arr.type == field.type: - return arr - mask = struct.is_null() if struct.null_count else None - return pa.StructArray.from_arrays(children, fields=list(field.type), mask=mask) - - -def conform_batch(batch: pa.RecordBatch, schema: pa.Schema) -> pa.RecordBatch: - """Return ``batch`` with ``schema``'s nullability, after checking it may have it. - - The column names and types must equal the schema's (nullability aside). Every - non-nullable column, and every non-nullable field of a struct column, must hold - no nulls. Nothing is cast or coerced. - - Raises: - SchemaMismatchError: on a column or type difference. - NullInRequiredColumnError: on a null in a non-nullable column. - """ - if batch.schema.names != schema.names: - raise SchemaMismatchError( - f"batch columns {batch.schema.names} differ from {schema.names}" - ) - arrays = [ - _conform_array(col, field) - for col, field in zip(batch.columns, schema, strict=True) - ] - return pa.RecordBatch.from_arrays(arrays, schema=schema) - - -# --------------------------------------------------------------------- source reads - - -def _qualified(subset: Subset, table: str) -> str: - """Return the identifier of ``table`` in the subset's dataset.""" - return f"{subset.dataset}.{table}" - - -def _window_filter(table: Any, ts_type: pa.DataType, window: tuple[int, int]) -> Any: - """Return a Spiral filter for ``window`` on ``table``'s ``timestamp`` column. - - The literals carry the column's exact Arrow type: SpiralDB refuses to compare - ``u64`` with an ``i64`` literal. - """ - start, stop = window - col = table["timestamp"] - return (col >= pa.scalar(start, ts_type)) & (col < pa.scalar(stop, ts_type)) - - -def _row_mask( - batch: pa.RecordBatch, - mode: str, - window: tuple[int, int], - unit_ids: pa.Array[Any], - trials: pa.Array[Any] | None, -) -> pa.Array[Any] | None: - """Return the client-side row mask for ``mode`` (``None`` keeps every row). - - The window is also applied here, so a row outside it is dropped even if a - server-side filter let it through. - """ - masks = [] - if mode in ("window", "window_units"): - ts = batch.column(batch.schema.get_field_index("timestamp")) - ts_type = ts.type - masks.append( - pc.and_( - pc.greater_equal(ts, pa.scalar(window[0], ts_type)), - pc.less(ts, pa.scalar(window[1], ts_type)), - ) - ) - if mode in ("window_units", "units"): - col = batch.column(batch.schema.get_field_index("unit_id")) - masks.append(pc.is_in(col, value_set=unit_ids.cast(col.type))) - if mode == "trials": - if trials is None: - raise ToolError("trial selection needs the referenced trials first") - col = batch.column(batch.schema.get_field_index("trial_index")) - masks.append(pc.is_in(col, value_set=trials.cast(col.type))) - if not masks: - return None - mask = masks[0] - for m in masks[1:]: - mask = pc.and_(mask, m) - return mask - - -def read_selection( - client: Any, - project: Any, - subset: Subset, - table: str, - mode: str, - trials: Sequence[int] | None = None, -) -> Iterator[pa.RecordBatch]: - """Yield the selected rows of one source table, in key order. - - Args: - client: A ``spiral.Spiral`` client (or a fake with the same surface). - project: The source ``Project``. - subset: The subset definition. - table: Table name (without dataset). - mode: The table's selection mode. - trials: The trial indexes to keep (``trials`` mode only). - """ - t = project.table(_qualified(subset, table)) - where = None - if mode in ("window", "window_units"): - ts_type = t.schema().to_arrow().field("timestamp").type - where = _window_filter(t, ts_type, subset.window) - unit_ids = pa.array(subset.unit_ids, pa.int64()) - trial_arr = pa.array(sorted(trials), pa.int64()) if trials is not None else None - scan = client.scan(t, where=where) if where is not None else client.scan(t) - for batch in scan.to_record_batches(): - mask = _row_mask(batch, mode, subset.window, unit_ids, trial_arr) - out = batch if mask is None else batch.filter(mask) - if out.num_rows: - yield out - - -def overlapping_trials( - client: Any, project: Any, subset: Subset -) -> tuple[set[int], set[int]]: - """Return the trials that overlap the window, and every trial in the table. - - A trial overlaps ``[start, stop)`` if it starts before ``stop`` and ends at or - after ``start``; a trial without an end overlaps if it starts inside the window. - """ - start, stop = subset.window - keep: set[int] = set() - every: set[int] = set() - for batch in client.scan( - project.table(_qualified(subset, subset.trial_table)) - ).to_record_batches(): - rows = batch.select( - ["trial_index", "trial_start_timestamp", "trial_end_timestamp"] - ).to_pylist() - for r in rows: - every.add(r["trial_index"]) - begin, end = r["trial_start_timestamp"], r["trial_end_timestamp"] - if end is None: - if start <= begin < stop: - keep.add(r["trial_index"]) - elif begin < stop and end >= start: - keep.add(r["trial_index"]) - return keep, every - - -def kept_trials(client: Any, project: Any, subset: Subset) -> list[int]: - """Return the trials to keep: those overlapping the window, plus every trial a - kept row of another table references through ``trial_index``. - - Raises: - ToolError: if a kept row references a trial the trial table does not hold. - """ - referenced: set[int] = set() - for name in table_order(subset): - mode = subset.tables[name] - if mode == "trials": - continue - schema = project.table(_qualified(subset, name)).schema().to_arrow() - if "trial_index" not in schema.names: - continue - for batch in read_selection(client, project, subset, name, mode): - referenced.update( - x - for x in pc.unique(batch.column("trial_index")).to_pylist() - if x is not None - ) - overlap, every = overlapping_trials(client, project, subset) - unknown = referenced - every - if unknown: - raise ToolError( - f"kept rows reference trials missing from {subset.trial_table}: {sorted(unknown)}" - ) - return sorted(overlap | referenced) - - -def table_order(subset: Subset) -> list[str]: - """Return the copy order: trial table last, so trial references are known.""" - names = sorted(subset.tables) - trial_tables = [n for n in names if subset.tables[n] == "trials"] - return [n for n in names if subset.tables[n] != "trials"] + trial_tables - - -# ------------------------------------------------------------- integrity bookkeeping - - -@dataclasses.dataclass -class TableResult: - """What the pipeline produced for one table.""" - - name: str - rows: int = 0 - nbytes: int = 0 - action: str = "" - - -@dataclasses.dataclass -class Integrity: - """Distinct key tuples, for checking foreign keys after every table is read.""" - - parent_keys: dict[str, set[tuple[Any, ...]]] = dataclasses.field( - default_factory=dict - ) - child_refs: dict[tuple[str, ForeignKey], set[tuple[Any, ...]]] = dataclasses.field( - default_factory=dict - ) - - def observe(self, nmap: NullabilityMap, table: str, batch: pa.RecordBatch) -> None: - """Record the key tuples of ``batch`` that foreign keys need.""" - spec = nmap.tables[table] - if any( - fk.parent == table for t in nmap.tables.values() for fk in t.foreign_keys - ): - self.parent_keys.setdefault(table, set()).update(_tuples(batch, spec.keys)) - for fk in spec.foreign_keys: - if fk.parent in nmap.tables: - self.child_refs.setdefault((table, fk), set()).update( - t for t in _tuples(batch, fk.columns) if None not in t - ) - - def problems(self, present: Iterable[str]) -> list[str]: - """Return every unresolved reference among the tables in ``present``. - - References a foreign key lists as accepted are left out; ``accepted`` reports - them. - """ - return self._unresolved(set(present), accepted=False) - - def accepted(self, present: Iterable[str]) -> list[str]: - """Return the unresolved references the map accepts, among ``present``.""" - return self._unresolved(set(present), accepted=True) - - def _unresolved(self, present: set[str], *, accepted: bool) -> list[str]: - """Return the unresolved references among the tables in ``present``. - - Args: - present: Tables the project holds. - accepted: Report the key tuples the map accepts as unresolved instead of the - others (a parent table missing from the project is reported only when - false). - - Returns: - One line per foreign key with unresolved key tuples, with a sample of them. - """ - out = [] - for (child, fk), refs in sorted( - self.child_refs.items(), key=lambda kv: (kv[0][0], kv[0][1].parent) - ): - if child not in present: - continue - if fk.parent not in present: - if not accepted: - out.append( - f"{child} -> {fk.parent}: parent table is not in the project" - ) - continue - missing = { - k - for k in refs - self.parent_keys.get(fk.parent, set()) - if (_json_key(k) in fk.accepted_unresolved) == accepted - } - if missing: - sample = sorted(missing, key=repr)[:3] - out.append( - f"{child} -> {fk.parent} {list(fk.columns)}: {len(missing)} " - f"{'accepted ' if accepted else ''}unresolved key(s), e.g. {sample}" - ) - return out - - -def _json_key(key: tuple[Any, ...]) -> tuple[Any, ...]: - """Return ``key`` as the map writes it: timestamps and dates in ISO format.""" - return tuple(v.isoformat() if hasattr(v, "isoformat") else v for v in key) - - -def _tuples(batch: pa.RecordBatch, columns: Sequence[str]) -> set[tuple[Any, ...]]: - """Return the distinct tuples of ``columns`` across the rows of ``batch``.""" - if not batch.num_rows: - return set() - cols = [batch.column(batch.schema.get_field_index(c)).to_pylist() for c in columns] - return set(zip(*cols, strict=True)) - - -# ----------------------------------------------------------------------- the sinks - - -class DryRunSink: - """Accepts every batch and writes nothing. - - Args: - replace: Tables a real copy would drop and copy again. - """ - - skipped_rows = 0 - - def __init__(self, replace: Iterable[str] = ()) -> None: - """Remember the tables a real copy would replace.""" - self._replace = frozenset(replace) - - def open_table( - self, table: str, schema: pa.Schema, keys: pa.Schema, digest: str - ) -> str: - """Return the action a real copy would take for ``table``.""" - return "would replace" if table in self._replace else "would write" - - def write(self, batch: pa.RecordBatch) -> None: - """Discard ``batch``.""" - - def commit(self, rows: int) -> None: - """Do nothing.""" - - def abort(self) -> None: - """Do nothing.""" - - -class SpiralSink: - """Writes each table to the target project in one transaction. - - Every table it creates carries the build marker in its metadata, set before any - data is written. When a table's transaction commits, its status metadata records - the row count and the subset digest, so a re-run can skip it. - """ - - def __init__( - self, - project: Any, - subset: Subset, - marker: Mapping[str, Any], - replace: Iterable[str] = (), - ) -> None: - """Hold the target project and the subset, and encode the build marker. - - Args: - project: The target Spiral project. - subset: The subset being copied. - marker: The build marker, stored as JSON in each created table's metadata. - replace: Tables to drop and copy again. - """ - self._project = project - self._subset = subset - self._replace = frozenset(replace) - self._marker = json.dumps(marker, sort_keys=True).encode() - self._table: Any = None - self._txn: Any = None - self._digest = "" - self.skipped_rows = 0 - - def open_table( - self, table: str, schema: pa.Schema, keys: pa.Schema, digest: str - ) -> str: - """Prepare ``table`` for writing; return ``"skip"`` if it is already complete. - - An existing table that is not complete is dropped and created again, so no - row of an earlier attempt survives. - - Raises: - TargetRefusedError: if the table exists without this build's marker. - StaleTableError: if it is complete for a different digest. - """ - ident = _qualified(self._subset, table) - existing = {f"{r.dataset}.{r.table}" for r in self._project.list_tables()} - if ident in existing and table in self._replace: - if ( - self._project.table(ident).get_metadata().get(MARKER_KEY) - != self._marker - ): - raise TargetRefusedError(f"{ident} exists without this build's marker") - self._project.drop_table(ident) - existing.discard(ident) - replaced = True - else: - replaced = False - if ident in existing: - t = self._project.table(ident) - meta = t.get_metadata() - if meta.get(MARKER_KEY) != self._marker: - raise TargetRefusedError(f"{ident} exists without this build's marker") - status = json.loads(meta[STATUS_KEY]) if STATUS_KEY in meta else None - if status and status.get("complete"): - if status.get("digest") != digest: - raise StaleTableError( - f"{ident} is complete for a different subset or map; " - "drop it and copy again" - ) - self.skipped_rows = int(status.get("rows", 0)) - return "skip" - # An incomplete table may hold rows of an earlier attempt (its data - # committed, its status did not), possibly for another subset: start over. - self._project.drop_table(ident) - t = self._project.create_table(ident, key_schema=keys) - t.set_metadata({MARKER_KEY: self._marker}) - action = "rewrite" - else: - t = self._project.create_table(ident, key_schema=keys) - t.set_metadata({MARKER_KEY: self._marker}) - action = "replace" if replaced else "create" - self._table = t - self._digest = digest - self._txn = t.txn() - return action - - def write(self, batch: pa.RecordBatch) -> None: - """Write ``batch`` inside the open transaction.""" - self._txn.write(batch) - - def commit(self, rows: int) -> None: - """Commit the transaction and record the table as complete.""" - self._txn.commit() - status = {"complete": True, "rows": rows, "digest": self._digest} - self._table.set_metadata({STATUS_KEY: json.dumps(status).encode()}) - self._txn = None - - def abort(self) -> None: - """Abort the open transaction, if any.""" - if self._txn is not None: - self._txn.abort() - self._txn = None - - -# --------------------------------------------------------------------- the pipeline - - -@dataclasses.dataclass -class Report: - """The outcome of ``plan``, ``copy`` or ``verify``.""" - - tables: list[TableResult] - kept_trials: list[int] - pending: dict[str, list[str]] - problems: list[str] - accepted: list[str] = dataclasses.field(default_factory=list) - notes: list[str] = dataclasses.field(default_factory=list) - - def render(self) -> str: - """Return a human-readable summary.""" - lines = [f"{'table':40s} {'rows':>10s} {'MB':>9s} action"] - for t in self.tables: - lines.append( - f"{t.name:40s} {t.rows:>10,d} {t.nbytes / 1e6:>9.1f} {t.action}" - ) - total = sum(t.nbytes for t in self.tables) - lines.append( - f"{'total':40s} {sum(t.rows for t in self.tables):>10,d} {total / 1e6:>9.1f}" - ) - lines.append(f"kept trials: {self.kept_trials}") - if self.pending: - lines.append("pending decisions (source nullability kept in this dry run):") - lines.extend(f" {t}: {cols}" for t, cols in self.pending.items()) - if self.accepted: - lines.append("accepted exceptions (recorded in the map):") - lines.extend(f" {a}" for a in self.accepted) - if self.notes: - lines.append("notes:") - lines.extend(f" {n}" for n in self.notes) - if self.problems: - lines.append("problems:") - lines.extend(f" {p}" for p in self.problems) - return "\n".join(lines) - - -def _run( - client: Any, - source: Any, - nmap: NullabilityMap, - subset: Subset, - sink: DryRunSink | SpiralSink, - *, - allow_pending: bool, - on_table: Callable[[TableResult], None] | None = None, - only: Iterable[str] | None = None, -) -> Report: - """Read every selected source table, check it, and hand it to ``sink``. - - ``only`` restricts the run to those tables; foreign keys are then left to - ``verify``, which sees every table. - """ - missing = sorted(set(subset.tables) - set(nmap.tables)) - if missing: - raise ToolError(f"tables in the subset but not in the map: {missing}") - integrity = Integrity() - trials = ( - kept_trials(client, source, subset) - if "trials" in subset.tables.values() - else [] - ) - results = [] - selected = set(subset.tables) if only is None else set(only) - for name in table_order(subset): - if name not in selected: - continue - mode = subset.tables[name] - spec = nmap.tables[name] - src_table = source.table(_qualified(subset, name)) - src_schema = src_table.schema().to_arrow() - result = TableResult(name) - try: - src_keys = list(src_table.key_schema.to_arrow().names) - if src_keys != list(spec.keys): - raise SchemaMismatchError( - f"source key columns {src_keys} differ from the map's " - f"{list(spec.keys)}" - ) - schema = target_schema(src_schema, spec, allow_pending=allow_pending) - keys = key_schema(schema, spec.keys) - result.action = sink.open_table( - name, schema, keys, table_digest(nmap, subset, name) - ) - if result.action == "skip": - result.rows = sink.skipped_rows - results.append(result) - continue - try: - for batch in read_selection(client, source, subset, name, mode, trials): - out = conform_batch(batch, schema) - integrity.observe(nmap, name, out) - sink.write(out) - result.rows += out.num_rows - result.nbytes += out.nbytes - if not result.rows: - raise ToolError( - "the selection holds no rows; an empty table means the subset " - "is wrong" - ) - except BaseException: - sink.abort() - raise - sink.commit(result.rows) - except ToolError as err: - err.add_note(f"while processing table {name!r}") - raise - results.append(result) - if on_table is not None: - on_table(result) - complete = only is None and not any(r.action == "skip" for r in results) - problems = integrity.problems(subset.tables) if complete else [] - accepted = integrity.accepted(subset.tables) if complete else [] - return Report(results, trials, nmap.pending(), problems, accepted) - - -def plan( - client: Any, source_project: str, nmap: NullabilityMap, subset: Subset -) -> Report: - """Run the whole copy against the source without writing anything. - - Pending nullability decisions keep the source's nullability and are listed in - the report. - """ - source = client.project(source_project) - return _run(client, source, nmap, subset, DryRunSink(), allow_pending=True) - - -# ------------------------------------------------------------------ target guards - - -def check_target_name(target: str | None, allow_writes: str | None, source: str) -> str: - """Return ``target`` if the names alone allow writing to it. - - Raises: - TargetRefusedError: if no target is given, ``allow_writes`` differs from it, - it equals the source, or it starts with a forbidden prefix. - """ - if not target: - raise TargetRefusedError("--target-project is required") - if allow_writes != target: - raise TargetRefusedError( - "--allow-writes must repeat --target-project exactly " - f"(got {allow_writes!r} for {target!r})" - ) - if target == source: - raise TargetRefusedError("the target project equals the source project") - if target.startswith(FORBIDDEN_TARGET_PREFIXES): - raise TargetRefusedError( - f"target {target!r} starts with one of {FORBIDDEN_TARGET_PREFIXES}" - ) - return target - - -def read_marker(project: Any) -> dict[str, Any]: - """Return the build marker of a project this tool created. - - Raises: - TargetRefusedError: if the project has no marker table, or holds a table - without the same marker. - """ - tables = {f"{r.dataset}.{r.table}" for r in project.list_tables()} - if MARKER_TABLE not in tables: - raise TargetRefusedError( - f"project has no {MARKER_TABLE} table: it was not made by this tool's create step" - ) - raw = project.table(MARKER_TABLE).get_metadata().get(MARKER_KEY) - if raw is None: - raise TargetRefusedError(f"{MARKER_TABLE} carries no {MARKER_KEY} metadata") - foreign = sorted( - ident - for ident in tables - {MARKER_TABLE} - if project.table(ident).get_metadata().get(MARKER_KEY) != raw - ) - if foreign: - raise TargetRefusedError( - f"project holds tables this build did not write: {foreign}" - ) - return json.loads(raw) - - -def open_target( - client: Any, target: str, allow_writes: str | None, source: str -) -> tuple[Any, dict[str, Any]]: - """Return the target ``Project`` and its marker after every guard has passed.""" - check_target_name(target, allow_writes, source) - project = client.project(target) - return project, read_marker(project) - - -def create_project( - client: Any, - *, - id_prefix: str, - allow_create: str | None, - description: str, - volume_id: str | None, - source_project: str, -) -> str: - """Create the target project and its marker table; return the new project ID. - - Raises: - TargetRefusedError: if ``allow_create`` differs from ``id_prefix``, the prefix - is forbidden, or the new project is not empty. - """ - if allow_create != id_prefix: - raise TargetRefusedError("--allow-create must repeat --id-prefix exactly") - if id_prefix.startswith(FORBIDDEN_TARGET_PREFIXES) or id_prefix == source_project: - raise TargetRefusedError(f"id prefix {id_prefix!r} is forbidden") - before = {p.id for p in client.list_projects()} - created = client.create_project( - id_prefix=id_prefix, description=description, volume_id=volume_id - ) - if created.id in before: - raise TargetRefusedError(f"project {created.id} existed before create") - project = client.project(created.id) - if list(project.list_tables()): - raise TargetRefusedError(f"new project {created.id} is not empty") - marker = { - "build_id": uuid.uuid4().hex, - "tool": TOOL_NAME, - "source_project": source_project, - "project_id": created.id, - } - t = project.create_table( - MARKER_TABLE, - key_schema=pa.schema([pa.field("build_id", pa.large_string(), False)]), - ) - t.set_metadata({MARKER_KEY: json.dumps(marker, sort_keys=True).encode()}) - return created.id - - -def copy( - client: Any, - *, - source_project: str, - target_project: str | None, - allow_writes: str | None, - nmap: NullabilityMap, - subset: Subset, - dry_run: bool = False, - on_table: Callable[[TableResult], None] | None = None, - tables: Sequence[str] | None = None, - replace: bool = False, -) -> Report: - """Copy the subset from the source into the target project. - - With ``dry_run`` the target guards still run, but nothing is written. ``tables`` - restricts the copy to those tables; with ``replace`` each of them is dropped (it - must carry this build's marker) and copied again from the source, which picks up - a change made in the source after the first copy. - - Raises: - TargetRefusedError: if the target fails a guard. - PendingDecisionError: if the map holds pending decisions. - NullInRequiredColumnError: if a selected row has a null where the map - forbids one (that table's transaction is aborted). - """ - project, marker = open_target( - client, target_project or "", allow_writes, source_project - ) - if marker.get("source_project") != source_project: - raise TargetRefusedError( - f"the target was created for source {marker.get('source_project')!r}" - ) - pending = nmap.pending() - if pending: - raise PendingDecisionError( - f"decide these in the nullability map first: {pending}" - ) - if replace and not tables: - raise ToolError("--replace needs --tables naming the tables to copy again") - unknown = sorted(set(tables or ()) - set(subset.tables)) - if unknown: - raise ToolError(f"tables not in the subset: {unknown}") - replacing = tuple(tables or ()) if replace else () - source = client.project(source_project) - sink: DryRunSink | SpiralSink = ( - DryRunSink(replacing) - if dry_run - else SpiralSink(project, subset, marker, replacing) - ) - report = _run( - client, - source, - nmap, - subset, - sink, - allow_pending=False, - on_table=on_table, - only=tables, - ) - record = provenance(client, source_project, nmap, subset, report) - existing = project.table(MARKER_TABLE).get_metadata().get(PROVENANCE_KEY) - if tables and existing is not None: - record["rows"] = {**json.loads(existing).get("rows", {}), **record["rows"]} - encoded = json.dumps(record, sort_keys=True, default=str).encode() - if len(PROVENANCE_KEY) + len(encoded) > METADATA_LIMIT: - raise ToolError(f"provenance is {len(encoded)} bytes, over SpiralDB's 8 KiB") - marker_table = project.table(MARKER_TABLE) - if ( - not dry_run - and not report.problems - and marker_table.get_metadata().get(PROVENANCE_KEY) != encoded - ): - marker_table.set_metadata({PROVENANCE_KEY: encoded}) - return report - - -def provenance( - client: Any, - source_project: str, - nmap: NullabilityMap, - subset: Subset, - report: Report, -) -> dict[str, Any]: - """Return the provenance record ``copy`` stores on the marker table. - - It names the source project (with its description), the specification version, - the window, units and trials, the row count of every table, and the source's - ``sample_info`` row, which is not copied because it describes the whole sample - (its ``sources_json`` is replaced by a SHA-256 digest to stay under 8 KiB). - """ - descriptions = {p.id: p.description for p in client.list_projects()} - record: dict[str, Any] = { - "source_project": source_project, - "source_description": descriptions.get(source_project), - "spec_version": nmap.spec_version, - "window_us": list(subset.window), - "unit_ids": list(subset.unit_ids), - "kept_trials": report.kept_trials, - "rows": {t.name: t.rows for t in report.tables}, - "decisions": nmap.raw.get("decisions", {}).get("decided_by"), - } - source = client.project(source_project) - names = {r.table for r in source.list_tables() if r.dataset == subset.dataset} - if "sample_info" in names: - rows = client.scan(source.table(_qualified(subset, "sample_info"))).to_table() - info = [] - for row in rows.to_pylist(): - blob = row.pop("sources_json", None) - if blob is not None: - row["sources_json_sha256"] = hashlib.sha256(blob.encode()).hexdigest() - info.append(row) - record["source_sample_info"] = info - return record - - -# --------------------------------------------------------------------------- verify - - -def _rechunk(batches: Iterable[pa.RecordBatch], rows: int) -> Iterator[pa.Table]: - """Yield tables of exactly ``rows`` rows (the last may be shorter).""" - pending: list[pa.RecordBatch] = [] - count = 0 - for b in batches: - pending.append(b) - count += b.num_rows - while count >= rows: - t = pa.Table.from_batches(pending) - yield t.slice(0, rows) - rest = t.slice(rows) - pending = rest.to_batches() - count = rest.num_rows - if count: - yield pa.Table.from_batches(pending) - - -def verify( - client: Any, - *, - source_project: str, - target_project: str, - nmap: NullabilityMap, - subset: Subset, - read_record: Callable[[Mapping[str, bytes]], Any] | None = None, -) -> Report: - """Check the target project against the source, the map and the subset. - - With ``read_record`` (Orcapod's ``read_nullability_record``, ITL-936), each - table's nullability record must list exactly the non-nullable paths the map - declares; a table without a record is listed in the report's notes. - - Checks: the project holds exactly the subset's tables (plus the marker table); - each table's schema has the source's types exactly and the map's nullability, its - key columns are the source's, in order, and non-null; required columns hold no - nulls; every foreign key resolves; and every row equals the source selection. - - Raises: - VerificationError: listing every problem found. - """ - target = client.project(target_project) - read_marker(target) - stored = target.table(MARKER_TABLE).get_metadata().get(PROVENANCE_KEY) - source = client.project(source_project) - problems: list[str] = [] - if stored is None: - problems.append(f"{MARKER_TABLE} carries no {PROVENANCE_KEY} metadata") - else: - record = json.loads(stored) - if ( - record.get("source_project") != source_project - or record.get("window_us") != list(subset.window) - or record.get("unit_ids") != list(subset.unit_ids) - ): - problems.append("the provenance record does not match the subset") - present = {r.table for r in target.list_tables() if r.dataset == subset.dataset} - if present != set(subset.tables): - problems.append( - f"tables differ: missing {sorted(set(subset.tables) - present)}, " - f"unexpected {sorted(present - set(subset.tables))}" - ) - trials = ( - kept_trials(client, source, subset) - if "trials" in subset.tables.values() - else [] - ) - integrity = Integrity() - results = [] - for name in table_order(subset): - if name not in present: - continue - spec = nmap.tables[name] - t = target.table(_qualified(subset, name)) - src_schema = source.table(_qualified(subset, name)).schema().to_arrow() - want = target_schema(src_schema, spec, allow_pending=True) - got = t.schema().to_arrow() - if got != stored_schema(want): - problems.append( - f"{name}: schema {got} differs from expected {stored_schema(want)}" - ) - got_keys = t.key_schema.to_arrow() - src_keys = source.table(_qualified(subset, name)).key_schema.to_arrow().names - if got_keys.names != list(spec.keys) or got_keys.names != src_keys: - problems.append( - f"{name}: key columns {got_keys.names}; the map's {list(spec.keys)}, " - f"the source's {src_keys}" - ) - mode = subset.tables[name] - expected = ( - conform_batch(b, want) - for b in read_selection(client, source, subset, name, mode, trials) - ) - actual_batches: list[int] = [0] - - def observed( - batches: Iterable[pa.RecordBatch], table: str = name - ) -> Iterator[pa.RecordBatch]: - """Yield ``batches`` unchanged, counting rows and recording their keys.""" - for b in batches: - actual_batches[0] += b.num_rows - integrity.observe(nmap, table, b) - yield b - - actual = observed(client.scan(t).to_record_batches()) - result = TableResult(name, action="verified") - mismatch = _compare_streams(expected, actual, want) - result.rows = actual_batches[0] - if mismatch: - problems.append(f"{name}: {mismatch}") - results.append(result) - problems.extend(integrity.problems(present)) - notes: list[str] = [] - if read_record is not None: - for name in sorted(present & set(subset.tables)): - ident = _qualified(subset, name) - src_schema = source.table(ident).schema().to_arrow() - want = set(non_null_paths(target_schema(src_schema, nmap.tables[name]))) - record = read_record(target.table(ident).get_metadata()) - if record is None: - notes.append(f"{name}: no nullability record") - elif set(record) != want: - problems.append( - f"{name}: nullability record differs from the map: " - f"missing {sorted(want - set(record))}, " - f"extra {sorted(set(record) - want)}" - ) - report = Report( - results, trials, nmap.pending(), problems, integrity.accepted(present), notes - ) - if problems: - raise VerificationError(problems) - return report - - -def load_record_reader() -> Callable[[Mapping[str, bytes]], Any] | None: - """Return Orcapod's ``read_nullability_record`` (ITL-936), or ``None`` without it.""" - import orcapod.databases - - return getattr(orcapod.databases, "read_nullability_record", None) - - -_FLOAT_BITS = {16: np.uint16, 32: np.uint32, 64: np.uint64} - - -def _float_bits(arr: pa.Array[Any]) -> Any: - """Return the raw bit patterns of a float array's slots, as unsigned integers.""" - dtype = _FLOAT_BITS[arr.type.bit_width] - data = arr.buffers()[1] - assert data is not None - # to_pybytes copies; compared slices hold at most COMPARE_SLICE_ROWS rows - return np.frombuffer(data.to_pybytes(), dtype=dtype)[ - arr.offset : arr.offset + len(arr) - ] - - -def identical(a: pa.Array[Any], b: pa.Array[Any]) -> bool: - """Return whether two arrays hold the same values, comparing floats bit for bit. - - Arrow's ``equals`` treats NaN as unequal to itself, so a column holding NaN - (gaze positions during blinks, some unit quality metrics) never equals its own - copy. Here a float slot equals another when their bit patterns are identical: NaN - equals NaN with the same payload, and -0.0 differs from 0.0. Bit-exact is the - right test for a copy, which must not change a single value. Struct fields and - list items are compared the same way, recursively; other types use ``equals``. - """ - if a.type != b.type or len(a) != len(b): - return False - t = a.type - if not (pa.types.is_floating(t) or pa.types.is_struct(t) or _is_list(t)): - # Arrow's equals treats -0.0 as 0.0 and NaN as unequal to itself, so only - # types that cannot hold a float take it. - return a.equals(b) - if a.null_count != b.null_count or not a.is_null().equals(b.is_null()): - return False - if pa.types.is_floating(t): - valid = np.asarray(a.is_valid().to_numpy(zero_copy_only=False), dtype=bool) - return bool(np.array_equal(_float_bits(a)[valid], _float_bits(b)[valid])) - if pa.types.is_struct(t): - sa, sb = a, b - assert isinstance(sa, pa.StructArray) and isinstance(sb, pa.StructArray) - return all( - identical(x, y) for x, y in zip(sa.flatten(), sb.flatten(), strict=True) - ) - if not pc.list_value_length(a).equals(pc.list_value_length(b)): - return False - return identical(pc.list_flatten(a), pc.list_flatten(b)) - - -def _compare_streams( - expected: Iterable[pa.RecordBatch], - actual: Iterable[pa.RecordBatch], - schema: pa.Schema, -) -> str: - """Return a description of the first difference, or ``""`` if the rows are equal.""" - exp_iter = _rechunk(expected, COMPARE_SLICE_ROWS) - act_iter = _rechunk(actual, COMPARE_SLICE_ROWS) - offset = 0 - while True: - e = next(exp_iter, None) - a = next(act_iter, None) - if e is None and a is None: - return "" - if e is None or a is None or e.num_rows != a.num_rows: - e_rows = ( - offset - + (e.num_rows if e is not None else 0) - + sum(t.num_rows for t in exp_iter) - ) - a_rows = ( - offset - + (a.num_rows if a is not None else 0) - + sum(t.num_rows for t in act_iter) - ) - return f"row count {a_rows} differs from expected {e_rows}" - if a.schema.names != schema.names: - return f"columns {a.schema.names} differ from {schema.names}" - for i, name in enumerate(schema.names): - want_col = e.column(i) - got_col = a.column(i) - try: - for chunk in got_col.chunks: - _conform_array(chunk, schema.field(i)) - except NullInRequiredColumnError as err: - return str(err) - except SchemaMismatchError as err: - return str(err) - if want_col.type != got_col.type: - want_col = want_col.cast(got_col.type) - if not identical(got_col.combine_chunks(), want_col.combine_chunks()): - return f"column {name!r} differs from the source in rows {offset}..{offset + e.num_rows - 1}" - offset += e.num_rows - - -# ---------------------------------------------------------------------- annotate - - -RecordWriter = Callable[[Any, pa.Schema], bool] - - -def load_record_writer() -> RecordWriter: - """Return Orcapod's ``write_nullability_record`` (ITL-936). - - Raises: - ToolError: if the installed Orcapod does not have it. - """ - import orcapod.databases - - writer = getattr(orcapod.databases, "write_nullability_record", None) - if writer is None: - raise ToolError( - "this Orcapod has no orcapod.databases.write_nullability_record; annotate " - "needs Orcapod with ITL-936 (the SpiralDB nullability record)" - ) - return writer - - -def annotate( - client: Any, - *, - source_project: str, - target_project: str, - allow_writes: str | None, - nmap: NullabilityMap, - subset: Subset, - write_record: RecordWriter | None, - dry_run: bool = False, -) -> list[str]: - """Give each copied table Orcapod's nullability record (ITL-936). - - The record is written by ``write_record`` (Orcapod's ``write_nullability_record``) - from the exact schema ``copy`` wrote the table with: the source's types and the - map's nullability. The record's format is Orcapod's; this tool does not encode it. - Before recording a table, its data in the target is read and every column the - schema declares non-nullable is checked for nulls, since the helper reads no data. - Only table metadata is written, never data; a table whose record is current is - left alone. - - Args: - client: The Spiral client. - source_project: The source project (read for each table's schema). - target_project: The project to annotate (a project ``create`` made). - allow_writes: Must repeat ``target_project``. - nmap: The nullability map. - subset: The subset definition (which tables, which dataset). - write_record: Writes the record; returns whether it changed anything. Not - called (and may be ``None``) with ``dry_run``. - dry_run: Check the data and list each table's non-nullable paths, but write - nothing. - - Returns: - One line per table: ``recorded``, ``unchanged``, or with ``dry_run`` the - paths that would be recorded. - - Raises: - TargetRefusedError: if the target fails a guard. - ToolError: if a subset table is missing from the target. - NullInRequiredColumnError: if a table holds a null where the map forbids one - (nothing is recorded for it). - """ - project, _ = open_target(client, target_project, allow_writes, source_project) - source = client.project(source_project) - present = {f"{r.dataset}.{r.table}" for r in project.list_tables()} - lines = [] - for name in sorted(subset.tables): - ident = _qualified(subset, name) - if ident not in present: - raise ToolError(f"{ident} is not in the target; copy it first") - src_schema = source.table(ident).schema().to_arrow() - declared = target_schema(src_schema, nmap.tables[name]) - table = project.table(ident) - try: - for batch in client.scan(table).to_record_batches(): - conform_batch(batch.select(declared.names), declared) - except ToolError as err: - err.add_note(f"while checking table {name!r} before recording it") - raise - if dry_run or write_record is None: - paths = ["/".join(p) for p in non_null_paths(declared)] - lines.append(f"{name}: would record {len(paths)} non-null path(s): {paths}") - continue - changed = write_record(table, declared) - lines.append(f"{name}: {'recorded' if changed else 'unchanged'}") - return lines - - -# ---------------------------------------------------------------- the Orcapod check - - -def _declared_nullable(spec: TableSpec, column: str) -> bool | None: - """Return the nullability ``spec`` declares for ``column`` (``None`` if unknown).""" - if column in spec.columns: - return spec.columns[column].nullable - if column in spec.groups: - return spec.groups[column].nullable - return None - - -def orcapod_check( - target_project: str, - nmap: NullabilityMap, - subset: Subset, - make_source: Callable[[str, str, str], Any], -) -> tuple[list[str], list[str]]: - """Load every target table as an Orcapod source and check its column types. - - A column declared non-nullable must have a plain Python type ``T``; a nullable - one must be ``T | None``. Tables with a ``fixed_size_list`` column are skipped: - Orcapod cannot hash that type until starfix supports it (ITL-927). - - Args: - target_project: The project to read. - nmap: The nullability map. - subset: The subset definition (which tables, which dataset). - make_source: Builds a source from ``(project_id, table, dataset)``; the CLI - passes ``SpiralDBTableSource`` (eager). - - Returns: - One line per table describing the outcome, and the list of problems. - """ - lines: list[str] = [] - problems: list[str] = [] - for name in sorted(subset.tables): - spec = nmap.tables[name] - types = [c.arrow_type for c in spec.columns.values()] - if any(t.startswith("fixed_size_list") for t in types): - lines.append(f"{name}: skipped (fixed_size_list, ITL-927)") - continue - source = make_source(target_project, name, subset.dataset) - tag_schema, data_schema = source.output_schema() - wrong = [] - for column, py_type in {**tag_schema, **data_schema}.items(): - optional = type(None) in typing.get_args(py_type) - declared = _declared_nullable(spec, column) - if declared is not None and optional != declared: - wrong.append(f"{column}: {py_type}") - if wrong: - problems.append(f"{name}: nullability differs from the map: {wrong}") - lines.append( - f"{name}: {len(tag_schema)} tag + {len(data_schema)} data columns" - + (f", WRONG {wrong}" if wrong else ", nullability as the map") - ) - return lines, problems - - -# ------------------------------------------------------------------------ the probe - - -def _probe_batch( - k: int, v: int | None, x: int | None, *, v_nullable: bool, x_nullable: bool -) -> pa.RecordBatch: - """Return one probe row: key ``k``, top-level ``v`` and nested ``g.x``.""" - group = pa.struct( - [pa.field("x", pa.int32(), x_nullable), pa.field("y", pa.int32(), True)] - ) - items = pa.list_(pa.field("item", pa.float32(), False), 2) - schema = pa.schema( - [ - pa.field("k", pa.int64(), False), - pa.field("v", pa.int32(), v_nullable), - pa.field("w", pa.int32(), True), - pa.field("g", group, False), - pa.field("l", items, False), - ] - ) - g = pa.StructArray.from_arrays( - [pa.array([x], pa.int32()), pa.array([None], pa.int32())], - fields=list(group), - ) - return pa.RecordBatch.from_arrays( - [ - pa.array([k], pa.int64()), - pa.array([v], pa.int32()), - pa.array([None], pa.int32()), - g, - # pyarrow-stubs types `type` as None; pyarrow accepts a FixedSizeListType - pa.FixedSizeListArray.from_arrays( - pa.array([1.0, 2.0], pa.float32()), - type=items, # pyright: ignore[reportArgumentType] - ), - ], - schema=schema, - ) - - -def probe_nullability( - client: Any, *, target_project: str, allow_writes: str | None, source_project: str -) -> dict[str, Any]: - """Record how SpiralDB treats non-nullable columns, in a scratch table. - - Writes to ``meta.nullability_probe`` in the target (a project ``create`` made) and - drops it afterwards. The table has a non-null key ``k``, a non-null top-level - column ``v``, a nullable ``w``, a struct column ``g`` with a non-null field ``x``, - and a ``fixed_size_list`` ``l`` with non-null items. Reported: the schema SpiralDB - keeps (its own repr, where ``?`` marks nullable, and the Arrow form), and what each - write of a null does, declared non-nullable and declared nullable, at the top - level and in the struct. - """ - project, _ = open_target(client, target_project, allow_writes, source_project) - tables = {f"{r.dataset}.{r.table}" for r in project.list_tables()} - if PROBE_TABLE in tables: - project.drop_table(PROBE_TABLE) - out: dict[str, Any] = {} - t = project.create_table( - PROBE_TABLE, key_schema=pa.schema([pa.field("k", pa.int64(), False)]) - ) - # Marked like every table this build writes, so a probe killed before its - # cleanup leaves a table the next command accepts and the next probe drops. - t.set_metadata({MARKER_KEY: project.table(MARKER_TABLE).get_metadata()[MARKER_KEY]}) - - def state() -> dict[str, Any]: - """Return the probe table's Spiral and Arrow schemas, scan schema and rows.""" - return { - "spiral_schema": repr(t.schema()), - "arrow_schema": str(t.schema().to_arrow()), - "scan_schema": str(client.scan(t).to_table().schema), - "rows": client.scan(t).to_table().to_pylist(), - } - - try: - out["key_schema_after_create"] = repr(t.key_schema) - with t.txn() as txn: - txn.write(_probe_batch(1, 10, 100, v_nullable=False, x_nullable=False)) - out["after_non_null_write"] = state() - variants = { - "null_v_declared_non_null": (2, None, 200, False, False), - "null_v_declared_nullable": (3, None, 300, True, False), - "null_x_declared_non_null": (4, 40, None, False, False), - "null_x_declared_nullable": (5, 50, None, False, True), - } - for label, (k, v, x, v_nullable, x_nullable) in variants.items(): - try: - batch = _probe_batch( - k, v, x, v_nullable=v_nullable, x_nullable=x_nullable - ) - with t.txn() as txn: - txn.write(batch) - out[label] = {"raised": None, **state()} - except Exception as err: # noqa: BLE001 - the probe records whatever SpiralDB raises - out[label] = {"raised": f"{type(err).__name__}: {err}"} - finally: - project.drop_table(PROBE_TABLE) - return out - - -# ---------------------------------------------------------------- deriving the map - -_SPEC_PRIMITIVES = { - "utf8": pa.string(), - "u8": pa.uint8(), - "u16": pa.uint16(), - "u32": pa.uint32(), - "u64": pa.uint64(), - "i16": pa.int16(), - "i32": pa.int32(), - "i64": pa.int64(), - "f32": pa.float32(), - "f64": pa.float64(), - "bool": pa.bool_(), - "datetime": pa.timestamp("us"), - "date": pa.date32(), -} - - -def spec_arrow_type(spec_type: str) -> tuple[pa.DataType, bool]: - """Return the Arrow type and nullability a specification type denotes. - - A trailing ``?`` marks a nullable value, at any nesting level. - - Raises: - ToolError: for a type this tool does not know. - """ - s = spec_type.strip().strip("`") - nullable = s.endswith("?") - if nullable: - s = s[:-1] - if s in _SPEC_PRIMITIVES: - return _SPEC_PRIMITIVES[s], nullable - if m := re.fullmatch(r"fixed_size_list\((.*)\)\[(\d+)\]", s): - inner, inner_nullable = spec_arrow_type(m.group(1)) - return pa.list_( - pa.field("item", inner, inner_nullable), int(m.group(2)) - ), nullable - if m := re.fullmatch(r"list\((.*)\)", s): - inner, inner_nullable = spec_arrow_type(m.group(1)) - return pa.list_(pa.field("item", inner, inner_nullable)), nullable - raise ToolError(f"unknown specification type {spec_type!r}") - - -def parse_spec_tables(text: str) -> dict[str, list[dict[str, Any]]]: - """Return the column tables of ``edi_schema_spec.md`` by table name. - - Each column is ``{"column", "type", "pk", "fk", "fk_parents", "group"}``; a - ``**... (`name` column group)**`` row starts a column group. - """ - tables: dict[str, list[dict[str, Any]]] = {} - current: str | None = None - group: str | None = None - for line in text.splitlines(): - if m := re.match(r"^####\s+(\w+)\s*$", line): - current, group = m.group(1), None - tables[m.group(1)] = [] - continue - if current is None or not line.startswith("|"): - continue - cells = [c.strip() for c in line.strip().strip("|").split("|")] - if len(cells) < 3 or cells[0] == "Column" or set(cells[0]) <= set("-: "): - continue - if m := re.search(r"`(\w+)` column group", cells[0]): - group = m.group(1) - continue - desc = cells[3] if len(cells) > 3 else "" - tables[current].append( - { - "column": cells[0].strip("`"), - "type": cells[1], - "pk": "PK" in cells[2], - "fk": "FK" in cells[2], - "fk_parents": re.findall(r"FK →\s*`(\w+)", desc), - "group": group, - } - ) - return tables - - -# References the specification does not declare as foreign keys, checked because the -# subset keeps whole trials and units. Each is ``child: [(parent, note)]``. -EXTRA_REFERENCES: dict[str, list[tuple[str, str]]] = { - "experiment_freeviewing_event": [ - ("experiment_freeviewing_trial", "trial_index; not an FK in the spec") - ], - "experiment_screen_capture_event": [ - ("experiment_freeviewing_trial", "trial_index; not an FK in the spec"), - ( - "experiment", - "the spec's parent experiment_screen_capture is not in the sample", - ), - ], - "experiment_screen_capture_frame_270p": [("experiment", "table not in the spec")], - "sample_unit": [("unit", "table not in the spec")], -} - - -def apply_decisions( - raw: Mapping[str, Any], decisions: Mapping[str, Any] -) -> dict[str, Any]: - """Return a copy of the map's JSON form with the developer's decisions applied. - - ``decisions`` holds: - - * ``tables_not_in_spec``: ``{"keys": "required" | "nullable", "values": ...}``, - applied to every pending column of a table the specification does not define; - * ``column_group_structs``: ``"required"`` or ``"nullable"``, for every pending - column group struct; - * ``accepted_unresolved``: ``[{"child", "parent", "keys", "evidence"}]``, key - tuples known not to resolve in the source. - - Raises: - ToolError: if an accepted reference names a foreign key the map lacks, or a - value is neither ``"required"`` nor ``"nullable"``. - """ - out = json.loads(json.dumps(raw)) - - def nullable(choice: str) -> bool: - """Return whether ``choice`` means nullable. - - Raises: - ToolError: if ``choice`` is neither ``"required"`` nor ``"nullable"``. - """ - if choice not in ("required", "nullable"): - raise ToolError(f"expected 'required' or 'nullable', got {choice!r}") - return choice == "nullable" - - outside = decisions.get("tables_not_in_spec") - groups = decisions.get("column_group_structs") - for table in out["tables"].values(): - if outside is not None and not table["in_spec"]: - for c in table["columns"].values(): - if c["nullable"] is None: - c["nullable"] = nullable(outside["keys" if c["key"] else "values"]) - if groups is not None: - for g in table["groups"].values(): - if g["nullable"] is None: - g["nullable"] = nullable(groups) - for item in decisions.get("accepted_unresolved", []): - fks = out["tables"][item["child"]]["foreign_keys"] - match = [fk for fk in fks if fk["parent"] == item["parent"]] - if not match: - raise ToolError(f"{item['child']} has no foreign key to {item['parent']}") - match[0]["accepted_unresolved"] = item["keys"] - out["decisions"] = dict(decisions) - return out - - -def derive_map( - spec_text: str, - spec_version: str, - source_project: str, - source_schemas: Mapping[str, tuple[pa.Schema, Sequence[str]]], - decisions: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Build the JSON form of the nullability map. - - Args: - spec_text: The text of ``edi_schema_spec.md``. - spec_version: The specification's version, recorded in the map. - source_project: The source project's ID, recorded in the map. - source_schemas: Per table, its Arrow schema and key columns as SpiralDB - reports them. - - Returns: - The map. A column the specification defines gets the specification's - nullability; a column or table it does not define gets ``nullable: null`` - (pending), as does each column group struct (the specification types only - the group's fields). Specification types that disagree with the source's - are listed under ``mismatches``. ``decisions``, if given, resolves pending - entries (see ``apply_decisions``). - """ - spec = parse_spec_tables(spec_text) - out: dict[str, Any] = { - "spec_version": spec_version, - "source_project": source_project, - "mismatches": [], - "absent_from_sample": sorted(t for t in spec if t not in source_schemas), - "tables": {}, - } - for table, (schema, keys) in sorted(source_schemas.items()): - rows = {r["column"]: r for r in spec.get(table, [])} - in_spec = table in spec - entry: dict[str, Any] = { - "in_spec": in_spec, - "keys": list(keys), - "columns": {}, - "groups": {}, - } - - def column( - field: pa.Field[Any], row: dict[str, Any] | None, *, is_key: bool - ) -> dict[str, Any]: - """Return the map entry for ``field``, recording any spec difference.""" - if row is None: - if in_spec: - out["mismatches"].append(f"{table}.{field.name}: not in the spec") - return { - "arrow_type": str(field.type), - "nullable": None, - "spec_type": None, - "key": is_key, - } - want, nullable = spec_arrow_type(row["type"]) - if want != field.type: - out["mismatches"].append( - f"{table}.{field.name}: spec {row['type']} is {want}, source is {field.type}" - ) - if row["pk"] != is_key: - out["mismatches"].append( - f"{table}.{field.name}: spec key={row['pk']}, source key={is_key}" - ) - return { - "arrow_type": str(field.type), - "nullable": nullable, - "spec_type": row["type"], - "key": is_key, - } - - for field in schema: - if pa.types.is_struct(field.type) and field.name not in rows: - entry["groups"][field.name] = { - "arrow_type": str(field.type), - "nullable": None, - "fields": { - child.name: column(child, rows.get(child.name), is_key=False) - for child in field.type - }, - } - else: - entry["columns"][field.name] = column( - field, rows.get(field.name), is_key=field.name in keys - ) - if in_spec: - spec_keys = [r["column"] for r in spec[table] if r["pk"]] - if spec_keys != list(keys): - out["mismatches"].append( - f"{table}: spec key {spec_keys}, source key {list(keys)}" - ) - listed = set(entry["columns"]) | { - f for g in entry["groups"].values() for f in g["fields"] - } - for name in rows: - if name not in listed: - out["mismatches"].append( - f"{table}.{name}: in the spec, not in the source" - ) - parents: list[tuple[str, str]] = [] - for r in spec.get(table, []): - parents.extend((p, "") for p in r["fk_parents"] if p != table) - parents.extend(EXTRA_REFERENCES.get(table, [])) - fks = [] - seen: set[str] = set() - for parent, note in parents: - if parent in seen: - continue - seen.add(parent) - if parent not in source_schemas: - fks.append( - { - "parent": parent, - "columns": [], - "note": "parent table not in the sample; not checked", - } - ) - continue - pkeys = list(source_schemas[parent][1]) - if not set(pkeys) <= set(schema.names): - out["mismatches"].append( - f"{table} -> {parent}: child lacks parent key columns {sorted(set(pkeys) - set(schema.names))}" - ) - continue - fks.append({"parent": parent, "columns": pkeys, "note": note}) - entry["foreign_keys"] = [fk for fk in fks if fk["columns"]] - entry["unchecked_references"] = [fk for fk in fks if not fk["columns"]] - out["tables"][table] = entry - return apply_decisions(out, decisions) if decisions is not None else out - - -def render_map(nmap: NullabilityMap) -> str: - """Return the map as Markdown tables, one row per column.""" - lines = [] - for name, t in sorted(nmap.tables.items()): - lines.append( - f"### `{name}`{'' if t.in_spec else ' (not in the specification)'}\n" - ) - lines.append("| Column | Arrow type | Spec type | Key | Declared |") - lines.append("| --- | --- | --- | --- | --- |") - - def row(col: str, c: ColumnSpec) -> str: - """Return the Markdown table row for column ``col``.""" - declared = ( - "pending" - if c.nullable is None - else ("nullable" if c.nullable else "**required**") - ) - return f"| `{col}` | `{c.arrow_type}` | {f'`{c.spec_type}`' if c.spec_type else '—'} | {'key' if c.key else ''} | {declared} |" - - for col, c in t.columns.items(): - lines.append(row(col, c)) - for g, group in t.groups.items(): - declared = ( - "pending" - if group.nullable is None - else ("nullable" if group.nullable else "**required**") - ) - lines.append(f"| `{g}` (column group) | struct | — | | {declared} |") - for col, c in group.fields.items(): - lines.append(row(f"{g}.{col}", c)) - if t.foreign_keys: - lines.append("") - lines.extend( - f"References `{fk.parent}` on {', '.join(f'`{c}`' for c in fk.columns)}" - + (f" ({fk.note})" if fk.note else "") - + "." - for fk in t.foreign_keys - ) - lines.append("") - return "\n".join(lines) - - -# ---------------------------------------------------------------------------- CLI - - -def connect() -> Any: - """Return a ``spiral.Spiral`` client, after checking the pyspiral version. - - Raises: - ToolError: if pyspiral is missing or older than 0.16.33. - """ - try: - from importlib.metadata import version - - import spiral # pyright: ignore[reportMissingImports] # optional; see connect() - except ImportError as err: - raise ToolError( - "pyspiral is not installed; run with `uv run --with pyspiral==0.16.33 ...`" - ) from err - found = tuple(int(x) for x in re.findall(r"\d+", version("pyspiral"))[:3]) - if found < MIN_PYSPIRAL: - raise ToolError( - f"pyspiral {version('pyspiral')} is too old: 0.16.33 or newer uses the " - "`spiral login` credentials" - ) - return spiral.Spiral() - - -def _source_schemas( - client: Any, project_id: str, dataset: str -) -> dict[str, tuple[pa.Schema, list[str]]]: - """Return the schema and key column names of each table in ``dataset``. - - Args: - client: The Spiral client. - project_id: The project to read. - dataset: The dataset whose tables are returned. - - Returns: - Table name (without the dataset) to ``(schema, key column names)``. - """ - project = client.project(project_id) - out = {} - for r in project.list_tables(): - if r.dataset != dataset: - continue - t = project.table(f"{dataset}.{r.table}") - out[r.table] = (t.schema().to_arrow(), list(t.key_schema.to_arrow().names)) - return out - - -def _parser() -> argparse.ArgumentParser: - """Build the command-line parser and its subcommands.""" - p = argparse.ArgumentParser( - prog="edi_tutorial_dataset.py", - description="Build the workshop Spiral project from the V1 EDI sample.", - ) - p.add_argument("--map", type=Path, default=MAP_PATH, help="nullability map JSON") - p.add_argument("--subset", type=Path, default=SUBSET_PATH, help="subset JSON") - p.add_argument("--source-project", default=DEFAULT_SOURCE) - sub = p.add_subparsers(dest="command", required=True) - - sub.add_parser("plan", help="read-only dry run of the copy") - - c = sub.add_parser("create", help="create the target project and its marker table") - c.add_argument("--id-prefix", required=True) - c.add_argument("--allow-create", required=True, help="repeat --id-prefix") - c.add_argument("--description", required=True) - c.add_argument("--volume", default=None, help="Spiral Volume ID for the new tables") - - for name, helptext in ( - ("probe", "check how SpiralDB treats non-nullable columns (scratch table)"), - ("copy", "copy the subset into the target"), - ): - s = sub.add_parser(name, help=helptext) - s.add_argument("--target-project", required=True) - s.add_argument("--allow-writes", required=True, help="repeat --target-project") - if name == "copy": - s.add_argument("--dry-run", action="store_true") - s.add_argument( - "--tables", default=None, help="comma-separated: copy only these tables" - ) - s.add_argument( - "--replace", - action="store_true", - help="drop the --tables and copy them again from the source", - ) - - a = sub.add_parser( - "annotate", - help="give each table Orcapod's nullability record (ITL-936; metadata only)", - ) - a.add_argument("--target-project", required=True) - a.add_argument("--allow-writes", required=True, help="repeat --target-project") - a.add_argument( - "--dry-run", action="store_true", help="list the paths, write nothing" - ) - - v = sub.add_parser("verify", help="check the target against the source and the map") - v.add_argument("--target-project", required=True) - - o = sub.add_parser( - "orcapod-check", help="load each target table as an Orcapod source (read-only)" - ) - o.add_argument("--target-project", required=True) - - d = sub.add_parser("derive-map", help="build the nullability map from the spec") - d.add_argument("--spec", type=Path, required=True, help="edi_schema_spec.md") - d.add_argument("--spec-version", required=True) - d.add_argument("--output", type=Path, required=True) - d.add_argument( - "--decisions", - type=Path, - default=DECISIONS_PATH, - help="the developer's decisions (default: decisions.json)", - ) - - sub.add_parser("render-map", help="print the map as Markdown") - return p - - -def main( - argv: Sequence[str] | None = None, client_factory: Callable[[], Any] = connect -) -> int: - """Run the command line; return the process exit code.""" - args = _parser().parse_args(argv) - try: - if args.command == "render-map": - print(render_map(load_map(args.map))) - return 0 - client = client_factory() - if args.command == "derive-map": - subset = load_subset(args.subset) - raw = derive_map( - args.spec.read_text(encoding="utf-8"), - args.spec_version, - args.source_project, - _source_schemas(client, args.source_project, subset.dataset), - json.loads(args.decisions.read_text(encoding="utf-8")), - ) - args.output.write_text(json.dumps(raw, indent=2) + "\n", encoding="utf-8") - print(f"wrote {args.output}; {len(raw['mismatches'])} mismatch(es)") - return 0 - if args.command == "create": - pid = create_project( - client, - id_prefix=args.id_prefix, - allow_create=args.allow_create, - description=args.description, - volume_id=args.volume, - source_project=args.source_project, - ) - print(f"created project {pid}") - return 0 - if args.command == "probe": - result = probe_nullability( - client, - target_project=args.target_project, - allow_writes=args.allow_writes, - source_project=args.source_project, - ) - print(json.dumps(result, indent=2, default=str)) - return 0 - nmap, subset = load_map(args.map), load_subset(args.subset) - if args.command == "annotate": - print( - "\n".join( - annotate( - client, - source_project=args.source_project, - target_project=args.target_project, - allow_writes=args.allow_writes, - nmap=nmap, - subset=subset, - write_record=None if args.dry_run else load_record_writer(), - dry_run=args.dry_run, - ) - ) - ) - return 0 - if args.command == "orcapod-check": - from orcapod.sources import SpiralDBTableSource - - lines, problems = orcapod_check( - args.target_project, - nmap, - subset, - lambda project, table, dataset: SpiralDBTableSource( - project, table, dataset=dataset - ), - ) - print("\n".join(lines)) - if problems: - print("problems:\n " + "\n ".join(problems)) - return 1 if problems else 0 - if args.command == "plan": - report = plan(client, args.source_project, nmap, subset) - elif args.command == "copy": - report = copy( - client, - source_project=args.source_project, - target_project=args.target_project, - allow_writes=args.allow_writes, - nmap=nmap, - subset=subset, - dry_run=args.dry_run, - tables=args.tables.split(",") if args.tables else None, - replace=args.replace, - on_table=lambda r: print( - f"{r.action:12s} {r.name} ({r.rows:,d} rows)", flush=True - ), - ) - else: - report = verify( - client, - source_project=args.source_project, - target_project=args.target_project, - nmap=nmap, - subset=subset, - read_record=load_record_reader(), - ) - print(report.render()) - return 1 if report.problems else 0 - except ToolError as err: - print(f"error: {err}", file=sys.stderr) - for note in getattr(err, "__notes__", []): - print(f" {note}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/workshop/edi_tutorial_dataset/nullability_map.json b/workshop/edi_tutorial_dataset/nullability_map.json deleted file mode 100644 index 10075e40f..000000000 --- a/workshop/edi_tutorial_dataset/nullability_map.json +++ /dev/null @@ -1,2955 +0,0 @@ -{ - "spec_version": "0.3.0", - "source_project": "edi-sample-v1-855632", - "mismatches": [], - "absent_from_sample": [ - "atlas_info", - "atlas_preference_history", - "experiment_dotmapping_event", - "experiment_dotmapping_params", - "experiment_dotmapping_trial", - "experiment_gaze_timeseries", - "experiment_hardware", - "experiment_preprocessing_run", - "experiment_screen_capture", - "experiment_sync_event", - "eye_tracker_hardware", - "ingestion_run", - "lfp_extraction_run", - "localization_run", - "mri_dataset", - "neural_hardware", - "spike_extraction_run" - ], - "tables": { - "atlas": { - "in_spec": true, - "keys": [ - "atlas_id" - ], - "columns": { - "atlas_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_description": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "atlas_name": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": false - }, - "atlas_version": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "source_url": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [], - "unchecked_references": [] - }, - "atlas_label": { - "in_spec": true, - "keys": [ - "atlas_id", - "atlas_label_region_short" - ], - "columns": { - "atlas_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_short": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_full": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "atlas_label_region_group": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "atlas_label_region_index": { - "arrow_type": "int64", - "nullable": true, - "spec_type": "i64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "atlas", - "columns": [ - "atlas_id" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "chamber": { - "in_spec": true, - "keys": [ - "subject_id", - "chamber_id" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "chamber_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "chamber_implant_date": { - "arrow_type": "date32[day]", - "nullable": true, - "spec_type": "date?", - "key": false - }, - "chamber_location": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "subject", - "columns": [ - "subject_id" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "experiment": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "correct_trials": { - "arrow_type": "int64", - "nullable": true, - "spec_type": "i64?", - "key": false - }, - "experiment_start_timestamp": { - "arrow_type": "uint64", - "nullable": true, - "spec_type": "u64?", - "key": false - }, - "experiment_stop_timestamp": { - "arrow_type": "uint64", - "nullable": true, - "spec_type": "u64?", - "key": false - }, - "incorrect_trials": { - "arrow_type": "int64", - "nullable": true, - "spec_type": "i64?", - "key": false - }, - "task_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": false - }, - "task_version": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": false - }, - "total_trials": { - "arrow_type": "int64", - "nullable": true, - "spec_type": "i64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "session", - "columns": [ - "subject_id", - "session_start_datetime" - ], - "note": "" - }, - { - "parent": "task", - "columns": [ - "task_id", - "task_version" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "experiment_preprocessing_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "experiment_display_params": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "monitor_bg_color": { - "arrow_type": "fixed_size_list[3]", - "nullable": true, - "spec_type": "fixed_size_list(u8?)[3]?", - "key": false - }, - "monitor_center_px": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(i16?)[2]?", - "key": false - }, - "monitor_curve_radius_mm": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[2]?", - "key": false - }, - "monitor_distance_mm": { - "arrow_type": "int16", - "nullable": true, - "spec_type": "i16?", - "key": false - }, - "monitor_gamma_table": { - "arrow_type": "fixed_size_list[256]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[256]?", - "key": false - }, - "monitor_refresh_rate_hz": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "monitor_resolution_px": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(u16?)[2]?", - "key": false - }, - "monitor_size_mm": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[2]?", - "key": false - }, - "monitor_type": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "experiment_freeviewing_event": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "timestamp", - "name" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": true - }, - "name": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "trial_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "" - }, - { - "parent": "experiment_freeviewing_trial", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "trial_index" - ], - "note": "trial_index; not an FK in the spec" - } - ], - "unchecked_references": [] - }, - "experiment_freeviewing_params": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "blank_duration_ms": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "fix_acquire_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "fix_hold_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "fix_spot_color": { - "arrow_type": "fixed_size_list[3]", - "nullable": true, - "spec_type": "fixed_size_list(u8?)[3]?", - "key": false - }, - "fix_spot_location_px": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(i16?)[2]?", - "key": false - }, - "fix_spot_ring_thickness_px": { - "arrow_type": "int16", - "nullable": true, - "spec_type": "i16?", - "key": false - }, - "fix_spot_size_px": { - "arrow_type": "int16", - "nullable": true, - "spec_type": "i16?", - "key": false - }, - "movie_dir": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "post_stimulus_duration_ms": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "stimulus_height_px": { - "arrow_type": "int16", - "nullable": true, - "spec_type": "i16?", - "key": false - }, - "stimulus_location_px": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(i16?)[2]?", - "key": false - }, - "stimulus_selection_hash": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "stimulus_selection_type": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "stimulus_width_px": { - "arrow_type": "int16", - "nullable": true, - "spec_type": "i16?", - "key": false - }, - "use_native_frame_rate": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "experiment_freeviewing_trial": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "trial_index" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "trial_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "block_frame_num": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "correct_response": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "delay_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "delayed_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "dest_rect_px": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(i32?)?", - "key": false - }, - "dropped_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "dropped_prestim_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "eye_params": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "fixation_radius_px": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "free": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "has_fixation_nidaq_time": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "intertrial_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "pause": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "played_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "played_prestim_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "pre_stimulus_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "response_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "reward_amount_ml": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "stim_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "test_block_frame_num": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "test_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "total_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "total_prestim_frames": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "trial_end_timestamp": { - "arrow_type": "uint64", - "nullable": true, - "spec_type": "u64?", - "key": false - }, - "trial_id": { - "arrow_type": "int32", - "nullable": true, - "spec_type": "i32?", - "key": false - }, - "trial_start_timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": false - }, - "trial_type": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "valid_trial": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "video_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "videos_end_frame": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(i32?)?", - "key": false - }, - "videos_fps": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "videos_frame_rate_hz": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "videos_hash": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(utf8?)?", - "key": false - }, - "videos_n_frames": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(i32?)?", - "key": false - }, - "videos_name": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(utf8?)?", - "key": false - }, - "videos_start_frame": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(i32?)?", - "key": false - }, - "videos_trial_type": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(utf8?)?", - "key": false - }, - "videos_video_path": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(utf8?)?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "experiment_gaze_online_timeseries": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "timestamp" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": true - }, - "eye_tracking_software": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "eye_tracking_software_version": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "eye_x_deg": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "eye_x_px": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "eye_y_deg": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "eye_y_px": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "gaze_unreliable": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "pupil_size_norm": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "experiment_screen_capture_event": { - "in_spec": true, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "timestamp", - "name" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": true - }, - "name": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "collection": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "displayed_movie": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "displayed_movie_frame_number": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "frame_bg_color": { - "arrow_type": "fixed_size_list[3]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[3]?", - "key": false - }, - "frame_display": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "frame_fix_spot_color": { - "arrow_type": "fixed_size_list[3]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[3]?", - "key": false - }, - "frame_fix_spot_location_px": { - "arrow_type": "fixed_size_list[2]", - "nullable": true, - "spec_type": "fixed_size_list(f32?)[2]?", - "key": false - }, - "frame_fix_spot_ring_thickness_px": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "frame_fix_spot_size_px": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "frame_index": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "screen_capture_fps": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "source_movie": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "source_movie_frame_number": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "stimulus_type": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "trial_index": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment_freeviewing_trial", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "trial_index" - ], - "note": "trial_index; not an FK in the spec" - }, - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "the spec's parent experiment_screen_capture is not in the sample" - } - ], - "unchecked_references": [ - { - "parent": "experiment_screen_capture", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "experiment_screen_capture_frame_270p": { - "in_spec": false, - "keys": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index", - "screen_capture_import_version_id", - "timestamp", - "frame_index" - ], - "columns": { - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": null, - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": null, - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": null, - "key": true - }, - "experiment_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": null, - "key": true - }, - "screen_capture_import_version_id": { - "arrow_type": "int64", - "nullable": false, - "spec_type": null, - "key": true - }, - "timestamp": { - "arrow_type": "int64", - "nullable": false, - "spec_type": null, - "key": true - }, - "frame_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": null, - "key": true - }, - "frame_jpeg": { - "arrow_type": "large_binary", - "nullable": true, - "spec_type": null, - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "experiment", - "columns": [ - "experiment_preprocessing_hash", - "subject_id", - "session_start_datetime", - "experiment_index" - ], - "note": "table not in the spec" - } - ], - "unchecked_references": [] - }, - "lfp_acquisition": { - "in_spec": true, - "keys": [ - "lfp_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "columns": { - "lfp_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "lfp_channel_ids": { - "arrow_type": "list", - "nullable": false, - "spec_type": "list(utf8?)", - "key": false - }, - "lfp_n_channels": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "lfp_n_samples": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "lfp_sampling_frequency_hz": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "recording_start_timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "penetration", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "lfp_extraction_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "lfp_location": { - "in_spec": true, - "keys": [ - "lfp_extraction_hash", - "localization_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "probe_channel_id", - "atlas_id", - "atlas_label_region_short" - ], - "columns": { - "lfp_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "localization_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_channel_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_short": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_full": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "chamber_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "chamber_x_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "chamber_y_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "chamber_z_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "lfp_acquisition", - "columns": [ - "lfp_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - }, - { - "parent": "atlas", - "columns": [ - "atlas_id" - ], - "note": "" - }, - { - "parent": "atlas_label", - "columns": [ - "atlas_id", - "atlas_label_region_short" - ], - "note": "" - }, - { - "parent": "chamber", - "columns": [ - "subject_id", - "chamber_id" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "localization_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "lfp_timeseries": { - "in_spec": true, - "keys": [ - "lfp_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "timestamp" - ], - "columns": { - "lfp_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": true - }, - "lfp_voltage_uv": { - "arrow_type": "fixed_size_list[384]", - "nullable": false, - "spec_type": "fixed_size_list(f32?)[384]", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "penetration", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "lfp_extraction_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "penetration": { - "in_spec": true, - "keys": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "chamber_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "chassis_index": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "port_index": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "probe_channel_config": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "probe_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": false - }, - "probe_nominal_chamber_x_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "probe_nominal_chamber_y_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "probe_nominal_chamber_z_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "probe_plan_chamber_x_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "probe_plan_chamber_y_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "probe_plan_chamber_z_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "slot_index": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "recording", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index" - ], - "note": "" - }, - { - "parent": "probe", - "columns": [ - "probe_id" - ], - "note": "" - }, - { - "parent": "chamber", - "columns": [ - "subject_id", - "chamber_id" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "probe": { - "in_spec": true, - "keys": [ - "probe_id" - ], - "columns": { - "probe_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "probe_model": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "probe_name": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "serial_number": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [], - "unchecked_references": [] - }, - "probe_location": { - "in_spec": true, - "keys": [ - "localization_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "atlas_id" - ], - "columns": { - "localization_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "atlas_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "chamber_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "tip_chamber_x_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "tip_chamber_y_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "tip_chamber_z_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "penetration", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - }, - { - "parent": "atlas", - "columns": [ - "atlas_id" - ], - "note": "" - }, - { - "parent": "chamber", - "columns": [ - "subject_id", - "chamber_id" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "localization_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "probe_map": { - "in_spec": true, - "keys": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "channel_acquisition_index": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(u32?)?", - "key": false - }, - "probe_channel_ids": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(utf8?)?", - "key": false - }, - "sites_physical": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(bool?)?", - "key": false - }, - "sites_shank_ids": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(u8?)?", - "key": false - }, - "sites_x_um": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f64?)?", - "key": false - }, - "sites_y_um": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f64?)?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "penetration", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "recording": { - "in_spec": true, - "keys": [ - "subject_id", - "session_start_datetime", - "recording_index" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "neural_acquisition_software": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "neural_acquisition_software_version": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "neural_hardware_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "recording_end_timestamp": { - "arrow_type": "uint64", - "nullable": true, - "spec_type": "u64?", - "key": false - }, - "recording_start_timestamp": { - "arrow_type": "uint64", - "nullable": true, - "spec_type": "u64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "session", - "columns": [ - "subject_id", - "session_start_datetime" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "neural_hardware", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "sample_info": { - "in_spec": false, - "keys": [ - "region" - ], - "columns": { - "region": { - "arrow_type": "string", - "nullable": false, - "spec_type": null, - "key": true - }, - "built_at_utc": { - "arrow_type": "timestamp[us]", - "nullable": true, - "spec_type": null, - "key": false - }, - "dataset": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "experiment_index": { - "arrow_type": "int64", - "nullable": true, - "spec_type": null, - "key": false - }, - "experiment_preprocessing_hash": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "git_sha": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "n_units": { - "arrow_type": "int64", - "nullable": true, - "spec_type": null, - "key": false - }, - "probe_index": { - "arrow_type": "int64", - "nullable": true, - "spec_type": null, - "key": false - }, - "recording_index": { - "arrow_type": "int64", - "nullable": true, - "spec_type": null, - "key": false - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": true, - "spec_type": null, - "key": false - }, - "sources_json": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "spike_extraction_hash": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "subject_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "unit_filter": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - } - }, - "groups": {}, - "foreign_keys": [], - "unchecked_references": [] - }, - "sample_unit": { - "in_spec": false, - "keys": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id" - ], - "columns": { - "spike_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": null, - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": null, - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": null, - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": null, - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": null, - "key": true - }, - "unit_id": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": null, - "key": true - }, - "atlas_label_region_short": { - "arrow_type": "string", - "nullable": true, - "spec_type": null, - "key": false - }, - "fano_factor_detrended": { - "arrow_type": "double", - "nullable": true, - "spec_type": null, - "key": false - }, - "firing_rate_hz": { - "arrow_type": "double", - "nullable": true, - "spec_type": null, - "key": false - }, - "presence_ratio": { - "arrow_type": "double", - "nullable": true, - "spec_type": null, - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "unit", - "columns": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id" - ], - "note": "table not in the spec" - } - ], - "unchecked_references": [] - }, - "session": { - "in_spec": true, - "keys": [ - "subject_id", - "session_start_datetime" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "booth": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": false - }, - "booth_platform_code": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "session_constants": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "session_end_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": false - }, - "session_notes": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "subject", - "columns": [ - "subject_id" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "spike_event": { - "in_spec": true, - "keys": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "timestamp", - "unit_id" - ], - "columns": { - "spike_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "timestamp": { - "arrow_type": "uint64", - "nullable": false, - "spec_type": "u64", - "key": true - }, - "unit_id": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "spike_amplitude_uv": { - "arrow_type": "float", - "nullable": false, - "spec_type": "f32", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "unit", - "columns": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id" - ], - "note": "" - } - ], - "unchecked_references": [] - }, - "subject": { - "in_spec": true, - "keys": [ - "subject_id" - ], - "columns": { - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "date_of_birth": { - "arrow_type": "date32[day]", - "nullable": true, - "spec_type": "date?", - "key": false - }, - "sex": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "species": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [], - "unchecked_references": [] - }, - "task": { - "in_spec": true, - "keys": [ - "task_id", - "task_version" - ], - "columns": { - "task_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "task_version": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "task_code_commit": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "task_description": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "task_repository": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [], - "unchecked_references": [] - }, - "unit": { - "in_spec": true, - "keys": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id" - ], - "columns": { - "spike_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "unit_id": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - } - }, - "groups": { - "labels": { - "arrow_type": "struct", - "nullable": false, - "fields": { - "aind_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "bombcell_label": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "brian_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "kilosort_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "rigor_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "tian_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - }, - "unitrefine_label": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "unitrefine_monkey_label": { - "arrow_type": "bool", - "nullable": true, - "spec_type": "bool?", - "key": false - } - } - }, - "quality_metrics": { - "arrow_type": "struct", - "nullable": false, - "fields": { - "amplitude_cutoff": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "amplitude_cv_median": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "amplitude_cv_range": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "amplitude_median_uv": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "d_prime": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "drift_mad_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "drift_ptp_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "drift_std_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "firing_range_hz": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "firing_rate_hz": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "isi_violations_count": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "isi_violations_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "isolation_distance": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "l_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "max_waveform_channel_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "nn_hit_rate": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "nn_miss_rate": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "noise_cutoff": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "noise_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "num_spikes": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "presence_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "rp_contamination": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "rp_violations": { - "arrow_type": "uint32", - "nullable": true, - "spec_type": "u32?", - "key": false - }, - "sd_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "silhouette": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "sliding_rp_violation": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "snr": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "sync_spike_2": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "sync_spike_4": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "sync_spike_8": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "unit_depth_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - } - } - }, - "rolling_quality_metrics": { - "arrow_type": "struct, rolling_firing_rate_hz: list, rolling_isi_violations_ratio: list, rolling_nn_hit_rate: list, rolling_nn_miss_rate: list, rolling_quality_metrics_timestamps: list, rolling_snr: list>", - "nullable": false, - "fields": { - "rolling_amplitude_median_uv": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "rolling_firing_rate_hz": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "rolling_isi_violations_ratio": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "rolling_nn_hit_rate": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "rolling_nn_miss_rate": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "rolling_quality_metrics_timestamps": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(u64?)?", - "key": false - }, - "rolling_snr": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - } - } - }, - "waveform_metrics": { - "arrow_type": "struct, trough_half_width_s: float, waveform_baseline_flatness: float>", - "nullable": false, - "fields": { - "exp_decay_per_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "main_peak_to_trough_ratio": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "main_to_next_extremum_duration_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "num_negative_peaks": { - "arrow_type": "uint8", - "nullable": true, - "spec_type": "u8?", - "key": false - }, - "num_positive_peaks": { - "arrow_type": "uint8", - "nullable": true, - "spec_type": "u8?", - "key": false - }, - "peak_half_width_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "spread_um": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "template_waveform_uv": { - "arrow_type": "list", - "nullable": true, - "spec_type": "list(f32?)?", - "key": false - }, - "trough_half_width_s": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - }, - "waveform_baseline_flatness": { - "arrow_type": "float", - "nullable": true, - "spec_type": "f32?", - "key": false - } - } - } - }, - "foreign_keys": [ - { - "parent": "penetration", - "columns": [ - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "spike_extraction_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - }, - "unit_location": { - "in_spec": true, - "keys": [ - "spike_extraction_hash", - "localization_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id", - "atlas_id", - "atlas_label_region_short" - ], - "columns": { - "spike_extraction_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "localization_hash": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "subject_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "session_start_datetime": { - "arrow_type": "timestamp[us]", - "nullable": false, - "spec_type": "datetime", - "key": true - }, - "recording_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "probe_index": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "unit_id": { - "arrow_type": "uint32", - "nullable": false, - "spec_type": "u32", - "key": true - }, - "atlas_id": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_short": { - "arrow_type": "string", - "nullable": false, - "spec_type": "utf8", - "key": true - }, - "atlas_label_region_full": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "chamber_id": { - "arrow_type": "string", - "nullable": true, - "spec_type": "utf8?", - "key": false - }, - "chamber_x_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "chamber_y_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - }, - "chamber_z_um": { - "arrow_type": "double", - "nullable": true, - "spec_type": "f64?", - "key": false - } - }, - "groups": {}, - "foreign_keys": [ - { - "parent": "unit", - "columns": [ - "spike_extraction_hash", - "subject_id", - "session_start_datetime", - "recording_index", - "probe_index", - "unit_id" - ], - "note": "" - }, - { - "parent": "atlas", - "columns": [ - "atlas_id" - ], - "note": "" - }, - { - "parent": "atlas_label", - "columns": [ - "atlas_id", - "atlas_label_region_short" - ], - "note": "" - }, - { - "parent": "chamber", - "columns": [ - "subject_id", - "chamber_id" - ], - "note": "" - } - ], - "unchecked_references": [ - { - "parent": "localization_run", - "columns": [], - "note": "parent table not in the sample; not checked" - } - ] - } - }, - "decisions": { - "decided_by": "developer, November 11, 2026 (ITL-933)", - "tables_not_in_spec": { - "keys": "required", - "values": "nullable" - }, - "column_group_structs": "required", - "accepted_unresolved": [] - } -} diff --git a/workshop/edi_tutorial_dataset/subset.json b/workshop/edi_tutorial_dataset/subset.json deleted file mode 100644 index 2c90d9ada..000000000 --- a/workshop/edi_tutorial_dataset/subset.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "source_project": "edi-sample-v1-855632", - "dataset": "default", - "window_us": [ - 5226928854, - 5370814965 - ], - "unit_ids": [ - 77, - 92, - 100, - 103, - 105, - 106, - 108, - 109, - 110, - 112, - 113, - 114, - 119, - 123, - 127, - 131, - 135, - 139, - 140, - 148, - 156, - 158, - 159, - 160, - 161, - 162, - 163, - 164, - 167, - 169, - 173, - 175 - ], - "trial_table": "experiment_freeviewing_trial", - "notes": { - "window": "Half-open, in µs relative to session_start_datetime (one clock for every timestamp column): from trial_start_timestamp of trial 215 to trial_end_timestamp of trial 225 plus 1 µs, i.e. 143.9 s holding 11 consecutive valid trials. The LFP table (600 Hz) bounds the window: 100,000 rows is about 166 s.", - "units": "The 32 units with the most spikes inside the window among the units whose labels.bombcell_label is not 'noise'.", - "trials": "Trials overlapping the window, plus every trial a kept row references through trial_index.", - "sample_info": "Not copied (developer, November 11, 2026): it describes the whole sample (n_units 100, unit_filter, sources_json). copy records its row in the provenance metadata of meta.edi_tutorial_build instead." - }, - "tables": { - "atlas": { - "mode": "whole" - }, - "atlas_label": { - "mode": "whole" - }, - "chamber": { - "mode": "whole" - }, - "experiment": { - "mode": "whole" - }, - "experiment_display_params": { - "mode": "whole" - }, - "experiment_freeviewing_event": { - "mode": "window" - }, - "experiment_freeviewing_params": { - "mode": "whole" - }, - "experiment_freeviewing_trial": { - "mode": "trials" - }, - "experiment_gaze_online_timeseries": { - "mode": "window" - }, - "experiment_screen_capture_event": { - "mode": "window" - }, - "experiment_screen_capture_frame_270p": { - "mode": "window" - }, - "lfp_acquisition": { - "mode": "whole" - }, - "lfp_location": { - "mode": "whole" - }, - "lfp_timeseries": { - "mode": "window" - }, - "penetration": { - "mode": "whole" - }, - "probe": { - "mode": "whole" - }, - "probe_location": { - "mode": "whole" - }, - "probe_map": { - "mode": "whole" - }, - "recording": { - "mode": "whole" - }, - "sample_unit": { - "mode": "units" - }, - "session": { - "mode": "whole" - }, - "spike_event": { - "mode": "window_units" - }, - "subject": { - "mode": "whole" - }, - "task": { - "mode": "whole" - }, - "unit": { - "mode": "units" - }, - "unit_location": { - "mode": "units" - } - } -}