From 0601f55c91e0ff341b1d9a03e3c0b2279f130033 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 15 Jul 2026 14:56:31 +0200 Subject: [PATCH 1/8] feat: versioned DatasetStorage for reading/writing datasets across protocol versions Mirror the syft-job migration/storage architecture in syft-datasets: - DatasetStorage: single choke point for all dataset filesystem IO. Reads select a ProtocolCodec by on-disk protocol version and upgrade to the latest object version in memory; writes downgrade to the target protocol and the codec persists it. Nests a MigrationRegistry, peer_schemas, and codecs. - On-disk protocol version: new datasets can live under a v path segment (public/syft_datasets/v//). Protocol 0 = the last release (syft-client 0.1.117 / syft-dataset 0.1.20): flat layout, no v, no canonical_name/version fields; read + written back byte-compatibly. - Versioned models: Dataset / PrivateDatasetConfig are now MigratableObjects under models//v1.py. Object schemas are unchanged since 0.1.117, so only v1 exists; the protocol bump is purely path-layout + identity fields. - DatasetRef: path-derived identity (owner, name, protocol_version). - Broadcast write: a public dataset is one copy read by the whole audience, so the write protocol(s) come from the audience's versions (target_protocol_versions_for_peers), defaulting to the widest-compatible (oldest) protocol when no peers are known. peer_schemas is injectable but not yet populated by syft-client, so today datasets are written in protocol 0 and the v1 layout ships dormant. - SyftDatasetManager routes all reads/writes through DatasetStorage and supports multi-version write (one copy per protocol version in the audience); get_all dedupes copies across layouts preferring the newest protocol. - migrations/history + release scripts (export_release_artifact.py, generate_release_fixture.py); protocol-0/0.1.20 artifacts and fixtures derived as if that release had emitted them. - Tests mirror syft-job/tests/migrations (unit + p2p, incl. byte-exact cross-release round-trip and a multi-version-write test); CI + justfile wired. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/unit-tests.yml | 19 ++ .pre-commit-config.yaml | 2 +- Justfile | 4 + packages/syft-datasets/pyproject.toml | 3 +- .../scripts/export_release_artifact.py | 41 +++ .../scripts/generate_release_fixture.py | 126 ++++++++ .../src/syft_datasets/__init__.py | 26 +- .../syft-datasets/src/syft_datasets/config.py | 126 +++++++- .../src/syft_datasets/dataset.py | 252 +-------------- .../src/syft_datasets/dataset_manager.py | 296 +++++++++--------- .../src/syft_datasets/dataset_ref.py | 23 ++ .../src/syft_datasets/dataset_storage.py | 216 +++++++++++++ .../src/syft_datasets/migrations/__init__.py | 5 + .../src/syft_datasets/migrations/history.py | 32 ++ .../syft-dataset-0.1.20.json | 149 +++++++++ .../syft-dataset-0.1.21.json | 149 +++++++++ .../history/protocols/protocol-0.json | 144 +++++++++ .../history/protocols/protocol-1.json | 144 +++++++++ .../src/syft_datasets/migrations/registry.py | 22 ++ .../src/syft_datasets/models/__init__.py | 11 + .../syft_datasets/models/dataset/__init__.py | 9 + .../src/syft_datasets/models/dataset/v1.py | 214 +++++++++++++ .../models/private_dataset_config/__init__.py | 9 + .../models/private_dataset_config/v1.py | 24 ++ .../syft_datasets/protocolcodecs/__init__.py | 16 + .../src/syft_datasets/protocolcodecs/base.py | 53 ++++ .../src/syft_datasets/protocolcodecs/v0.py | 52 +++ .../src/syft_datasets/protocolcodecs/v1.py | 47 +++ .../src/syft_datasets/version.py | 10 + .../tests/migrations/__init__.py | 0 .../tests/migrations/p2p/__init__.py | 0 .../private/syft_datasets/demo/private.csv | 3 + .../syft_datasets/demo/private_metadata.yaml | 2 + .../private/syft_datasets/demo/syft.pub.yaml | 8 + .../public/syft_datasets/demo/dataset.yaml | 13 + .../public/syft_datasets/demo/mock.csv | 3 + .../public/syft_datasets/demo/readme.md | 1 + .../public/syft_datasets/demo/syft.pub.yaml | 8 + .../private/syft_datasets/v1/demo/private.csv | 3 + .../v1/demo/private_metadata.yaml | 4 + .../syft_datasets/v1/demo/syft.pub.yaml | 8 + .../public/syft_datasets/v1/demo/dataset.yaml | 15 + .../public/syft_datasets/v1/demo/mock.csv | 3 + .../public/syft_datasets/v1/demo/readme.md | 1 + .../syft_datasets/v1/demo/syft.pub.yaml | 8 + .../migrations/p2p/test_current_protocol.py | 109 +++++++ .../p2p/test_older_protocol_compatibility.py | 122 ++++++++ .../migrations/p2p/test_protocol_codecs.py | 143 +++++++++ .../p2p/test_protocol_sanity_checks.py | 63 ++++ .../tests/migrations/unit/__init__.py | 0 .../migrations/unit/fixtures/Dataset/v1.yaml | 15 + .../fixtures/PrivateDatasetConfig/v1.yaml | 4 + .../tests/migrations/unit/mocks.py | 35 +++ .../migrations/unit/test_history_artifacts.py | 100 ++++++ .../tests/migrations/unit/test_migrations.py | 72 +++++ .../migrations/unit/test_mock_migrations.py | 107 +++++++ .../unit/test_objects_registered.py | 58 ++++ .../migrations/unit/test_serialization.py | 40 +++ .../migrations/unit/test_upgrade_paths.py | 28 ++ uv.lock | 4 +- 60 files changed, 2789 insertions(+), 415 deletions(-) create mode 100644 packages/syft-datasets/scripts/export_release_artifact.py create mode 100644 packages/syft-datasets/scripts/generate_release_fixture.py create mode 100644 packages/syft-datasets/src/syft_datasets/dataset_ref.py create mode 100644 packages/syft-datasets/src/syft_datasets/dataset_storage.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history.py create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json create mode 100644 packages/syft-datasets/src/syft_datasets/migrations/registry.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/dataset/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py create mode 100644 packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py create mode 100644 packages/syft-datasets/src/syft_datasets/version.py create mode 100644 packages/syft-datasets/tests/migrations/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md create mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py create mode 100644 packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py create mode 100644 packages/syft-datasets/tests/migrations/unit/__init__.py create mode 100644 packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml create mode 100644 packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml create mode 100644 packages/syft-datasets/tests/migrations/unit/mocks.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_migrations.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_objects_registered.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_serialization.py create mode 100644 packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 2f0a61d816a..bce442931be 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -111,6 +111,25 @@ jobs: - name: Run job tests run: uv run pytest packages/syft-job/tests/ -v + dataset-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install the project + run: uv sync --all-extras + + - name: Run dataset tests + run: uv run pytest packages/syft-datasets/tests/ -v + enclave-tests: runs-on: ubuntu-latest steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6522797eb1b..5519bc46191 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,4 +47,4 @@ repos: rev: 'v3.0.0-alpha.9-for-vscode' hooks: - id: prettier - exclude: (\.min\.(js|css)$|^ci/|^test_outputs/) + exclude: (\.min\.(js|css)$|^ci/|^test_outputs/|^packages/syft-datasets/tests/migrations/(p2p|unit)/fixtures/) diff --git a/Justfile b/Justfile index 132ada82358..74fcb344912 100644 --- a/Justfile +++ b/Justfile @@ -30,6 +30,10 @@ test-unit-job: #!/bin/bash uv run pytest -v ./packages/syft-job/tests +test-unit-datasets: + #!/bin/bash + uv run pytest -n auto ./packages/syft-datasets/tests + test-unit-migration: #!/bin/bash uv run pytest -n auto ./packages/syft-migration/tests diff --git a/packages/syft-datasets/pyproject.toml b/packages/syft-datasets/pyproject.toml index 03af882f014..8ed83693736 100644 --- a/packages/syft-datasets/pyproject.toml +++ b/packages/syft-datasets/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "syft-dataset" -version = "0.1.20" +version = "0.1.21" description = "Syft Datasets" authors = [{ name = "OpenMined", email = "info@openmined.org" }] license = { text = "Apache-2.0" } @@ -10,6 +10,7 @@ dependencies = [ "pyyaml>=6.0.3", "syft-notebook-ui", "syft-perms==0.1.14", + "syft-migration", ] diff --git a/packages/syft-datasets/scripts/export_release_artifact.py b/packages/syft-datasets/scripts/export_release_artifact.py new file mode 100644 index 00000000000..f6a97db3bbd --- /dev/null +++ b/packages/syft-datasets/scripts/export_release_artifact.py @@ -0,0 +1,41 @@ +"""Export the release artifacts for the current syft-dataset version. + +Run on EVERY release (uv run python scripts/export_release_artifact.py): +always writes the package release info; additionally writes the protocol +artifact when this release introduces a new protocol version. +""" + +import sys + +from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION, dataset_registry +from syft_datasets.version import __version__ + + +def main() -> None: + # Import the models so every versioned object is registered. + import syft_datasets # noqa: F401 + + if dataset_registry.protocol_changed_without_bump(): + sys.exit( + "The dataset protocol changed compared to the released " + f"protocol-{DATASET_PROTOCOL_VERSION}.json; bump " + "DATASET_PROTOCOL_VERSION in syft_datasets/migrations/registry.py " + "before releasing." + ) + + PACKAGE_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + PROTOCOLS_DIR.mkdir(parents=True, exist_ok=True) + + info_path = PACKAGE_ARTIFACTS_DIR / f"syft-dataset-{__version__}.json" + dataset_registry.compute_released_package_protocol_info().save(info_path) + print(f"Wrote {info_path}") + + protocol_path = PROTOCOLS_DIR / f"protocol-{DATASET_PROTOCOL_VERSION}.json" + if not protocol_path.exists(): + dataset_registry.compute_released_protocol().save(protocol_path) + print(f"Wrote {protocol_path} (new protocol version)") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-datasets/scripts/generate_release_fixture.py b/packages/syft-datasets/scripts/generate_release_fixture.py new file mode 100644 index 00000000000..594fda59c01 --- /dev/null +++ b/packages/syft-datasets/scripts/generate_release_fixture.py @@ -0,0 +1,126 @@ +"""Generate a p2p backward-compatibility fixture for a syft-dataset release. + +Run on EVERY release, after bumping the version: + + uv run python scripts/generate_release_fixture.py + +Writes a SyftBox tree exactly as this release serializes datasets to disk, into + + tests/migrations/p2p/fixtures/syft_dataset--protocol

_syftbox/ + +Future releases loop over these fixtures (test_older_protocol_compatibility.py) +to prove they can still read and round-trip older on-disk data. + +Protocol 0 / release 0.1.20 predates this script; its fixture +(syft_dataset-0.1.20-protocol0_syftbox) is generated by passing +``--protocol 0 --version 0.1.20`` and is treated as hand-authored, like +protocol-0.json. +""" + +import argparse +import shutil +import tempfile +from pathlib import Path + +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_datasets.version import __version__ + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + +FIXTURES_DIR = ( + Path(__file__).resolve().parents[1] / "tests" / "migrations" / "p2p" / "fixtures" +) + + +def _seed_syftbox(syftbox: Path, src_dir: Path, protocol_version: str) -> None: + """One dataset written in the given protocol layout (mock + private + readme).""" + mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + # Force the target protocol by advertising a peer at that version. + mgr.storage.peer_schemas = { + DS_EMAIL: dataset_registry.schema_for_protocol_version(protocol_version) + } + + mock_path = src_dir / "mock.csv" + mock_path.write_text("id,value\n1,10\n2,20\n") + private_path = src_dir / "private.csv" + private_path.write_text("id,secret\n1,alpha\n2,beta\n") + readme_path = src_dir / "readme.md" + readme_path.write_text("# demo dataset\n") + + mgr.create( + name="demo", + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + summary="demo dataset", + tags=["demo"], + users=[DS_EMAIL], + ) + + +def _normalize(target: Path) -> None: + """Strip machine-specific values so the committed fixture is portable. + + Drop the executable bit (fixtures are data, not scripts) and blank the + private config's absolute ``data_dir`` (not part of the on-disk format + peers see). + """ + import yaml + + for path in target.rglob("*"): + if path.is_file(): + path.chmod(0o644) + + for config in target.rglob("private_metadata.yaml"): + data = yaml.safe_load(config.read_text()) or {} + if data.get("data_dir"): + data["data_dir"] = "" + config.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + + +def _build_fixture(target: Path, protocol_version: str) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + src_dir = tmp_path / "src" + src_dir.mkdir() + _seed_syftbox(syftbox, src_dir, protocol_version) + shutil.copytree(syftbox, target) + _normalize(target) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--force", action="store_true", help="overwrite an existing fixture" + ) + parser.add_argument( + "--protocol", + default=DATASET_PROTOCOL_VERSION, + help="protocol version to write (default: current)", + ) + parser.add_argument( + "--version", + default=__version__, + help="release version to stamp in the fixture name (default: current)", + ) + args = parser.parse_args() + + target = ( + FIXTURES_DIR / f"syft_dataset-{args.version}-protocol{args.protocol}_syftbox" + ) + if target.exists(): + if not args.force: + raise SystemExit(f"{target} already exists; pass --force to regenerate.") + shutil.rmtree(target) + + _build_fixture(target, args.protocol) + print(f"Wrote {target}") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-datasets/src/syft_datasets/__init__.py b/packages/syft-datasets/src/syft_datasets/__init__.py index 405e3e4fe68..6d2b75b44e0 100644 --- a/packages/syft-datasets/src/syft_datasets/__init__.py +++ b/packages/syft-datasets/src/syft_datasets/__init__.py @@ -1,4 +1,26 @@ +# __version__ comes from the installed distribution metadata (see version.py). +from .version import __version__ + from .config import SyftBoxConfig +from .dataset_ref import DatasetRef +from .dataset_storage import DatasetStorage +from .migrations import dataset_registry +from .migrations.history import register_historic_schemas +from .models import Dataset, DatasetV1, PrivateDatasetConfig, PrivateDatasetConfigV1 + +# Historic schemas list object versions that must already be registered, which +# happens when the models above are imported. +register_historic_schemas() -__version__ = "0.1.7" -__all__ = ["SyftBoxConfig"] +__all__ = [ + "__version__", + "SyftBoxConfig", + "DatasetRef", + "DatasetStorage", + "dataset_registry", + # Models + "Dataset", + "DatasetV1", + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/config.py b/packages/syft-datasets/src/syft_datasets/config.py index 28c3ae4251c..52674ebcce1 100644 --- a/packages/syft-datasets/src/syft_datasets/config.py +++ b/packages/syft-datasets/src/syft_datasets/config.py @@ -1,9 +1,27 @@ +import re +from typing import Optional + from pydantic import BaseModel, Field from pathlib import Path from .url import SyftBoxURL +from .migrations.registry import DATASET_PROTOCOL_VERSION SYFT_DATASETS_FOLDER_NAME = "syft_datasets" METADATA_FILENAME = "dataset.yaml" +PRIVATE_METADATA_FILENAME = "private_metadata.yaml" + +# Datasets under public/syft_datasets/ and private/syft_datasets/ live inside a +# protocol-version segment ("v1", "v2", ...); protocol 0 (<= 0.1.20) had none. +PROTOCOL_DIR_RE = re.compile(r"^v\d+$") + + +def is_protocol_dir_name(name: str) -> bool: + return PROTOCOL_DIR_RE.match(name) is not None + + +def protocol_dir_name(protocol_version: str) -> Optional[str]: + """The path segment for a protocol version; None for protocol 0.""" + return None if protocol_version == "0" else f"v{protocol_version}" class SyftBoxConfig(BaseModel): @@ -20,45 +38,119 @@ def private_dir(self) -> Path: def public_dir(self) -> Path: return self.syftbox_folder / self.email / "public" + def public_datasets_root_for_datasite(self, datasite: str) -> Path: + """The syft_datasets scan root for a datasite (no protocol segment).""" + return self.syftbox_folder / datasite / "public" / SYFT_DATASETS_FOLDER_NAME + + def private_datasets_root(self) -> Path: + """The private syft_datasets scan root (no protocol segment).""" + return self.private_dir / SYFT_DATASETS_FOLDER_NAME + + def private_datasets_root_for_owner(self, owner: str) -> Path: + """The private syft_datasets scan root under an owner's datasite.""" + return self.syftbox_folder / owner / "private" / SYFT_DATASETS_FOLDER_NAME + + def get_private_dataset_dir( + self, + owner: str, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.private_datasets_root_for_owner(owner), protocol_version, dataset_name + ) + def public_datasets_dir_for_datasite(self, datasite: str) -> Path: - dir = self.syftbox_folder / datasite / "public" / SYFT_DATASETS_FOLDER_NAME + # Backwards-compatible accessor; ensures the scan root exists. + dir = self.public_datasets_root_for_datasite(datasite) dir.mkdir(parents=True, exist_ok=True) return dir - def private_dir_for_my_dataset(self, dataset_name: str) -> Path: - return self.private_dir / SYFT_DATASETS_FOLDER_NAME / dataset_name + def _with_segment(self, root: Path, protocol_version: str, name: str) -> Path: + segment = protocol_dir_name(protocol_version) + return (root / segment / name) if segment else (root / name) - def get_my_mock_dataset_dir(self, dataset_name: str) -> Path: - return self.public_datasets_dir_for_datasite(self.email) / dataset_name + def private_dir_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.private_datasets_root(), protocol_version, dataset_name + ) - def get_mock_dataset_dir(self, dataset_name: str, datasite: str) -> Path: - return self.public_datasets_dir_for_datasite(datasite) / dataset_name + def get_my_mock_dataset_dir( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self.get_mock_dataset_dir( + dataset_name=dataset_name, + datasite=self.email, + protocol_version=protocol_version, + ) + + def get_mock_dataset_dir( + self, + dataset_name: str, + datasite: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return self._with_segment( + self.public_datasets_root_for_datasite(datasite), + protocol_version, + dataset_name, + ) - def get_mock_url_for_my_dataset(self, dataset_name: str) -> SyftBoxURL: + def get_mock_url_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name), + path=self.get_my_mock_dataset_dir(dataset_name, protocol_version), syftbox_folder=self.syftbox_folder, ) - def get_private_url_for_my_dataset(self, dataset_name: str) -> SyftBoxURL: + def get_private_url_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.private_dir_for_my_dataset(dataset_name), + path=self.private_dir_for_my_dataset(dataset_name, protocol_version), syftbox_folder=self.syftbox_folder, ) def get_readme_url_for_my_dataset( - self, dataset_name: str, readme_name: str + self, + dataset_name: str, + readme_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, ) -> SyftBoxURL: return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name) / readme_name, + path=self.get_my_mock_dataset_dir(dataset_name, protocol_version) + / readme_name, syftbox_folder=self.syftbox_folder, ) - def public_metadata_filename_for_my_dataset(self, dataset_name: str) -> str: + def public_metadata_filename_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: # TODO: not sure why the absolute is needed here return ( - self.get_my_mock_dataset_dir(dataset_name) / METADATA_FILENAME + self.get_my_mock_dataset_dir(dataset_name, protocol_version) + / METADATA_FILENAME ).absolute() - def private_metadata_filename_for_my_dataset(self, dataset_name: str) -> str: - return self.private_dir_for_my_dataset(dataset_name) / "private_metadata.yaml" + def private_metadata_filename_for_my_dataset( + self, + dataset_name: str, + protocol_version: str = DATASET_PROTOCOL_VERSION, + ) -> Path: + return ( + self.private_dir_for_my_dataset(dataset_name, protocol_version) + / PRIVATE_METADATA_FILENAME + ) diff --git a/packages/syft-datasets/src/syft_datasets/dataset.py b/packages/syft-datasets/src/syft_datasets/dataset.py index ba5eea9e5eb..20a94ffa999 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset.py +++ b/packages/syft-datasets/src/syft_datasets/dataset.py @@ -1,243 +1,15 @@ -from datetime import datetime, timezone -from functools import cached_property -from pathlib import Path -from typing import ClassVar -from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME -from typing_extensions import Self -from uuid import UUID, uuid4 +"""Backwards-compatible re-exports. -import yaml -from pydantic import BaseModel, Field -from syft_notebook_ui.formatter_mixin import ( - ANSIPydanticFormatter, - PydanticFormatter, - PydanticFormatterMixin, -) +The dataset models now live under ``syft_datasets.models`` as versioned +``MigratableObject``s. This module keeps the historical import paths +(``from syft_datasets.dataset import Dataset, PrivateDatasetConfig``) working. +""" -from .types import PathLike, to_path -from .url import SyftBoxURL -from .config import SyftBoxConfig +from .models import Dataset, DatasetV1, PrivateDatasetConfig, PrivateDatasetConfigV1 - -def _utcnow(): - return datetime.now(tz=timezone.utc) - - -class DatasetBase(BaseModel): - __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter() - _syftbox_config: SyftBoxConfig | None = None - - def save(self, filepath: PathLike) -> None: - filepath = to_path(filepath) - if not filepath.suffix == ".yaml": - raise ValueError("Model must be saved as a .yaml file.") - - if not filepath.parent.exists(): - filepath.parent.mkdir(parents=True, exist_ok=True) - - data = self.model_dump(mode="json") - yaml_dump = yaml.safe_dump(data, indent=2, sort_keys=False) - filepath.write_text(yaml_dump) - - @classmethod - def load( - cls, filepath: PathLike, syftbox_config: SyftBoxConfig | None = None - ) -> Self: - filepath = to_path(filepath) - if not filepath.exists(): - raise FileNotFoundError(f"Config file not found: {filepath}") - - data = yaml.safe_load(filepath.read_text()) - res = cls.model_validate(data) - res._syftbox_config = syftbox_config - return res - - def __str__(self) -> str: - return self.__display_formatter__.format_str(self) - - def __repr__(self) -> str: - return self.__display_formatter__.format_repr(self) - - def _repr_html_(self) -> str: - return self.__display_formatter__.format_html(self) - - def _repr_markdown_(self) -> str: - return self.__display_formatter__.format_markdown(self) - - -class PrivateDatasetConfig(DatasetBase, PydanticFormatterMixin): - """Used to store private dataset metadata, outside of the sync folder.""" - - uid: UUID # id for this dataset - data_dir: Path - - -class Dataset(DatasetBase, PydanticFormatterMixin): - __table_extra_fields__ = [ - "name", - "owner", - ] - - uid: UUID = Field(default_factory=uuid4) - created_at: datetime = Field(default_factory=_utcnow) - updated_at: datetime = Field(default_factory=_utcnow) - name: str - summary: str | None = None - tags: list[str] = [] - location: str | None = None - - mock_url: SyftBoxURL - private_url: SyftBoxURL - readme_url: SyftBoxURL | None = None - - # URLs to uploaded files (excluding metadata files) - mock_files_urls: list[SyftBoxURL] = Field(default_factory=list) - - @property - def owner(self) -> str: - return self.mock_url.host - - @property - def syftbox_config(self) -> SyftBoxConfig: - if self._syftbox_config is None: - raise ValueError("SyftBox config is not set.") - return self._syftbox_config - - def _url_to_path(self, url: SyftBoxURL) -> Path: - return url.to_local_path( - syftbox_folder=self.syftbox_config.syftbox_folder, - ) - - @property - def readme_path(self) -> Path | None: - if self.readme_url is None: - return None - return self._url_to_path(self.readme_url) - - def get_readme(self) -> str | None: - """Get the content of the README file.""" - if self.readme_path and self.readme_path.exists(): - return self.readme_path.read_text() - return None - - @property - def mock_dir(self) -> Path: - return self._url_to_path(self.mock_url) - - @property - def private_config_path(self) -> Path: - if self.syftbox_config.email != self.owner: - raise ValueError( - "Cannot access private config for a dataset owned by another user." - ) - return self._private_metadata_dir / "private_metadata.yaml" - - @cached_property - def private_config(self) -> PrivateDatasetConfig: - config_path = self.private_config_path - if not config_path.exists(): - raise FileNotFoundError( - f"Private dataset config not found at {config_path}" - ) - - return PrivateDatasetConfig.load( - filepath=config_path, syftbox_config=self._syftbox_config - ) - - @property - def private_dir(self) -> Path: - if self._syftbox_config is None: - raise ValueError("SyftBox config is not set.") - private_dir = ( - self.syftbox_config.syftbox_folder - / self.owner - / "private" - / "syft_datasets" - / self.name - ) - return private_dir - - @property - def _private_metadata_dir(self) -> Path: - if self.syftbox_config.email != self.owner: - raise ValueError( - "Cannot access private data for a dataset owned by another user." - ) - return self.private_dir - - @property - def mock_files(self) -> list[Path]: - """ - Get absolute paths to all mock files uploaded during dataset.create. - Excludes dataset.yaml and readme.md files. - """ - return [self._url_to_path(url) for url in self.mock_files_urls] - - @property - def private_files(self) -> list[Path]: - """ - Get absolute paths to all private files. - - For owners: returns paths from dataset.create (private_files_paths). - For non-owners (e.g. enclave): returns files from shared_private_dir. - """ - return [ - f - for f in self.private_dir.iterdir() - if f.is_file() - and f.name not in (PERMISSION_FILE_NAME, "private_metadata.yaml") - ] - - @property - def files(self) -> list[Path]: - """ - Get absolute paths to all files (both mock and private) uploaded during dataset.create. - """ - return self.mock_files + self.private_files - - def _generate_description_html(self) -> str: - from syft_notebook_ui.pydantic_html_repr import create_html_repr - - fields_to_include = ["name", "created_at", "summary", "tags", "location"] - - paths_to_include = [] - try: - paths_to_include.append("mock_dir") - except Exception: - fields_to_include.append("mock_url") - - try: - private_dir = self.private_dir - if private_dir.is_dir(): - paths_to_include.append("private_dir") - except Exception: - pass - - try: - readme_path = self.readme_path - if readme_path and readme_path.exists(): - paths_to_include.append("readme_path") - except Exception: - fields_to_include.append("readme_url") - - description = create_html_repr( - obj=self, - fields=fields_to_include, - display_paths=paths_to_include, - ) - - hint = ( - '

' - "💡 Use .mock_files to access mock data" - "
" - ) - return description + hint - - def describe(self) -> None: - from IPython.display import HTML, display - - description = self._generate_description_html() - display(HTML(description)) - - def _repr_html_(self) -> str: - return self._generate_description_html() +__all__ = [ + "Dataset", + "DatasetV1", + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index 13cb012dde8..2060eb6fde4 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -11,12 +11,13 @@ from typing_extensions import Literal from syft_datasets.dataset import Dataset, PrivateDatasetConfig +from syft_datasets.dataset_ref import DatasetRef +from syft_datasets.dataset_storage import DatasetStorage from syft_datasets.file_utils import copy_dir_contents, copy_paths, is_empty_dir -from .url import SyftBoxURL -from .config import SyftBoxConfig +from .config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig from .permissions import set_mock_dataset_permissions, set_private_dataset_permissions -from .config import METADATA_FILENAME +from .url import SyftBoxURL DATASET_COLLECTION_PREFIX = "syft_datasetcollection" PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" @@ -28,6 +29,10 @@ def __init__(self, syftbox_folder_path: PathLike, email: str): self.syftbox_config = SyftBoxConfig( syftbox_folder=to_path(syftbox_folder_path), email=email ) + # peer_schemas (peer email -> dataset ProtocolSchema) will be filled in by + # syft-client later; until then every peer resolves to the widest- + # compatible protocol, so datasets are written in that layout. + self.storage = DatasetStorage(config=self.syftbox_config) @classmethod def from_config(cls, config: SyftBoxConfig) -> Self: @@ -39,11 +44,10 @@ def _validate_dataset_name(self, dataset_name: str) -> None: raise ValueError( f"Invalid dataset name '{dataset_name}'. Only alphanumeric characters, underscores, and hyphens are allowed." ) + # Reject names that collide with a protocol-version directory (v1, v2, ...). + self.storage.validate_dataset_name(dataset_name) - def _prepare_mock_data(self, dataset_name: str, src_path: Path) -> list[Path]: - target_mock_dir = self.syftbox_config.get_my_mock_dataset_dir( - dataset_name=dataset_name - ) + def _prepare_mock_data(self, target_mock_dir: Path, src_path: Path) -> list[Path]: # Validate src data if not src_path.exists(): raise FileNotFoundError(f"Could not find mock data at {src_path}") @@ -80,14 +84,7 @@ def _prepare_mock_data(self, dataset_name: str, src_path: Path) -> list[Path]: return copied_files - def _prepare_private_data( - self, - dataset_name: str, - src_path: Path, - ) -> list[Path]: - private_dir = self.syftbox_config.private_dir_for_my_dataset( - dataset_name=dataset_name - ) + def _prepare_private_data(self, private_dir: Path, src_path: Path) -> list[Path]: private_dir.mkdir(parents=True, exist_ok=True) copied_files = [] @@ -111,36 +108,9 @@ def _prepare_private_data( return copied_files - def _prepare_private_config( - self, - dataset_name: str, - dataset_uid: UUID, - ) -> None: - """ - The private dataset config is used to store private metadata separately from the public dataset metadata. - """ - private_metadata_path: Path = ( - self.syftbox_config.private_metadata_filename_for_my_dataset( - dataset_name=dataset_name - ) - ) - if private_metadata_path.exists(): - raise FileExistsError( - f"Private metadata file {private_metadata_path} already exists." - ) - - private_config = PrivateDatasetConfig( - uid=dataset_uid, - data_dir=private_metadata_path.parent, - ) - - private_metadata_path.parent.mkdir(parents=True, exist_ok=True) - private_config.save(filepath=private_metadata_path) - - def _prepare_readme(self, dataset_name: str, src_file: Path | None) -> list[Path]: - target_mock_dir = self.syftbox_config.get_my_mock_dataset_dir( - dataset_name=dataset_name - ) + def _prepare_readme( + self, target_mock_dir: Path, src_file: Path | None + ) -> list[Path]: copied_files = [] if src_file is not None: if not src_file.is_file(): @@ -166,86 +136,106 @@ def create( users: list[str] | str | None = None, # copy_private_data: bool = True, # TODO ) -> Dataset: - """_summary_ + """Create a dataset, writing it in each protocol version its audience can read. Args: - name (str): Unique of the dataset to create. - mock_path (PathLike): Path to the existing mock data. This can be a file or a directory. - private_path (PathLike): Path to the existing private data. This can be a file or a directory. + name (str): Unique name of the dataset to create. + mock_path (PathLike): Path to the existing mock data (file or directory). + private_path (PathLike): Path to the existing private data (file or directory). summary (str | None, optional): Short summary of the dataset. Defaults to None. - readme_path (Path | None, optional): Markdown README in the public dataset. Defaults to None. - location (str | None, optional): Location identifier for the dataset, e.g. 'high-side-1234'. - Only required for datasets that are hosted on a remote location and require manual syncing. - Defaults to None. - tags (list[str] | None, optional): Optional tags for the dataset. Defaults to None. - users (list[str] | str | None, optional): Users to share dataset with. Can be list of emails, SHARE_WITH_ANY, or None (default, share with no one). + readme_path (Path | None, optional): Markdown README in the public dataset. + location (str | None, optional): Location identifier for the dataset. + tags (list[str] | None, optional): Optional tags for the dataset. + users (list[str] | str | None, optional): Users to share dataset with. Can be + a list of emails, SHARE_WITH_ANY, or None (default, share with no one). Returns: - Dataset: The created Dataset object. + Dataset: The created Dataset object (the newest protocol version written). """ - mock_path: Path = to_path(mock_path) - private_path: Path = to_path(private_path) - readme_path: Path | None = to_path(readme_path) if readme_path else None + # Reject names that collide with a protocol-version directory (v1, v2, ...). + # (The stricter _validate_dataset_name regex is intentionally not enforced + # here to preserve historical behavior, e.g. names with spaces.) + self.storage.validate_dataset_name(name) + mock_path = to_path(mock_path) + private_path = to_path(private_path) + readme_path = to_path(readme_path) if readme_path else None tags = tags or [] - mock_url = self.syftbox_config.get_mock_url_for_my_dataset(dataset_name=name) + target_versions = self._target_protocol_versions(users) + dataset_uid = uuid4() + + created: dict[str, Dataset] = {} + for protocol_version in sorted(target_versions, key=int): + created[protocol_version] = self._create_one_version( + name=name, + protocol_version=protocol_version, + dataset_uid=dataset_uid, + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + summary=summary, + location=location, + tags=tags, + users=users, + ) + + # Return the newest protocol version written (richest layout). + newest = max(created, key=int) + return created[newest] + + def _target_protocol_versions(self, users: list[str] | str | None) -> set[str]: + """The protocol versions to write so the audience can read the dataset.""" + peer_emails = None if users in (None, SHARE_WITH_ANY) else list(users) + return self.storage.target_protocol_versions_for_peers(peer_emails) + + def _create_one_version( + self, + name: str, + protocol_version: str, + dataset_uid: UUID, + mock_path: Path, + private_path: Path, + readme_path: Path | None, + summary: str | None, + location: str | None, + tags: list[str], + users: list[str] | str | None, + ) -> Dataset: + ref = self.storage.new_dataset_ref(name, protocol_version) + + mock_url = self.syftbox_config.get_mock_url_for_my_dataset( + name, protocol_version + ) readme_url = ( self.syftbox_config.get_readme_url_for_my_dataset( - dataset_name=name, readme_name=readme_path.name + name, readme_path.name, protocol_version ) if readme_path else None ) - - # Generate private_url for the dataset Private URLs use a simple path format private_url = self.syftbox_config.get_private_url_for_my_dataset( - dataset_name=name - ) - - # Prepare mock data and collect file paths - all_mock_file_paths = self._prepare_mock_data( - dataset_name=name, - src_path=mock_path, + name, protocol_version ) - # Mock files exclude dataset.yaml and readme.md - # Convert absolute paths to SyftBoxURLs - - # Prepare readme and collect file paths - readme_files = self._prepare_readme( - dataset_name=name, - src_file=readme_path, - ) - - public_metadata_path = ( - self.syftbox_config.public_metadata_filename_for_my_dataset( - dataset_name=name - ) - ) + target_mock_dir = self.storage.public_dataset_dir(ref) + all_mock_file_paths = self._prepare_mock_data(target_mock_dir, mock_path) + readme_files = self._prepare_readme(target_mock_dir, readme_path) + public_metadata_path = target_mock_dir / METADATA_FILENAME mock_file_paths = [ f for f in all_mock_file_paths if f != public_metadata_path and f not in readme_files ] - mock_files_urls = [ SyftBoxURL.from_path( - path=file_path, - syftbox_folder=self.syftbox_config.syftbox_folder, + path=file_path, syftbox_folder=self.syftbox_config.syftbox_folder ) for file_path in mock_file_paths ] - self._prepare_private_data( - dataset_name=name, - src_path=private_path, - ) - - # TODO enable adding private data without copying to SyftBox - # e.g. private_data_dir = dataset._private_metadata_dir if copy_private_data else private_path - dataset_uid = uuid4() - self._prepare_private_config(dataset_uid=dataset_uid, dataset_name=name) + private_dir = self.storage.private_dataset_dir(ref) + self._prepare_private_data(private_dir, private_path) dataset = Dataset( uid=dataset_uid, @@ -257,14 +247,14 @@ def create( location=location, tags=tags, mock_files_urls=mock_files_urls, - _syftbox_config=self.syftbox_config, ) - # needs to set since its a private attr dataset._syftbox_config = self.syftbox_config + dataset._protocol_version = protocol_version - # Save dataset metadata - dataset.save(filepath=public_metadata_path) - # Set permissions on mock and private directories + self.storage.write_dataset(ref, dataset) + self.storage.write_private_config( + ref, PrivateDatasetConfig(uid=dataset_uid, data_dir=private_dir) + ) self._set_new_dataset_permissions(dataset=dataset, users=users) return dataset @@ -289,24 +279,11 @@ def _set_new_dataset_permissions( dataset.private_dir, ) - def _load_dataset_from_dir(self, dataset_dir: Path) -> Dataset: - metadata_path = dataset_dir / METADATA_FILENAME - if not metadata_path.exists(): - raise FileNotFoundError(f"Dataset metadata not found at {metadata_path}") - - return Dataset.load( - filepath=metadata_path, - syftbox_config=self.syftbox_config, - ) - def get(self, name: str, datasite: str | None = None) -> Dataset: datasite = datasite or self.syftbox_config.email - mock_dir = self.syftbox_config.get_mock_dataset_dir( - dataset_name=name, - datasite=datasite, - ) - - if not mock_dir.exists(): + try: + ref = self.storage.find_dataset_ref(datasite, name) + except FileNotFoundError: available = self.get_all() if available: listing = "\n".join( @@ -323,7 +300,7 @@ def get(self, name: str, datasite: str | None = None) -> Dataset: f" Available datasets:\n" f"{listing}" ) - return self._load_dataset_from_dir(mock_dir) + return self.storage.read_dataset(ref) def __getitem__(self, key: str | int) -> Dataset: if isinstance(key, int): @@ -345,6 +322,15 @@ def _repr_html_(self) -> str: return dataset_manager_repr_html(self.get_all()) + def _datasites_to_check(self, datasite: str | None) -> list[str]: + if datasite: + return [datasite] + syftbox_folder = self.syftbox_config.syftbox_folder + # All directories with "@" in the name are peer/owner email directories + return [ + d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name + ] + def get_all( self, datasite: str | None = None, @@ -353,30 +339,24 @@ def get_all( order_by: str | None = None, sort_order: Literal["asc", "desc"] = "asc", ) -> list[Dataset]: - all_datasets = [] + # A dataset may exist on disk in more than one protocol layout; keep only + # the newest protocol per (owner, name). + best_ref: dict[tuple[str, str], DatasetRef] = {} + for datasite in self._datasites_to_check(datasite): + for ref in self.storage.iter_dataset_refs(datasite): + key = (ref.owner, ref.name) + current = best_ref.get(key) + if current is None or int(ref.protocol_version) > int( + current.protocol_version + ): + best_ref[key] = ref - if datasite: - datasites_to_check = [datasite] - else: - syftbox_folder = self.syftbox_config.syftbox_folder - # All directories with "@" in the name are peer/owner email directories - datasites_to_check = [ - d.name for d in syftbox_folder.iterdir() if d.is_dir() and "@" in d.name - ] - - for datasite in datasites_to_check: - public_datasets_dir = self.syftbox_config.public_datasets_dir_for_datasite( - datasite - ) - if not public_datasets_dir.exists(): + all_datasets = [] + for ref in best_ref.values(): + try: + all_datasets.append(self.storage.read_dataset(ref)) + except Exception: continue - for dataset_dir in public_datasets_dir.iterdir(): - if dataset_dir.is_dir(): - try: - dataset = self._load_dataset_from_dir(dataset_dir) - all_datasets.append(dataset) - except Exception: - continue if order_by is not None: all_datasets.sort( @@ -447,22 +427,27 @@ def delete( def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: """Get private dataset files as {path_in_datasite: content}. - Returns paths relative to the datasite (e.g. private/syft_datasets/{name}/{file}). - For private_metadata.yaml, clears data_dir before including it. + Returns paths relative to the datasite (e.g. + private/syft_datasets/[v/]{name}/{file}). For private_metadata.yaml, + clears data_dir before including it. """ - dataset = self.get(name=name, datasite=self.syftbox_config.email) - private_dir = dataset.private_dir + datasite = self.syftbox_config.email + ref = self.storage.find_dataset_ref(datasite, name) + private_dir = self.storage.private_dataset_dir(ref) if not private_dir.exists(): raise ValueError(f"Private data directory not found: {private_dir}") + datasite_root = self.syftbox_config.syftbox_folder / datasite + private_rel_root = private_dir.relative_to(datasite_root) + files = {} for f in private_dir.rglob("*"): if not f.is_file(): continue rel = f.relative_to(private_dir) - path_in_datasite = Path(f"private/syft_datasets/{name}") / rel - if f.name == "private_metadata.yaml" and rel == Path(f.name): - files[path_in_datasite] = self._private_config_without_data_dir(f) + path_in_datasite = private_rel_root / rel + if f.name == PRIVATE_METADATA_FILENAME and rel == Path(f.name): + files[path_in_datasite] = self._private_config_without_data_dir(ref) else: files[path_in_datasite] = f.read_bytes() @@ -470,10 +455,15 @@ def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: raise ValueError(f"No private files found for dataset '{name}'") return files - def _private_config_without_data_dir(self, config_path: Path) -> bytes: - """Load private_metadata.yaml and return it with data_dir cleared.""" - config = PrivateDatasetConfig.load(filepath=config_path) + def _private_config_without_data_dir(self, ref: DatasetRef) -> bytes: + """Serialize the dataset's private_metadata.yaml with data_dir cleared. + + Written in the dataset's on-disk protocol format so peers can read it. + """ + config = self.storage.read_private_config(ref) config.data_dir = Path("") - return yaml.safe_dump( - config.model_dump(mode="json"), indent=2, sort_keys=False - ).encode() + data = config.disk_dict() + if ref.protocol_version == "0": + data.pop("canonical_name", None) + data.pop("version", None) + return yaml.safe_dump(data, indent=2, sort_keys=False).encode() diff --git a/packages/syft-datasets/src/syft_datasets/dataset_ref.py b/packages/syft-datasets/src/syft_datasets/dataset_ref.py new file mode 100644 index 00000000000..a1bcca87ee3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/dataset_ref.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + + +class DatasetNotFoundError(FileNotFoundError): + """Raised when a dataset's metadata (dataset.yaml) does not exist.""" + + +class PrivateConfigNotFoundError(FileNotFoundError): + """Raised when a dataset's private_metadata.yaml does not exist.""" + + +@dataclass(frozen=True) +class DatasetRef: + """One dataset on disk: who owns it and its protocol layout. + + A dataset is single-owner and broadcast-read (one public copy read by the + whole audience), so there is no per-reader field: identity is + (owner, name, protocol_version). + """ + + owner: str # datasite email of the DO who owns the dataset + name: str + protocol_version: str # "0" (no path segment) or "1"+ (v segment) diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py new file mode 100644 index 00000000000..9ee96dbb515 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -0,0 +1,216 @@ +from pathlib import Path +from typing import Iterator, Optional + +from syft_migration import ( + MigratableObject, + MigrationError, + MigrationRegistry, + MigrationService, + ProtocolSchema, +) + +from .config import SyftBoxConfig, is_protocol_dir_name +from .dataset_ref import DatasetNotFoundError, DatasetRef, PrivateConfigNotFoundError +from .migrations.registry import DATASET_PROTOCOL_VERSION, dataset_registry +from .models import Dataset, PrivateDatasetConfig +from .protocolcodecs import CODECS, ProtocolCodec + +__all__ = [ + "DatasetRef", + "DatasetNotFoundError", + "PrivateConfigNotFoundError", + "DatasetStorage", +] + + +class DatasetStorage: + """Dataset filesystem IO and path resolution, delegating disk layout to codecs. + + DatasetStorage owns migration: reads upgrade objects to the latest registered + version in memory; writes downgrade them to what a reading peer understands. + Each on-disk storage format lives behind a ProtocolCodec, selected by the + protocol version (from a dataset's on-disk layout for reads; the negotiated + version for new datasets). The last codec is the current protocol. + + Unlike jobs (each copy targets one peer), a public dataset is a single copy + read by the whole audience, so new datasets are written at the version(s) the + audience can read (see ``target_protocol_versions_for_peers``), defaulting to + the widest-compatible (oldest) protocol when no peers are known. + """ + + def __init__( + self, + config: SyftBoxConfig, + registry: MigrationRegistry = dataset_registry, + peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + ) -> None: + self.config = config + self.registry = registry + self.service = MigrationService(registry=registry) + # peer email -> dataset ProtocolSchema; filled in by syft-client later. + # Peers without an entry cannot be assumed to read the current layout, so + # they resolve to the widest-compatible (oldest) protocol. + self.peer_schemas: dict[str, ProtocolSchema] = peer_schemas or {} + self.codecs = [cls(config) for cls in CODECS] + + @property + def _codec_by_protocol_version(self) -> dict[str, ProtocolCodec]: + # protocol version -> codec; one codec may serve several versions. + return { + protocol_version: codec + for codec in self.codecs + for protocol_version in codec.protocol_versions + } + + def _codec_for(self, protocol_version: str) -> ProtocolCodec: + codec = self._codec_by_protocol_version.get(protocol_version) + if codec is None: + raise MigrationError( + f"No codec for dataset protocol version {protocol_version!r}" + ) + return codec + + @property + def _widest_protocol_version(self) -> str: + """The oldest protocol version any current client can read (widest compat).""" + return min(self._codec_by_protocol_version, key=int) + + # -- peers / protocol ---------------------------------------------------- + def negotiated_protocol_version_for_peer( + self, peer_email: str, raise_on_unknown: bool = True + ) -> str: + """The dataset protocol version to speak with ``peer_email``. + + Negotiated as the minimum of our own protocol version and the peer's, so + both sides use a version they can read. A peer without a known schema + raises by default; with ``raise_on_unknown=False`` it is assumed to run + the current protocol. + """ + schema = self.peer_schemas.get(peer_email) + if schema is not None: + return min(DATASET_PROTOCOL_VERSION, schema.version, key=int) + if raise_on_unknown: + raise MigrationError( + f"No dataset protocol schema known for peer {peer_email!r}" + ) + return DATASET_PROTOCOL_VERSION + + def target_protocol_versions_for_peers( + self, peer_emails: Optional[list[str]] = None + ) -> set[str]: + """The set of protocol versions to write so every peer can read a copy. + + A dataset is written once per distinct version in the audience. A known + peer contributes ``min(ours, theirs)``; an unknown peer (or no audience) + contributes the widest-compatible protocol, since we cannot assume it can + read a newer layout. + """ + if not peer_emails: + return {self._widest_protocol_version} + versions: set[str] = set() + for email in peer_emails: + schema = self.peer_schemas.get(email) + if schema is not None: + versions.add(min(DATASET_PROTOCOL_VERSION, schema.version, key=int)) + else: + versions.add(self._widest_protocol_version) + return versions + + def new_dataset_ref(self, name: str, protocol_version: str) -> DatasetRef: + """A ref for a new dataset owned by the current user.""" + return DatasetRef( + owner=self.config.email, name=name, protocol_version=protocol_version + ) + + # -- naming -------------------------------------------------------------- + @staticmethod + def validate_dataset_name(dataset_name: str) -> None: + if is_protocol_dir_name(dataset_name): + raise ValueError( + f"Dataset name {dataset_name!r} is reserved for protocol version directories" + ) + + # -- path resolution ----------------------------------------------------- + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).public_dataset_dir(ref) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).private_dataset_dir(ref) + + def metadata_path(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).metadata_path(ref) + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self._codec_for(ref.protocol_version).private_metadata_path(ref) + + # -- scanning (union over all protocol layouts) -------------------------- + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + """Yield a ref per dataset under public/syft_datasets/, across all codecs.""" + for codec in self.codecs: + yield from codec.iter_dataset_refs(datasite_email) + + def find_dataset_ref(self, datasite_email: str, name: str) -> DatasetRef: + """The ref for ``name`` in a datasite; prefers the highest protocol layout.""" + matches = [ + ref for ref in self.iter_dataset_refs(datasite_email) if ref.name == name + ] + if not matches: + raise DatasetNotFoundError(f"Dataset '{name}' not found") + return max(matches, key=lambda r: int(r.protocol_version)) + + # -- model IO ------------------------------------------------------------ + def read_dataset(self, ref: DatasetRef) -> Dataset: + """Load a dataset's dataset.yaml, upgraded to the latest version.""" + codec = self._codec_for(ref.protocol_version) + data = codec.read(codec.metadata_path(ref), "Dataset") + dataset = self._upgrade(data, "Dataset") + # Name comes from the path (the dataset dir), never a spoofable file. + dataset.name = ref.name + dataset._syftbox_config = self.config + dataset._protocol_version = ref.protocol_version + return dataset + + def read_private_config(self, ref: DatasetRef) -> PrivateDatasetConfig: + """Load a dataset's private_metadata.yaml, upgraded to the latest version. + + Raises PrivateConfigNotFoundError if the file does not exist. + """ + codec = self._codec_for(ref.protocol_version) + path = codec.private_metadata_path(ref) + if not path.exists(): + raise PrivateConfigNotFoundError( + f"Private config not found for dataset {ref.name}" + ) + return self._upgrade( + codec.read(path, "PrivateDatasetConfig"), "PrivateDatasetConfig" + ) + + def write_dataset(self, ref: DatasetRef, dataset: Dataset) -> Path: + """Write dataset.yaml in the version/format for this ref's protocol.""" + codec = self._codec_for(ref.protocol_version) + return self._write(codec, codec.metadata_path(ref), dataset, ref) + + def write_private_config( + self, ref: DatasetRef, config: PrivateDatasetConfig + ) -> Path: + """Write private_metadata.yaml in the version/format for this ref's protocol.""" + codec = self._codec_for(ref.protocol_version) + return self._write(codec, codec.private_metadata_path(ref), config, ref) + + # -- internals ----------------------------------------------------------- + def _upgrade(self, data: dict, canonical_name: str) -> MigratableObject: + obj = self.service.load(data) + return self.service.migrate(obj, self.registry.latest_version(canonical_name)) + + def _write( + self, + codec: ProtocolCodec, + path: Path, + obj: MigratableObject, + ref: DatasetRef, + ) -> Path: + """Downgrade ``obj`` to the ref's protocol schema, then let the codec persist it.""" + schema = self.registry.schema_for_protocol_version(ref.protocol_version) + downgraded = self.service.migrate_to_schema(obj, schema) + codec.write(path, downgraded) + return path diff --git a/packages/syft-datasets/src/syft_datasets/migrations/__init__.py b/packages/syft-datasets/src/syft_datasets/migrations/__init__.py new file mode 100644 index 00000000000..22b0db6f7ad --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/__init__.py @@ -0,0 +1,5 @@ +from .registry import dataset_registry + +__all__ = [ + "dataset_registry", +] diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history.py b/packages/syft-datasets/src/syft_datasets/migrations/history.py new file mode 100644 index 00000000000..583f67e9689 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from syft_migration import ReleasedPackageProtocolInfo, ReleasedProtocol + +from .registry import dataset_registry + +# Release artifacts of past syft-dataset releases: +# package-artifacts/syft-dataset-.json (every release) +# protocols/protocol-.json (only when the protocol changed) +# Generated by export_release_artifact.py; 0.1.20 / protocol 0 predate the +# artifact mechanism, so their files are hardcoded as if that release had +# emitted them. +HISTORY_DIR = Path(__file__).parent / "history" +PACKAGE_ARTIFACTS_DIR = HISTORY_DIR / "package-artifacts" +PROTOCOLS_DIR = HISTORY_DIR / "protocols" + + +def register_historic_schemas() -> None: + """Register the release artifacts of past releases into the dataset registry. + + Must run after the versioned models are imported: with + ``raise_for_unknown_objects`` an artifact listing an object version this + release cannot load fails at import time instead of at migration time. + """ + for path in sorted(PACKAGE_ARTIFACTS_DIR.glob("*.json")): + dataset_registry.register_released_package_protocol_info( + ReleasedPackageProtocolInfo.load(path), raise_for_unknown_objects=True + ) + for path in sorted(PROTOCOLS_DIR.glob("*.json")): + dataset_registry.register_released_protocol( + ReleasedProtocol.load(path), raise_for_unknown_objects=True + ) diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json new file mode 100644 index 00000000000..cd2f31cc7e3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.20.json @@ -0,0 +1,149 @@ +{ + "package_info": { + "package_name": "syft-dataset", + "version": "0.1.20", + "protocol_version": "0" + }, + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "0", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json new file mode 100644 index 00000000000..dbcf520d6a7 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json @@ -0,0 +1,149 @@ +{ + "package_info": { + "package_name": "syft-dataset", + "version": "0.1.21", + "protocol_version": "1" + }, + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "1", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json new file mode 100644 index 00000000000..c01c931dfc9 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-0.json @@ -0,0 +1,144 @@ +{ + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "0", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json new file mode 100644 index 00000000000..b81ab76a29b --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json @@ -0,0 +1,144 @@ +{ + "protocol_schema": { + "protocol_name": "syft-dataset", + "version": "1", + "supported_versions": { + "PrivateDatasetConfig": ["1"], + "Dataset": ["1"] + }, + "current_object_schemas": { + "PrivateDatasetConfig": { + "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", + "properties": { + "canonical_name": { + "default": "PrivateDatasetConfig", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "data_dir": { + "format": "path", + "title": "Data Dir", + "type": "string" + } + }, + "required": ["uid", "data_dir"], + "title": "PrivateDatasetConfigV1", + "type": "object" + }, + "Dataset": { + "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", + "properties": { + "canonical_name": { + "default": "Dataset", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "uid": { + "format": "uuid", + "title": "Uid", + "type": "string" + }, + "created_at": { + "format": "date-time", + "title": "Created At", + "type": "string" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Summary" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Location" + }, + "mock_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Mock Url", + "type": "string" + }, + "private_url": { + "description": "A SyftBox URL", + "format": "uri", + "title": "Private Url", + "type": "string" + }, + "readme_url": { + "anyOf": [ + { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Readme Url" + }, + "mock_files_urls": { + "items": { + "description": "A SyftBox URL", + "format": "uri", + "type": "string" + }, + "title": "Mock Files Urls", + "type": "array" + } + }, + "required": ["name", "mock_url", "private_url"], + "title": "DatasetV1", + "type": "object" + } + } + } +} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/registry.py b/packages/syft-datasets/src/syft_datasets/migrations/registry.py new file mode 100644 index 00000000000..661be107d98 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/migrations/registry.py @@ -0,0 +1,22 @@ +from syft_migration import MigrationRegistry + +from ..version import PACKAGE_NAME, __version__ + +# Hardcoded, language-agnostic identifier for the syft-dataset protocol; +# intentionally distinct from the package name. +PROTOCOL_NAME = "syft-dataset" + +# Incrementing version of the dataset protocol. Protocol 0 is the last release +# without versioning (syft-client 0.1.117 / syft-dataset 0.1.20, no v path +# segment); protocol >= 1 stores datasets under a v segment after the +# syft_datasets folder (see config.protocol_dir_name). +DATASET_PROTOCOL_VERSION = "1" + +# Package-local registry for all versioned syft-dataset objects. The current +# protocol schema is computed from the objects registered into it. +dataset_registry = MigrationRegistry( + protocol_name=PROTOCOL_NAME, + package_name=PACKAGE_NAME, + package_version=__version__, + protocol_version=DATASET_PROTOCOL_VERSION, +) diff --git a/packages/syft-datasets/src/syft_datasets/models/__init__.py b/packages/syft-datasets/src/syft_datasets/models/__init__.py new file mode 100644 index 00000000000..427185b28e9 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/__init__.py @@ -0,0 +1,11 @@ +from .dataset import Dataset, DatasetV1 +from .private_dataset_config import PrivateDatasetConfig, PrivateDatasetConfigV1 + +__all__ = [ + # Current-version aliases + "Dataset", + "PrivateDatasetConfig", + # Versioned objects + "DatasetV1", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py b/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py new file mode 100644 index 00000000000..afbb706ab44 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/__init__.py @@ -0,0 +1,9 @@ +from .v1 import DatasetV1 + +# The current version of the dataset object. +Dataset = DatasetV1 + +__all__ = [ + "Dataset", + "DatasetV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py new file mode 100644 index 00000000000..2978a35d051 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from functools import cached_property +from pathlib import Path +from typing import ClassVar +from uuid import UUID, uuid4 + +import yaml +from pydantic import Field +from syft_migration import MigratableObject +from syft_notebook_ui.formatter_mixin import ( + ANSIPydanticFormatter, + PydanticFormatter, + PydanticFormatterMixin, +) +from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME + +from ...config import ( + PRIVATE_METADATA_FILENAME, + SYFT_DATASETS_FOLDER_NAME, + SyftBoxConfig, + protocol_dir_name, +) +from ...migrations import dataset_registry +from ...migrations.registry import DATASET_PROTOCOL_VERSION +from ...url import SyftBoxURL +from ..private_dataset_config.v1 import PrivateDatasetConfigV1 + + +def _utcnow() -> datetime: + return datetime.now(tz=timezone.utc) + + +class DatasetV1(MigratableObject, PydanticFormatterMixin, registry=dataset_registry): + """Public dataset metadata, stored as dataset.yaml under + SyftBox//public/syft_datasets/[v/]/.""" + + __display_formatter__: ClassVar[PydanticFormatter] = ANSIPydanticFormatter() + __table_extra_fields__: ClassVar[list[str]] = ["name", "owner"] + + canonical_name: str = "Dataset" + version: str = "1" + + uid: UUID = Field(default_factory=uuid4) + created_at: datetime = Field(default_factory=_utcnow) + updated_at: datetime = Field(default_factory=_utcnow) + name: str + summary: str | None = None + tags: list[str] = [] + location: str | None = None + + mock_url: SyftBoxURL + private_url: SyftBoxURL + readme_url: SyftBoxURL | None = None + + # URLs to uploaded files (excluding metadata files) + mock_files_urls: list[SyftBoxURL] = Field(default_factory=list) + + # Runtime-only: set by DatasetStorage / the manager, never serialized. The + # on-disk protocol layout the dataset lives in (governs the v segment). + _syftbox_config: SyftBoxConfig | None = None + _protocol_version: str = DATASET_PROTOCOL_VERSION + + def disk_dict(self) -> dict: + """The on-disk form of the dataset metadata.""" + return self.model_dump(mode="json") + + @property + def owner(self) -> str: + return self.mock_url.host + + @property + def syftbox_config(self) -> SyftBoxConfig: + if self._syftbox_config is None: + raise ValueError("SyftBox config is not set.") + return self._syftbox_config + + def _url_to_path(self, url: SyftBoxURL) -> Path: + return url.to_local_path(syftbox_folder=self.syftbox_config.syftbox_folder) + + @property + def readme_path(self) -> Path | None: + if self.readme_url is None: + return None + return self._url_to_path(self.readme_url) + + def get_readme(self) -> str | None: + """Get the content of the README file.""" + if self.readme_path and self.readme_path.exists(): + return self.readme_path.read_text() + return None + + @property + def mock_dir(self) -> Path: + return self._url_to_path(self.mock_url) + + @property + def private_config_path(self) -> Path: + if self.syftbox_config.email != self.owner: + raise ValueError( + "Cannot access private config for a dataset owned by another user." + ) + return self._private_metadata_dir / PRIVATE_METADATA_FILENAME + + @cached_property + def private_config(self) -> PrivateDatasetConfigV1: + config_path = self.private_config_path + if not config_path.exists(): + raise FileNotFoundError( + f"Private dataset config not found at {config_path}" + ) + data = yaml.safe_load(config_path.read_text()) or {} + data.setdefault("canonical_name", "PrivateDatasetConfig") + data.setdefault("version", "1") + return PrivateDatasetConfigV1(**data) + + @property + def private_dir(self) -> Path: + """The private data dir for this dataset, under the dataset's protocol layout. + + Derived from the path (owner + name + protocol) rather than the stored + URL so it stays correct across on-disk layouts. + """ + segment = protocol_dir_name(self._protocol_version) + root = ( + self.syftbox_config.syftbox_folder + / self.owner + / "private" + / SYFT_DATASETS_FOLDER_NAME + ) + return (root / segment / self.name) if segment else (root / self.name) + + @property + def _private_metadata_dir(self) -> Path: + if self.syftbox_config.email != self.owner: + raise ValueError( + "Cannot access private data for a dataset owned by another user." + ) + return self.private_dir + + @property + def mock_files(self) -> list[Path]: + """Absolute paths to all mock files uploaded during dataset.create. + + Excludes dataset.yaml and readme.md files. + """ + return [self._url_to_path(url) for url in self.mock_files_urls] + + @property + def private_files(self) -> list[Path]: + """Absolute paths to all private files. + + For owners: returns paths from dataset.create (private_files_paths). + For non-owners (e.g. enclave): returns files from shared_private_dir. + """ + return [ + f + for f in self.private_dir.iterdir() + if f.is_file() + and f.name not in (PERMISSION_FILE_NAME, PRIVATE_METADATA_FILENAME) + ] + + @property + def files(self) -> list[Path]: + """Absolute paths to all files (both mock and private).""" + return self.mock_files + self.private_files + + def _generate_description_html(self) -> str: + from syft_notebook_ui.pydantic_html_repr import create_html_repr + + fields_to_include = ["name", "created_at", "summary", "tags", "location"] + + paths_to_include = [] + try: + paths_to_include.append("mock_dir") + except Exception: + fields_to_include.append("mock_url") + + try: + private_dir = self.private_dir + if private_dir.is_dir(): + paths_to_include.append("private_dir") + except Exception: + pass + + try: + readme_path = self.readme_path + if readme_path and readme_path.exists(): + paths_to_include.append("readme_path") + except Exception: + fields_to_include.append("readme_url") + + description = create_html_repr( + obj=self, + fields=fields_to_include, + display_paths=paths_to_include, + ) + + hint = ( + '
' + "💡 Use .mock_files to access mock data" + "
" + ) + return description + hint + + def describe(self) -> None: + from IPython.display import HTML, display + + description = self._generate_description_html() + display(HTML(description)) + + def _repr_html_(self) -> str: + return self._generate_description_html() diff --git a/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py new file mode 100644 index 00000000000..d391d1f9f79 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/__init__.py @@ -0,0 +1,9 @@ +from .v1 import PrivateDatasetConfigV1 + +# The current version of the private dataset config object. +PrivateDatasetConfig = PrivateDatasetConfigV1 + +__all__ = [ + "PrivateDatasetConfig", + "PrivateDatasetConfigV1", +] diff --git a/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py new file mode 100644 index 00000000000..fbc7f080dfd --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/models/private_dataset_config/v1.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +from syft_migration import MigratableObject +from syft_notebook_ui.formatter_mixin import PydanticFormatterMixin + +from ...migrations import dataset_registry + + +class PrivateDatasetConfigV1( + MigratableObject, PydanticFormatterMixin, registry=dataset_registry +): + """Private dataset metadata, stored as private_metadata.yaml outside the sync folder.""" + + canonical_name: str = "PrivateDatasetConfig" + version: str = "1" + + uid: UUID # id for this dataset + data_dir: Path + + def disk_dict(self) -> dict: + return self.model_dump(mode="json") diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py new file mode 100644 index 00000000000..50c78dccd44 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py @@ -0,0 +1,16 @@ +from .base import ProtocolCodec +from .v0 import ProtocolCodecV0 +from .v1 import ProtocolCodecV1 + +# Ordered oldest -> newest. The last codec is the one the current dataset +# protocol uses (CODECS[-1].version == DATASET_PROTOCOL_VERSION); adding a new +# on-disk format means appending a ProtocolCodecV here and bumping +# DATASET_PROTOCOL_VERSION. +CODECS = [ProtocolCodecV0, ProtocolCodecV1] + +__all__ = [ + "ProtocolCodec", + "ProtocolCodecV0", + "ProtocolCodecV1", + "CODECS", +] diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py new file mode 100644 index 00000000000..1e21dc2174a --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Iterator + +from syft_migration import MigratableObject + +from ..config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig +from ..dataset_ref import DatasetRef + + +class ProtocolCodec(ABC): + """Interface for reading/writing/listing datasets in one on-disk storage format. + + A codec owns raw disk layout + serialization for the protocol versions it + lists in ``protocol_versions`` (one codec can serve several — the layout is + derived from each dataset's ``protocol_version``). DatasetStorage owns + migration and selects a codec by protocol version; a codec never maps back + to one. The codec carries its own ``version`` independent of the protocols + it handles. + + A dataset is a directory with a public ``dataset.yaml`` (the discovery + marker, synced to peers) plus mock files, and a separate + ``private_metadata.yaml`` under the owner's private/ tree (never synced). + """ + + version: str # the codec's own version ("0", "1", ...) + protocol_versions: list[str] # protocol versions this codec reads/writes + metadata_marker = METADATA_FILENAME + private_metadata_marker = PRIVATE_METADATA_FILENAME + + def __init__(self, config: SyftBoxConfig) -> None: + self.config = config + + @abstractmethod + def public_dataset_dir(self, ref: DatasetRef) -> Path: ... + + @abstractmethod + def private_dataset_dir(self, ref: DatasetRef) -> Path: ... + + def metadata_path(self, ref: DatasetRef) -> Path: + return self.public_dataset_dir(ref) / self.metadata_marker + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self.private_dataset_dir(ref) / self.private_metadata_marker + + @abstractmethod + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: ... + + @abstractmethod + def read(self, path: Path, canonical_name: str) -> dict: ... + + @abstractmethod + def write(self, path: Path, obj: MigratableObject) -> None: ... diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py new file mode 100644 index 00000000000..d2bb91920aa --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py @@ -0,0 +1,52 @@ +from pathlib import Path +from typing import Iterator + +import yaml +from syft_migration import MigratableObject + +from ..config import is_protocol_dir_name +from ..dataset_ref import DatasetRef +from .base import ProtocolCodec + + +class ProtocolCodecV0(ProtocolCodec): + """Pre-versioning layout (<= 0.1.20): datasets flat under syft_datasets/, no identity fields.""" + + version = "0" + protocol_versions = ["0"] + + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_mock_dataset_dir( + ref.name, ref.owner, ref.protocol_version + ) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_private_dataset_dir( + ref.owner, ref.name, ref.protocol_version + ) + + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + root = self.config.public_datasets_root_for_datasite(datasite_email) + if not root.exists(): + return + for entry in sorted(p for p in root.iterdir() if p.is_dir()): + # v/ segments belong to a versioned codec, not the flat layout. + if is_protocol_dir_name(entry.name): + continue + if (entry / self.metadata_marker).exists(): + yield DatasetRef(datasite_email, entry.name, "0") + + def read(self, path: Path, canonical_name: str) -> dict: + data = yaml.safe_load(path.read_text()) or {} + # Protocol-0 files predate the identity fields; they are all version 1. + data.setdefault("canonical_name", canonical_name) + data.setdefault("version", "1") + return data + + def write(self, path: Path, obj: MigratableObject) -> None: + data = obj.disk_dict() + # Byte-match the pre-versioning (<= 0.1.20) on-disk format. + data.pop("canonical_name", None) + data.pop("version", None) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py new file mode 100644 index 00000000000..1c3119a38c3 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Iterator + +import yaml +from syft_migration import MigratableObject + +from ..config import protocol_dir_name +from ..dataset_ref import DatasetRef +from .base import ProtocolCodec + + +class ProtocolCodecV1(ProtocolCodec): + """Versioned layout (>= 0.1.21): datasets under syft_datasets/v/, identity fields on disk.""" + + version = "1" + protocol_versions = ["1"] + + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_mock_dataset_dir( + ref.name, ref.owner, ref.protocol_version + ) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.config.get_private_dataset_dir( + ref.owner, ref.name, ref.protocol_version + ) + + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + root = self.config.public_datasets_root_for_datasite(datasite_email) + if not root.exists(): + return + for protocol_version in self.protocol_versions: + version_dir = root / protocol_dir_name(protocol_version) + if not version_dir.is_dir(): + continue + for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): + if (entry / self.metadata_marker).exists(): + yield DatasetRef(datasite_email, entry.name, protocol_version) + + def read(self, path: Path, canonical_name: str) -> dict: + # Files already carry canonical_name/version on disk. + return yaml.safe_load(path.read_text()) or {} + + def write(self, path: Path, obj: MigratableObject) -> None: + data = obj.disk_dict() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) diff --git a/packages/syft-datasets/src/syft_datasets/version.py b/packages/syft-datasets/src/syft_datasets/version.py new file mode 100644 index 00000000000..d2941ebc140 --- /dev/null +++ b/packages/syft-datasets/src/syft_datasets/version.py @@ -0,0 +1,10 @@ +from importlib.metadata import version + +# Distribution name from pyproject ([project].name). The import package is +# "syft_datasets" but the distribution is "syft-dataset" (singular), so this is +# hardcoded rather than derived from __package__. +PACKAGE_NAME = "syft-dataset" + +# Derived from the installed distribution metadata (pyproject.toml's version) +# so it cannot drift from the released package version. +__version__ = version(PACKAGE_NAME) diff --git a/packages/syft-datasets/tests/migrations/__init__.py b/packages/syft-datasets/tests/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/p2p/__init__.py b/packages/syft-datasets/tests/migrations/p2p/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv new file mode 100644 index 00000000000..f5c2e52a5a1 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private.csv @@ -0,0 +1,3 @@ +id,secret +1,alpha +2,beta diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml new file mode 100644 index 00000000000..e30855a6492 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/private_metadata.yaml @@ -0,0 +1,2 @@ +uid: 4362bc0f-000f-44db-88c0-be508aa814fa +data_dir: '' diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml new file mode 100644 index 00000000000..c362f440602 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/private/syft_datasets/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - do@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml new file mode 100644 index 00000000000..96f5fcf8ef3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/dataset.yaml @@ -0,0 +1,13 @@ +uid: 4362bc0f-000f-44db-88c0-be508aa814fa +created_at: '2026-07-15T12:54:05.124932Z' +updated_at: '2026-07-15T12:54:05.125289Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/demo +private_url: syft://do@test.org/private/syft_datasets/demo +readme_url: syft://do@test.org/public/syft_datasets/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv new file mode 100644 index 00000000000..ea6be9205d3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/mock.csv @@ -0,0 +1,3 @@ +id,value +1,10 +2,20 diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md new file mode 100644 index 00000000000..c5b7a1f37d7 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/readme.md @@ -0,0 +1 @@ +# demo dataset diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml new file mode 100644 index 00000000000..676a3c8f2e3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.20-protocol0_syftbox/do@test.org/public/syft_datasets/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - ds@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv new file mode 100644 index 00000000000..f5c2e52a5a1 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv @@ -0,0 +1,3 @@ +id,secret +1,alpha +2,beta diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml new file mode 100644 index 00000000000..9362a9fd6cf --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml @@ -0,0 +1,4 @@ +canonical_name: PrivateDatasetConfig +version: '1' +uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 +data_dir: '' diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml new file mode 100644 index 00000000000..c362f440602 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - do@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml new file mode 100644 index 00000000000..503efd923a4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml @@ -0,0 +1,15 @@ +canonical_name: Dataset +version: '1' +uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 +created_at: '2026-07-15T12:54:04.399452Z' +updated_at: '2026-07-15T12:54:04.399619Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/v1/demo +private_url: syft://do@test.org/private/syft_datasets/v1/demo +readme_url: syft://do@test.org/public/syft_datasets/v1/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/v1/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv new file mode 100644 index 00000000000..ea6be9205d3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv @@ -0,0 +1,3 @@ +id,value +1,10 +2,20 diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md new file mode 100644 index 00000000000..c5b7a1f37d7 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md @@ -0,0 +1 @@ +# demo dataset diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml new file mode 100644 index 00000000000..676a3c8f2e3 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml @@ -0,0 +1,8 @@ +rules: +- access: + admin: [] + read: + - ds@test.org + write: [] + pattern: '**' +terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py new file mode 100644 index 00000000000..b8573eb74b9 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -0,0 +1,109 @@ +"""End-to-end dataset flow through SyftDatasetManager. + +Datasets are broadcast (one public copy read by the whole audience), so by +default they are written in the widest-compatible (oldest) protocol so every +current peer can read them; the new v layout is written only for peers that +advertise support for it. +""" + +from pathlib import Path + +import yaml +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.migrations import dataset_registry + +DO_EMAIL = "do@test.org" +DS0 = "old@test.org" +DS1 = "new@test.org" + + +def _src(tmp_path: Path) -> tuple[Path, Path, Path]: + src = tmp_path / "src" + src.mkdir() + mock = src / "mock.csv" + mock.write_text("id,value\n1,10\n") + private = src / "private.csv" + private.write_text("id,secret\n1,x\n") + readme = src / "readme.md" + readme.write_text("# demo\n") + return mock, private, readme + + +def _manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + if peer_schemas is not None: + mgr.storage.peer_schemas = peer_schemas + return mgr + + +def test_default_create_writes_protocol_0(tmp_path: Path): + mgr = _manager(tmp_path) + mock, private, readme = _src(tmp_path) + + dataset = mgr.create( + name="demo", mock_path=mock, private_path=private, readme_path=readme + ) + # Flat layout, no v, no identity fields (byte-compatible with 0.1.20). + assert dataset.mock_dir.parent.name == "syft_datasets" + raw = yaml.safe_load((dataset.mock_dir / "dataset.yaml").read_text()) + assert "canonical_name" not in raw + + got = mgr.get("demo") + assert got.name == "demo" + assert got.version == dataset_registry.latest_version("Dataset") + assert [p.name for p in got.mock_files] == ["mock.csv"] + assert [p.name for p in got.private_files] == ["private.csv"] + + +def test_create_for_protocol1_peer_writes_v1(tmp_path: Path): + schema1 = dataset_registry.schema_for_protocol_version("1") + mgr = _manager(tmp_path, peer_schemas={DS1: schema1}) + mock, private, readme = _src(tmp_path) + + dataset = mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + users=[DS1], + ) + # Versioned layout, identity fields present. + assert dataset.mock_dir.parent.name == "v1" + raw = yaml.safe_load((dataset.mock_dir / "dataset.yaml").read_text()) + assert raw["canonical_name"] == "Dataset" and raw["version"] == "1" + + # The shared peer has read access to the (v1) mock dir. + perm = dataset.mock_dir / "syft.pub.yaml" + assert perm.exists() + assert DS1 in perm.read_text() + + got = mgr.get("demo") + assert got.name == "demo" and got._protocol_version == "1" + + +def test_multi_version_write_for_mixed_audience(tmp_path: Path): + schema0 = dataset_registry.schema_for_protocol_version("0") + schema1 = dataset_registry.schema_for_protocol_version("1") + mgr = _manager(tmp_path, peer_schemas={DS0: schema0, DS1: schema1}) + mock, private, readme = _src(tmp_path) + + mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + users=[DS0, DS1], + ) + + root = mgr.syftbox_config.public_datasets_root_for_datasite(DO_EMAIL) + # Both layouts written: flat (protocol 0) and v1 (protocol 1). + assert (root / "demo" / "dataset.yaml").exists() + assert (root / "v1" / "demo" / "dataset.yaml").exists() + + # get_all() dedupes the two on-disk copies to one, preferring the newest. + all_datasets = mgr.get_all() + assert len(all_datasets) == 1 + assert all_datasets[0]._protocol_version == "1" + assert mgr.get("demo")._protocol_version == "1" diff --git a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py new file mode 100644 index 00000000000..5840603ba5f --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -0,0 +1,122 @@ +"""Reading/writing datasets against every released syft-dataset on-disk layout. + +Each fixture tree under fixtures/ is named +``syft_dataset--protocol

_syftbox`` and holds a SyftBox exactly as +that release wrote a dataset to disk (a ``demo`` dataset: mock files + readme + +dataset.yaml, plus a private_metadata.yaml). The current code must still read, +upgrade and byte-exactly round-trip all of them. + +Protocol 0 is the last release (0.1.20) without a v path segment or +canonical_name/version fields; protocol >= 1 stores datasets under a v +segment and writes those identity fields. Fixtures are generated by +scripts/generate_release_fixture.py. +""" + +import re +import shutil +from pathlib import Path + +import pytest +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations import dataset_registry + +DO_EMAIL = "do@test.org" +DATASET = "demo" + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +RELEASE_FIXTURES = sorted(FIXTURES_DIR.glob("syft_dataset-*-protocol*_syftbox")) + +released_syftbox_fixtures = pytest.mark.parametrize( + "fixture", RELEASE_FIXTURES, ids=lambda f: f.name +) + + +def _protocol_of(fixture: Path) -> str: + return re.search(r"-protocol(\d+)_syftbox$", fixture.name).group(1) + + +def _has_identity(protocol: str) -> bool: + """canonical_name/version are written only from protocol 1 onwards.""" + return protocol != "0" + + +def _syftbox_w_old_datasets(fixture: Path, tmp_path: Path) -> Path: + syftbox = tmp_path / "SyftBox" + shutil.copytree(fixture, syftbox) + return syftbox + + +def _manager(syftbox: Path) -> SyftDatasetManager: + return SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) + + +def _storage(syftbox: Path) -> DatasetStorage: + return DatasetStorage(config=SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL)) + + +@released_syftbox_fixtures +def test_old_datasets_load_and_upgrade(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + mgr = _manager(syftbox) + + datasets = mgr.get_all() + assert {d.name for d in datasets} == {DATASET} + dataset = datasets[0] + # Loads into the latest registered version in memory. + assert dataset.version == dataset_registry.latest_version("Dataset") + assert dataset.owner == DO_EMAIL + assert dataset._protocol_version == _protocol_of(fixture) + # Private config also loads/upgrades to latest. + ref = mgr.storage.find_dataset_ref(DO_EMAIL, DATASET) + private_config = mgr.storage.read_private_config(ref) + assert private_config.version == dataset_registry.latest_version( + "PrivateDatasetConfig" + ) + + +@released_syftbox_fixtures +def test_write_back_is_byte_exact(fixture: Path, tmp_path: Path): + protocol = _protocol_of(fixture) + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + storage = _storage(syftbox) + + ref = storage.find_dataset_ref(DO_EMAIL, DATASET) + metadata_path = storage.metadata_path(ref) + original_bytes = metadata_path.read_text() + + # Read (upgrade in memory) then write back in that release's layout/format. + dataset = storage.read_dataset(ref) + storage.write_dataset(ref, dataset) + + assert metadata_path.read_text() == original_bytes + raw = metadata_path.read_text() + assert ("canonical_name" in raw) is _has_identity(protocol) + + +@released_syftbox_fixtures +def test_mixed_protocol_listing(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + mgr = _manager(syftbox) + + # A newly created dataset (default protocol) coexists with the old one. + src = tmp_path / "src" + src.mkdir() + mock = src / "mock.csv" + mock.write_text("id\n1\n") + private = src / "private.csv" + private.write_text("id\n1\n") + mgr.create(name="fresh", mock_path=mock, private_path=private) + + assert {d.name for d in mgr.get_all()} == {DATASET, "fresh"} + + +@released_syftbox_fixtures +def test_dataset_reprs_do_not_error(fixture: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(fixture, tmp_path) + datasets = _manager(syftbox).get_all() + + for render in (repr, str, lambda o: o._repr_html_()): + assert render(datasets) + assert render(datasets[0]) diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py new file mode 100644 index 00000000000..556e16d69f4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py @@ -0,0 +1,143 @@ +"""Structure, coverage, and behavior of the ProtocolCodecs behind DatasetStorage. + +A codec is selected by PROTOCOL version and may serve several of them +(``protocol_versions``); it also carries its own ``version``. The invariants +here are about protocol-version coverage: every protocol version the registry +understands is handled by exactly one codec. +""" + +import importlib +import inspect +import pkgutil +from pathlib import Path + +import yaml +from syft_datasets import protocolcodecs +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_storage import DatasetRef, DatasetStorage +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_datasets.models import Dataset +from syft_datasets.protocolcodecs import ProtocolCodec + +DO_EMAIL = "do@test.org" + + +def _storage(tmp_path: Path) -> DatasetStorage: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) + return DatasetStorage(config=config) + + +def _mock_dataset(storage: DatasetStorage, ref: DatasetRef) -> Dataset: + cfg = storage.config + return Dataset( + name=ref.name, + mock_url=cfg.get_mock_url_for_my_dataset(ref.name, ref.protocol_version), + private_url=cfg.get_private_url_for_my_dataset(ref.name, ref.protocol_version), + ) + + +# -- structural invariant ------------------------------------------------------ +def test_codec_versions_unique_and_protocol_versions_disjoint(tmp_path: Path): + codecs = _storage(tmp_path).codecs + + codec_versions = [c.version for c in codecs] + assert len(codec_versions) == len(set(codec_versions)), ( + "codec versions must be unique" + ) + + seen: set[str] = set() + for codec in codecs: + for protocol_version in codec.protocol_versions: + assert protocol_version not in seen, ( + f"protocol {protocol_version} handled by more than one codec" + ) + seen.add(protocol_version) + + # The current protocol is handled by the last (newest) codec. + assert DATASET_PROTOCOL_VERSION in codecs[-1].protocol_versions + + +def _all_concrete_codec_classes() -> set[type[ProtocolCodec]]: + for module_info in pkgutil.iter_modules(protocolcodecs.__path__): + importlib.import_module(f"{protocolcodecs.__name__}.{module_info.name}") + + def descendants(cls: type) -> set[type]: + subs = set(cls.__subclasses__()) + return subs.union(*(descendants(s) for s in subs)) + + return {cls for cls in descendants(ProtocolCodec) if not inspect.isabstract(cls)} + + +# -- registration -------------------------------------------------------------- +def test_all_codecs_in_codebase_are_registered_in_dataset_storage(tmp_path: Path): + registered_codecs = {type(codec) for codec in _storage(tmp_path).codecs} + defined_codecs = _all_concrete_codec_classes() + + missing = defined_codecs - registered_codecs + assert not missing, ( + f"codec(s) defined but not registered in DatasetStorage/CODECS: " + f"{sorted(cls.__name__ for cls in missing)}" + ) + + +# -- coverage ------------------------------------------------------------------ +def test_every_known_protocol_version_is_covered_by_exactly_one_codec(tmp_path: Path): + storage = _storage(tmp_path) + + known = set(dataset_registry.protocol_version_history) | {DATASET_PROTOCOL_VERSION} + covered = {pv for codec in storage.codecs for pv in codec.protocol_versions} + assert covered == known + + for protocol_version in known: + assert storage._codec_for(protocol_version) is not None + + +# -- behavior 1: on-disk format per codec -------------------------------------- +def test_v0_writes_flat_no_identity_v1_nests_with_identity(tmp_path: Path): + storage = _storage(tmp_path) + + ref0 = DatasetRef(DO_EMAIL, "flat", "0") + ref1 = DatasetRef(DO_EMAIL, "nested", "1") + p0 = storage.write_dataset(ref0, _mock_dataset(storage, ref0)) + p1 = storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + + # protocol 0: public/syft_datasets//dataset.yaml (no v), identity stripped + assert p0.parent.parent.name == "syft_datasets" + raw0 = yaml.safe_load(p0.read_text()) + assert "canonical_name" not in raw0 and "version" not in raw0 + + # protocol 1: public/syft_datasets/v1//dataset.yaml, identity present + assert p1.parent.parent.name == "v1" + raw1 = yaml.safe_load(p1.read_text()) + assert raw1["canonical_name"] == "Dataset" and raw1["version"] == "1" + + for ref in (ref0, ref1): + loaded = storage.read_dataset(ref) + assert loaded.name == ref.name + assert loaded.version == dataset_registry.latest_version("Dataset") + + +# -- behavior 2: each codec scans only its own layout -------------------------- +def test_scan_partitions_by_layout(tmp_path: Path): + storage = _storage(tmp_path) + v0_codec, v1_codec = storage._codec_for("0"), storage._codec_for("1") + + ref0 = DatasetRef(DO_EMAIL, "flat", "0") + ref1 = DatasetRef(DO_EMAIL, "nested", "1") + storage.write_dataset(ref0, _mock_dataset(storage, ref0)) + storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + + v0_refs = list(v0_codec.iter_dataset_refs(DO_EMAIL)) + v1_refs = list(v1_codec.iter_dataset_refs(DO_EMAIL)) + assert [(r.name, r.protocol_version) for r in v0_refs] == [("flat", "0")] + assert [(r.name, r.protocol_version) for r in v1_refs] == [("nested", "1")] + + all_refs = list(storage.iter_dataset_refs(DO_EMAIL)) + assert {(r.name, r.protocol_version) for r in all_refs} == { + ("flat", "0"), + ("nested", "1"), + } + assert len(all_refs) == 2 diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py new file mode 100644 index 00000000000..34b3a437ea4 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_sanity_checks.py @@ -0,0 +1,63 @@ +"""Sanity checks around the protocol-versioned dataset layout.""" + +from pathlib import Path + +import pytest +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_migration import MigrationError + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + + +def _storage(tmp_path: Path, peer_schemas=None) -> DatasetStorage: + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) + return DatasetStorage(config=config, peer_schemas=peer_schemas) + + +def test_reserved_dataset_name_rejected(tmp_path: Path): + storage = _storage(tmp_path) + with pytest.raises(ValueError, match="reserved"): + storage.validate_dataset_name("v2") + # Names that merely resemble the segment are fine. + storage.validate_dataset_name("v2x") + storage.validate_dataset_name("version1") + + +def test_negotiated_protocol_version_for_peer(tmp_path: Path): + protocol0_schema = dataset_registry.schema_for_protocol_version("0") + storage = _storage(tmp_path, peer_schemas={DS_EMAIL: protocol0_schema}) + + # An older peer negotiates down to the version both sides can read. + assert storage.negotiated_protocol_version_for_peer(DS_EMAIL) == "0" + with pytest.raises(MigrationError): + storage.negotiated_protocol_version_for_peer("stranger@test.org") + # Opting out assumes the current protocol. + assert ( + storage.negotiated_protocol_version_for_peer( + "stranger@test.org", raise_on_unknown=False + ) + == DATASET_PROTOCOL_VERSION + ) + + +def test_target_protocol_versions_for_peers(tmp_path: Path): + schema0 = dataset_registry.schema_for_protocol_version("0") + schema1 = dataset_registry.schema_for_protocol_version("1") + storage = _storage( + tmp_path, peer_schemas={"old@test.org": schema0, "new@test.org": schema1} + ) + + # No audience -> widest-compatible (oldest) protocol. + assert storage.target_protocol_versions_for_peers() == {"0"} + # Unknown peer -> also widest-compatible. + assert storage.target_protocol_versions_for_peers(["stranger@test.org"]) == {"0"} + # Mixed audience -> a copy per distinct version. + assert storage.target_protocol_versions_for_peers( + ["old@test.org", "new@test.org"] + ) == {"0", "1"} diff --git a/packages/syft-datasets/tests/migrations/unit/__init__.py b/packages/syft-datasets/tests/migrations/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml b/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml new file mode 100644 index 00000000000..7666a9aabd6 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/fixtures/Dataset/v1.yaml @@ -0,0 +1,15 @@ +canonical_name: Dataset +version: '1' +uid: 00000000-0000-0000-0000-000000000001 +created_at: '2026-07-01T00:00:00Z' +updated_at: '2026-07-01T00:00:00Z' +name: demo +summary: demo dataset +tags: +- demo +location: null +mock_url: syft://do@test.org/public/syft_datasets/demo +private_url: syft://do@test.org/private/syft_datasets/demo +readme_url: syft://do@test.org/public/syft_datasets/demo/readme.md +mock_files_urls: +- syft://do@test.org/public/syft_datasets/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml b/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml new file mode 100644 index 00000000000..d6575f6a9fd --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/fixtures/PrivateDatasetConfig/v1.yaml @@ -0,0 +1,4 @@ +canonical_name: PrivateDatasetConfig +version: '1' +uid: 00000000-0000-0000-0000-000000000001 +data_dir: . diff --git a/packages/syft-datasets/tests/migrations/unit/mocks.py b/packages/syft-datasets/tests/migrations/unit/mocks.py new file mode 100644 index 00000000000..c994191729d --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/mocks.py @@ -0,0 +1,35 @@ +"""Shared mock objects for the syft-dataset migration tests.""" + +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 +from syft_datasets.url import SyftBoxURL + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" +DATASET_NAME = "demo" +DATASET_UID = UUID("00000000-0000-0000-0000-000000000001") + + +def create_mock_dataset() -> DatasetV1: + base = f"syft://{DO_EMAIL}/public/syft_datasets/{DATASET_NAME}" + return DatasetV1( + uid=DATASET_UID, + created_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 7, 1, tzinfo=timezone.utc), + name=DATASET_NAME, + summary="demo dataset", + tags=["demo"], + mock_url=SyftBoxURL(base), + private_url=SyftBoxURL( + f"syft://{DO_EMAIL}/private/syft_datasets/{DATASET_NAME}" + ), + readme_url=SyftBoxURL(f"{base}/readme.md"), + mock_files_urls=[SyftBoxURL(f"{base}/mock.csv")], + ) + + +def create_mock_private_config() -> PrivateDatasetConfigV1: + return PrivateDatasetConfigV1(uid=DATASET_UID, data_dir=Path("")) diff --git a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py new file mode 100644 index 00000000000..82c816dd21e --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py @@ -0,0 +1,100 @@ +"""The hardcoded release artifacts of past syft-dataset releases.""" + +from syft_migration import ( + MigrationService, + ReleasedPackageProtocolInfo, + ReleasedProtocol, +) + +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft_datasets.models import DatasetV1 + + +def test_0_1_20_artifact_file_loads(): + artifact = ReleasedPackageProtocolInfo.load( + PACKAGE_ARTIFACTS_DIR / "syft-dataset-0.1.20.json" + ) + assert artifact.package_info.package_name == "syft-dataset" + assert artifact.package_info.version == "0.1.20" + assert artifact.package_info.protocol_version == "0" + assert artifact.protocol_schema.protocol_name == "syft-dataset" + assert artifact.protocol_schema.version == "0" + expected_versions = {"Dataset": ["1"], "PrivateDatasetConfig": ["1"]} + assert artifact.protocol_schema.supported_versions == expected_versions + + +def test_all_released_protocols_load(): + protocol_paths = sorted(PROTOCOLS_DIR.glob("*.json")) + assert protocol_paths # at least protocol-0.json exists + + for path in protocol_paths: + released = ReleasedProtocol.load(path) + schema = released.protocol_schema + # The filename encodes the protocol version: protocol-.json. + assert path.name == f"protocol-{schema.version}.json" + assert schema.protocol_name == "syft-dataset" + assert schema.supported_versions + assert set(schema.current_object_schemas) == set(schema.supported_versions) + for canonical_name in schema.supported_versions: + assert schema.current_schema(canonical_name) + + +def test_protocol_0_released_protocol_loads(): + released = ReleasedProtocol.load(PROTOCOLS_DIR / "protocol-0.json") + schema = released.protocol_schema + assert schema.version == "0" + assert set(schema.current_object_schemas) == {"Dataset", "PrivateDatasetConfig"} + assert schema.current_schema("Dataset") == "1" + assert schema.current_schema("PrivateDatasetConfig") == "1" + + +def test_released_object_schemas_unchanged(): + # Released object versions are frozen forever; see the failure message. + drift = dataset_registry.find_schema_drift() + assert drift == [], ( + f"Released object schemas changed: {drift} " + "(canonical_name, object_version, protocol_version).\n" + "A class that shipped in a released protocol was modified in place. " + "Released versions are frozen forever, because peers on old releases " + "still read/write them. To fix:\n" + " 1. Revert your change to the released class (e.g. DatasetV1).\n" + " 2. Create the next version instead: copy the class into " + "models//v.py with version='' and apply your change " + "there.\n" + " 3. Point the current-version alias in models//__init__.py " + "at the new class.\n" + " 4. Register migrations in BOTH directions (v -> v and back) " + "so old peers stay supported; test_upgrade_paths enforces this.\n" + " 5. Add a serialized fixture: tests/migrations/unit/fixtures/" + "/v.yaml.\n" + " 6. Bump DATASET_PROTOCOL_VERSION in syft_datasets/migrations/" + "registry.py — a new object version is a protocol change.\n" + "If NO class was edited and this failure appeared after a pydantic " + "upgrade, model_json_schema() output changed cosmetically: review the " + "diff carefully and regenerate the files under migrations/history/." + ) + + +def test_protocol_bumped_when_changed(): + # Fires when the dataset protocol changes (e.g. a new object version registers) + # without bumping DATASET_PROTOCOL_VERSION. + assert not dataset_registry.protocol_changed_without_bump() + + +def test_historic_schemas_registered_on_import(): + # syft_datasets/__init__ registers every artifact in migrations/history/. + assert dataset_registry.package_version_history["0"].version == "0.1.20" + schema = dataset_registry.schema_for_protocol_version("0") + assert schema.current_schema("Dataset") == "1" + assert schema.current_schema("PrivateDatasetConfig") == "1" + + +def test_downgrade_for_last_released_protocol_version(): + service = MigrationService(registry=dataset_registry) + from .mocks import create_mock_dataset + + dataset = create_mock_dataset() + downgraded = service.downgrade_for_protocol_version(dataset, "0") + assert isinstance(downgraded, DatasetV1) + assert downgraded.name == "demo" diff --git a/packages/syft-datasets/tests/migrations/unit/test_migrations.py b/packages/syft-datasets/tests/migrations/unit/test_migrations.py new file mode 100644 index 00000000000..6201667e87c --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_migrations.py @@ -0,0 +1,72 @@ +"""Migrations against the REAL dataset registry, using on-disk fixtures. + +Every registered version of each object has a serialized fixture under +``fixtures//v.yaml``; each fixture loads and upgrades +in memory to the latest version, and the latest version downgrades to every +registered version and round-trips through disk. With only V1 registered these +are no-ops; when a newer version is registered, these tests fail until a fixture +for it is added and then exercise the real migration paths automatically. +""" + +from pathlib import Path + +import yaml +from syft_migration import MigrationService + +from syft_datasets.migrations import dataset_registry + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +def _load_fixture(canonical_name: str, version: str) -> dict: + path = FIXTURES_DIR / canonical_name / f"v{version}.yaml" + assert path.exists(), ( + f"Missing fixture {path}; add one for every registered version" + ) + return yaml.safe_load(path.read_text()) + + +def _assert_all_upgrade_to_latest(canonical_name: str): + service = MigrationService(registry=dataset_registry) + latest = dataset_registry.latest_version(canonical_name) + latest_cls = dataset_registry.get_class(canonical_name, latest) + + for version in dataset_registry.versions(canonical_name): + upgraded = service.load( + _load_fixture(canonical_name, version), target_version=latest + ) + assert type(upgraded) is latest_cls + + +def _assert_latest_downgrades_to_all(canonical_name: str, tmp_path: Path): + service = MigrationService(registry=dataset_registry) + latest = dataset_registry.latest_version(canonical_name) + newest = service.load(_load_fixture(canonical_name, latest)) + + for version in dataset_registry.versions(canonical_name): + version_cls = dataset_registry.get_class(canonical_name, version) + downgraded = service.migrate(newest, target_version=version) + assert type(downgraded) is version_cls + + path = tmp_path / f"{canonical_name}_v{version}.yaml" + path.write_text(yaml.safe_dump(downgraded.disk_dict())) + reloaded = service.load( + yaml.safe_load(path.read_text()), target_version=version + ) + assert reloaded == downgraded + + +def test_all_dataset_versions_upgrade_from_disk_to_latest(): + _assert_all_upgrade_to_latest("Dataset") + + +def test_latest_dataset_downgrades_to_all_versions(tmp_path: Path): + _assert_latest_downgrades_to_all("Dataset", tmp_path) + + +def test_all_private_config_versions_upgrade_from_disk_to_latest(): + _assert_all_upgrade_to_latest("PrivateDatasetConfig") + + +def test_latest_private_config_downgrades_to_all_versions(tmp_path: Path): + _assert_latest_downgrades_to_all("PrivateDatasetConfig", tmp_path) diff --git a/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py b/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py new file mode 100644 index 00000000000..5a8d7a12392 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_mock_migrations.py @@ -0,0 +1,107 @@ +"""Upgrading and downgrading dataset objects across versions. + +Simulates the NEXT release: V2 objects and migrations, in test scope only. +""" + +import yaml +from syft_migration import MigrationRegistry, MigrationService + +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 + +from .mocks import create_mock_dataset, create_mock_private_config + + +def _register_dataset_v2(registry: MigrationRegistry) -> type: + class DatasetV2(DatasetV1, registry=registry): + version: str = "2" + license: str = "unknown" # new in v2 + + registry.register_migration( + canonical_name="Dataset", + from_version="1", + to_version="2", + fn=lambda obj: DatasetV2(**obj.model_dump(exclude={"version"})), + ) + registry.register_migration( + canonical_name="Dataset", + from_version="2", + to_version="1", + fn=lambda obj: DatasetV1(**obj.model_dump(exclude={"version", "license"})), + ) + return DatasetV2 + + +def _register_private_config_v2(registry: MigrationRegistry) -> type: + class PrivateDatasetConfigV2(PrivateDatasetConfigV1, registry=registry): + version: str = "2" + checksum: str = "" # new in v2 + + registry.register_migration( + canonical_name="PrivateDatasetConfig", + from_version="1", + to_version="2", + fn=lambda obj: PrivateDatasetConfigV2(**obj.model_dump(exclude={"version"})), + ) + registry.register_migration( + canonical_name="PrivateDatasetConfig", + from_version="2", + to_version="1", + fn=lambda obj: PrivateDatasetConfigV1( + **obj.model_dump(exclude={"version", "checksum"}) + ), + ) + return PrivateDatasetConfigV2 + + +def _version_registry_with_migrations() -> tuple[MigrationRegistry, type, type]: + """A fresh registry holding the current objects plus test-scope V2 versions.""" + registry = MigrationRegistry( + protocol_name="syft-dataset", + package_name="syft-dataset", + package_version="test-next-release", + protocol_version="test-next", + ) + registry.register_object_version(DatasetV1) + registry.register_object_version(PrivateDatasetConfigV1) + dataset_v2 = _register_dataset_v2(registry) + private_config_v2 = _register_private_config_v2(registry) + return registry, dataset_v2, private_config_v2 + + +def test_dataset_upgrades_from_disk(tmp_path): + registry, DatasetV2, _ = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + # An old (v1) dataset file on disk loads and migrates to the next version. + path = tmp_path / "dataset.yaml" + path.write_text(yaml.safe_dump(create_mock_dataset().disk_dict())) + upgraded = service.load(yaml.safe_load(path.read_text()), target_version="2") + assert type(upgraded) is DatasetV2 + assert upgraded.name == "demo" + assert upgraded.license == "unknown" + + +def test_dataset_downgrades(): + registry, DatasetV2, _ = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + v2_only = DatasetV2( + **create_mock_dataset().model_dump(exclude={"version"}), license="mit" + ) + downgraded = service.migrate(v2_only, target_version="1") + assert type(downgraded) is DatasetV1 + assert downgraded.name == "demo" + assert not hasattr(downgraded, "license") + + +def test_private_config_upgrades_and_downgrades(): + registry, _, PrivateDatasetConfigV2 = _version_registry_with_migrations() + service = MigrationService(registry=registry) + + upgraded = service.migrate(create_mock_private_config(), target_version="2") + assert type(upgraded) is PrivateDatasetConfigV2 + assert upgraded.checksum == "" + + downgraded = service.migrate(upgraded, target_version="1") + assert type(downgraded) is PrivateDatasetConfigV1 + assert not hasattr(downgraded, "checksum") diff --git a/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py b/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py new file mode 100644 index 00000000000..bf7a12eb019 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_objects_registered.py @@ -0,0 +1,58 @@ +"""Every versioned syft-dataset object is known to the package registry.""" + +import importlib +import pkgutil + +from syft_migration import MigratableObject + +import syft_datasets +from syft_datasets.migrations import dataset_registry +from syft_datasets.models import ( + Dataset, + DatasetV1, + PrivateDatasetConfig, + PrivateDatasetConfigV1, +) + + +def test_versioned_objects_registered_and_aliased(): + # Both objects have at least one version registered in the package registry. + assert dataset_registry.versions("Dataset") + assert dataset_registry.versions("PrivateDatasetConfig") + + # The current-version aliases resolve to the V1 classes. + assert Dataset is DatasetV1 + assert PrivateDatasetConfig is PrivateDatasetConfigV1 + + # The protocol schema covers both objects and resolves a current version. + schema = dataset_registry.compute_protocol_schema() + assert {"Dataset", "PrivateDatasetConfig"} <= set(schema.supported_versions) + assert schema.current_schema(canonical_name="Dataset") + assert schema.current_schema(canonical_name="PrivateDatasetConfig") + + +def _all_subclasses(cls: type) -> set[type]: + subclasses = set(cls.__subclasses__()) + for sub in cls.__subclasses__(): + subclasses |= _all_subclasses(sub) + return subclasses + + +def test_all_migratable_objects_in_package_are_registered(): + # Import every syft_datasets module so all MigratableObject subclasses are defined. + for module_info in pkgutil.walk_packages( + syft_datasets.__path__, prefix="syft_datasets." + ): + importlib.import_module(module_info.name) + + package_objects = [ + cls + for cls in _all_subclasses(MigratableObject) + if cls.__module__.startswith("syft_datasets.") + ] + assert len(package_objects) >= 2 # the scan actually found the dataset objects + + for cls in package_objects: + canonical_name = cls.model_fields["canonical_name"].default + version = cls.model_fields["version"].default + assert dataset_registry.get_class(canonical_name, version) is cls diff --git a/packages/syft-datasets/tests/migrations/unit/test_serialization.py b/packages/syft-datasets/tests/migrations/unit/test_serialization.py new file mode 100644 index 00000000000..cbee4d50032 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_serialization.py @@ -0,0 +1,40 @@ +"""Versioned syft-dataset objects serialize with their identity and load back.""" + +import yaml +from syft_migration import MigrationService + +from syft_datasets.migrations import dataset_registry +from syft_datasets.models import DatasetV1, PrivateDatasetConfigV1 + +from .mocks import create_mock_dataset, create_mock_private_config + + +def test_dataset_serialization_roundtrip(): + dataset = create_mock_dataset() + data = yaml.safe_load(yaml.safe_dump(dataset.disk_dict())) + loaded = DatasetV1(**data) + assert loaded == dataset + assert loaded.canonical_name == "Dataset" + assert loaded.version == "1" + + +def test_private_config_serialization_roundtrip(): + config = create_mock_private_config() + data = yaml.safe_load(yaml.safe_dump(config.disk_dict())) + loaded = PrivateDatasetConfigV1(**data) + assert loaded == config + assert loaded.canonical_name == "PrivateDatasetConfig" + assert loaded.version == "1" + + +def test_migration_service_loads_into_versioned_class(): + service = MigrationService(registry=dataset_registry) + + dataset = create_mock_dataset() + loaded = service.load(dataset.disk_dict()) + assert isinstance(loaded, DatasetV1) + assert loaded.name == "demo" + + config = create_mock_private_config() + loaded_config = service.load(config.disk_dict()) + assert isinstance(loaded_config, PrivateDatasetConfigV1) diff --git a/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py b/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py new file mode 100644 index 00000000000..06baccba6d0 --- /dev/null +++ b/packages/syft-datasets/tests/migrations/unit/test_upgrade_paths.py @@ -0,0 +1,28 @@ +"""Every registered object version can migrate up to latest and down to any lower.""" + +from syft_datasets.migrations import dataset_registry + + +def test_every_version_has_upgrade_path_to_latest(): + assert dataset_registry.objects # sanity: the registry is populated + + for canonical_name, versions in dataset_registry.objects.items(): + for version in versions: + assert dataset_registry.has_upgradeable_path_to_latest( + canonical_name=canonical_name, from_version=version + ), f"No upgrade path for {canonical_name!r} v{version} to latest" + + +def test_every_version_has_downgrade_path_to_all_lower_versions(): + assert dataset_registry.objects # sanity: the registry is populated + + for canonical_name, versions in dataset_registry.objects.items(): + for higher in versions: + for lower in versions: + if lower >= higher: + continue + assert dataset_registry.has_migration_path( + canonical_name=canonical_name, + from_version=higher, + to_version=lower, + ), f"No downgrade path for {canonical_name!r} v{higher} to v{lower}" diff --git a/uv.lock b/uv.lock index 4156afcaf5a..cfb24ccb349 100644 --- a/uv.lock +++ b/uv.lock @@ -4620,10 +4620,11 @@ wheels = [ [[package]] name = "syft-dataset" -version = "0.1.20" +version = "0.1.21" source = { editable = "packages/syft-datasets" } dependencies = [ { name = "pyyaml" }, + { name = "syft-migration" }, { name = "syft-notebook-ui" }, { name = "syft-perms" }, ] @@ -4636,6 +4637,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "syft-migration", editable = "packages/syft-migration" }, { name = "syft-notebook-ui", editable = "packages/syft-notebook-ui" }, { name = "syft-perms", editable = "packages/syft-perms" }, ] From 066d617b11928057321eb372a8a06f899f215fff Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 15 Jul 2026 16:28:14 +0200 Subject: [PATCH 2/8] fix: resolve dataset private dir at its actual on-disk protocol layout The enclave inference example computed private dataset dirs using the default protocol version (v1 segment), but datasets shared with 'any' are written at the widest-compatible protocol (0, no segment). This mismatch broke the log-sink path and the ensure_logs_dataset idempotency check. Resolve the actual on-disk ref via DatasetStorage, falling back to the widest-compatible protocol for datasets not yet synced. --- .../src/enclave_model_api/logs_dataset.py | 14 ++++++++---- .../src/enclave_model_api/paths.py | 22 ++++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py b/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py index 8a001fc94cd..422c8c29f08 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/logs_dataset.py @@ -10,7 +10,10 @@ import tempfile from pathlib import Path +from syft_datasets.dataset_ref import DatasetNotFoundError + from enclave_model_api.log_writer import LOG_FILE_NAME +from enclave_model_api.paths import resolve_private_dataset_dir LOGS_DATASET_SUMMARY = ( "Inference request logs (prompt, completion, stats) collected by the " @@ -41,9 +44,12 @@ def ensure_logs_dataset(client, name: str) -> Path: Idempotent so reboots with fresh_state=false keep the existing logs. """ - config = client.datasets.syftbox_config - if (config.get_my_mock_dataset_dir(name) / "dataset.yaml").exists(): - return config.private_dir_for_my_dataset(name) + storage = client.datasets.storage + email = storage.config.email + try: + return storage.private_dataset_dir(storage.find_dataset_ref(email, name)) + except DatasetNotFoundError: + pass mock_dir, private_dir = _seed_dirs() client.create_dataset( @@ -56,4 +62,4 @@ def ensure_logs_dataset(client, name: str) -> Path: sync=False, ) client.sync() - return config.private_dir_for_my_dataset(name) + return resolve_private_dataset_dir(storage, email, name) diff --git a/packages/enclave-model-api-example/src/enclave_model_api/paths.py b/packages/enclave-model-api-example/src/enclave_model_api/paths.py index 7f3eea44856..499609e1d72 100644 --- a/packages/enclave-model-api-example/src/enclave_model_api/paths.py +++ b/packages/enclave-model-api-example/src/enclave_model_api/paths.py @@ -9,6 +9,8 @@ from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_ref import DatasetNotFoundError, DatasetRef +from syft_datasets.dataset_storage import DatasetStorage def default_syftbox_folder(email: str) -> Path: @@ -16,12 +18,30 @@ def default_syftbox_folder(email: str) -> Path: return get_jupyter_default_syftbox_folder(email) +def resolve_private_dataset_dir(storage: DatasetStorage, owner: str, name: str) -> Path: + """Private dir at the dataset's actual on-disk protocol layout. + + A dataset may live at protocol 0 (flat) or under a ``v`` segment; the + written layout depends on what the audience can read, not the current + default. Datasets not yet on disk (e.g. weights still syncing) fall back to + the widest-compatible protocol — where a peer running any current release + writes them for us. + """ + try: + ref = storage.find_dataset_ref(owner, name) + except DatasetNotFoundError: + (widest,) = storage.target_protocol_versions_for_peers(None) + ref = DatasetRef(owner=owner, name=name, protocol_version=widest) + return storage.private_dataset_dir(ref) + + def private_dataset_dir( syftbox_folder: Path | str, datasite: str, dataset_name: str ) -> Path: """Private dir of *dataset_name* on *datasite* inside *syftbox_folder*.""" config = SyftBoxConfig(syftbox_folder=Path(syftbox_folder), email=datasite) - return config.private_dir_for_my_dataset(dataset_name) + storage = DatasetStorage(config=config) + return resolve_private_dataset_dir(storage, datasite, dataset_name) def find_checkpoint_dir(weights_dir: Path | str) -> Path | None: From 5adfc2d96c586e165805e9ffe42c5aa4593e1e1f Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 20 Jul 2026 11:32:05 +0200 Subject: [PATCH 3/8] chore: drop premature release artifacts for unreleased 0.1.21 package-artifacts/ and protocols/ hold released history only. Only 0.1.20 (protocol 0) shipped to PyPI, so the frozen 0.1.21/protocol-1 artifacts were premature. The unreleased protocol 1 already lives in the live registry and will be frozen by export_release_artifact.py at release time (mirrors syft-job). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../syft-dataset-0.1.21.json | 149 ------------------ .../history/protocols/protocol-1.json | 144 ----------------- 2 files changed, 293 deletions(-) delete mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json delete mode 100644 packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json b/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json deleted file mode 100644 index dbcf520d6a7..00000000000 --- a/packages/syft-datasets/src/syft_datasets/migrations/history/package-artifacts/syft-dataset-0.1.21.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "package_info": { - "package_name": "syft-dataset", - "version": "0.1.21", - "protocol_version": "1" - }, - "protocol_schema": { - "protocol_name": "syft-dataset", - "version": "1", - "supported_versions": { - "PrivateDatasetConfig": ["1"], - "Dataset": ["1"] - }, - "current_object_schemas": { - "PrivateDatasetConfig": { - "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", - "properties": { - "canonical_name": { - "default": "PrivateDatasetConfig", - "title": "Canonical Name", - "type": "string" - }, - "version": { - "default": "1", - "title": "Version", - "type": "string" - }, - "uid": { - "format": "uuid", - "title": "Uid", - "type": "string" - }, - "data_dir": { - "format": "path", - "title": "Data Dir", - "type": "string" - } - }, - "required": ["uid", "data_dir"], - "title": "PrivateDatasetConfigV1", - "type": "object" - }, - "Dataset": { - "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", - "properties": { - "canonical_name": { - "default": "Dataset", - "title": "Canonical Name", - "type": "string" - }, - "version": { - "default": "1", - "title": "Version", - "type": "string" - }, - "uid": { - "format": "uuid", - "title": "Uid", - "type": "string" - }, - "created_at": { - "format": "date-time", - "title": "Created At", - "type": "string" - }, - "updated_at": { - "format": "date-time", - "title": "Updated At", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "summary": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Summary" - }, - "tags": { - "default": [], - "items": { - "type": "string" - }, - "title": "Tags", - "type": "array" - }, - "location": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Location" - }, - "mock_url": { - "description": "A SyftBox URL", - "format": "uri", - "title": "Mock Url", - "type": "string" - }, - "private_url": { - "description": "A SyftBox URL", - "format": "uri", - "title": "Private Url", - "type": "string" - }, - "readme_url": { - "anyOf": [ - { - "description": "A SyftBox URL", - "format": "uri", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Readme Url" - }, - "mock_files_urls": { - "items": { - "description": "A SyftBox URL", - "format": "uri", - "type": "string" - }, - "title": "Mock Files Urls", - "type": "array" - } - }, - "required": ["name", "mock_url", "private_url"], - "title": "DatasetV1", - "type": "object" - } - } - } -} diff --git a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json b/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json deleted file mode 100644 index b81ab76a29b..00000000000 --- a/packages/syft-datasets/src/syft_datasets/migrations/history/protocols/protocol-1.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "protocol_schema": { - "protocol_name": "syft-dataset", - "version": "1", - "supported_versions": { - "PrivateDatasetConfig": ["1"], - "Dataset": ["1"] - }, - "current_object_schemas": { - "PrivateDatasetConfig": { - "description": "Private dataset metadata, stored as private_metadata.yaml outside the sync folder.", - "properties": { - "canonical_name": { - "default": "PrivateDatasetConfig", - "title": "Canonical Name", - "type": "string" - }, - "version": { - "default": "1", - "title": "Version", - "type": "string" - }, - "uid": { - "format": "uuid", - "title": "Uid", - "type": "string" - }, - "data_dir": { - "format": "path", - "title": "Data Dir", - "type": "string" - } - }, - "required": ["uid", "data_dir"], - "title": "PrivateDatasetConfigV1", - "type": "object" - }, - "Dataset": { - "description": "Public dataset metadata, stored as dataset.yaml under\nSyftBox//public/syft_datasets/[v/]/.", - "properties": { - "canonical_name": { - "default": "Dataset", - "title": "Canonical Name", - "type": "string" - }, - "version": { - "default": "1", - "title": "Version", - "type": "string" - }, - "uid": { - "format": "uuid", - "title": "Uid", - "type": "string" - }, - "created_at": { - "format": "date-time", - "title": "Created At", - "type": "string" - }, - "updated_at": { - "format": "date-time", - "title": "Updated At", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "summary": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Summary" - }, - "tags": { - "default": [], - "items": { - "type": "string" - }, - "title": "Tags", - "type": "array" - }, - "location": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Location" - }, - "mock_url": { - "description": "A SyftBox URL", - "format": "uri", - "title": "Mock Url", - "type": "string" - }, - "private_url": { - "description": "A SyftBox URL", - "format": "uri", - "title": "Private Url", - "type": "string" - }, - "readme_url": { - "anyOf": [ - { - "description": "A SyftBox URL", - "format": "uri", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Readme Url" - }, - "mock_files_urls": { - "items": { - "description": "A SyftBox URL", - "format": "uri", - "type": "string" - }, - "title": "Mock Files Urls", - "type": "array" - } - }, - "required": ["name", "mock_url", "private_url"], - "title": "DatasetV1", - "type": "object" - } - } - } -} From 53eb6e6e6cac8635f4bd7ea54e52e97155068808 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 20 Jul 2026 13:52:45 +0200 Subject: [PATCH 4/8] test: rename dataset test helpers for clarity Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migrations/p2p/test_current_protocol.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py index b8573eb74b9..eb3fe6ce169 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -17,7 +17,7 @@ DS1 = "new@test.org" -def _src(tmp_path: Path) -> tuple[Path, Path, Path]: +def _create_dataset_files(tmp_path: Path) -> tuple[Path, Path, Path]: src = tmp_path / "src" src.mkdir() mock = src / "mock.csv" @@ -29,7 +29,7 @@ def _src(tmp_path: Path) -> tuple[Path, Path, Path]: return mock, private, readme -def _manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: +def _dataset_manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: syftbox = tmp_path / "SyftBox" syftbox.mkdir() mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) @@ -39,8 +39,8 @@ def _manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: def test_default_create_writes_protocol_0(tmp_path: Path): - mgr = _manager(tmp_path) - mock, private, readme = _src(tmp_path) + mgr = _dataset_manager(tmp_path) + mock, private, readme = _create_dataset_files(tmp_path) dataset = mgr.create( name="demo", mock_path=mock, private_path=private, readme_path=readme @@ -59,8 +59,8 @@ def test_default_create_writes_protocol_0(tmp_path: Path): def test_create_for_protocol1_peer_writes_v1(tmp_path: Path): schema1 = dataset_registry.schema_for_protocol_version("1") - mgr = _manager(tmp_path, peer_schemas={DS1: schema1}) - mock, private, readme = _src(tmp_path) + mgr = _dataset_manager(tmp_path, peer_schemas={DS1: schema1}) + mock, private, readme = _create_dataset_files(tmp_path) dataset = mgr.create( name="demo", @@ -86,8 +86,8 @@ def test_create_for_protocol1_peer_writes_v1(tmp_path: Path): def test_multi_version_write_for_mixed_audience(tmp_path: Path): schema0 = dataset_registry.schema_for_protocol_version("0") schema1 = dataset_registry.schema_for_protocol_version("1") - mgr = _manager(tmp_path, peer_schemas={DS0: schema0, DS1: schema1}) - mock, private, readme = _src(tmp_path) + mgr = _dataset_manager(tmp_path, peer_schemas={DS0: schema0, DS1: schema1}) + mock, private, readme = _create_dataset_files(tmp_path) mgr.create( name="demo", From 93844c9b52494516a8cbbfb6e9cf650667214a42 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 21 Jul 2026 13:57:11 +0200 Subject: [PATCH 5/8] refactor: move dataset write/migrate orchestration into DatasetStorage DatasetStorage now owns materializing a dataset on disk (the multi-version broadcast write loop + file copying), so the client layer no longer needs to know per-version on-disk layout. - Add DatasetSourceFiles (unversioned DTO) as the create input; storage builds the per-version Dataset objects (URLs are layout-dependent, so a caller can't hand in a finished Dataset). - Add DatasetStorage.create_dataset (multi-version write, single shared identity across copies) and migrate_dataset (rewrite an existing dataset into another protocol layout, preserving uid/timestamps), sharing a private _materialize_version core. File-copy helpers move into storage as _copy_mock_data/_copy_private_data/_copy_readme; copy_dir_contents gains exclude_names. - SyftDatasetManager.create delegates to storage and applies permissions per written copy; add a migrate() wrapper; fix users bare-string char-split. - Add v0->v1 migrate round-trip test; byte-exact protocol-0 round-trip preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/syft-datasets/pyproject.toml | 2 +- .../syft-datasets/src/syft_datasets/config.py | 132 +------- .../src/syft_datasets/dataset.py | 15 - .../src/syft_datasets/dataset_manager.py | 266 ++++----------- .../src/syft_datasets/dataset_storage.py | 306 +++++++++++++++++- .../src/syft_datasets/file_utils.py | 10 +- .../src/syft_datasets/models/dataset/v1.py | 21 +- .../syft_datasets/protocolcodecs/__init__.py | 24 +- .../src/syft_datasets/protocolcodecs/base.py | 53 +-- .../src/syft_datasets/protocolcodecs/v0.py | 57 +++- .../src/syft_datasets/protocolcodecs/v1.py | 67 +++- .../private/syft_datasets/v1/demo/private.csv | 3 - .../v1/demo/private_metadata.yaml | 4 - .../syft_datasets/v1/demo/syft.pub.yaml | 8 - .../public/syft_datasets/v1/demo/dataset.yaml | 15 - .../public/syft_datasets/v1/demo/mock.csv | 3 - .../public/syft_datasets/v1/demo/readme.md | 1 - .../syft_datasets/v1/demo/syft.pub.yaml | 8 - .../migrations/p2p/test_current_protocol.py | 75 ++++- .../p2p/test_older_protocol_compatibility.py | 40 +-- .../migrations/p2p/test_protocol_codecs.py | 35 +- .../migrations/unit/test_history_artifacts.py | 37 +-- syft_client/utils.py | 2 +- .../with_unit_coverage/test_sync_manager.py | 2 +- tests/unit/test_create_dataset_cleanup.py | 10 +- tests/unit/test_sync_manager.py | 2 +- uv.lock | 2 +- 27 files changed, 679 insertions(+), 521 deletions(-) delete mode 100644 packages/syft-datasets/src/syft_datasets/dataset.py delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md delete mode 100644 packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml diff --git a/packages/syft-datasets/pyproject.toml b/packages/syft-datasets/pyproject.toml index 8ed83693736..a4a49fea3ac 100644 --- a/packages/syft-datasets/pyproject.toml +++ b/packages/syft-datasets/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "syft-dataset" -version = "0.1.21" +version = "0.1.20" description = "Syft Datasets" authors = [{ name = "OpenMined", email = "info@openmined.org" }] license = { text = "Apache-2.0" } diff --git a/packages/syft-datasets/src/syft_datasets/config.py b/packages/syft-datasets/src/syft_datasets/config.py index 52674ebcce1..e06c2f8b195 100644 --- a/packages/syft-datasets/src/syft_datasets/config.py +++ b/packages/syft-datasets/src/syft_datasets/config.py @@ -3,8 +3,6 @@ from pydantic import BaseModel, Field from pathlib import Path -from .url import SyftBoxURL -from .migrations.registry import DATASET_PROTOCOL_VERSION SYFT_DATASETS_FOLDER_NAME = "syft_datasets" METADATA_FILENAME = "dataset.yaml" @@ -12,6 +10,8 @@ # Datasets under public/syft_datasets/ and private/syft_datasets/ live inside a # protocol-version segment ("v1", "v2", ...); protocol 0 (<= 0.1.20) had none. +# The naming convention is shared; the per-protocol layout that uses it lives in +# the codecs' DatasetConfig. PROTOCOL_DIR_RE = re.compile(r"^v\d+$") @@ -25,6 +25,13 @@ def protocol_dir_name(protocol_version: str) -> Optional[str]: class SyftBoxConfig(BaseModel): + """Environment: which SyftBox and whose datasite. Protocol-agnostic. + + Dataset on-disk layout (folders, v segments, filenames) is *not* here; it + is versioned per protocol and owned by the codecs' DatasetConfig. This only + knows the SyftBox tree, which is stable across dataset protocol versions. + """ + syftbox_folder: Path = Field( ..., description="Path to the SyftBox folder on the local filesystem." ) @@ -38,119 +45,10 @@ def private_dir(self) -> Path: def public_dir(self) -> Path: return self.syftbox_folder / self.email / "public" - def public_datasets_root_for_datasite(self, datasite: str) -> Path: - """The syft_datasets scan root for a datasite (no protocol segment).""" - return self.syftbox_folder / datasite / "public" / SYFT_DATASETS_FOLDER_NAME - - def private_datasets_root(self) -> Path: - """The private syft_datasets scan root (no protocol segment).""" - return self.private_dir / SYFT_DATASETS_FOLDER_NAME - - def private_datasets_root_for_owner(self, owner: str) -> Path: - """The private syft_datasets scan root under an owner's datasite.""" - return self.syftbox_folder / owner / "private" / SYFT_DATASETS_FOLDER_NAME - - def get_private_dataset_dir( - self, - owner: str, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - return self._with_segment( - self.private_datasets_root_for_owner(owner), protocol_version, dataset_name - ) - - def public_datasets_dir_for_datasite(self, datasite: str) -> Path: - # Backwards-compatible accessor; ensures the scan root exists. - dir = self.public_datasets_root_for_datasite(datasite) - dir.mkdir(parents=True, exist_ok=True) - return dir - - def _with_segment(self, root: Path, protocol_version: str, name: str) -> Path: - segment = protocol_dir_name(protocol_version) - return (root / segment / name) if segment else (root / name) - - def private_dir_for_my_dataset( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - return self._with_segment( - self.private_datasets_root(), protocol_version, dataset_name - ) - - def get_my_mock_dataset_dir( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - return self.get_mock_dataset_dir( - dataset_name=dataset_name, - datasite=self.email, - protocol_version=protocol_version, - ) - - def get_mock_dataset_dir( - self, - dataset_name: str, - datasite: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - return self._with_segment( - self.public_datasets_root_for_datasite(datasite), - protocol_version, - dataset_name, - ) - - def get_mock_url_for_my_dataset( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> SyftBoxURL: - return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name, protocol_version), - syftbox_folder=self.syftbox_folder, - ) - - def get_private_url_for_my_dataset( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> SyftBoxURL: - return SyftBoxURL.from_path( - path=self.private_dir_for_my_dataset(dataset_name, protocol_version), - syftbox_folder=self.syftbox_folder, - ) - - def get_readme_url_for_my_dataset( - self, - dataset_name: str, - readme_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> SyftBoxURL: - return SyftBoxURL.from_path( - path=self.get_my_mock_dataset_dir(dataset_name, protocol_version) - / readme_name, - syftbox_folder=self.syftbox_folder, - ) - - def public_metadata_filename_for_my_dataset( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - # TODO: not sure why the absolute is needed here - return ( - self.get_my_mock_dataset_dir(dataset_name, protocol_version) - / METADATA_FILENAME - ).absolute() + def datasite_public_root(self, datasite: str) -> Path: + """The public/ root under a datasite (no dataset layout applied).""" + return self.syftbox_folder / datasite / "public" - def private_metadata_filename_for_my_dataset( - self, - dataset_name: str, - protocol_version: str = DATASET_PROTOCOL_VERSION, - ) -> Path: - return ( - self.private_dir_for_my_dataset(dataset_name, protocol_version) - / PRIVATE_METADATA_FILENAME - ) + def datasite_private_root(self, datasite: str) -> Path: + """The private/ root under a datasite (no dataset layout applied).""" + return self.syftbox_folder / datasite / "private" diff --git a/packages/syft-datasets/src/syft_datasets/dataset.py b/packages/syft-datasets/src/syft_datasets/dataset.py deleted file mode 100644 index 20a94ffa999..00000000000 --- a/packages/syft-datasets/src/syft_datasets/dataset.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Backwards-compatible re-exports. - -The dataset models now live under ``syft_datasets.models`` as versioned -``MigratableObject``s. This module keeps the historical import paths -(``from syft_datasets.dataset import Dataset, PrivateDatasetConfig``) working. -""" - -from .models import Dataset, DatasetV1, PrivateDatasetConfig, PrivateDatasetConfigV1 - -__all__ = [ - "Dataset", - "DatasetV1", - "PrivateDatasetConfig", - "PrivateDatasetConfigV1", -] diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index 2060eb6fde4..ef43d6dd818 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -1,7 +1,5 @@ import re -import shutil from pathlib import Path -from uuid import UUID, uuid4 from typing_extensions import Self import yaml @@ -10,14 +8,12 @@ from syft_notebook_ui.types import TableList from typing_extensions import Literal -from syft_datasets.dataset import Dataset, PrivateDatasetConfig +from syft_datasets.models import Dataset from syft_datasets.dataset_ref import DatasetRef -from syft_datasets.dataset_storage import DatasetStorage -from syft_datasets.file_utils import copy_dir_contents, copy_paths, is_empty_dir +from syft_datasets.dataset_storage import DatasetSourceFiles, DatasetStorage -from .config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig +from .config import PRIVATE_METADATA_FILENAME, SyftBoxConfig from .permissions import set_mock_dataset_permissions, set_private_dataset_permissions -from .url import SyftBoxURL DATASET_COLLECTION_PREFIX = "syft_datasetcollection" PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" @@ -47,83 +43,6 @@ def _validate_dataset_name(self, dataset_name: str) -> None: # Reject names that collide with a protocol-version directory (v1, v2, ...). self.storage.validate_dataset_name(dataset_name) - def _prepare_mock_data(self, target_mock_dir: Path, src_path: Path) -> list[Path]: - # Validate src data - if not src_path.exists(): - raise FileNotFoundError(f"Could not find mock data at {src_path}") - - if (src_path / METADATA_FILENAME).exists(): - raise ValueError( - f"Mock data at {src_path} contains reserved file {METADATA_FILENAME}. Please rename it and try again." - ) - - # Validate dir we're making on Syftbox - if target_mock_dir.exists() and not is_empty_dir(target_mock_dir): - raise FileExistsError( - f"Mock dir {target_mock_dir} already exists and is not empty." - ) - target_mock_dir.mkdir(parents=True, exist_ok=True) - - copied_files = [] - if src_path.is_dir(): - copied_files = copy_dir_contents( - src=src_path, - dst=target_mock_dir, - exists_ok=True, - ) - elif src_path.is_file(): - copied_files = copy_paths( - files=[src_path], - dst=target_mock_dir, - exists_ok=True, - ) - else: - raise ValueError( - f"Mock data path {src_path} must be an existing file or directory." - ) - - return copied_files - - def _prepare_private_data(self, private_dir: Path, src_path: Path) -> list[Path]: - private_dir.mkdir(parents=True, exist_ok=True) - - copied_files = [] - if src_path.is_dir(): - # TODO: Implementing without copying private data to `SyftBox/private`` - copied_files = copy_dir_contents( - src=src_path, - dst=private_dir, - exists_ok=True, - ) - elif src_path.is_file(): - copied_files = copy_paths( - files=[src_path], - dst=private_dir, - exists_ok=True, - ) - else: - raise ValueError( - f"Private data path {src_path} must be an existing file or directory." - ) - - return copied_files - - def _prepare_readme( - self, target_mock_dir: Path, src_file: Path | None - ) -> list[Path]: - copied_files = [] - if src_file is not None: - if not src_file.is_file(): - raise FileNotFoundError(f"Could not find README at {src_file}") - if not src_file.suffix.lower() == ".md": - raise ValueError("readme file must be a markdown (.md) file.") - copied_files = copy_paths( - files=[src_file], - dst=target_mock_dir, - exists_ok=True, - ) - return copied_files - def create( self, name: str, @@ -152,111 +71,50 @@ def create( Returns: Dataset: The created Dataset object (the newest protocol version written). """ - # Reject names that collide with a protocol-version directory (v1, v2, ...). - # (The stricter _validate_dataset_name regex is intentionally not enforced - # here to preserve historical behavior, e.g. names with spaces.) - self.storage.validate_dataset_name(name) - mock_path = to_path(mock_path) - private_path = to_path(private_path) - readme_path = to_path(readme_path) if readme_path else None - tags = tags or [] - - target_versions = self._target_protocol_versions(users) - dataset_uid = uuid4() - - created: dict[str, Dataset] = {} - for protocol_version in sorted(target_versions, key=int): - created[protocol_version] = self._create_one_version( - name=name, - protocol_version=protocol_version, - dataset_uid=dataset_uid, - mock_path=mock_path, - private_path=private_path, - readme_path=readme_path, - summary=summary, - location=location, - tags=tags, - users=users, - ) - - # Return the newest protocol version written (richest layout). - newest = max(created, key=int) - return created[newest] - - def _target_protocol_versions(self, users: list[str] | str | None) -> set[str]: - """The protocol versions to write so the audience can read the dataset.""" - peer_emails = None if users in (None, SHARE_WITH_ANY) else list(users) - return self.storage.target_protocol_versions_for_peers(peer_emails) - - def _create_one_version( - self, - name: str, - protocol_version: str, - dataset_uid: UUID, - mock_path: Path, - private_path: Path, - readme_path: Path | None, - summary: str | None, - location: str | None, - tags: list[str], - users: list[str] | str | None, - ) -> Dataset: - ref = self.storage.new_dataset_ref(name, protocol_version) - - mock_url = self.syftbox_config.get_mock_url_for_my_dataset( - name, protocol_version + source = DatasetSourceFiles( + mock=to_path(mock_path), + private=to_path(private_path), + readme=to_path(readme_path) if readme_path else None, ) - readme_url = ( - self.syftbox_config.get_readme_url_for_my_dataset( - name, readme_path.name, protocol_version - ) - if readme_path - else None - ) - private_url = self.syftbox_config.get_private_url_for_my_dataset( - name, protocol_version - ) - - target_mock_dir = self.storage.public_dataset_dir(ref) - all_mock_file_paths = self._prepare_mock_data(target_mock_dir, mock_path) - readme_files = self._prepare_readme(target_mock_dir, readme_path) - - public_metadata_path = target_mock_dir / METADATA_FILENAME - mock_file_paths = [ - f - for f in all_mock_file_paths - if f != public_metadata_path and f not in readme_files - ] - mock_files_urls = [ - SyftBoxURL.from_path( - path=file_path, syftbox_folder=self.syftbox_config.syftbox_folder - ) - for file_path in mock_file_paths - ] - - private_dir = self.storage.private_dataset_dir(ref) - self._prepare_private_data(private_dir, private_path) - - dataset = Dataset( - uid=dataset_uid, + created = self.storage.create_dataset( name=name, - mock_url=mock_url, - private_url=private_url, - readme_url=readme_url, + source=source, summary=summary, location=location, tags=tags, - mock_files_urls=mock_files_urls, + peer_emails=self._peer_emails(users), ) - dataset._syftbox_config = self.syftbox_config - dataset._protocol_version = protocol_version + for dataset in created.values(): + self._set_new_dataset_permissions(dataset=dataset, users=users) + # Return the newest protocol version written (richest layout). + return created[max(created, key=int)] - self.storage.write_dataset(ref, dataset) - self.storage.write_private_config( - ref, PrivateDatasetConfig(uid=dataset_uid, data_dir=private_dir) - ) - self._set_new_dataset_permissions(dataset=dataset, users=users) - return dataset + def migrate( + self, + name: str, + to_version: str, + users: list[str] | str | None = None, + ) -> Dataset: + """Rewrite an owned dataset into another protocol layout, re-applying permissions. + + Storage copies the files + writes metadata for the new layout; the manager + re-applies read permissions. The audience (``users``) must be supplied by + the caller as it is on create — granted readers are not recoverable from + disk via the permissions API. + """ + ref = self.storage.find_dataset_ref(self.syftbox_config.email, name) + migrated = self.storage.migrate_dataset(ref, to_version) + self._set_new_dataset_permissions(dataset=migrated, users=users) + return migrated + + @staticmethod + def _peer_emails(users: list[str] | str | None) -> list[str] | None: + """Audience emails for protocol negotiation; None means no/any peers.""" + if users is None or users == SHARE_WITH_ANY: + return None + if isinstance(users, str): + return [users] + return list(users) def _set_new_dataset_permissions( self, dataset: Dataset, users: list[str] | str | None @@ -322,9 +180,7 @@ def _repr_html_(self) -> str: return dataset_manager_repr_html(self.get_all()) - def _datasites_to_check(self, datasite: str | None) -> list[str]: - if datasite: - return [datasite] + def _all_syftbox_datasites(self) -> list[str]: syftbox_folder = self.syftbox_config.syftbox_folder # All directories with "@" in the name are peer/owner email directories return [ @@ -339,24 +195,22 @@ def get_all( order_by: str | None = None, sort_order: Literal["asc", "desc"] = "asc", ) -> list[Dataset]: - # A dataset may exist on disk in more than one protocol layout; keep only - # the newest protocol per (owner, name). - best_ref: dict[tuple[str, str], DatasetRef] = {} - for datasite in self._datasites_to_check(datasite): - for ref in self.storage.iter_dataset_refs(datasite): - key = (ref.owner, ref.name) - current = best_ref.get(key) - if current is None or int(ref.protocol_version) > int( - current.protocol_version - ): - best_ref[key] = ref + # storage.iter_dataset_refs already yields one ref per dataset in its + # preferred protocol layout, so there is nothing to dedupe here. + datasites = ( + [datasite] if datasite is not None else self._all_syftbox_datasites() + ) all_datasets = [] - for ref in best_ref.values(): - try: - all_datasets.append(self.storage.read_dataset(ref)) - except Exception: - continue + for datasite in datasites: + for ref in self.storage.iter_dataset_refs(datasite): + try: + all_datasets.append(self.storage.read_dataset(ref)) + except Exception: + print( + f"Error reading dataset {ref.name} from {ref.owner}, skipping", + ) + continue if order_by is not None: all_datasets.sort( @@ -418,11 +272,8 @@ def delete( print("Dataset deletion cancelled.") return - # Delete the dataset directories - if dataset.mock_dir.exists(): - shutil.rmtree(dataset.mock_dir) - if dataset._private_metadata_dir.exists(): - shutil.rmtree(dataset._private_metadata_dir) + # Remove every on-disk copy (all protocol versions) via the storage layer. + self.storage.delete_dataset(datasite, name) def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: """Get private dataset files as {path_in_datasite: content}. @@ -463,7 +314,4 @@ def _private_config_without_data_dir(self, ref: DatasetRef) -> bytes: config = self.storage.read_private_config(ref) config.data_dir = Path("") data = config.disk_dict() - if ref.protocol_version == "0": - data.pop("canonical_name", None) - data.pop("version", None) return yaml.safe_dump(data, indent=2, sort_keys=False).encode() diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py index 9ee96dbb515..31528bbedea 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_storage.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -1,5 +1,9 @@ +import shutil +from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path from typing import Iterator, Optional +from uuid import UUID, uuid4 from syft_migration import ( MigratableObject, @@ -8,20 +12,67 @@ MigrationService, ProtocolSchema, ) +from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME -from .config import SyftBoxConfig, is_protocol_dir_name +from .config import ( + METADATA_FILENAME, + PRIVATE_METADATA_FILENAME, + SyftBoxConfig, + is_protocol_dir_name, +) from .dataset_ref import DatasetNotFoundError, DatasetRef, PrivateConfigNotFoundError +from .file_utils import copy_dir_contents, copy_paths, is_empty_dir from .migrations.registry import DATASET_PROTOCOL_VERSION, dataset_registry from .models import Dataset, PrivateDatasetConfig from .protocolcodecs import CODECS, ProtocolCodec +from .url import SyftBoxURL __all__ = [ "DatasetRef", "DatasetNotFoundError", "PrivateConfigNotFoundError", "DatasetStorage", + "DatasetSourceFiles", ] +# On-disk files that are part of the dataset representation, not payload data; +# skipped when re-copying an existing dataset into a new layout during migration. +_NON_PAYLOAD_FILES = frozenset( + {METADATA_FILENAME, PRIVATE_METADATA_FILENAME, PERMISSION_FILE_NAME} +) + + +def _utcnow() -> datetime: + return datetime.now(tz=timezone.utc) + + +@dataclass(frozen=True) +class DatasetSourceFiles: + """Source paths a caller hands to ``create_dataset`` (ephemeral input). + + Unversioned by design: these paths never persist. The on-disk layout is + versioned by the codecs' DatasetConfig, and the persisted file manifest rides + the versioned Dataset (``mock_files_urls``). A new file category is just a new + field here plus a new Dataset version + codec layout entry. + """ + + mock: Path # file or directory + private: Path # file or directory + readme: Optional[Path] = None # .md file + + +@dataclass(frozen=True) +class _DatasetFields: + """Layout-free dataset fields, built once and shared across every version written.""" + + uid: UUID + created_at: datetime + updated_at: datetime + name: str + summary: Optional[str] = None + location: Optional[str] = None + tags: list[str] = field(default_factory=list) + class DatasetStorage: """Dataset filesystem IO and path resolution, delegating disk layout to codecs. @@ -122,6 +173,193 @@ def new_dataset_ref(self, name: str, protocol_version: str) -> DatasetRef: owner=self.config.email, name=name, protocol_version=protocol_version ) + # -- create / migrate ---------------------------------------------------- + def create_dataset( + self, + *, + name: str, + source: DatasetSourceFiles, + uid: Optional[UUID] = None, + summary: Optional[str] = None, + location: Optional[str] = None, + tags: Optional[list[str]] = None, + peer_emails: Optional[list[str]] = None, + ) -> dict[str, Dataset]: + """Write a new dataset in every protocol version its audience can read. + + Copies the source files into each version's on-disk layout and writes the + metadata/private config. Returns {protocol_version: written Dataset}. With + no/unknown peers, writes only the widest-compatible protocol. + """ + self.validate_dataset_name(name) + if source.mock.is_dir() and (source.mock / METADATA_FILENAME).exists(): + raise ValueError( + f"Mock data at {source.mock} contains reserved file " + f"{METADATA_FILENAME}. Please rename it and try again." + ) + now = _utcnow() + fields = _DatasetFields( + uid=uid or uuid4(), + created_at=now, + updated_at=now, + name=name, + summary=summary, + location=location, + tags=tags or [], + ) + created: dict[str, Dataset] = {} + for protocol_version in sorted( + self.target_protocol_versions_for_peers(peer_emails), key=int + ): + ref = self.new_dataset_ref(name, protocol_version) + created[protocol_version] = self._materialize_version(ref, fields, source) + return created + + def migrate_dataset(self, ref: DatasetRef, target_protocol_version: str) -> Dataset: + """Rewrite an existing on-disk dataset into another protocol layout. + + Copies the source layout's files into the target layout and writes the + metadata/private config there, preserving identity (uid/timestamps). + Does not delete the source copy. Owner-only. + """ + if ref.owner != self.config.email: + raise ValueError("Can only migrate datasets you own.") + old = self.read_dataset(ref) + source = DatasetSourceFiles( + mock=self.public_dataset_dir(ref), + private=self.private_dataset_dir(ref), + readme=old.readme_path, + ) + fields = _DatasetFields( + uid=old.uid, + created_at=old.created_at, + updated_at=old.updated_at, + name=old.name, + summary=old.summary, + location=old.location, + tags=old.tags, + ) + target_ref = self.new_dataset_ref(ref.name, target_protocol_version) + return self._materialize_version( + target_ref, fields, source, exclude_names=_NON_PAYLOAD_FILES + ) + + def _materialize_version( + self, + ref: DatasetRef, + fields: _DatasetFields, + source: DatasetSourceFiles, + exclude_names: frozenset[str] = frozenset(), + ) -> Dataset: + """Copy files + write metadata/config for a single protocol version.""" + target_mock_dir = self.public_dataset_dir(ref) + private_dir = self.private_dataset_dir(ref) + + all_mock_file_paths = self._copy_mock_data(ref, source.mock, exclude_names) + readme_files = self._copy_readme(ref, source.readme) + + # mock_files_urls is the copied mock payload minus the metadata file and + # the readme (which peers read via their own URL fields). + public_metadata_path = target_mock_dir / METADATA_FILENAME + mock_file_paths = [ + f + for f in all_mock_file_paths + if f != public_metadata_path and f not in readme_files + ] + + dataset = Dataset( + uid=fields.uid, + created_at=fields.created_at, + updated_at=fields.updated_at, + name=fields.name, + mock_url=self._url_for(target_mock_dir), + private_url=self._url_for(private_dir), + readme_url=( + self._url_for(target_mock_dir / source.readme.name) + if source.readme + else None + ), + summary=fields.summary, + location=fields.location, + tags=fields.tags, + mock_files_urls=[self._url_for(f) for f in mock_file_paths], + ) + dataset._syftbox_config = self.config + dataset._protocol_version = ref.protocol_version + + self._copy_private_data(ref, source.private, exclude_names) + + self.write_dataset_metadata(ref, dataset) + self.write_private_config( + ref, PrivateDatasetConfig(uid=fields.uid, data_dir=private_dir) + ) + return dataset + + def _url_for(self, path: Path) -> SyftBoxURL: + return SyftBoxURL.from_path( + path=path, syftbox_folder=self.config.syftbox_folder + ) + + def _copy_mock_data( + self, + ref: DatasetRef, + src_path: Path, + exclude_names: frozenset[str] = frozenset(), + ) -> list[Path]: + target_dir = self.public_dataset_dir(ref) + if not src_path.exists(): + raise FileNotFoundError(f"Could not find mock data at {src_path}") + if target_dir.exists() and not is_empty_dir(target_dir): + raise FileExistsError( + f"Mock dir {target_dir} already exists and is not empty." + ) + target_dir.mkdir(parents=True, exist_ok=True) + if src_path.is_dir(): + return copy_dir_contents( + src=src_path, + dst=target_dir, + exists_ok=True, + exclude_names=exclude_names, + ) + if src_path.is_file(): + return copy_paths(files=[src_path], dst=target_dir, exists_ok=True) + raise ValueError( + f"Mock data path {src_path} must be an existing file or directory." + ) + + def _copy_private_data( + self, + ref: DatasetRef, + src_path: Path, + exclude_names: frozenset[str] = frozenset(), + ) -> list[Path]: + private_target_dir = self.private_dataset_dir(ref) + private_target_dir.mkdir(parents=True, exist_ok=True) + if src_path.is_dir(): + # TODO: Implementing without copying private data to `SyftBox/private`` + return copy_dir_contents( + src=src_path, + dst=private_target_dir, + exists_ok=True, + exclude_names=exclude_names, + ) + if src_path.is_file(): + return copy_paths(files=[src_path], dst=private_target_dir, exists_ok=True) + raise ValueError( + f"Private data path {src_path} must be an existing file or directory." + ) + + def _copy_readme(self, ref: DatasetRef, src_file: Optional[Path]) -> list[Path]: + if src_file is None: + return [] + if not src_file.is_file(): + raise FileNotFoundError(f"Could not find README at {src_file}") + if not src_file.suffix.lower() == ".md": + raise ValueError("readme file must be a markdown (.md) file.") + return copy_paths( + files=[src_file], dst=self.public_dataset_dir(ref), exists_ok=True + ) + # -- naming -------------------------------------------------------------- @staticmethod def validate_dataset_name(dataset_name: str) -> None: @@ -143,20 +381,66 @@ def metadata_path(self, ref: DatasetRef) -> Path: def private_metadata_path(self, ref: DatasetRef) -> Path: return self._codec_for(ref.protocol_version).private_metadata_path(ref) - # -- scanning (union over all protocol layouts) -------------------------- - def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: - """Yield a ref per dataset under public/syft_datasets/, across all codecs.""" + # -- scanning ------------------------------------------------------------ + def iter_dataset_refs_all_protocols( + self, datasite_email: str + ) -> Iterator[DatasetRef]: + """Yield a ref per on-disk dataset copy, across every protocol layout. + + A dataset broadcast to a mixed audience is written once per protocol, so + the same (owner, name) can appear multiple times. Storage-internal; use + it when you must touch every copy (e.g. deletion). Callers outside + storage should use ``iter_dataset_refs``. + """ for codec in self.codecs: yield from codec.iter_dataset_refs(datasite_email) + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + """Yield one ref per dataset, in its preferred (newest) protocol layout. + + Collapses the per-protocol copies of a dataset to the single layout a + reader should use, so callers see each dataset once. + """ + best: dict[tuple[str, str], DatasetRef] = {} + for ref in self.iter_dataset_refs_all_protocols(datasite_email): + key = (ref.owner, ref.name) + current = best.get(key) + if current is None or int(ref.protocol_version) > int( + current.protocol_version + ): + best[key] = ref + yield from best.values() + def find_dataset_ref(self, datasite_email: str, name: str) -> DatasetRef: - """The ref for ``name`` in a datasite; prefers the highest protocol layout.""" - matches = [ - ref for ref in self.iter_dataset_refs(datasite_email) if ref.name == name + """The ref for ``name`` in a datasite, in its preferred protocol layout.""" + for ref in self.iter_dataset_refs(datasite_email): + if ref.name == name: + return ref + raise DatasetNotFoundError(f"Dataset '{name}' not found") + + # -- deletion ------------------------------------------------------------ + def delete_dataset(self, datasite_email: str, name: str) -> list[Path]: + """Remove every on-disk copy of a dataset, across all protocol layouts. + + A dataset may be written in several protocol versions at once, so this + removes both the public dataset dir and the private metadata dir for each + version present on disk. Returns the directories that were removed. + """ + refs = [ + ref + for ref in self.iter_dataset_refs_all_protocols(datasite_email) + if ref.name == name ] - if not matches: - raise DatasetNotFoundError(f"Dataset '{name}' not found") - return max(matches, key=lambda r: int(r.protocol_version)) + removed: list[Path] = [] + for ref in refs: + for directory in ( + self.public_dataset_dir(ref), + self.private_dataset_dir(ref), + ): + if directory.exists(): + shutil.rmtree(directory) + removed.append(directory) + return removed # -- model IO ------------------------------------------------------------ def read_dataset(self, ref: DatasetRef) -> Dataset: @@ -185,7 +469,7 @@ def read_private_config(self, ref: DatasetRef) -> PrivateDatasetConfig: codec.read(path, "PrivateDatasetConfig"), "PrivateDatasetConfig" ) - def write_dataset(self, ref: DatasetRef, dataset: Dataset) -> Path: + def write_dataset_metadata(self, ref: DatasetRef, dataset: Dataset) -> Path: """Write dataset.yaml in the version/format for this ref's protocol.""" codec = self._codec_for(ref.protocol_version) return self._write(codec, codec.metadata_path(ref), dataset, ref) diff --git a/packages/syft-datasets/src/syft_datasets/file_utils.py b/packages/syft-datasets/src/syft_datasets/file_utils.py index 37e464b9ec0..1c8933e9eb1 100644 --- a/packages/syft-datasets/src/syft_datasets/file_utils.py +++ b/packages/syft-datasets/src/syft_datasets/file_utils.py @@ -3,10 +3,16 @@ from typing import Iterator -def copy_dir_contents(src: Path, dst: Path, exists_ok: bool = False) -> list[Path]: +def copy_dir_contents( + src: Path, + dst: Path, + exists_ok: bool = False, + exclude_names: frozenset[str] = frozenset(), +) -> list[Path]: if not src.is_dir(): raise ValueError(f"Source path {src} is not a directory.") - return copy_paths(src.iterdir(), dst, exists_ok) + files = (p for p in src.iterdir() if p.name not in exclude_names) + return copy_paths(files, dst, exists_ok) def copy_paths(files: Iterator[Path], dst: Path, exists_ok: bool = False) -> list[Path]: diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py index 2978a35d051..5d035e690d4 100644 --- a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py @@ -18,10 +18,9 @@ from ...config import ( PRIVATE_METADATA_FILENAME, - SYFT_DATASETS_FOLDER_NAME, SyftBoxConfig, - protocol_dir_name, ) +from ...dataset_ref import DatasetRef from ...migrations import dataset_registry from ...migrations.registry import DATASET_PROTOCOL_VERSION from ...url import SyftBoxURL @@ -120,16 +119,18 @@ def private_dir(self) -> Path: """The private data dir for this dataset, under the dataset's protocol layout. Derived from the path (owner + name + protocol) rather than the stored - URL so it stays correct across on-disk layouts. + URL so it stays correct across on-disk layouts. Delegates to the codec's + DatasetConfig so layout lives in exactly one place. """ - segment = protocol_dir_name(self._protocol_version) - root = ( - self.syftbox_config.syftbox_folder - / self.owner - / "private" - / SYFT_DATASETS_FOLDER_NAME + from ...protocolcodecs import dataset_config_for_protocol + + ref = DatasetRef( + owner=self.owner, name=self.name, protocol_version=self._protocol_version + ) + layout = dataset_config_for_protocol( + self._protocol_version, self.syftbox_config ) - return (root / segment / self.name) if segment else (root / self.name) + return layout.private_dataset_dir(ref) @property def _private_metadata_dir(self) -> Path: diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py index 50c78dccd44..522418b855d 100644 --- a/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/__init__.py @@ -1,6 +1,11 @@ +from typing import TYPE_CHECKING + from .base import ProtocolCodec -from .v0 import ProtocolCodecV0 -from .v1 import ProtocolCodecV1 +from .v0 import DatasetConfigV0, ProtocolCodecV0 +from .v1 import DatasetConfigV1, ProtocolCodecV1 + +if TYPE_CHECKING: + from ..config import SyftBoxConfig # Ordered oldest -> newest. The last codec is the one the current dataset # protocol uses (CODECS[-1].version == DATASET_PROTOCOL_VERSION); adding a new @@ -8,9 +13,24 @@ # DATASET_PROTOCOL_VERSION. CODECS = [ProtocolCodecV0, ProtocolCodecV1] + +def dataset_config_for_protocol( + protocol_version: str, syftbox_config: "SyftBoxConfig" +) -> "DatasetConfigV0 | DatasetConfigV1": + """The layout that stores a given protocol version (mirrors codec selection).""" + for codec_cls in CODECS: + config_cls = codec_cls.dataset_config_cls + if protocol_version in config_cls.protocol_versions: + return config_cls(syftbox_config) + raise ValueError(f"No dataset layout for protocol version {protocol_version!r}") + + __all__ = [ + "DatasetConfigV0", + "DatasetConfigV1", "ProtocolCodec", "ProtocolCodecV0", "ProtocolCodecV1", "CODECS", + "dataset_config_for_protocol", ] diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py index 1e21dc2174a..770d97e22c6 100644 --- a/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/base.py @@ -4,47 +4,48 @@ from syft_migration import MigratableObject -from ..config import METADATA_FILENAME, PRIVATE_METADATA_FILENAME, SyftBoxConfig +from ..config import SyftBoxConfig from ..dataset_ref import DatasetRef class ProtocolCodec(ABC): - """Interface for reading/writing/listing datasets in one on-disk storage format. - - A codec owns raw disk layout + serialization for the protocol versions it - lists in ``protocol_versions`` (one codec can serve several — the layout is - derived from each dataset's ``protocol_version``). DatasetStorage owns - migration and selects a codec by protocol version; a codec never maps back - to one. The codec carries its own ``version`` independent of the protocols - it handles. - - A dataset is a directory with a public ``dataset.yaml`` (the discovery - marker, synced to peers) plus mock files, and a separate - ``private_metadata.yaml`` under the owner's private/ tree (never synced). + """Reads/writes dataset metadata for one on-disk layout. + + The codec owns serialization (``read``/``write``); its ``dataset_config`` owns + disk layout (paths, filenames, iteration) and nests the SyftBoxConfig it + resolves against. DatasetStorage owns migration and selects a codec by + protocol version. Path/iteration methods here delegate to the config, so + DatasetStorage keeps a single object to talk to. """ - version: str # the codec's own version ("0", "1", ...) - protocol_versions: list[str] # protocol versions this codec reads/writes - metadata_marker = METADATA_FILENAME - private_metadata_marker = PRIVATE_METADATA_FILENAME + # Each concrete codec sets its DatasetConfigV class (see v0.py / v1.py). + dataset_config_cls: type def __init__(self, config: SyftBoxConfig) -> None: - self.config = config + self.dataset_config = self.dataset_config_cls(config) - @abstractmethod - def public_dataset_dir(self, ref: DatasetRef) -> Path: ... + @property + def version(self) -> str: + return self.dataset_config.version - @abstractmethod - def private_dataset_dir(self, ref: DatasetRef) -> Path: ... + @property + def protocol_versions(self) -> list[str]: + return self.dataset_config.protocol_versions + + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.dataset_config.public_dataset_dir(ref) + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.dataset_config.private_dataset_dir(ref) def metadata_path(self, ref: DatasetRef) -> Path: - return self.public_dataset_dir(ref) / self.metadata_marker + return self.dataset_config.metadata_path(ref) def private_metadata_path(self, ref: DatasetRef) -> Path: - return self.private_dataset_dir(ref) / self.private_metadata_marker + return self.dataset_config.private_metadata_path(ref) - @abstractmethod - def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: ... + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + return self.dataset_config.iter_dataset_refs(datasite_email) @abstractmethod def read(self, path: Path, canonical_name: str) -> dict: ... diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py index d2bb91920aa..36091c48d21 100644 --- a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v0.py @@ -4,38 +4,69 @@ import yaml from syft_migration import MigratableObject -from ..config import is_protocol_dir_name +from ..config import ( + METADATA_FILENAME, + PRIVATE_METADATA_FILENAME, + SYFT_DATASETS_FOLDER_NAME, + SyftBoxConfig, + is_protocol_dir_name, +) from ..dataset_ref import DatasetRef from .base import ProtocolCodec -class ProtocolCodecV0(ProtocolCodec): - """Pre-versioning layout (<= 0.1.20): datasets flat under syft_datasets/, no identity fields.""" +class DatasetConfigV0: + """Pre-versioning layout (<= 0.1.20): datasets flat under syft_datasets/, no segment.""" version = "0" protocol_versions = ["0"] + datasets_folder_name = SYFT_DATASETS_FOLDER_NAME + metadata_filename = METADATA_FILENAME + private_metadata_filename = PRIVATE_METADATA_FILENAME - def public_dataset_dir(self, ref: DatasetRef) -> Path: - return self.config.get_mock_dataset_dir( - ref.name, ref.owner, ref.protocol_version + def __init__(self, syftbox_config: SyftBoxConfig) -> None: + self.syftbox_config = syftbox_config + + def public_all_datasets_folder(self, datasite: str) -> Path: + return ( + self.syftbox_config.datasite_public_root(datasite) + / self.datasets_folder_name ) - def private_dataset_dir(self, ref: DatasetRef) -> Path: - return self.config.get_private_dataset_dir( - ref.owner, ref.name, ref.protocol_version + def private_all_datasets_folder(self, owner: str) -> Path: + return ( + self.syftbox_config.datasite_private_root(owner) / self.datasets_folder_name ) + def public_dataset_dir(self, ref: DatasetRef) -> Path: + return self.public_all_datasets_folder(ref.owner) / ref.name + + def private_dataset_dir(self, ref: DatasetRef) -> Path: + return self.private_all_datasets_folder(ref.owner) / ref.name + + def metadata_path(self, ref: DatasetRef) -> Path: + return self.public_dataset_dir(ref) / self.metadata_filename + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self.private_dataset_dir(ref) / self.private_metadata_filename + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: - root = self.config.public_datasets_root_for_datasite(datasite_email) - if not root.exists(): + root = self.public_all_datasets_folder(datasite_email) + if not root.is_dir(): return for entry in sorted(p for p in root.iterdir() if p.is_dir()): - # v/ segments belong to a versioned codec, not the flat layout. + # v/ segments belong to a versioned layout, not the flat one. if is_protocol_dir_name(entry.name): continue - if (entry / self.metadata_marker).exists(): + if (entry / self.metadata_filename).exists(): yield DatasetRef(datasite_email, entry.name, "0") + +class ProtocolCodecV0(ProtocolCodec): + """Reads/writes the flat layout, stripping the identity fields off disk.""" + + dataset_config_cls = DatasetConfigV0 + def read(self, path: Path, canonical_name: str) -> dict: data = yaml.safe_load(path.read_text()) or {} # Protocol-0 files predate the identity fields; they are all version 1. diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py index 1c3119a38c3..c9fb11b1af8 100644 --- a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py @@ -4,38 +4,73 @@ import yaml from syft_migration import MigratableObject -from ..config import protocol_dir_name +from ..config import ( + METADATA_FILENAME, + PRIVATE_METADATA_FILENAME, + SYFT_DATASETS_FOLDER_NAME, + SyftBoxConfig, +) from ..dataset_ref import DatasetRef from .base import ProtocolCodec -class ProtocolCodecV1(ProtocolCodec): - """Versioned layout (>= 0.1.21): datasets under syft_datasets/v/, identity fields on disk.""" +class DatasetConfigV1: + """Versioned layout (>= 0.1.21): datasets under syft_datasets/v1/, identity fields on disk.""" version = "1" protocol_versions = ["1"] + protocol_segment = "v1" + datasets_folder_name = SYFT_DATASETS_FOLDER_NAME + metadata_filename = METADATA_FILENAME + private_metadata_filename = PRIVATE_METADATA_FILENAME + + def __init__(self, syftbox_config: SyftBoxConfig) -> None: + self.syftbox_config = syftbox_config + + def public_all_dataset_folder(self, datasite: str) -> Path: + return ( + self.syftbox_config.datasite_public_root(datasite) + / self.datasets_folder_name + ) + + def private_all_datatset_folder(self, owner: str) -> Path: + return ( + self.syftbox_config.datasite_private_root(owner) / self.datasets_folder_name + ) def public_dataset_dir(self, ref: DatasetRef) -> Path: - return self.config.get_mock_dataset_dir( - ref.name, ref.owner, ref.protocol_version + return ( + self.public_all_dataset_folder(ref.owner) / self.protocol_segment / ref.name ) def private_dataset_dir(self, ref: DatasetRef) -> Path: - return self.config.get_private_dataset_dir( - ref.owner, ref.name, ref.protocol_version + return ( + self.private_all_datatset_folder(ref.owner) + / self.protocol_segment + / ref.name ) + def metadata_path(self, ref: DatasetRef) -> Path: + return self.public_dataset_dir(ref) / self.metadata_filename + + def private_metadata_path(self, ref: DatasetRef) -> Path: + return self.private_dataset_dir(ref) / self.private_metadata_filename + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: - root = self.config.public_datasets_root_for_datasite(datasite_email) - if not root.exists(): + version_dir = ( + self.public_all_dataset_folder(datasite_email) / self.protocol_segment + ) + if not version_dir.is_dir(): return - for protocol_version in self.protocol_versions: - version_dir = root / protocol_dir_name(protocol_version) - if not version_dir.is_dir(): - continue - for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): - if (entry / self.metadata_marker).exists(): - yield DatasetRef(datasite_email, entry.name, protocol_version) + for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): + if (entry / self.metadata_filename).exists(): + yield DatasetRef(datasite_email, entry.name, "1") + + +class ProtocolCodecV1(ProtocolCodec): + """Reads/writes the versioned layout with identity fields intact.""" + + dataset_config_cls = DatasetConfigV1 def read(self, path: Path, canonical_name: str) -> dict: # Files already carry canonical_name/version on disk. diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv deleted file mode 100644 index f5c2e52a5a1..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,secret -1,alpha -2,beta diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml deleted file mode 100644 index 9362a9fd6cf..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/private_metadata.yaml +++ /dev/null @@ -1,4 +0,0 @@ -canonical_name: PrivateDatasetConfig -version: '1' -uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 -data_dir: '' diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml deleted file mode 100644 index c362f440602..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/private/syft_datasets/v1/demo/syft.pub.yaml +++ /dev/null @@ -1,8 +0,0 @@ -rules: -- access: - admin: [] - read: - - do@test.org - write: [] - pattern: '**' -terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml deleted file mode 100644 index 503efd923a4..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/dataset.yaml +++ /dev/null @@ -1,15 +0,0 @@ -canonical_name: Dataset -version: '1' -uid: fddcb47f-1d56-46d7-bf54-99be3ccc1e24 -created_at: '2026-07-15T12:54:04.399452Z' -updated_at: '2026-07-15T12:54:04.399619Z' -name: demo -summary: demo dataset -tags: -- demo -location: null -mock_url: syft://do@test.org/public/syft_datasets/v1/demo -private_url: syft://do@test.org/private/syft_datasets/v1/demo -readme_url: syft://do@test.org/public/syft_datasets/v1/demo/readme.md -mock_files_urls: -- syft://do@test.org/public/syft_datasets/v1/demo/mock.csv diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv deleted file mode 100644 index ea6be9205d3..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/mock.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,value -1,10 -2,20 diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md deleted file mode 100644 index c5b7a1f37d7..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/readme.md +++ /dev/null @@ -1 +0,0 @@ -# demo dataset diff --git a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml b/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml deleted file mode 100644 index 676a3c8f2e3..00000000000 --- a/packages/syft-datasets/tests/migrations/p2p/fixtures/syft_dataset-0.1.21-protocol1_syftbox/do@test.org/public/syft_datasets/v1/demo/syft.pub.yaml +++ /dev/null @@ -1,8 +0,0 @@ -rules: -- access: - admin: [] - read: - - ds@test.org - write: [] - pattern: '**' -terminal: false diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py index eb3fe6ce169..6a610d8f6fd 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -38,6 +38,47 @@ def _dataset_manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: return mgr +def test_migrate_dataset_v0_to_v1_preserves_identity(tmp_path: Path): + # No peers => created in the widest-compatible (flat, protocol 0) layout. + mgr = _dataset_manager(tmp_path) + mock, private, readme = _create_dataset_files(tmp_path) + mgr.create(name="demo", mock_path=mock, private_path=private, readme_path=readme) + + ref0 = mgr.storage.find_dataset_ref(DO_EMAIL, "demo") + assert ref0.protocol_version == "0" + old = mgr.storage.read_dataset(ref0) + + migrated = mgr.storage.migrate_dataset(ref0, "1") + + public_root = mgr.syftbox_config.datasite_public_root(DO_EMAIL) / "syft_datasets" + # Source (v0) copy is left intact; the v1 layout is created alongside it. + assert (public_root / "demo" / "dataset.yaml").exists() + assert (public_root / "v1" / "demo" / "dataset.yaml").exists() + + # Identity is preserved across the migration (datasets are immutable). + assert migrated.uid == old.uid + assert migrated.created_at == old.created_at + assert migrated._protocol_version == "1" + + # v1 dataset.yaml carries the identity fields (unlike flat protocol 0). + raw = yaml.safe_load((public_root / "v1" / "demo" / "dataset.yaml").read_text()) + assert raw["canonical_name"] == "Dataset" and raw["version"] == "1" + + # Payload copied; metadata/permission/readme files excluded from mock_files. + mock_names = sorted(p.name for p in migrated.mock_files) + assert mock_names == ["mock.csv"] + + # Private data + a fresh private config land in the v1 layout. + target_ref = mgr.storage.new_dataset_ref("demo", "1") + assert mgr.storage.private_dataset_dir(target_ref).joinpath("private.csv").exists() + assert mgr.storage.read_private_config(target_ref).uid == old.uid + + # get_all() dedupes the two on-disk copies, preferring the newest (v1). + all_datasets = mgr.get_all() + assert len(all_datasets) == 1 + assert all_datasets[0]._protocol_version == "1" + + def test_default_create_writes_protocol_0(tmp_path: Path): mgr = _dataset_manager(tmp_path) mock, private, readme = _create_dataset_files(tmp_path) @@ -97,7 +138,7 @@ def test_multi_version_write_for_mixed_audience(tmp_path: Path): users=[DS0, DS1], ) - root = mgr.syftbox_config.public_datasets_root_for_datasite(DO_EMAIL) + root = mgr.syftbox_config.datasite_public_root(DO_EMAIL) / "syft_datasets" # Both layouts written: flat (protocol 0) and v1 (protocol 1). assert (root / "demo" / "dataset.yaml").exists() assert (root / "v1" / "demo" / "dataset.yaml").exists() @@ -107,3 +148,35 @@ def test_multi_version_write_for_mixed_audience(tmp_path: Path): assert len(all_datasets) == 1 assert all_datasets[0]._protocol_version == "1" assert mgr.get("demo")._protocol_version == "1" + + +def test_delete_removes_all_protocol_versions(tmp_path: Path): + schema0 = dataset_registry.schema_for_protocol_version("0") + schema1 = dataset_registry.schema_for_protocol_version("1") + mgr = _dataset_manager(tmp_path, peer_schemas={DS0: schema0, DS1: schema1}) + mock, private, readme = _create_dataset_files(tmp_path) + + mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + users=[DS0, DS1], + ) + + public_root = mgr.syftbox_config.datasite_public_root(DO_EMAIL) / "syft_datasets" + private_root = mgr.syftbox_config.datasite_private_root(DO_EMAIL) / "syft_datasets" + # Both layouts exist on disk before deletion. + assert (public_root / "demo").exists() + assert (public_root / "v1" / "demo").exists() + assert (private_root / "demo").exists() + assert (private_root / "v1" / "demo").exists() + + mgr.delete(name="demo", require_confirmation=False) + + # Every protocol version is gone, public and private. + assert not (public_root / "demo").exists() + assert not (public_root / "v1" / "demo").exists() + assert not (private_root / "demo").exists() + assert not (private_root / "v1" / "demo").exists() + assert mgr.get_all() == [] diff --git a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py index 5840603ba5f..c35460dc9c6 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -29,7 +29,7 @@ RELEASE_FIXTURES = sorted(FIXTURES_DIR.glob("syft_dataset-*-protocol*_syftbox")) released_syftbox_fixtures = pytest.mark.parametrize( - "fixture", RELEASE_FIXTURES, ids=lambda f: f.name + "released_syftbox_path", RELEASE_FIXTURES, ids=lambda f: f.name ) @@ -37,7 +37,7 @@ def _protocol_of(fixture: Path) -> str: return re.search(r"-protocol(\d+)_syftbox$", fixture.name).group(1) -def _has_identity(protocol: str) -> bool: +def _has_identity_in_path(protocol: str) -> bool: """canonical_name/version are written only from protocol 1 onwards.""" return protocol != "0" @@ -48,18 +48,18 @@ def _syftbox_w_old_datasets(fixture: Path, tmp_path: Path) -> Path: return syftbox -def _manager(syftbox: Path) -> SyftDatasetManager: +def _dataset_manager(syftbox: Path) -> SyftDatasetManager: return SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) -def _storage(syftbox: Path) -> DatasetStorage: +def _dataset_storage(syftbox: Path) -> DatasetStorage: return DatasetStorage(config=SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL)) @released_syftbox_fixtures -def test_old_datasets_load_and_upgrade(fixture: Path, tmp_path: Path): - syftbox = _syftbox_w_old_datasets(fixture, tmp_path) - mgr = _manager(syftbox) +def test_old_datasets_load_and_upgrade(released_syftbox_path: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(released_syftbox_path, tmp_path) + mgr = _dataset_manager(syftbox) datasets = mgr.get_all() assert {d.name for d in datasets} == {DATASET} @@ -67,7 +67,7 @@ def test_old_datasets_load_and_upgrade(fixture: Path, tmp_path: Path): # Loads into the latest registered version in memory. assert dataset.version == dataset_registry.latest_version("Dataset") assert dataset.owner == DO_EMAIL - assert dataset._protocol_version == _protocol_of(fixture) + assert dataset._protocol_version == _protocol_of(released_syftbox_path) # Private config also loads/upgrades to latest. ref = mgr.storage.find_dataset_ref(DO_EMAIL, DATASET) private_config = mgr.storage.read_private_config(ref) @@ -77,10 +77,10 @@ def test_old_datasets_load_and_upgrade(fixture: Path, tmp_path: Path): @released_syftbox_fixtures -def test_write_back_is_byte_exact(fixture: Path, tmp_path: Path): - protocol = _protocol_of(fixture) - syftbox = _syftbox_w_old_datasets(fixture, tmp_path) - storage = _storage(syftbox) +def test_write_back_is_byte_exact(released_syftbox_path: Path, tmp_path: Path): + protocol = _protocol_of(released_syftbox_path) + syftbox = _syftbox_w_old_datasets(released_syftbox_path, tmp_path) + storage = _dataset_storage(syftbox) ref = storage.find_dataset_ref(DO_EMAIL, DATASET) metadata_path = storage.metadata_path(ref) @@ -88,17 +88,17 @@ def test_write_back_is_byte_exact(fixture: Path, tmp_path: Path): # Read (upgrade in memory) then write back in that release's layout/format. dataset = storage.read_dataset(ref) - storage.write_dataset(ref, dataset) + storage.write_dataset_metadata(ref, dataset) assert metadata_path.read_text() == original_bytes raw = metadata_path.read_text() - assert ("canonical_name" in raw) is _has_identity(protocol) + assert ("canonical_name" in raw) is _has_identity_in_path(protocol) @released_syftbox_fixtures -def test_mixed_protocol_listing(fixture: Path, tmp_path: Path): - syftbox = _syftbox_w_old_datasets(fixture, tmp_path) - mgr = _manager(syftbox) +def test_mixed_protocol_listing(released_syftbox_path: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(released_syftbox_path, tmp_path) + mgr = _dataset_manager(syftbox) # A newly created dataset (default protocol) coexists with the old one. src = tmp_path / "src" @@ -113,9 +113,9 @@ def test_mixed_protocol_listing(fixture: Path, tmp_path: Path): @released_syftbox_fixtures -def test_dataset_reprs_do_not_error(fixture: Path, tmp_path: Path): - syftbox = _syftbox_w_old_datasets(fixture, tmp_path) - datasets = _manager(syftbox).get_all() +def test_dataset_reprs_do_not_error(released_syftbox_path: Path, tmp_path: Path): + syftbox = _syftbox_w_old_datasets(released_syftbox_path, tmp_path) + datasets = _dataset_manager(syftbox).get_all() for render in (repr, str, lambda o: o._repr_html_()): assert render(datasets) diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py index 556e16d69f4..23892bd2519 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py @@ -19,6 +19,7 @@ from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION from syft_datasets.models import Dataset from syft_datasets.protocolcodecs import ProtocolCodec +from syft_datasets.url import SyftBoxURL DO_EMAIL = "do@test.org" @@ -31,11 +32,11 @@ def _storage(tmp_path: Path) -> DatasetStorage: def _mock_dataset(storage: DatasetStorage, ref: DatasetRef) -> Dataset: - cfg = storage.config + folder = storage.config.syftbox_folder return Dataset( name=ref.name, - mock_url=cfg.get_mock_url_for_my_dataset(ref.name, ref.protocol_version), - private_url=cfg.get_private_url_for_my_dataset(ref.name, ref.protocol_version), + mock_url=SyftBoxURL.from_path(storage.public_dataset_dir(ref), folder), + private_url=SyftBoxURL.from_path(storage.private_dataset_dir(ref), folder), ) @@ -101,8 +102,8 @@ def test_v0_writes_flat_no_identity_v1_nests_with_identity(tmp_path: Path): ref0 = DatasetRef(DO_EMAIL, "flat", "0") ref1 = DatasetRef(DO_EMAIL, "nested", "1") - p0 = storage.write_dataset(ref0, _mock_dataset(storage, ref0)) - p1 = storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + p0 = storage.write_dataset_metadata(ref0, _mock_dataset(storage, ref0)) + p1 = storage.write_dataset_metadata(ref1, _mock_dataset(storage, ref1)) # protocol 0: public/syft_datasets//dataset.yaml (no v), identity stripped assert p0.parent.parent.name == "syft_datasets" @@ -127,17 +128,35 @@ def test_scan_partitions_by_layout(tmp_path: Path): ref0 = DatasetRef(DO_EMAIL, "flat", "0") ref1 = DatasetRef(DO_EMAIL, "nested", "1") - storage.write_dataset(ref0, _mock_dataset(storage, ref0)) - storage.write_dataset(ref1, _mock_dataset(storage, ref1)) + storage.write_dataset_metadata(ref0, _mock_dataset(storage, ref0)) + storage.write_dataset_metadata(ref1, _mock_dataset(storage, ref1)) v0_refs = list(v0_codec.iter_dataset_refs(DO_EMAIL)) v1_refs = list(v1_codec.iter_dataset_refs(DO_EMAIL)) assert [(r.name, r.protocol_version) for r in v0_refs] == [("flat", "0")] assert [(r.name, r.protocol_version) for r in v1_refs] == [("nested", "1")] - all_refs = list(storage.iter_dataset_refs(DO_EMAIL)) + all_refs = list(storage.iter_dataset_refs_all_protocols(DO_EMAIL)) assert {(r.name, r.protocol_version) for r in all_refs} == { ("flat", "0"), ("nested", "1"), } assert len(all_refs) == 2 + + +def test_iter_dataset_refs_prefers_newest_protocol(tmp_path: Path): + storage = _storage(tmp_path) + + # Same dataset written in two protocol layouts (as for a mixed audience). + ref0 = DatasetRef(DO_EMAIL, "demo", "0") + ref1 = DatasetRef(DO_EMAIL, "demo", "1") + storage.write_dataset_metadata(ref0, _mock_dataset(storage, ref0)) + storage.write_dataset_metadata(ref1, _mock_dataset(storage, ref1)) + + # all-protocols sees both copies; the public iterator collapses to the newest. + all_refs = list(storage.iter_dataset_refs_all_protocols(DO_EMAIL)) + assert {r.protocol_version for r in all_refs} == {"0", "1"} + + preferred = list(storage.iter_dataset_refs(DO_EMAIL)) + assert [(r.name, r.protocol_version) for r in preferred] == [("demo", "1")] + assert storage.find_dataset_ref(DO_EMAIL, "demo").protocol_version == "1" diff --git a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py index 82c816dd21e..be7b2814160 100644 --- a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py +++ b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py @@ -11,17 +11,23 @@ from syft_datasets.models import DatasetV1 -def test_0_1_20_artifact_file_loads(): - artifact = ReleasedPackageProtocolInfo.load( - PACKAGE_ARTIFACTS_DIR / "syft-dataset-0.1.20.json" - ) - assert artifact.package_info.package_name == "syft-dataset" - assert artifact.package_info.version == "0.1.20" - assert artifact.package_info.protocol_version == "0" - assert artifact.protocol_schema.protocol_name == "syft-dataset" - assert artifact.protocol_schema.version == "0" - expected_versions = {"Dataset": ["1"], "PrivateDatasetConfig": ["1"]} - assert artifact.protocol_schema.supported_versions == expected_versions +def test_all_released_package_artifacts_load(): + artifact_paths = sorted(PACKAGE_ARTIFACTS_DIR.glob("*.json")) + assert artifact_paths # at least syft-dataset-0.1.20.json exists + + for path in artifact_paths: + artifact = ReleasedPackageProtocolInfo.load(path) + info = artifact.package_info + # The filename encodes the package version: syft-dataset-.json. + assert path.name == f"{info.package_name}-{info.version}.json" + assert info.package_name == "syft-dataset" + schema = artifact.protocol_schema + assert schema.protocol_name == "syft-dataset" + assert schema.version == info.protocol_version + assert schema.supported_versions + assert set(schema.current_object_schemas) == set(schema.supported_versions) + for canonical_name in schema.supported_versions: + assert schema.current_schema(canonical_name) def test_all_released_protocols_load(): @@ -40,15 +46,6 @@ def test_all_released_protocols_load(): assert schema.current_schema(canonical_name) -def test_protocol_0_released_protocol_loads(): - released = ReleasedProtocol.load(PROTOCOLS_DIR / "protocol-0.json") - schema = released.protocol_schema - assert schema.version == "0" - assert set(schema.current_object_schemas) == {"Dataset", "PrivateDatasetConfig"} - assert schema.current_schema("Dataset") == "1" - assert schema.current_schema("PrivateDatasetConfig") == "1" - - def test_released_object_schemas_unchanged(): # Released object versions are frozen forever; see the failure message. drift = dataset_registry.find_schema_drift() diff --git a/syft_client/utils.py b/syft_client/utils.py index 065b19a45a0..e8e25f9a149 100644 --- a/syft_client/utils.py +++ b/syft_client/utils.py @@ -12,7 +12,7 @@ from typing_extensions import TYPE_CHECKING -from syft_datasets.dataset import Dataset +from syft_datasets import Dataset from syft_datasets.dataset_manager import SyftDatasetManager if TYPE_CHECKING: diff --git a/tests/integration/with_unit_coverage/test_sync_manager.py b/tests/integration/with_unit_coverage/test_sync_manager.py index 31294201492..9e509060e32 100644 --- a/tests/integration/with_unit_coverage/test_sync_manager.py +++ b/tests/integration/with_unit_coverage/test_sync_manager.py @@ -13,7 +13,7 @@ - test_file_deletion_do_to_ds -> test_file_deletion_do_to_ds """ -from syft_datasets.dataset import Dataset +from syft_datasets import Dataset import os from pathlib import Path import time diff --git a/tests/unit/test_create_dataset_cleanup.py b/tests/unit/test_create_dataset_cleanup.py index ac938dee596..e1f89401e8e 100644 --- a/tests/unit/test_create_dataset_cleanup.py +++ b/tests/unit/test_create_dataset_cleanup.py @@ -69,10 +69,12 @@ def test_cleanup_on_private_upload_failure(self): """If private upload fails, both GDrive mock folder and local dataset are cleaned up.""" do_manager = self._make_do_manager() - # Compute expected local paths before the test so we can verify deletion - syftbox_config = do_manager.dataset_manager.syftbox_config - mock_dir = syftbox_config.get_my_mock_dataset_dir("testdataset") - private_metadata_dir = syftbox_config.private_dir_for_my_dataset("testdataset") + # Compute expected local paths before the test so we can verify deletion. + # A dataset with no peers is written at the widest-compatible protocol. + storage = do_manager.dataset_manager.storage + ref = storage.new_dataset_ref("testdataset", storage._widest_protocol_version) + mock_dir = storage.public_dataset_dir(ref) + private_metadata_dir = storage.private_dataset_dir(ref) with patch.object( do_manager, diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index f9b3aeae50f..57944534718 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -16,7 +16,7 @@ from syft_client.sync.sync.caches.datasite_owner_cache import ( ProposedEventFileOutdatedException, ) -from syft_datasets.dataset import Dataset +from syft_datasets import Dataset from tests.unit.utils import ( create_tmp_dataset_files, create_tmp_dataset_files_with_parquet, diff --git a/uv.lock b/uv.lock index cfb24ccb349..51ca3f63bbc 100644 --- a/uv.lock +++ b/uv.lock @@ -4620,7 +4620,7 @@ wheels = [ [[package]] name = "syft-dataset" -version = "0.1.21" +version = "0.1.20" source = { editable = "packages/syft-datasets" } dependencies = [ { name = "pyyaml" }, From 586468eb3a3ccd80fd01f1bdcecb3132559ed28a Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 21 Jul 2026 14:20:28 +0200 Subject: [PATCH 6/8] refactor: store DatasetRef on Dataset instead of _protocol_version DatasetStorage passes the ref when reading/creating a dataset; the model's owner property now derives from _ref.owner (was mock_url.host) and private_dir uses _ref directly. _ref is a private attr with no default. Also drop the dead _validate_dataset_name from SyftDatasetManager (never called; storage's validate_dataset_name still runs via create_dataset). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/syft_datasets/dataset_manager.py | 10 -------- .../src/syft_datasets/dataset_storage.py | 4 +-- .../src/syft_datasets/models/dataset/v1.py | 25 ++++++++----------- .../migrations/p2p/test_current_protocol.py | 10 ++++---- .../p2p/test_older_protocol_compatibility.py | 2 +- 5 files changed, 19 insertions(+), 32 deletions(-) diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index ef43d6dd818..2c98643aa0f 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -1,4 +1,3 @@ -import re from pathlib import Path from typing_extensions import Self @@ -34,15 +33,6 @@ def __init__(self, syftbox_folder_path: PathLike, email: str): def from_config(cls, config: SyftBoxConfig) -> Self: return cls(syftbox_folder_path=config.syftbox_folder, email=config.email) - def _validate_dataset_name(self, dataset_name: str) -> None: - # Returns True if the dataset is a valid path name on unix or windows. - if not re.match(r"^[\w-]+$", dataset_name): - raise ValueError( - f"Invalid dataset name '{dataset_name}'. Only alphanumeric characters, underscores, and hyphens are allowed." - ) - # Reject names that collide with a protocol-version directory (v1, v2, ...). - self.storage.validate_dataset_name(dataset_name) - def create( self, name: str, diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py index 31528bbedea..6a4c5ade03a 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_storage.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -285,7 +285,7 @@ def _materialize_version( mock_files_urls=[self._url_for(f) for f in mock_file_paths], ) dataset._syftbox_config = self.config - dataset._protocol_version = ref.protocol_version + dataset._ref = ref self._copy_private_data(ref, source.private, exclude_names) @@ -451,7 +451,7 @@ def read_dataset(self, ref: DatasetRef) -> Dataset: # Name comes from the path (the dataset dir), never a spoofable file. dataset.name = ref.name dataset._syftbox_config = self.config - dataset._protocol_version = ref.protocol_version + dataset._ref = ref return dataset def read_private_config(self, ref: DatasetRef) -> PrivateDatasetConfig: diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py index 5d035e690d4..a98ca667e1e 100644 --- a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py @@ -7,7 +7,7 @@ from uuid import UUID, uuid4 import yaml -from pydantic import Field +from pydantic import Field, PrivateAttr from syft_migration import MigratableObject from syft_notebook_ui.formatter_mixin import ( ANSIPydanticFormatter, @@ -22,7 +22,6 @@ ) from ...dataset_ref import DatasetRef from ...migrations import dataset_registry -from ...migrations.registry import DATASET_PROTOCOL_VERSION from ...url import SyftBoxURL from ..private_dataset_config.v1 import PrivateDatasetConfigV1 @@ -56,10 +55,11 @@ class DatasetV1(MigratableObject, PydanticFormatterMixin, registry=dataset_regis # URLs to uploaded files (excluding metadata files) mock_files_urls: list[SyftBoxURL] = Field(default_factory=list) - # Runtime-only: set by DatasetStorage / the manager, never serialized. The - # on-disk protocol layout the dataset lives in (governs the v segment). + # Runtime-only: set by DatasetStorage when the dataset is read/created, never + # serialized. _ref carries the dataset's identity (owner, name) and its + # on-disk protocol layout; it has no default and must be set before use. _syftbox_config: SyftBoxConfig | None = None - _protocol_version: str = DATASET_PROTOCOL_VERSION + _ref: DatasetRef = PrivateAttr() def disk_dict(self) -> dict: """The on-disk form of the dataset metadata.""" @@ -67,7 +67,7 @@ def disk_dict(self) -> dict: @property def owner(self) -> str: - return self.mock_url.host + return self._ref.owner @property def syftbox_config(self) -> SyftBoxConfig: @@ -116,21 +116,18 @@ def private_config(self) -> PrivateDatasetConfigV1: @property def private_dir(self) -> Path: - """The private data dir for this dataset, under the dataset's protocol layout. + """The private data dir for this dataset, under its on-disk protocol layout. - Derived from the path (owner + name + protocol) rather than the stored - URL so it stays correct across on-disk layouts. Delegates to the codec's + Derived from the ref (owner + name + protocol) rather than the stored URL + so it stays correct across on-disk layouts. Delegates to the codec's DatasetConfig so layout lives in exactly one place. """ from ...protocolcodecs import dataset_config_for_protocol - ref = DatasetRef( - owner=self.owner, name=self.name, protocol_version=self._protocol_version - ) layout = dataset_config_for_protocol( - self._protocol_version, self.syftbox_config + self._ref.protocol_version, self.syftbox_config ) - return layout.private_dataset_dir(ref) + return layout.private_dataset_dir(self._ref) @property def _private_metadata_dir(self) -> Path: diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py index 6a610d8f6fd..37ca6fc8114 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -58,7 +58,7 @@ def test_migrate_dataset_v0_to_v1_preserves_identity(tmp_path: Path): # Identity is preserved across the migration (datasets are immutable). assert migrated.uid == old.uid assert migrated.created_at == old.created_at - assert migrated._protocol_version == "1" + assert migrated._ref.protocol_version == "1" # v1 dataset.yaml carries the identity fields (unlike flat protocol 0). raw = yaml.safe_load((public_root / "v1" / "demo" / "dataset.yaml").read_text()) @@ -76,7 +76,7 @@ def test_migrate_dataset_v0_to_v1_preserves_identity(tmp_path: Path): # get_all() dedupes the two on-disk copies, preferring the newest (v1). all_datasets = mgr.get_all() assert len(all_datasets) == 1 - assert all_datasets[0]._protocol_version == "1" + assert all_datasets[0]._ref.protocol_version == "1" def test_default_create_writes_protocol_0(tmp_path: Path): @@ -121,7 +121,7 @@ def test_create_for_protocol1_peer_writes_v1(tmp_path: Path): assert DS1 in perm.read_text() got = mgr.get("demo") - assert got.name == "demo" and got._protocol_version == "1" + assert got.name == "demo" and got._ref.protocol_version == "1" def test_multi_version_write_for_mixed_audience(tmp_path: Path): @@ -146,8 +146,8 @@ def test_multi_version_write_for_mixed_audience(tmp_path: Path): # get_all() dedupes the two on-disk copies to one, preferring the newest. all_datasets = mgr.get_all() assert len(all_datasets) == 1 - assert all_datasets[0]._protocol_version == "1" - assert mgr.get("demo")._protocol_version == "1" + assert all_datasets[0]._ref.protocol_version == "1" + assert mgr.get("demo")._ref.protocol_version == "1" def test_delete_removes_all_protocol_versions(tmp_path: Path): diff --git a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py index c35460dc9c6..c644d9a7b61 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -67,7 +67,7 @@ def test_old_datasets_load_and_upgrade(released_syftbox_path: Path, tmp_path: Pa # Loads into the latest registered version in memory. assert dataset.version == dataset_registry.latest_version("Dataset") assert dataset.owner == DO_EMAIL - assert dataset._protocol_version == _protocol_of(released_syftbox_path) + assert dataset._ref.protocol_version == _protocol_of(released_syftbox_path) # Private config also loads/upgrades to latest. ref = mgr.storage.find_dataset_ref(DO_EMAIL, DATASET) private_config = mgr.storage.read_private_config(ref) From b23deabaf49be66446bcadf598185b91c89d6ddb Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 21 Jul 2026 14:37:56 +0200 Subject: [PATCH 7/8] feat: allow passing explicit protocol_versions to dataset create DatasetStorage.create_dataset (and SyftDatasetManager.create) accept an optional protocol_versions list; when provided, the dataset is written in exactly those versions instead of inferring them from the audience. Includes a test and a generate_release_fixture.py tweak. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scripts/generate_release_fixture.py | 12 +++++------ .../src/syft_datasets/dataset_manager.py | 4 ++++ .../src/syft_datasets/dataset_storage.py | 16 ++++++++++----- .../migrations/p2p/test_current_protocol.py | 20 +++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/packages/syft-datasets/scripts/generate_release_fixture.py b/packages/syft-datasets/scripts/generate_release_fixture.py index 594fda59c01..f21d9c20487 100644 --- a/packages/syft-datasets/scripts/generate_release_fixture.py +++ b/packages/syft-datasets/scripts/generate_release_fixture.py @@ -35,7 +35,9 @@ ) -def _seed_syftbox(syftbox: Path, src_dir: Path, protocol_version: str) -> None: +def create_syftbox_with_datasets_created_by_manager( + syftbox: Path, src_dir: Path, protocol_version: str +) -> None: """One dataset written in the given protocol layout (mock + private + readme).""" mgr = SyftDatasetManager(syftbox_folder_path=syftbox, email=DO_EMAIL) # Force the target protocol by advertising a peer at that version. @@ -70,10 +72,6 @@ def _normalize(target: Path) -> None: """ import yaml - for path in target.rglob("*"): - if path.is_file(): - path.chmod(0o644) - for config in target.rglob("private_metadata.yaml"): data = yaml.safe_load(config.read_text()) or {} if data.get("data_dir"): @@ -88,7 +86,9 @@ def _build_fixture(target: Path, protocol_version: str) -> None: syftbox.mkdir() src_dir = tmp_path / "src" src_dir.mkdir() - _seed_syftbox(syftbox, src_dir, protocol_version) + create_syftbox_with_datasets_created_by_manager( + syftbox, src_dir, protocol_version + ) shutil.copytree(syftbox, target) _normalize(target) diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index 2c98643aa0f..28c9383da9c 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -43,6 +43,7 @@ def create( location: str | None = None, tags: list[str] | None = None, users: list[str] | str | None = None, + protocol_versions: list[str] | None = None, # copy_private_data: bool = True, # TODO ) -> Dataset: """Create a dataset, writing it in each protocol version its audience can read. @@ -57,6 +58,8 @@ def create( tags (list[str] | None, optional): Optional tags for the dataset. users (list[str] | str | None, optional): Users to share dataset with. Can be a list of emails, SHARE_WITH_ANY, or None (default, share with no one). + protocol_versions (list[str] | None, optional): Write exactly these + protocol versions instead of inferring them from ``users``. Returns: Dataset: The created Dataset object (the newest protocol version written). @@ -73,6 +76,7 @@ def create( location=location, tags=tags, peer_emails=self._peer_emails(users), + protocol_versions=protocol_versions, ) for dataset in created.values(): self._set_new_dataset_permissions(dataset=dataset, users=users) diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py index 6a4c5ade03a..a6c0bcbfb65 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_storage.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -184,12 +184,16 @@ def create_dataset( location: Optional[str] = None, tags: Optional[list[str]] = None, peer_emails: Optional[list[str]] = None, + protocol_versions: Optional[list[str]] = None, ) -> dict[str, Dataset]: """Write a new dataset in every protocol version its audience can read. Copies the source files into each version's on-disk layout and writes the - metadata/private config. Returns {protocol_version: written Dataset}. With - no/unknown peers, writes only the widest-compatible protocol. + metadata/private config. Returns {protocol_version: written Dataset}. + + By default the versions are inferred from ``peer_emails`` (no/unknown peers + => the widest-compatible protocol). Pass ``protocol_versions`` to write + exactly those versions instead, skipping inference. """ self.validate_dataset_name(name) if source.mock.is_dir() and (source.mock / METADATA_FILENAME).exists(): @@ -197,6 +201,10 @@ def create_dataset( f"Mock data at {source.mock} contains reserved file " f"{METADATA_FILENAME}. Please rename it and try again." ) + if protocol_versions is None: + protocol_versions = list( + self.target_protocol_versions_for_peers(peer_emails) + ) now = _utcnow() fields = _DatasetFields( uid=uid or uuid4(), @@ -208,9 +216,7 @@ def create_dataset( tags=tags or [], ) created: dict[str, Dataset] = {} - for protocol_version in sorted( - self.target_protocol_versions_for_peers(peer_emails), key=int - ): + for protocol_version in sorted(protocol_versions, key=int): ref = self.new_dataset_ref(name, protocol_version) created[protocol_version] = self._materialize_version(ref, fields, source) return created diff --git a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py index 37ca6fc8114..5d9d2f428b1 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_current_protocol.py @@ -38,6 +38,26 @@ def _dataset_manager(tmp_path: Path, peer_schemas=None) -> SyftDatasetManager: return mgr +def test_create_with_explicit_protocol_versions_skips_inference(tmp_path: Path): + # No peers => inference would write only protocol 0; explicit versions override. + mgr = _dataset_manager(tmp_path) + mock, private, readme = _create_dataset_files(tmp_path) + + mgr.create( + name="demo", + mock_path=mock, + private_path=private, + readme_path=readme, + protocol_versions=["1"], + ) + + public_root = mgr.syftbox_config.datasite_public_root(DO_EMAIL) / "syft_datasets" + # Exactly the requested version is written: v1 layout, not the flat default. + assert (public_root / "v1" / "demo" / "dataset.yaml").exists() + assert not (public_root / "demo").exists() + assert mgr.get("demo")._ref.protocol_version == "1" + + def test_migrate_dataset_v0_to_v1_preserves_identity(tmp_path: Path): # No peers => created in the widest-compatible (flat, protocol 0) layout. mgr = _dataset_manager(tmp_path) From 060c37caa847f0dbd665cf5959d7d18a9533bbb9 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 21 Jul 2026 15:34:04 +0200 Subject: [PATCH 8/8] refactor: derive v1 codec segment from the ref, serve multiple protocols DatasetConfigV1 no longer hardcodes protocol_segment: public/private_dataset_dir derive the v segment from ref.protocol_version (raising if the version has no segment), so a single codec serves every version in protocol_versions. Renamed its iteration to iter_dataset_refs_all_supported_protocols (loops all supported protocols); ProtocolCodecV1 overrides iter_dataset_refs to call it, leaving v0 and the base untouched. A protocol bump without a layout change is now just appending to protocol_versions. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/syft_datasets/protocolcodecs/v1.py | 56 +++++++++++++------ .../migrations/p2p/test_protocol_codecs.py | 33 +++++++++++ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py index c9fb11b1af8..b3ca28cf85b 100644 --- a/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py +++ b/packages/syft-datasets/src/syft_datasets/protocolcodecs/v1.py @@ -9,17 +9,23 @@ PRIVATE_METADATA_FILENAME, SYFT_DATASETS_FOLDER_NAME, SyftBoxConfig, + protocol_dir_name, ) from ..dataset_ref import DatasetRef from .base import ProtocolCodec class DatasetConfigV1: - """Versioned layout (>= 0.1.21): datasets under syft_datasets/v1/, identity fields on disk.""" + """Versioned layout (>= 0.1.21): datasets under syft_datasets/v/, identity fields on disk. + + Serves every protocol version in ``protocol_versions``. The v segment is + derived from the protocol version (of the ref, or of each version when + scanning), never hardcoded, so a protocol bump that doesn't change the layout + is just appending the new version to ``protocol_versions``. + """ version = "1" protocol_versions = ["1"] - protocol_segment = "v1" datasets_folder_name = SYFT_DATASETS_FOLDER_NAME metadata_filename = METADATA_FILENAME private_metadata_filename = PRIVATE_METADATA_FILENAME @@ -38,16 +44,24 @@ def private_all_datatset_folder(self, owner: str) -> Path: self.syftbox_config.datasite_private_root(owner) / self.datasets_folder_name ) + def _segment(self, ref: DatasetRef) -> str: + # Derived from the ref, never hardcoded. This codec only handles + # versioned (segmented) protocols, so a ref that resolves to no segment + # (e.g. protocol 0) must never reach here. + segment = protocol_dir_name(ref.protocol_version) + if segment is None: + raise ValueError( + "DatasetConfigV1 cannot resolve a path for protocol version " + f"{ref.protocol_version!r}: it has no versioned segment" + ) + return segment + def public_dataset_dir(self, ref: DatasetRef) -> Path: - return ( - self.public_all_dataset_folder(ref.owner) / self.protocol_segment / ref.name - ) + return self.public_all_dataset_folder(ref.owner) / self._segment(ref) / ref.name def private_dataset_dir(self, ref: DatasetRef) -> Path: return ( - self.private_all_datatset_folder(ref.owner) - / self.protocol_segment - / ref.name + self.private_all_datatset_folder(ref.owner) / self._segment(ref) / ref.name ) def metadata_path(self, ref: DatasetRef) -> Path: @@ -56,15 +70,18 @@ def metadata_path(self, ref: DatasetRef) -> Path: def private_metadata_path(self, ref: DatasetRef) -> Path: return self.private_dataset_dir(ref) / self.private_metadata_filename - def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: - version_dir = ( - self.public_all_dataset_folder(datasite_email) / self.protocol_segment - ) - if not version_dir.is_dir(): - return - for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): - if (entry / self.metadata_filename).exists(): - yield DatasetRef(datasite_email, entry.name, "1") + def iter_dataset_refs_all_supported_protocols( + self, datasite_email: str + ) -> Iterator[DatasetRef]: + """Yield refs for datasets stored under any supported protocol's segment.""" + root = self.public_all_dataset_folder(datasite_email) + for protocol_version in self.protocol_versions: + version_dir = root / protocol_dir_name(protocol_version) + if not version_dir.is_dir(): + continue + for entry in sorted(p for p in version_dir.iterdir() if p.is_dir()): + if (entry / self.metadata_filename).exists(): + yield DatasetRef(datasite_email, entry.name, protocol_version) class ProtocolCodecV1(ProtocolCodec): @@ -72,6 +89,11 @@ class ProtocolCodecV1(ProtocolCodec): dataset_config_cls = DatasetConfigV1 + def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: + return self.dataset_config.iter_dataset_refs_all_supported_protocols( + datasite_email + ) + def read(self, path: Path, canonical_name: str) -> dict: # Files already carry canonical_name/version on disk. return yaml.safe_load(path.read_text()) or {} diff --git a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py index 23892bd2519..2863534e423 100644 --- a/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py +++ b/packages/syft-datasets/tests/migrations/p2p/test_protocol_codecs.py @@ -11,6 +11,7 @@ import pkgutil from pathlib import Path +import pytest import yaml from syft_datasets import protocolcodecs from syft_datasets.config import SyftBoxConfig @@ -19,6 +20,7 @@ from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION from syft_datasets.models import Dataset from syft_datasets.protocolcodecs import ProtocolCodec +from syft_datasets.protocolcodecs.v1 import DatasetConfigV1 from syft_datasets.url import SyftBoxURL DO_EMAIL = "do@test.org" @@ -160,3 +162,34 @@ def test_iter_dataset_refs_prefers_newest_protocol(tmp_path: Path): preferred = list(storage.iter_dataset_refs(DO_EMAIL)) assert [(r.name, r.protocol_version) for r in preferred] == [("demo", "1")] assert storage.find_dataset_ref(DO_EMAIL, "demo").protocol_version == "1" + + +# -- behavior 3: one codec serves many protocol versions, segment from the ref -- +def test_v1_codec_serves_multiple_protocols_with_derived_segment(tmp_path: Path): + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + config = SyftBoxConfig(syftbox_folder=syftbox, email=DO_EMAIL) + cfg = DatasetConfigV1(config) + # Simulate a protocol bump that did not change the layout: the same codec now + # serves protocol 2 as well, purely by extending its supported versions. + cfg.protocol_versions = ["1", "2"] + + # The v segment is derived from each ref's protocol version, not hardcoded. + assert cfg.public_dataset_dir(DatasetRef(DO_EMAIL, "a", "1")).parent.name == "v1" + assert cfg.public_dataset_dir(DatasetRef(DO_EMAIL, "b", "2")).parent.name == "v2" + + # A ref with no versioned segment (e.g. protocol 0) must not resolve here. + with pytest.raises(ValueError): + cfg.public_dataset_dir(DatasetRef(DO_EMAIL, "x", "0")) + + # Lay a dataset down in each segment; iteration finds both, tagged correctly. + for name, version in (("a", "1"), ("b", "2")): + dataset_dir = cfg.public_dataset_dir(DatasetRef(DO_EMAIL, name, version)) + dataset_dir.mkdir(parents=True) + (dataset_dir / cfg.metadata_filename).write_text("{}") + + found = { + (r.name, r.protocol_version) + for r in cfg.iter_dataset_refs_all_supported_protocols(DO_EMAIL) + } + assert found == {("a", "1"), ("b", "2")}