Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9,415 changes: 4,708 additions & 4,707 deletions pixi.lock

Large diffs are not rendered by default.

9 changes: 0 additions & 9 deletions resources/shapes/README.md

This file was deleted.

Binary file removed resources/shapes/mne_admin0.parquet
Binary file not shown.
Binary file removed resources/shapes/prt_admin1.parquet
Binary file not shown.
15 changes: 15 additions & 0 deletions src/clio_tools/data_module/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,19 @@
import yaml
from pydantic import BaseModel, Field, model_validator

SEMVER_REGEX = (
r"^v?"
r"(?P<major>0|[1-9]\d*)\."
r"(?P<minor>0|[1-9]\d*)\."
r"(?P<patch>0|[1-9]\d*)"
r"(?:-(?P<prerelease>"
r"(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*"
r"))?"
r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+"
r"(?:\.[0-9a-zA-Z-]+)*))?$"
)


def _find_between(text: str, brackets: str) -> list[str]:
"""Helper to find text inside different bracket configurations."""
Expand Down Expand Up @@ -89,6 +102,8 @@ class ModuleInterface(BaseModel):
"Snakemake pathvars, allowing module input re-wiring."
wildcards: dict[str, str] = Field(default_factory=dict)
"Module wildcards. If provided, these must be present in the keys of either module resources or results."
convention_version: str = Field(pattern=SEMVER_REGEX)
"Modelblocks convention in semantic versioning."

@classmethod
def from_yaml(cls, path: str | Path):
Expand Down
71 changes: 44 additions & 27 deletions tests/data_module_test/io_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from clio_tools.data_module import ModuleInterface, modular_rulegraph_png


def load_yaml(path: Path):
def load_yaml(path: Path) -> dict:
"""File with no wildcards."""
return yaml.safe_load(path.read_text())

Expand All @@ -21,52 +21,69 @@ class TestModuleInterface:
@pytest.fixture(
params=["interface_simple", "interface_wildcard", "interface_no_resources"]
)
def path_interface_test(self, request):
"""Name of the testfile to use."""
@staticmethod
def interface_file_path(request) -> Path:
"""Path fixture for all testfiles."""
return Path(__file__).parent / f"utils/{request.param}.yaml"

def test_from_dict(self, path_interface_test):
"""Loading a simple yaml configuration file."""
data = load_yaml(path_interface_test)
ModuleInterface(**data)
@pytest.fixture
@staticmethod
def interface_dict(interface_file_path: Path) -> dict:
"""Loaded testfile."""
return load_yaml(interface_file_path)

only_interface_with_wildcards = pytest.mark.parametrize(
"interface_file_path", ["interface_wildcard"], indirect=True
)

def test_from_dict(self, interface_dict: dict):
"""Dictionary loading should work."""
assert ModuleInterface(**interface_dict)

def test_from_path(self, path_interface_test):
def test_from_path(self, interface_file_path: Path, interface_dict: dict):
"""Loading data from YAML files should result in no alterations."""
assert ModuleInterface.from_yaml(path_interface_test) == ModuleInterface(
**load_yaml(path_interface_test)
assert ModuleInterface.from_yaml(interface_file_path) == ModuleInterface(
**interface_dict
)

def test_to_mermaid_flowchart(self, path_interface_test):
def test_to_mermaid_flowchart(self, interface_dict: dict):
"""Mermaid graph generation should include all file I/O elements."""
data = load_yaml(path_interface_test)
interface = ModuleInterface(**data)
mermaid_txt = interface.to_mermaid_flowchart(path_interface_test.name)
interface = ModuleInterface(**interface_dict)
mermaid_txt = interface.to_mermaid_flowchart("test")
assert all([i in mermaid_txt for i in interface.pathvars.user_resources])
assert all([i in mermaid_txt for i in interface.pathvars.results])

@pytest.fixture
def interface_w_wilcards(self):
"""File with wildcards configured."""
return load_yaml(Path(__file__).parent / "utils/interface_wildcard.yaml")

def test_wildcard_section_missing(self, interface_w_wilcards):
@pytest.mark.parametrize(
"semver", ["0.1.0", "2.0.0", "3.0.0-alpha", "1.0.0-alpha.beta"]
)
def test_modelblocks_convention_semver(self, semver: str, interface_dict: dict):
"""Modelblocks convention should accept semver, with or without 'v'."""
interface_dict["convention_version"] = semver
assert ModuleInterface(**interface_dict)
interface_dict["convention_version"] = f"v{semver}"
assert ModuleInterface(**interface_dict)

@only_interface_with_wildcards
def test_wildcard_section_missing(self, interface_dict):
"""If filenames specify wildcards, they should appear in the wildcards section."""
del interface_w_wilcards["wildcards"]
del interface_dict["wildcards"]
with pytest.raises(
ValidationError,
match="Wildcards not specified in 'user_resources' or 'results' pathvars:",
):
ModuleInterface(**interface_w_wilcards)
ModuleInterface(**interface_dict)

def test_wildcard_not_in_filename(self, interface_w_wilcards):
@only_interface_with_wildcards
def test_wildcard_not_in_filename(self, interface_dict):
"""All values in the wildcards section should appear in filenames at least once."""
interface_w_wilcards["pathvars"]["user_resources"]["text"]["default"] = (
interface_dict["pathvars"]["user_resources"]["text"]["default"] = (
"<resources>/user/no_wildcard.txt"
)
with pytest.raises(ValidationError, match="Unused wildcards found"):
ModuleInterface(**interface_w_wilcards)
ModuleInterface(**interface_dict)

def test_mermaid_flow_diagram_text(self, interface_w_wilcards):
@only_interface_with_wildcards
def test_mermaid_flow_diagram_text(self, interface_dict):
"""The generated diagram should be correct and use 4 space indentation."""
expected = dedent("""\
---
Expand All @@ -82,7 +99,7 @@ def test_mermaid_flow_diagram_text(self, interface_w_wilcards):
stuff
more_stuff
`")""")
interface = ModuleInterface(**interface_w_wilcards)
interface = ModuleInterface(**interface_dict)
generated = interface.to_mermaid_flowchart("biomass")
assert expected == generated

Expand Down
1 change: 1 addition & 0 deletions tests/data_module_test/utils/interface_no_resources.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
convention_version: 1.0.0
pathvars:
snakemake_defaults:
logs:
Expand Down
1 change: 1 addition & 0 deletions tests/data_module_test/utils/interface_simple.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
convention_version: 1.0.0
pathvars:
snakemake_defaults:
logs:
Expand Down
1 change: 1 addition & 0 deletions tests/data_module_test/utils/interface_wildcard.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
convention_version: 1.0.0
pathvars:
snakemake_defaults:
logs:
Expand Down
Loading