From 1325da53cfa4c9eeee9adc2a5a0d16d943923a0b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 08:06:36 +0000 Subject: [PATCH 01/12] feat(gis_datasets): catalog module for uploading & managing geospatial datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new ``gis_datasets`` module that lets users upload arbitrary GIS files (GeoJSON, Shapefile, KML, GeoTIFF, CSV, etc.), browse them with extracted metadata (CRS, bbox, feature/band counts), and download them. * Local-filesystem storage abstracted behind ``LocalDatasetStorage`` so an S3 backend can drop in later without touching the service. * Best-effort metadata extraction (stdlib JSON for GeoJSON; optional ``fiona``/``rasterio`` paths for Shapefile/KML/raster) — never blocks an upload when a parser is missing. * REST API (``/api/gis_datasets``) + Inertia views (Browse/Create/Edit/Show), permissions group, sidebar menu item, locale namespace, and a health check that probes the storage dir. * Alembic migration for the new ``gis_datasets_dataset`` table with ``branch_labels=("gis_datasets",)``. Framework changes: * ``i18n_manifest._serialize`` emits single-quoted TS string literals to match the project's biome ``quoteStyle: "single"``, so the generated ``keys.generated.ts`` is biome-clean without a follow-up format pass. * ``test_health.test_ready_no_checks_is_healthy`` now tolerates module- registered checks (the old assertion broke whenever a new module registered one). https://claude.ai/code/session_01W384SpjDjwzuZmQq2WV4Qv --- .gitignore | 4 + .../simple_module_hosting/i18n_manifest.py | 33 +- framework/hosting/tests/test_health.py | 7 +- framework/hosting/tests/test_i18n_manifest.py | 14 +- ...e09a1f47_add_gis_datasets_dataset_table.py | 73 +++++ host/pyproject.toml | 2 + modules/gis_datasets/gis_datasets/__init__.py | 1 + .../gis_datasets/contracts/__init__.py | 20 ++ .../gis_datasets/contracts/events.py | 19 ++ .../gis_datasets/contracts/schemas.py | 66 ++++ .../gis_datasets/contracts/service.py | 16 + modules/gis_datasets/gis_datasets/deps.py | 30 ++ .../gis_datasets/endpoints/__init__.py | 0 .../gis_datasets/endpoints/api.py | 155 ++++++++++ .../gis_datasets/endpoints/views.py | 68 ++++ .../gis_datasets/gis_datasets/extractors.py | 180 +++++++++++ .../gis_datasets/gis_datasets/locales/en.json | 67 ++++ modules/gis_datasets/gis_datasets/models.py | 41 +++ modules/gis_datasets/gis_datasets/module.py | 90 ++++++ .../gis_datasets/pages/Browse.tsx | 174 +++++++++++ .../gis_datasets/pages/Create.tsx | 144 +++++++++ .../gis_datasets/gis_datasets/pages/Edit.tsx | 145 +++++++++ .../gis_datasets/gis_datasets/pages/Show.tsx | 112 +++++++ modules/gis_datasets/gis_datasets/py.typed | 0 modules/gis_datasets/gis_datasets/service.py | 155 ++++++++++ modules/gis_datasets/gis_datasets/services.py | 22 ++ modules/gis_datasets/gis_datasets/settings.py | 26 ++ modules/gis_datasets/gis_datasets/storage.py | 83 +++++ modules/gis_datasets/pyproject.toml | 23 ++ .../gis_datasets/tests/test_gis_datasets.py | 290 ++++++++++++++++++ packages/i18n/src/generated-resources.ts | 47 +++ packages/i18n/src/keys.generated.ts | 67 ++++ pyproject.toml | 3 +- 33 files changed, 2155 insertions(+), 22 deletions(-) create mode 100644 host/migrations/versions/7385e09a1f47_add_gis_datasets_dataset_table.py create mode 100644 modules/gis_datasets/gis_datasets/__init__.py create mode 100644 modules/gis_datasets/gis_datasets/contracts/__init__.py create mode 100644 modules/gis_datasets/gis_datasets/contracts/events.py create mode 100644 modules/gis_datasets/gis_datasets/contracts/schemas.py create mode 100644 modules/gis_datasets/gis_datasets/contracts/service.py create mode 100644 modules/gis_datasets/gis_datasets/deps.py create mode 100644 modules/gis_datasets/gis_datasets/endpoints/__init__.py create mode 100644 modules/gis_datasets/gis_datasets/endpoints/api.py create mode 100644 modules/gis_datasets/gis_datasets/endpoints/views.py create mode 100644 modules/gis_datasets/gis_datasets/extractors.py create mode 100644 modules/gis_datasets/gis_datasets/locales/en.json create mode 100644 modules/gis_datasets/gis_datasets/models.py create mode 100644 modules/gis_datasets/gis_datasets/module.py create mode 100644 modules/gis_datasets/gis_datasets/pages/Browse.tsx create mode 100644 modules/gis_datasets/gis_datasets/pages/Create.tsx create mode 100644 modules/gis_datasets/gis_datasets/pages/Edit.tsx create mode 100644 modules/gis_datasets/gis_datasets/pages/Show.tsx create mode 100644 modules/gis_datasets/gis_datasets/py.typed create mode 100644 modules/gis_datasets/gis_datasets/service.py create mode 100644 modules/gis_datasets/gis_datasets/services.py create mode 100644 modules/gis_datasets/gis_datasets/settings.py create mode 100644 modules/gis_datasets/gis_datasets/storage.py create mode 100644 modules/gis_datasets/pyproject.toml create mode 100644 modules/gis_datasets/tests/test_gis_datasets.py diff --git a/.gitignore b/.gitignore index 203e4492..ec3c2dc3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ node_modules/ *.db *.sqlite3 +# Module-managed runtime state (e.g. uploaded GIS dataset files, +# default storage_dir for SM_GIS_DATASETS_STORAGE_DIR). +var/ + # Vite host/static/dist/ diff --git a/framework/hosting/simple_module_hosting/i18n_manifest.py b/framework/hosting/simple_module_hosting/i18n_manifest.py index 49b08f12..69309da1 100644 --- a/framework/hosting/simple_module_hosting/i18n_manifest.py +++ b/framework/hosting/simple_module_hosting/i18n_manifest.py @@ -13,7 +13,6 @@ from __future__ import annotations -import json import logging from pathlib import Path from typing import Any @@ -76,15 +75,12 @@ def emit_frontend_types(registry: I18nRegistry, project_root: Path) -> None: logger.exception("Failed to write generated-resources.ts — frontend types will be stale") -_RESOURCES_HEADER = """\ -// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand. -// Regenerate by booting the host in development mode. -""" +_RESOURCES_HEADER = ( + "// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand.\n" + "// Regenerate by booting the host in development mode." +) -_KEYS_HEADER = """\ -// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand. -// Regenerate by booting the host in development mode. -""" +_KEYS_HEADER = _RESOURCES_HEADER _PLURAL_SUFFIXES: tuple[str, ...] = tuple(f"_{c}" for c in PLURAL_CATEGORIES) @@ -126,7 +122,7 @@ def _render_resources(keys: list[str]) -> str: def _render_keys(keys: list[str]) -> str: """Render the nested ``keys`` constant tree.""" tree = _build_key_tree(keys) - return f"{_KEYS_HEADER}\nexport const keys = {_serialize(tree, indent=0)} as const;\n" + return f"{_KEYS_HEADER}\n\nexport const keys = {_serialize(tree, indent=0)} as const;\n" def _build_key_tree(flat_keys: list[str]) -> dict[str, Any]: @@ -179,21 +175,32 @@ def _insert(tree: dict[str, Any], path: list[str], value: str) -> None: def _serialize(node: Any, *, indent: int) -> str: - """Emit a dict-of-dicts tree as pretty-printed TypeScript object literal.""" + """Emit a dict-of-dicts tree as pretty-printed TypeScript object literal. + + Strings render with single quotes to match the project's biome config + (``javascript.formatter.quoteStyle = "single"``) so the generated file + is biome-clean without a follow-up format pass. + """ if isinstance(node, str): - return json.dumps(node) + return _ts_string_literal(node) if not isinstance(node, dict) or not node: return "{}" pad = " " * indent inner_pad = " " * (indent + 1) lines = ["{"] for k in sorted(node.keys()): - key_repr = k if _is_valid_js_identifier(k) else json.dumps(k) + key_repr = k if _is_valid_js_identifier(k) else _ts_string_literal(k) lines.append(f"{inner_pad}{key_repr}: {_serialize(node[k], indent=indent + 1)},") lines.append(f"{pad}}}") return "\n".join(lines) +def _ts_string_literal(value: str) -> str: + """Render ``value`` as a single-quoted TS string literal.""" + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + def _is_valid_js_identifier(name: str) -> bool: """True if ``name`` can be an unquoted object key in JS (ASCII subset).""" if not name or not (name[0].isalpha() or name[0] in "_$"): diff --git a/framework/hosting/tests/test_health.py b/framework/hosting/tests/test_health.py index b1413d5e..0192b553 100644 --- a/framework/hosting/tests/test_health.py +++ b/framework/hosting/tests/test_health.py @@ -76,7 +76,12 @@ async def check_broken(): assert "connection refused" in data["checks"]["broken_service"]["detail"] async def test_ready_no_checks_is_healthy(self, client: httpx.AsyncClient): + # Modules may register their own health checks (e.g. gis_datasets + # probes its storage dir). Assert the aggregate is healthy and + # every reported check is healthy, rather than requiring zero + # checks — which would break whenever a new module joins boot. resp = await client.get("/health/ready") data = resp.json() assert data["status"] == "healthy" - assert data["checks"] == {} + for name, payload in data["checks"].items(): + assert payload["status"] == "healthy", f"{name} not healthy: {payload}" diff --git a/framework/hosting/tests/test_i18n_manifest.py b/framework/hosting/tests/test_i18n_manifest.py index 4b44734d..e6450ccc 100644 --- a/framework/hosting/tests/test_i18n_manifest.py +++ b/framework/hosting/tests/test_i18n_manifest.py @@ -70,8 +70,8 @@ def test_keys_tree_is_nested(tmp_path: Path) -> None: # Nested structure: keys = { auth: { errors: { not_authenticated: ... } }, products: { ... } } assert "auth:" in text assert "errors:" in text - assert 'not_authenticated: "auth.errors.not_authenticated"' in text - assert 'title: "products.browse.title"' in text + assert "not_authenticated: 'auth.errors.not_authenticated'" in text + assert "title: 'products.browse.title'" in text def test_keys_tree_adds_plural_stems(tmp_path: Path) -> None: @@ -86,9 +86,9 @@ def test_keys_tree_adds_plural_stems(tmp_path: Path) -> None: write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() # Both the concrete variants AND the virtual stem must be emitted. - assert 'count_one: "products.browse.count_one"' in text - assert 'count_other: "products.browse.count_other"' in text - assert 'count: "products.browse.count"' in text + assert "count_one: 'products.browse.count_one'" in text + assert "count_other: 'products.browse.count_other'" in text + assert "count: 'products.browse.count'" in text def test_keys_tree_quotes_non_identifier_segments(tmp_path: Path) -> None: @@ -97,7 +97,7 @@ def test_keys_tree_quotes_non_identifier_segments(tmp_path: Path) -> None: reg._messages = {"en": {"ui.switcher.a-b": "X"}} # hyphen in leaf write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() - assert '"a-b":' in text + assert "'a-b':" in text def test_keys_tree_does_not_overwrite_real_key_with_stem(tmp_path: Path) -> None: @@ -113,4 +113,4 @@ def test_keys_tree_does_not_overwrite_real_key_with_stem(tmp_path: Path) -> None write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() # The real key retains its value; the virtual stem is skipped. - assert 'count: "products.browse.count"' in text + assert "count: 'products.browse.count'" in text diff --git a/host/migrations/versions/7385e09a1f47_add_gis_datasets_dataset_table.py b/host/migrations/versions/7385e09a1f47_add_gis_datasets_dataset_table.py new file mode 100644 index 00000000..9354b2fa --- /dev/null +++ b/host/migrations/versions/7385e09a1f47_add_gis_datasets_dataset_table.py @@ -0,0 +1,73 @@ +"""add gis_datasets dataset table + +Revision ID: 7385e09a1f47 +Revises: b7e1af4c9d02 +Create Date: 2026-04-19 07:50:34.660432 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "7385e09a1f47" +down_revision: str | None = "b7e1af4c9d02" +branch_labels: str | Sequence[str] | None = ("gis_datasets",) +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "gis_datasets_dataset", + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_by", sa.String(length=255), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("slug", sa.String(length=200), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("description", sa.String(length=2000), nullable=True), + sa.Column("original_filename", sa.String(length=255), nullable=False), + sa.Column("mime_type", sa.String(length=127), nullable=True), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("storage_key", sa.String(length=512), nullable=False), + sa.Column("crs", sa.String(length=64), nullable=True), + sa.Column("bbox_min_x", sa.Float(), nullable=True), + sa.Column("bbox_min_y", sa.Float(), nullable=True), + sa.Column("bbox_max_x", sa.Float(), nullable=True), + sa.Column("bbox_max_y", sa.Float(), nullable=True), + sa.Column("feature_count", sa.Integer(), nullable=True), + sa.Column("band_count", sa.Integer(), nullable=True), + sa.Column("extraction_status", sa.String(length=16), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_gis_datasets_dataset")), + ) + op.create_index( + "ix_gis_datasets_dataset_is_deleted", "gis_datasets_dataset", ["is_deleted"], unique=False + ) + op.create_index( + op.f("ix_gis_datasets_dataset_kind"), "gis_datasets_dataset", ["kind"], unique=False + ) + op.create_index( + op.f("ix_gis_datasets_dataset_slug"), "gis_datasets_dataset", ["slug"], unique=True + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_gis_datasets_dataset_slug"), table_name="gis_datasets_dataset") + op.drop_index(op.f("ix_gis_datasets_dataset_kind"), table_name="gis_datasets_dataset") + op.drop_index("ix_gis_datasets_dataset_is_deleted", table_name="gis_datasets_dataset") + op.drop_table("gis_datasets_dataset") + # ### end Alembic commands ### diff --git a/host/pyproject.toml b/host/pyproject.toml index ed15268a..97a9511b 100644 --- a/host/pyproject.toml +++ b/host/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "auth", "dashboard", "products", + "gis-datasets", "python-multipart>=0.0.6", ] @@ -16,3 +17,4 @@ simple-module-hosting = { workspace = true } auth = { workspace = true } products = { workspace = true } dashboard = { workspace = true } +gis-datasets = { workspace = true } diff --git a/modules/gis_datasets/gis_datasets/__init__.py b/modules/gis_datasets/gis_datasets/__init__.py new file mode 100644 index 00000000..d2cb5bfd --- /dev/null +++ b/modules/gis_datasets/gis_datasets/__init__.py @@ -0,0 +1 @@ +"""GisDatasets module.""" diff --git a/modules/gis_datasets/gis_datasets/contracts/__init__.py b/modules/gis_datasets/gis_datasets/contracts/__init__.py new file mode 100644 index 00000000..9c91cf63 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/contracts/__init__.py @@ -0,0 +1,20 @@ +"""GisDatasets contracts — public interface for other modules.""" + +from gis_datasets.contracts.events import DatasetDeleted, DatasetUploaded +from gis_datasets.contracts.schemas import ( + KIND_VALUES, + DatasetKind, + DatasetOut, + DatasetUpdate, +) +from gis_datasets.contracts.service import IDatasetService + +__all__ = [ + "KIND_VALUES", + "DatasetDeleted", + "DatasetKind", + "DatasetOut", + "DatasetUpdate", + "DatasetUploaded", + "IDatasetService", +] diff --git a/modules/gis_datasets/gis_datasets/contracts/events.py b/modules/gis_datasets/gis_datasets/contracts/events.py new file mode 100644 index 00000000..5976dd8b --- /dev/null +++ b/modules/gis_datasets/gis_datasets/contracts/events.py @@ -0,0 +1,19 @@ +"""Domain events emitted by the GisDatasets module.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from simple_module_core.events import Event + + +@dataclass +class DatasetUploaded(Event): + dataset_id: int + name: str + kind: str + + +@dataclass +class DatasetDeleted(Event): + dataset_id: int diff --git a/modules/gis_datasets/gis_datasets/contracts/schemas.py b/modules/gis_datasets/gis_datasets/contracts/schemas.py new file mode 100644 index 00000000..e12cf8d9 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/contracts/schemas.py @@ -0,0 +1,66 @@ +"""SQLModel DTOs for the GisDatasets module.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import ConfigDict +from sqlmodel import Field, SQLModel + +DatasetKind = Literal[ + "vector_geojson", + "vector_shapefile", + "vector_kml", + "raster_geotiff", + "tabular_csv", + "other", +] + + +KIND_VALUES: tuple[str, ...] = ( + "vector_geojson", + "vector_shapefile", + "vector_kml", + "raster_geotiff", + "tabular_csv", + "other", +) + + +class DatasetOut(SQLModel): + """Dataset metadata returned by the API.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + name: str + slug: str + kind: str + description: str | None = None + original_filename: str + mime_type: str | None = None + size_bytes: int + crs: str | None = None + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None + feature_count: int | None = None + band_count: int | None = None + extraction_status: str + created_at: datetime | None = None + updated_at: datetime | None = None + + +class DatasetUpdate(SQLModel): + """Patchable metadata. Files are immutable after upload — re-upload to replace.""" + + name: str | None = Field(default=None, min_length=1, max_length=200) + description: str | None = Field(default=None, max_length=2000) + kind: DatasetKind | None = None + crs: str | None = Field(default=None, max_length=64) + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None diff --git a/modules/gis_datasets/gis_datasets/contracts/service.py b/modules/gis_datasets/gis_datasets/contracts/service.py new file mode 100644 index 00000000..5d2633ac --- /dev/null +++ b/modules/gis_datasets/gis_datasets/contracts/service.py @@ -0,0 +1,16 @@ +"""Dataset service protocol — the public contract other modules depend on.""" + +from __future__ import annotations + +from typing import Protocol + +from gis_datasets.contracts.schemas import DatasetOut, DatasetUpdate + + +class IDatasetService(Protocol): + """Interface for dataset operations.""" + + async def get_all(self) -> list[DatasetOut]: ... + async def get_by_id(self, dataset_id: int) -> DatasetOut | None: ... + async def update(self, dataset_id: int, data: DatasetUpdate) -> DatasetOut | None: ... + async def delete(self, dataset_id: int) -> bool: ... diff --git a/modules/gis_datasets/gis_datasets/deps.py b/modules/gis_datasets/gis_datasets/deps.py new file mode 100644 index 00000000..1e57c594 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/deps.py @@ -0,0 +1,30 @@ +"""FastAPI dependencies for the GisDatasets module.""" + +from __future__ import annotations + +from fastapi import Depends, Request +from simple_module_core.events import EventBus +from simple_module_db.deps import get_db +from sqlalchemy.ext.asyncio import AsyncSession + +from gis_datasets.service import DatasetService +from gis_datasets.storage import LocalDatasetStorage + + +def get_storage(request: Request) -> LocalDatasetStorage: + return request.app.state.gis_datasets.storage + + +async def get_dataset_service( + db: AsyncSession = Depends(get_db), + storage: LocalDatasetStorage = Depends(get_storage), +) -> DatasetService: + return DatasetService(db, storage) + + +def get_event_bus(request: Request) -> EventBus: + return request.app.state.sm.event_bus + + +def get_max_upload_bytes(request: Request) -> int: + return request.app.state.gis_datasets.settings.max_upload_mb * 1024 * 1024 diff --git a/modules/gis_datasets/gis_datasets/endpoints/__init__.py b/modules/gis_datasets/gis_datasets/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/gis_datasets/gis_datasets/endpoints/api.py b/modules/gis_datasets/gis_datasets/endpoints/api.py new file mode 100644 index 00000000..49de549f --- /dev/null +++ b/modules/gis_datasets/gis_datasets/endpoints/api.py @@ -0,0 +1,155 @@ +"""REST API endpoints for the GisDatasets module.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from fastapi.responses import FileResponse +from simple_module_core.events import EventBus +from simple_module_hosting.permissions import RequiresPermission + +from gis_datasets.contracts.events import DatasetDeleted, DatasetUploaded +from gis_datasets.contracts.schemas import KIND_VALUES, DatasetOut, DatasetUpdate +from gis_datasets.deps import ( + get_dataset_service, + get_event_bus, + get_max_upload_bytes, + get_storage, +) +from gis_datasets.service import DatasetService, UploadInput +from gis_datasets.storage import LocalDatasetStorage + +router = APIRouter() + + +@router.get("/", response_model=list[DatasetOut]) +async def list_datasets( + service: DatasetService = Depends(get_dataset_service), +) -> list[DatasetOut]: + return await service.get_all() + + +@router.get("/{dataset_id}", response_model=DatasetOut) +async def get_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), +) -> DatasetOut: + item = await service.get_by_id(dataset_id) + if item is None: + raise HTTPException(status_code=404, detail="Dataset not found") + return item + + +@router.get("/{dataset_id}/download", response_class=FileResponse) +async def download_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), + storage: LocalDatasetStorage = Depends(get_storage), +) -> FileResponse: + info = await service.get_storage_key(dataset_id) + if info is None: + raise HTTPException(status_code=404, detail="Dataset not found") + storage_key, original_filename, mime_type = info + path = storage.absolute(storage_key) + if not path.exists(): + raise HTTPException(status_code=410, detail="Dataset file is missing on disk") + return FileResponse( + path, + filename=original_filename, + media_type=mime_type or "application/octet-stream", + ) + + +@router.post( + "/", + response_model=DatasetOut, + status_code=201, + dependencies=[Depends(RequiresPermission("gis_datasets.upload"))], +) +async def upload_dataset( + name: str = Form(..., min_length=1, max_length=200), + description: str | None = Form(default=None, max_length=2000), + kind: str | None = Form(default=None), + file: UploadFile = File(...), + service: DatasetService = Depends(get_dataset_service), + storage: LocalDatasetStorage = Depends(get_storage), + bus: EventBus = Depends(get_event_bus), + max_upload_bytes: int = Depends(get_max_upload_bytes), +) -> DatasetOut: + if kind is not None and kind not in KIND_VALUES: + raise HTTPException(status_code=422, detail=f"Unknown kind: {kind}") + + storage.ensure_root() + original_filename = file.filename or "upload.bin" + bytes_written = 0 + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = Path(tmp.name) + try: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + bytes_written += len(chunk) + if bytes_written > max_upload_bytes: + raise HTTPException( + status_code=413, + detail=f"Upload exceeds {max_upload_bytes} bytes", + ) + tmp.write(chunk) + except HTTPException: + tmp.close() + tmp_path.unlink(missing_ok=True) + raise + + try: + dataset = await service.register_upload( + UploadInput( + name=name, + original_filename=original_filename, + temp_path=tmp_path, + size_bytes=bytes_written, + mime_type=file.content_type, + description=description, + kind=kind, + ) + ) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + await bus.publish(DatasetUploaded(dataset_id=dataset.id, name=dataset.name, kind=dataset.kind)) + return dataset + + +@router.patch( + "/{dataset_id}", + response_model=DatasetOut, + dependencies=[Depends(RequiresPermission("gis_datasets.edit"))], +) +async def update_dataset( + dataset_id: int, + data: DatasetUpdate, + service: DatasetService = Depends(get_dataset_service), +) -> DatasetOut: + item = await service.update(dataset_id, data) + if item is None: + raise HTTPException(status_code=404, detail="Dataset not found") + return item + + +@router.delete( + "/{dataset_id}", + status_code=204, + dependencies=[Depends(RequiresPermission("gis_datasets.delete"))], +) +async def delete_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), + bus: EventBus = Depends(get_event_bus), +) -> None: + deleted = await service.delete(dataset_id) + if not deleted: + raise HTTPException(status_code=404, detail="Dataset not found") + await bus.publish(DatasetDeleted(dataset_id=dataset_id)) diff --git a/modules/gis_datasets/gis_datasets/endpoints/views.py b/modules/gis_datasets/gis_datasets/endpoints/views.py new file mode 100644 index 00000000..04b98da8 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/endpoints/views.py @@ -0,0 +1,68 @@ +"""Inertia view endpoints for the GisDatasets module.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from inertia import InertiaResponse +from simple_module_hosting.inertia_deps import InertiaDep +from simple_module_hosting.permissions import RequiresPermission + +from gis_datasets.deps import get_dataset_service +from gis_datasets.service import DatasetService + +router = APIRouter() + + +@router.get("/", response_model=None) +async def browse( + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + items = await service.get_all() + return await inertia.render( + "GisDatasets/Browse", + {"datasets": [item.model_dump(mode="json") for item in items]}, + ) + + +@router.get( + "/create", + response_model=None, + dependencies=[Depends(RequiresPermission("gis_datasets.upload"))], +) +async def create_view(inertia: InertiaDep) -> InertiaResponse: + return await inertia.render("GisDatasets/Create") + + +@router.get("/{dataset_id}", response_model=None) +async def show_view( + dataset_id: int, + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + item = await service.get_by_id(dataset_id) + if item is None: + return await inertia.render( + "GisDatasets/Browse", + {"datasets": [], "error": "Dataset not found"}, + ) + return await inertia.render("GisDatasets/Show", {"dataset": item.model_dump(mode="json")}) + + +@router.get( + "/{dataset_id}/edit", + response_model=None, + dependencies=[Depends(RequiresPermission("gis_datasets.edit"))], +) +async def edit_view( + dataset_id: int, + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + item = await service.get_by_id(dataset_id) + if item is None: + return await inertia.render( + "GisDatasets/Browse", + {"datasets": [], "error": "Dataset not found"}, + ) + return await inertia.render("GisDatasets/Edit", {"dataset": item.model_dump(mode="json")}) diff --git a/modules/gis_datasets/gis_datasets/extractors.py b/modules/gis_datasets/gis_datasets/extractors.py new file mode 100644 index 00000000..64a9024f --- /dev/null +++ b/modules/gis_datasets/gis_datasets/extractors.py @@ -0,0 +1,180 @@ +"""Best-effort metadata extraction for uploaded GIS datasets. + +The stdlib JSON path covers GeoJSON without any extra dependency. Optional +``fiona`` and ``rasterio`` extras enrich extraction for shapefiles, KML, and +rasters when available. Every extractor degrades to ``status="manual"`` so +the catalog never blocks an upload because a parser is missing. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class ExtractedMeta: + crs: str | None = None + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None + feature_count: int | None = None + band_count: int | None = None + status: str = "manual" + + +def kind_for_filename(filename: str) -> str: + """Map a filename to a coarse kind label. + + The mapping is intentionally cheap — anything we don't recognise becomes + ``"other"`` so the upload still lands and the user can correct the kind + via the edit form. + """ + suffix = Path(filename).suffix.lower() + if suffix == ".geojson" or suffix == ".json": + return "vector_geojson" + if suffix in {".shp", ".zip"}: + return "vector_shapefile" + if suffix in {".kml", ".kmz"}: + return "vector_kml" + if suffix in {".tif", ".tiff"}: + return "raster_geotiff" + if suffix == ".csv": + return "tabular_csv" + return "other" + + +def extract_metadata(path: Path, kind: str) -> ExtractedMeta: + """Dispatch to a kind-specific extractor. + + Never raises — extraction failures collapse to ``status="failed"`` so + the upload itself succeeds and the user can fill the metadata in. + """ + try: + if kind == "vector_geojson": + return _extract_geojson(path) + if kind == "raster_geotiff": + return _extract_raster(path) + if kind in {"vector_shapefile", "vector_kml"}: + return _extract_via_fiona(path) + except Exception: + return ExtractedMeta(status="failed") + return ExtractedMeta(status="manual") + + +def _extract_geojson(path: Path) -> ExtractedMeta: + with path.open("rb") as fp: + doc = json.load(fp) + + features = _features(doc) + bbox = doc.get("bbox") if isinstance(doc, dict) else None + if not bbox: + bbox = _compute_bbox(features) + + if bbox is None: + return ExtractedMeta( + crs="EPSG:4326", + feature_count=len(features), + status="partial", + ) + return ExtractedMeta( + crs="EPSG:4326", + bbox_min_x=float(bbox[0]), + bbox_min_y=float(bbox[1]), + bbox_max_x=float(bbox[2]), + bbox_max_y=float(bbox[3]), + feature_count=len(features), + status="ok", + ) + + +def _features(doc: object) -> list[dict]: + if isinstance(doc, dict): + if doc.get("type") == "FeatureCollection": + features = doc.get("features", []) + return [f for f in features if isinstance(f, dict)] + if doc.get("type") == "Feature": + return [doc] + return [] + + +def _compute_bbox(features: list[dict]) -> tuple[float, float, float, float] | None: + min_x = min_y = float("inf") + max_x = max_y = float("-inf") + for feature in features: + geom = feature.get("geometry") if isinstance(feature, dict) else None + if not isinstance(geom, dict): + continue + for x, y in _iter_coords(geom): + min_x = min(min_x, x) + min_y = min(min_y, y) + max_x = max(max_x, x) + max_y = max(max_y, y) + if min_x == float("inf"): + return None + return min_x, min_y, max_x, max_y + + +def _iter_coords(geom: dict): + coords = geom.get("coordinates") + geom_type = geom.get("type") + if geom_type == "GeometryCollection": + for sub in geom.get("geometries", []): + if isinstance(sub, dict): + yield from _iter_coords(sub) + return + yield from _walk(coords) + + +def _walk(node: object): + if ( + isinstance(node, list) + and len(node) >= 2 + and all(isinstance(v, (int, float)) for v in node[:2]) + ): + yield float(node[0]), float(node[1]) + return + if isinstance(node, list): + for child in node: + yield from _walk(child) + + +def _extract_via_fiona(path: Path) -> ExtractedMeta: + try: + import fiona # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError: + return ExtractedMeta(status="manual") + + with fiona.open(str(path)) as src: + bounds = src.bounds + crs = src.crs.get("init") if hasattr(src.crs, "get") else str(src.crs) if src.crs else None + return ExtractedMeta( + crs=crs, + bbox_min_x=float(bounds[0]), + bbox_min_y=float(bounds[1]), + bbox_max_x=float(bounds[2]), + bbox_max_y=float(bounds[3]), + feature_count=len(src), + status="ok", + ) + + +def _extract_raster(path: Path) -> ExtractedMeta: + try: + import rasterio # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError: + return ExtractedMeta(status="manual") + + with rasterio.open(str(path)) as src: + bounds = src.bounds + return ExtractedMeta( + crs=str(src.crs) if src.crs else None, + bbox_min_x=float(bounds.left), + bbox_min_y=float(bounds.bottom), + bbox_max_x=float(bounds.right), + bbox_max_y=float(bounds.top), + band_count=src.count, + status="ok", + ) diff --git a/modules/gis_datasets/gis_datasets/locales/en.json b/modules/gis_datasets/gis_datasets/locales/en.json new file mode 100644 index 00000000..90da5f5e --- /dev/null +++ b/modules/gis_datasets/gis_datasets/locales/en.json @@ -0,0 +1,67 @@ +{ + "browse": { + "title": "GIS Datasets", + "description": "Upload and manage geospatial datasets", + "new_button": "Upload Dataset", + "empty_title": "No datasets yet", + "empty_description": "Upload your first geospatial dataset to get started.", + "create_button": "Upload Dataset" + }, + "table": { + "name": "Name", + "kind": "Kind", + "crs": "CRS", + "bbox": "Bounding Box", + "size": "Size", + "actions": "Actions" + }, + "form": { + "name_label": "Name", + "name_placeholder": "Descriptive name for the dataset", + "description_label": "Description", + "description_placeholder": "Optional notes about the dataset", + "kind_label": "Kind", + "kind_auto": "Auto-detect from filename", + "file_label": "File", + "crs_label": "CRS", + "cancel_button": "Cancel" + }, + "create": { + "title": "Upload Dataset", + "description": "Add a geospatial dataset to the catalog", + "submit_button": "Upload", + "submitting_button": "Uploading..." + }, + "edit": { + "title": "Edit: {name}", + "description": "Update dataset metadata", + "back_button": "Back to Datasets", + "submit_button": "Save Changes", + "submitting_button": "Saving..." + }, + "show": { + "download": "Download", + "edit": "Edit", + "kind": "Kind", + "original_file": "Original filename", + "size": "Size", + "mime_type": "MIME type", + "crs": "CRS", + "bbox": "Bounding box", + "features": "Feature count", + "bands": "Band count", + "extraction_status": "Extraction status" + }, + "toasts": { + "created": "Dataset uploaded", + "updated": "Dataset updated", + "deleted": "Dataset deleted" + }, + "validation": { + "name_required": "Name is required", + "file_required": "Please choose a file to upload" + }, + "errors": { + "not_found": "Dataset not found" + } +} diff --git a/modules/gis_datasets/gis_datasets/models.py b/modules/gis_datasets/gis_datasets/models.py new file mode 100644 index 00000000..7b00e4a9 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/models.py @@ -0,0 +1,41 @@ +"""SQLModel tables for the GisDatasets module.""" + +from __future__ import annotations + +from simple_module_db.base import create_module_base +from simple_module_db.mixins import AuditMixin, SoftDeleteMixin +from sqlalchemy import Index +from sqlmodel import Field + +# Provider is auto-detected from SM_DATABASE_URL (falls back to SQLite). +# On PostgreSQL this gives the module its own `gis_datasets` schema; on SQLite +# all modules share one schema, so __tablename__ is prefixed for isolation. +Base = create_module_base("gis_datasets") + + +class Dataset(Base, AuditMixin, SoftDeleteMixin, table=True): # ty: ignore[unsupported-base] + """A geospatial dataset uploaded into the catalog.""" + + __tablename__ = "gis_datasets_dataset" + + id: int | None = Field(default=None, primary_key=True) + name: str = Field(max_length=200) + slug: str = Field(max_length=200, unique=True, index=True) + kind: str = Field(max_length=32, index=True) + description: str | None = Field(default=None, max_length=2000) + + original_filename: str = Field(max_length=255) + mime_type: str | None = Field(default=None, max_length=127) + size_bytes: int = Field(default=0) + storage_key: str = Field(max_length=512) + + crs: str | None = Field(default=None, max_length=64) + bbox_min_x: float | None = Field(default=None) + bbox_min_y: float | None = Field(default=None) + bbox_max_x: float | None = Field(default=None) + bbox_max_y: float | None = Field(default=None) + feature_count: int | None = Field(default=None) + band_count: int | None = Field(default=None) + extraction_status: str = Field(default="manual", max_length=16) + + __table_args__ = (Index("ix_gis_datasets_dataset_is_deleted", "is_deleted"),) diff --git a/modules/gis_datasets/gis_datasets/module.py b/modules/gis_datasets/gis_datasets/module.py new file mode 100644 index 00000000..0f95d03d --- /dev/null +++ b/modules/gis_datasets/gis_datasets/module.py @@ -0,0 +1,90 @@ +"""GisDatasets module definition.""" + +from __future__ import annotations + +import importlib.resources +from pathlib import Path + +from fastapi import APIRouter, FastAPI +from simple_module_core.health import HealthCheck, HealthCheckResult, HealthRegistry, HealthStatus +from simple_module_core.menu import MenuItem, MenuRegistry, MenuSection +from simple_module_core.module import ModuleBase, ModuleMeta +from simple_module_core.permissions import PermissionRegistry + +from gis_datasets.storage import LocalDatasetStorage + + +class GisDatasetsModule(ModuleBase): + meta = ModuleMeta( + name="GisDatasets", + route_prefix="/api/gis_datasets", + view_prefix="/gis_datasets", + ) + + def __init__(self) -> None: + self._storage: LocalDatasetStorage | None = None + + def register_settings(self, app: FastAPI) -> None: + from gis_datasets.services import GisDatasetsServices + from gis_datasets.settings import GisDatasetsSettings + + settings = GisDatasetsSettings() + storage = LocalDatasetStorage(Path(settings.storage_dir)) + app.state.gis_datasets = GisDatasetsServices(settings=settings, storage=storage) + self._storage = storage + + def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: + from gis_datasets.endpoints.api import router as api + from gis_datasets.endpoints.views import router as views + + api_router.include_router(api) + view_router.include_router(views) + + def register_menu_items(self, registry: MenuRegistry) -> None: + registry.add( + MenuItem( + label="GIS Datasets", + url="/gis_datasets", + icon="layers", + order=40, + section=MenuSection.SIDEBAR, + ) + ) + + def register_permissions(self, registry: PermissionRegistry) -> None: + registry.add_group( + "GIS Datasets", + [ + "gis_datasets.view", + "gis_datasets.upload", + "gis_datasets.edit", + "gis_datasets.delete", + ], + ) + + def register_health_checks(self, registry: HealthRegistry) -> None: + storage = self._storage + + async def check_storage() -> HealthCheckResult: + if storage is None: + return HealthCheckResult( + status=HealthStatus.UNHEALTHY, + detail="Storage backend not initialised", + ) + if not storage.is_writable(): + return HealthCheckResult( + status=HealthStatus.UNHEALTHY, + detail=f"Storage directory not writable: {storage.root}", + ) + return HealthCheckResult(status=HealthStatus.HEALTHY) + + registry.add(HealthCheck(name="gis_datasets.storage", check=check_storage)) + + def locale_dirs(self) -> dict[str, Path]: + base = Path(str(importlib.resources.files(__package__) / "locales")) + return {"gis_datasets": base} + + async def on_startup(self, app: FastAPI) -> None: + # Make sure the storage directory exists once the app is fully booted + # so the health check has a writable target on first request. + app.state.gis_datasets.storage.ensure_root() diff --git a/modules/gis_datasets/gis_datasets/pages/Browse.tsx b/modules/gis_datasets/gis_datasets/pages/Browse.tsx new file mode 100644 index 00000000..4c10c179 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/pages/Browse.tsx @@ -0,0 +1,174 @@ +import { Link, router, usePage } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { PageShell } from '@simple-module/ui/components/PageShell'; +import { Badge } from '@simple-module/ui/components/ui/badge'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Card } from '@simple-module/ui/components/ui/card'; +import { + Empty, + EmptyDescription, + EmptyMedia, + EmptyTitle, +} from '@simple-module/ui/components/ui/empty'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@simple-module/ui/components/ui/table'; +import { usePermissions } from '@simple-module/ui/hooks/use-permissions'; +import { AuthenticatedLayout } from '@simple-module/ui/layouts/AuthenticatedLayout'; +import { Layers, Plus, Trash2 } from 'lucide-react'; + +interface Dataset { + id: number; + name: string; + slug: string; + kind: string; + size_bytes: number; + crs: string | null; + bbox_min_x: number | null; + bbox_min_y: number | null; + bbox_max_x: number | null; + bbox_max_y: number | null; + extraction_status: string; + created_at: string | null; +} + +interface Props { + datasets: Dataset[]; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function formatBbox(d: Dataset): string { + if ( + d.bbox_min_x === null || + d.bbox_min_y === null || + d.bbox_max_x === null || + d.bbox_max_y === null + ) { + return '—'; + } + return `[${d.bbox_min_x.toFixed(3)}, ${d.bbox_min_y.toFixed(3)}, ${d.bbox_max_x.toFixed(3)}, ${d.bbox_max_y.toFixed(3)}]`; +} + +function Browse() { + const { datasets } = usePage<{ props: Props }>().props as unknown as Props; + const { t } = useT(); + const { can } = usePermissions(); + const canUpload = can('gis_datasets.upload'); + const canDelete = can('gis_datasets.delete'); + + function handleDelete(dataset: Dataset) { + router.delete(`/api/gis_datasets/${dataset.id}`, { preserveScroll: true }); + } + + return ( + + + + {t(keys.gis_datasets.browse.new_button)} + + + ) : undefined + } + > + + + + + {t(keys.gis_datasets.table.name)} + {t(keys.gis_datasets.table.kind)} + + {t(keys.gis_datasets.table.crs)} + + + {t(keys.gis_datasets.table.bbox)} + + {t(keys.gis_datasets.table.size)} + + {t(keys.gis_datasets.table.actions)} + + + + + {datasets.map((dataset) => ( + + + + {dataset.name} + + + + {dataset.kind} + + + {dataset.crs || '—'} + + + {formatBbox(dataset)} + + + {formatBytes(dataset.size_bytes)} + + + {canDelete && ( + + )} + + + ))} + {datasets.length === 0 && ( + + + + + + + {t(keys.gis_datasets.browse.empty_title)} + + {t(keys.gis_datasets.browse.empty_description)} + + {canUpload && ( + + )} + + + + )} + +
+ + + ); +} + +Browse.layout = (page: React.ReactNode) => {page}; +export default Browse; diff --git a/modules/gis_datasets/gis_datasets/pages/Create.tsx b/modules/gis_datasets/gis_datasets/pages/Create.tsx new file mode 100644 index 00000000..0d886668 --- /dev/null +++ b/modules/gis_datasets/gis_datasets/pages/Create.tsx @@ -0,0 +1,144 @@ +import { Link, router } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { PageShell } from '@simple-module/ui/components/PageShell'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Card, CardContent } from '@simple-module/ui/components/ui/card'; +import { Input } from '@simple-module/ui/components/ui/input'; +import { Label } from '@simple-module/ui/components/ui/label'; +import { Textarea } from '@simple-module/ui/components/ui/textarea'; +import { AuthenticatedLayout } from '@simple-module/ui/layouts/AuthenticatedLayout'; +import { useState } from 'react'; +import { toast } from 'sonner'; + +const KINDS = [ + '', + 'vector_geojson', + 'vector_shapefile', + 'vector_kml', + 'raster_geotiff', + 'tabular_csv', + 'other', +]; + +function Create() { + const { t } = useT(); + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [kind, setKind] = useState(''); + const [file, setFile] = useState(null); + const [submitting, setSubmitting] = useState(false); + + function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + if (!file) { + toast.error(t(keys.gis_datasets.validation.file_required)); + return; + } + if (!name.trim()) { + toast.error(t(keys.gis_datasets.validation.name_required)); + return; + } + const data = new FormData(); + data.append('name', name); + if (description) data.append('description', description); + if (kind) data.append('kind', kind); + data.append('file', file); + + setSubmitting(true); + router.post('/api/gis_datasets/', data, { + forceFormData: true, + onSuccess: () => toast.success(t(keys.gis_datasets.toasts.created)), + onError: (errs) => { + const first = Object.values(errs)[0]; + if (first) toast.error(String(first)); + }, + onFinish: () => setSubmitting(false), + }); + } + + return ( + + {t(keys.gis_datasets.form.cancel_button)} + + } + > + + +
+
+ + setName(e.target.value)} + placeholder={t(keys.gis_datasets.form.name_placeholder)} + maxLength={200} + required + /> +
+ +
+ + +
+ +
+ + setFile(e.target.files?.[0] ?? null)} + required + /> +
+ +
+ +