diff --git a/.gitignore b/.gitignore index 5a970b4c..5bca149a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ node_modules/ *.db *.sqlite3 +# Module-managed runtime state (e.g. uploaded dataset files, +# default storage_dir for SM_DATASETS_STORAGE_DIR). +var/ + # file_storage filesystem backend default root (override via SM_FILE_STORAGE_FS_ROOT_PATH). uploads/ diff --git a/framework/hosting/simple_module_hosting/i18n_manifest.py b/framework/hosting/simple_module_hosting/i18n_manifest.py index 37b01243..ec1e1a44 100644 --- a/framework/hosting/simple_module_hosting/i18n_manifest.py +++ b/framework/hosting/simple_module_hosting/i18n_manifest.py @@ -75,15 +75,12 @@ def emit_frontend_types(registry: I18nRegistry, project_root: Path) -> None: logger.exception("Failed to write generated-resources.ts — frontend types will be stale") -_RESOURCES_HEADER = """\ -// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand. -// Regenerate by booting the host in development mode. -""" +_RESOURCES_HEADER = ( + "// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand.\n" + "// Regenerate by booting the host in development mode." +) -_KEYS_HEADER = """\ -// AUTO-GENERATED by simple_module_hosting.i18n_manifest — do not edit by hand. -// Regenerate by booting the host in development mode. -""" +_KEYS_HEADER = _RESOURCES_HEADER _PLURAL_SUFFIXES: tuple[str, ...] = tuple(f"_{c}" for c in PLURAL_CATEGORIES) @@ -125,7 +122,7 @@ def _render_resources(keys: list[str]) -> str: def _render_keys(keys: list[str]) -> str: """Render the nested ``keys`` constant tree.""" tree = _build_key_tree(keys) - return f"{_KEYS_HEADER}\nexport const keys = {_serialize(tree, indent=0)} as const;\n" + return f"{_KEYS_HEADER}\n\nexport const keys = {_serialize(tree, indent=0)} as const;\n" def _build_key_tree(flat_keys: list[str]) -> dict[str, Any]: diff --git a/framework/hosting/tests/test_health.py b/framework/hosting/tests/test_health.py index b1413d5e..0192b553 100644 --- a/framework/hosting/tests/test_health.py +++ b/framework/hosting/tests/test_health.py @@ -76,7 +76,12 @@ async def check_broken(): assert "connection refused" in data["checks"]["broken_service"]["detail"] async def test_ready_no_checks_is_healthy(self, client: httpx.AsyncClient): + # Modules may register their own health checks (e.g. gis_datasets + # probes its storage dir). Assert the aggregate is healthy and + # every reported check is healthy, rather than requiring zero + # checks — which would break whenever a new module joins boot. resp = await client.get("/health/ready") data = resp.json() assert data["status"] == "healthy" - assert data["checks"] == {} + for name, payload in data["checks"].items(): + assert payload["status"] == "healthy", f"{name} not healthy: {payload}" diff --git a/host/migrations/versions/dad9a134290f_add_datasets_table.py b/host/migrations/versions/dad9a134290f_add_datasets_table.py new file mode 100644 index 00000000..8114b9bb --- /dev/null +++ b/host/migrations/versions/dad9a134290f_add_datasets_table.py @@ -0,0 +1,69 @@ +"""add datasets table + +Revision ID: dad9a134290f +Revises: b7e1af4c9d02 +Create Date: 2026-04-19 11:51:19.145277 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "dad9a134290f" +down_revision: str | None = "b7e1af4c9d02" +branch_labels: str | Sequence[str] | None = ("datasets",) +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "datasets_dataset", + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_by", sa.String(length=255), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column("slug", sa.String(length=200), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("description", sa.String(length=2000), nullable=True), + sa.Column("original_filename", sa.String(length=255), nullable=False), + sa.Column("mime_type", sa.String(length=127), nullable=True), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("storage_key", sa.String(length=512), nullable=False), + sa.Column("crs", sa.String(length=64), nullable=True), + sa.Column("bbox_min_x", sa.Float(), nullable=True), + sa.Column("bbox_min_y", sa.Float(), nullable=True), + sa.Column("bbox_max_x", sa.Float(), nullable=True), + sa.Column("bbox_max_y", sa.Float(), nullable=True), + sa.Column("feature_count", sa.Integer(), nullable=True), + sa.Column("band_count", sa.Integer(), nullable=True), + sa.Column("extraction_status", sa.String(length=16), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_datasets_dataset")), + ) + op.create_index( + "ix_datasets_dataset_is_deleted", "datasets_dataset", ["is_deleted"], unique=False + ) + op.create_index(op.f("ix_datasets_dataset_kind"), "datasets_dataset", ["kind"], unique=False) + op.create_index(op.f("ix_datasets_dataset_slug"), "datasets_dataset", ["slug"], unique=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_datasets_dataset_slug"), table_name="datasets_dataset") + op.drop_index(op.f("ix_datasets_dataset_kind"), table_name="datasets_dataset") + op.drop_index("ix_datasets_dataset_is_deleted", table_name="datasets_dataset") + op.drop_table("datasets_dataset") + # ### end Alembic commands ### diff --git a/host/pyproject.toml b/host/pyproject.toml index 4e29e6d3..2b74ec5f 100644 --- a/host/pyproject.toml +++ b/host/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "auth", "dashboard", "products", + "datasets", "permissions", "background-tasks", "file-storage", @@ -21,6 +22,7 @@ simple-module-hosting = { workspace = true } auth = { workspace = true } products = { workspace = true } dashboard = { workspace = true } +datasets = { workspace = true } permissions = { workspace = true } background-tasks = { workspace = true } file-storage = { workspace = true } diff --git a/modules/datasets/datasets/__init__.py b/modules/datasets/datasets/__init__.py new file mode 100644 index 00000000..fb7083d7 --- /dev/null +++ b/modules/datasets/datasets/__init__.py @@ -0,0 +1 @@ +"""Datasets module.""" diff --git a/modules/datasets/datasets/constants.py b/modules/datasets/datasets/constants.py new file mode 100644 index 00000000..584604a4 --- /dev/null +++ b/modules/datasets/datasets/constants.py @@ -0,0 +1,146 @@ +"""Stable identifiers for the datasets module. + +Every string that identifies a permission, role, page, module dependency, +event, route, or config key lives here. Everything else in the module +imports from this file, so ``rg`` can prove the module has no scattered +magic strings. +""" + +from __future__ import annotations + +from typing import Final + +# ── Module identity ────────────────────────────────────────────────── +MODULE_NAME: Final = "datasets" +MODULE_PASCAL: Final = "Datasets" +MODULE_DISPLAY_NAME: Final = "Datasets" + +# ── Configuration ──────────────────────────────────────────────────── +ENV_PREFIX: Final = "SM_DATASETS_" + +# ── Routing ────────────────────────────────────────────────────────── +ROUTE_PREFIX_API: Final = "/api/datasets" +ROUTE_PREFIX_VIEW: Final = "/datasets" + +# REST sub-paths (joined to ROUTE_PREFIX_API by the router) +PATH_DOWNLOAD: Final = "/{dataset_id}/download" +PATH_DATASET: Final = "/{dataset_id}" + +# View routes (relative to ROUTE_PREFIX_VIEW, used in redirects) +REDIRECT_BROWSE: Final = "/datasets/" + +# ── Module dependencies (for ModuleMeta.depends_on) ────────────────── +MODULE_FILE_STORAGE: Final = "FileStorage" +MODULE_BACKGROUND_TASKS: Final = "BackgroundTasks" + +# ── i18n ───────────────────────────────────────────────────────────── +LOCALE_NAMESPACE: Final = MODULE_NAME + +# ── Inertia page identifiers ───────────────────────────────────────── +PAGE_BROWSE: Final = "Datasets/Browse" +PAGE_CREATE: Final = "Datasets/Create" +PAGE_EDIT: Final = "Datasets/Edit" +PAGE_SHOW: Final = "Datasets/Show" + +# ── Permissions ────────────────────────────────────────────────────── +PERM_DATASETS_VIEW: Final = "datasets.view" +PERM_DATASETS_UPLOAD: Final = "datasets.upload" +PERM_DATASETS_EDIT: Final = "datasets.edit" +PERM_DATASETS_DELETE: Final = "datasets.delete" + +PERMISSION_GROUP: Final = "Datasets" +ALL_PERMISSIONS: Final = ( + PERM_DATASETS_VIEW, + PERM_DATASETS_UPLOAD, + PERM_DATASETS_EDIT, + PERM_DATASETS_DELETE, +) + +# ── Menu ───────────────────────────────────────────────────────────── +MENU_LABEL: Final = "Datasets" +MENU_ICON: Final = "layers" +MENU_ORDER: Final = 40 + +# ── Celery tasks ───────────────────────────────────────────────────── +TASK_EXTRACT_METADATA: Final = "datasets.extract_metadata" + +# ── Feature flags ──────────────────────────────────────────────────── +FLAG_AUTO_EXTRACT: Final = "datasets.auto_extract" +FLAG_ALLOW_RASTER_UPLOADS: Final = "datasets.allow_raster_uploads" + +# ── Settings keys (registered with the settings module) ────────────── +SETTING_MAX_UPLOAD_MB: Final = "datasets.max_upload_mb" +SETTING_DEFAULT_KIND: Final = "datasets.default_kind" + +# ── Roles ──────────────────────────────────────────────────────────── +ROLE_USER: Final = "user" +ROLE_ADMIN: Final = "admin" + +# Permissions granted to plain ``user`` role via ``map_role``. ``admin`` +# inherits everything through the framework's wildcard, so we don't list +# it explicitly here. +USER_ROLE_PERMISSIONS: Final = ( + PERM_DATASETS_VIEW, + PERM_DATASETS_UPLOAD, +) + +# ── Health check ───────────────────────────────────────────────────── +HEALTH_CHECK_STORAGE: Final = "datasets.storage" + +# ── Storage ────────────────────────────────────────────────────────── +STORAGE_KEY_PREFIX: Final = "datasets/" + +# ── Database ───────────────────────────────────────────────────────── +SCHEMA_NAME: Final = "datasets" +TABLE_DATASET: Final = "datasets_dataset" + +# ── Defaults ───────────────────────────────────────────────────────── +DEFAULT_MAX_UPLOAD_MB: Final = 256 +DEFAULT_UPLOAD_CHUNK_SIZE: Final = 1024 * 1024 # 1 MB +DEFAULT_PRESIGN_TTL_SECONDS: Final = 300 # 5 minutes +DEFAULT_FALLBACK_FILENAME: Final = "upload.bin" +DEFAULT_MIME_TYPE: Final = "application/octet-stream" +DEFAULT_GEOJSON_CRS: Final = "EPSG:4326" + + +class DatasetKind: + """Built-in dataset kinds. + + The ``kind`` column is a free-form string so new providers can register + their own labels, but these are the ones the shipped extractors and + the UI know about. + """ + + VECTOR_GEOJSON: Final = "vector_geojson" + VECTOR_SHAPEFILE: Final = "vector_shapefile" + VECTOR_KML: Final = "vector_kml" + RASTER_GEOTIFF: Final = "raster_geotiff" + TABULAR_CSV: Final = "tabular_csv" + OTHER: Final = "other" + + +ALL_KINDS: Final = ( + DatasetKind.VECTOR_GEOJSON, + DatasetKind.VECTOR_SHAPEFILE, + DatasetKind.VECTOR_KML, + DatasetKind.RASTER_GEOTIFF, + DatasetKind.TABULAR_CSV, + DatasetKind.OTHER, +) + + +class ExtractionStatus: + """Values stored in ``Dataset.extraction_status``. + + ``pending`` is set by the upload endpoint; ``ok``/``partial``/``failed`` + are set by the Celery worker once extraction completes. ``manual`` is + the fallback for kinds the extractor doesn't understand; ``not_found`` + is only used as a task-result marker (never stored). + """ + + PENDING: Final = "pending" + OK: Final = "ok" + PARTIAL: Final = "partial" + FAILED: Final = "failed" + MANUAL: Final = "manual" + NOT_FOUND: Final = "not_found" diff --git a/modules/datasets/datasets/contracts/__init__.py b/modules/datasets/datasets/contracts/__init__.py new file mode 100644 index 00000000..8889bec9 --- /dev/null +++ b/modules/datasets/datasets/contracts/__init__.py @@ -0,0 +1,51 @@ +"""Datasets contracts — public interface for other modules. + +Downstream modules should import from here, not from +``datasets.models`` or ``datasets.service`` internals. The SM009 +diagnostic enforces framework→plugin purity; this package is the +supported surface for plugin→plugin coupling:: + + from datasets.contracts import ( + DatasetOut, # DTO returned by service lookups + DatasetFile, # handle for stored file access + DatasetUploaded, # event subscribers listen for + download_url, # URL helper for UIs + ) + +For the FastAPI dependency, prefer +``from datasets.deps import DatasetServiceDep``. Type-hint against the +concrete ``DatasetService`` — the module is single-impl, so there's no +Protocol abstraction (matching the framework's "ship a Protocol only +for real extension points" rule). +""" + +from datasets.contracts.events import DatasetDeleted, DatasetUploaded +from datasets.contracts.files import DatasetFile +from datasets.contracts.schemas import ( + KIND_VALUES, + DatasetKind, + DatasetOut, + DatasetUpdate, +) +from datasets.contracts.urls import ( + API_PREFIX, + VIEW_PREFIX, + detail_url, + download_url, + show_url, +) + +__all__ = [ + "API_PREFIX", + "KIND_VALUES", + "VIEW_PREFIX", + "DatasetDeleted", + "DatasetFile", + "DatasetKind", + "DatasetOut", + "DatasetUpdate", + "DatasetUploaded", + "detail_url", + "download_url", + "show_url", +] diff --git a/modules/datasets/datasets/contracts/events.py b/modules/datasets/datasets/contracts/events.py new file mode 100644 index 00000000..19f4c5c9 --- /dev/null +++ b/modules/datasets/datasets/contracts/events.py @@ -0,0 +1,28 @@ +"""Domain events emitted by the Datasets module. + +Events carry the handful of fields subscribers most often route on +(``slug``, ``kind``) so downstream handlers don't have to make a round +trip to ``IDatasetService`` just to decide whether the event is for +them. Anything beyond this — CRS, bbox, feature counts — is still a +``get_by_id`` away. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from simple_module_core.events import Event + + +@dataclass +class DatasetUploaded(Event): + dataset_id: int + name: str + slug: str + kind: str + + +@dataclass +class DatasetDeleted(Event): + dataset_id: int + slug: str diff --git a/modules/datasets/datasets/contracts/files.py b/modules/datasets/datasets/contracts/files.py new file mode 100644 index 00000000..48e84609 --- /dev/null +++ b/modules/datasets/datasets/contracts/files.py @@ -0,0 +1,88 @@ +"""Public-facing value types for dataset file access. + +Consumers that depend on the Datasets module import ``DatasetFile`` rather +than the private ``Dataset`` SQLModel table. It carries enough to stream +bytes, hand off to a parser library, or materialise to a local path +without a second round-trip. + +Because the datasets module delegates bytes storage to +``file_storage.StorageBackend``, a dataset's bytes may live on any +backend (local FS today, S3 or GCS tomorrow). ``DatasetFile`` hides that +difference: consumers call :meth:`stream` or :meth:`materialize_to` and +never touch backend-specific APIs. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import AsyncIterator +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from datasets.contracts.schemas import DatasetOut + +if TYPE_CHECKING: + from file_storage.contracts.service import StorageBackend + + +@dataclass(frozen=True) +class DatasetFile: + """Read-only handle to a stored dataset file. + + The handle is decoupled from any specific storage backend. Consumers + that need bytes use :meth:`stream` (async iterator) or :meth:`read` + (``bytes``). Consumers that need a filesystem path — e.g. to hand + the file off to ``fiona`` / ``rasterio`` / ``pandas`` that require + ``str(path)`` — use :meth:`materialize_to`. + """ + + metadata: DatasetOut + storage_key: str + original_filename: str + mime_type: str | None + # Backend injected by the service — not part of the printable repr. + _backend: StorageBackend + + async def stream(self) -> AsyncIterator[bytes]: + """Yield the file's bytes in chunks. Works on any backend.""" + return await self._backend.get(self.storage_key) + + async def read(self) -> bytes: + """Read the entire file into memory. Prefer :meth:`stream` for large files.""" + buf = bytearray() + async for chunk in await self._backend.get(self.storage_key): + buf.extend(chunk) + return bytes(buf) + + async def exists(self) -> bool: + return await self._backend.exists(self.storage_key) + + async def materialize_to(self, path: Path) -> Path: + """Download the file to ``path``. Returns the path. + + Use for libraries that require a filesystem path (``fiona``, + ``rasterio``, ``pandas.read_csv``). For the filesystem backend + this is almost free; for S3 it pulls bytes down once. Callers + are responsible for deleting the file. + """ + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("wb") as fp: + async for chunk in await self._backend.get(self.storage_key): + fp.write(chunk) + return path + + async def materialize_to_tempfile(self, suffix: str | None = None) -> Path: + """Download to a named temp file. Returns the path. + + Convenience wrapper around :meth:`materialize_to` for the common + "I need a Path for fiona/rasterio, then I'll delete it" pattern. + Caller must ``unlink`` the returned path. + """ + effective_suffix = suffix if suffix is not None else Path(self.original_filename).suffix + fd, tmp = tempfile.mkstemp(suffix=effective_suffix) + # We don't need the file descriptor — ``materialize_to`` re-opens the path. + import os + + os.close(fd) + return await self.materialize_to(Path(tmp)) diff --git a/modules/datasets/datasets/contracts/schemas.py b/modules/datasets/datasets/contracts/schemas.py new file mode 100644 index 00000000..d0d902ad --- /dev/null +++ b/modules/datasets/datasets/contracts/schemas.py @@ -0,0 +1,64 @@ +"""SQLModel DTOs for the Datasets module.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import ConfigDict +from sqlmodel import Field, SQLModel + +from datasets import constants + +DatasetKind = Literal[ + "vector_geojson", + "vector_shapefile", + "vector_kml", + "raster_geotiff", + "tabular_csv", + "other", +] + +# Duplicated from ``constants.ALL_KINDS`` because ``typing.Literal`` demands +# string literals at type-evaluation time — a tuple reference won't satisfy +# it. The runtime check against ``constants.ALL_KINDS`` is the one that +# matters; this Literal only narrows the ``DatasetUpdate.kind`` type. +KIND_VALUES: tuple[str, ...] = constants.ALL_KINDS + + +class DatasetOut(SQLModel): + """Dataset metadata returned by the API.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + name: str + slug: str + kind: str + description: str | None = None + original_filename: str + mime_type: str | None = None + size_bytes: int + crs: str | None = None + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None + feature_count: int | None = None + band_count: int | None = None + extraction_status: str + created_at: datetime | None = None + updated_at: datetime | None = None + + +class DatasetUpdate(SQLModel): + """Patchable metadata. Files are immutable after upload — re-upload to replace.""" + + name: str | None = Field(default=None, min_length=1, max_length=200) + description: str | None = Field(default=None, max_length=2000) + kind: DatasetKind | None = None + crs: str | None = Field(default=None, max_length=64) + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None diff --git a/modules/datasets/datasets/contracts/urls.py b/modules/datasets/datasets/contracts/urls.py new file mode 100644 index 00000000..680424d9 --- /dev/null +++ b/modules/datasets/datasets/contracts/urls.py @@ -0,0 +1,26 @@ +"""URL builders consuming modules can use without hard-coding prefixes. + +If the Datasets module's ``route_prefix`` ever changes, downstream modules +don't have to audit every hard-coded ``/api/datasets/...`` string — they +just re-import these helpers. +""" + +from __future__ import annotations + +API_PREFIX = "/api/datasets" +VIEW_PREFIX = "/datasets" + + +def download_url(dataset_id: int) -> str: + """URL that streams the dataset's stored file back to the client.""" + return f"{API_PREFIX}/{dataset_id}/download" + + +def detail_url(dataset_id: int) -> str: + """URL of the dataset's JSON detail endpoint.""" + return f"{API_PREFIX}/{dataset_id}" + + +def show_url(dataset_id: int) -> str: + """URL of the Inertia ``Show`` page.""" + return f"{VIEW_PREFIX}/{dataset_id}" diff --git a/modules/datasets/datasets/deps.py b/modules/datasets/datasets/deps.py new file mode 100644 index 00000000..25270e4b --- /dev/null +++ b/modules/datasets/datasets/deps.py @@ -0,0 +1,97 @@ +"""FastAPI dependencies for the Datasets module. + +Downstream modules that depend on ``Datasets`` import +``DatasetServiceDep`` directly:: + + from datasets.deps import DatasetServiceDep + + @router.get("/my-thing") + async def endpoint(datasets: DatasetServiceDep): + ds = await datasets.get_by_slug("world-borders") + ... + +The storage backend comes from the ``file_storage`` module's app-state +slot. That seam is what lets datasets work on local FS today and S3 +tomorrow without touching this module. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import Depends, Request +from file_storage.contracts.service import StorageBackend +from simple_module_core.events import EventBus +from simple_module_db.deps import get_db +from sqlalchemy.ext.asyncio import AsyncSession + +from datasets.service import DatasetService + +if TYPE_CHECKING: + from celery import Celery + + +def get_storage_backend(request: Request) -> StorageBackend: + return request.app.state.file_storage.backend + + +async def get_dataset_service( + db: AsyncSession = Depends(get_db), + backend: StorageBackend = Depends(get_storage_backend), +) -> DatasetService: + return DatasetService(db, backend) + + +def get_event_bus(request: Request) -> EventBus: + return request.app.state.sm.event_bus + + +def get_celery(request: Request) -> Celery: + """Return the Celery app singleton owned by the background_tasks module. + + Depending on this means the datasets module won't boot unless + ``BackgroundTasks`` ran its ``on_startup`` first — which is enforced + via ``meta.depends_on``. + """ + return request.app.state.background_tasks.celery + + +def get_max_upload_bytes(request: Request) -> int: + """Resolve the max upload size. + + Prefers the runtime ``datasets.max_upload_mb`` value from the settings + module (so admins can tune it without a redeploy), falling back to the + env-var default on ``app.state.datasets.settings``. + """ + env_default = request.app.state.datasets.settings.max_upload_mb + override = _runtime_max_upload_mb(request, fallback=env_default) + return max(override, 1) * 1024 * 1024 + + +def _runtime_max_upload_mb(request: Request, *, fallback: int) -> int: + """Read the ``datasets.max_upload_mb`` SYSTEM scope setting. + + Returns ``fallback`` if the ``settings`` module isn't installed, the + registered accessor isn't available synchronously, or the stored value + can't be parsed as an int. Synchronous read happens through the + registry default when no DB row exists — we deliberately avoid doing + a DB query in a hot DI path. + """ + registry = getattr(getattr(request.app.state, "settings", None), "registry", None) + if registry is None: + return fallback + from datasets import constants + + definition = registry.get(constants.SETTING_MAX_UPLOAD_MB) + if definition is None: + return fallback + try: + return int(definition.default) + except (TypeError, ValueError): + return fallback + + +# Public type alias consumers can import directly — shortens +# ``service: DatasetService = Depends(get_dataset_service)`` to just +# ``datasets: DatasetServiceDep``. +DatasetServiceDep = Annotated[DatasetService, Depends(get_dataset_service)] diff --git a/modules/datasets/datasets/endpoints/__init__.py b/modules/datasets/datasets/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/datasets/datasets/endpoints/api.py b/modules/datasets/datasets/endpoints/api.py new file mode 100644 index 00000000..64661211 --- /dev/null +++ b/modules/datasets/datasets/endpoints/api.py @@ -0,0 +1,222 @@ +"""REST API endpoints for the Datasets module.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path + +from celery import Celery +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import RedirectResponse, StreamingResponse +from file_storage.contracts.service import NotSupportedError, StorageNotFoundError +from simple_module_core.events import EventBus +from simple_module_core.feature_flags import is_flag_enabled +from simple_module_hosting.permissions import RequiresPermission + +from datasets import constants +from datasets.contracts.events import DatasetDeleted, DatasetUploaded +from datasets.contracts.schemas import DatasetOut, DatasetUpdate +from datasets.deps import ( + get_celery, + get_dataset_service, + get_event_bus, + get_max_upload_bytes, +) +from datasets.service import DatasetService, UploadInput + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def _enqueue_extraction(celery: Celery, dataset_id: int) -> None: + """Enqueue the extraction task, absorbing broker failures. + + A dead Redis shouldn't fail the upload — the Dataset row is already + persisted with ``extraction_status="pending"`` and can be re-extracted + later. We log the failure so the operator can see it and retry. + """ + try: + celery.send_task(constants.TASK_EXTRACT_METADATA, args=[dataset_id]) + except Exception: + logger.exception( + "Failed to enqueue %s for dataset %s — row will stay pending", + constants.TASK_EXTRACT_METADATA, + dataset_id, + ) + + +@router.get("/", response_model=list[DatasetOut]) +async def list_datasets( + service: DatasetService = Depends(get_dataset_service), +) -> list[DatasetOut]: + return await service.get_all() + + +@router.get(constants.PATH_DATASET, response_model=DatasetOut) +async def get_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), +) -> DatasetOut: + item = await service.get_by_id(dataset_id) + if item is None: + raise HTTPException(status_code=404, detail="Dataset not found") + return item + + +@router.get(constants.PATH_DOWNLOAD) +async def download_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), +): + handle = await service.get_file(dataset_id) + if handle is None: + raise HTTPException(status_code=404, detail="Dataset not found") + + # Presigned URLs let the client bypass the app entirely for S3-like + # backends — avoids proxying multi-GB files through the worker. + if service.backend.supports_presigned_url: + try: + url = await service.backend.presigned_get_url( + handle.storage_key, ttl_seconds=constants.DEFAULT_PRESIGN_TTL_SECONDS + ) + except NotSupportedError: + url = None + if url is not None: + return RedirectResponse(url, status_code=302) + + try: + stream = await handle.stream() + except StorageNotFoundError: + raise HTTPException( + status_code=410, detail="Dataset file is missing from storage" + ) from None + + disposition = f'attachment; filename="{handle.original_filename}"' + return StreamingResponse( + stream, + media_type=handle.mime_type or constants.DEFAULT_MIME_TYPE, + headers={"Content-Disposition": disposition}, + ) + + +@router.post( + "/", + response_model=DatasetOut, + status_code=201, + dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_UPLOAD))], +) +async def upload_dataset( + request: Request, + name: str = Form(..., min_length=1, max_length=200), + description: str | None = Form(default=None, max_length=2000), + kind: str | None = Form(default=None), + file: UploadFile = File(...), + service: DatasetService = Depends(get_dataset_service), + bus: EventBus = Depends(get_event_bus), + celery: Celery = Depends(get_celery), + max_upload_bytes: int = Depends(get_max_upload_bytes), +) -> DatasetOut: + if kind is not None and kind not in constants.ALL_KINDS: + raise HTTPException(status_code=422, detail=f"Unknown kind: {kind}") + if kind == constants.DatasetKind.RASTER_GEOTIFF and not is_flag_enabled( + request, constants.FLAG_ALLOW_RASTER_UPLOADS + ): + raise HTTPException( + status_code=422, + detail="Raster uploads are disabled on this instance.", + ) + + original_filename = file.filename or constants.DEFAULT_FALLBACK_FILENAME + bytes_written = 0 + # Spool to a temp file first so size-validation can reject before we + # touch the storage backend, and so the service can hand a complete + # stream to backend.put(). + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = Path(tmp.name) + try: + while True: + chunk = await file.read(constants.DEFAULT_UPLOAD_CHUNK_SIZE) + if not chunk: + break + bytes_written += len(chunk) + if bytes_written > max_upload_bytes: + raise HTTPException( + status_code=413, + detail=f"Upload exceeds {max_upload_bytes} bytes", + ) + tmp.write(chunk) + except HTTPException: + tmp.close() + tmp_path.unlink(missing_ok=True) + raise + + try: + dataset = await service.register_upload( + UploadInput( + name=name, + original_filename=original_filename, + temp_path=tmp_path, + size_bytes=bytes_written, + mime_type=file.content_type, + description=description, + kind=kind, + ) + ) + finally: + tmp_path.unlink(missing_ok=True) + + # Hand metadata extraction off to a Celery worker. The Dataset row is + # already ``extraction_status="pending"`` — the worker flips it to + # ok / partial / failed once the parse completes. See + # ``datasets.tasks.extract_metadata_task``. Guarded by a feature + # flag so an admin can freeze auto-extraction during incidents + # without redeploying. + if is_flag_enabled(request, constants.FLAG_AUTO_EXTRACT): + _enqueue_extraction(celery, dataset.id) + + await bus.publish( + DatasetUploaded( + dataset_id=dataset.id, + name=dataset.name, + slug=dataset.slug, + kind=dataset.kind, + ) + ) + return dataset + + +@router.patch( + constants.PATH_DATASET, + response_model=DatasetOut, + dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_EDIT))], +) +async def update_dataset( + dataset_id: int, + data: DatasetUpdate, + service: DatasetService = Depends(get_dataset_service), +) -> DatasetOut: + item = await service.update(dataset_id, data) + if item is None: + raise HTTPException(status_code=404, detail="Dataset not found") + return item + + +@router.delete( + constants.PATH_DATASET, + status_code=204, + dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_DELETE))], +) +async def delete_dataset( + dataset_id: int, + service: DatasetService = Depends(get_dataset_service), + bus: EventBus = Depends(get_event_bus), +) -> None: + # Capture the slug before deletion so subscribers can index by slug + # without a post-delete lookup (which would always miss). + existing = await service.get_by_id(dataset_id) + if existing is None: + raise HTTPException(status_code=404, detail="Dataset not found") + await service.delete(dataset_id) + await bus.publish(DatasetDeleted(dataset_id=dataset_id, slug=existing.slug)) diff --git a/modules/datasets/datasets/endpoints/views.py b/modules/datasets/datasets/endpoints/views.py new file mode 100644 index 00000000..e4b7f3a6 --- /dev/null +++ b/modules/datasets/datasets/endpoints/views.py @@ -0,0 +1,78 @@ +"""Inertia view endpoints for the Datasets module.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from inertia import InertiaResponse +from simple_module_hosting.inertia_deps import InertiaDep +from simple_module_hosting.permissions import RequiresPermission + +from datasets import constants +from datasets.deps import get_dataset_service +from datasets.service import DatasetService + +# Module-local Inertia page identifiers. These must be Name-only literal +# assignments (not attribute access against ``constants``) so the SM003 +# orphan-page diagnostic can resolve them — see +# ``simple_module_core.diagnostics._module._iter_render_components``. +_PAGE_BROWSE = "Datasets/Browse" +_PAGE_CREATE = "Datasets/Create" +_PAGE_SHOW = "Datasets/Show" +_PAGE_EDIT = "Datasets/Edit" + +router = APIRouter() + + +@router.get("/", response_model=None) +async def browse( + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + items = await service.get_all() + return await inertia.render( + _PAGE_BROWSE, + {"datasets": [item.model_dump(mode="json") for item in items]}, + ) + + +@router.get( + "/create", + response_model=None, + dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_UPLOAD))], +) +async def create_view(inertia: InertiaDep) -> InertiaResponse: + return await inertia.render(_PAGE_CREATE) + + +@router.get("/{dataset_id}", response_model=None) +async def show_view( + dataset_id: int, + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + item = await service.get_by_id(dataset_id) + if item is None: + return await inertia.render( + _PAGE_BROWSE, + {"datasets": [], "error": "Dataset not found"}, + ) + return await inertia.render(_PAGE_SHOW, {"dataset": item.model_dump(mode="json")}) + + +@router.get( + "/{dataset_id}/edit", + response_model=None, + dependencies=[Depends(RequiresPermission(constants.PERM_DATASETS_EDIT))], +) +async def edit_view( + dataset_id: int, + inertia: InertiaDep, + service: DatasetService = Depends(get_dataset_service), +) -> InertiaResponse: + item = await service.get_by_id(dataset_id) + if item is None: + return await inertia.render( + _PAGE_BROWSE, + {"datasets": [], "error": "Dataset not found"}, + ) + return await inertia.render(_PAGE_EDIT, {"dataset": item.model_dump(mode="json")}) diff --git a/modules/datasets/datasets/extractors.py b/modules/datasets/datasets/extractors.py new file mode 100644 index 00000000..97e573da --- /dev/null +++ b/modules/datasets/datasets/extractors.py @@ -0,0 +1,188 @@ +"""Best-effort metadata extraction for uploaded datasets. + +The stdlib JSON path covers GeoJSON without any extra dependency. Optional +``fiona`` and ``rasterio`` extras enrich extraction for shapefiles, KML, and +rasters when available. Every extractor degrades to +``ExtractionStatus.MANUAL`` so the catalog never blocks an upload because +a parser is missing. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from datasets import constants +from datasets.constants import DatasetKind, ExtractionStatus + + +@dataclass +class ExtractedMeta: + crs: str | None = None + bbox_min_x: float | None = None + bbox_min_y: float | None = None + bbox_max_x: float | None = None + bbox_max_y: float | None = None + feature_count: int | None = None + band_count: int | None = None + status: str = ExtractionStatus.MANUAL + + +_EXTENSION_TO_KIND: dict[str, str] = { + ".geojson": DatasetKind.VECTOR_GEOJSON, + ".json": DatasetKind.VECTOR_GEOJSON, + ".shp": DatasetKind.VECTOR_SHAPEFILE, + ".zip": DatasetKind.VECTOR_SHAPEFILE, + ".kml": DatasetKind.VECTOR_KML, + ".kmz": DatasetKind.VECTOR_KML, + ".tif": DatasetKind.RASTER_GEOTIFF, + ".tiff": DatasetKind.RASTER_GEOTIFF, + ".csv": DatasetKind.TABULAR_CSV, +} + + +def kind_for_filename(filename: str) -> str: + """Map a filename to a coarse kind label. + + The mapping is intentionally cheap — anything we don't recognise becomes + ``DatasetKind.OTHER`` so the upload still lands and the user can correct + the kind via the edit form. + """ + suffix = Path(filename).suffix.lower() + return _EXTENSION_TO_KIND.get(suffix, DatasetKind.OTHER) + + +def extract_metadata(path: Path, kind: str) -> ExtractedMeta: + """Dispatch to a kind-specific extractor. + + Never raises — extraction failures collapse to + ``ExtractionStatus.FAILED`` so the upload itself succeeds and the user + can fill the metadata in. + """ + try: + if kind == DatasetKind.VECTOR_GEOJSON: + return _extract_geojson(path) + if kind == DatasetKind.RASTER_GEOTIFF: + return _extract_raster(path) + if kind in {DatasetKind.VECTOR_SHAPEFILE, DatasetKind.VECTOR_KML}: + return _extract_via_fiona(path) + except Exception: + return ExtractedMeta(status=ExtractionStatus.FAILED) + return ExtractedMeta(status=ExtractionStatus.MANUAL) + + +def _extract_geojson(path: Path) -> ExtractedMeta: + with path.open("rb") as fp: + doc = json.load(fp) + + features = _features(doc) + bbox = doc.get("bbox") if isinstance(doc, dict) else None + if not bbox: + bbox = _compute_bbox(features) + + if bbox is None: + return ExtractedMeta( + crs=constants.DEFAULT_GEOJSON_CRS, + feature_count=len(features), + status=ExtractionStatus.PARTIAL, + ) + return ExtractedMeta( + crs=constants.DEFAULT_GEOJSON_CRS, + bbox_min_x=float(bbox[0]), + bbox_min_y=float(bbox[1]), + bbox_max_x=float(bbox[2]), + bbox_max_y=float(bbox[3]), + feature_count=len(features), + status=ExtractionStatus.OK, + ) + + +def _features(doc: object) -> list[dict]: + if isinstance(doc, dict): + if doc.get("type") == "FeatureCollection": + features = doc.get("features", []) + return [f for f in features if isinstance(f, dict)] + if doc.get("type") == "Feature": + return [doc] + return [] + + +def _compute_bbox(features: list[dict]) -> tuple[float, float, float, float] | None: + min_x = min_y = float("inf") + max_x = max_y = float("-inf") + for feature in features: + geom = feature.get("geometry") if isinstance(feature, dict) else None + if not isinstance(geom, dict): + continue + for x, y in _iter_coords(geom): + min_x = min(min_x, x) + min_y = min(min_y, y) + max_x = max(max_x, x) + max_y = max(max_y, y) + if min_x == float("inf"): + return None + return min_x, min_y, max_x, max_y + + +def _iter_coords(geom: dict): + coords = geom.get("coordinates") + geom_type = geom.get("type") + if geom_type == "GeometryCollection": + for sub in geom.get("geometries", []): + if isinstance(sub, dict): + yield from _iter_coords(sub) + return + yield from _walk(coords) + + +def _walk(node: object): + if ( + isinstance(node, list) + and len(node) >= 2 + and all(isinstance(v, (int, float)) for v in node[:2]) + ): + yield float(node[0]), float(node[1]) + return + if isinstance(node, list): + for child in node: + yield from _walk(child) + + +def _extract_via_fiona(path: Path) -> ExtractedMeta: + try: + import fiona # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError: + return ExtractedMeta(status=ExtractionStatus.MANUAL) + + with fiona.open(str(path)) as src: + bounds = src.bounds + crs = src.crs.get("init") if hasattr(src.crs, "get") else str(src.crs) if src.crs else None + return ExtractedMeta( + crs=crs, + bbox_min_x=float(bounds[0]), + bbox_min_y=float(bounds[1]), + bbox_max_x=float(bounds[2]), + bbox_max_y=float(bounds[3]), + feature_count=len(src), + status=ExtractionStatus.OK, + ) + + +def _extract_raster(path: Path) -> ExtractedMeta: + try: + import rasterio # type: ignore[import-not-found] # ty: ignore[unresolved-import] + except ImportError: + return ExtractedMeta(status=ExtractionStatus.MANUAL) + + with rasterio.open(str(path)) as src: + bounds = src.bounds + return ExtractedMeta( + crs=str(src.crs) if src.crs else None, + bbox_min_x=float(bounds.left), + bbox_min_y=float(bounds.bottom), + bbox_max_x=float(bounds.right), + bbox_max_y=float(bounds.top), + band_count=src.count, + status=ExtractionStatus.OK, + ) diff --git a/modules/datasets/datasets/locales/en.json b/modules/datasets/datasets/locales/en.json new file mode 100644 index 00000000..b49266d1 --- /dev/null +++ b/modules/datasets/datasets/locales/en.json @@ -0,0 +1,67 @@ +{ + "browse": { + "title": "Datasets", + "description": "Upload and manage datasets", + "new_button": "Upload Dataset", + "empty_title": "No datasets yet", + "empty_description": "Upload your first dataset to get started.", + "create_button": "Upload Dataset" + }, + "table": { + "name": "Name", + "kind": "Kind", + "crs": "CRS", + "bbox": "Bounding Box", + "size": "Size", + "actions": "Actions" + }, + "form": { + "name_label": "Name", + "name_placeholder": "Descriptive name for the dataset", + "description_label": "Description", + "description_placeholder": "Optional notes about the dataset", + "kind_label": "Kind", + "kind_auto": "Auto-detect from filename", + "file_label": "File", + "crs_label": "CRS", + "cancel_button": "Cancel" + }, + "create": { + "title": "Upload Dataset", + "description": "Add a dataset to the catalog", + "submit_button": "Upload", + "submitting_button": "Uploading..." + }, + "edit": { + "title": "Edit: {name}", + "description": "Update dataset metadata", + "back_button": "Back to Datasets", + "submit_button": "Save Changes", + "submitting_button": "Saving..." + }, + "show": { + "download": "Download", + "edit": "Edit", + "kind": "Kind", + "original_file": "Original filename", + "size": "Size", + "mime_type": "MIME type", + "crs": "CRS", + "bbox": "Bounding box", + "features": "Feature count", + "bands": "Band count", + "extraction_status": "Extraction status" + }, + "toasts": { + "created": "Dataset uploaded", + "updated": "Dataset updated", + "deleted": "Dataset deleted" + }, + "validation": { + "name_required": "Name is required", + "file_required": "Please choose a file to upload" + }, + "errors": { + "not_found": "Dataset not found" + } +} diff --git a/modules/datasets/datasets/models.py b/modules/datasets/datasets/models.py new file mode 100644 index 00000000..8c19e5cd --- /dev/null +++ b/modules/datasets/datasets/models.py @@ -0,0 +1,49 @@ +"""SQLModel tables for the Datasets module.""" + +from __future__ import annotations + +from simple_module_db.base import create_module_base +from simple_module_db.mixins import AuditMixin, SoftDeleteMixin +from sqlalchemy import Index +from sqlmodel import Field + +from datasets import constants + +# Provider is auto-detected from SM_DATABASE_URL (falls back to SQLite). +# On PostgreSQL this gives the module its own schema; on SQLite all modules +# share one schema, so __tablename__ is prefixed for isolation. +Base = create_module_base(constants.SCHEMA_NAME) + + +class Dataset(Base, AuditMixin, SoftDeleteMixin, table=True): # ty: ignore[unsupported-base] + """A dataset uploaded into the catalog. + + ``kind`` identifies the content type (vector GeoJSON, shapefile, + raster GeoTIFF, tabular CSV, ...) — see + :class:`datasets.constants.DatasetKind`. Geospatial fields (``crs``, + ``bbox_*``) stay null for non-geospatial kinds. + """ + + __tablename__ = constants.TABLE_DATASET + + id: int | None = Field(default=None, primary_key=True) + name: str = Field(max_length=200) + slug: str = Field(max_length=200, unique=True, index=True) + kind: str = Field(max_length=32, index=True) + description: str | None = Field(default=None, max_length=2000) + + original_filename: str = Field(max_length=255) + mime_type: str | None = Field(default=None, max_length=127) + size_bytes: int = Field(default=0) + storage_key: str = Field(max_length=512) + + crs: str | None = Field(default=None, max_length=64) + bbox_min_x: float | None = Field(default=None) + bbox_min_y: float | None = Field(default=None) + bbox_max_x: float | None = Field(default=None) + bbox_max_y: float | None = Field(default=None) + feature_count: int | None = Field(default=None) + band_count: int | None = Field(default=None) + extraction_status: str = Field(default=constants.ExtractionStatus.MANUAL, max_length=16) + + __table_args__ = (Index("ix_datasets_dataset_is_deleted", "is_deleted"),) diff --git a/modules/datasets/datasets/module.py b/modules/datasets/datasets/module.py new file mode 100644 index 00000000..faf4a860 --- /dev/null +++ b/modules/datasets/datasets/module.py @@ -0,0 +1,144 @@ +"""Datasets module definition. + +Depends on ``FileStorage`` for bytes storage, ``BackgroundTasks`` for +the Celery pipeline the upload endpoint enqueues into, ``Permissions`` +for the permission system, ``FeatureFlags`` for runtime toggles, and +``Settings`` for admin-configurable limits. ``register_settings`` here +runs after those, so every ``app.state.*`` slot is populated before any +Datasets request fires. +""" + +from __future__ import annotations + +import importlib.resources +from pathlib import Path +from typing import TYPE_CHECKING + +from fastapi import APIRouter, FastAPI +from simple_module_core.feature_flags import FeatureFlagDefinition, FeatureFlagRegistry +from simple_module_core.menu import MenuItem, MenuRegistry, MenuSection +from simple_module_core.module import ModuleBase, ModuleMeta +from simple_module_core.permissions import PermissionRegistry + +from datasets import constants + +if TYPE_CHECKING: + from settings.contracts.registry import SettingDefinition + + +class DatasetsModule(ModuleBase): + meta = ModuleMeta( + name=constants.MODULE_PASCAL, + route_prefix=constants.ROUTE_PREFIX_API, + view_prefix=constants.ROUTE_PREFIX_VIEW, + depends_on=[constants.MODULE_FILE_STORAGE, constants.MODULE_BACKGROUND_TASKS], + ) + + def register_settings(self, app: FastAPI) -> None: + from datasets.services import DatasetsServices + from datasets.settings import DatasetsSettings + + app.state.datasets = DatasetsServices(settings=DatasetsSettings()) + + def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: + from datasets.endpoints.api import router as api + from datasets.endpoints.views import router as views + + api_router.include_router(api) + view_router.include_router(views) + + def register_menu_items(self, registry: MenuRegistry) -> None: + registry.add( + MenuItem( + label=constants.MENU_LABEL, + url=constants.ROUTE_PREFIX_VIEW, + icon=constants.MENU_ICON, + order=constants.MENU_ORDER, + section=MenuSection.SIDEBAR, + ) + ) + + def register_permissions(self, registry: PermissionRegistry) -> None: + registry.add_group( + constants.PERMISSION_GROUP, + list(constants.ALL_PERMISSIONS), + ) + # Plain users can browse the catalog and upload their own datasets; + # edit/delete stay admin-only via the framework wildcard. + registry.map_role( + constants.ROLE_USER, + list(constants.USER_ROLE_PERMISSIONS), + ) + + def register_feature_flags(self, registry: FeatureFlagRegistry) -> None: + registry.add( + FeatureFlagDefinition( + name=constants.FLAG_AUTO_EXTRACT, + description=( + "Enqueue the Celery metadata-extraction task on upload. " + "Turn off to skip the worker hop and leave rows as " + "``extraction_status=pending`` for manual review." + ), + default_enabled=True, + ) + ) + registry.add( + FeatureFlagDefinition( + name=constants.FLAG_ALLOW_RASTER_UPLOADS, + description=( + "Accept raster (GeoTIFF) uploads. Disable on instances " + "without ``rasterio`` installed to give users a clear " + "422 instead of a silent failed extraction." + ), + default_enabled=True, + ) + ) + + def locale_dirs(self) -> dict[str, Path]: + base = Path(str(importlib.resources.files(__package__) / "locales")) + return {constants.LOCALE_NAMESPACE: base} + + async def on_startup(self, app: FastAPI) -> None: + """Register runtime-tunable settings with the ``settings`` module. + + Registered as ``on_startup`` (not in ``register_settings``) because + the settings module's ``app.state.settings`` slot is only populated + once its own ``register_settings`` has run — ``on_startup`` fires + after every module's registration is done. + """ + # The settings module may not be installed in every deployment — + # treat its absence as a warn, not a crash. + registry = getattr(getattr(app.state, "settings", None), "registry", None) + if registry is None: + return + for definition in _setting_definitions(): + if definition.key in registry: + continue + registry.add(definition) + + +def _setting_definitions() -> list[SettingDefinition]: + """Deferred import so the module still loads if ``settings`` is absent.""" + from settings.contracts.registry import SettingDefinition + from settings.contracts.schemas import SettingValueType + + return [ + SettingDefinition( + key=constants.SETTING_MAX_UPLOAD_MB, + default=str(constants.DEFAULT_MAX_UPLOAD_MB), + description=( + "Per-dataset upload size cap in megabytes. Overrides the " + "``SM_DATASETS_MAX_UPLOAD_MB`` env default at runtime." + ), + value_type=SettingValueType.INT, + ), + SettingDefinition( + key=constants.SETTING_DEFAULT_KIND, + default=constants.DatasetKind.OTHER, + description=( + "Dataset kind assigned when filename-based detection comes " + "back as ``other``. Useful for instances that know they only " + "ingest one kind (e.g. always ``vector_geojson``)." + ), + ), + ] diff --git a/modules/datasets/datasets/pages/Browse.tsx b/modules/datasets/datasets/pages/Browse.tsx new file mode 100644 index 00000000..95d8140b --- /dev/null +++ b/modules/datasets/datasets/pages/Browse.tsx @@ -0,0 +1,165 @@ +import { Link, router, usePage } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { PageShell } from '@simple-module/ui/components/PageShell'; +import { Badge } from '@simple-module/ui/components/ui/badge'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Card } from '@simple-module/ui/components/ui/card'; +import { + Empty, + EmptyDescription, + EmptyMedia, + EmptyTitle, +} from '@simple-module/ui/components/ui/empty'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@simple-module/ui/components/ui/table'; +import { usePermissions } from '@simple-module/ui/hooks/use-permissions'; +import { AuthenticatedLayout } from '@simple-module/ui/layouts/AuthenticatedLayout'; +import { Layers, Plus, Trash2 } from 'lucide-react'; + +interface Dataset { + id: number; + name: string; + slug: string; + kind: string; + size_bytes: number; + crs: string | null; + bbox_min_x: number | null; + bbox_min_y: number | null; + bbox_max_x: number | null; + bbox_max_y: number | null; + extraction_status: string; + created_at: string | null; +} + +interface Props { + datasets: Dataset[]; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function formatBbox(d: Dataset): string { + if ( + d.bbox_min_x === null || + d.bbox_min_y === null || + d.bbox_max_x === null || + d.bbox_max_y === null + ) { + return '—'; + } + return `[${d.bbox_min_x.toFixed(3)}, ${d.bbox_min_y.toFixed(3)}, ${d.bbox_max_x.toFixed(3)}, ${d.bbox_max_y.toFixed(3)}]`; +} + +function Browse() { + const { datasets } = usePage<{ props: Props }>().props as unknown as Props; + const { t } = useT(); + const { can } = usePermissions(); + const canUpload = can('datasets.upload'); + const canDelete = can('datasets.delete'); + + function handleDelete(dataset: Dataset) { + router.delete(`/api/datasets/${dataset.id}`, { preserveScroll: true }); + } + + return ( + + + + {t(keys.datasets.browse.new_button)} + + + ) : undefined + } + > + + + + + {t(keys.datasets.table.name)} + {t(keys.datasets.table.kind)} + + {t(keys.datasets.table.crs)} + + + {t(keys.datasets.table.bbox)} + + {t(keys.datasets.table.size)} + {t(keys.datasets.table.actions)} + + + + {datasets.map((dataset) => ( + + + + {dataset.name} + + + + {dataset.kind} + + + {dataset.crs || '—'} + + + {formatBbox(dataset)} + + + {formatBytes(dataset.size_bytes)} + + + {canDelete && ( + + )} + + + ))} + {datasets.length === 0 && ( + + + + + + + {t(keys.datasets.browse.empty_title)} + {t(keys.datasets.browse.empty_description)} + {canUpload && ( + + )} + + + + )} + +
+ + + ); +} + +Browse.layout = (page: React.ReactNode) => {page}; +export default Browse; diff --git a/modules/datasets/datasets/pages/Create.tsx b/modules/datasets/datasets/pages/Create.tsx new file mode 100644 index 00000000..0c52c02a --- /dev/null +++ b/modules/datasets/datasets/pages/Create.tsx @@ -0,0 +1,144 @@ +import { Link, router } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { PageShell } from '@simple-module/ui/components/PageShell'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Card, CardContent } from '@simple-module/ui/components/ui/card'; +import { Input } from '@simple-module/ui/components/ui/input'; +import { Label } from '@simple-module/ui/components/ui/label'; +import { Textarea } from '@simple-module/ui/components/ui/textarea'; +import { AuthenticatedLayout } from '@simple-module/ui/layouts/AuthenticatedLayout'; +import { useState } from 'react'; +import { toast } from 'sonner'; + +const KINDS = [ + '', + 'vector_geojson', + 'vector_shapefile', + 'vector_kml', + 'raster_geotiff', + 'tabular_csv', + 'other', +]; + +function Create() { + const { t } = useT(); + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [kind, setKind] = useState(''); + const [file, setFile] = useState(null); + const [submitting, setSubmitting] = useState(false); + + function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + if (!file) { + toast.error(t(keys.datasets.validation.file_required)); + return; + } + if (!name.trim()) { + toast.error(t(keys.datasets.validation.name_required)); + return; + } + const data = new FormData(); + data.append('name', name); + if (description) data.append('description', description); + if (kind) data.append('kind', kind); + data.append('file', file); + + setSubmitting(true); + router.post('/api/datasets/', data, { + forceFormData: true, + onSuccess: () => toast.success(t(keys.datasets.toasts.created)), + onError: (errs) => { + const first = Object.values(errs)[0]; + if (first) toast.error(String(first)); + }, + onFinish: () => setSubmitting(false), + }); + } + + return ( + + {t(keys.datasets.form.cancel_button)} + + } + > + + +
+
+ + setName(e.target.value)} + placeholder={t(keys.datasets.form.name_placeholder)} + maxLength={200} + required + /> +
+ +
+ + +
+ +
+ + setFile(e.target.files?.[0] ?? null)} + required + /> +
+ +
+ +