diff --git a/.gitignore b/.gitignore index 203e4492..5a970b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ node_modules/ *.db *.sqlite3 +# file_storage filesystem backend default root (override via SM_FILE_STORAGE_FS_ROOT_PATH). +uploads/ + # Vite host/static/dist/ diff --git a/framework/hosting/simple_module_hosting/i18n_manifest.py b/framework/hosting/simple_module_hosting/i18n_manifest.py index 49b08f12..37b01243 100644 --- a/framework/hosting/simple_module_hosting/i18n_manifest.py +++ b/framework/hosting/simple_module_hosting/i18n_manifest.py @@ -13,7 +13,6 @@ from __future__ import annotations -import json import logging from pathlib import Path from typing import Any @@ -114,7 +113,7 @@ def write_generated_resources(registry: I18nRegistry, output_dir: Path) -> Path: def _render_resources(keys: list[str]) -> str: - lines = [_RESOURCES_HEADER, "", "export default {", " translation: {"] + lines = [_RESOURCES_HEADER, "export default {", " translation: {"] for key in keys: lines.append(f" '{key}': '',") lines.append(" },") @@ -179,16 +178,21 @@ def _insert(tree: dict[str, Any], path: list[str], value: str) -> None: def _serialize(node: Any, *, indent: int) -> str: - """Emit a dict-of-dicts tree as pretty-printed TypeScript object literal.""" + """Emit a dict-of-dicts tree as pretty-printed TypeScript object literal. + + Strings are wrapped in single quotes to match the project's biome config + (``quoteStyle: "single"``); dotted i18n keys never contain single quotes, + so plain wrapping is safe. + """ if isinstance(node, str): - return json.dumps(node) + return f"'{node}'" if not isinstance(node, dict) or not node: return "{}" pad = " " * indent inner_pad = " " * (indent + 1) lines = ["{"] for k in sorted(node.keys()): - key_repr = k if _is_valid_js_identifier(k) else json.dumps(k) + key_repr = k if _is_valid_js_identifier(k) else f"'{k}'" lines.append(f"{inner_pad}{key_repr}: {_serialize(node[k], indent=indent + 1)},") lines.append(f"{pad}}}") return "\n".join(lines) diff --git a/framework/hosting/tests/test_i18n_manifest.py b/framework/hosting/tests/test_i18n_manifest.py index 4b44734d..e6450ccc 100644 --- a/framework/hosting/tests/test_i18n_manifest.py +++ b/framework/hosting/tests/test_i18n_manifest.py @@ -70,8 +70,8 @@ def test_keys_tree_is_nested(tmp_path: Path) -> None: # Nested structure: keys = { auth: { errors: { not_authenticated: ... } }, products: { ... } } assert "auth:" in text assert "errors:" in text - assert 'not_authenticated: "auth.errors.not_authenticated"' in text - assert 'title: "products.browse.title"' in text + assert "not_authenticated: 'auth.errors.not_authenticated'" in text + assert "title: 'products.browse.title'" in text def test_keys_tree_adds_plural_stems(tmp_path: Path) -> None: @@ -86,9 +86,9 @@ def test_keys_tree_adds_plural_stems(tmp_path: Path) -> None: write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() # Both the concrete variants AND the virtual stem must be emitted. - assert 'count_one: "products.browse.count_one"' in text - assert 'count_other: "products.browse.count_other"' in text - assert 'count: "products.browse.count"' in text + assert "count_one: 'products.browse.count_one'" in text + assert "count_other: 'products.browse.count_other'" in text + assert "count: 'products.browse.count'" in text def test_keys_tree_quotes_non_identifier_segments(tmp_path: Path) -> None: @@ -97,7 +97,7 @@ def test_keys_tree_quotes_non_identifier_segments(tmp_path: Path) -> None: reg._messages = {"en": {"ui.switcher.a-b": "X"}} # hyphen in leaf write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() - assert '"a-b":' in text + assert "'a-b':" in text def test_keys_tree_does_not_overwrite_real_key_with_stem(tmp_path: Path) -> None: @@ -113,4 +113,4 @@ def test_keys_tree_does_not_overwrite_real_key_with_stem(tmp_path: Path) -> None write_generated_resources(reg, tmp_path) text = (tmp_path / "keys.generated.ts").read_text() # The real key retains its value; the virtual stem is skipped. - assert 'count: "products.browse.count"' in text + assert "count: 'products.browse.count'" in text diff --git a/host/migrations/versions/5d44218ee368_create_file_storage_tables.py b/host/migrations/versions/5d44218ee368_create_file_storage_tables.py new file mode 100644 index 00000000..cff58dc1 --- /dev/null +++ b/host/migrations/versions/5d44218ee368_create_file_storage_tables.py @@ -0,0 +1,77 @@ +"""create file_storage tables + +Revision ID: 5d44218ee368 +Revises: b7e1af4c9d02 +Create Date: 2026-04-19 11:20:24.247357 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "5d44218ee368" +down_revision: str | None = "b7e1af4c9d02" +branch_labels: str | Sequence[str] | None = ("file_storage",) +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + if op.get_context().dialect.name == "postgresql": + op.execute("CREATE SCHEMA IF NOT EXISTS file_storage") + + op.create_table( + "file_storage_stored_file", + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("deleted_by", sa.String(length=255), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=True), + sa.Column("updated_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("key", sa.String(length=512), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("content_type", sa.String(length=128), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column("backend", sa.String(length=32), nullable=False), + sa.Column("checksum_sha256", sa.String(length=64), nullable=False), + sa.Column("extra_metadata", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_file_storage_stored_file")), + ) + op.create_index( + "ix_file_storage_stored_file_created_by", + "file_storage_stored_file", + ["created_by"], + unique=False, + ) + op.create_index( + "ix_file_storage_stored_file_is_deleted", + "file_storage_stored_file", + ["is_deleted"], + unique=False, + ) + op.create_index( + "ix_file_storage_stored_file_key", + "file_storage_stored_file", + ["key"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("ix_file_storage_stored_file_key", table_name="file_storage_stored_file") + op.drop_index("ix_file_storage_stored_file_is_deleted", table_name="file_storage_stored_file") + op.drop_index("ix_file_storage_stored_file_created_by", table_name="file_storage_stored_file") + op.drop_table("file_storage_stored_file") + + if op.get_context().dialect.name == "postgresql": + op.execute("DROP SCHEMA IF EXISTS file_storage") diff --git a/host/pyproject.toml b/host/pyproject.toml index ed15268a..76eda86b 100644 --- a/host/pyproject.toml +++ b/host/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "auth", "dashboard", "products", + "file-storage", "python-multipart>=0.0.6", ] @@ -16,3 +17,4 @@ simple-module-hosting = { workspace = true } auth = { workspace = true } products = { workspace = true } dashboard = { workspace = true } +file-storage = { workspace = true } diff --git a/modules/file_storage/file_storage/__init__.py b/modules/file_storage/file_storage/__init__.py new file mode 100644 index 00000000..eeb03fa9 --- /dev/null +++ b/modules/file_storage/file_storage/__init__.py @@ -0,0 +1 @@ +"""FileStorage module.""" diff --git a/modules/file_storage/file_storage/backends/__init__.py b/modules/file_storage/file_storage/backends/__init__.py new file mode 100644 index 00000000..02ecf40f --- /dev/null +++ b/modules/file_storage/file_storage/backends/__init__.py @@ -0,0 +1,77 @@ +"""Storage backend registry — extension point for storage providers. + +Built-in providers (filesystem, S3) self-register on import. Third-party +packages add new providers via :func:`register_backend`:: + + from file_storage.backends import register_backend + from file_storage.contracts.service import StorageBackend + + @register_backend("azure_blob") + def _build(settings) -> StorageBackend: + return AzureBlobBackend(...) + +The factory receives the parsed :class:`FileStorageSettings` so providers +can read their own ``SM_FILE_STORAGE_*`` keys (or extend the settings +class with extra fields). :func:`build_backend` resolves the active +factory by ``settings.backend`` and raises :class:`ConfigurationError` +on an unknown id, listing all currently registered providers. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from file_storage.contracts.service import ConfigurationError + +if TYPE_CHECKING: + from file_storage.contracts.service import StorageBackend + from file_storage.settings import FileStorageSettings + +BackendFactory = Callable[["FileStorageSettings"], "StorageBackend"] + +_REGISTRY: dict[str, BackendFactory] = {} + + +def register_backend(backend_id: str) -> Callable[[BackendFactory], BackendFactory]: + """Decorator that registers a factory under ``backend_id``.""" + + def decorator(factory: BackendFactory) -> BackendFactory: + _REGISTRY[backend_id] = factory + return factory + + return decorator + + +def unregister_backend(backend_id: str) -> None: + """Remove a registered backend. Primarily for tests.""" + _REGISTRY.pop(backend_id, None) + + +def registered_backends() -> list[str]: + """Return all currently registered backend ids, sorted.""" + return sorted(_REGISTRY) + + +def build_backend(settings: FileStorageSettings) -> StorageBackend: + """Construct the backend selected by ``settings.backend``.""" + factory = _REGISTRY.get(settings.backend) + if factory is None: + raise ConfigurationError( + f"Unknown storage backend {settings.backend!r}. Registered: {registered_backends()}." + ) + return factory(settings) + + +# Trigger self-registration of built-in providers. Order is irrelevant; both +# add themselves to the same module-global registry. +from file_storage.backends import filesystem as _filesystem # noqa: E402,F401 +from file_storage.backends import s3 as _s3 # noqa: E402,F401 + +__all__ = [ + "BackendFactory", + "build_backend", + "register_backend", + "registered_backends", + "unregister_backend", +] diff --git a/modules/file_storage/file_storage/backends/filesystem.py b/modules/file_storage/file_storage/backends/filesystem.py new file mode 100644 index 00000000..9b4d863d --- /dev/null +++ b/modules/file_storage/file_storage/backends/filesystem.py @@ -0,0 +1,84 @@ +"""Filesystem storage backend — writes objects under a configurable root.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from pathlib import Path + +import aiofiles + +from file_storage import constants +from file_storage.backends import register_backend +from file_storage.contracts.service import ( + NotSupportedError, + StorageBackendError, + StorageNotFoundError, +) +from file_storage.settings import FileStorageSettings + + +class FilesystemBackend: + """Stores objects on the local filesystem under ``root``. + + Keys are sharded by their first two characters to keep any single + directory from accumulating millions of entries (which slows ``readdir`` + on most filesystems). A key like ``2026/04/19/abc123.png`` is written to + ``/20/2026/04/19/abc123.png``. + """ + + backend_id = constants.BackendId.FILESYSTEM + supports_presigned_url = False + + def __init__(self, root: Path) -> None: + self.root = root + + def _resolve(self, key: str) -> Path: + # Reject path traversal: ".." segments and absolute paths can escape root. + if key.startswith("/") or ".." in Path(key).parts: + raise StorageBackendError(f"Invalid storage key: {key!r}") + shard = key[:2] if len(key) >= 2 else "_" + return self.root / shard / key + + async def put( + self, + key: str, + stream: AsyncIterator[bytes], + *, + content_type: str, + size: int, + ) -> None: + path = self._resolve(key) + path.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(path, "wb") as fh: + async for chunk in stream: + await fh.write(chunk) + + async def get(self, key: str) -> AsyncIterator[bytes]: + path = self._resolve(key) + if not path.exists(): + raise StorageNotFoundError(key) + return _stream_file(path) + + async def delete(self, key: str) -> None: + path = self._resolve(key) + path.unlink(missing_ok=True) + + async def exists(self, key: str) -> bool: + return self._resolve(key).exists() + + async def presigned_get_url(self, key: str, ttl_seconds: int) -> str: + raise NotSupportedError("Filesystem backend cannot mint presigned URLs.") + + +async def _stream_file(path: Path) -> AsyncIterator[bytes]: + async with aiofiles.open(path, "rb") as fh: + while True: + chunk = await fh.read(constants.DEFAULT_CHUNK_SIZE) + if not chunk: + break + yield chunk + + +@register_backend(constants.BackendId.FILESYSTEM) +def _build(settings: FileStorageSettings) -> FilesystemBackend: + return FilesystemBackend(root=settings.resolved_fs_root()) diff --git a/modules/file_storage/file_storage/backends/s3.py b/modules/file_storage/file_storage/backends/s3.py new file mode 100644 index 00000000..b8b5057d --- /dev/null +++ b/modules/file_storage/file_storage/backends/s3.py @@ -0,0 +1,159 @@ +"""S3-compatible storage backend (AWS S3, MinIO, Cloudflare R2, ...). + +``aioboto3`` is imported lazily at backend construction so the module loads +even when the optional S3 extra isn't installed; the failure mode is a clear +:class:`ConfigurationError` at boot rather than a confusing import error. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from tempfile import SpooledTemporaryFile +from typing import TYPE_CHECKING, Any + +from file_storage import constants +from file_storage.backends import register_backend +from file_storage.contracts.service import ( + ConfigurationError, + StorageBackendError, + StorageNotFoundError, +) +from file_storage.settings import FileStorageSettings + +if TYPE_CHECKING: + pass + + +class S3Backend: + """S3-compatible object storage. + + Each operation opens a fresh ``aioboto3`` client context (cheap; the + underlying ``aiobotocore`` session is reused via ``self._session``). We + deliberately do not hold one long-lived client across requests because + aioboto3 client contexts are per-event-loop and tests run multiple loops. + """ + + backend_id = constants.BackendId.S3 + supports_presigned_url = True + + def __init__(self, *, bucket: str, region: str, client_kwargs: dict[str, Any]) -> None: + try: + import aioboto3 + except ImportError as exc: + raise ConfigurationError( + "S3 backend requires the 'aioboto3' package. " + "Install with: uv add --optional s3 aioboto3" + ) from exc + self._aioboto3 = aioboto3 + self._session = aioboto3.Session() + self.bucket = bucket + self.region = region + self.client_kwargs = client_kwargs + + def _client(self): + return self._session.client("s3", **self.client_kwargs) + + async def put( + self, + key: str, + stream: AsyncIterator[bytes], + *, + content_type: str, + size: int, + ) -> None: + # Spool to memory up to SPOOL_MAX_SIZE_BYTES, then to a temp file. + # put_object needs a seekable file-like; multipart upload would be + # the next step for very large files (deferred to v2). + with SpooledTemporaryFile(max_size=constants.SPOOL_MAX_SIZE_BYTES) as buf: + async for chunk in stream: + buf.write(chunk) + length = buf.tell() + buf.seek(0) + try: + async with self._client() as client: + await client.put_object( + Bucket=self.bucket, + Key=key, + Body=buf, + ContentType=content_type, + ContentLength=length, + ) + except Exception as exc: + raise StorageBackendError(f"S3 put_object failed: {exc}") from exc + + async def get(self, key: str) -> AsyncIterator[bytes]: + try: + async with self._client() as client: + resp = await client.get_object(Bucket=self.bucket, Key=key) + # ``Body.read()`` works uniformly across aioboto3 (real) and + # moto's patched aiobotocore (tests); ``iter_chunks`` is sync + # under moto, breaking ``async for``. + data = await resp["Body"].read() + except Exception as exc: + if _is_not_found(exc): + raise StorageNotFoundError(key) from exc + raise StorageBackendError(f"S3 get_object failed: {exc}") from exc + + for offset in range(0, len(data), constants.DEFAULT_CHUNK_SIZE): + yield data[offset : offset + constants.DEFAULT_CHUNK_SIZE] + + async def delete(self, key: str) -> None: + try: + async with self._client() as client: + await client.delete_object(Bucket=self.bucket, Key=key) + except Exception as exc: + raise StorageBackendError(f"S3 delete_object failed: {exc}") from exc + + async def exists(self, key: str) -> bool: + try: + async with self._client() as client: + await client.head_object(Bucket=self.bucket, Key=key) + return True + except Exception as exc: + if _is_not_found(exc): + return False + raise StorageBackendError(f"S3 head_object failed: {exc}") from exc + + async def presigned_get_url(self, key: str, ttl_seconds: int) -> str: + try: + async with self._client() as client: + return await client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": key}, + ExpiresIn=ttl_seconds, + ) + except Exception as exc: + raise StorageBackendError(f"presign failed: {exc}") from exc + + +def _is_not_found(exc: Exception) -> bool: + """Detect a 404 from botocore's heterogeneous error shapes.""" + response = getattr(exc, "response", None) + if isinstance(response, dict): + code = response.get("Error", {}).get("Code") + if code in {"NoSuchKey", "404", "NotFound"}: + return True + return getattr(exc, "__class__", type(exc)).__name__ in { + "NoSuchKey", + "ClientError", + } and "404" in str(exc) + + +@register_backend(constants.BackendId.S3) +def _build(settings: FileStorageSettings) -> S3Backend: + if not settings.s3_bucket or not settings.s3_region: + raise ConfigurationError( + f"S3 backend requires {constants.ENV_PREFIX}S3_BUCKET and " + f"{constants.ENV_PREFIX}S3_REGION." + ) + client_kwargs: dict[str, Any] = {"region_name": settings.s3_region} + if settings.s3_endpoint_url: + client_kwargs["endpoint_url"] = settings.s3_endpoint_url + if settings.s3_access_key_id and settings.s3_secret_access_key: + client_kwargs["aws_access_key_id"] = settings.s3_access_key_id + client_kwargs["aws_secret_access_key"] = settings.s3_secret_access_key + return S3Backend( + bucket=settings.s3_bucket, + region=settings.s3_region, + client_kwargs=client_kwargs, + ) diff --git a/modules/file_storage/file_storage/constants.py b/modules/file_storage/file_storage/constants.py new file mode 100644 index 00000000..bd56fe6c --- /dev/null +++ b/modules/file_storage/file_storage/constants.py @@ -0,0 +1,107 @@ +"""Single source of truth for every literal in the file_storage module. + +Permissions, event topics, route prefixes, env-var prefix, page names, table +names, error codes, default config values — all live here. Other files in the +module import from this one rather than embedding string literals, so renames +stay localised and `rg` can prove there are no magic strings outside of this +file. +""" + +from __future__ import annotations + +from typing import Final + +# ── Module identity ────────────────────────────────────────────────── +# ``MODULE_NAME`` is the snake_case package name; ``MODULE_PASCAL`` matches the +# Inertia page-prefix convention (PascalCase of the module dir). ``meta.name`` +# uses ``MODULE_PASCAL`` so the diagnostic code SM003 lines up against +# ``FileStorage/Browse.tsx``. ``MODULE_DISPLAY_NAME`` is the user-visible label +# (menu, permission group) — keep it free-form. +MODULE_NAME: Final = "file_storage" +MODULE_PASCAL: Final = "FileStorage" +MODULE_DISPLAY_NAME: Final = "Files" + +# ── Configuration ──────────────────────────────────────────────────── +ENV_PREFIX: Final = "SM_FILE_STORAGE_" + +# ── Routing ────────────────────────────────────────────────────────── +ROUTE_PREFIX_API: Final = "/api/file-storage" +ROUTE_PREFIX_VIEW: Final = "/file-storage" + +# REST sub-paths (joined to ROUTE_PREFIX_API by the router) +PATH_UPLOAD: Final = "/upload" +PATH_FILES: Final = "/files" +PATH_FILE_BY_ID: Final = "/files/{file_id}" +PATH_FILE_DOWNLOAD: Final = "/files/{file_id}/download" + +# ── Inertia page names ─────────────────────────────────────────────── +PAGE_BROWSE: Final = "FileStorage/Browse" + +# ── Database ───────────────────────────────────────────────────────── +SCHEMA_NAME: Final = "file_storage" +TABLE_STORED_FILE: Final = "file_storage_stored_file" + +# ── i18n ───────────────────────────────────────────────────────────── +LOCALE_NAMESPACE: Final = MODULE_NAME + + +class BackendId: + """Stable identifiers for built-in storage providers. + + Third-party providers register themselves under any string id, but the + built-ins are referenced from settings, the model, and tests, so they + deserve named constants. + """ + + FILESYSTEM: Final = "filesystem" + S3: Final = "s3" + + +class Permission: + UPLOAD: Final = "file_storage.upload" + DOWNLOAD: Final = "file_storage.download" + DELETE: Final = "file_storage.delete" + MANAGE: Final = "file_storage.manage" + + +class Event: + FILE_UPLOADED: Final = "file_storage.file.uploaded" + FILE_DELETED: Final = "file_storage.file.deleted" + + +class FeatureFlag: + PUBLIC_UPLOADS: Final = "file_storage.public_uploads" + + +class ErrorCode: + """Machine-readable codes returned in 4xx response bodies.""" + + TOO_LARGE: Final = "file_storage.too_large" + BAD_TYPE: Final = "file_storage.bad_type" + NOT_FOUND: Final = "file_storage.not_found" + BACKEND_ERROR: Final = "file_storage.backend_error" + + +class I18nKey: + """Translator keys (full dotted paths). Mirror locales/en.json shape.""" + + ERR_NOT_FOUND: Final = "file_storage.errors.not_found" + ERR_TOO_LARGE: Final = "file_storage.errors.too_large" + ERR_BAD_TYPE: Final = "file_storage.errors.bad_type" + ERR_BACKEND: Final = "file_storage.errors.backend_error" + + +# ── Defaults ───────────────────────────────────────────────────────── +DEFAULT_BACKEND: Final = BackendId.FILESYSTEM +DEFAULT_FS_ROOT: Final = "./uploads" +DEFAULT_MAX_FILE_SIZE_BYTES: Final = 100 * 1024 * 1024 # 100 MB +DEFAULT_PRESIGN_TTL_SECONDS: Final = 300 # 5 minutes +DEFAULT_CHUNK_SIZE: Final = 64 * 1024 # 64 KB +SPOOL_MAX_SIZE_BYTES: Final = 10 * 1024 * 1024 # 10 MB before disk-spill + +# ── Menu ───────────────────────────────────────────────────────────── +MENU_ICON: Final = "files" +MENU_ORDER: Final = 40 +MENU_ROLES: Final = ("admin",) +ADMIN_ROLE: Final = "admin" +USER_ROLE: Final = "user" diff --git a/modules/file_storage/file_storage/contracts/__init__.py b/modules/file_storage/file_storage/contracts/__init__.py new file mode 100644 index 00000000..c5ba7f4e --- /dev/null +++ b/modules/file_storage/file_storage/contracts/__init__.py @@ -0,0 +1,25 @@ +"""file_storage contracts — public interface for other modules.""" + +from file_storage.contracts.events import FileDeleted, FileUploaded +from file_storage.contracts.schemas import StoredFileListOut, StoredFileOut +from file_storage.contracts.service import ( + ConfigurationError, + NotSupportedError, + StorageBackend, + StorageBackendError, + StorageError, + StorageNotFoundError, +) + +__all__ = [ + "ConfigurationError", + "FileDeleted", + "FileUploaded", + "NotSupportedError", + "StorageBackend", + "StorageBackendError", + "StorageError", + "StorageNotFoundError", + "StoredFileListOut", + "StoredFileOut", +] diff --git a/modules/file_storage/file_storage/contracts/events.py b/modules/file_storage/file_storage/contracts/events.py new file mode 100644 index 00000000..9171acdf --- /dev/null +++ b/modules/file_storage/file_storage/contracts/events.py @@ -0,0 +1,23 @@ +"""file_storage domain events.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass + +from simple_module_core.events import Event + + +@dataclass +class FileUploaded(Event): + file_id: uuid.UUID + key: str + backend: str + size_bytes: int + uploaded_by: str | None + + +@dataclass +class FileDeleted(Event): + file_id: uuid.UUID + key: str diff --git a/modules/file_storage/file_storage/contracts/schemas.py b/modules/file_storage/file_storage/contracts/schemas.py new file mode 100644 index 00000000..a0ce5e35 --- /dev/null +++ b/modules/file_storage/file_storage/contracts/schemas.py @@ -0,0 +1,37 @@ +"""SQLModel DTOs for the file_storage module.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import ConfigDict +from sqlmodel import Field, SQLModel + + +class StoredFileOut(SQLModel): + """Metadata returned for a stored file.""" + + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + key: str + filename: str + content_type: str + size_bytes: int + backend: str + checksum_sha256: str + uploaded_by: str | None = Field( + default=None, + description="User id from AuditMixin.created_by — populated by the audit listener.", + ) + created_at: datetime | None = None + + +class StoredFileListOut(SQLModel): + """Paginated list response.""" + + items: list[StoredFileOut] + total: int + page: int + per_page: int diff --git a/modules/file_storage/file_storage/contracts/service.py b/modules/file_storage/file_storage/contracts/service.py new file mode 100644 index 00000000..ada82c57 --- /dev/null +++ b/modules/file_storage/file_storage/contracts/service.py @@ -0,0 +1,82 @@ +"""Storage backend contract — extension point for new providers. + +A storage provider is anything that satisfies :class:`StorageBackend`. Adding +a new provider (Azure Blob, GCS, R2 with custom auth, in-memory for tests) +means writing one module that implements this Protocol and self-registers via +:func:`file_storage.backends.register_backend`. + +The Protocol is deliberately narrow: ``put``, ``get``, ``delete``, ``exists``, +plus capability flags so the service layer can dispatch (e.g. presigned-URL +download vs proxied stream) without inspecting backend identity. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Protocol, runtime_checkable + + +class StorageError(Exception): + """Base class for all storage backend errors.""" + + +class StorageNotFoundError(StorageError): + """Raised when an object key does not exist in the backend.""" + + +class StorageBackendError(StorageError): + """Raised when the backend itself fails (network, IO, auth).""" + + +class NotSupportedError(StorageError): + """Raised when a backend does not implement an optional capability.""" + + +class ConfigurationError(StorageError): + """Raised at backend construction when required config is missing.""" + + +@runtime_checkable +class StorageBackend(Protocol): + """Async object storage abstraction. + + Implementations must be safe to share across requests — they're held as + a singleton on ``app.state.file_storage.backend``. + """ + + backend_id: str + """Stable identifier matching the registry key (e.g. ``"filesystem"``).""" + + supports_presigned_url: bool + """Whether :meth:`presigned_get_url` returns a usable URL. + + The download endpoint reads this flag to choose between proxying bytes + and issuing a 302 redirect — no `if backend == "s3"` branching. + """ + + async def put( + self, + key: str, + stream: AsyncIterator[bytes], + *, + content_type: str, + size: int, + ) -> None: + """Persist ``stream`` under ``key``. ``size`` is the total byte count.""" + + async def get(self, key: str) -> AsyncIterator[bytes]: + """Yield the object's bytes in chunks. Raises :class:`StorageNotFoundError`.""" + + async def delete(self, key: str) -> None: + """Remove the object at ``key``. No-op if absent.""" + + async def exists(self, key: str) -> bool: + """Return whether an object exists at ``key``.""" + + async def presigned_get_url(self, key: str, ttl_seconds: int) -> str: + """Return a short-lived URL the client can fetch directly. + + Implementations without presigned-URL support must raise + :class:`NotSupportedError`. The endpoint will not call this on + backends with ``supports_presigned_url=False``. + """ diff --git a/modules/file_storage/file_storage/deps.py b/modules/file_storage/file_storage/deps.py new file mode 100644 index 00000000..13c2004f --- /dev/null +++ b/modules/file_storage/file_storage/deps.py @@ -0,0 +1,26 @@ +"""FastAPI dependencies for the file_storage module.""" + +from __future__ import annotations + +from fastapi import Depends, Request +from simple_module_core.events import EventBus +from simple_module_db.deps import get_db +from sqlalchemy.ext.asyncio import AsyncSession + +from file_storage.service import FileStorageService +from file_storage.services import FileStorageServices + + +def get_file_storage_services(request: Request) -> FileStorageServices: + return request.app.state.file_storage + + +async def get_file_storage_service( + db: AsyncSession = Depends(get_db), + services: FileStorageServices = Depends(get_file_storage_services), +) -> FileStorageService: + return FileStorageService(db, services.backend, services.settings) + + +def get_event_bus(request: Request) -> EventBus: + return request.app.state.sm.event_bus diff --git a/modules/file_storage/file_storage/endpoints/__init__.py b/modules/file_storage/file_storage/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/file_storage/file_storage/endpoints/api.py b/modules/file_storage/file_storage/endpoints/api.py new file mode 100644 index 00000000..ae063094 --- /dev/null +++ b/modules/file_storage/file_storage/endpoints/api.py @@ -0,0 +1,179 @@ +"""REST endpoints for the file_storage module.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status +from fastapi.responses import RedirectResponse, StreamingResponse +from simple_module_core.events import EventBus +from simple_module_hosting.i18n_deps import TranslatorDep +from simple_module_hosting.permissions import RequiresPermission + +from file_storage import constants +from file_storage.contracts.events import FileDeleted, FileUploaded +from file_storage.contracts.schemas import StoredFileListOut, StoredFileOut +from file_storage.deps import get_event_bus, get_file_storage_service +from file_storage.service import ( + ContentTypeNotAllowedError, + FileStorageService, + FileTooLargeError, + RedirectDownload, + StoredFileNotFoundError, + StreamDownload, +) + +router = APIRouter() + + +@router.post( + constants.PATH_UPLOAD, + response_model=StoredFileOut, + status_code=status.HTTP_201_CREATED, + dependencies=[Depends(RequiresPermission(constants.Permission.UPLOAD))], +) +async def upload_file( + t: TranslatorDep, + file: UploadFile = File(...), + service: FileStorageService = Depends(get_file_storage_service), + bus: EventBus = Depends(get_event_bus), +) -> StoredFileOut: + try: + out = await service.upload(file) + except FileTooLargeError as exc: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail={ + "code": constants.ErrorCode.TOO_LARGE, + "message": t.t(constants.I18nKey.ERR_TOO_LARGE), + }, + ) from exc + except ContentTypeNotAllowedError as exc: + raise HTTPException( + status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + detail={ + "code": constants.ErrorCode.BAD_TYPE, + "message": t.t(constants.I18nKey.ERR_BAD_TYPE), + }, + ) from exc + + await bus.publish( + FileUploaded( + file_id=out.id, + key=out.key, + backend=out.backend, + size_bytes=out.size_bytes, + uploaded_by=out.uploaded_by, + ) + ) + return out + + +@router.get( + constants.PATH_FILES, + response_model=StoredFileListOut, + dependencies=[Depends(RequiresPermission(constants.Permission.DOWNLOAD))], +) +async def list_files( + page: int = 1, + per_page: int = 20, + service: FileStorageService = Depends(get_file_storage_service), +) -> StoredFileListOut: + items, total = await service.list_files(page=page, per_page=per_page) + return StoredFileListOut(items=items, total=total, page=page, per_page=per_page) + + +@router.get( + constants.PATH_FILE_BY_ID, + response_model=StoredFileOut, + dependencies=[Depends(RequiresPermission(constants.Permission.DOWNLOAD))], +) +async def get_file( + file_id: uuid.UUID, + t: TranslatorDep, + service: FileStorageService = Depends(get_file_storage_service), +) -> StoredFileOut: + try: + row = await service.get(file_id) + except StoredFileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": constants.ErrorCode.NOT_FOUND, + "message": t.t(constants.I18nKey.ERR_NOT_FOUND), + }, + ) from exc + return StoredFileOut.model_validate( + { + "id": row.id, + "key": row.key, + "filename": row.filename, + "content_type": row.content_type, + "size_bytes": row.size_bytes, + "backend": row.backend, + "checksum_sha256": row.checksum_sha256, + "uploaded_by": row.created_by, + "created_at": row.created_at, + } + ) + + +@router.get( + constants.PATH_FILE_DOWNLOAD, + response_model=None, + dependencies=[Depends(RequiresPermission(constants.Permission.DOWNLOAD))], +) +async def download_file( + file_id: uuid.UUID, + t: TranslatorDep, + service: FileStorageService = Depends(get_file_storage_service), +) -> RedirectResponse | StreamingResponse: + try: + download = await service.download(file_id) + except StoredFileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": constants.ErrorCode.NOT_FOUND, + "message": t.t(constants.I18nKey.ERR_NOT_FOUND), + }, + ) from exc + + if isinstance(download, RedirectDownload): + return RedirectResponse(url=download.url, status_code=status.HTTP_302_FOUND) + + assert isinstance(download, StreamDownload) + row = download.file + return StreamingResponse( + download.body, + media_type=row.content_type, + headers={ + "Content-Disposition": f'attachment; filename="{row.filename}"', + "Content-Length": str(row.size_bytes), + "ETag": f'"{row.checksum_sha256}"', + }, + ) + + +@router.delete( + constants.PATH_FILE_BY_ID, + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(RequiresPermission(constants.Permission.DELETE))], +) +async def delete_file( + file_id: uuid.UUID, + t: TranslatorDep, + service: FileStorageService = Depends(get_file_storage_service), + bus: EventBus = Depends(get_event_bus), +) -> None: + try: + row = await service.delete(file_id) + except StoredFileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "code": constants.ErrorCode.NOT_FOUND, + "message": t.t(constants.I18nKey.ERR_NOT_FOUND), + }, + ) from exc + await bus.publish(FileDeleted(file_id=row.id, key=row.key)) diff --git a/modules/file_storage/file_storage/endpoints/views.py b/modules/file_storage/file_storage/endpoints/views.py new file mode 100644 index 00000000..5098cb28 --- /dev/null +++ b/modules/file_storage/file_storage/endpoints/views.py @@ -0,0 +1,41 @@ +"""Inertia view endpoints for the file_storage module.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from inertia import InertiaResponse +from simple_module_hosting.inertia_deps import InertiaDep +from simple_module_hosting.permissions import RequiresPermission + +from file_storage import constants +from file_storage.deps import get_file_storage_service +from file_storage.service import FileStorageService + +router = APIRouter() + +_PER_PAGE = 20 + + +@router.get( + "/", + response_model=None, + dependencies=[Depends(RequiresPermission(constants.Permission.DOWNLOAD))], +) +async def browse( + inertia: InertiaDep, + page: int = 1, + service: FileStorageService = Depends(get_file_storage_service), +) -> InertiaResponse: + items, total = await service.list_files(page=page, per_page=_PER_PAGE) + # The page name is hard-coded as a literal here (rather than via + # ``constants.PAGE_BROWSE``) so the SM003/SM004 diagnostics — which do + # static AST analysis and cannot resolve attribute access — pair this + # call with ``pages/Browse.tsx``. A unit test asserts the literal + # matches ``constants.PAGE_BROWSE``. + return await inertia.render( + "FileStorage/Browse", + { + "files": [item.model_dump(mode="json") for item in items], + "pagination": {"page": page, "perPage": _PER_PAGE, "total": total}, + }, + ) diff --git a/modules/file_storage/file_storage/locales/en.json b/modules/file_storage/file_storage/locales/en.json new file mode 100644 index 00000000..8814ff8e --- /dev/null +++ b/modules/file_storage/file_storage/locales/en.json @@ -0,0 +1,42 @@ +{ + "browse": { + "title": "Files", + "description": "Manage uploaded files.", + "upload_button": "Upload file", + "uploading": "Uploading...", + "empty_title": "No files yet", + "empty_description": "Upload your first file to get started.", + "count_one": "{count} file", + "count_other": "{count} files" + }, + "table": { + "filename": "Name", + "size": "Size", + "type": "Type", + "uploaded_by": "Uploaded by", + "uploaded_at": "Uploaded", + "actions": "Actions" + }, + "actions": { + "download": "Download", + "delete": "Delete" + }, + "delete_dialog": { + "title": "Delete \"{name}\"?", + "description": "The file will be removed from storage. This cannot be undone.", + "cancel": "Cancel", + "confirm": "Delete" + }, + "toasts": { + "uploaded": "\"{name}\" uploaded", + "upload_failed": "Upload failed", + "deleted": "\"{name}\" deleted", + "delete_failed": "Failed to delete file" + }, + "errors": { + "not_found": "File not found", + "too_large": "File exceeds the maximum allowed size", + "bad_type": "This file type is not allowed", + "backend_error": "Storage backend error" + } +} diff --git a/modules/file_storage/file_storage/models.py b/modules/file_storage/file_storage/models.py new file mode 100644 index 00000000..4e15eaeb --- /dev/null +++ b/modules/file_storage/file_storage/models.py @@ -0,0 +1,45 @@ +"""SQLModel tables for the file_storage module.""" + +from __future__ import annotations + +import uuid + +import sqlalchemy as sa +from simple_module_db.base import create_module_base +from simple_module_db.mixins import AuditMixin, SoftDeleteMixin +from sqlalchemy import Index +from sqlmodel import Field + +from file_storage import constants + +# PostgreSQL → ``file_storage`` schema; SQLite → table name is prefixed below. +Base = create_module_base(constants.MODULE_NAME) + + +class StoredFile(Base, AuditMixin, SoftDeleteMixin, table=True): # ty: ignore[unsupported-base] + """A file persisted to a configured storage backend. + + The ``backend`` column records which provider holds the bytes — important + if the active backend is changed after ingest, since old rows still need + to be located on their original provider until they're migrated. + """ + + __tablename__ = constants.TABLE_STORED_FILE + + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + key: str = Field(max_length=512) + filename: str = Field(max_length=255) + content_type: str = Field(max_length=128) + size_bytes: int = Field() + backend: str = Field(max_length=32) + checksum_sha256: str = Field(max_length=64) + extra_metadata: dict = Field( + default_factory=dict, + sa_type=sa.JSON, + ) + + __table_args__ = ( + Index(f"ix_{constants.TABLE_STORED_FILE}_key", "key", unique=True), + Index(f"ix_{constants.TABLE_STORED_FILE}_created_by", "created_by"), + Index(f"ix_{constants.TABLE_STORED_FILE}_is_deleted", "is_deleted"), + ) diff --git a/modules/file_storage/file_storage/module.py b/modules/file_storage/file_storage/module.py new file mode 100644 index 00000000..d428b9d8 --- /dev/null +++ b/modules/file_storage/file_storage/module.py @@ -0,0 +1,105 @@ +"""file_storage module — pluggable object storage with filesystem + S3 backends.""" + +from __future__ import annotations + +import importlib.resources +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from simple_module_core.feature_flags import FeatureFlagDefinition, FeatureFlagRegistry +from simple_module_core.menu import MenuItem, MenuRegistry, MenuSection +from simple_module_core.module import ModuleBase, ModuleMeta +from simple_module_core.permissions import PermissionRegistry + +from file_storage import constants + +if TYPE_CHECKING: + from fastapi import FastAPI + +logger = logging.getLogger(__name__) + + +class FileStorageModule(ModuleBase): + meta = ModuleMeta( + name=constants.MODULE_PASCAL, + route_prefix=constants.ROUTE_PREFIX_API, + view_prefix=constants.ROUTE_PREFIX_VIEW, + ) + + def register_settings(self, app: FastAPI) -> None: + from file_storage.backends import build_backend + from file_storage.services import FileStorageServices + from file_storage.settings import FileStorageSettings + + settings = FileStorageSettings() + backend = build_backend(settings) + app.state.file_storage = FileStorageServices(settings=settings, backend=backend) + + def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: + from file_storage.endpoints.api import router as api + from file_storage.endpoints.views import router as views + + api_router.include_router(api) + view_router.include_router(views) + + def register_permissions(self, registry: PermissionRegistry) -> None: + registry.add_group( + constants.MODULE_DISPLAY_NAME, + [ + constants.Permission.UPLOAD, + constants.Permission.DOWNLOAD, + constants.Permission.DELETE, + constants.Permission.MANAGE, + ], + ) + registry.map_role( + constants.USER_ROLE, + [ + constants.Permission.UPLOAD, + constants.Permission.DOWNLOAD, + constants.Permission.DELETE, + ], + ) + + def register_menu_items(self, registry: MenuRegistry) -> None: + registry.add( + MenuItem( + label=constants.MODULE_DISPLAY_NAME, + url=constants.ROUTE_PREFIX_VIEW, + icon=constants.MENU_ICON, + order=constants.MENU_ORDER, + section=MenuSection.SIDEBAR, + roles=list(constants.MENU_ROLES), + ) + ) + + def register_feature_flags(self, registry: FeatureFlagRegistry) -> None: + registry.add( + FeatureFlagDefinition( + name=constants.FeatureFlag.PUBLIC_UPLOADS, + description="Allow unauthenticated uploads (future).", + default_enabled=False, + ) + ) + + def locale_dirs(self) -> dict[str, Path]: + base = Path(str(importlib.resources.files(__package__) / "locales")) + return {constants.LOCALE_NAMESPACE: base} + + async def on_startup(self, app: FastAPI) -> None: + """Ensure the filesystem backend's root exists; probe S3 bucket reachability.""" + services = app.state.file_storage + settings = services.settings + if settings.backend == constants.BackendId.FILESYSTEM: + root = settings.resolved_fs_root() + root.mkdir(parents=True, exist_ok=True) + logger.info("file_storage filesystem backend ready at %s", root) + elif settings.backend == constants.BackendId.S3: + logger.info( + "file_storage S3 backend configured for bucket=%s region=%s endpoint=%s", + settings.s3_bucket, + settings.s3_region, + settings.s3_endpoint_url or "(default)", + ) diff --git a/modules/file_storage/file_storage/pages/Browse.tsx b/modules/file_storage/file_storage/pages/Browse.tsx new file mode 100644 index 00000000..c78ea8d0 --- /dev/null +++ b/modules/file_storage/file_storage/pages/Browse.tsx @@ -0,0 +1,198 @@ +import { router, usePage } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { PageShell } from '@simple-module/ui/components/PageShell'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from '@simple-module/ui/components/ui/alert-dialog'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Card } from '@simple-module/ui/components/ui/card'; +import { + Empty, + EmptyDescription, + EmptyMedia, + EmptyTitle, +} from '@simple-module/ui/components/ui/empty'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@simple-module/ui/components/ui/table'; +import { usePermissions } from '@simple-module/ui/hooks/use-permissions'; +import { AuthenticatedLayout } from '@simple-module/ui/layouts/AuthenticatedLayout'; +import { Download, FileBox, Trash2 } from 'lucide-react'; +import { toast } from 'sonner'; +import { UploadDropzone } from './components/UploadDropzone'; +import { PERMISSIONS, ROUTES, UNKNOWN_UPLOADER } from './constants'; + +interface StoredFile { + id: string; + filename: string; + size_bytes: number; + content_type: string; + uploaded_by: string | null; + created_at: string | null; +} + +interface Pagination { + page: number; + perPage: number; + total: number; +} + +interface Props { + files: StoredFile[]; + pagination: Pagination; +} + +interface SharedAuth { + csrf_token?: string; +} + +function formatBytes(n: number): string { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; +} + +function Browse() { + const page = usePage<{ props: Props } & { csrf_token?: string }>(); + const { files, pagination } = page.props as unknown as Props; + const csrf = ((page.props as unknown as SharedAuth).csrf_token ?? '') as string; + const { t } = useT(); + const { can } = usePermissions(); + const canUpload = can(PERMISSIONS.UPLOAD); + const canDelete = can(PERMISSIONS.DELETE); + + function handleDelete(file: StoredFile) { + fetch(ROUTES.apiFile(file.id), { + method: 'DELETE', + headers: { 'X-CSRF-Token': csrf }, + }) + .then((resp) => { + if (!resp.ok) throw new Error('delete failed'); + toast.success(t(keys.file_storage.toasts.deleted, { name: file.filename })); + router.reload({ only: ['files', 'pagination'] }); + }) + .catch(() => toast.error(t(keys.file_storage.toasts.delete_failed))); + } + + return ( + : undefined} + > + + + + + {t(keys.file_storage.table.filename)} + + {t(keys.file_storage.table.type)} + + {t(keys.file_storage.table.size)} + + {t(keys.file_storage.table.uploaded_by)} + + + {t(keys.file_storage.table.actions)} + + + + + {files.map((file) => ( + + {file.filename} + + {file.content_type} + + + {formatBytes(file.size_bytes)} + + + {file.uploaded_by ?? UNKNOWN_UPLOADER} + + +
+ + {canDelete && ( + + + + + + + + {t(keys.file_storage.delete_dialog.title, { name: file.filename })} + + + {t(keys.file_storage.delete_dialog.description)} + + + + + {t(keys.file_storage.delete_dialog.cancel)} + + handleDelete(file)} + className="bg-destructive text-white hover:bg-destructive/90" + > + {t(keys.file_storage.delete_dialog.confirm)} + + + + + )} +
+ + + ))} + {files.length === 0 && pagination.total === 0 && ( + + + + + + + {t(keys.file_storage.browse.empty_title)} + + {t(keys.file_storage.browse.empty_description)} + + + + + )} + +
+ + + ); +} + +Browse.layout = (page: React.ReactNode) => {page}; +export default Browse; diff --git a/modules/file_storage/file_storage/pages/components/UploadDropzone.tsx b/modules/file_storage/file_storage/pages/components/UploadDropzone.tsx new file mode 100644 index 00000000..806e6a7e --- /dev/null +++ b/modules/file_storage/file_storage/pages/components/UploadDropzone.tsx @@ -0,0 +1,59 @@ +import { router } from '@inertiajs/react'; +import { keys, useT } from '@simple-module/i18n'; +import { Button } from '@simple-module/ui/components/ui/button'; +import { Upload } from 'lucide-react'; +import { useRef, useState } from 'react'; +import { toast } from 'sonner'; + +import { ROUTES } from '../constants'; + +interface Props { + csrfToken: string; +} + +export function UploadDropzone({ csrfToken }: Props) { + const { t } = useT(); + const inputRef = useRef(null); + const [busy, setBusy] = useState(false); + + async function handleFiles(files: FileList | null) { + if (!files || files.length === 0) return; + setBusy(true); + try { + for (const file of Array.from(files)) { + const form = new FormData(); + form.append('file', file); + const resp = await fetch(ROUTES.API_UPLOAD, { + method: 'POST', + body: form, + headers: { 'X-CSRF-Token': csrfToken }, + }); + if (!resp.ok) { + toast.error(t(keys.file_storage.toasts.upload_failed)); + } else { + toast.success(t(keys.file_storage.toasts.uploaded, { name: file.name })); + } + } + router.reload({ only: ['files', 'pagination'] }); + } finally { + setBusy(false); + if (inputRef.current) inputRef.current.value = ''; + } + } + + return ( + <> + handleFiles(e.target.files)} + /> + + + ); +} diff --git a/modules/file_storage/file_storage/pages/constants.ts b/modules/file_storage/file_storage/pages/constants.ts new file mode 100644 index 00000000..f6b056b1 --- /dev/null +++ b/modules/file_storage/file_storage/pages/constants.ts @@ -0,0 +1,21 @@ +// TSX-side mirror of file_storage/constants.py — keep in sync. +// All routes, permission strings, and i18n keys used by Browse.tsx / +// UploadDropzone.tsx are declared here so the JSX has no magic strings. + +export const ROUTES = { + VIEW_BROWSE: '/file-storage', + API_UPLOAD: '/api/file-storage/upload', + API_LIST: '/api/file-storage/files', + apiFile: (id: string) => `/api/file-storage/files/${id}`, + apiDownload: (id: string) => `/api/file-storage/files/${id}/download`, +} as const; + +export const PERMISSIONS = { + UPLOAD: 'file_storage.upload', + DOWNLOAD: 'file_storage.download', + DELETE: 'file_storage.delete', + MANAGE: 'file_storage.manage', +} as const; + +// Empty placeholder used when the row's uploader is missing (anonymous backfill). +export const UNKNOWN_UPLOADER = '—'; diff --git a/modules/file_storage/file_storage/py.typed b/modules/file_storage/file_storage/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/modules/file_storage/file_storage/service.py b/modules/file_storage/file_storage/service.py new file mode 100644 index 00000000..05cfb0af --- /dev/null +++ b/modules/file_storage/file_storage/service.py @@ -0,0 +1,208 @@ +"""file_storage service — upload, list, download, delete orchestration.""" + +from __future__ import annotations + +import contextlib +import hashlib +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from fastapi import UploadFile +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from file_storage import constants +from file_storage.contracts.schemas import StoredFileOut +from file_storage.contracts.service import StorageNotFoundError +from file_storage.models import StoredFile + +if TYPE_CHECKING: + from file_storage.contracts.service import StorageBackend + from file_storage.settings import FileStorageSettings + + +class FileTooLargeError(Exception): + """Raised when an upload exceeds the configured size limit.""" + + +class ContentTypeNotAllowedError(Exception): + """Raised when the upload's content type isn't in the allow-list.""" + + +class StoredFileNotFoundError(Exception): + """Raised when a StoredFile lookup misses; endpoints map this to HTTP 404.""" + + +@dataclass +class StreamDownload: + """Response shape for backends that proxy bytes through the app.""" + + file: StoredFile + body: AsyncIterator[bytes] + + +@dataclass +class RedirectDownload: + """Response shape for backends that issue presigned URLs.""" + + file: StoredFile + url: str + + +Download = StreamDownload | RedirectDownload + + +class FileStorageService: + """Orchestrates validation, hashing, backend IO, and DB lifecycle.""" + + def __init__( + self, + db: AsyncSession, + backend: StorageBackend, + settings: FileStorageSettings, + ) -> None: + self.db = db + self.backend = backend + self.settings = settings + + # ── Upload ─────────────────────────────────────────────────────── + + async def upload(self, upload: UploadFile) -> StoredFileOut: + """Validate, stream-hash, persist to backend, and record metadata.""" + content_type = upload.content_type or "application/octet-stream" + self._check_content_type(content_type) + + size = 0 + sha = hashlib.sha256() + max_size = self.settings.max_file_size_bytes + + async def _hashing_stream() -> AsyncIterator[bytes]: + nonlocal size + while True: + chunk = await upload.read(constants.DEFAULT_CHUNK_SIZE) + if not chunk: + break + size += len(chunk) + if size > max_size: + raise FileTooLargeError(f"Upload exceeds {max_size} bytes.") + sha.update(chunk) + yield chunk + + key = _generate_key(upload.filename or "file") + await self.backend.put( + key, + _hashing_stream(), + content_type=content_type, + size=0, # unknown until stream is drained; backends that need it can spool + ) + + # Compensation: on DB failure, drop the just-uploaded object. + try: + row = StoredFile( + key=key, + filename=upload.filename or key, + content_type=content_type, + size_bytes=size, + backend=self.backend.backend_id, + checksum_sha256=sha.hexdigest(), + ) + self.db.add(row) + await self.db.flush() + await self.db.refresh(row) + except Exception: + await self.backend.delete(key) + raise + + return StoredFileOut.model_validate(_to_out_dict(row)) + + def _check_content_type(self, content_type: str) -> None: + allowed = self.settings.allowed_content_types + if allowed is not None and content_type not in allowed: + raise ContentTypeNotAllowedError(f"Content-Type {content_type!r} not in allow-list.") + + # ── Read ───────────────────────────────────────────────────────── + + async def list_files( + self, + *, + page: int = 1, + per_page: int = 20, + created_by: str | None = None, + ) -> tuple[list[StoredFileOut], int]: + base = select(StoredFile) + count_q = select(func.count()).select_from(StoredFile) + if created_by is not None: + base = base.where(StoredFile.created_by == created_by) + count_q = count_q.where(StoredFile.created_by == created_by) + + total = (await self.db.execute(count_q)).scalar() or 0 + result = await self.db.execute( + base.order_by(StoredFile.created_at.desc()) + .offset((page - 1) * per_page) + .limit(per_page) + ) + rows = result.scalars().all() + items = [StoredFileOut.model_validate(_to_out_dict(r)) for r in rows] + return items, total + + async def get(self, file_id: uuid.UUID) -> StoredFile: + row = await self.db.get(StoredFile, file_id) + if row is None: + raise StoredFileNotFoundError(str(file_id)) + return row + + async def download(self, file_id: uuid.UUID) -> Download: + """Return either a streamed body or a redirect URL. + + Dispatch on ``backend.supports_presigned_url`` so the service stays + provider-agnostic — adding a new backend that supports presigning + works without touching this method. + """ + row = await self.get(file_id) + if self.backend.supports_presigned_url: + url = await self.backend.presigned_get_url( + row.key, self.settings.s3_presign_ttl_seconds + ) + return RedirectDownload(file=row, url=url) + body = await self.backend.get(row.key) + return StreamDownload(file=row, body=body) + + # ── Delete ─────────────────────────────────────────────────────── + + async def delete(self, file_id: uuid.UUID) -> StoredFile: + row = await self.get(file_id) + # Soft-delete in DB first; if the backend delete fails afterwards we + # still have a row marked deleted that can be reaped by a janitor. + row.is_deleted = True + row.deleted_at = datetime.now(UTC) + await self.db.flush() + # Object is acceptably absent — eg. a previous delete partially succeeded. + with contextlib.suppress(StorageNotFoundError): + await self.backend.delete(row.key) + return row + + +def _generate_key(filename: str) -> str: + """Build a date-sharded, collision-proof key from the original filename.""" + today = datetime.now(UTC) + suffix = Path(filename).suffix + return f"{today:%Y/%m/%d}/{uuid.uuid4().hex}{suffix}" + + +def _to_out_dict(row: StoredFile) -> dict: + """Project ORM row → DTO dict, mapping ``created_by`` to ``uploaded_by``.""" + return { + "id": row.id, + "key": row.key, + "filename": row.filename, + "content_type": row.content_type, + "size_bytes": row.size_bytes, + "backend": row.backend, + "checksum_sha256": row.checksum_sha256, + "uploaded_by": row.created_by, + "created_at": row.created_at, + } diff --git a/modules/file_storage/file_storage/services.py b/modules/file_storage/file_storage/services.py new file mode 100644 index 00000000..63160020 --- /dev/null +++ b/modules/file_storage/file_storage/services.py @@ -0,0 +1,26 @@ +"""Module-scoped state container. + +Stored as ``app.state.file_storage`` by +:meth:`FileStorageModule.register_settings`. + +Holds the resolved :class:`StorageBackend` so endpoints don't reconstruct it +per request — provider handles (S3 client sessions, open file descriptors) +are pooled inside the backend instance. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from file_storage.contracts.service import StorageBackend + from file_storage.settings import FileStorageSettings + + +@dataclass +class FileStorageServices: + """file_storage module singletons. Single slot at ``app.state.file_storage``.""" + + settings: FileStorageSettings + backend: StorageBackend diff --git a/modules/file_storage/file_storage/settings.py b/modules/file_storage/file_storage/settings.py new file mode 100644 index 00000000..2c0215e1 --- /dev/null +++ b/modules/file_storage/file_storage/settings.py @@ -0,0 +1,72 @@ +"""file_storage settings loaded from SM_FILE_STORAGE_* environment variables.""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from file_storage import constants + + +class FileStorageSettings(BaseSettings): + """Configuration for the file_storage module. + + The active backend is selected by ``backend`` (matches a key in the + backend registry). All ``s3_*`` fields are optional unless ``backend`` + selects S3, in which case the validator below enforces presence of + bucket + region. Custom backends supply their own validation by + subclassing or by reading additional env vars at registration time. + """ + + model_config = SettingsConfigDict( + env_prefix=constants.ENV_PREFIX, + env_file=".env", + extra="ignore", + ) + + backend: str = constants.DEFAULT_BACKEND + + # Filesystem backend + fs_root_path: str = constants.DEFAULT_FS_ROOT + + # S3-compatible backend (works with AWS S3, MinIO, R2, etc.) + s3_bucket: str = "" + s3_region: str = "" + s3_access_key_id: str = "" + s3_secret_access_key: str = "" + s3_endpoint_url: str = "" # custom endpoint for MinIO / R2 + s3_presign_ttl_seconds: int = constants.DEFAULT_PRESIGN_TTL_SECONDS + + # Limits + max_file_size_bytes: int = constants.DEFAULT_MAX_FILE_SIZE_BYTES + allowed_content_types: list[str] | None = Field( + default=None, + description="Whitelist of MIME types. None = any type allowed.", + ) + + @model_validator(mode="after") + def _validate_backend_config(self) -> FileStorageSettings: + """Enforce per-backend required fields. + + We validate at config time rather than at backend construction so + misconfigured prod boots fail fast with a clear message instead of + deferring the error until the first upload. + """ + if self.backend == constants.BackendId.S3: + missing = [ + name + for name, value in ( + (f"{constants.ENV_PREFIX}S3_BUCKET", self.s3_bucket), + (f"{constants.ENV_PREFIX}S3_REGION", self.s3_region), + ) + if not value + ] + if missing: + raise ValueError(f"S3 backend selected but {', '.join(missing)} is not set.") + return self + + def resolved_fs_root(self) -> Path: + """Return ``fs_root_path`` as an absolute Path.""" + return Path(self.fs_root_path).expanduser().resolve() diff --git a/modules/file_storage/pyproject.toml b/modules/file_storage/pyproject.toml new file mode 100644 index 00000000..581ca8f5 --- /dev/null +++ b/modules/file_storage/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "file-storage" +version = "0.1.0" +description = "The FileStorage module" +authors = [] +requires-python = ">=3.12" +dependencies = [ + "simple-module-core", + "simple-module-db", + "simple-module-hosting", + "aiofiles>=23", +] + +[project.optional-dependencies] +s3 = ["aioboto3>=13"] + +[project.entry-points.simple_module] +file_storage = "file_storage.module:FileStorageModule" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.uv.sources] +simple-module-core = { workspace = true } +simple-module-db = { workspace = true } +simple-module-hosting = { workspace = true } diff --git a/modules/file_storage/tests/test_api.py b/modules/file_storage/tests/test_api.py new file mode 100644 index 00000000..1af6c7d5 --- /dev/null +++ b/modules/file_storage/tests/test_api.py @@ -0,0 +1,66 @@ +"""End-to-end tests for the file_storage REST API via authenticated_client.""" + +from __future__ import annotations + +import httpx +from file_storage import constants + + +async def test_upload_then_list(authenticated_client: httpx.AsyncClient): + files = {"file": ("hello.txt", b"hi", "text/plain")} + resp = await authenticated_client.post( + f"{constants.ROUTE_PREFIX_API}{constants.PATH_UPLOAD}", + files=files, + ) + assert resp.status_code == 201, resp.text + body = resp.json() + assert body["filename"] == "hello.txt" + assert body["content_type"] == "text/plain" + assert body["backend"] == constants.BackendId.FILESYSTEM + assert body["size_bytes"] == 2 + + list_resp = await authenticated_client.get( + f"{constants.ROUTE_PREFIX_API}{constants.PATH_FILES}" + ) + assert list_resp.status_code == 200 + listing = list_resp.json() + assert listing["total"] == 1 + assert listing["items"][0]["id"] == body["id"] + + +async def test_download_streams_filesystem_bytes(authenticated_client: httpx.AsyncClient): + payload = b"streamed content" + files = {"file": ("doc.bin", payload, "application/octet-stream")} + upload = await authenticated_client.post( + f"{constants.ROUTE_PREFIX_API}{constants.PATH_UPLOAD}", + files=files, + ) + file_id = upload.json()["id"] + + resp = await authenticated_client.get(f"{constants.ROUTE_PREFIX_API}/files/{file_id}/download") + assert resp.status_code == 200 + assert resp.content == payload + assert "attachment" in resp.headers.get("content-disposition", "") + + +async def test_delete_removes_file(authenticated_client: httpx.AsyncClient): + upload = await authenticated_client.post( + f"{constants.ROUTE_PREFIX_API}{constants.PATH_UPLOAD}", + files={"file": ("gone.txt", b"x", "text/plain")}, + ) + file_id = upload.json()["id"] + + delete = await authenticated_client.delete(f"{constants.ROUTE_PREFIX_API}/files/{file_id}") + assert delete.status_code == 204 + + fetch = await authenticated_client.get(f"{constants.ROUTE_PREFIX_API}/files/{file_id}") + assert fetch.status_code == 404 + + +async def test_get_unknown_id_returns_404(authenticated_client: httpx.AsyncClient): + import uuid + + resp = await authenticated_client.get(f"{constants.ROUTE_PREFIX_API}/files/{uuid.uuid4()}") + # Framework converts 4xx/5xx HTTPException into an Inertia error page; + # status code is preserved but the body is HTML. + assert resp.status_code == 404 diff --git a/modules/file_storage/tests/test_backend_registry.py b/modules/file_storage/tests/test_backend_registry.py new file mode 100644 index 00000000..d5369aab --- /dev/null +++ b/modules/file_storage/tests/test_backend_registry.py @@ -0,0 +1,93 @@ +"""Tests for the backend registry — the extension point for new providers.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +import pytest +from file_storage import constants +from file_storage.backends import ( + build_backend, + register_backend, + registered_backends, + unregister_backend, +) +from file_storage.contracts.service import ConfigurationError, StorageBackend +from file_storage.settings import FileStorageSettings + + +class _DummyBackend: + """Bare-minimum StorageBackend for registry tests.""" + + backend_id = "dummy" + supports_presigned_url = False + + async def put(self, key: str, stream: AsyncIterator[bytes], **kwargs: Any) -> None: + async for _ in stream: + pass + + async def get(self, key: str) -> AsyncIterator[bytes]: # pragma: no cover - unused + async def empty() -> AsyncIterator[bytes]: + if False: + yield b"" + + return empty() + + async def delete(self, key: str) -> None: ... + async def exists(self, key: str) -> bool: + return False + + async def presigned_get_url(self, key: str, ttl_seconds: int) -> str: # pragma: no cover + raise NotImplementedError + + +def test_builtin_backends_are_registered(): + ids = registered_backends() + assert constants.BackendId.FILESYSTEM in ids + assert constants.BackendId.S3 in ids + + +def test_dummy_backend_can_register_and_resolve(tmp_path): + @register_backend("dummy") + def _factory(settings: FileStorageSettings) -> StorageBackend: + return _DummyBackend() + + try: + settings = FileStorageSettings(backend="dummy", fs_root_path=str(tmp_path)) + backend = build_backend(settings) + assert backend.backend_id == "dummy" + assert "dummy" in registered_backends() + finally: + unregister_backend("dummy") + + +def test_unknown_backend_raises_configuration_error(tmp_path): + settings = FileStorageSettings(backend="bogus", fs_root_path=str(tmp_path)) + with pytest.raises(ConfigurationError) as exc_info: + build_backend(settings) + # Error must list the available backends so misconfigured prods can self-diagnose. + msg = str(exc_info.value) + assert constants.BackendId.FILESYSTEM in msg + assert constants.BackendId.S3 in msg + + +def test_inertia_page_literal_matches_constant(): + """SM003/SM004 require a string literal in inertia.render(); guard the duplication.""" + from file_storage.endpoints import views + + text = Path(views.__file__).read_text() + assert f'"{constants.PAGE_BROWSE}"' in text, ( + f"views.py must contain the literal {constants.PAGE_BROWSE!r} to satisfy SM003." + ) + + +def test_filesystem_factory_resolves_root(tmp_path): + settings = FileStorageSettings( + backend=constants.BackendId.FILESYSTEM, + fs_root_path=str(tmp_path), + ) + backend = build_backend(settings) + assert backend.backend_id == constants.BackendId.FILESYSTEM + assert backend.supports_presigned_url is False diff --git a/modules/file_storage/tests/test_filesystem_backend.py b/modules/file_storage/tests/test_filesystem_backend.py new file mode 100644 index 00000000..40a75bb0 --- /dev/null +++ b/modules/file_storage/tests/test_filesystem_backend.py @@ -0,0 +1,67 @@ +"""Tests for the filesystem storage backend.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest +from file_storage.backends.filesystem import FilesystemBackend +from file_storage.contracts.service import ( + NotSupportedError, + StorageBackendError, + StorageNotFoundError, +) + + +async def _bytes_stream(data: bytes) -> AsyncIterator[bytes]: + yield data + + +async def test_put_then_get_roundtrip(tmp_path): + backend = FilesystemBackend(root=tmp_path) + payload = b"hello world" + await backend.put( + "ab/file1.bin", _bytes_stream(payload), content_type="text/plain", size=len(payload) + ) + + body = await backend.get("ab/file1.bin") + out = b"" + async for chunk in body: + out += chunk + assert out == payload + + +async def test_exists_and_delete(tmp_path): + backend = FilesystemBackend(root=tmp_path) + await backend.put("aa/x", _bytes_stream(b"x"), content_type="text/plain", size=1) + + assert await backend.exists("aa/x") is True + await backend.delete("aa/x") + assert await backend.exists("aa/x") is False + + +async def test_get_missing_raises_not_found(tmp_path): + backend = FilesystemBackend(root=tmp_path) + with pytest.raises(StorageNotFoundError): + await backend.get("missing/key") + + +async def test_path_traversal_rejected(tmp_path): + backend = FilesystemBackend(root=tmp_path) + with pytest.raises(StorageBackendError): + await backend.put("../escape", _bytes_stream(b"x"), content_type="text/plain", size=1) + with pytest.raises(StorageBackendError): + await backend.put("/etc/passwd", _bytes_stream(b"x"), content_type="text/plain", size=1) + + +async def test_presigned_url_not_supported(tmp_path): + backend = FilesystemBackend(root=tmp_path) + with pytest.raises(NotSupportedError): + await backend.presigned_get_url("any", 60) + + +async def test_two_char_shard_layout(tmp_path): + backend = FilesystemBackend(root=tmp_path) + await backend.put("ab/file.bin", _bytes_stream(b"x"), content_type="text/plain", size=1) + # Stored at /ab/ab/file.bin (2-char shard prefix from key[:2]) + assert (tmp_path / "ab" / "ab" / "file.bin").exists() diff --git a/modules/file_storage/tests/test_s3_backend.py b/modules/file_storage/tests/test_s3_backend.py new file mode 100644 index 00000000..ec3d9589 --- /dev/null +++ b/modules/file_storage/tests/test_s3_backend.py @@ -0,0 +1,103 @@ +"""S3 backend tests against a real ``moto.server.ThreadedMotoServer``. + +We intentionally avoid the in-process ``@mock_aws`` decorator: it patches +``botocore`` but leaves ``aiobotocore``'s response handling tripping on +"object bytes can't be used in 'await' expression" for ``Body.read``. Running +moto as an HTTP server and pointing the S3 backend at it via +``s3_endpoint_url`` exercises the real aioboto3 client end-to-end. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import AsyncIterator, Generator + +import boto3 +import pytest +from file_storage import constants +from file_storage.backends import build_backend +from file_storage.contracts.service import StorageNotFoundError +from file_storage.settings import FileStorageSettings +from moto.server import ThreadedMotoServer + +_BUCKET = "test-bucket" +_REGION = "us-east-1" + + +@pytest.fixture(scope="module") +def moto_endpoint() -> Generator[str, None, None]: + server = ThreadedMotoServer(port=0) + server.start() + host, port = server.get_host_and_port() + yield f"http://{host}:{port}" + server.stop() + + +@pytest.fixture +def s3_settings(moto_endpoint: str) -> FileStorageSettings: + return FileStorageSettings( + backend=constants.BackendId.S3, + s3_bucket=_BUCKET, + s3_region=_REGION, + s3_access_key_id="test", + s3_secret_access_key="test", + s3_endpoint_url=moto_endpoint, + ) + + +@pytest.fixture(autouse=True) +def reset_bucket(moto_endpoint: str): + """Recreate the bucket fresh per test (moto persists across tests in server mode).""" + client = boto3.client( + "s3", + region_name=_REGION, + endpoint_url=moto_endpoint, + aws_access_key_id="test", + aws_secret_access_key="test", + ) + with contextlib.suppress(client.exceptions.ClientError): + client.delete_bucket(Bucket=_BUCKET) + client.create_bucket(Bucket=_BUCKET) + yield + + +async def _bytes_stream(data: bytes) -> AsyncIterator[bytes]: + yield data + + +async def _drain(stream: AsyncIterator[bytes]) -> bytes: + out = b"" + async for chunk in stream: + out += chunk + return out + + +async def test_s3_put_get_delete_roundtrip(s3_settings): + backend = build_backend(s3_settings) + payload = b"contents" + await backend.put( + "2026/04/19/abc.bin", + _bytes_stream(payload), + content_type="application/octet-stream", + size=len(payload), + ) + assert await backend.exists("2026/04/19/abc.bin") is True + out = await _drain(backend.get("2026/04/19/abc.bin")) + assert out == payload + + await backend.delete("2026/04/19/abc.bin") + assert await backend.exists("2026/04/19/abc.bin") is False + + +async def test_s3_get_missing_raises_not_found(s3_settings): + backend = build_backend(s3_settings) + with pytest.raises(StorageNotFoundError): + await _drain(backend.get("missing")) + + +async def test_s3_presigned_url_format(s3_settings): + backend = build_backend(s3_settings) + url = await backend.presigned_get_url("any/key.txt", ttl_seconds=300) + assert url.startswith("http") + assert "any/key.txt" in url + assert "X-Amz-Signature" in url or "Signature" in url diff --git a/modules/file_storage/tests/test_service.py b/modules/file_storage/tests/test_service.py new file mode 100644 index 00000000..16b6491e --- /dev/null +++ b/modules/file_storage/tests/test_service.py @@ -0,0 +1,204 @@ +"""Tests for FileStorageService — validation, hashing, lifecycle, capability dispatch.""" + +from __future__ import annotations + +import hashlib +from collections.abc import AsyncIterator +from io import BytesIO +from typing import Any + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from fastapi import UploadFile +from file_storage import constants +from file_storage.backends.filesystem import FilesystemBackend +from file_storage.contracts.service import StorageBackend +from file_storage.models import StoredFile +from file_storage.service import ( + ContentTypeNotAllowedError, + FileStorageService, + FileTooLargeError, + RedirectDownload, + StoredFileNotFoundError, + StreamDownload, +) +from file_storage.settings import FileStorageSettings +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + + +def _upload(name: str, data: bytes, content_type: str = "application/octet-stream") -> UploadFile: + return UploadFile(filename=name, file=BytesIO(data), headers={"content-type": content_type}) # type: ignore[arg-type] + + +def _settings(tmp_path, **overrides) -> FileStorageSettings: + return FileStorageSettings( + backend=constants.BackendId.FILESYSTEM, + fs_root_path=str(tmp_path), + **overrides, + ) + + +async def test_upload_persists_metadata_and_bytes(tmp_path, db_session: AsyncSession): + settings = _settings(tmp_path) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + payload = b"hello" + out = await svc.upload(_upload("note.txt", payload, "text/plain")) + + assert out.filename == "note.txt" + assert out.content_type == "text/plain" + assert out.size_bytes == len(payload) + assert out.checksum_sha256 == hashlib.sha256(payload).hexdigest() + assert out.backend == constants.BackendId.FILESYSTEM + + # Backend round-trip + body = await backend.get(out.key) + out_bytes = b"" + async for chunk in body: + out_bytes += chunk + assert out_bytes == payload + + +async def test_upload_rejects_oversize(tmp_path, db_session: AsyncSession): + settings = _settings(tmp_path, max_file_size_bytes=4) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + with pytest.raises(FileTooLargeError): + await svc.upload(_upload("big.bin", b"way too large")) + + +async def test_upload_rejects_disallowed_content_type(tmp_path, db_session: AsyncSession): + settings = _settings(tmp_path, allowed_content_types=["text/plain"]) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + with pytest.raises(ContentTypeNotAllowedError): + await svc.upload(_upload("evil.exe", b"x", "application/octet-stream")) + + +async def test_upload_compensates_on_db_failure(tmp_path, db_session: AsyncSession): + """If the DB write fails, the just-uploaded object must be removed from the backend.""" + settings = _settings(tmp_path) + + deleted_keys: list[str] = [] + + class _RecordingBackend(FilesystemBackend): + async def delete(self, key: str) -> None: # type: ignore[override] + deleted_keys.append(key) + await super().delete(key) + + backend = _RecordingBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + # Patch flush to fail once, succeed thereafter — provokes the compensation path. + real_flush = db_session.flush + call_count = {"n": 0} + + async def flaky_flush(*args: Any, **kwargs: Any) -> None: + call_count["n"] += 1 + if call_count["n"] == 1: + raise RuntimeError("boom") + return await real_flush(*args, **kwargs) + + monkeypatched = MonkeyPatch() + monkeypatched.setattr(db_session, "flush", flaky_flush) + try: + with pytest.raises(RuntimeError): + await svc.upload(_upload("doomed.bin", b"x")) + finally: + monkeypatched.undo() + + assert len(deleted_keys) == 1 + + +async def test_list_files_paginates_and_filters_by_user(tmp_path, db_session: AsyncSession): + settings = _settings(tmp_path) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + for i in range(3): + await svc.upload(_upload(f"f{i}.bin", f"x{i}".encode())) + + items, total = await svc.list_files(page=1, per_page=2) + assert total == 3 + assert len(items) == 2 + + items_p2, _ = await svc.list_files(page=2, per_page=2) + assert len(items_p2) == 1 + + +async def test_delete_marks_row_and_removes_object(tmp_path, db_session: AsyncSession): + settings = _settings(tmp_path) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + + out = await svc.upload(_upload("doomed.bin", b"x")) + await svc.delete(out.id) + + # Bytes are gone from the backend. + assert await backend.exists(out.key) is False + + # Row exists but is_deleted=True (use include_deleted to bypass the soft-delete filter). + stmt = select(StoredFile).where(StoredFile.id == out.id).execution_options(include_deleted=True) + row = (await db_session.execute(stmt)).scalar_one() + assert row.is_deleted is True + + +async def test_get_missing_raises_not_found(tmp_path, db_session: AsyncSession): + import uuid + + settings = _settings(tmp_path) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + with pytest.raises(StoredFileNotFoundError): + await svc.get(uuid.uuid4()) + + +async def test_download_dispatches_on_backend_capability(tmp_path, db_session: AsyncSession): + """The service picks Stream vs Redirect from supports_presigned_url, never from backend_id.""" + settings = _settings(tmp_path) + backend = FilesystemBackend(root=tmp_path) + svc = FileStorageService(db_session, backend, settings) + out = await svc.upload(_upload("a.bin", b"x")) + + download = await svc.download(out.id) + assert isinstance(download, StreamDownload) + + # Swap in a presigning backend; download should now redirect. + class _PresigningBackend: + backend_id = "presign-test" + supports_presigned_url = True + + async def put(self, key, stream, **kwargs): ... + async def get(self, key): # pragma: no cover + async def empty(): + if False: + yield b"" + + return empty() + + async def delete(self, key): ... + async def exists(self, key): + return True + + async def presigned_get_url(self, key, ttl_seconds): + return f"https://signed/{key}" + + pb: StorageBackend = _PresigningBackend() # type: ignore[assignment] + svc.backend = pb + download2 = await svc.download(out.id) + assert isinstance(download2, RedirectDownload) + assert download2.url.startswith("https://signed/") + + +async def _drain(stream: AsyncIterator[bytes]) -> bytes: + out = b"" + async for chunk in stream: + out += chunk + return out + + +_unused: Any = None diff --git a/packages/i18n/src/generated-resources.ts b/packages/i18n/src/generated-resources.ts index 18d4a801..e23bd634 100644 --- a/packages/i18n/src/generated-resources.ts +++ b/packages/i18n/src/generated-resources.ts @@ -18,6 +18,34 @@ export default { 'dashboard.home.title': '', 'dashboard.home.welcome_card_title': '', 'dashboard.home.welcome_message': '', + 'file_storage.actions.delete': '', + 'file_storage.actions.download': '', + 'file_storage.browse.count_one': '', + 'file_storage.browse.count_other': '', + 'file_storage.browse.description': '', + 'file_storage.browse.empty_description': '', + 'file_storage.browse.empty_title': '', + 'file_storage.browse.title': '', + 'file_storage.browse.upload_button': '', + 'file_storage.browse.uploading': '', + 'file_storage.delete_dialog.cancel': '', + 'file_storage.delete_dialog.confirm': '', + 'file_storage.delete_dialog.description': '', + 'file_storage.delete_dialog.title': '', + 'file_storage.errors.backend_error': '', + 'file_storage.errors.bad_type': '', + 'file_storage.errors.not_found': '', + 'file_storage.errors.too_large': '', + 'file_storage.table.actions': '', + 'file_storage.table.filename': '', + 'file_storage.table.size': '', + 'file_storage.table.type': '', + 'file_storage.table.uploaded_at': '', + 'file_storage.table.uploaded_by': '', + 'file_storage.toasts.delete_failed': '', + 'file_storage.toasts.deleted': '', + 'file_storage.toasts.upload_failed': '', + 'file_storage.toasts.uploaded': '', 'host.error.forbidden_description': '', 'host.error.forbidden_title': '', 'host.error.generic_description': '', diff --git a/packages/i18n/src/keys.generated.ts b/packages/i18n/src/keys.generated.ts index d0ee311e..b73dcd47 100644 --- a/packages/i18n/src/keys.generated.ts +++ b/packages/i18n/src/keys.generated.ts @@ -29,6 +29,49 @@ export const keys = { welcome_message: 'dashboard.home.welcome_message', }, }, + file_storage: { + actions: { + delete: 'file_storage.actions.delete', + download: 'file_storage.actions.download', + }, + browse: { + count: 'file_storage.browse.count', + count_one: 'file_storage.browse.count_one', + count_other: 'file_storage.browse.count_other', + description: 'file_storage.browse.description', + empty_description: 'file_storage.browse.empty_description', + empty_title: 'file_storage.browse.empty_title', + title: 'file_storage.browse.title', + upload_button: 'file_storage.browse.upload_button', + uploading: 'file_storage.browse.uploading', + }, + delete_dialog: { + cancel: 'file_storage.delete_dialog.cancel', + confirm: 'file_storage.delete_dialog.confirm', + description: 'file_storage.delete_dialog.description', + title: 'file_storage.delete_dialog.title', + }, + errors: { + backend_error: 'file_storage.errors.backend_error', + bad_type: 'file_storage.errors.bad_type', + not_found: 'file_storage.errors.not_found', + too_large: 'file_storage.errors.too_large', + }, + table: { + actions: 'file_storage.table.actions', + filename: 'file_storage.table.filename', + size: 'file_storage.table.size', + type: 'file_storage.table.type', + uploaded_at: 'file_storage.table.uploaded_at', + uploaded_by: 'file_storage.table.uploaded_by', + }, + toasts: { + delete_failed: 'file_storage.toasts.delete_failed', + deleted: 'file_storage.toasts.deleted', + upload_failed: 'file_storage.toasts.upload_failed', + uploaded: 'file_storage.toasts.uploaded', + }, + }, host: { error: { forbidden_description: 'host.error.forbidden_description', diff --git a/pyproject.toml b/pyproject.toml index 478569dd..75b44afa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ dev = [ "ty>=0.0.29", "memray>=1.14; sys_platform != 'win32'", "locust>=2.31", + "aioboto3>=13", + "moto[s3]>=5", ] [tool.ruff] @@ -64,6 +66,7 @@ extra-paths = [ "modules/dashboard", "modules/products", "modules/users", + "modules/file_storage", "host", ] @@ -95,7 +98,7 @@ invalid-argument-type = "ignore" [tool.pytest.ini_options] asyncio_mode = "auto" -testpaths = ["framework/core/tests", "framework/db/tests", "framework/hosting/tests", "framework/testing/tests", "host/tests", "modules/auth/tests", "modules/dashboard/tests", "modules/products/tests", "modules/users/tests", "tests/integration", "tests/e2e", "tests/benchmarks"] +testpaths = ["framework/core/tests", "framework/db/tests", "framework/hosting/tests", "framework/testing/tests", "host/tests", "modules/auth/tests", "modules/dashboard/tests", "modules/products/tests", "modules/users/tests", "modules/file_storage/tests", "tests/integration", "tests/e2e", "tests/benchmarks"] markers = [ "e2e: end-to-end tests requiring a live browser", "perf: performance benchmarks (opt-in; run via `make bench`)",