diff --git a/.env.example b/.env.example index 2e53491..69863e0 100644 --- a/.env.example +++ b/.env.example @@ -6,9 +6,11 @@ POSTGRES_PASSWORD=flowix POSTGRES_DB=flowix # MinIO +# ⚠️ dev-only значения — обязательно заменить в prod (issue #55: дефолтные креды minioadmin запрещены, +# compose не поднимется без MINIO_ACCESS_KEY/MINIO_SECRET_KEY) MINIO_ENDPOINT=localhost:9000 -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin +MINIO_ACCESS_KEY=change-me-minio-access-key +MINIO_SECRET_KEY=change-me-minio-secret-key MINIO_SECURE=false VIDEO_STORAGE_BUCKET=videos @@ -31,10 +33,9 @@ UPLOAD_MAX_BYTES=5368709120 # Transcoder (Phase 10) — limit CPU/RAM per job, sequential renditions FFMPEG_THREADS=2 FFMPEG_PRESET=veryfast -# Phase 12: adaptive ladder + HW + pipe streaming +# Phase 12: adaptive ladder + HW ENCODE_MODE=cbr FFMPEG_HWACCEL=auto -TRANSCODER_PIPE_INPUT=true # Phase 15 cost: delete raw/{id}/original.mp4 after ready unless true; lifecycle 7d via mc ilm is safety net KEEP_RAW=false # Phase 15: set DATABASE_URL to pgbouncer to enable pooled connections (transaction mode, statement_cache_size=0) @@ -60,6 +61,10 @@ MINIO_PUBLIC_ENDPOINT=http://localhost:9000 MINIO_API_CORS_ALLOW_ORIGIN=* # Redis — rate-limit storage (auth slowapi + gateway fixed-window, issue #52) +# issue #55: requirepass обязателен — compose не поднимется без REDIS_PASSWORD. +# Сервисы в compose получают redis://:$REDIS_PASSWORD@redis:6379/0 автоматически; +# для dev на хосте задай REDIS_URL=redis://:@localhost:6379/0 +REDIS_PASSWORD=change-me-redis-password REDIS_URL=redis://redis:6379/0 REDIS_PORT=6379 @@ -72,16 +77,17 @@ CORS_ALLOWED_ORIGINS=http://localhost:3000 #TRUSTED_PROXY_CIDRS= # App ports +# issue #55: auth/metadata/upload/transcoder НЕ публикуются на хост — клиенты идут +# через gateway (:8080), Prometheus скрейпит по внутренней сети compose. +# Локальный dev: make dev-auth (порт 8001 фикс. в uvicorn), dev-metadata, dev-upload GATEWAY_PORT=8080 -AUTH_PORT=8001 -METADATA_PORT=8002 -UPLOAD_PORT=8003 # Issue #43: nginx-vod is internal-only (no published port) — HLS goes through gateway /hls/* # NGINX_VOD_PORT=8081 FRONTEND_PORT=3000 PROMETHEUS_PORT=9090 GRAFANA_PORT=3001 -GRAFANA_USER=admin -GRAFANA_PASSWORD=admin +# ⚠️ dev-only значения — обязательно заменить в prod (issue #55: дефолт admin/admin запрещён) +GRAFANA_USER=change-me-grafana-user +GRAFANA_PASSWORD=change-me-grafana-password TRANSCODER_METRICS_PORT=8004 LOKI_PORT=3100 diff --git a/AGENTS.md b/AGENTS.md index 67e0ef4..15c3f21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ Strong success criteria let you loop independently. Weak criteria ("make it work ### Стек и структура (кратко) - **Go (chi `go-chi/chi/v5`):** `services/gateway`, `services/metadata`, `services/upload` — `golang:1.27-alpine`, `pgx`, `zerolog`, `amqp091-go`. Версионирование `/api/v1`. -- **Python (FastAPI + Celery + uv):** `services/auth` (`src/main.py:1`), `services/transcoder` (`app/celery_app.py:1`, `app/tasks.py:1`) — `python:3.14-slim`, `uv`, `argon2`, `aiobotocore`, `ffmpeg-python`. +- **Python (FastAPI + pika + uv):** `services/auth` (`src/main.py:1`), `services/transcoder` (`app/consumer.py:1`) — `python:3.14-slim`, `uv`, `argon2`, `aiobotocore`, `ffmpeg-python`. - **Infra:** `deploy/docker-compose.yml:1` — `postgres:16-alpine :5432`, `minio/minio :9000/:9001`, `rabbitmq:3-management :5672/:15672`, `nginx-vod :8081`, `gateway :8080`, `frontend :3000`. - **Frontend:** `frontend/` — Next.js 14 App Router + `hls.js` + `zustand` + `tailwind`. - **Transcoding JIT:** храним 3 MP4/рип (360/720/1080) с `-force_key_frames "expr:gte(t,n_forced*2)"` и `-sc_threshold 0` (`docs/spec.md:82`), nginx-vod режет на сегменты. @@ -98,7 +98,7 @@ make swagger # swag init для metadata+upload make e2e # scripts/e2e.sh — upload→transcode→hls ``` -Локально без Docker: `make dev-auth` (`:8001`), `make dev-transcoder` (Celery), `make dev-metadata` (`:8002`), `make dev-upload` (`:8003`), `make dev-gateway` (`:8080`), `make dev-frontend` (`:3000`). +Локально без Docker: `make dev-auth` (`:8001`), `make dev-transcoder` (pika-воркер), `make dev-metadata` (`:8002`), `make dev-upload` (`:8003`), `make dev-gateway` (`:8080`), `make dev-frontend` (`:3000`). ### Кодстайл diff --git a/CLAUDE.md b/CLAUDE.md index 67e0ef4..15c3f21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,7 @@ Strong success criteria let you loop independently. Weak criteria ("make it work ### Стек и структура (кратко) - **Go (chi `go-chi/chi/v5`):** `services/gateway`, `services/metadata`, `services/upload` — `golang:1.27-alpine`, `pgx`, `zerolog`, `amqp091-go`. Версионирование `/api/v1`. -- **Python (FastAPI + Celery + uv):** `services/auth` (`src/main.py:1`), `services/transcoder` (`app/celery_app.py:1`, `app/tasks.py:1`) — `python:3.14-slim`, `uv`, `argon2`, `aiobotocore`, `ffmpeg-python`. +- **Python (FastAPI + pika + uv):** `services/auth` (`src/main.py:1`), `services/transcoder` (`app/consumer.py:1`) — `python:3.14-slim`, `uv`, `argon2`, `aiobotocore`, `ffmpeg-python`. - **Infra:** `deploy/docker-compose.yml:1` — `postgres:16-alpine :5432`, `minio/minio :9000/:9001`, `rabbitmq:3-management :5672/:15672`, `nginx-vod :8081`, `gateway :8080`, `frontend :3000`. - **Frontend:** `frontend/` — Next.js 14 App Router + `hls.js` + `zustand` + `tailwind`. - **Transcoding JIT:** храним 3 MP4/рип (360/720/1080) с `-force_key_frames "expr:gte(t,n_forced*2)"` и `-sc_threshold 0` (`docs/spec.md:82`), nginx-vod режет на сегменты. @@ -98,7 +98,7 @@ make swagger # swag init для metadata+upload make e2e # scripts/e2e.sh — upload→transcode→hls ``` -Локально без Docker: `make dev-auth` (`:8001`), `make dev-transcoder` (Celery), `make dev-metadata` (`:8002`), `make dev-upload` (`:8003`), `make dev-gateway` (`:8080`), `make dev-frontend` (`:3000`). +Локально без Docker: `make dev-auth` (`:8001`), `make dev-transcoder` (pika-воркер), `make dev-metadata` (`:8002`), `make dev-upload` (`:8003`), `make dev-gateway` (`:8080`), `make dev-frontend` (`:3000`). ### Кодстайл diff --git a/Makefile b/Makefile index be53735..07d3bae 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ COMPOSE=docker compose --env-file .env -f deploy/docker-compose.yml up: $(COMPOSE) up --build -d - @echo "infra: postgres :5432, minio :9000/:9001, rabbit :5672/:15672, gateway :8080 (hls; nginx-vod internal), prometheus :9090, grafana :3001, loki :3100, transcoder :8004/metrics" + @echo "infra: postgres :5432 (loopback), minio :9000/:9001 (loopback), rabbit :5672/:15672 (loopback), redis :6379 (loopback, requirepass), gateway :8080, frontend :3000, prometheus :9090, grafana :3001, loki :3100 (loopback); auth/metadata/upload 8001-8003 не публикуются (issue #55) — e2e через gateway" up-frontend: $(COMPOSE) up --build -d frontend @@ -100,8 +100,6 @@ dev-auth: dev-transcoder: uv run --project services/transcoder python -m app.consumer -dev-transcoder-celery: # legacy, Phase 10 — celery deprecated, use dev-transcoder (pika) - uv run --project services/transcoder celery -A app.celery_app worker --loglevel=info dev-metadata: set -a; . ./.env 2>/dev/null || true; export ENV=dev; cd services/metadata && go run ./cmd/server diff --git a/README.md b/README.md index 6613763..2fb54d1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - **JIT packaging** — only 3 MP4s stored, HLS/DASH segmented on-the-fly by `nginx-vod` (`vod_mode mapped`, `vod_segment_duration 2000`) - **Async pipeline** — `upload → RabbitMQ (video.uploaded) → pika + FFmpeg (sequential, -threads 2 -preset veryfast, heartbeat 600) → MinIO → metadata (ready) → HLS` - **Secure & scalable** — `INTERNAL_TOKEN` for `/internal/*`, `MaxBytesReader 5GB` (`UPLOAD_MAX_BYTES`), `private` raw bucket (`renditions/thumbnails` download), owner attribution (`owner_email`) -- **Microservices** — Go (`chi`) for gateway/metadata/upload, Python (`FastAPI` + `pika`/`Celery` legacy) for auth/transcoder +- **Microservices** — Go (`chi`) for gateway/metadata/upload, Python (`FastAPI` + `pika`) for auth/transcoder - **Modern frontend** — Next.js 14 App Router + `hls.js` + `zustand` + `tailwind`, `playbackRate 0.5–2×`, owner badges, delete (`DELETE /api/v1/videos/:id` owner only + S3 cleanup) - **Production-ready infra** — Postgres 16 + `golang-migrate` (`deploy/migrations`), MinIO, RabbitMQ, Docker Compose, healthchecks, JWT (HS256, Argon2) @@ -42,7 +42,7 @@ | Layer | Tech | |-------|------| | Gateway / Metadata / Upload | Go 1.27, `go-chi/chi/v5`, `pgx`, `zerolog`, `amqp091-go`, `minio-go` (metadata delete) | -| Auth / Transcoder | Python 3.14, FastAPI, `pika` (primary) / `Celery` legacy, `uv`, `argon2`, `minio`, `ffmpeg` (`-threads 2 -preset veryfast`, `heartbeat 600`) | +| Auth / Transcoder | Python 3.14, FastAPI, `pika`, `uv`, `argon2`, `minio`, `ffmpeg` (`-threads 2 -preset veryfast`, `heartbeat 600`) | | Infra | Postgres 16 + `golang-migrate` (`deploy/migrations`), MinIO (private raw, download renditions), RabbitMQ 3, `nginx-vod-module` (`envsubst` INTERNAL_TOKEN), FFmpeg | | Frontend | Next.js 14, `hls.js` 1.5 (`maxBufferLength 4`), `zustand` 4, Tailwind | | Tooling | Docker Compose, `golangci-lint`, `black`/`ruff`/`mypy`, `eslint`/`prettier` | @@ -93,7 +93,7 @@ uv sync --project services/transcoder make migrate-up # golang-migrate up (if postgres is running without compose) make dev-auth # uvicorn src.main:app --reload --port 8001 -make dev-transcoder # python -m app.consumer (pika, heartbeat 600) — legacy: make dev-transcoder-celery +make dev-transcoder # python -m app.consumer (pika, heartbeat 600) make dev-metadata # :8002 (needs MINIO_ENDPOINT, INTERNAL_TOKEN) make dev-upload # :8003 make dev-gateway # :8080 diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 33f1887..3be136d 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -51,7 +51,6 @@ services: FFMPEG_PRESET: ${FFMPEG_PRESET:-veryfast} ENCODE_MODE: ${ENCODE_MODE:-cbr} FFMPEG_HWACCEL: ${FFMPEG_HWACCEL:-auto} - TRANSCODER_PIPE_INPUT: ${TRANSCODER_PIPE_INPUT:-true} deploy: resources: limits: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 66aaa5c..e178a04 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -9,8 +9,10 @@ services: # Phase 15: pgbouncer uses md5, so postgres must store md5 hashes (scram breaks pgbouncer md5) POSTGRES_INITDB_ARGS: "--auth-host=md5 --auth-local=md5" POSTGRES_HOST_AUTH_METHOD: md5 + # issue #55: postgres/pgbouncer/rabbit/redis/loki/prometheus/grafana публикуются + # только на loopback — dev-доступ с хоста, но не с внешних интерфейсов ports: - - "${POSTGRES_PORT:-5432}:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" volumes: - pgdata:/var/lib/postgresql/data # Legacy fallback for fresh volume — primary migrations are golang-migrate (deploy/migrations) @@ -39,7 +41,7 @@ services: SERVER_RESET_QUERY: DISCARD ALL IGNORE_STARTUP_PARAMETERS: extra_float_digits,search_path ports: - - "${PGBOUNCER_PORT:-6432}:5432" + - "127.0.0.1:${PGBOUNCER_PORT:-6432}:5432" depends_on: postgres: condition: service_healthy @@ -63,8 +65,9 @@ services: image: minio/minio:latest command: server /data --console-address ":9001" environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-minioadmin} + # issue #55: дефолтные креды minioadmin запрещены — значения обязательны + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required — set it in .env (see .env.example)} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required — set it in .env (see .env.example)} MINIO_API_CORS_ALLOW_ORIGIN: ${MINIO_API_CORS_ALLOW_ORIGIN:-*} # Issue #43: loopback-only publication — reachable for browser presigned PUT/GET # (MINIO_PUBLIC_ENDPOINT), not exposed on external interfaces. Bucket is fully private. @@ -97,14 +100,20 @@ services: exit 0 " environment: - MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-minioadmin} - MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-minioadmin} + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required — set it in .env (see .env.example)} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required — set it in .env (see .env.example)} VIDEO_STORAGE_BUCKET: ${VIDEO_STORAGE_BUCKET:-videos} redis: image: redis:7-alpine + # issue #55: Redis без requirepass на 0.0.0.0 — классический путь компрометации; + # пароль обязателен (compose упадёт без REDIS_PASSWORD), порт — только loopback + command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?REDIS_PASSWORD is required — set it in .env (see .env.example)}"] + environment: + # redis-cli в healthcheck аутентифицируется через REDISCLI_AUTH + REDISCLI_AUTH: ${REDIS_PASSWORD:?REDIS_PASSWORD is required — set it in .env (see .env.example)} ports: - - "${REDIS_PORT:-6379}:6379" + - "127.0.0.1:${REDIS_PORT:-6379}:6379" volumes: - redisdata:/data healthcheck: @@ -118,8 +127,8 @@ services: RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER:-flowix} RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS:-flowix} ports: - - "${RABBITMQ_PORT:-5672}:5672" - - "${RABBITMQ_MGMT_PORT:-15672}:15672" + - "127.0.0.1:${RABBITMQ_PORT:-5672}:5672" + - "127.0.0.1:${RABBITMQ_MGMT_PORT:-15672}:15672" volumes: - rabbitdata:/var/lib/rabbitmq healthcheck: @@ -150,7 +159,7 @@ services: build: ../services/auth environment: DATABASE_URL: postgres://flowix:flowix@postgres:5432/flowix?sslmode=disable - REDIS_URL: redis://redis:6379/0 + REDIS_URL: redis://:${REDIS_PASSWORD:?REDIS_PASSWORD is required — set it in .env (see .env.example)}@redis:6379/0 # Phase 15: set PGBOUNCER_ENABLED=true to route via pgbouncer:5432 (pooled) DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-5} DATABASE_MAX_OVERFLOW: ${DATABASE_MAX_OVERFLOW:-10} @@ -163,10 +172,9 @@ services: # X-Real-IP от gateway (внутри docker-сети) — ключ auth-лимитера по # реальному клиенту; см. services/auth/src/core/limiter.py TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-172.16.0.0/12} - ports: - # ревью фазы 17 (H1): auth не для внешнего доступа — клиенты идут через - # gateway (rate-limit + CORS); порт только для localhost (e2e/dev) - - "127.0.0.1:${AUTH_PORT:-8001}:8001" + # issue #55: auth/metadata/upload не публикуются — клиенты идут только через + # gateway (rate-limit + CORS); Prometheus скрейпит по внутренней сети, + # e2e ходит через gateway. Локальный dev: make dev-auth (:8001 на хосте) depends_on: postgres: condition: service_healthy @@ -200,8 +208,7 @@ services: VIDEO_STORAGE_BUCKET: ${VIDEO_STORAGE_BUCKET:-videos} # Issue #43: host for presigned thumbnail URLs handed to browsers (loopback-published MinIO) MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000} - ports: - - "${METADATA_PORT:-8002}:8002" + # issue #55: порт не публикуется — доступ только через gateway (см. auth) depends_on: postgres: condition: service_healthy @@ -235,8 +242,7 @@ services: MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000} # grace для Shutdown меньше stop_grace_period (issue #63) SHUTDOWN_GRACE: 4m - ports: - - "${UPLOAD_PORT:-8003}:8003" + # issue #55: порт не публикуется — доступ только через gateway (см. auth) depends_on: postgres: condition: service_healthy @@ -268,11 +274,9 @@ services: FFMPEG_PRESET: ${FFMPEG_PRESET:-veryfast} ENCODE_MODE: ${ENCODE_MODE:-cbr} FFMPEG_HWACCEL: ${FFMPEG_HWACCEL:-auto} - TRANSCODER_PIPE_INPUT: ${TRANSCODER_PIPE_INPUT:-true} KEEP_RAW: ${KEEP_RAW:-false} METRICS_PORT: ${TRANSCODER_METRICS_PORT:-8004} - ports: - - "${TRANSCODER_METRICS_PORT:-8004}:8004" + # issue #55: метрики скрейпит Prometheus по внутренней сети — порт не публикуется depends_on: rabbitmq: condition: service_healthy @@ -309,7 +313,7 @@ services: # Если появится LB — указать его CIDR в .env. CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000} TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-} - REDIS_URL: redis://redis:6379/0 + REDIS_URL: redis://:${REDIS_PASSWORD:?REDIS_PASSWORD is required — set it in .env (see .env.example)}@redis:6379/0 ports: - "${GATEWAY_PORT:-8080}:8080" depends_on: @@ -358,7 +362,7 @@ services: - ./prometheus/alert.rules.yml:/etc/prometheus/alert.rules.yml:ro - promdata:/prometheus ports: - - "${PROMETHEUS_PORT:-9090}:9090" + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"] @@ -369,15 +373,15 @@ services: grafana: image: grafana/grafana:11.4.0 environment: - GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:-admin} - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin} + GF_SECURITY_ADMIN_USER: ${GRAFANA_USER:?GRAFANA_USER is required — set it in .env (see .env.example)} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:?GRAFANA_PASSWORD is required — set it in .env (see .env.example)} GF_SERVER_HTTP_PORT: 3000 volumes: - grafanadata:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/etc/grafana/dashboards:ro ports: - - "${GRAFANA_PORT:-3001}:3000" + - "127.0.0.1:${GRAFANA_PORT:-3001}:3000" depends_on: prometheus: condition: service_healthy @@ -397,7 +401,7 @@ services: - ./loki/loki.yml:/etc/loki/loki.yml:ro - lokidata:/loki ports: - - "${LOKI_PORT:-3100}:3100" + - "127.0.0.1:${LOKI_PORT:-3100}:3100" restart: unless-stopped healthcheck: test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready | grep -q ready || curl -f http://localhost:3100/ready"] diff --git a/docs/services-pipeline.md b/docs/services-pipeline.md index 4a66084..51607a8 100644 --- a/docs/services-pipeline.md +++ b/docs/services-pipeline.md @@ -22,12 +22,12 @@ Flowix — MVP видеоплатформы с адаптивным стрими | **auth** | Python FastAPI | `:8001` | Регистрация, логин, refresh, `GET /me`. Выпускает JWT (`HS256`, access 15м / refresh 7д, Argon2) | `postgres.users` | | **metadata** | Go + `chi` | `:8002` | CRUD видео: `GET/POST /api/v1/videos`, `GET/PATCH/DELETE /api/v1/videos/:id`. Внутренний `PATCH /internal/videos/:id/status` для transcoder'а. Валидация, пагинация | `postgres.videos`, `video_renditions` | | **upload** | Go + `chi` | `:8003` | Принимает `multipart/form-data` на `POST /api/v1/videos/upload`, льёт оригинал в MinIO `raw/{id}/original.mp4`, создаёт запись `status=uploaded` через metadata, публикует `video.uploaded` в RabbitMQ | MinIO + RabbitMQ + metadata | -| **transcoder** | Python Celery + FFmpeg | — (воркер) | Слушает `video.uploaded`, качает оригинал, `ffprobe` → 3× FFmpeg параллельно (360p/720p/1080p), льёт `renditions/{id}/{quality}.mp4` в MinIO, `PATCH metadata status=ready`, публикует `video.transcoded`. Тут же делает превью (`-ss 1 -vframes 1`) | MinIO, RabbitMQ, metadata | +| **transcoder** | Python pika + FFmpeg | — (воркер) | Слушает `video.uploaded`, качает оригинал, `ffprobe` → 3× FFmpeg параллельно (360p/720p/1080p), льёт `renditions/{id}/{quality}.mp4` в MinIO, `PATCH metadata status=ready`, публикует `video.transcoded`. Тут же делает превью (`-ss 1 -vframes 1`) | MinIO, RabbitMQ, metadata | | **streaming (nginx-vod)** | nginx + `kaltura/nginx-vod-module` | internal (без публикации порта, issue #43) | Отдаёт HLS/DASH на лету через gateway `/hls/*` (HLSAuth): `GET /hls/{id}/master.m3u8` склеивает 3 MP4 в мастер-манифест, сегменты режет по ключевым кадрам. JIT — храним только MP4, сегменты не прегенерим. `vod_mode mapped`; MP4 читает из MinIO по presigned GET из mapping (`metadata /internal/videos/:id/vod`) | Читает MinIO (presigned) | | **frontend** | Next.js 14 + `hls.js` + `zustand` + `tailwind` | `:3000` | Лента `/`, просмотр `/watch/[id]` (`new Hls().loadSource(master.m3u8)` + `playbackRate 0.5–2x`), загрузка `/upload` (multipart + прогресс) | Gateway + HLS | | **infra: Postgres** | `postgres:16-alpine` | `:5432` | `users`, `videos (status: uploaded/processing/ready/failed)`, `video_renditions` — см. `deploy/postgres/init.sql:1` | — | | **infra: MinIO** | `minio/minio` | `:9000/:9001` | S3-совместимое хранилище: `raw/` оригиналы, `renditions/` готовые MP4, превью | — | -| **infra: RabbitMQ** | `rabbitmq:3-management` | `:5672/:15672` | Очередь `video.uploaded` / `video.transcoded`, Celery брокер. UI на `:15672` | — | +| **infra: RabbitMQ** | `rabbitmq:3-management` | `:5672/:15672` | Очередь `video.uploaded` / `video.transcoded` (брокер pipeline-событий). UI на `:15672` | — | Каждый сервис — свой `Dockerfile` + `go.mod` / `pyproject.toml` (uv), собирается независимо. Локально — `make up` / `docker compose --env-file .env -f deploy/docker-compose.yml up --build -d`. @@ -57,7 +57,7 @@ Flowix — MVP видеоплатформы с адаптивным стрими │ ▼ ┌──────────┐ RabbitMQ video.uploaded - │transcoder│ Celery worker + │transcoder│ pika worker └──────────┘ │ ├─ download raw/{id}/original.mp4 ├─ ffprobe (fps/duration проверка) @@ -139,7 +139,7 @@ ffmpeg -i in.mp4 -vn -c:a aac -b:a 128k -ar 48000 -ac 2 audio.m4a ``` services/gateway|metadata|upload — Go chi (1.27-alpine), см. их README.md services/auth — FastAPI + uv, src/routers/auth.py -services/transcoder — Celery app/{celery_app.py,tasks.py} +services/transcoder — pika-воркер app/consumer.py frontend/ — Next 14 App Router deploy/docker-compose.yml — infra + все сервисы (healthcheck service_healthy) deploy/docker-compose.prod.yml — prod overrides (CDN headers, limits) diff --git a/docs/spec.md b/docs/spec.md index c823c86..f916fb2 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -41,7 +41,7 @@ The backend is split into microservices written in **Go** and **Python** to leve │ ├── metadata/ # Go (Metadata Service) │ ├── upload/ # Go (Upload Service) │ ├── auth/ # Python (FastAPI or Django) -│ ├── transcoder/ # Python (Celery worker) +│ ├── transcoder/ # Python (pika worker) │ └── ... (others as needed) ├── frontend/ # Next app ├── deploy/ # Dockerfiles, docker-compose.yml, nginx configs @@ -81,7 +81,7 @@ Each service has its own `Dockerfile` and can be developed independently. Go-с ### 5. Transcoding Worker (Python) - **Purpose**: Consume `video.uploaded` events, download original from MinIO, run FFmpeg to produce multiple renditions, upload results, publish `video.transcoded` event. -- **Framework**: Celery with Redis/RabbitMQ broker. +- **Framework**: pika consumer with RabbitMQ broker (`video.uploaded`, DLX/retry). - **Key FFmpeg requirements**: - Output formats: H.264/AAC in MP4 container. - Multiple renditions (e.g., 360p, 720p, 1080p) with appropriate bitrates. @@ -164,7 +164,7 @@ Refer to `.env.example` for all variables. Key ones: ### Python Services - Use FastAPI for new services; type hints are mandatory. -- For Celery workers, define tasks in `tasks.py` and use Pydantic models for payloads. +- Worker (transcoder) — pika consumer in `app/consumer.py`; event payloads as JSON dicts. - Use `aiobotocore` for async MinIO operations if needed. - Logging: standard `logging` with JSON formatter in production. @@ -221,7 +221,7 @@ go run ./cmd/server ### Run transcoding worker locally (with hot reload) ```bash cd services/transcoder -celery -A app.celery worker --loglevel=info +python -m app.consumer ``` ### Run frontend dev server diff --git a/go.work.sum b/go.work.sum index 72fff74..33de720 100644 --- a/go.work.sum +++ b/go.work.sum @@ -48,6 +48,9 @@ github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9N github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 1acf37b..a61b377 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -6,7 +6,9 @@ # Phase-5 only: VIDEO_ID= ./scripts/e2e.sh # skip upload, assert HLS for a ready video. # # Endpoints (override via env): -# AUTH, UPLOAD, METADATA, GATEWAY +# GATEWAY (and optionally AUTH/UPLOAD/METADATA for non-default deployments). +# issue #55: app ports 8001-8003 are not published — auth/upload/metadata calls +# default to the gateway, which proxies /api/v1/* to the internal services. # HLS assertions go through the gateway (:8080) — since issue #43 nginx-vod :8081 # is not published; the gateway /hls/* proxy (HLSAuth) is the only playback path. # @@ -15,10 +17,10 @@ # GATEWAY=http://localhost:8080 bash scripts/e2e.sh set -euo pipefail -AUTH=${AUTH:-http://localhost:8001} -UPLOAD=${UPLOAD:-http://localhost:8003} -METADATA=${METADATA:-http://localhost:8002} GATEWAY=${GATEWAY:-http://localhost:8080} +AUTH=${AUTH:-$GATEWAY} +UPLOAD=${UPLOAD:-$GATEWAY} +METADATA=${METADATA:-$GATEWAY} # VOD legacy override kept for back-compat; default = gateway (nginx-vod is internal now) VOD=${VOD:-$GATEWAY} @@ -77,14 +79,7 @@ say "endpoints: auth=$AUTH upload=$UPLOAD metadata=$METADATA gateway=$GATEWAY" # ── 1. health ───────────────────────────────────────────────────────────── say "1) health" [ "$(http_get "$GATEWAY/health")" = "200" ] || fail "gateway not healthy at $GATEWAY/health" -say " gateway: ok" -if [ -z "$VIDEO_ID" ]; then - for pair in "auth:$AUTH" "upload:$UPLOAD" "metadata:$METADATA"; do - name=${pair%%:*}; url=${pair#*:} - [ "$(http_get "$url/health")" = "200" ] || fail "$name not healthy at $url/health" - say " $name: ok" - done -fi +say " gateway: ok (auth/upload/metadata behind gateway, issue #55)" # ── 2. auth + upload (skipped when VIDEO_ID is provided) ─────────────────── if [ -z "$VIDEO_ID" ]; then diff --git a/services/metadata/internal/handler/video.go b/services/metadata/internal/handler/video.go index d0c16d7..473e169 100644 --- a/services/metadata/internal/handler/video.go +++ b/services/metadata/internal/handler/video.go @@ -3,6 +3,7 @@ package handler import ( "context" "encoding/json" + "errors" "log/slog" "net/http" "net/url" @@ -11,6 +12,7 @@ import ( "time" "flowix/metadata/internal/model" + "flowix/metadata/internal/repository" pkgmw "flowix/pkg/middleware" @@ -19,7 +21,7 @@ import ( // VideoStore abstracts persistence — allows in-memory fake for tests (T2 stepwise). type VideoStore interface { - Create(ctx context.Context, ownerID, title, description string) (*model.Video, error) + Create(ctx context.Context, ownerID, title, description string, visibility *model.Visibility) (*model.Video, error) GetByID(ctx context.Context, id string) (*model.Video, error) List(ctx context.Context, limit, offset int, viewerID string) ([]model.Video, error) Update(ctx context.Context, id, ownerID string, req model.UpdateVideoRequest) (*model.Video, error) @@ -119,20 +121,21 @@ func (h *VideoHandler) Create(w http.ResponseWriter, r *http.Request) { writeError(w, r, http.StatusBadRequest, "title required") return } - v, err := h.repo.Create(r.Context(), ownerID, req.Title, req.Description) - if err != nil { - slog.Error("create video failed", "error", err, "owner_id", ownerID) - writeError(w, r, http.StatusInternalServerError, "internal error") + // issue #58: visibility валидируется ДО вставки — невалидное значение не создаёт строку + if req.Visibility != nil && !req.Visibility.Valid() { + writeError(w, r, http.StatusBadRequest, "invalid visibility") return } - if req.Visibility != nil { - if !req.Visibility.Valid() { + v, err := h.repo.Create(r.Context(), ownerID, req.Title, req.Description, req.Visibility) + if err != nil { + // защита на уровне репозитория (проверка до INSERT) — тоже 400 + if errors.Is(err, repository.ErrInvalidVisibility) { writeError(w, r, http.StatusBadRequest, "invalid visibility") return } - if upd, err := h.repo.Update(r.Context(), v.ID, ownerID, model.UpdateVideoRequest{Visibility: req.Visibility}); err == nil { - v = upd - } + slog.Error("create video failed", "error", err, "owner_id", ownerID) + writeError(w, r, http.StatusInternalServerError, "internal error") + return } h.presignThumbnail(r.Context(), v) writeJSON(w, r, http.StatusCreated, v) @@ -168,7 +171,13 @@ func (h *VideoHandler) getVideo(w http.ResponseWriter, r *http.Request, enforceV id := chi.URLParam(r, "id") v, err := h.repo.GetByID(r.Context(), id) if err != nil { - writeError(w, r, http.StatusNotFound, "not found") + // issue #58: только отсутствие записи — 404; ошибка БД — 500 + if errors.Is(err, repository.ErrNotFound) { + writeError(w, r, http.StatusNotFound, "not found") + return + } + slog.Error("get video failed", "error", err, "video_id", id) + writeError(w, r, http.StatusInternalServerError, "internal error") return } if enforceVisibility && v.Visibility == model.VisibilityPrivate && pkgmw.UserIDFromCtx(r.Context()) != v.OwnerID { @@ -196,7 +205,17 @@ type vodClip struct { // Phase 12: adaptive ladder — accept 1..3 renditions (not fixed 3). func (h *VideoHandler) GetVODMapping(w http.ResponseWriter, r *http.Request) { v, err := h.repo.GetByID(r.Context(), chi.URLParam(r, "id")) - if err != nil || v.Status != model.StatusReady || len(v.Renditions) == 0 { + if err != nil { + // issue #58: ошибка БД — 500, отсутствие записи — 404 + if errors.Is(err, repository.ErrNotFound) { + writeError(w, r, http.StatusNotFound, "video not ready") + return + } + slog.Error("vod mapping get failed", "error", err) + writeError(w, r, http.StatusInternalServerError, "internal error") + return + } + if v.Status != model.StatusReady || len(v.Renditions) == 0 { writeError(w, r, http.StatusNotFound, "video not ready") return } @@ -290,15 +309,18 @@ func (h *VideoHandler) Update(w http.ResponseWriter, r *http.Request) { } v, err := h.repo.Update(r.Context(), id, ownerID, req) if err != nil { - if err.Error() == "forbidden" { + // issue #58: sentinel-ошибки через errors.Is; прочие ошибки БД — 500, не 404 + switch { + case errors.Is(err, repository.ErrForbidden): writeError(w, r, http.StatusForbidden, "forbidden") - return - } - if err.Error() == "invalid visibility" { + case errors.Is(err, repository.ErrInvalidVisibility): writeError(w, r, http.StatusBadRequest, "invalid visibility") - return + case errors.Is(err, repository.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not found") + default: + slog.Error("update video failed", "error", err, "video_id", id) + writeError(w, r, http.StatusInternalServerError, "internal error") } - writeError(w, r, http.StatusNotFound, "not found") return } h.presignThumbnail(r.Context(), v) @@ -335,11 +357,16 @@ func (h *VideoHandler) Delete(w http.ResponseWriter, r *http.Request) { } } if err := h.repo.Delete(r.Context(), id, ownerID); err != nil { - if err.Error() == "forbidden" { + // issue #58: sentinel-ошибки через errors.Is; прочие ошибки БД — 500, не 404 + switch { + case errors.Is(err, repository.ErrForbidden): writeError(w, r, http.StatusForbidden, "forbidden") - return + case errors.Is(err, repository.ErrNotFound): + writeError(w, r, http.StatusNotFound, "not found") + default: + slog.Error("delete video failed", "error", err, "video_id", id) + writeError(w, r, http.StatusInternalServerError, "internal error") } - writeError(w, r, http.StatusNotFound, "not found") return } if h.storage != nil { diff --git a/services/metadata/internal/handler/video_test.go b/services/metadata/internal/handler/video_test.go index f9530d6..c2406a1 100644 --- a/services/metadata/internal/handler/video_test.go +++ b/services/metadata/internal/handler/video_test.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "testing" "time" "flowix/metadata/internal/model" + "flowix/metadata/internal/repository" pkgmw "flowix/pkg/middleware" @@ -36,15 +38,23 @@ type fakeStore struct { func newFake() *fakeStore { return &fakeStore{videos: map[string]*model.Video{}} } -func (f *fakeStore) Create(_ context.Context, ownerID, title, description string) (*model.Video, error) { - v := &model.Video{ID: "vid-" + title, OwnerID: ownerID, Title: title, Description: description, Status: model.StatusUploaded, Visibility: model.VisibilityPublic} +func (f *fakeStore) Create(_ context.Context, ownerID, title, description string, visibility *model.Visibility) (*model.Video, error) { + // mirror repo.Create (issue #58): validate visibility BEFORE insert + if visibility != nil && !visibility.Valid() { + return nil, repository.ErrInvalidVisibility + } + vis := model.VisibilityPublic + if visibility != nil { + vis = *visibility + } + v := &model.Video{ID: "vid-" + title, OwnerID: ownerID, Title: title, Description: description, Status: model.StatusUploaded, Visibility: vis} f.videos[v.ID] = v return v, nil } func (f *fakeStore) GetByID(_ context.Context, id string) (*model.Video, error) { v, ok := f.videos[id] if !ok { - return nil, errNotFound() + return nil, repository.ErrNotFound } return v, nil } @@ -74,10 +84,10 @@ func (f *fakeStore) List(_ context.Context, limit, offset int, viewerID string) func (f *fakeStore) Update(_ context.Context, id, ownerID string, req model.UpdateVideoRequest) (*model.Video, error) { v, ok := f.videos[id] if !ok { - return nil, errNotFound() + return nil, repository.ErrNotFound } if v.OwnerID != ownerID { - return nil, errForbidden() + return nil, repository.ErrForbidden } if req.Title != nil { v.Title = *req.Title @@ -87,7 +97,7 @@ func (f *fakeStore) Update(_ context.Context, id, ownerID string, req model.Upda } if req.Visibility != nil { if !req.Visibility.Valid() { - return nil, &fakeErr{"invalid visibility"} + return nil, repository.ErrInvalidVisibility } v.Visibility = *req.Visibility } @@ -96,10 +106,10 @@ func (f *fakeStore) Update(_ context.Context, id, ownerID string, req model.Upda func (f *fakeStore) Delete(_ context.Context, id, ownerID string) error { v, ok := f.videos[id] if !ok { - return errNotFound() + return repository.ErrNotFound } if v.OwnerID != ownerID { - return errForbidden() + return repository.ErrForbidden } delete(f.videos, id) return nil @@ -107,7 +117,7 @@ func (f *fakeStore) Delete(_ context.Context, id, ownerID string) error { func (f *fakeStore) UpdateStatus(_ context.Context, id string, status model.VideoStatus, renditions []model.Rendition) error { v, ok := f.videos[id] if !ok { - return errNotFound() + return repository.ErrNotFound } // idempotency: don't downgrade ready → processing/uploaded if v.Status == model.StatusReady && (status == model.StatusProcessing || status == model.StatusUploaded) { @@ -122,7 +132,7 @@ func (f *fakeStore) UpdateStatus(_ context.Context, id string, status model.Vide func (f *fakeStore) UpdateThumbnail(_ context.Context, id string, thumbnailS3Key string) error { v, ok := f.videos[id] if !ok { - return errNotFound() + return repository.ErrNotFound } v.ThumbnailS3Key = &thumbnailS3Key u := "/thumbnails/" + id + "/thumb.jpg" @@ -130,12 +140,24 @@ func (f *fakeStore) UpdateThumbnail(_ context.Context, id string, thumbnailS3Key return nil } -func errNotFound() error { return &fakeErr{"not found"} } -func errForbidden() error { return &fakeErr{"forbidden"} } +// Issue #58: wrappers that simulate a failing DB for handler 500-mapping tests. +type failingGetStore struct { + *fakeStore + getErr error +} + +func (f *failingGetStore) GetByID(_ context.Context, _ string) (*model.Video, error) { + return nil, f.getErr +} -type fakeErr struct{ s string } +type failingUpdateStore struct { + *fakeStore + updateErr error +} -func (e *fakeErr) Error() string { return e.s } +func (f *failingUpdateStore) Update(_ context.Context, _ string, _ string, _ model.UpdateVideoRequest) (*model.Video, error) { + return nil, f.updateErr +} // helper to build router with auth where needed func testRouter(store VideoStore) chi.Router { @@ -224,6 +246,69 @@ func TestCreateNoAuth(t *testing.T) { } } +// Issue #58: невалидная visibility → 400, записи в БД нет (валидация до вставки). +func TestCreateInvalidVisibilityNoWrite(t *testing.T) { + store := newFake() + r := testRouter(store) + body, _ := json.Marshal(map[string]string{"title": "x", "visibility": "bogus"}) + req := httptest.NewRequest("POST", "/api/v1/videos", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+signToken("owner1")) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 400 { + t.Fatalf("want 400 got %d %s", w.Code, w.Body.String()) + } + if len(store.videos) != 0 { + t.Fatalf("invalid visibility must not create a row, got %d", len(store.videos)) + } +} + +// Issue #58: visibility сохраняется одним INSERT'ом при создании (без UPDATE-довески). +func TestCreateWithVisibility(t *testing.T) { + store := newFake() + r := testRouter(store) + body, _ := json.Marshal(map[string]string{"title": "vis", "visibility": "private"}) + req := httptest.NewRequest("POST", "/api/v1/videos", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+signToken("owner1")) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 201 { + t.Fatalf("want 201 got %d %s", w.Code, w.Body.String()) + } + if v := store.videos["vid-vis"]; v == nil || v.Visibility != model.VisibilityPrivate { + t.Fatalf("visibility private expected, got %+v", store.videos["vid-vis"]) + } +} + +// Issue #58: ошибка БД при Update → 500, а не 404. +func TestUpdateDBErrorIs500(t *testing.T) { + store := &failingUpdateStore{fakeStore: newFake(), updateErr: errors.New("db down")} + r := testRouter(store) + body, _ := json.Marshal(map[string]string{"title": "new"}) + req := httptest.NewRequest("PATCH", "/api/v1/videos/vid-1", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+signToken("owner1")) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 500 { + t.Fatalf("want 500 got %d %s", w.Code, w.Body.String()) + } +} + +// Issue #58: ошибка БД при Get → 500, а не 404. +func TestGetDBErrorIs500(t *testing.T) { + store := &failingGetStore{fakeStore: newFake(), getErr: errors.New("db down")} + r := testRouter(store) + req := httptest.NewRequest("GET", "/api/v1/videos/vid-1", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 500 { + t.Fatalf("want 500 got %d %s", w.Code, w.Body.String()) + } +} + func TestGetAndList(t *testing.T) { store := newFake() store.videos["vid-1"] = &model.Video{ID: "vid-1", OwnerID: "o1", Title: "t1", Status: model.StatusUploaded} diff --git a/services/metadata/internal/repository/video.go b/services/metadata/internal/repository/video.go index 44038a8..4abffef 100644 --- a/services/metadata/internal/repository/video.go +++ b/services/metadata/internal/repository/video.go @@ -3,10 +3,12 @@ package repository import ( "context" + "errors" "fmt" "flowix/metadata/internal/model" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -16,17 +18,27 @@ type VideoRepo struct { func NewVideoRepo(pool *pgxpool.Pool) *VideoRepo { return &VideoRepo{pool: pool} } -func (r *VideoRepo) Create(ctx context.Context, ownerID, title, description string) (*model.Video, error) { +// Sentinel errors — handler маппит их на HTTP-статусы через errors.Is (issue #58): +// любые прочие ошибки считаются ошибками БД и отдаются как 500. +var ( + ErrNotFound = errors.New("video not found") + ErrForbidden = errors.New("forbidden") + ErrInvalidVisibility = errors.New("invalid visibility") +) + +func (r *VideoRepo) Create(ctx context.Context, ownerID, title, description string, visibility *model.Visibility) (*model.Video, error) { + // issue #58: visibility валидируется ДО вставки — невалидное значение не создаёт + // строку; значение передаётся одним INSERT'ом (без последующего UPDATE) + if visibility != nil && !visibility.Valid() { + return nil, ErrInvalidVisibility + } id := uuid.New().String() - vis := model.VisibilityPublic - q := `INSERT INTO videos (id, owner_id, title, description, status, visibility) VALUES ($1,$2,$3,$4,'uploaded',$5) RETURNING id, owner_id, title, description, duration, status, visibility, thumbnail_s3_key, created_at, updated_at` + q := `INSERT INTO videos (id, owner_id, title, description, status, visibility) VALUES ($1,$2,$3,$4,'uploaded',COALESCE($5::video_visibility,'public')) RETURNING id, owner_id, title, description, duration, status, visibility::text, thumbnail_s3_key, created_at, updated_at` v := &model.Video{} - err := r.pool.QueryRow(ctx, q, id, ownerID, title, description, vis).Scan(&v.ID, &v.OwnerID, &v.Title, &v.Description, &v.Duration, &v.Status, &v.Visibility, &v.ThumbnailS3Key, &v.CreatedAt, &v.UpdatedAt) + err := r.pool.QueryRow(ctx, q, id, ownerID, title, description, visibility).Scan(&v.ID, &v.OwnerID, &v.Title, &v.Description, &v.Duration, &v.Status, &v.Visibility, &v.ThumbnailS3Key, &v.CreatedAt, &v.UpdatedAt) if err != nil { return nil, fmt.Errorf("create video: %w", err) } - // fetch owner_email for response (best-effort) - _ = r.pool.QueryRow(ctx, `SELECT email FROM users WHERE id=$1`, ownerID).Scan(&v.OwnerEmail) if v.ThumbnailS3Key != nil { u := "/thumbnails/" + v.ID + "/thumb.jpg" v.ThumbnailURL = &u @@ -39,6 +51,9 @@ func (r *VideoRepo) GetByID(ctx context.Context, id string) (*model.Video, error v := &model.Video{} err := r.pool.QueryRow(ctx, q, id).Scan(&v.ID, &v.OwnerID, &v.OwnerEmail, &v.Title, &v.Description, &v.Duration, &v.Status, &v.Visibility, &v.ThumbnailS3Key, &v.CreatedAt, &v.UpdatedAt) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } return nil, err } if v.ThumbnailS3Key != nil { @@ -87,36 +102,38 @@ func (r *VideoRepo) List(ctx context.Context, limit, offset int, viewerID string } func (r *VideoRepo) Update(ctx context.Context, id, ownerID string, req model.UpdateVideoRequest) (*model.Video, error) { - // only owner can update - v, err := r.GetByID(ctx, id) - if err != nil { - return nil, err - } - if v.OwnerID != ownerID { - return nil, fmt.Errorf("forbidden") - } - if req.Title != nil { - _, err = r.pool.Exec(ctx, `UPDATE videos SET title=$1 WHERE id=$2`, *req.Title, id) - if err != nil { - return nil, err + // issue #57: один атомарный UPDATE вместо read-check-write (TOCTOU) и 3 отдельных + // UPDATE. owner в WHERE закрывает forbidden-проверку, COALESCE не трогает поля, + // не переданные в запросе — конкурентные PATCH не затирают изменения друг друга. + // Email берётся подзапросом в RETURNING, чтобы остаться в одном запросе. + if req.Visibility != nil && !req.Visibility.Valid() { + return nil, ErrInvalidVisibility + } + q := `UPDATE videos v SET title=COALESCE($1, v.title), description=COALESCE($2, v.description), visibility=COALESCE($3::video_visibility, v.visibility) WHERE v.id=$4 AND v.owner_id=$5 RETURNING v.id, v.owner_id, (SELECT email FROM users WHERE id=v.owner_id), v.title, v.description, v.duration, v.status, v.visibility::text, v.thumbnail_s3_key, v.created_at, v.updated_at` + v := &model.Video{} + err := r.pool.QueryRow(ctx, q, req.Title, req.Description, req.Visibility, id, ownerID).Scan(&v.ID, &v.OwnerID, &v.OwnerEmail, &v.Title, &v.Description, &v.Duration, &v.Status, &v.Visibility, &v.ThumbnailS3Key, &v.CreatedAt, &v.UpdatedAt) + if errors.Is(err, pgx.ErrNoRows) { + // no-row в WHERE неразличим: чужое видео или не существует — классифицируем + // отдельным чтением, чтобы сохранить контракт ответов 403/404 + var exists bool + e := r.pool.QueryRow(ctx, `SELECT true FROM videos WHERE id=$1`, id).Scan(&exists) + if e == nil { + return nil, ErrForbidden } - } - if req.Description != nil { - _, err = r.pool.Exec(ctx, `UPDATE videos SET description=$1 WHERE id=$2`, *req.Description, id) - if err != nil { - return nil, err + if !errors.Is(e, pgx.ErrNoRows) { + // transient DB error при классификации не маскируем в 404 + return nil, e } + return nil, ErrNotFound } - if req.Visibility != nil { - if !req.Visibility.Valid() { - return nil, fmt.Errorf("invalid visibility") - } - _, err = r.pool.Exec(ctx, `UPDATE videos SET visibility=$1 WHERE id=$2`, *req.Visibility, id) - if err != nil { - return nil, err - } + if err != nil { + return nil, err } - return r.GetByID(ctx, id) + if v.ThumbnailS3Key != nil { + u := "/thumbnails/" + v.ID + "/thumb.jpg" + v.ThumbnailURL = &u + } + return v, nil } func (r *VideoRepo) Delete(ctx context.Context, id, ownerID string) error { @@ -125,7 +142,7 @@ func (r *VideoRepo) Delete(ctx context.Context, id, ownerID string) error { return err } if v.OwnerID != ownerID { - return fmt.Errorf("forbidden") + return ErrForbidden } _, err = r.pool.Exec(ctx, `DELETE FROM videos WHERE id=$1`, id) return err diff --git a/services/transcoder/README.md b/services/transcoder/README.md index 32afe0c..f4e38ab 100644 --- a/services/transcoder/README.md +++ b/services/transcoder/README.md @@ -7,9 +7,9 @@ ```bash # Локально (нужен RabbitMQ + MinIO + FFmpeg) uv sync --project services/transcoder -python -m app.consumer # pika BlockingConnection, не celery (celery — legacy, будет удален) +python -m app.consumer # pika BlockingConnection (issue #62: celery-стабы удалены) # или -make dev-transcoder # python -m app.consumer (celery legacy: make dev-transcoder-celery) +make dev-transcoder # python -m app.consumer # Через OrbStack docker compose -f deploy/docker-compose.yml up -d transcoder rabbitmq minio @@ -46,10 +46,10 @@ FFMPEG_THREADS=2 FFMPEG_PRESET=veryfast python -m app.consumer ## Лимиты - Последовательный рендеринг (было 3× parallel → OOM), `timeout 900`, `fps` из probe (≤30 preserve, >30 cap 30), `-threads 2 -preset veryfast -maxrate 1.10×` -- Pipe-режим: при ошибке стрима/таймауте ffmpeg SIGKILL-ится и реапится (`_kill_and_reap`, issue #51) — без зомби +- `subprocess.run(timeout=900)` на таймауте сам убивает и дожидается ffmpeg — зомби не остаётся (issue #51; pipe-режим с ручным Popen удалён в #62) - Heartbeat: `_HeartbeatKeeper` (daemon-поток, `process_data_events` каждые 30с) держит pika-соединение живым во время длинных транскодов (issue #51) - `shutil.disk_usage` чек, `SIGTERM` graceful, `deploy/docker-compose.yml:179` + `prod.yml:44` limits `cpus 2 / mem 4G` -- `.env.example:30` `FFMPEG_THREADS`/`FFMPEG_PRESET`, `make dev-transcoder` теперь `python -m app.consumer` (celery legacy `make dev-transcoder-celery`) +- `.env.example:30` `FFMPEG_THREADS`/`FFMPEG_PRESET`, `make dev-transcoder` = `python -m app.consumer` ## Zed IDE Аналогично `services/auth/README.md` — `.zed/settings.json:1` → Python `format_on_save` + `ruff` импорты. diff --git a/services/transcoder/app/celery_app.py b/services/transcoder/app/celery_app.py deleted file mode 100644 index debf12c..0000000 --- a/services/transcoder/app/celery_app.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -from celery import Celery # type: ignore[import-untyped] - -broker_url = os.getenv("RABBITMQ_URL", "amqp://flowix:flowix@localhost:5672//") -result_backend = os.getenv("CELERY_RESULT_BACKEND", "rpc://") - -app = Celery("transcoder", broker=broker_url, backend=result_backend) -celery = app # alias for `celery -A app.celery_app` (expects `celery` attr) - -app.conf.update( - task_serializer="json", - accept_content=["json"], - result_serializer="json", - timezone="UTC", - enable_utc=True, - task_track_started=True, - worker_prefetch_multiplier=1, -) - -# Import tasks so Celery discovers them (stub for Phase 4) -try: - import app.tasks # noqa: F401 -except ImportError: - pass diff --git a/services/transcoder/app/consumer.py b/services/transcoder/app/consumer.py index 40444a5..ad37f22 100644 --- a/services/transcoder/app/consumer.py +++ b/services/transcoder/app/consumer.py @@ -53,7 +53,6 @@ FFMPEG_PRESET = os.getenv("FFMPEG_PRESET", "veryfast") ENCODE_MODE = os.getenv("ENCODE_MODE", "cbr") # cbr (default, compat) or crf FFMPEG_HWACCEL = os.getenv("FFMPEG_HWACCEL", "auto") # auto | nvenc | none -TRANSCODE_PIPE = os.getenv("TRANSCODER_PIPE_INPUT", "true").lower() == "true" # Phase 12 HW/thumbnail tuning THUMBNAIL_FROM_RENDITION = True @@ -63,9 +62,6 @@ ("720p", 1280, 720, 2500, 23), ("1080p", 1920, 1080, 5000, 23), ] -# fan-out per-rendition queues — Phase 12: 3 workers × prefetch 1 -FANOUT_QUEUES = ["video.transcode.360p", "video.transcode.720p", "video.transcode.1080p"] -FANOUT_RQ_MAP = {"360p": FANOUT_QUEUES[0], "720p": FANOUT_QUEUES[1], "1080p": FANOUT_QUEUES[2]} # One shared audio bitrate — see encode_audio() for why it must not vary per rendition. AUDIO_BITRATE = "128k" @@ -293,202 +289,31 @@ def encode_audio(input_path: str, output_path: str): subprocess.run(cmd, check=True, capture_output=True, timeout=300) -def transcode_one( +def build_ffmpeg_argv( input_path: str, audio_path: str | None, output_path: str, - width: int, - height: int, - bitrate_k: int, - fps: int = 30, - crf: int = 23, -): - """Single rendition with aligned GOP for JIT HLS (phase 4 spec, Phase 10 limits + Phase 12 CRF/HW).""" - vf = f"scale=-2:{height}:flags=lanczos" - codec_args, enc = _get_video_codec_and_extra(fps) - # input handling — pipe:0 is used for streaming without /tmp (Phase 12) - if input_path == "pipe:0": - cmd = ["ffmpeg", "-y", "-i", "pipe:0"] - else: - # for file input keep threads via codec_args (threads already in codec_args for libx264) - cmd = ["ffmpeg", "-y", "-i", input_path] - # prepend threads for file path is inside codec_args; for pipe we ignore threads (nvenc doesn't use) - if enc == "libx264" and input_path != "pipe:0": - # codec_args already contains threads/preset for libx264 file path - pass - if audio_path: - cmd += ["-i", audio_path, "-map", "0:v:0", "-map", "1:a:0"] - else: - # when using pipe:0 video is stream 0, no audio mapping needed - if input_path == "pipe:0": - cmd += ["-an"] - else: - cmd += ["-an"] - # video codec args - if enc == "libx264": - # codec_args for libx264 already includes -threads etc., but for pipe:0 we need to inject them before -i which we already handled - # Re-build to ensure correct order: ffmpeg -y [-threads X] -i pipe:0 ... - if input_path == "pipe:0": - # inject threads before input if libx264 - threads = ( - FFMPEG_THREADS - if FFMPEG_THREADS.isdigit() and 1 <= int(FFMPEG_THREADS) <= 8 - else "2" - ) - # rebuild cmd with threads before pipe input - base = ["ffmpeg", "-y", "-threads", threads, "-i", "pipe:0"] - if audio_path: - base += ["-i", audio_path, "-map", "0:v:0", "-map", "1:a:0"] - else: - base += ["-an"] - cmd = base - else: - cmd += ["-c:v"] + codec_args[:1] # libx264 - # append remaining codec_args after c:v (preset/threads) - # codec_args is ["libx264","-preset",preset,"-threads",threads] -> split - for a in codec_args[1:]: - cmd.append(a) - cmd += [ - "-profile:v", - "high", - "-pix_fmt", - "yuv420p", - "-r", - str(fps), - "-g", - str(fps * 2), - "-keyint_min", - str(fps * 2), - "-sc_threshold", - "0", - "-force_key_frames", - "expr:gte(t,n_forced*2)", - "-vf", - vf, - ] - if ENCODE_MODE == "crf": - cmd += [ - "-crf", - str(crf), - "-maxrate", - f"{int(bitrate_k * 1.10)}k", - "-bufsize", - f"{bitrate_k * 2}k", - ] - else: - cmd += [ - "-b:v", - f"{bitrate_k}k", - "-maxrate", - f"{int(bitrate_k * 1.10)}k", - "-bufsize", - f"{bitrate_k * 2}k", - ] - else: - # nvenc path - cmd += ["-c:v", "h264_nvenc", "-preset", "p4", "-rc", "vbr", "-cq", str(crf)] - cmd += [ - "-profile:v", - "high", - "-pix_fmt", - "yuv420p", - "-r", - str(fps), - "-g", - str(fps * 2), - "-keyint_min", - str(fps * 2), - "-sc_threshold", - "0", - "-force_key_frames", - "expr:gte(t,n_forced*2)", - "-vf", - vf, - "-b:v", - f"{bitrate_k}k", - "-maxrate", - f"{int(bitrate_k * 1.10)}k", - "-bufsize", - f"{bitrate_k * 2}k", - ] - if audio_path: - cmd += ["-c:a", "copy"] - cmd += ["-movflags", "+faststart", output_path] - log.info( - "ffmpeg %dx%d %dk crf=%d %dfps enc=%s mode=%s: %s", - width, - height, - bitrate_k, - crf, - fps, - enc, - ENCODE_MODE, - " ".join(cmd), - ) - start = time.time() - subprocess.run(cmd, check=True, capture_output=True, timeout=900) - if METRICS_AVAILABLE and ffmpeg_duration_metric is not None: - try: - ffmpeg_duration_metric.labels(quality=f"{height}p").observe(time.time() - start) - except Exception: - pass - - -def _kill_and_reap(proc: subprocess.Popen) -> None: - """SIGKILL an ffmpeg subprocess and reap it so it cannot linger as a zombie (issue #51). - - Safe to call on an already-exited process (skips kill, still waits to reap). - """ - try: - if proc.poll() is None: - proc.kill() - proc.wait(timeout=10) - except Exception as e: - log.error("failed to kill/reap ffmpeg pid=%s: %s", proc.pid, e) - - -def transcode_one_pipe( - get_object_stream, - audio_path: str | None, - output_path: str, - width: int, height: int, bitrate_k: int, fps: int = 30, crf: int = 23, -): - """Phase 12: stream MinIO object via pipe:0 to avoid /tmp disk usage. +) -> list[str]: + """Единая сборка ffmpeg argv для одной рендишны (issue #62). - get_object_stream should be a file-like object with read() or an iterable of bytes. - We feed it to ffmpeg stdin via Popen. + Раньше file- и pipe-пути дублировали сборку (~60%) и разъехались: pipe + игнорировал ENCODE_MODE=crf при nvenc. Все рендишны обязаны получить + идентичный GOP/таймлайн сегментов (fps*2, keyint_min, sc_threshold=0, + force_key_frames каждые 2s) — менять аргументы можно только для всех сразу. """ vf = f"scale=-2:{height}:flags=lanczos" codec_args, enc = _get_video_codec_and_extra(fps) - # Build cmd similar to transcode_one with pipe:0 - if enc == "libx264": - threads = ( - FFMPEG_THREADS if FFMPEG_THREADS.isdigit() and 1 <= int(FFMPEG_THREADS) <= 8 else "2" - ) - cmd = ["ffmpeg", "-y", "-threads", threads, "-i", "pipe:0"] - if audio_path: - cmd += ["-i", audio_path, "-map", "0:v:0", "-map", "1:a:0"] - else: - cmd += ["-an"] - cmd += ["-c:v", "libx264"] - preset = ( - FFMPEG_PRESET - if FFMPEG_PRESET - in ("ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow") - else "veryfast" - ) - cmd += ["-preset", preset] + cmd = ["ffmpeg", "-y", "-i", input_path] + if audio_path: + cmd += ["-i", audio_path, "-map", "0:v:0", "-map", "1:a:0"] else: - cmd = ["ffmpeg", "-y", "-i", "pipe:0"] - if audio_path: - cmd += ["-i", audio_path, "-map", "0:v:0", "-map", "1:a:0"] - else: - cmd += ["-an"] - cmd += ["-c:v", "h264_nvenc", "-preset", "p4", "-rc", "vbr", "-cq", str(crf)] + cmd += ["-an"] + # libx264: [codec, -preset, -threads]; nvenc: [codec, -preset, -rc, -cq] + cmd += ["-c:v"] + codec_args cmd += [ "-profile:v", "high", @@ -528,72 +353,42 @@ def transcode_one_pipe( if audio_path: cmd += ["-c:a", "copy"] cmd += ["-movflags", "+faststart", output_path] - log.info("ffmpeg pipe %dx%d %dk enc=%s: %s", width, height, bitrate_k, enc, " ".join(cmd)) - start = time.time() - proc = subprocess.Popen( - cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + return cmd + + +def transcode_one( + input_path: str, + audio_path: str | None, + output_path: str, + width: int, + height: int, + bitrate_k: int, + fps: int = 30, + crf: int = 23, +): + """Single rendition with aligned GOP for JIT HLS (phase 4 spec, Phase 10 limits + Phase 12 CRF/HW). + + subprocess.run на таймауте сам убивает и дожидается процесс — зомби не + остаётся (issue #51); pipe-вариант с ручным Popen удалён (issue #62): + он дублировал сборку argv и всё равно скачивал файл для probe/audio. + """ + cmd = build_ffmpeg_argv( + input_path, audio_path, output_path, height, bitrate_k, fps=fps, crf=crf ) - stdout: bytes | None = None - stderr: bytes | None = None - try: - try: - # stream MinIO object to ffmpeg stdin in chunks - chunk_size = 256 * 1024 - # get_object_stream may be response object with stream() or read() - if hasattr(get_object_stream, "stream"): - # minio get_object returns HTTPResponse with stream() - assert proc.stdin is not None - for chunk in get_object_stream.stream(chunk_size): - if chunk: - proc.stdin.write(chunk) - proc.stdin.close() - elif hasattr(get_object_stream, "read"): - assert proc.stdin is not None - while True: - chunk = get_object_stream.read(chunk_size) - if not chunk: - break - proc.stdin.write(chunk) - proc.stdin.close() - else: - # iterable - assert proc.stdin is not None - for chunk in get_object_stream: - proc.stdin.write(chunk) - proc.stdin.close() - stdout, stderr = proc.communicate(timeout=900) - except subprocess.TimeoutExpired: - # issue #51: ffmpeg outlived its budget — kill it, collect output, re-raise - log.warning("ffmpeg pipe timed out after 900s, killing pid=%s", proc.pid) - proc.kill() - stdout, stderr = proc.communicate() - raise subprocess.TimeoutExpired(cmd, 900, output=stdout, stderr=stderr) - except Exception: - # issue #51: stream error mid-feed (or communicate failure) — never leave ffmpeg running - _kill_and_reap(proc) - raise - if proc.returncode != 0: - raise subprocess.CalledProcessError(proc.returncode, cmd, output=stdout, stderr=stderr) - if METRICS_AVAILABLE and ffmpeg_duration_metric is not None: - try: - ffmpeg_duration_metric.labels(quality=f"{height}p").observe(time.time() - start) - except Exception: - pass - finally: - # belt-and-braces (issue #51): reap anything that survived the paths above — no zombies - if proc.poll() is None: - _kill_and_reap(proc) - try: - if proc.stdin and not proc.stdin.closed: - proc.stdin.close() - except Exception: - pass - # ensure stream released + log.info( + "ffmpeg %dx%d %dk fps=%d enc-mode=%s: %s", + width, + height, + bitrate_k, + fps, + ENCODE_MODE, + " ".join(cmd), + ) + start = time.time() + subprocess.run(cmd, check=True, capture_output=True, timeout=900) + if METRICS_AVAILABLE and ffmpeg_duration_metric is not None: try: - if hasattr(get_object_stream, "close"): - get_object_stream.close() - elif hasattr(get_object_stream, "release_conn"): - get_object_stream.release_conn() + ffmpeg_duration_metric.labels(quality=f"{height}p").observe(time.time() - start) except Exception: pass @@ -633,7 +428,7 @@ def transcode_thumbnail_from_rendition(rendition_path: str, output_path: str): def declare_topology(channel): - """Declare DLX + DLQ + retry queue + main queue + fan-out rendition queues (idempotent). Phase 10b+12.""" + """Declare DLX + DLQ + retry queue + main queue (idempotent). Phase 10b+12.""" channel.exchange_declare(exchange=DLX_EXCHANGE, exchange_type="direct", durable=True) channel.queue_declare(queue=DLQ, durable=True) try: @@ -671,23 +466,6 @@ def declare_topology(channel): except Exception: pass raise - # Phase 12 fan-out queues: one per rendition for parallel workers (prefetch 1 each) - for fq in FANOUT_QUEUES: - try: - channel.queue_declare( - queue=fq, - durable=True, - arguments={ - "x-dead-letter-exchange": DLX_EXCHANGE, - "x-dead-letter-routing-key": DLQ, - }, - ) - except Exception as e: - msg = str(e) - if "PRECONDITION" in msg or "inequivalent" in msg: - log.warning("fanout queue %s args mismatch: %s", fq, e) - raise - log.warning("fanout queue declare %s failed: %s", fq, e) def process_message(body: bytes): @@ -717,19 +495,12 @@ def process_message(body: bytes): mc = get_minio() # ensure object exists (stat will raise if missing) stat = mc.stat_object(BUCKET, s3_key) - # detect large file for ultrafast preset — handle MagicMock in tests file_size = 0 try: sz = getattr(stat, "size", 0) - # MagicMock (tests) should not trigger pipe path - if sz is not None and not str(type(sz)).endswith("MagicMock'>"): - if isinstance(sz, int): - file_size = sz - else: - try: - file_size = int(sz) - except Exception: - file_size = 0 + # не-int (отсутствует или mock в тестах) → 0, реальный размер возьмём с файла + if isinstance(sz, int) and not isinstance(sz, bool): + file_size = sz except Exception: file_size = 0 @@ -745,23 +516,19 @@ def process_message(body: bytes): raise log.warning("disk_usage check failed: %s", e) - # Download raw for probe (need file for ffprobe). For large files we keep file but stream for transcode if enabled. + # Raw нужен на диске для ffprobe + общего аудиотрека и переиспользуется + # всеми рендишнами (issue #62: pipe-стриминг удалён — он качал файл всё + # равно и добавлял по сетевому чтению на каждую рендишну). raw_path = os.path.join(tmp, "original.mp4") - use_pipe = TRANSCODE_PIPE and hasattr(mc, "get_object") - # Download only if not using pipe for probe fallback; if pipe enabled we still need file for probe + audio - log.info("downloading s3://%s/%s -> %s (pipe=%s)", BUCKET, s3_key, raw_path, use_pipe) - try: - mc.fget_object(BUCKET, s3_key, raw_path) - except AttributeError: - log.warning("fget_object not available, falling back to copy test stub") - raise + log.info("downloading s3://%s/%s -> %s", BUCKET, s3_key, raw_path) + mc.fget_object(BUCKET, s3_key, raw_path) probe = probe_video(raw_path) fps = _fps_from_probe(probe) # Phase 12 large-file preset override global _current_file_size _current_file_size = file_size - # also check raw_path size if stat was 0 (e.g., MagicMock in tests -> use file size) + # stat может не содержать размер — берём фактический размер скачанного файла try: if file_size == 0 and os.path.exists(raw_path): _current_file_size = os.path.getsize(raw_path) @@ -793,27 +560,11 @@ def process_message(body: bytes): log.warning("no usable audio track (%s), renditions will be video-only", e) # Phase 12: sequential transcode 720p first for faster HLS, adaptive ladder - # If pipe streaming enabled, close raw file after audio extraction and stream per rendition renditions: list[dict] = [] renditions_so_far: list[dict] = [] for q, w, h, br, crf in ordered: - log.info("transcoding %s sequentially (fps=%d crf=%d pipe=%s)", q, fps, crf, use_pipe) - if use_pipe and file_size > 0: - # Phase 12 streaming without /tmp: stream raw from MinIO per rendition via pipe:0 - # Need fresh stream per rendition (MinIO get_object is not reusable) - try: - stream = mc.get_object(BUCKET, s3_key) - tmp_out = outputs[q] - # For pipe we need audio separately — transcode_one_pipe handles stdin streaming - # We reuse transcode_one_pipe with file audio if exists - transcode_one_pipe(stream, audio_path, tmp_out, w, h, br, fps=fps, crf=crf) - except Exception as e: - log.warning( - "pipe transcode failed for %s (%s), falling back to file input", q, e - ) - transcode_one(raw_path, audio_path, outputs[q], w, h, br, fps=fps, crf=crf) - else: - transcode_one(raw_path, audio_path, outputs[q], w, h, br, fps=fps, crf=crf) + log.info("transcoding %s sequentially (fps=%d crf=%d)", q, fps, crf) + transcode_one(raw_path, audio_path, outputs[q], w, h, br, fps=fps, crf=crf) # upload rendition immediately and do incremental status update (720p first → HLS available sooner) rk = f"renditions/{video_id}/{q}.mp4" diff --git a/services/transcoder/app/tasks.py b/services/transcoder/app/tasks.py deleted file mode 100644 index f9f30c8..0000000 --- a/services/transcoder/app/tasks.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging - -from .celery_app import app as celery_app - -log = logging.getLogger(__name__) - - -@celery_app.task(name="transcoder.ping") # type: ignore[attr-defined] -def ping() -> str: - log.info("transcoder ping") - return "pong" - - -@celery_app.task(name="transcoder.transcode") # type: ignore[attr-defined] -def transcode(video_id: str, s3_key: str) -> dict: - """Stub for Phase 4 — real FFmpeg logic will be here (aligned segments).""" - log.info("transcode stub: video_id=%s s3_key=%s", video_id, s3_key) - return {"video_id": video_id, "status": "stub"} diff --git a/services/transcoder/pyproject.toml b/services/transcoder/pyproject.toml index 978523e..a29ee78 100644 --- a/services/transcoder/pyproject.toml +++ b/services/transcoder/pyproject.toml @@ -1,16 +1,11 @@ [project] name = "flowix-transcoder" version = "0.1.0" -description = "Flowix Transcoding Worker — Celery + FFmpeg" +description = "Flowix Transcoding Worker — pika + FFmpeg" requires-python = ">=3.14" dependencies = [ - # celery+redis kept as legacy for make dev-transcoder-celery, will be removed in Phase 10b (pika is primary) - "celery>=5.4,<6", - "boto3>=1.35,<2", + # issue #62: celery/redis/boto3/pydantic удалены — воркер на pika, стабы не используются "minio>=7.2,<8", - "pydantic>=2.9,<3", - "pydantic-settings>=2.6,<3", - "redis>=5", "pika>=1.3", "requests>=2.32", "prometheus_client>=0.21", diff --git a/services/transcoder/tests/test_consumer.py b/services/transcoder/tests/test_consumer.py index a8d97ed..816432f 100644 --- a/services/transcoder/tests/test_consumer.py +++ b/services/transcoder/tests/test_consumer.py @@ -1,5 +1,4 @@ import json -import subprocess import time from unittest.mock import MagicMock, patch @@ -242,14 +241,12 @@ def test_declare_topology(): mock_ch.exchange_declare.assert_called_once_with( exchange=cons.DLX_EXCHANGE, exchange_type="direct", durable=True ) - # dlq, retry, main + 3 fan-out queues (Phase 12) - assert mock_ch.queue_declare.call_count == 6 + # issue #62: dlq, retry, main — fan-out очереди удалены (никто не потреблял) + assert mock_ch.queue_declare.call_count == 3 calls = [c[1].get("queue") for c in mock_ch.queue_declare.call_args_list] assert cons.DLQ in calls assert cons.RETRY_QUEUE in calls assert cons.QUEUE in calls - for fq in cons.FANOUT_QUEUES: - assert fq in calls # retry queue has TTL retry_call = [ c for c in mock_ch.queue_declare.call_args_list if c[1].get("queue") == cons.RETRY_QUEUE @@ -473,105 +470,45 @@ def test_handle_message_moves_to_dlq_after_max_retries(): ch.basic_publish.assert_not_called() -class _FakeProc: - """Minimal Popen stand-in with tracked kill/wait/communicate (issue #51). - - Behaves like a real process: poll() returns None while alive, kill() makes - it exit (SIGKILL), wait()/communicate() reap it. - """ - - def __init__(self, returncode: int = 0, communicate_raises: Exception | None = None): - self.stdin = MagicMock() - self.pid = 4242 - self.returncode: int | None = None - self._exit_code = returncode - self._communicate_raises = communicate_raises - self.kill_calls = 0 - self.wait_calls = 0 - self.communicate_calls = 0 - - def poll(self) -> int | None: - return self.returncode - - def kill(self) -> None: - self.kill_calls += 1 - self.returncode = -9 - - def wait(self, timeout: float | None = None) -> int: - self.wait_calls += 1 - assert self.returncode is not None, "wait() on a live process would block" - return self.returncode - - def communicate(self, timeout: float | None = None) -> tuple[bytes, bytes]: - self.communicate_calls += 1 - if self._communicate_raises is not None and self.returncode is None: - raise self._communicate_raises - if self.returncode is None: - self.returncode = self._exit_code - return b"out", b"err" - - -class _BrokenStream: - """MinIO-like stream that dies mid-feed with an error.""" - - def __init__(self): - self.closed = False - - def stream(self, chunk_size: int): - yield b"chunk" - raise OSError("stream broken") - - def close(self) -> None: - self.closed = True - - -def test_transcode_one_pipe_success_does_not_kill_ffmpeg(): - proc = _FakeProc(returncode=0) - stream = MagicMock() - with patch("app.consumer.subprocess.Popen", return_value=proc): - cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) - assert proc.kill_calls == 0 - assert proc.wait_calls == 0 - proc.stdin.close.assert_called_once() - stream.close.assert_called_once() - - -def test_transcode_one_pipe_stream_error_kills_and_reaps_ffmpeg(): - # issue #51 verify: stream failure mid-feed -> ffmpeg killed AND waited/reaped, no zombie - proc = _FakeProc() - stream = _BrokenStream() - with patch("app.consumer.subprocess.Popen", return_value=proc): - with pytest.raises(OSError, match="stream broken"): - cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) - assert proc.kill_calls == 1 - assert proc.wait_calls == 1 - assert proc.returncode == -9 # reaped - assert stream.closed - - -def test_transcode_one_pipe_timeout_kills_and_reaps_ffmpeg(): - # issue #51: communicate(timeout=900) fires -> kill, collect output via - # communicate() again (reaps), and propagate TimeoutExpired - proc = _FakeProc(communicate_raises=subprocess.TimeoutExpired("ffmpeg", 900)) - stream = MagicMock() - with patch("app.consumer.subprocess.Popen", return_value=proc): - with pytest.raises(subprocess.TimeoutExpired): - cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) - assert proc.kill_calls == 1 - assert proc.communicate_calls == 2 # retry after kill to drain output and reap - assert proc.returncode == -9 - - -def test_transcode_one_pipe_nonzero_exit_raises_and_skips_kill(): - # ffmpeg already exited on its own -> only CalledProcessError, no kill needed - proc = _FakeProc(returncode=1) - stream = MagicMock() - with patch("app.consumer.subprocess.Popen", return_value=proc): - with pytest.raises(subprocess.CalledProcessError): - cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) - assert proc.kill_calls == 0 - assert proc.wait_calls == 0 - stream.close.assert_called_once() +# Issue #62: единый build_ffmpeg_argv — одна сборка argv для всех рендишн. +def test_build_ffmpeg_argv_cbr_vs_crf(monkeypatch): + monkeypatch.setattr(cons, "_nvenc_checked", True) + monkeypatch.setattr(cons, "_nvenc_available", False) # libx264 path + monkeypatch.setattr(cons, "ENCODE_MODE", "cbr") + cbr = cons.build_ffmpeg_argv("/tmp/in.mp4", None, "/tmp/out.mp4", 720, 2500) + assert "-b:v" in cbr and cbr[cbr.index("-b:v") + 1] == "2500k" + assert "-crf" not in cbr + monkeypatch.setattr(cons, "ENCODE_MODE", "crf") + crf_cmd = cons.build_ffmpeg_argv("/tmp/in.mp4", None, "/tmp/out.mp4", 720, 2500, crf=23) + assert "-crf" in crf_cmd and crf_cmd[crf_cmd.index("-crf") + 1] == "23" + assert "-b:v" not in crf_cmd + + +def test_build_ffmpeg_argv_identical_gop_for_all_renditions(monkeypatch): + # aligned segments: GOP/сегментные аргументы не зависят от качества рендишны + monkeypatch.setattr(cons, "_nvenc_checked", True) + monkeypatch.setattr(cons, "_nvenc_available", False) + cmds = [ + cons.build_ffmpeg_argv("/tmp/in.mp4", None, f"/tmp/{h}.mp4", h, br) + for _, _, h, br, _ in cons.RENDITIONS_SPEC + ] + for cmd in cmds: + assert cmd[cmd.index("-g") + 1] == "60" + assert cmd[cmd.index("-keyint_min") + 1] == "60" + assert "-force_key_frames" in cmd and "expr:gte(t,n_forced*2)" in cmd + assert "-sc_threshold" in cmd and cmd[cmd.index("-sc_threshold") + 1] == "0" + + +def test_build_ffmpeg_argv_audio_copy_or_video_only(monkeypatch): + monkeypatch.setattr(cons, "_nvenc_checked", True) + monkeypatch.setattr(cons, "_nvenc_available", False) + with_audio = cons.build_ffmpeg_argv( + "/tmp/in.mp4", "/tmp/audio.m4a", "/tmp/out.mp4", 720, 2500 + ) + assert with_audio[with_audio.index("-c:a") + 1] == "copy" + assert "-map" in with_audio and "1:a:0" in with_audio + without_audio = cons.build_ffmpeg_argv("/tmp/in.mp4", None, "/tmp/out.mp4", 720, 2500) + assert "-an" in without_audio and "-c:a" not in without_audio def test_heartbeat_keeper_pokes_connection_periodically(): diff --git a/services/transcoder/uv.lock b/services/transcoder/uv.lock index 18d7e1d..4f3f655 100644 --- a/services/transcoder/uv.lock +++ b/services/transcoder/uv.lock @@ -6,27 +6,6 @@ resolution-markers = [ "python_full_version < '3.15'", ] -[[package]] -name = "amqp" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, -] - [[package]] name = "argon2-cffi" version = "25.1.0" @@ -145,15 +124,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/25/6c/b400476d3ceba681ab929787edc9554f6d88fcc69435eb681b00fc0457a5/ast_serialize-0.8.0-cp39-abi3-win_arm64.whl", hash = "sha256:b2a5978662fd4db463dfb4b974d2b10ac6430b98f5333aabc7051909df3561d0", size = 1083655, upload-time = "2026-08-07T11:29:00.349Z" }, ] -[[package]] -name = "billiard" -version = "4.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, -] - [[package]] name = "black" version = "26.5.1" @@ -176,54 +146,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, ] -[[package]] -name = "boto3" -version = "1.43.78" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/be/55/e026c943f7f1ed6d2f5e6035713f21233bbe9ee975008662dc64ca0d4ced/boto3-1.43.78.tar.gz", hash = "sha256:2fa59116e298171ef59e7600a8be6c01177faef8af4b9a4314b7a57a04009ada", size = 112679, upload-time = "2026-08-21T19:37:36.84Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/55/288b1f0d987e15db0417f5ea43da564bfa042063f508c259eec893134628/boto3-1.43.78-py3-none-any.whl", hash = "sha256:893f06a171469618e17de78dc927aca6e74fcf45a70d2c5918e9ac9919e96cc9", size = 140028, upload-time = "2026-08-21T19:37:35.358Z" }, -] - -[[package]] -name = "botocore" -version = "1.43.78" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/67/71/490aaa384855bf3b69405ded52ae77a0e5f4eeb2165044fa65466c4d3a73/botocore-1.43.78.tar.gz", hash = "sha256:e8238d22c1e1342025d75d2e33d154a375e7caad0fc67f77d77faa2d82668b94", size = 15982895, upload-time = "2026-08-21T19:37:32.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/5b/b7c5c22767c0e8eb4bb2bd0a972d77bd0be3dd9908e15d4487094a31fa4b/botocore-1.43.78-py3-none-any.whl", hash = "sha256:ddd020493235e264b3bd12606f239a3d3b2dd7cfb1d25a0691061183c290c228", size = 15677214, upload-time = "2026-08-21T19:37:28.894Z" }, -] - -[[package]] -name = "celery" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "billiard" }, - { name = "click" }, - { name = "click-didyoumean" }, - { name = "click-plugins" }, - { name = "click-repl" }, - { name = "kombu" }, - { name = "python-dateutil" }, - { name = "tzlocal" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, -] - [[package]] name = "certifi" version = "2026.7.22" @@ -398,43 +320,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "click-didyoumean" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, -] - -[[package]] -name = "click-plugins" -version = "1.1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, -] - -[[package]] -name = "click-repl" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "prompt-toolkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -463,14 +348,9 @@ name = "flowix-transcoder" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "boto3" }, - { name = "celery" }, { name = "minio" }, { name = "pika" }, { name = "prometheus-client" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "redis" }, { name = "requests" }, ] @@ -485,14 +365,9 @@ dev = [ [package.metadata] requires-dist = [ - { name = "boto3", specifier = ">=1.35,<2" }, - { name = "celery", specifier = ">=5.4,<6" }, { name = "minio", specifier = ">=7.2,<8" }, { name = "pika", specifier = ">=1.3" }, { name = "prometheus-client", specifier = ">=0.21" }, - { name = "pydantic", specifier = ">=2.9,<3" }, - { name = "pydantic-settings", specifier = ">=2.6,<3" }, - { name = "redis", specifier = ">=5" }, { name = "requests", specifier = ">=2.32" }, ] @@ -523,30 +398,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "jmespath" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, -] - -[[package]] -name = "kombu" -version = "5.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "packaging" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, -] - [[package]] name = "librt" version = "0.15.0" @@ -747,18 +598,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, ] -[[package]] -name = "prompt-toolkit" -version = "3.0.53" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, -] - [[package]] name = "pycodestyle" version = "2.14.0" @@ -796,76 +635,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, ] -[[package]] -name = "pydantic" -version = "2.13.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, - { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, - { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, -] - -[[package]] -name = "pydantic-settings" -version = "2.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, -] - [[package]] name = "pyflakes" version = "3.4.0" @@ -900,27 +669,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, -] - [[package]] name = "pytokens" version = "0.4.1" @@ -940,15 +688,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] -[[package]] -name = "redis" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, -] - [[package]] name = "requests" version = "2.34.2" @@ -989,27 +728,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, ] -[[package]] -name = "s3transfer" -version = "0.19.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - [[package]] name = "typing-extensions" version = "4.16.0" @@ -1019,39 +737,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] -[[package]] -name = "typing-inspection" -version = "0.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, -] - -[[package]] -name = "tzdata" -version = "2026.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, -] - -[[package]] -name = "tzlocal" -version = "5.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, -] - [[package]] name = "urllib3" version = "2.7.0" @@ -1060,21 +745,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] - -[[package]] -name = "vine" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, -] diff --git a/services/upload/internal/handler/ownership.go b/services/upload/internal/handler/ownership.go index 8458b0c..bd6b251 100644 --- a/services/upload/internal/handler/ownership.go +++ b/services/upload/internal/handler/ownership.go @@ -16,19 +16,20 @@ type OwnershipChecker interface { // requireOwnership verifies userID owns videoID. On failure it writes the // response (403 wrong owner, 404 unknown video, 502 metadata unavailable — -// fail closed) and returns false. -func requireOwnership(checker OwnershipChecker, videoID, userID string, w http.ResponseWriter) bool { +// fail closed) and returns false. Issue #59: ошибки через writeError (JSON, +// детали — только в лог). +func requireOwnership(checker OwnershipChecker, videoID, userID string, w http.ResponseWriter, r *http.Request) bool { owner, err := checker.GetVideoOwner(videoID) if err != nil { if strings.Contains(err.Error(), "404") { - http.Error(w, `{"error":"video not found"}`, http.StatusNotFound) + writeError(w, r, http.StatusNotFound, "video not found", err) return false } - http.Error(w, `{"error":"metadata unavailable"}`, http.StatusBadGateway) + writeError(w, r, http.StatusBadGateway, "metadata unavailable", err) return false } if owner != userID { - http.Error(w, `{"error":"forbidden"}`, http.StatusForbidden) + writeError(w, r, http.StatusForbidden, "forbidden", nil) return false } return true diff --git a/services/upload/internal/handler/presign.go b/services/upload/internal/handler/presign.go index 3bed9e8..9f08177 100644 --- a/services/upload/internal/handler/presign.go +++ b/services/upload/internal/handler/presign.go @@ -68,7 +68,7 @@ func (h *PresignHandler) Presign(w http.ResponseWriter, r *http.Request) { } var req presignRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, `{"error":"invalid body"}`, 400) + writeError(w, r, http.StatusBadRequest, "invalid body", nil) return } title := strings.TrimSpace(req.Title) @@ -80,13 +80,13 @@ func (h *PresignHandler) Presign(w http.ResponseWriter, r *http.Request) { } ct := strings.TrimSpace(req.ContentType) if ct != "" && !isAllowedContentType(ct) { - http.Error(w, `{"error":"unsupported content type: `+ct+`"}`, 400) + writeError(w, r, http.StatusBadRequest, "unsupported content type: "+ct, nil) return } // create metadata first videoID, err := h.metadata.CreateVideo(token, title, req.Description) if err != nil { - http.Error(w, `{"error":"metadata: `+err.Error()+`"}`, http.StatusBadGateway) + writeError(w, r, http.StatusBadGateway, "metadata unavailable", err) return } _ = ownerID // owner stored in metadata via token @@ -100,12 +100,10 @@ func (h *PresignHandler) Presign(w http.ResponseWriter, r *http.Request) { expires := time.Hour urlStr, err := h.storage.PresignedPutObjectExternal(r.Context(), s3Key, expires, publicEndpoint) if err != nil { - http.Error(w, `{"error":"presign: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "presign failed", err) return } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(201) - _ = json.NewEncoder(w).Encode(presignResponse{ + writeJSON(w, r, http.StatusCreated, presignResponse{ ID: videoID, VideoID: videoID, S3Key: s3Key, @@ -138,36 +136,35 @@ func (h *PresignHandler) Complete(w http.ResponseWriter, r *http.Request) { } } if videoID == "" { - http.Error(w, `{"error":"video_id required"}`, 400) + writeError(w, r, http.StatusBadRequest, "video_id required", nil) return } // IDOR (issue #46): only the owner may complete an upload for a video id. - if !requireOwnership(h.owner, videoID, ownerID, w) { + if !requireOwnership(h.owner, videoID, ownerID, w, r) { return } s3Key := fmt.Sprintf("raw/%s/original.mp4", videoID) // Verify the uploaded object matches a real video PUT (issue #46). size, contentType, err := h.storage.StatObjectInfo(r.Context(), s3Key) if err != nil { - http.Error(w, `{"error":"object not found, upload via presigned URL first"}`, 404) + writeError(w, r, http.StatusNotFound, "object not found, upload via presigned URL first", nil) return } if size == 0 { - http.Error(w, `{"error":"empty object"}`, 400) + writeError(w, r, http.StatusBadRequest, "empty object", nil) return } if contentType != "" && !strings.HasPrefix(contentType, "video/") { - http.Error(w, `{"error":"unexpected content type: `+contentType+`"}`, 400) + writeError(w, r, http.StatusBadRequest, "unexpected content type: "+contentType, nil) return } metrics.UploadBytes.Inc() ev := VideoUploadedEvent{VideoID: videoID, S3Key: s3Key, OwnerID: ownerID} if err := h.publisher.Publish(r.Context(), ev); err != nil { - http.Error(w, `{"error":"queue: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "queue error", err) return } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{ + writeJSON(w, r, http.StatusOK, map[string]interface{}{ "id": videoID, "s3_key": s3Key, "status": "uploaded", diff --git a/services/upload/internal/handler/response.go b/services/upload/internal/handler/response.go new file mode 100644 index 0000000..e0cab9a --- /dev/null +++ b/services/upload/internal/handler/response.go @@ -0,0 +1,24 @@ +package handler + +import ( + "log/slog" + "net/http" + + "flowix/pkg/httputil" +) + +// writeJSON — общий JSON-ответ через pkg/httputil (как в metadata, issue #63). +func writeJSON(w http.ResponseWriter, r *http.Request, status int, v any) { + httputil.WriteJSON(w, r, status, v) +} + +// writeError отдаёт клиенту безопасное сообщение через json.NewEncoder +// (issue #59: err.Error() не интерполируется в рукописный JSON — спецсимволы +// не ломают тело, детали инфры (endpoint'ы MinIO/metadata) не утекают). +// Детали ошибки — только в лог. +func writeError(w http.ResponseWriter, r *http.Request, status int, msg string, err error) { + if err != nil { + slog.Error("upload handler error", "error", err, "path", r.URL.Path, "status", status) + } + writeJSON(w, r, status, map[string]string{"error": msg}) +} diff --git a/services/upload/internal/handler/resumable.go b/services/upload/internal/handler/resumable.go index e099eaa..51a816a 100644 --- a/services/upload/internal/handler/resumable.go +++ b/services/upload/internal/handler/resumable.go @@ -3,7 +3,6 @@ package handler import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -52,11 +51,11 @@ type statusResponse struct { func (h *ResumableHandler) Status(w http.ResponseWriter, r *http.Request) { videoID := chi.URLParam(r, "id") if videoID == "" { - http.Error(w, `{"error":"video_id required"}`, 400) + writeError(w, r, http.StatusBadRequest, "video_id required", nil) return } // IDOR (issue #46): only the owner may read the upload offset. - if !requireOwnership(h.owner, videoID, mw.UserIDFromCtx(r.Context()), w) { + if !requireOwnership(h.owner, videoID, mw.UserIDFromCtx(r.Context()), w, r) { return } key := fmt.Sprintf("raw/%s/original.mp4", videoID) @@ -73,8 +72,7 @@ func (h *ResumableHandler) Status(w http.ResponseWriter, r *http.Request) { size = 0 } } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(statusResponse{Uploaded: size}) + writeJSON(w, r, http.StatusOK, statusResponse{Uploaded: size}) } // Content-Range format: "bytes start-end/total" or "bytes start-end/*" or "bytes */total" @@ -107,11 +105,11 @@ func parseContentRange(v string) (start, end, total int64, hasTotal bool, err er func (h *ResumableHandler) Upload(w http.ResponseWriter, r *http.Request) { videoID := chi.URLParam(r, "id") if videoID == "" { - http.Error(w, `{"error":"video_id required"}`, 400) + writeError(w, r, http.StatusBadRequest, "video_id required", nil) return } // IDOR (issue #46): only the owner may append/overwrite the raw object. - if !requireOwnership(h.owner, videoID, mw.UserIDFromCtx(r.Context()), w) { + if !requireOwnership(h.owner, videoID, mw.UserIDFromCtx(r.Context()), w, r) { return } maxBytes := int64(5 << 30) // 5GB @@ -134,27 +132,26 @@ func (h *ResumableHandler) Upload(w http.ResponseWriter, r *http.Request) { if err != nil { var mbe *http.MaxBytesError if errors.As(err, &mbe) { - http.Error(w, `{"error":"file too large (max `+strconv.FormatInt(maxBytes, 10)+` bytes)"}`, http.StatusRequestEntityTooLarge) + writeError(w, r, http.StatusRequestEntityTooLarge, "file too large (max "+strconv.FormatInt(maxBytes, 10)+" bytes)", err) return } - http.Error(w, `{"error":"read body"}`, 400) + writeError(w, r, http.StatusBadRequest, "read body", err) return } if err := h.storage.PutObject(r.Context(), key, bytes.NewReader(data), int64(len(data)), ct); err != nil { - http.Error(w, `{"error":"put: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "storage error", err) return } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok", "uploaded": strconv.Itoa(len(data))}) + writeJSON(w, r, http.StatusOK, map[string]string{"status": "ok", "uploaded": strconv.Itoa(len(data))}) return } start, end, total, hasTotal, err := parseContentRange(cr) if err != nil { - http.Error(w, `{"error":"`+err.Error()+`"}`, 400) + writeError(w, r, http.StatusBadRequest, "invalid Content-Range", err) return } if start < 0 || end < start { - http.Error(w, `{"error":"invalid range"}`, 400) + writeError(w, r, http.StatusBadRequest, "invalid range", nil) return } chunkSize := end - start + 1 @@ -170,7 +167,10 @@ func (h *ResumableHandler) Upload(w http.ResponseWriter, r *http.Request) { if existing == 0 { w.Header().Set("Range", "bytes=0-0") } - http.Error(w, fmt.Sprintf(`{"error":"range mismatch: expected start %d got %d","uploaded":%d}`, existing, start, existing), http.StatusRequestedRangeNotSatisfiable) + writeJSON(w, r, http.StatusRequestedRangeNotSatisfiable, map[string]interface{}{ + "error": fmt.Sprintf("range mismatch: expected start %d got %d", existing, start), + "uploaded": existing, + }) return } ct := r.Header.Get("Content-Type") @@ -178,21 +178,21 @@ func (h *ResumableHandler) Upload(w http.ResponseWriter, r *http.Request) { ct = "video/mp4" } if existing+chunkSize > maxBytes { - http.Error(w, `{"error":"file too large (max `+strconv.FormatInt(maxBytes, 10)+` bytes)"}`, http.StatusRequestEntityTooLarge) + writeError(w, r, http.StatusRequestEntityTooLarge, "file too large (max "+strconv.FormatInt(maxBytes, 10)+" bytes)", nil) return } chunk, err := io.ReadAll(io.LimitReader(r.Body, chunkSize+1)) if err != nil { var mbe *http.MaxBytesError if errors.As(err, &mbe) { - http.Error(w, `{"error":"file too large (max `+strconv.FormatInt(maxBytes, 10)+` bytes)"}`, http.StatusRequestEntityTooLarge) + writeError(w, r, http.StatusRequestEntityTooLarge, "file too large (max "+strconv.FormatInt(maxBytes, 10)+" bytes)", err) return } - http.Error(w, `{"error":"read chunk"}`, 400) + writeError(w, r, http.StatusBadRequest, "read chunk", err) return } if int64(len(chunk)) != chunkSize { - http.Error(w, fmt.Sprintf(`{"error":"chunk size mismatch: expected %d got %d"}`, chunkSize, len(chunk)), 400) + writeError(w, r, http.StatusBadRequest, fmt.Sprintf("chunk size mismatch: expected %d got %d", chunkSize, len(chunk)), nil) return } // Append: if existing==0 just put, else get existing + append @@ -202,39 +202,34 @@ func (h *ResumableHandler) Upload(w http.ResponseWriter, r *http.Request) { } else { rc, err := h.storage.GetObject(r.Context(), key) if err != nil { - http.Error(w, `{"error":"get existing: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "get existing failed", err) return } existingData, err := io.ReadAll(rc) _ = rc.Close() if err != nil { - http.Error(w, `{"error":"read existing"}`, 500) + writeError(w, r, http.StatusInternalServerError, "read existing failed", err) return } toPut = append(existingData, chunk...) } if err := h.storage.PutObject(r.Context(), key, bytes.NewReader(toPut), int64(len(toPut)), ct); err != nil { - http.Error(w, `{"error":"put: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "storage error", err) return } uploaded := int64(len(toPut)) if hasTotal && uploaded == total { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "complete", "uploaded": uploaded, "total": total}) + writeJSON(w, r, http.StatusOK, map[string]interface{}{"status": "complete", "uploaded": uploaded, "total": total}) return } if hasTotal && uploaded < total { w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", uploaded-1)) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(308) - _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "resume", "uploaded": uploaded, "total": total}) + writeJSON(w, r, http.StatusPermanentRedirect, map[string]interface{}{"status": "resume", "uploaded": uploaded, "total": total}) return } // no total -> 308 w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", uploaded-1)) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(308) - _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "resume", "uploaded": uploaded}) + writeJSON(w, r, http.StatusPermanentRedirect, map[string]interface{}{"status": "resume", "uploaded": uploaded}) } // Ensure interface compliance at compile time diff --git a/services/upload/internal/handler/upload.go b/services/upload/internal/handler/upload.go index 1864a84..943fc7a 100644 --- a/services/upload/internal/handler/upload.go +++ b/services/upload/internal/handler/upload.go @@ -2,7 +2,6 @@ package handler import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -81,22 +80,22 @@ func (h *UploadHandler) Upload(w http.ResponseWriter, r *http.Request) { if err := r.ParseMultipartForm(32 << 20); err != nil { var mbe *http.MaxBytesError if errors.As(err, &mbe) { - http.Error(w, `{"error":"file too large (max `+strconv.FormatInt(maxBytes, 10)+` bytes)"}`, http.StatusRequestEntityTooLarge) + writeError(w, r, http.StatusRequestEntityTooLarge, "file too large (max "+strconv.FormatInt(maxBytes, 10)+" bytes)", nil) return } - http.Error(w, `{"error":"parse form: `+err.Error()+`"}`, 400) + writeError(w, r, http.StatusBadRequest, "invalid form", err) return } file, header, err := r.FormFile("file") if err != nil { - http.Error(w, `{"error":"file required (field 'file')"}`, 400) + writeError(w, r, http.StatusBadRequest, "file required (field 'file')", nil) return } defer func() { _ = file.Close() }() // Minimal MIME validation — header can be spoofed, deep check via ffprobe is in transcoder if ct := header.Header.Get("Content-Type"); ct != "" && !isAllowedContentType(ct) { - http.Error(w, `{"error":"unsupported content type: `+ct+`"}`, 400) + writeError(w, r, http.StatusBadRequest, "unsupported content type: "+ct, nil) return } @@ -112,7 +111,7 @@ func (h *UploadHandler) Upload(w http.ResponseWriter, r *http.Request) { // 1. create metadata record first to get video_id videoID, err := h.metadata.CreateVideo(token, title, description) if err != nil { - http.Error(w, `{"error":"metadata: `+err.Error()+`"}`, http.StatusBadGateway) + writeError(w, r, http.StatusBadGateway, "metadata unavailable", err) return } @@ -126,10 +125,10 @@ func (h *UploadHandler) Upload(w http.ResponseWriter, r *http.Request) { if err := h.storage.PutObject(r.Context(), s3Key, file, header.Size, contentType); err != nil { // MaxBytesReader returns error on too large body if strings.Contains(err.Error(), "http: request body too large") { - http.Error(w, `{"error":"file too large"}`, http.StatusRequestEntityTooLarge) + writeError(w, r, http.StatusRequestEntityTooLarge, "file too large", err) return } - http.Error(w, `{"error":"storage: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "storage error", err) return } @@ -141,13 +140,11 @@ func (h *UploadHandler) Upload(w http.ResponseWriter, r *http.Request) { if err := h.publisher.Publish(r.Context(), ev); err != nil { // log but don't fail upload — transcoder will be triggered manually if needed // for MVP return 500 so client knows - http.Error(w, `{"error":"queue: `+err.Error()+`"}`, 500) + writeError(w, r, http.StatusInternalServerError, "queue error", err) return } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(201) - _ = json.NewEncoder(w).Encode(map[string]interface{}{ + writeJSON(w, r, http.StatusCreated, map[string]interface{}{ "id": videoID, "s3_key": s3Key, "status": "uploaded", diff --git a/services/upload/internal/handler/upload_test.go b/services/upload/internal/handler/upload_test.go index 8049731..bc56840 100644 --- a/services/upload/internal/handler/upload_test.go +++ b/services/upload/internal/handler/upload_test.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -197,3 +198,56 @@ type fakeErr string func (e fakeErr) Error() string { return string(e) } func errFake(s string) error { return fakeErr(s) } + +// Issue #59: ошибка с кавычками и деталями инфры не ломает JSON и не утекает +// в тело ответа — клиент получает только безопасное сообщение. +func TestUploadStorageErrorJSONSafe(t *testing.T) { + meta := &fakeMeta{id: "vid-1"} + leak := `minio put failed: "quoted" msg http://minio:9000/videos \backslash` + st := &fakeStorage{err: errFake(leak)} + pub := &fakePub{} + h := NewUploadHandler(st, pub, meta) + body, ctype := newMultipart("file", "test.mp4", "c", nil) + req := httptest.NewRequest("POST", "/api/v1/videos/upload", body) + req.Header.Set("Content-Type", ctype) + req.Header.Set("Authorization", "Bearer "+token("owner1")) + req = authContext(req, "owner1") + w := httptest.NewRecorder() + h.Upload(w, req) + if w.Code != 500 { + t.Fatalf("want 500 got %d %s", w.Code, w.Body.String()) + } + var out map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("response is not valid json: %v (body %s)", err, w.Body.String()) + } + if out["error"] != "storage error" { + t.Fatalf("want generic error message, got %q", out["error"]) + } + if strings.Contains(w.Body.String(), "minio") || strings.Contains(w.Body.String(), "quoted") { + t.Fatalf("internal details leaked to client: %s", w.Body.String()) + } +} + +// Issue #59: ошибки metadata тоже не содержат деталей upstream. +func TestUploadMetadataErrorJSONSafe(t *testing.T) { + meta := &fakeMeta{err: errFake(`metadata 502: http://metadata:8002 "oops"`)} + h := NewUploadHandler(&fakeStorage{}, &fakePub{}, meta) + body, ctype := newMultipart("file", "test.mp4", "c", nil) + req := httptest.NewRequest("POST", "/api/v1/videos/upload", body) + req.Header.Set("Content-Type", ctype) + req.Header.Set("Authorization", "Bearer "+token("owner1")) + req = authContext(req, "owner1") + w := httptest.NewRecorder() + h.Upload(w, req) + if w.Code != 502 { + t.Fatalf("want 502 got %d %s", w.Code, w.Body.String()) + } + var out map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("response is not valid json: %v (body %s)", err, w.Body.String()) + } + if strings.Contains(w.Body.String(), "metadata:8002") || strings.Contains(w.Body.String(), "oops") { + t.Fatalf("internal details leaked to client: %s", w.Body.String()) + } +}