diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a1c20bd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Build context для Go-сервисов = корень репо (issue #63: общий модуль pkg/). +# Оставляем только исходники трёх Go-сервисов и pkg — остальное в контекст +# не нужно (.git, node_modules, .env и т.д. в образ не должны попадать). +.git +.env +data/ +tmp/ +docs/ +deploy/ +frontend/ +scripts/ +tests/ +services/auth/ +services/transcoder/ +**/__pycache__/ +**/node_modules/ +**/.venv/ diff --git a/.env.example b/.env.example index c94943a..2e53491 100644 --- a/.env.example +++ b/.env.example @@ -59,10 +59,18 @@ RABBITMQ_MGMT_PORT=15672 MINIO_PUBLIC_ENDPOINT=http://localhost:9000 MINIO_API_CORS_ALLOW_ORIGIN=* -# Redis for auth rate-limit (backlog) — 5/min/IP on /auth/login +# Redis — rate-limit storage (auth slowapi + gateway fixed-window, issue #52) REDIS_URL=redis://redis:6379/0 REDIS_PORT=6379 +# Gateway (issue #52) +# CORS — явный allowlist origin через запятую; пусто = без CORS-заголовков (без wildcard) +CORS_ALLOWED_ORIGINS=http://localhost:3000 +# CIDR прокси, которым gateway доверяет X-Forwarded-For. Пусто = никому +# (gateway — edge, LB перед ним нет). Для auth эта же переменная задаёт доверие +# X-Real-IP от gateway: compose по умолчанию подставляет 172.16.0.0/12. +#TRUSTED_PROXY_CIDRS= + # App ports GATEWAY_PORT=8080 AUTH_PORT=8001 diff --git a/.golangci.strict.yaml b/.golangci.strict.yaml new file mode 100644 index 0000000..8f3d91e --- /dev/null +++ b/.golangci.strict.yaml @@ -0,0 +1,19 @@ +# Строгий набор под gopls в Zed (см. lsp.gopls в settings.json): +# staticcheck с полным набором проверок (включая ST*) + govet shadow. +# Используется в `make lint-report`, чтобы его вывод совпадал с диагностиками Zed. +version: "2" +linters: + default: none + enable: + - staticcheck + - govet + - errcheck + - ineffassign + - unused + settings: + staticcheck: + checks: + - all + govet: + enable: + - shadow diff --git a/Makefile b/Makefile index cbaf83a..be53735 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # .env — единственный в корне, compose явно указывает на него (--env-file), deploy/.env не нужен COMPOSE=docker compose --env-file .env -f deploy/docker-compose.yml -.PHONY: up down logs ps build lint fmt test e2e e2e-file swagger swagger-install sync-py migrate-up migrate-down migrate-create migrate-alembic-up metrics loki-logs grafana prometheus +.PHONY: up down logs ps build lint lint-report fmt test e2e e2e-file swagger swagger-install sync-py migrate-up migrate-down migrate-create migrate-alembic-up metrics loki-logs grafana prometheus up: $(COMPOSE) up --build -d @@ -41,27 +41,27 @@ build: $(COMPOSE) build # Go — локально если установлено, иначе через Docker (golang:1.27, golangci-lint) -# Каждый сервис — отдельный go.mod, поэтому линтим per-service +# Каждый модуль — отдельный go.mod (pkg + сервисы), поэтому линтим per-module lint-go: - @for svc in metadata gateway upload; do \ - echo "==> lint $$svc"; \ + @for dir in pkg services/metadata services/gateway services/upload; do \ + echo "==> lint $$dir"; \ if which golangci-lint >/dev/null 2>&1; then \ - (cd services/$$svc && golangci-lint run ./...) || exit 1; \ + (cd $$dir && golangci-lint run ./...) || exit 1; \ else \ - docker run --rm -v $(PWD):/app -w /app/services/$$svc golangci/golangci-lint:latest golangci-lint run ./... || exit 1; \ + docker run --rm -v $(PWD):/app -w /app/$$dir golangci/golangci-lint:latest golangci-lint run ./... || exit 1; \ fi; \ done fmt-go: - @which gofmt >/dev/null 2>&1 && gofmt -w services/ || docker run --rm -v $(PWD):/app -w /app golang:1.27-alpine gofmt -w ./services + @which gofmt >/dev/null 2>&1 && gofmt -w services/ pkg/ || docker run --rm -v $(PWD):/app -w /app golang:1.27-alpine gofmt -w ./services ./pkg test-go: - @for svc in metadata gateway upload; do \ - echo "==> test $$svc"; \ + @for dir in pkg services/metadata services/gateway services/upload; do \ + echo "==> test $$dir"; \ if which go >/dev/null 2>&1; then \ - (cd services/$$svc && go test ./...) || exit 1; \ + (cd $$dir && go test ./...) || exit 1; \ else \ - docker run --rm -v $(PWD):/app -w /app/services/$$svc golang:1.27-alpine go test ./... || exit 1; \ + docker run --rm -v $(PWD):/app -w /app/$$dir golang:1.27-alpine go test ./... || exit 1; \ fi; \ done @@ -119,6 +119,29 @@ dev-frontend: lint-front: cd frontend && npm run lint +# Ошибки и варнинги всех проектов одним списком (file:line: message) — удобно копировать целиком. +# Не падает на первой ошибке; полный отчёт дублируется в /tmp/flowix-diagnostics.txt +lint-report: + @: > /tmp/flowix-diagnostics.txt + @ROOT=$$(pwd); \ + for dir in pkg services/metadata services/gateway services/upload; do \ + echo "==> go: $$dir" | tee -a /tmp/flowix-diagnostics.txt; \ + if which golangci-lint >/dev/null 2>&1; then \ + (cd $$dir && golangci-lint run -c $$ROOT/.golangci.strict.yaml ./...) 2>&1 | tee -a /tmp/flowix-diagnostics.txt; \ + else \ + echo "golangci-lint not installed, skip go ($$dir)" | tee -a /tmp/flowix-diagnostics.txt; \ + fi; \ + done + @echo "==> py: auth (flake8 + mypy)" | tee -a /tmp/flowix-diagnostics.txt + @uv run --project services/auth flake8 services/auth/src 2>&1 | tee -a /tmp/flowix-diagnostics.txt + @uv run --project services/auth mypy services/auth/src 2>&1 | tee -a /tmp/flowix-diagnostics.txt + @echo "==> py: transcoder (flake8 + mypy)" | tee -a /tmp/flowix-diagnostics.txt + @uv run --project services/transcoder flake8 services/transcoder/app 2>&1 | tee -a /tmp/flowix-diagnostics.txt + @uv run --project services/transcoder mypy services/transcoder/app 2>&1 | tee -a /tmp/flowix-diagnostics.txt + @echo "==> frontend (next lint)" | tee -a /tmp/flowix-diagnostics.txt + @(cd frontend && npm run lint) 2>&1 | tee -a /tmp/flowix-diagnostics.txt + @echo "report saved: /tmp/flowix-diagnostics.txt" + fmt-front: cd frontend && npm run format || npx prettier --write . diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 24524c3..66aaa5c 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -160,8 +160,13 @@ services: MINIO_ENDPOINT: minio:9000 MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + # X-Real-IP от gateway (внутри docker-сети) — ключ auth-лимитера по + # реальному клиенту; см. services/auth/src/core/limiter.py + TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-172.16.0.0/12} ports: - - "${AUTH_PORT:-8001}:8001" + # ревью фазы 17 (H1): auth не для внешнего доступа — клиенты идут через + # gateway (rate-limit + CORS); порт только для localhost (e2e/dev) + - "127.0.0.1:${AUTH_PORT:-8001}:8001" depends_on: postgres: condition: service_healthy @@ -178,7 +183,10 @@ services: start_period: 15s metadata: - build: ../services/metadata + # issue #63: context = repo root — Dockerfile копирует общий модуль pkg/ + build: + context: .. + dockerfile: services/metadata/Dockerfile environment: DATABASE_URL: postgres://flowix:flowix@postgres:5432/flowix?sslmode=disable DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-5} @@ -208,7 +216,11 @@ services: start_period: 10s upload: - build: ../services/upload + # issue #63: context = repo root; graceful shutdown ждёт in-flight загрузки + stop_grace_period: 5m + build: + context: .. + dockerfile: services/upload/Dockerfile environment: DATABASE_URL: postgres://flowix:flowix@postgres:5432/flowix?sslmode=disable RABBITMQ_URL: amqp://flowix:flowix@rabbitmq:5672/ @@ -221,6 +233,8 @@ services: JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required — set it in .env (see .env.example)} UPLOAD_MAX_BYTES: ${UPLOAD_MAX_BYTES:-5368709120} MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000} + # grace для Shutdown меньше stop_grace_period (issue #63) + SHUTDOWN_GRACE: 4m ports: - "${UPLOAD_PORT:-8003}:8003" depends_on: @@ -275,7 +289,11 @@ services: start_period: 20s gateway: - build: ../services/gateway + # issue #63: context = repo root; graceful shutdown ждёт in-flight загрузки + stop_grace_period: 5m + build: + context: .. + dockerfile: services/gateway/Dockerfile environment: JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required — set it in .env (see .env.example)} INTERNAL_TOKEN: ${INTERNAL_TOKEN:?INTERNAL_TOKEN is required — set it in .env (see .env.example)} @@ -284,6 +302,14 @@ services: METADATA_URL: http://metadata:8002 UPLOAD_URL: http://upload:8003 VOD_URL: http://nginx-vod:80 + # grace для Shutdown меньше stop_grace_period (issue #63) + SHUTDOWN_GRACE: 4m + # issue #52: CORS allowlist, Redis rate-limit. TRUSTED_PROXY_CIDRS пуст — + # gateway является edge (LB перед ним нет), клиентский XFF не доверяем. + # Если появится LB — указать его CIDR в .env. + CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000} + TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-} + REDIS_URL: redis://redis:6379/0 ports: - "${GATEWAY_PORT:-8080}:8080" depends_on: @@ -295,6 +321,8 @@ services: condition: service_healthy nginx-vod: condition: service_healthy + redis: + condition: service_healthy restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/health"] diff --git a/docs/spec.md b/docs/spec.md index 2c05e73..c823c86 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -33,6 +33,9 @@ The backend is split into microservices written in **Go** and **Python** to leve ``` . +├── go.work # Go workspace: pkg + три Go-сервиса (issue #63) +├── pkg/ # Shared Go module (flowix/pkg): auth middleware, request logger, +│ # metrics, httputil.WriteJSON, httpserver (таймауты + graceful shutdown) ├── services/ │ ├── gateway/ # Go (API Gateway) │ ├── metadata/ # Go (Metadata Service) @@ -47,7 +50,7 @@ The backend is split into microservices written in **Go** and **Python** to leve └── README.md ``` -Each service has its own `Dockerfile` and can be developed independently. +Each service has its own `Dockerfile` and can be developed independently. Go-сервисы используют общий модуль `pkg/` (через `replace` в go.mod + `go.work` локально; build context в compose — корень репо). Все HTTP-серверы запускаются через `pkg/httpserver` — таймауты Read/Write/Idle и graceful shutdown по SIGTERM/SIGINT (issue #63). --- @@ -55,6 +58,7 @@ Each service has its own `Dockerfile` and can be developed independently. ### 1. API Gateway (Go) - **Purpose**: Single entry point for client requests; routes to appropriate services; handles CORS, rate limiting, request aggregation. +- **Security (issue #52)**: CORS — явный allowlist origin (`CORS_ALLOWED_ORIGINS`, без wildcard с credentials); rate-limit — fixed-window в Redis (`REDIS_URL`), fail-open при недоступности Redis; `X-Forwarded-For`/`X-Real-IP` доверяются только от trusted прокси (`TRUSTED_PROXY_CIDRS`), downstream получает проверенный `X-Real-IP` (по нему ключует лимитер auth). - **Framework**: Gin, Echo, or chi. - **Communication**: REST or gRPC to internal services. diff --git a/go.work b/go.work new file mode 100644 index 0000000..938ba0e --- /dev/null +++ b/go.work @@ -0,0 +1,8 @@ +go 1.25.0 + +use ( + ./pkg + ./services/gateway + ./services/metadata + ./services/upload +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..72fff74 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,219 @@ +4d63.com/gocheckcompilerdirectives v1.3.0/go.mod h1:ofsJ4zx2QAuIP/NO/NAh1ig6R1Fb18/GI7RVMwz7kAY= +4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= +charm.land/lipgloss/v2 v2.0.3/go.mod h1:7myLU9iG/3xluAWzpY/fSxYYHCgoKTie7laxk6ATwXA= +codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= +codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= +dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= +dev.gaijin.team/go/golib v0.6.0/go.mod h1:uY1mShx8Z/aNHWDyAkZTkX+uCi5PdX7KsG1eDQa2AVE= +github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= +github.com/Abirdcfly/dupword v0.1.7/go.mod h1:K0DkBeOebJ4VyOICFdppB23Q0YMOgVafM0zYW0n9lF4= +github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= +github.com/AlwxSin/noinlineerr v1.0.5/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= +github.com/Antonboom/errname v1.1.1/go.mod h1:gjhe24xoxXp0ScLtHzjiXp0Exi1RFLKJb0bVBtWKCWQ= +github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= +github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/ClickHouse/clickhouse-go-linter v1.2.0/go.mod h1:pLorS7ffPTfuUV9M0SJgfHA/h/WQPQUk2FWG9x74cQ4= +github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/chroma/v2 v2.24.1/go.mod h1:l+ohZ9xRXIbGe7cIW+YZgOGbvuVLjMps/FYN/CwuabI= +github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= +github.com/alexkohler/prealloc v1.1.0/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= +github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= +github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= +github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= +github.com/ashanbrown/forbidigo/v2 v2.3.1/go.mod h1:2QDkLTzU6TV937eFROamXrW92M3paehdae4HCDCOZCM= +github.com/ashanbrown/makezero/v2 v2.2.1/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= +github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= +github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= +github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= +github.com/bombsimon/wsl/v5 v5.8.0/go.mod h1:AbOLsulgkqP4ZnitHf9gwPtCOGlrzkk0jb0uNxRSY0o= +github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= +github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= +github.com/butuzov/ireturn v0.4.1/go.mod h1:q+DXKzTDV5guNuXLnIab9fKXizTn2miZHLhxH7V/GB4= +github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= +github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= +github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= +github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318/go.mod h1:Y6kE2GzHfkyQQVCSL9r2hwokSrIlHGzZG+71+wDYSZI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= +github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= +github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= +github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= +github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= +github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= +github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= +github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= +github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= +github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= +github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= +github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= +github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= +github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= +github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= +github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d/go.mod h1:ivJ9QDg0XucIkmwhzCDsqcnxxlDStoTl89jDMIoNxKY= +github.com/golangci/golangci-lint/v2 v2.12.2/go.mod h1:opqHHuIcTG2R+4akzWMd4o1BnD9/1LcjICWOujr91U8= +github.com/golangci/golines v0.15.0/go.mod h1:AZjXd23tbHMpowhtnGlj9KCNsysj72aeZVVHnVcZx10= +github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= +github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= +github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= +github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sCBNcpRmhJCtbFGz49+IM3ETTFf7QdJ30AeYCd43NKk= +github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= +github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= +github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= +github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= +github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= +github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jgautheron/goconst v1.10.0/go.mod h1:0p+wv1lFOiUr0IlNNT1nrm6+8DB8u2sU6KHGzFRXHDc= +github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= +github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= +github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= +github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= +github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= +github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= +github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= +github.com/ldez/gomoddirectives v0.8.0/go.mod h1:jutzamvZR4XYJLr0d5Honycp4Gy6GEg2mS9+2YX3F1Q= +github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= +github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= +github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= +github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= +github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= +github.com/manuelarte/funcorder v0.6.0/go.mod h1:id3NDhXdQBmeqXH7eVC6Z89xS6JxvZ8kF9xUxpArU/g= +github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= +github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= +github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= +github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= +github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= +github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= +github.com/ryancurrah/gomodguard/v2 v2.1.3/go.mod h1:CQicdLGatWMxLX53JzoBjYlsNZhHbmLv2AVa0s2aivU= +github.com/ryanrolds/sqlclosecheck v0.6.0/go.mod h1:xyX16hsDaCMXHrMJ3JMzGf5OpDfHTOTTQrT7HOFUmeU= +github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= +github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= +github.com/securego/gosec/v2 v2.26.1/go.mod h1:57UW4p0uoP3kxoTkhoo3axLdVAi+OWrLg/Ax/kdqtPE= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= +github.com/sonatard/noctx v0.5.1/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= +github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= +github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tetafro/godot v1.5.6/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= +github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4/go.mod h1:sDHLK7rb/59v/ZxZ7KtymgcoxuUMxjXq8gtu9VMOK8M= +github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= +github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= +github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= +github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= +github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/uudashr/gocognit v1.2.1/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q= +github.com/uudashr/iface v1.4.2/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= +github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= +github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= +github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= +go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= +go-simpler.org/sloglint v0.12.0/go.mod h1:jBjjC2bm8rYrs88oTRlFX497kWjJsyZWYoNaXkGRI6I= +go.augendre.info/arangolint v0.4.0/go.mod h1:l+f/b4plABuFISuKnTGD4RioXiCCgghv2xqst/xOvAA= +go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358/go.mod h1:4Mzdyp/6jzw9auFDJ3OMF5qksa7UvPnzKqTVGcb04ms= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/go.mod b/pkg/go.mod new file mode 100644 index 0000000..03c0be8 --- /dev/null +++ b/pkg/go.mod @@ -0,0 +1,24 @@ +module flowix/pkg + +go 1.25.0 + +require ( + github.com/go-chi/chi/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/prometheus/client_golang v1.22.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.30.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect +) diff --git a/pkg/go.sum b/pkg/go.sum new file mode 100644 index 0000000..b9eddbd --- /dev/null +++ b/pkg/go.sum @@ -0,0 +1,43 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= +github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/httpserver/server.go b/pkg/httpserver/server.go new file mode 100644 index 0000000..33db17a --- /dev/null +++ b/pkg/httpserver/server.go @@ -0,0 +1,108 @@ +// Package httpserver — общий bootstrap HTTP-сервера Go-сервисов Flowix: +// таймауты http.Server + graceful shutdown по SIGTERM/SIGINT (issue #63). +package httpserver + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/rs/zerolog/log" +) + +const ( + DefaultReadHeaderTimeout = 15 * time.Second + DefaultReadTimeout = 15 * time.Second + DefaultWriteTimeout = 60 * time.Second + DefaultIdleTimeout = 120 * time.Second + DefaultShutdownGrace = 30 * time.Second +) + +// Timeouts задаёт таймауты http.Server. Нулевые ReadTimeout/WriteTimeout +// означают «без лимита» — так нужно сервисам, через которые стримятся +// большие тела запросов (upload/gateway: тела до 5 ГБ не влезают в +// Read/Write timeout — иначе деплой/таймаут обрывает in-flight загрузки). +type Timeouts struct { + ReadHeaderTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + ShutdownGrace time.Duration +} + +// Defaults — литеральные значения issue #63: Read 15s, Write 60s, Idle 120s. +// Подходит сервисам с маленькими JSON-телами (metadata). +func Defaults() Timeouts { + return Timeouts{ + ReadHeaderTimeout: DefaultReadHeaderTimeout, + ReadTimeout: DefaultReadTimeout, + WriteTimeout: DefaultWriteTimeout, + IdleTimeout: DefaultIdleTimeout, + ShutdownGrace: DefaultShutdownGrace, + } +} + +// Streaming — таймауты для сервисов, проксирующих/принимающих большие тела +// (upload 5 ГБ): лимитируем только заголовки, тело стримим без лимита. +func Streaming() Timeouts { + return Timeouts{ + ReadHeaderTimeout: DefaultReadHeaderTimeout, + IdleTimeout: DefaultIdleTimeout, + ShutdownGrace: DefaultShutdownGrace, + } +} + +// Run запускает http.Server и блокируется до SIGTERM/SIGINT или ошибки. +// Shutdown ждёт завершения in-flight запросов (загрузки не обрываются), +// но не дольше ShutdownGrace. SHUTDOWN_GRACE (например «4m») переопределяет +// грейс из env — должен быть меньше stop_grace_period в docker-compose. +func Run(addr string, h http.Handler, t Timeouts) error { + if t.ShutdownGrace <= 0 { + t.ShutdownGrace = DefaultShutdownGrace + } + if v := os.Getenv("SHUTDOWN_GRACE"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + t.ShutdownGrace = d + } + } + srv := newServer(addr, h, t) + + errCh := make(chan error, 1) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + }() + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + select { + case err := <-errCh: + return err + case <-quit: + } + + log.Info().Dur("grace", t.ShutdownGrace).Msg("shutdown signal received, waiting for in-flight requests") + ctx, cancel := context.WithTimeout(context.Background(), t.ShutdownGrace) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + return fmt.Errorf("graceful shutdown: %w", err) + } + return nil +} + +func newServer(addr string, h http.Handler, t Timeouts) *http.Server { + return &http.Server{ + Addr: addr, + Handler: h, + ReadHeaderTimeout: t.ReadHeaderTimeout, + ReadTimeout: t.ReadTimeout, + WriteTimeout: t.WriteTimeout, + IdleTimeout: t.IdleTimeout, + } +} diff --git a/pkg/httpserver/server_test.go b/pkg/httpserver/server_test.go new file mode 100644 index 0000000..a38f526 --- /dev/null +++ b/pkg/httpserver/server_test.go @@ -0,0 +1,36 @@ +package httpserver + +import ( + "net/http" + "testing" + "time" +) + +func TestDefaultsMatchIssue(t *testing.T) { + d := Defaults() + if d.ReadTimeout != 15*time.Second || d.WriteTimeout != 60*time.Second || d.IdleTimeout != 120*time.Second || d.ReadHeaderTimeout != 15*time.Second { + t.Fatalf("issue #63 timeouts mismatch: %+v", d) + } + if d.ShutdownGrace <= 0 { + t.Fatalf("shutdown grace must be positive: %+v", d) + } +} + +func TestStreamingHasNoBodyLimits(t *testing.T) { + s := Streaming() + if s.ReadTimeout != 0 || s.WriteTimeout != 0 { + t.Fatalf("streaming config must not cap body read/write: %+v", s) + } + if s.ReadHeaderTimeout != 15*time.Second || s.IdleTimeout != 120*time.Second { + t.Fatalf("streaming config mismatch: %+v", s) + } +} + +func TestNewServerAppliesTimeouts(t *testing.T) { + tm := Defaults() + srv := newServer(":0", http.NotFoundHandler(), tm) + if srv.ReadTimeout != tm.ReadTimeout || srv.WriteTimeout != tm.WriteTimeout || + srv.IdleTimeout != tm.IdleTimeout || srv.ReadHeaderTimeout != tm.ReadHeaderTimeout { + t.Fatalf("server timeouts not applied: %+v", srv) + } +} diff --git a/pkg/httputil/json.go b/pkg/httputil/json.go new file mode 100644 index 0000000..c67580a --- /dev/null +++ b/pkg/httputil/json.go @@ -0,0 +1,51 @@ +// Package httputil — общие HTTP-хелперы Go-сервисов Flowix (JSON-ответы). +package httputil + +import ( + "encoding/json" + "log/slog" + "net/http" + "os" + "strconv" +) + +// WriteJSON centralizes JSON encoding per golang-code-style + golang-error-handling. +// +// - Sets Content-Type explicitly (security: correct MIME, no sniffing) +// - Uses compact JSON by default (production: меньше трафика) +// - Pretty-print только по ?pretty=1 или PRETTY_JSON=true (dev: читаемость в Swagger/curl) +// - Логирует ошибку кодирования через slog (single handling rule: log server-side, не отдавать стек клиенту) +func WriteJSON(w http.ResponseWriter, r *http.Request, status int, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + + if shouldPretty(r) { + enc.SetIndent("", " ") + } + + if err := enc.Encode(v); err != nil { + // Encode after WriteHeader уже не поправить статус — только логируем + slog.Error("encode json failed", "error", err, "path", r.URL.Path) + } +} + +func shouldPretty(r *http.Request) bool { + // 1. явный ?pretty=1 / ?pretty=true + if v := r.URL.Query().Get("pretty"); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + // ?pretty без значения — считаем включенным + return true + } + // 2. глобальный флаг для dev/Swagger: PRETTY_JSON=true + if v := os.Getenv("PRETTY_JSON"); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return false +} diff --git a/pkg/httputil/json_test.go b/pkg/httputil/json_test.go new file mode 100644 index 0000000..1d29270 --- /dev/null +++ b/pkg/httputil/json_test.go @@ -0,0 +1,57 @@ +package httputil + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestWriteJSONSetsContentTypeAndBody(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + WriteJSON(w, r, http.StatusOK, map[string]string{"status": "ok"}) + + if ct := w.Header().Get("Content-Type"); ct != "application/json; charset=utf-8" { + t.Fatalf("content-type %q", ct) + } + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + var out map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("body is not json: %v (%s)", err, w.Body.String()) + } + if out["status"] != "ok" { + t.Fatalf("body %s", w.Body.String()) + } +} + +func TestWriteJSONPrettyByQuery(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/?pretty=true", nil) + WriteJSON(w, r, http.StatusOK, map[string]string{"a": "b"}) + if !strings.Contains(w.Body.String(), "\n") || !strings.Contains(w.Body.String(), " \"a\"") { + t.Fatalf("expected indented json, got %q", w.Body.String()) + } +} + +func TestWriteJSONPrettyByEnv(t *testing.T) { + t.Setenv("PRETTY_JSON", "true") + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + WriteJSON(w, r, http.StatusOK, map[string]string{"a": "b"}) + if !strings.Contains(w.Body.String(), "\n") { + t.Fatalf("expected indented json via PRETTY_JSON, got %q", w.Body.String()) + } +} + +func TestWriteJSONEscapeHTMLDisabled(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/", nil) + WriteJSON(w, r, http.StatusOK, map[string]string{"u": "http://x/?a=1&b=2"}) + if strings.Contains(w.Body.String(), "\\u0026") { + t.Fatalf("HTML escaping must be disabled, got %s", w.Body.String()) + } +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..57c3d29 --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,29 @@ +// Package logger — единая инициализация zerolog для Go-сервисов Flowix +// (issue #63: один bootstrap вместо копий в каждом main.go). +package logger + +import ( + "os" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// Setup настраивает глобальный zerolog (console в dev / json в prod, уровень +// из LOG_LEVEL) и возвращает логгер с полем service=. +func Setup(service string) zerolog.Logger { + // zerolog console in dev, json in prod + if strings.ToLower(os.Getenv("LOG_FORMAT")) == "console" || os.Getenv("ENV") == "dev" { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}) + } else { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + } + zerolog.SetGlobalLevel(zerolog.InfoLevel) + if lvl := os.Getenv("LOG_LEVEL"); lvl != "" { + if l, err := zerolog.ParseLevel(lvl); err == nil { + zerolog.SetGlobalLevel(l) + } + } + return log.With().Timestamp().Str("service", service).Logger() +} diff --git a/services/gateway/internal/metrics/metrics.go b/pkg/metrics/metrics.go similarity index 66% rename from services/gateway/internal/metrics/metrics.go rename to pkg/metrics/metrics.go index 458429d..9a895c9 100644 --- a/services/gateway/internal/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -1,3 +1,5 @@ +// Package metrics — общие Prometheus-метрики Go-сервисов Flowix +// (issue #63: единый набор collectors вместо копий в каждом сервисе). package metrics import ( @@ -8,6 +10,10 @@ import ( ) var ( + UploadBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "upload_bytes", + Help: "Total uploaded bytes via gateway/upload", + }) RabbitMQQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "rabbitmq_queue_depth", Help: "Current depth of video.uploaded queue", @@ -16,10 +22,6 @@ var ( Name: "vod_cache_hit", Help: "Number of VOD cache hits", }) - UploadBytes = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "upload_bytes", - Help: "Total uploaded bytes via gateway", - }) FfmpegDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "ffmpeg_duration_seconds", Help: "FFmpeg transcoding duration", @@ -28,13 +30,16 @@ var ( ) func init() { - prometheus.MustRegister(RabbitMQQueueDepth, VodCacheHit, UploadBytes, FfmpegDuration) + prometheus.MustRegister(UploadBytes, RabbitMQQueueDepth, VodCacheHit, FfmpegDuration) } +// Handler — /metrics endpoint. func Handler() http.Handler { return promhttp.Handler() } +// Middleware — счётчик VodCacheHit на каждый запрос (используется gateway +// на /hls/*). func Middleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { VodCacheHit.Inc() diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 0000000..0b7a472 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMiddlewareIncrementsVodCacheHit(t *testing.T) { + before := testutil.ToFloat64(VodCacheHit) + h := Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + req := httptest.NewRequest(http.MethodGet, "/hls/x", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if after := testutil.ToFloat64(VodCacheHit); after != before+1 { + t.Fatalf("vod_cache_hit want %v got %v", before+1, after) + } +} + +func TestHandlerNonNil(t *testing.T) { + if Handler() == nil { + t.Fatal("metrics handler must not be nil") + } +} diff --git a/services/gateway/internal/middleware/auth.go b/pkg/middleware/auth.go similarity index 89% rename from services/gateway/internal/middleware/auth.go rename to pkg/middleware/auth.go index 2a1e99e..a9f3459 100644 --- a/services/gateway/internal/middleware/auth.go +++ b/pkg/middleware/auth.go @@ -1,4 +1,5 @@ -// Package middleware provides HTTP middleware for gateway authentication. +// Package middleware provides HTTP middleware shared by the Flowix Go +// services (issue #63: единая копия вместо дубликата в gateway/metadata/upload). package middleware import ( @@ -14,8 +15,7 @@ type ctxKey string const UserIDKey ctxKey = "user_id" // AuthMiddleware проверяет JWT Bearer и кладёт sub (user_id) в контекст -// и заголовок X-User-ID для downstream сервисов. Пропускает без токена -// только если вызывающий явно не требует аутентификации. +// и заголовок X-User-ID для downstream сервисов. func AuthMiddleware(secret string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -60,6 +60,7 @@ func AuthMiddleware(secret string) func(http.Handler) http.Handler { // и кладёт в контекст, если нет — пропускает. Удобно для публичных GET. // Невалидный/просроченный токен — тоже анонимный проход (issue #44): публичная // лента не должна падать 401, когда фронт прислал истёкший access-токен. +// Клиентский X-User-ID недоверенный — убираем, ставим только после валидации JWT. func OptionalAuth(secret string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -97,6 +98,7 @@ func OptionalAuth(secret string) func(http.Handler) http.Handler { } } +// UserIDFromCtx возвращает sub из контекста (пустую строку для анонимов). func UserIDFromCtx(ctx context.Context) string { v, _ := ctx.Value(UserIDKey).(string) return v diff --git a/services/gateway/internal/middleware/auth_test.go b/pkg/middleware/auth_test.go similarity index 84% rename from services/gateway/internal/middleware/auth_test.go rename to pkg/middleware/auth_test.go index bc22faf..c470290 100644 --- a/services/gateway/internal/middleware/auth_test.go +++ b/pkg/middleware/auth_test.go @@ -139,9 +139,10 @@ func TestOptionalAuthStripsForgedUserID(t *testing.T) { func TestOptionalAuthSetsUserIDFromJWT(t *testing.T) { secret := "test-secret" - var sawHeader string + var sawHeader, sawCtx string h := OptionalAuth(secret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sawHeader = r.Header.Get("X-User-ID") + sawCtx = UserIDFromCtx(r.Context()) w.WriteHeader(200) })) req := httptest.NewRequest("GET", "/", nil) @@ -149,8 +150,8 @@ func TestOptionalAuthSetsUserIDFromJWT(t *testing.T) { req.Header.Set("Authorization", "Bearer "+mustToken(secret, "u1")) w := httptest.NewRecorder() h.ServeHTTP(w, req) - if w.Code != 200 || sawHeader != "u1" { - t.Fatalf("valid JWT must set X-User-ID=u1, got %q (code %d)", sawHeader, w.Code) + if w.Code != 200 || sawHeader != "u1" || sawCtx != "u1" { + t.Fatalf("valid JWT must set X-User-ID=u1, got header %q ctx %q (code %d)", sawHeader, sawCtx, w.Code) } } @@ -199,3 +200,26 @@ func TestOptionalAuthRejectsRefreshToken(t *testing.T) { t.Fatalf("refresh token must be anonymous, got code %d called %v header %q", w.Code, called, sawHeader) } } + +func TestClientIPForwardedFor(t *testing.T) { + // issue #52: клиентский XFF подделываем — не используем его для логов + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("X-Forwarded-For", " 203.0.113.7 , 10.0.0.1") + r.RemoteAddr = "192.0.2.10:5555" + if got := ClientIP(r); got != "192.0.2.10" { + t.Fatalf("XFF must be ignored, want RemoteAddr ip, got %q", got) + } + r2 := httptest.NewRequest("GET", "/", nil) + r2.Header.Set("X-Real-IP", "198.51.100.2") + r2.RemoteAddr = "192.0.2.10:5555" + if got := ClientIP(r2); got != "198.51.100.2" { + t.Fatalf("want X-Real-IP, got %q", got) + } + // не-IP в X-Real-IP — не доверяем, падаем на RemoteAddr + r3 := httptest.NewRequest("GET", "/", nil) + r3.Header.Set("X-Real-IP", "not-an-ip") + r3.RemoteAddr = "192.0.2.10:5555" + if got := ClientIP(r3); got != "192.0.2.10" { + t.Fatalf("invalid X-Real-IP must be ignored, got %q", got) + } +} diff --git a/pkg/middleware/logger.go b/pkg/middleware/logger.go new file mode 100644 index 0000000..692af55 --- /dev/null +++ b/pkg/middleware/logger.go @@ -0,0 +1,95 @@ +package middleware + +import ( + "net" + "net/http" + "strings" + "time" + + chimw "github.com/go-chi/chi/v5/middleware" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// RequestLogger — structured logging через zerolog. Логирует метод, путь, +// статус, длительность, IP и trace_id; service тегирует сервис-источник. +// Работает с chi RequestID (из контекста), иначе берёт id из заголовков +// клиента; найденный id прокидывается в заголовки запроса (для proxy в +// апстримы) и ответа (клиент может скопировать id для поиска в Grafana). +// Ошибки >=500 — уровнем Error. +func RequestLogger(service string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + reqID := chimw.GetReqID(r.Context()) + if reqID == "" { + reqID = r.Header.Get("X-Request-ID") + } + if reqID == "" { + reqID = r.Header.Get("X-Request-Id") + } + if reqID == "" { + reqID = r.Header.Get("X-Correlation-ID") + } + if reqID != "" { + r.Header.Set("X-Request-ID", reqID) + w.Header().Set("X-Request-Id", reqID) + } + + ww := &respWriter{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(ww, r) + dur := time.Since(start) + + ev := log.Info() + if ww.status >= 500 { + ev = log.Error() + } else if ww.status >= 400 { + ev = log.Warn() + } + ev.Str("method", r.Method). + Str("path", r.URL.Path). + Int("status", ww.status). + Dur("duration", dur). + Str("ip", ClientIP(r)). + Str("trace_id", reqID). + Str("service", service). + Msg("request") + // также zerolog global logger доступен как zerolog.Ctx + _ = zerolog.Ctx(r.Context()) + }) + } +} + +// respWriter захватывает статус ответа и пробрасывает Flush в нижележащий +// writer (http.Flusher) — иначе стриминг сквозь обёртку (HLS-прокси, +// streaming-ответы) буферизуется, а ReverseProxy не может сбрасывать буфер. +type respWriter struct { + http.ResponseWriter + status int +} + +func (w *respWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +func (w *respWriter) Flush() { + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// ClientIP извлекает IP клиента для логов. Клиентский X-Forwarded-For не +// доверяем (первый IP подделывается, issue #52): берём X-Real-IP — его +// выставляет gateway-RealIP из проверенного peer'а и затирает клиентские +// подделки, — иначе RemoteAddr (за gateway это peer самого gateway). +func ClientIP(r *http.Request) string { + if xri := strings.TrimSpace(r.Header.Get("X-Real-IP")); xri != "" && net.ParseIP(xri) != nil { + return xri + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} diff --git a/services/gateway/internal/middleware/logger_test.go b/pkg/middleware/logger_test.go similarity index 50% rename from services/gateway/internal/middleware/logger_test.go rename to pkg/middleware/logger_test.go index ec54f76..beab88b 100644 --- a/services/gateway/internal/middleware/logger_test.go +++ b/pkg/middleware/logger_test.go @@ -25,7 +25,7 @@ func TestRequestLoggerPropagatesGeneratedRequestID(t *testing.T) { defer restore() var gotHeader string - h := chimw.RequestID(RequestLogger(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := chimw.RequestID(RequestLogger("gateway")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotHeader = r.Header.Get("X-Request-ID") w.WriteHeader(http.StatusOK) }))) @@ -50,7 +50,7 @@ func TestRequestLoggerKeepsClientRequestID(t *testing.T) { defer restore() var gotHeader string - h := chimw.RequestID(RequestLogger(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := chimw.RequestID(RequestLogger("metadata")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotHeader = r.Header.Get("X-Request-ID") w.WriteHeader(http.StatusOK) }))) @@ -66,4 +66,49 @@ func TestRequestLoggerKeepsClientRequestID(t *testing.T) { if !strings.Contains(buf.String(), `"trace_id":"client-trace-1"`) { t.Fatalf("client trace_id missing in log: %s", buf.String()) } + if !strings.Contains(buf.String(), `"service":"metadata"`) { + t.Fatalf("service tag missing in log: %s", buf.String()) + } +} + +// respWriter must expose http.Flusher to handlers behind the logging wrapper — +// otherwise ReverseProxy (HLS) cannot flush streaming responses (issue #63). +func TestRequestLoggerSupportsFlusher(t *testing.T) { + flushed := false + h := RequestLogger("upload")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f, ok := w.(http.Flusher) + if !ok { + t.Fatal("response writer behind RequestLogger does not implement http.Flusher") + } + _, _ = w.Write([]byte("chunk")) + f.Flush() + flushed = true + })) + req := httptest.NewRequest(http.MethodGet, "/", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if !flushed { + t.Fatal("Flush was not called") + } + if w.Body.String() != "chunk" { + t.Fatalf("body lost through wrapper: %q", w.Body.String()) + } +} + +func TestRequestLoggerStatusCaptured(t *testing.T) { + buf, restore := captureLog(t) + defer restore() + + h := RequestLogger("gateway")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + req := httptest.NewRequest(http.MethodGet, "/", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if !strings.Contains(buf.String(), `"status":403`) { + t.Fatalf("status 403 missing in log: %s", buf.String()) + } + if !strings.Contains(buf.String(), `"level":"warn"`) { + t.Fatalf("4xx must log at warn level: %s", buf.String()) + } } diff --git a/scripts/e2e.sh b/scripts/e2e.sh index c89e788..1acf37b 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -109,6 +109,11 @@ if [ -z "$VIDEO_ID" ]; then [ -n "$TOKEN" ] && [ "$TOKEN" != "null" ] || fail "no access_token in auth response" say " token acquired" + # regression guard: video list must not 500 (uuid = text in repo List, SQLSTATE 42883) + code=$(http_get "$GATEWAY/api/v1/videos?limit=1") + [ "$code" = "200" ] || fail "video list failed ($code): $(cat "$TMP/body")" + say " video list 200 ok" + if [ -z "$SAMPLE" ]; then need ffmpeg SAMPLE="$TMP/sample.mp4" diff --git a/services/auth/README.md b/services/auth/README.md index ad326ec..3827808 100644 --- a/services/auth/README.md +++ b/services/auth/README.md @@ -38,6 +38,14 @@ uv run --project services/auth pytest -q uv run --project services/auth pytest --cov ``` +## Rate-limit (issue #52) +`/api/v1/auth/login` — slowapi 5/minute, Redis storage (`REDIS_URL`). +Ключ лимитера — реальный IP клиента из `X-Real-IP`, который выставляет gateway +(затирая клиентские подделки). `X-Real-IP` доверяем только когда непосредственный +пир в `TRUSTED_PROXY_CIDRS` (compose задаёт docker-сеть `172.16.0.0/12`; порт auth +публикуется только на 127.0.0.1 — ревью фазы 17, H1), иначе ключ по peer IP. +Без этого один абузер блокировал бы всех (slowapi видел только IP gateway). + ## Zed IDE (при сохранении) Настроено в `.zed/settings.json:1`: - `Python` → `pyright` + `ruff`, `format_on_save: on` diff --git a/services/auth/src/core/limiter.py b/services/auth/src/core/limiter.py index a28fdc5..2af5035 100644 --- a/services/auth/src/core/limiter.py +++ b/services/auth/src/core/limiter.py @@ -1,5 +1,8 @@ +import ipaddress import os +from fastapi import Request + from .config import settings try: @@ -8,11 +11,50 @@ _redis_url = os.getenv("REDIS_URL") or settings.redis_url + def _parse_cidrs(raw: str) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: + """Список доверенных прокси из TRUSTED_PROXY_CIDRS (пусто = не доверять никому).""" + nets: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for part in raw.split(","): + part = part.strip() + if not part: + continue + try: + nets.append(ipaddress.ip_network(part)) + except ValueError: + continue + return nets + + _trusted_nets = _parse_cidrs(os.getenv("TRUSTED_PROXY_CIDRS", "")) + + def _client_ip(request: Request) -> str: + """Ключ лимитера — реальный IP клиента (issue #52). + + Gateway выставляет проверенный X-Real-IP (затирая клиентские + подделки) — ключуем по нему, иначе slowapi видит только IP gateway + и один абузер блокирует всех. X-Real-IP доверяем только когда + непосредственный пир в TRUSTED_PROXY_CIDRS: при прямом доступе к auth + клиент мог подделать заголовок и получить безлимит бакетов (ревью + фазы 17, H1). Некорректные значения и чужие пиры — ключ по peer IP. + """ + peer = request.client.host if request.client else "" + try: + peer_addr = ipaddress.ip_address(peer) + except ValueError: + return get_remote_address(request) + if any(peer_addr in net for net in _trusted_nets): + real_ip = request.headers.get("X-Real-IP", "") + try: + ipaddress.ip_address(real_ip) + return real_ip + except ValueError: + pass + return get_remote_address(request) + def _make_limiter(url: str | None): # type: ignore[no-untyped-def] try: if url and url.startswith("redis"): # probe redis connectivity — fallback to memory if unreachable - lim = Limiter(key_func=get_remote_address, storage_uri=url, default_limits=[]) # type: ignore[no-untyped-call] + lim = Limiter(key_func=_client_ip, storage_uri=url, default_limits=[]) # type: ignore[no-untyped-call] # quick storage check: try to get reset without network if possible try: # try a dummy operation; if redis not reachable, this will raise @@ -28,11 +70,11 @@ def _make_limiter(url: str | None): # type: ignore[no-untyped-def] s.close() except Exception: # redis not reachable — use memory - return Limiter(key_func=get_remote_address, default_limits=[]) # type: ignore[no-untyped-call] + return Limiter(key_func=_client_ip, default_limits=[]) # type: ignore[no-untyped-call] return lim - return Limiter(key_func=get_remote_address, default_limits=[]) # type: ignore[no-untyped-call] + return Limiter(key_func=_client_ip, default_limits=[]) # type: ignore[no-untyped-call] except Exception: - return Limiter(key_func=get_remote_address, default_limits=[]) # type: ignore[no-untyped-call] + return Limiter(key_func=_client_ip, default_limits=[]) # type: ignore[no-untyped-call] limiter: Limiter | None = _make_limiter(_redis_url) except ImportError: diff --git a/services/auth/tests/test_limiter.py b/services/auth/tests/test_limiter.py new file mode 100644 index 0000000..b1ed494 --- /dev/null +++ b/services/auth/tests/test_limiter.py @@ -0,0 +1,42 @@ +import ipaddress + +from fastapi import Request + +from src.core.limiter import _client_ip + + +def _request(peer: str | None, headers: dict[str, str] | None = None) -> Request: + scope = { + "type": "http", + "method": "POST", + "path": "/api/v1/auth/login", + "headers": [(k.lower().encode(), v.encode()) for k, v in (headers or {}).items()], + "client": (peer, 12345) if peer else None, + "query_string": b"", + } + return Request(scope) + + +def test_client_ip_trusted_peer_uses_real_ip(monkeypatch): + monkeypatch.setattr("src.core.limiter._trusted_nets", [ipaddress.ip_network("10.0.0.0/8")]) + req = _request("10.0.0.5", {"X-Real-IP": "203.0.113.9"}) + assert _client_ip(req) == "203.0.113.9" + + +def test_client_ip_untrusted_peer_cannot_spoof(monkeypatch): + # ревью фазы 17 (H1): прямой доступ к auth — подделанный X-Real-IP игнорируем + monkeypatch.setattr("src.core.limiter._trusted_nets", [ipaddress.ip_network("10.0.0.0/8")]) + req = _request("192.0.2.7", {"X-Real-IP": "203.0.113.9"}) + assert _client_ip(req) == "192.0.2.7" + + +def test_client_ip_invalid_real_ip_falls_back_to_peer(monkeypatch): + monkeypatch.setattr("src.core.limiter._trusted_nets", [ipaddress.ip_network("10.0.0.0/8")]) + req = _request("10.0.0.5", {"X-Real-IP": "not-an-ip"}) + assert _client_ip(req) == "10.0.0.5" + + +def test_client_ip_no_trusted_nets_keys_by_peer(monkeypatch): + monkeypatch.setattr("src.core.limiter._trusted_nets", []) + req = _request("192.0.2.7", {"X-Real-IP": "203.0.113.9"}) + assert _client_ip(req) == "192.0.2.7" diff --git a/services/gateway/Dockerfile b/services/gateway/Dockerfile index 8492171..c6be7b1 100644 --- a/services/gateway/Dockerfile +++ b/services/gateway/Dockerfile @@ -1,8 +1,14 @@ +# Issue #63: build context = repo root (см. deploy/docker-compose.yml) — +# рядом копируем общий модуль pkg/. go.work в образ не попадает: сборка +# идёт per-module через replace flowix/pkg => ../../pkg. FROM golang:1.27-alpine AS builder WORKDIR /app -COPY go.mod go.sum* ./ -RUN go mod download || true -COPY . . +COPY pkg/go.mod pkg/go.sum pkg/ +COPY services/gateway/go.mod services/gateway/go.sum services/gateway/ +RUN cd /app/pkg && go mod download && cd /app/services/gateway && go mod download +COPY pkg/ pkg/ +COPY services/gateway/ services/gateway/ +WORKDIR /app/services/gateway RUN CGO_ENABLED=0 go build -o /server ./cmd/server FROM alpine:3.19 diff --git a/services/gateway/README.md b/services/gateway/README.md index 30700f4..50cb373 100644 --- a/services/gateway/README.md +++ b/services/gateway/README.md @@ -17,5 +17,39 @@ make fmt-go make lint-go ``` +## CORS и rate-limit (issue #52) + +- **CORS** — только явный allowlist: `CORS_ALLOWED_ORIGINS` (comma-separated, + dev-дефолт `http://localhost:3000`). Origin из списка → отражаем его в + `Access-Control-Allow-Origin` (+ `Vary: Origin`, credentials включены). + Чужой origin или пустой список → ACAO-заголовков нет вообще; wildcard `*` + не поддерживается. +- **Rate-limit** — fixed-window 20rps/burst 40 на Redis (`REDIS_URL`, + атомарный Lua-скрипт): состояние общее для всех инстансов gateway. + Redis недоступен → fail-open (availability over strictness), warn в лог + не чаще раза в минуту. +- **XFF-доверие** — `X-Forwarded-For`/`X-Real-IP` учитываются только если + непосредственный пир в `TRUSTED_PROXY_CIDRS` (по умолчанию пусто: gateway — + edge, LB перед ним нет, клиентский XFF не доверяется никогда — ревью фазы 17, + M1); подделка XFF клиентом не влияет на ключ лимитера. Проверенный + клиентский IP выставляется в `X-Real-IP` для downstream — auth ключует свой + slowapi-лимитер по нему. + +## HLS-auth: metadata-кэш и общий http client (issue #54) + +`HLSAuth` больше не ходит в metadata на каждый `.ts`-сегмент: + +- **Один `http.Client`** (`hlsMetaClient`, timeout 3s, keep-alive пул) переиспользуется + и `HLSAuth`, и `HLSTokenHandler` — без нового клиента на каждый запрос. +- **LRU-кэш** `video_id → metadata` (4096 записей): позитивные ответы TTL 10s + (смена visibility видна в пределах 15s по ТЗ), негативные (metadata 404/403) + TTL 5s. Сеть/5xx от metadata не кэшируются. Кэш in-process, на инстанс middleware. +- Поведение authz не изменилось: private без валидного токена владельца → 403, + не найден в metadata → passthrough к vod (404), metadata недоступна → 502. + ## Zed IDE См. `services/metadata/README.md` — Go `gopls` автоформат при сохранении. + +## Общий Go-модуль pkg (issue #63) + +AuthMiddleware/OptionalAuth, RequestLogger, метрики Prometheus, `httputil.WriteJSON` и запуск сервера (таймауты Read/Write/Idle + graceful shutdown по SIGTERM/SIGINT) живут в общем модуле `pkg/` (`flowix/pkg`, подключён через `go.work` и `replace` в `go.mod`). Приватные копии в сервисе удалены — правки общих хелперов делай в `pkg/`, а не в сервисе. Таймауты и грейс shutdown: `pkg/httpserver` (env `SHUTDOWN_GRACE` переопределяет грейс, см. `stop_grace_period` в `deploy/docker-compose.yml`). diff --git a/services/gateway/cmd/server/main.go b/services/gateway/cmd/server/main.go index c343962..72c1da4 100644 --- a/services/gateway/cmd/server/main.go +++ b/services/gateway/cmd/server/main.go @@ -1,7 +1,7 @@ package main import ( - "encoding/json" + "net" "net/http" "net/url" "os" @@ -10,13 +10,17 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/rs/zerolog" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "flowix/gateway/internal/handler" - "flowix/gateway/internal/metrics" gwmw "flowix/gateway/internal/middleware" "flowix/gateway/internal/proxy" + pkghttp "flowix/pkg/httpserver" + "flowix/pkg/httputil" + pkglogger "flowix/pkg/logger" + "flowix/pkg/metrics" + pkgmw "flowix/pkg/middleware" ) type routerConfig struct { @@ -27,6 +31,10 @@ type routerConfig struct { metadataURL string uploadURL string vodURL string + // issue #52: CORS allowlist, trusted proxy CIDRs, Redis для rate-limit + corsOrigins []string + trusted []*net.IPNet + rdb *redis.Client } func main() { @@ -44,24 +52,31 @@ func main() { uploadURL := envOr("UPLOAD_URL", "http://upload:8003") vodURL := envOr("VOD_URL", "http://nginx-vod:80") + // issue #52: CORS — явный allowlist (пусто = никому, без wildcard), + // XFF доверяем только от trusted прокси, rate-limit в Redis + corsOrigins := parseCommaList(os.Getenv("CORS_ALLOWED_ORIGINS")) + // issue #52: пустой дефолт = не доверять ничьему XFF (gateway — edge, LB нет); + // если перед gateway появится LB — задать его CIDR в TRUSTED_PROXY_CIDRS + trusted, err := gwmw.ParseTrustedCIDRS(envOr("TRUSTED_PROXY_CIDRS", "")) + if err != nil { + log.Fatal().Err(err).Str("env", "TRUSTED_PROXY_CIDRS").Msg("invalid trusted proxy CIDR list") + } + var rdb *redis.Client + if redisURL := os.Getenv("REDIS_URL"); redisURL != "" { + opts, err := redis.ParseURL(redisURL) + if err != nil { + log.Fatal().Err(err).Str("env", "REDIS_URL").Msg("invalid redis url") + } + rdb = redis.NewClient(opts) + } + // also support legacy AUTH_URL without port fallback if authURL == "http://auth:8000" { authURL = "http://auth:8001" } - // zerolog console in dev, json in prod - if strings.ToLower(os.Getenv("LOG_FORMAT")) == "console" || os.Getenv("ENV") == "dev" { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}) - } else { - zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - } - zerolog.SetGlobalLevel(zerolog.InfoLevel) - if lvl := os.Getenv("LOG_LEVEL"); lvl != "" { - if l, err := zerolog.ParseLevel(lvl); err == nil { - zerolog.SetGlobalLevel(l) - } - } - logger := log.With().Str("service", "gateway").Logger() + // zerolog console in dev, json in prod — общий bootstrap (issue #63) + logger := pkglogger.Setup("gateway") r := newRouter(routerConfig{ jwtSecret: jwtSecret, @@ -71,8 +86,18 @@ func main() { metadataURL: metadataURL, uploadURL: uploadURL, vodURL: vodURL, + corsOrigins: corsOrigins, + trusted: trusted, + rdb: rdb, }) + if rdb == nil { + logger.Warn().Msg("REDIS_URL is empty — rate limit disabled (fail-open)") + } + if len(corsOrigins) == 0 { + logger.Warn().Msg("CORS_ALLOWED_ORIGINS is empty — no CORS headers will be sent") + } + logger.Info(). Str("port", port). Str("auth", authURL). @@ -81,7 +106,10 @@ func main() { Str("vod", vodURL). Msg("gateway starting") - if err := http.ListenAndServe(":"+port, r); err != nil { + // Streaming(): через gateway стримятся тела до 5 ГБ (upload-прокси) — + // лимитируем только заголовки, иначе Read/Write timeout обрывает загрузки. + // Run также делает graceful shutdown по SIGTERM/SIGINT (issue #63). + if err := pkghttp.Run(":"+port, r, pkghttp.Streaming()); err != nil { logger.Fatal().Err(err).Msg("gateway stopped") } } @@ -100,20 +128,23 @@ func newRouter(cfg routerConfig) *chi.Mux { r := chi.NewRouter() // базовые chi middleware r.Use(middleware.RequestID) - r.Use(middleware.RealIP) + // issue #52: вместо chi middleware.RealIP (слепо доверяет XFF) — trusted RealIP: + // RemoteAddr/X-Real-IP переписываются только от TRUSTED_PROXY_CIDRS + r.Use(gwmw.RealIP(cfg.trusted)) r.Use(middleware.Recoverer) // gateway middleware: CORS + RateLimit + RequestLogger (zerolog) - // CORS allow all for MVP (frontend :3000) - r.Use(gwmw.CORS([]string{"*"}, nil, nil)) - r.Use(gwmw.RateLimit(20, 40)) - r.Use(gwmw.RequestLogger) + // CORS — явный allowlist из CORS_ALLOWED_ORIGINS (issue #52, пусто = без CORS) + r.Use(gwmw.CORS(cfg.corsOrigins, nil, nil)) + // rate-limit — fixed-window в Redis, ключ по проверенному client IP (issue #52) + r.Use(gwmw.RateLimit(20, 40, cfg.rdb, cfg.trusted, log.With().Str("service", "gateway").Logger())) + r.Use(pkgmw.RequestLogger("gateway")) // health — без прокси, без rate-limit (rate-limit уже пропускает /health) r.Get("/health", healthHandler) r.Get("/healthz", healthHandler) r.Get("/metrics", metrics.Handler().ServeHTTP) r.Get("/", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"service": "gateway", "status": "ok"}) + httputil.WriteJSON(w, r, http.StatusOK, map[string]string{"service": "gateway", "status": "ok"}) }) // aggregated Swagger — единая точка на gateway, разделённая по сервисам (tags: auth/videos/upload) @@ -125,7 +156,7 @@ func newRouter(cfg routerConfig) *chi.Mux { r.Get("/openapi/metadata.json", docsH.HandleMetadataSpec) r.Get("/openapi/upload.json", docsH.HandleUploadSpec) - authMw := gwmw.AuthMiddleware(cfg.jwtSecret) + authMw := pkgmw.AuthMiddleware(cfg.jwtSecret) // --- Auth service: все /api/v1/auth/* публичные, без JWT --- // chi wildcard: /api/v1/auth/* захватывает /api/v1/auth/login etc. @@ -157,7 +188,7 @@ func newRouter(cfg routerConfig) *chi.Mux { // Публичные GET (лист и деталь) — без обязательного JWT, но с OptionalAuth: // валидный Bearer превращается в X-User-ID, чтобы metadata отдала приватные // видео владельцу (issue #44) - optAuth := gwmw.OptionalAuth(cfg.jwtSecret) + optAuth := pkgmw.OptionalAuth(cfg.jwtSecret) r.With(optAuth).Get("/api/v1/videos", metadataProxy.ServeHTTP) r.With(optAuth).Get("/api/v1/videos/*", metadataProxy.ServeHTTP) @@ -194,13 +225,7 @@ func newRouter(cfg routerConfig) *chi.Mux { } func healthHandler(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"status": "ok", "service": "gateway"}) -} - -func writeJSON(w http.ResponseWriter, status int, v any) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(v) + httputil.WriteJSON(w, r, http.StatusOK, map[string]string{"status": "ok", "service": "gateway"}) } func mustParseURL(s string) *url.URL { @@ -218,6 +243,18 @@ func envOr(k, def string) string { return def } +// parseCommaList — comma-separated env (CORS_ALLOWED_ORIGINS) → список, +// пустые элементы и пробелы отбрасываются. +func parseCommaList(s string) []string { + var out []string + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + // requireEnv reads a mandatory env var and exits with a clear error when it is // empty — unless ENV=dev (local `make dev-*` runs without .env, issue #53). func requireEnv(k string) string { diff --git a/services/gateway/go.mod b/services/gateway/go.mod index 44b61be..d288d80 100644 --- a/services/gateway/go.mod +++ b/services/gateway/go.mod @@ -3,11 +3,12 @@ module flowix/gateway go 1.25.0 require ( + flowix/pkg v0.0.0-00010101000000-000000000000 + github.com/alicebob/miniredis/v2 v2.39.0 github.com/go-chi/chi/v5 v5.2.1 github.com/golang-jwt/jwt/v5 v5.3.1 - github.com/prometheus/client_golang v1.22.0 + github.com/redis/go-redis/v9 v9.22.0 github.com/rs/zerolog v1.35.1 - golang.org/x/time v0.15.0 ) require ( @@ -16,9 +17,14 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.30.0 // indirect google.golang.org/protobuf v1.36.5 // indirect ) + +replace flowix/pkg => ../../pkg diff --git a/services/gateway/go.sum b/services/gateway/go.sum index fb306a9..6e12f4d 100644 --- a/services/gateway/go.sum +++ b/services/gateway/go.sum @@ -1,5 +1,11 @@ +github.com/alicebob/miniredis/v2 v2.39.0 h1:M7WbmV5BmV56L8KTG0rw6vEQ+woTOghpDgin2xv4A0g= +github.com/alicebob/miniredis/v2 v2.39.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -12,6 +18,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -30,15 +38,21 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/services/gateway/internal/middleware/clientip.go b/services/gateway/internal/middleware/clientip.go new file mode 100644 index 0000000..1f5bec5 --- /dev/null +++ b/services/gateway/internal/middleware/clientip.go @@ -0,0 +1,122 @@ +// Package middleware provides gateway-specific HTTP middleware: +// CORS allowlist, Redis rate limiting, trusted-proxy client IP, HLS auth. +package middleware + +import ( + "net" + "net/http" + "strings" +) + +// ParseTrustedCIDRS — парсит comma-separated список CIDR (env TRUSTED_PROXY_CIDRS, +// issue #52). X-Forwarded-For / X-Real-IP доверяем только если пирыый адрес +// запроса попадает в одну из этих сетей. +func ParseTrustedCIDRS(s string) ([]*net.IPNet, error) { + if strings.TrimSpace(s) == "" { + return nil, nil + } + var cidrs []*net.IPNet + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + // голый IP (например "10.0.0.1") трактуем как /32 или /128 + if !strings.Contains(part, "/") { + if ip := net.ParseIP(part); ip != nil && ip.To4() == nil { + part += "/128" + } else { + part += "/32" + } + } + _, n, err := net.ParseCIDR(part) + if err != nil { + return nil, err + } + cidrs = append(cidrs, n) + } + return cidrs, nil +} + +// ClientIP — реальный IP клиента (issue #52). Клиентский X-Forwarded-For +// подделываем: доверяем ему только когда непосредственный пир (r.RemoteAddr) +// из trusted CIDR (наш прокси/LB). Из XFF берём самый правый недоверенный IP +// (стандартный обход спуфинга через цепочку прокси). Иначе — RemoteAddr. +func ClientIP(r *http.Request, trusted []*net.IPNet) string { + peer := remoteHost(r) + if !isTrustedIP(peer, trusted) { + return peer + } + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + if ip := rightmostUntrusted(xff, trusted); ip != "" { + return ip + } + } + if xri := strings.TrimSpace(r.Header.Get("X-Real-IP")); xri != "" && net.ParseIP(xri) != nil { + return xri + } + return peer +} + +// RealIP — замена chi middleware.RealIP с проверкой доверия (issue #52): +// RemoteAddr переписывается по XFF/X-Real-IP только от trusted прокси, а +// проверенный клиентский IP всегда выставляется в X-Real-IP для downstream +// (auth ключует лимитер по нему) — клиентские подделки затираются. +func RealIP(trusted []*net.IPNet) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ip := ClientIP(r, trusted) + port := "0" + if _, p, err := net.SplitHostPort(r.RemoteAddr); err == nil && p != "" { + port = p + } + r.RemoteAddr = net.JoinHostPort(ip, port) + r.Header.Set("X-Real-IP", ip) + next.ServeHTTP(w, r) + }) + } +} + +// remoteHost — host-часть RemoteAddr (без порта). +func remoteHost(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +func isTrustedIP(ipStr string, trusted []*net.IPNet) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + for _, n := range trusted { + if n.Contains(ip) { + return true + } + } + return false +} + +// rightmostUntrusted — идём по XFF справа налево и возвращаем первый IP +// вне trusted CIDR (пропуская наши прокси, которые могли добавить свои хопы). +// Если все IP trusted — берём левый (ближайший к клиенту). +func rightmostUntrusted(xff string, trusted []*net.IPNet) string { + parts := strings.Split(xff, ",") + leftmost := "" + for i := len(parts) - 1; i >= 0; i-- { + p := strings.TrimSpace(parts[i]) + ip := net.ParseIP(p) + if ip == nil { + continue + } + if leftmost == "" { + leftmost = ip.String() + } + if !isTrustedIP(p, trusted) { + return ip.String() + } + } + return leftmost +} diff --git a/services/gateway/internal/middleware/clientip_test.go b/services/gateway/internal/middleware/clientip_test.go new file mode 100644 index 0000000..21883d6 --- /dev/null +++ b/services/gateway/internal/middleware/clientip_test.go @@ -0,0 +1,117 @@ +package middleware + +import ( + "net" + "net/http" + "net/http/httptest" + "testing" + + "github.com/rs/zerolog" +) + +func mustCIDRS(t *testing.T, s string) []*net.IPNet { + t.Helper() + cidrs, err := ParseTrustedCIDRS(s) + if err != nil { + t.Fatalf("ParseTrustedCIDRS(%q): %v", s, err) + } + return cidrs +} + +// issue #52: forged XFF от недоверенного пира не меняет client IP +func TestClientIPUntrustedPeerIgnoresXFF(t *testing.T) { + cidrs := mustCIDRS(t, "172.16.0.0/12") + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "5.5.5.5:1234" // пиер не из trusted сети + r.Header.Set("X-Forwarded-For", "1.2.3.4, 10.0.0.9") + r.Header.Set("X-Real-IP", "6.6.6.6") + if got := ClientIP(r, cidrs); got != "5.5.5.5" { + t.Fatalf("want peer 5.5.5.5, got %q", got) + } + // без trusted CIDR — тоже RemoteAddr + if got := ClientIP(r, nil); got != "5.5.5.5" { + t.Fatalf("want peer 5.5.5.5 with nil trusted, got %q", got) + } +} + +// issue #52: trusted proxy → берём самый правый недоверенный IP из XFF +func TestClientIPTrustedProxy(t *testing.T) { + cidrs := mustCIDRS(t, "172.16.0.0/12") + r := httptest.NewRequest("GET", "/", nil) + r.RemoteAddr = "172.18.0.5:443" // gateway за прокси в docker-сети + r.Header.Set("X-Forwarded-For", "1.2.3.4, 172.18.0.5") + if got := ClientIP(r, cidrs); got != "1.2.3.4" { + t.Fatalf("want 1.2.3.4, got %q", got) + } + // forged хвост (клиент сам подсунул 9.9.9.9, а прокси добавил реальный peer) + // → правый недоверенный всё равно 1.2.3.4? Нет: XFF = "1.2.3.4, 9.9.9.9" + // от trusted прокси означает что 9.9.9.9 видел прокси — доверяем правому. + r2 := httptest.NewRequest("GET", "/", nil) + r2.RemoteAddr = "172.18.0.5:443" + r2.Header.Set("X-Forwarded-For", "1.2.3.4, 9.9.9.9") + if got := ClientIP(r2, cidrs); got != "9.9.9.9" { + t.Fatalf("rightmost untrusted wins, want 9.9.9.9 got %q", got) + } + // X-Real-IP от trusted прокси без XFF + r3 := httptest.NewRequest("GET", "/", nil) + r3.RemoteAddr = "172.18.0.5:443" + r3.Header.Set("X-Real-IP", "7.7.7.7") + if got := ClientIP(r3, cidrs); got != "7.7.7.7" { + t.Fatalf("want 7.7.7.7 got %q", got) + } +} + +// issue #52: RealIP middleware переписывает RemoteAddr + выставляет проверенный X-Real-IP +func TestRealIPTrustedMiddleware(t *testing.T) { + cidrs := mustCIDRS(t, "172.16.0.0/12") + h := RealIP(cidrs)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if host := remoteHost(r); host != "1.2.3.4" { + t.Errorf("want RemoteAddr host 1.2.3.4 got %q", host) + } + if got := r.Header.Get("X-Real-IP"); got != "1.2.3.4" { + t.Errorf("want X-Real-IP 1.2.3.4 got %q", got) + } + })) + req := httptest.NewRequest("GET", "/", nil) + req.RemoteAddr = "172.18.0.5:5555" + req.Header.Set("X-Forwarded-For", "1.2.3.4") + h.ServeHTTP(httptest.NewRecorder(), req) +} + +// issue #52: недоверенный клиент не может подделать X-Real-IP — затирается +func TestRealIPUntrustedOverwritesForgedHeader(t *testing.T) { + h := RealIP(nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Real-IP"); got != "8.8.8.8" { + t.Errorf("forged X-Real-IP must be overwritten with peer, got %q", got) + } + })) + req := httptest.NewRequest("GET", "/", nil) + req.RemoteAddr = "8.8.8.8:9999" + req.Header.Set("X-Real-IP", "1.1.1.1") + req.Header.Set("X-Forwarded-For", "1.1.1.1") + h.ServeHTTP(httptest.NewRecorder(), req) +} + +// issue #52 (verify): подделка XFF не обходит лимит — два «разных» forged IP +// с одного недоверенного пира делят один бакет +func TestRateLimitForgedXFFSharesLimit(t *testing.T) { + rdb, _ := newTestRedis(t) + cidrs := mustCIDRS(t, "172.16.0.0/12") + h := RateLimit(1, 2, rdb, cidrs, zerolog.Nop())(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + // один и тот же peer 5.5.5.5 (не trusted), «разные» forged XFF + for i, xff := range []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} { + req := httptest.NewRequest("GET", "/api/v1/videos", nil) + req.RemoteAddr = "5.5.5.5:1000" + req.Header.Set("X-Forwarded-For", xff) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if i < 2 && w.Code != 200 { + t.Fatalf("iter %d want 200 (burst 2), got %d", i, w.Code) + } + if i == 2 && w.Code != 429 { + t.Fatalf("forged XFF must not bypass limit, want 429 got %d", w.Code) + } + } +} diff --git a/services/gateway/internal/middleware/cors.go b/services/gateway/internal/middleware/cors.go index c7afa91..f2c7369 100644 --- a/services/gateway/internal/middleware/cors.go +++ b/services/gateway/internal/middleware/cors.go @@ -4,8 +4,11 @@ import "net/http" // CORS возвращает middleware, который добавляет CORS-заголовки и обрабатывает preflight. // -// Логика: проверяем origin — если задан список allowedOrigins, то только они -// (или "*" для любого). Иначе отражаем origin запроса — простой подход для MVP. +// Логика (issue #52): только явный allowlist origin (CORS_ALLOWED_ORIGINS) — +// wildcard «*» вместе с Allow-Credentials запрещён. Origin из списка → +// отражаем его в Access-Control-Allow-Origin (+ Vary: Origin). Чужой origin → +// вообще без ACAO-заголовков. Пустой список → CORS-заголовков нет никому +// (safe default, без wildcard). func CORS(allowedOrigins []string, allowedMethods []string, allowedHeaders []string) func(http.Handler) http.Handler { if len(allowedMethods) == 0 { allowedMethods = []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"} @@ -13,11 +16,12 @@ func CORS(allowedOrigins []string, allowedMethods []string, allowedHeaders []str if len(allowedHeaders) == 0 { allowedHeaders = []string{"Authorization", "Content-Type", "Accept", "Origin", "X-Requested-With", "Range"} } - allowAll := len(allowedOrigins) == 0 + // точное совпадение origin; wildcard "*" не поддерживается — + // пустой/чужой список = без ACAO + allowed := make(map[string]struct{}, len(allowedOrigins)) for _, o := range allowedOrigins { - if o == "*" { - allowAll = true - break + if o != "" && o != "*" { + allowed[o] = struct{}{} } } // precompute header values @@ -39,26 +43,17 @@ func CORS(allowedOrigins []string, allowedMethods []string, allowedHeaders []str return } - allowed := allowAll - if !allowAll { - for _, o := range allowedOrigins { - if o == origin { - allowed = true - break - } - } - } - if allowed { + // для кэша вариантность нужна в обоих случаях (в т.ч. при отказе) + w.Header().Set("Vary", "Origin") + if _, ok := allowed[origin]; ok { w.Header().Set("Access-Control-Allow-Origin", origin) - // для кэша вариантности - w.Header().Set("Vary", "Origin") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Allow-Methods", methodsVal) + w.Header().Set("Access-Control-Allow-Headers", headersVal) + w.Header().Set("Access-Control-Max-Age", "86400") + // expose useful headers for hls / upload + w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range, Content-Type, X-Request-Id") } - w.Header().Set("Access-Control-Allow-Methods", methodsVal) - w.Header().Set("Access-Control-Allow-Headers", headersVal) - w.Header().Set("Access-Control-Allow-Credentials", "true") - w.Header().Set("Access-Control-Max-Age", "86400") - // expose useful headers for hls / upload - w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range, Content-Type, X-Request-Id") if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) diff --git a/services/gateway/internal/middleware/cors_test.go b/services/gateway/internal/middleware/cors_test.go index 9c841fc..bf3221e 100644 --- a/services/gateway/internal/middleware/cors_test.go +++ b/services/gateway/internal/middleware/cors_test.go @@ -7,7 +7,7 @@ import ( ) func TestCORSPreflight(t *testing.T) { - h := CORS([]string{"*"}, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := CORS([]string{"http://localhost:3000"}, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("should not call next for OPTIONS") })) req := httptest.NewRequest("OPTIONS", "/api/v1/videos", nil) @@ -27,19 +27,26 @@ func TestCORSPreflight(t *testing.T) { } func TestCORSSimple(t *testing.T) { - h := CORS([]string{"*"}, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := CORS([]string{"http://localhost:3000"}, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })) req := httptest.NewRequest("GET", "/api/v1/videos", nil) - req.Header.Set("Origin", "http://example.com") + req.Header.Set("Origin", "http://localhost:3000") w := httptest.NewRecorder() h.ServeHTTP(w, req) if w.Code != 200 { t.Fatalf("want 200 got %d", w.Code) } - if got := w.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" { + // issue #52: отражаем конкретный allowlisted origin, не wildcard + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "http://localhost:3000" { t.Fatalf("want origin got %q", got) } + if got := w.Header().Get("Vary"); got != "Origin" { + t.Fatalf("want Vary: Origin got %q", got) + } + if got := w.Header().Get("Access-Control-Allow-Credentials"); got != "true" { + t.Fatalf("credentials should stay enabled for allowed origin, got %q", got) + } } func TestCORSAllowedOriginsFilter(t *testing.T) { @@ -54,3 +61,34 @@ func TestCORSAllowedOriginsFilter(t *testing.T) { t.Fatalf("evil origin should not be allowed got %q", got) } } + +// issue #52: wildcard "*" больше не разрешает любой origin с credentials +func TestCORSWildcardRejected(t *testing.T) { + h := CORS([]string{"*"}, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("Origin", "http://evil.com") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "" { + t.Fatalf("wildcard must be ignored, got ACAO %q", got) + } + if got := w.Header().Get("Access-Control-Allow-Credentials"); got != "" { + t.Fatalf("credentials must not be set for unknown origin, got %q", got) + } +} + +// issue #52: пустой allowlist — safe default, никому не отдаём ACAO +func TestCORSEmptyAllowlist(t *testing.T) { + h := CORS(nil, nil, nil)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("Origin", "http://localhost:3000") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "" { + t.Fatalf("empty allowlist must not produce ACAO, got %q", got) + } +} diff --git a/services/gateway/internal/middleware/hlsauth.go b/services/gateway/internal/middleware/hlsauth.go index 142f1f6..d9819e8 100644 --- a/services/gateway/internal/middleware/hlsauth.go +++ b/services/gateway/internal/middleware/hlsauth.go @@ -1,15 +1,20 @@ package middleware import ( + "container/list" "encoding/json" + "errors" "fmt" "net/http" "regexp" "strings" + "sync" "time" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + + pkgmw "flowix/pkg/middleware" ) var hlsPathRe = regexp.MustCompile(`/hls/([0-9a-fA-F-]{36})`) @@ -95,8 +100,39 @@ type videoMeta struct { Status string `json:"status"` } +// hlsMetaClient is shared by all metadata calls (issue #54): one client with +// connection pooling instead of a new http.Client per segment request. +var hlsMetaClient = &http.Client{ + Timeout: 3 * time.Second, + Transport: func() http.RoundTripper { + t := http.DefaultTransport.(*http.Transport).Clone() + t.MaxIdleConns = 100 + t.MaxIdleConnsPerHost = 32 + t.IdleConnTimeout = 60 * time.Second + return t + }(), +} + +// statusError is a non-200 response from the metadata internal endpoint. +type statusError struct { + status int +} + +func (e *statusError) Error() string { return fmt.Sprintf("metadata status %d", e.status) } + +// TTLs are vars so tests can shorten them; positive TTL ≤15s keeps a +// visibility change observable within that window (issue #54). +var ( + metaCacheTTL = 10 * time.Second + metaNegCacheTTL = 5 * time.Second +) + +// metaCacheMaxEntries bounds the LRU (a few thousand videos is plenty for one +// gateway instance). +const metaCacheMaxEntries = 4096 + // fetchVideoMeta calls metadata internal endpoint to get visibility/owner. -func fetchVideoMeta(metadataURL, internalToken, videoID string) (*videoMeta, error) { +func fetchVideoMeta(client *http.Client, metadataURL, internalToken, videoID string) (*videoMeta, error) { url := strings.TrimSuffix(metadataURL, "/") + "/internal/videos/" + videoID req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -105,14 +141,13 @@ func fetchVideoMeta(metadataURL, internalToken, videoID string) (*videoMeta, err if internalToken != "" { req.Header.Set("X-Internal-Token", internalToken) } - client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Do(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("metadata status %d", resp.StatusCode) + return nil, &statusError{status: resp.StatusCode} } var v videoMeta if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { @@ -124,8 +159,99 @@ func fetchVideoMeta(metadataURL, internalToken, videoID string) (*videoMeta, err return &v, nil } +// metaCache is a concurrency-safe bounded LRU with per-entry TTL. +// A positive entry caches the metadata result; a negative entry (meta == nil) +// caches a metadata 404/403 for the shorter negTTL (issue #54). +type metaCache struct { + mu sync.Mutex + max int + ttl time.Duration + negTTL time.Duration + entries map[string]*list.Element + lru *list.List // front = most recently used +} + +type metaCacheEntry struct { + key string + meta *videoMeta // nil = negative entry + errStatus int // metadata status for negative entries + expires time.Time +} + +func newMetaCache(max int, ttl, negTTL time.Duration) *metaCache { + return &metaCache{ + max: max, + ttl: ttl, + negTTL: negTTL, + entries: make(map[string]*list.Element, max), + lru: list.New(), + } +} + +func (c *metaCache) get(videoID string) (metaCacheEntry, bool) { + c.mu.Lock() + defer c.mu.Unlock() + el, ok := c.entries[videoID] + if !ok { + return metaCacheEntry{}, false + } + e := el.Value.(*metaCacheEntry) + if time.Now().After(e.expires) { + c.lru.Remove(el) + delete(c.entries, videoID) + return metaCacheEntry{}, false + } + c.lru.MoveToFront(el) + // return a copy: put() may refresh the stored entry concurrently + return *e, true +} + +func (c *metaCache) put(videoID string, meta *videoMeta, errStatus int) { + c.mu.Lock() + defer c.mu.Unlock() + ttl := c.ttl + if errStatus != 0 { + ttl = c.negTTL + } + if el, ok := c.entries[videoID]; ok { + e := el.Value.(*metaCacheEntry) + e.meta, e.errStatus, e.expires = meta, errStatus, time.Now().Add(ttl) + c.lru.MoveToFront(el) + return + } + c.entries[videoID] = c.lru.PushFront(&metaCacheEntry{ + key: videoID, meta: meta, errStatus: errStatus, expires: time.Now().Add(ttl), + }) + if len(c.entries) > c.max { + if back := c.lru.Back(); back != nil { + delete(c.entries, back.Value.(*metaCacheEntry).key) + c.lru.Remove(back) + } + } +} + // HLSAuth protects /hls/* : private videos require owner JWT or hls signed token. +// Metadata lookups go through an LRU cache (TTL 10s, negative 404/403 5s, issue #54): +// one playback issues 100+ segment requests, they must not each hit metadata. func HLSAuth(jwtSecret, internalToken, metadataURL string) func(http.Handler) http.Handler { + cache := newMetaCache(metaCacheMaxEntries, metaCacheTTL, metaNegCacheTTL) + getMeta := func(videoID string) (*videoMeta, error) { + if e, ok := cache.get(videoID); ok { + if e.meta != nil { + return e.meta, nil + } + return nil, &statusError{status: e.errStatus} + } + meta, err := fetchVideoMeta(hlsMetaClient, metadataURL, internalToken, videoID) + var se *statusError + switch { + case err == nil: + cache.put(videoID, meta, 0) + case errors.As(err, &se) && (se.status == http.StatusNotFound || se.status == http.StatusForbidden): + cache.put(videoID, nil, se.status) // negative cache + } + return meta, err + } return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { videoID := extractVideoID(r.URL.Path) @@ -133,12 +259,13 @@ func HLSAuth(jwtSecret, internalToken, metadataURL string) func(http.Handler) ht next.ServeHTTP(w, r) return } - meta, err := fetchVideoMeta(metadataURL, internalToken, videoID) + meta, err := getMeta(videoID) if err != nil { // If video not found, let vod return 404; if fetch fails, propagate 502 // To avoid leaking existence, treat not-found as 404 from vod. // If metadata unavailable, return 502. - if strings.Contains(err.Error(), "404") { + var se *statusError + if errors.As(err, &se) && se.status == http.StatusNotFound { next.ServeHTTP(w, r) return } @@ -230,7 +357,7 @@ func HLSTokenHandler(jwtSecret, internalToken, metadataURL string) http.HandlerF } } } - userID := UserIDFromCtx(r.Context()) + userID := pkgmw.UserIDFromCtx(r.Context()) if userID == "" { // try via Authorization parsing as fallback (if middleware not applied) if sub, ok := validateAccessToken(r, jwtSecret); ok { @@ -245,7 +372,7 @@ func HLSTokenHandler(jwtSecret, internalToken, metadataURL string) http.HandlerF http.Error(w, `{"error":"missing video id"}`, http.StatusBadRequest) return } - meta, err := fetchVideoMeta(metadataURL, internalToken, videoID) + meta, err := fetchVideoMeta(hlsMetaClient, metadataURL, internalToken, videoID) if err != nil { http.Error(w, `{"error":"not found"}`, http.StatusNotFound) return diff --git a/services/gateway/internal/middleware/hlsauth_test.go b/services/gateway/internal/middleware/hlsauth_test.go new file mode 100644 index 0000000..53b4ed6 --- /dev/null +++ b/services/gateway/internal/middleware/hlsauth_test.go @@ -0,0 +1,182 @@ +package middleware + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +const testVideoID = "00000000-0000-0000-0000-000000000001" + +// shortenMetaCacheTTLs shrinks the cache TTLs for the test duration. +func shortenMetaCacheTTLs(t *testing.T) { + t.Helper() + oldTTL, oldNeg := metaCacheTTL, metaNegCacheTTL + metaCacheTTL, metaNegCacheTTL = 20*time.Millisecond, 10*time.Millisecond + t.Cleanup(func() { metaCacheTTL, metaNegCacheTTL = oldTTL, oldNeg }) +} + +// newMetaStub starts a metadata stub returning the given status/visibility +// and counting internal video requests. +func newMetaStub(t *testing.T, status int, visibility func() string) (*httptest.Server, *int32) { + t.Helper() + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + if status != http.StatusOK { + w.WriteHeader(status) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{ + "id": testVideoID, + "owner_id": "owner-1", + "visibility": visibility(), + "status": "ready", + }) + })) + t.Cleanup(srv.Close) + return srv, &calls +} + +func hlsRequest(t *testing.T, h http.Handler, path string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + return w +} + +func nextHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) +} + +// (a) 200 authz checks for the same video → ~1-2 metadata HTTP requests. +func TestHLSAuthCachesMetadataLookups(t *testing.T) { + srv, calls := newMetaStub(t, http.StatusOK, func() string { return "public" }) + h := HLSAuth("secret", "", srv.URL)(nextHandler()) + + for i := 0; i < 200; i++ { + w := hlsRequest(t, h, "/hls/"+testVideoID+"/seg-1.ts") + if w.Code != http.StatusOK { + t.Fatalf("req %d: want 200 got %d", i, w.Code) + } + } + if got := atomic.LoadInt32(calls); got > 2 { + t.Fatalf("want ≤2 metadata requests for 200 checks, got %d", got) + } + if got := atomic.LoadInt32(calls); got < 1 { + t.Fatal("expected at least one metadata request") + } +} + +// (b) visibility change at metadata is observed within the cache TTL. +func TestHLSAuthVisibilityChangeObservedWithinTTL(t *testing.T) { + shortenMetaCacheTTLs(t) + var visibility atomic.Value + visibility.Store("private") + srv, calls := newMetaStub(t, http.StatusOK, func() string { + return visibility.Load().(string) + }) + h := HLSAuth("secret", "", srv.URL)(nextHandler()) + path := "/hls/" + testVideoID + "/master.m3u8" + + // private without token → 403 + if w := hlsRequest(t, h, path); w.Code != http.StatusForbidden { + t.Fatalf("want 403 for private video, got %d", w.Code) + } + // flip to public: still cached as private within TTL → 403 + visibility.Store("public") + if w := hlsRequest(t, h, path); w.Code != http.StatusForbidden { + t.Fatalf("want cached 403 within TTL, got %d", w.Code) + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("want 1 metadata request before TTL expiry, got %d", got) + } + // after TTL the change is visible → 200 through to vod + time.Sleep(40 * time.Millisecond) + w := hlsRequest(t, h, path) + if w.Code != http.StatusOK { + t.Fatalf("want 200 after visibility change, got %d", w.Code) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("want 2 metadata requests after TTL expiry, got %d", got) + } +} + +// (c) a 404 from metadata is negatively cached for the short TTL. +func TestHLSAuthNegativeCache404(t *testing.T) { + shortenMetaCacheTTLs(t) + srv, calls := newMetaStub(t, http.StatusNotFound, nil) + var next int32 + h := HLSAuth("secret", "", srv.URL)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&next, 1) + w.WriteHeader(http.StatusNotFound) // vod would 404 + })) + + for i := 0; i < 10; i++ { + if w := hlsRequest(t, h, "/hls/"+testVideoID+"/seg.ts"); w.Code != http.StatusNotFound { + t.Fatalf("req %d: want 404 passthrough, got %d", i, w.Code) + } + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("want 1 metadata request (negatively cached), got %d", got) + } + // after the negative TTL a fresh lookup happens + time.Sleep(30 * time.Millisecond) + if w := hlsRequest(t, h, "/hls/"+testVideoID+"/seg.ts"); w.Code != http.StatusNotFound { + t.Fatalf("want 404 passthrough after TTL, got %d", w.Code) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("want 2 metadata requests after neg TTL, got %d", got) + } +} + +// (d) concurrent access is race-free (run with -race). +func TestHLSAuthConcurrentAccess(t *testing.T) { + srv, calls := newMetaStub(t, http.StatusOK, func() string { return "public" }) + h := HLSAuth("secret", "", srv.URL)(nextHandler()) + + var wg sync.WaitGroup + for g := 0; g < 32; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 25; i++ { + w := hlsRequest(t, h, fmt.Sprintf("/hls/%s/seg-%d-%d.ts", testVideoID, g, i)) + if w.Code != http.StatusOK { + t.Errorf("want 200 got %d", w.Code) + return + } + } + }() + } + wg.Wait() + if got := atomic.LoadInt32(calls); got < 1 { + t.Fatal("expected at least one metadata request") + } +} + +// authz logic unchanged: private video allows only the owner (hls token or bearer). +func TestHLSAuthPrivateVideoAuthz(t *testing.T) { + srv, _ := newMetaStub(t, http.StatusOK, func() string { return "private" }) + h := HLSAuth("secret", "", srv.URL)(nextHandler()) + path := "/hls/" + testVideoID + "/seg.ts" + + if w := hlsRequest(t, h, path); w.Code != http.StatusForbidden { + t.Fatalf("want 403 without token, got %d", w.Code) + } + tok, err := GenerateHLSToken(testVideoID, "owner-1", "secret") + if err != nil { + t.Fatal(err) + } + if w := hlsRequest(t, h, path+"?token="+tok); w.Code != http.StatusOK { + t.Fatalf("want 200 with owner hls token, got %d", w.Code) + } +} diff --git a/services/gateway/internal/middleware/logger.go b/services/gateway/internal/middleware/logger.go deleted file mode 100644 index 54d09e6..0000000 --- a/services/gateway/internal/middleware/logger.go +++ /dev/null @@ -1,61 +0,0 @@ -package middleware - -import ( - "net/http" - "time" - - chimw "github.com/go-chi/chi/v5/middleware" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -// RequestLogger — structured logging через zerolog. Логирует метод, путь, -// статус, длительность и IP. Ошибки >=500 — уровнем Error. -func RequestLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - // chi RequestID кладёт id только в context, не в r.Header и не - // в response. Прокидываем в header до обработки, чтобы proxy.New - // передал его во все апстримы, и в response — чтобы клиент мог - // скопировать id для поиска в Grafana. - reqID := chimw.GetReqID(r.Context()) - if reqID == "" { - reqID = r.Header.Get("X-Request-ID") - } - if reqID != "" { - r.Header.Set("X-Request-ID", reqID) - w.Header().Set("X-Request-Id", reqID) - } - - ww := &respWriter{ResponseWriter: w, status: http.StatusOK} - next.ServeHTTP(ww, r) - dur := time.Since(start) - - ev := log.Info() - if ww.status >= 500 { - ev = log.Error() - } else if ww.status >= 400 { - ev = log.Warn() - } - ev.Str("method", r.Method). - Str("path", r.URL.Path). - Int("status", ww.status). - Dur("duration", dur). - Str("ip", clientIP(r)). - Str("trace_id", reqID). - Str("service", "gateway"). - Msg("request") - // также zerolog global logger доступен как zerolog.Ctx - _ = zerolog.Ctx(r.Context()) - }) -} - -type respWriter struct { - http.ResponseWriter - status int -} - -func (w *respWriter) WriteHeader(code int) { - w.status = code - w.ResponseWriter.WriteHeader(code) -} diff --git a/services/gateway/internal/middleware/ratelimit.go b/services/gateway/internal/middleware/ratelimit.go index cbd2808..94685d3 100644 --- a/services/gateway/internal/middleware/ratelimit.go +++ b/services/gateway/internal/middleware/ratelimit.go @@ -1,60 +1,67 @@ package middleware import ( + "context" "net" "net/http" + "strconv" "sync" "time" - "golang.org/x/time/rate" + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog" ) -// RateLimit — per-IP токен-бакет. Параметры: +// rateLimitScript — атомарный fixed-window (issue #52): INCR + PEXPIRE в одном +// EVAL, без гонок между инстансами gateway. KEYS[1] = bucket key, +// ARGV[1] = window ms, ARGV[2] = limit. Возвращает 1 (пропустить) / 0 (лимит). +var rateLimitScript = redis.NewScript(` +local cnt = redis.call('INCR', KEYS[1]) +if cnt == 1 then + redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[1])) +end +if cnt > tonumber(ARGV[2]) then + return 0 +end +return 1 +`) + +// RateLimit — per-IP fixed-window лимитер на Redis (issue #52): состояние +// общее для всех инстансов gateway, ключ — реальный IP клиента (XFF доверяем +// только от trusted прокси, см. ClientIP). // -// rps — запросов в секунду (rate) -// burst — размер бакета +// Параметры: // -// Для gateway дефолт 20 rps burst 40 достаточно для MVP и hls сегментов -// (браузер грузит ~1 сегмент / 2 сек + метаданные). -func RateLimit(rps int, burst int) func(http.Handler) http.Handler { +// rps — запросов в секунду (sustained) +// burst — максимум за окно (окно = max(1s, burst/rps), т.е. ~40 запросов +// за 2с при 20rps/40burst — как прежний токен-бакет) +// rdb — Redis-клиент; nil или недоступный Redis → fail-open +// (availability over strictness), с warn в лог не чаще раза в минуту +func RateLimit(rps int, burst int, rdb *redis.Client, trusted []*net.IPNet, logger zerolog.Logger) func(http.Handler) http.Handler { if rps <= 0 { rps = 20 } if burst <= 0 { burst = 40 } - type entry struct { - limiter *rate.Limiter - lastSeen time.Time + window := time.Duration(burst/rps) * time.Second + if window < time.Second { + window = time.Second } + windowMs := strconv.Itoa(int(window.Milliseconds())) + limit := strconv.Itoa(burst) + var ( mu sync.Mutex - visitors = make(map[string]*entry) + lastWarn time.Time ) - // фоновая очистка старых лимитеров (раз в минуту — не более ~10k IP) - go func() { - ticker := time.NewTicker(time.Minute) - for range ticker.C { - mu.Lock() - for ip, e := range visitors { - if time.Since(e.lastSeen) > 3*time.Minute { - delete(visitors, ip) - } - } - mu.Unlock() - } - }() - - getLimiter := func(ip string) *rate.Limiter { + warnOpen := func(err error) { mu.Lock() defer mu.Unlock() - if e, ok := visitors[ip]; ok { - e.lastSeen = time.Now() - return e.limiter + if time.Since(lastWarn) > time.Minute { + lastWarn = time.Now() + logger.Warn().Err(err).Msg("redis rate limiter unavailable — failing open") } - lim := rate.NewLimiter(rate.Limit(rps), burst) - visitors[ip] = &entry{limiter: lim, lastSeen: time.Now()} - return lim } return func(next http.Handler) http.Handler { @@ -64,50 +71,21 @@ func RateLimit(rps int, burst int) func(http.Handler) http.Handler { next.ServeHTTP(w, r) return } - ip := clientIP(r) - lim := getLimiter(ip) - if !lim.Allow() { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Retry-After", "1") - http.Error(w, `{"error":"too many requests"}`, http.StatusTooManyRequests) - return + if rdb != nil { + ip := ClientIP(r, trusted) + bucket := time.Now().UnixMilli() / window.Milliseconds() + key := "ratelimit:" + ip + ":" + strconv.FormatInt(bucket, 10) + allowed, err := rateLimitScript.Run(context.Background(), rdb, []string{key}, windowMs, limit).Int() + if err != nil { + warnOpen(err) + } else if allowed == 0 { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Retry-After", "1") + http.Error(w, `{"error":"too many requests"}`, http.StatusTooManyRequests) + return + } } next.ServeHTTP(w, r) }) } } - -func clientIP(r *http.Request) string { - // учитываем X-Forwarded-For (когда gateway за CDN / LB) - if xff := r.Header.Get("X-Forwarded-For"); xff != "" { - // берём первый IP - for i, c := range xff { - if c == ',' { - return trimSpace(xff[:i]) - } - } - return trimSpace(xff) - } - if xri := r.Header.Get("X-Real-IP"); xri != "" { - return xri - } - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return r.RemoteAddr - } - return host -} - -func trimSpace(s string) string { - // inline strings.TrimSpace без импорта strings для одного места, - // но проще импорт — оставим ручную версию для отсутствия extra import - start := 0 - for start < len(s) && (s[start] == ' ' || s[start] == '\t') { - start++ - } - end := len(s) - for end > start && (s[end-1] == ' ' || s[end-1] == '\t') { - end-- - } - return s[start:end] -} diff --git a/services/gateway/internal/middleware/ratelimit_test.go b/services/gateway/internal/middleware/ratelimit_test.go index 716e865..210e6c5 100644 --- a/services/gateway/internal/middleware/ratelimit_test.go +++ b/services/gateway/internal/middleware/ratelimit_test.go @@ -4,13 +4,26 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog" ) +func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) { + t.Helper() + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = rdb.Close() }) + return rdb, mr +} + func TestRateLimitAllowsThenBlocks(t *testing.T) { - h := RateLimit(2, 2)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rdb, _ := newTestRedis(t) + // 2 burst → окно 1s, первые 2 проходят, 3-й — 429 + h := RateLimit(2, 2, rdb, nil, zerolog.Nop())(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })) - // 2 burst → первые 2 должны пройти, 3-й — 429 for i := 0; i < 2; i++ { req := httptest.NewRequest("GET", "/api/v1/videos", nil) req.RemoteAddr = "1.2.3.4:1234" @@ -30,7 +43,8 @@ func TestRateLimitAllowsThenBlocks(t *testing.T) { } func TestRateLimitBypassesHealth(t *testing.T) { - h := RateLimit(1, 1)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rdb, _ := newTestRedis(t) + h := RateLimit(1, 1, rdb, nil, zerolog.Nop())(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })) // исчерпываем лимит @@ -49,3 +63,52 @@ func TestRateLimitBypassesHealth(t *testing.T) { t.Fatalf("health should bypass rate limit got %d", w.Code) } } + +// issue #52: разные IP лимитируются независимо (state в Redis, ключ по ClientIP) +func TestRateLimitPerIP(t *testing.T) { + rdb, _ := newTestRedis(t) + h := RateLimit(1, 1, rdb, nil, zerolog.Nop())(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + req1 := httptest.NewRequest("GET", "/api/v1/videos", nil) + req1.RemoteAddr = "1.1.1.1:1000" + w1 := httptest.NewRecorder() + h.ServeHTTP(w1, req1) + if w1.Code != 200 { + t.Fatalf("first ip want 200 got %d", w1.Code) + } + // другой IP — свой бакет, не 429 + req2 := httptest.NewRequest("GET", "/api/v1/videos", nil) + req2.RemoteAddr = "2.2.2.2:1000" + w2 := httptest.NewRecorder() + h.ServeHTTP(w2, req2) + if w2.Code != 200 { + t.Fatalf("second ip should have own bucket got %d", w2.Code) + } +} + +// issue #52: Redis недоступен → fail-open (availability over strictness) +func TestRateLimitFailOpenWhenRedisDown(t *testing.T) { + mr := miniredis.RunT(t) + rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + _ = rdb.Close() // соединение рвём — все EVAL будут падать + mr.Close() + + called := false + h := RateLimit(1, 1, rdb, nil, zerolog.Nop())(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(200) + })) + for i := 0; i < 5; i++ { + req := httptest.NewRequest("GET", "/api/v1/videos", nil) + req.RemoteAddr = "3.3.3.3:1000" + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("redis down must fail-open, iter %d got %d", i, w.Code) + } + } + if !called { + t.Fatal("handler must be reached when redis is down") + } +} diff --git a/services/metadata/Dockerfile b/services/metadata/Dockerfile index ce865b0..86713e0 100644 --- a/services/metadata/Dockerfile +++ b/services/metadata/Dockerfile @@ -1,8 +1,14 @@ +# Issue #63: build context = repo root (см. deploy/docker-compose.yml) — +# рядом копируем общий модуль pkg/. go.work в образ не попадает: сборка +# идёт per-module через replace flowix/pkg => ../../pkg. FROM golang:1.27-alpine AS builder WORKDIR /app -COPY go.mod go.sum* ./ -RUN go mod download || true -COPY . . +COPY pkg/go.mod pkg/go.sum pkg/ +COPY services/metadata/go.mod services/metadata/go.sum services/metadata/ +RUN cd /app/pkg && go mod download && cd /app/services/metadata && go mod download +COPY pkg/ pkg/ +COPY services/metadata/ services/metadata/ +WORKDIR /app/services/metadata RUN CGO_ENABLED=0 go build -o /server ./cmd/server FROM alpine:3.19 diff --git a/services/metadata/README.md b/services/metadata/README.md index a2ec45b..7eb8f57 100644 --- a/services/metadata/README.md +++ b/services/metadata/README.md @@ -64,3 +64,7 @@ docker run --rm -v $PWD:/app -w /app golang:1.27-alpine go test ./services/metad - При `Cmd+S`: `gofmt` + `goimports` (импорты сортируются), `staticcheck` подсвечивает - Рефактор: `F2` (rename), `Cmd+Click` (go to def), `Cmd+.` (fix) - Если не форматирует: `Cmd+Shift+P` → `editor: restart language server`, проверь `gopls` установлен (`go install golang.org/x/tools/gopls@latest`) + +## Общий Go-модуль pkg (issue #63) + +AuthMiddleware/OptionalAuth, RequestLogger, метрики Prometheus, `httputil.WriteJSON` и запуск сервера (таймауты Read/Write/Idle + graceful shutdown по SIGTERM/SIGINT) живут в общем модуле `pkg/` (`flowix/pkg`, подключён через `go.work` и `replace` в `go.mod`). Приватные копии в сервисе удалены — правки общих хелперов делай в `pkg/`, а не в сервисе. Таймауты и грейс shutdown: `pkg/httpserver` (env `SHUTDOWN_GRACE` переопределяет грейс, см. `stop_grace_period` в `deploy/docker-compose.yml`). diff --git a/services/metadata/cmd/server/main.go b/services/metadata/cmd/server/main.go index ed45097..a7c931a 100644 --- a/services/metadata/cmd/server/main.go +++ b/services/metadata/cmd/server/main.go @@ -10,9 +10,7 @@ package main // @description Type "Bearer {token}" for JWT import ( "context" - "encoding/json" "log" - "log/slog" "net/http" "net/url" "os" @@ -21,16 +19,18 @@ import ( _ "flowix/metadata/docs" // generated by swag init "flowix/metadata/internal/handler" - "flowix/metadata/internal/metrics" mw "flowix/metadata/internal/middleware" "flowix/metadata/internal/repository" "flowix/metadata/internal/storage" + pkghttp "flowix/pkg/httpserver" + "flowix/pkg/httputil" + pkglogger "flowix/pkg/logger" + "flowix/pkg/metrics" + pkgmw "flowix/pkg/middleware" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" httpSwagger "github.com/swaggo/http-swagger" ) @@ -66,18 +66,8 @@ func main() { minioAccess := os.Getenv("MINIO_ACCESS_KEY") minioSecret := os.Getenv("MINIO_SECRET_KEY") secure, _ := strconv.ParseBool(os.Getenv("MINIO_SECURE")) - if strings.ToLower(os.Getenv("LOG_FORMAT")) == "console" || os.Getenv("ENV") == "dev" { - zlog.Logger = zlog.Output(zerolog.ConsoleWriter{Out: os.Stdout}) - } else { - zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - } - zerolog.SetGlobalLevel(zerolog.InfoLevel) - if lvl := os.Getenv("LOG_LEVEL"); lvl != "" { - if l, err := zerolog.ParseLevel(lvl); err == nil { - zerolog.SetGlobalLevel(l) - } - } - logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + // zerolog console in dev, json in prod — общий bootstrap (issue #63) + logger := pkglogger.Setup("metadata") // Phase 15: tune pool for PgBouncer (transaction mode requires simple protocol, no prepared statements) poolCfg, err := pgxpool.ParseConfig(dbURL) @@ -85,7 +75,7 @@ func main() { log.Fatalf("pgxpool parse: %v", err) } if v := os.Getenv("DATABASE_POOL_SIZE"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { + if n, parseErr := strconv.Atoi(v); parseErr == nil && n > 0 { poolCfg.MaxConns = int32(n) } } else { @@ -114,10 +104,10 @@ func main() { // Issue #43: presigned thumbnail URLs for browsers must embed the public host publicEndpoint := os.Getenv("MINIO_PUBLIC_ENDPOINT") if publicEndpoint != "" { - if err := store.EnablePublicPresign(publicEndpoint, minioAccess, minioSecret, strings.HasPrefix(publicEndpoint, "https://")); err == nil { + if presignErr := store.EnablePublicPresign(publicEndpoint, minioAccess, minioSecret, strings.HasPrefix(publicEndpoint, "https://")); presignErr == nil { logger.Info().Str("public_endpoint", publicEndpoint).Msg("metadata public presign enabled") } else { - logger.Warn().Err(err).Msg("public presign init failed, thumbnails will use internal endpoint") + logger.Warn().Err(presignErr).Msg("public presign init failed, thumbnails will use internal endpoint") } } vh = handler.NewVideoHandlerWithStorage(repo, store) @@ -131,14 +121,14 @@ func main() { } r := chi.NewRouter() - r.Use(middleware.RequestID, middleware.Recoverer, mw.RequestLogger) + r.Use(middleware.RequestID, middleware.Recoverer, pkgmw.RequestLogger("metadata")) r.Get("/health", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, r, http.StatusOK, map[string]string{"status": "ok", "service": "metadata"}) + httputil.WriteJSON(w, r, http.StatusOK, map[string]string{"status": "ok", "service": "metadata"}) }) r.Get("/metrics", metrics.Handler().ServeHTTP) r.Get("/", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, r, http.StatusOK, map[string]string{"service": "metadata", "version": "0.1"}) + httputil.WriteJSON(w, r, http.StatusOK, map[string]string{"service": "metadata", "version": "0.1"}) }) r.Get("/swagger/*", httpSwagger.WrapHandler) r.Get("/openapi.json", func(w http.ResponseWriter, r *http.Request) { @@ -146,7 +136,7 @@ func main() { }) // protected routes: need JWT r.Group(func(r chi.Router) { - r.Use(mw.AuthMiddleware(jwtSecret)) + r.Use(pkgmw.AuthMiddleware(jwtSecret)) r.Post("/api/v1/videos", vh.Create) r.Patch("/api/v1/videos/{id}", vh.Update) r.Delete("/api/v1/videos/{id}", vh.Delete) @@ -154,7 +144,7 @@ func main() { // public list/get: optional JWT — владелец видит свои private/unlisted // (issue #44). Идентичность только из собственного JWT (X-User-ID не // доверенный — metadata опубликована наружу). - optAuth := mw.OptionalAuth(jwtSecret) + optAuth := pkgmw.OptionalAuth(jwtSecret) r.With(optAuth).Get("/api/v1/videos", vh.List) r.With(optAuth).Get("/api/v1/videos/{id}", vh.Get) // internal — protected by X-Internal-Token (gateway/nginx/transcoder) @@ -166,7 +156,8 @@ func main() { }) log.Printf("metadata listening :%s", port) - if err := http.ListenAndServe(":"+port, r); err != nil { + // маленькие JSON-тела — литеральные таймауты issue #63 (Read 15s, Write 60s, Idle 120s) + if err := pkghttp.Run(":"+port, r, pkghttp.Defaults()); err != nil { log.Fatal(err) } } @@ -185,34 +176,6 @@ func stripSSLMode(s string) string { return u.String() } -func writeJSON(w http.ResponseWriter, r *http.Request, status int, v any) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(status) - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - if shouldPretty(r) { - enc.SetIndent("", " ") - } - if err := enc.Encode(v); err != nil { - slog.Error("encode json failed", "error", err, "path", r.URL.Path) - } -} - -func shouldPretty(r *http.Request) bool { - if v := r.URL.Query().Get("pretty"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - return true - } - if v := os.Getenv("PRETTY_JSON"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - } - return false -} - // requireEnv reads a mandatory env var and exits with a clear error when it is // empty — unless ENV=dev (local `make dev-*` runs without .env, issue #53). // Empty INTERNAL_TOKEN previously left /internal/* metadata open. diff --git a/services/metadata/go.mod b/services/metadata/go.mod index d31e469..84dca68 100644 --- a/services/metadata/go.mod +++ b/services/metadata/go.mod @@ -3,13 +3,13 @@ module flowix/metadata go 1.25.0 require ( + flowix/pkg v0.0.0-00010101000000-000000000000 github.com/go-chi/chi/v5 v5.2.1 - github.com/golang-jwt/jwt/v5 v5.2.2 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.5 github.com/minio/minio-go/v7 v7.3.0 - github.com/prometheus/client_golang v1.22.0 - github.com/rs/zerolog v1.33.0 + github.com/rs/zerolog v1.35.1 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.4 ) @@ -37,6 +37,7 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/philhofer/fwd v1.2.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect @@ -56,3 +57,5 @@ require ( gopkg.in/ini.v1 v1.67.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +replace flowix/pkg => ../../pkg diff --git a/services/metadata/go.sum b/services/metadata/go.sum index 35f89d1..849e149 100644 --- a/services/metadata/go.sum +++ b/services/metadata/go.sum @@ -4,7 +4,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -23,9 +22,8 @@ github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -60,11 +58,8 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= @@ -78,7 +73,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= @@ -91,11 +85,10 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -133,9 +126,7 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/services/metadata/internal/handler/response.go b/services/metadata/internal/handler/response.go index 7f05572..256ee45 100644 --- a/services/metadata/internal/handler/response.go +++ b/services/metadata/internal/handler/response.go @@ -1,55 +1,18 @@ +// Package handler implements HTTP handlers of the metadata API (videos CRUD, VOD mapping). package handler import ( - "encoding/json" - "log/slog" "net/http" - "os" - "strconv" + + "flowix/pkg/httputil" ) -// writeJSON centralizes JSON encoding per golang-code-style + golang-error-handling. -// -// - Sets Content-Type explicitly (security: correct MIME, no sniffing) -// - Uses compact JSON by default (production: меньше трафика) -// - Pretty-print только по ?pretty=1 или PRETTY_JSON=true (dev: читаемость в Swagger/curl) -// - Логирует ошибку кодирования через slog (single handling rule: log server-side, не отдавать стек клиенту) +// writeJSON — общий хелпер JSON-ответов (issue #63: единая реализация в pkg/httputil). func writeJSON(w http.ResponseWriter, r *http.Request, status int, v any) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(status) - - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - - if shouldPretty(r) { - enc.SetIndent("", " ") - } - - if err := enc.Encode(v); err != nil { - // Encode after WriteHeader уже не поправить статус — только логируем - slog.Error("encode json failed", "error", err, "path", r.URL.Path) - } + httputil.WriteJSON(w, r, status, v) } func writeError(w http.ResponseWriter, r *http.Request, status int, msg string) { // security: не отдаем err.Error() с деталями БД/стека — только generic msg writeJSON(w, r, status, map[string]string{"error": msg}) } - -func shouldPretty(r *http.Request) bool { - // 1. явный ?pretty=1 / ?pretty=true - if v := r.URL.Query().Get("pretty"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - // ?pretty без значения — считаем включенным - return true - } - // 2. глобальный флаг для dev/Swagger: PRETTY_JSON=true - if v := os.Getenv("PRETTY_JSON"); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } - } - return false -} diff --git a/services/metadata/internal/handler/video.go b/services/metadata/internal/handler/video.go index e55792a..d0c16d7 100644 --- a/services/metadata/internal/handler/video.go +++ b/services/metadata/internal/handler/video.go @@ -10,9 +10,10 @@ import ( "strconv" "time" - "flowix/metadata/internal/middleware" "flowix/metadata/internal/model" + pkgmw "flowix/pkg/middleware" + "github.com/go-chi/chi/v5" ) @@ -108,7 +109,7 @@ func (h *VideoHandler) Register(r chi.Router) { // @Failure 401 {string} string "unauthorized" // @Router /api/v1/videos [post] func (h *VideoHandler) Create(w http.ResponseWriter, r *http.Request) { - ownerID := middleware.UserIDFromCtx(r.Context()) + ownerID := pkgmw.UserIDFromCtx(r.Context()) var req model.CreateVideoRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, r, http.StatusBadRequest, "invalid body") @@ -170,7 +171,7 @@ func (h *VideoHandler) getVideo(w http.ResponseWriter, r *http.Request, enforceV writeError(w, r, http.StatusNotFound, "not found") return } - if enforceVisibility && v.Visibility == model.VisibilityPrivate && middleware.UserIDFromCtx(r.Context()) != v.OwnerID { + if enforceVisibility && v.Visibility == model.VisibilityPrivate && pkgmw.UserIDFromCtx(r.Context()) != v.OwnerID { writeError(w, r, http.StatusForbidden, "forbidden") return } @@ -250,7 +251,7 @@ func (h *VideoHandler) List(w http.ResponseWriter, r *http.Request) { if v, err := strconv.Atoi(r.URL.Query().Get("offset")); err == nil && v >= 0 { offset = v } - list, err := h.repo.List(r.Context(), limit, offset, middleware.UserIDFromCtx(r.Context())) + list, err := h.repo.List(r.Context(), limit, offset, pkgmw.UserIDFromCtx(r.Context())) if err != nil { slog.Error("list videos failed", "error", err) writeError(w, r, http.StatusInternalServerError, "internal error") @@ -276,7 +277,7 @@ func (h *VideoHandler) List(w http.ResponseWriter, r *http.Request) { // @Failure 404 {string} string "not found" // @Router /api/v1/videos/{id} [patch] func (h *VideoHandler) Update(w http.ResponseWriter, r *http.Request) { - ownerID := middleware.UserIDFromCtx(r.Context()) + ownerID := pkgmw.UserIDFromCtx(r.Context()) id := chi.URLParam(r, "id") var req model.UpdateVideoRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -314,7 +315,7 @@ func (h *VideoHandler) Update(w http.ResponseWriter, r *http.Request) { // @Failure 404 {string} string "not found" // @Router /api/v1/videos/{id} [delete] func (h *VideoHandler) Delete(w http.ResponseWriter, r *http.Request) { - ownerID := middleware.UserIDFromCtx(r.Context()) + ownerID := pkgmw.UserIDFromCtx(r.Context()) id := chi.URLParam(r, "id") // fetch first for S3 keys (need renditions + thumbnail before DB delete) var keys []string diff --git a/services/metadata/internal/handler/video_test.go b/services/metadata/internal/handler/video_test.go index 1f854c6..f9530d6 100644 --- a/services/metadata/internal/handler/video_test.go +++ b/services/metadata/internal/handler/video_test.go @@ -9,9 +9,10 @@ import ( "testing" "time" - "flowix/metadata/internal/middleware" "flowix/metadata/internal/model" + pkgmw "flowix/pkg/middleware" + "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" ) @@ -142,13 +143,13 @@ func testRouter(store VideoStore) chi.Router { vh := NewVideoHandler(store) // protected r.Group(func(r chi.Router) { - r.Use(middleware.AuthMiddleware(testSecret)) + r.Use(pkgmw.AuthMiddleware(testSecret)) r.Post("/api/v1/videos", vh.Create) r.Patch("/api/v1/videos/{id}", vh.Update) r.Delete("/api/v1/videos/{id}", vh.Delete) }) // public GET — optional JWT, как в cmd/server/main.go (issue #44) - optAuth := middleware.OptionalAuth(testSecret) + optAuth := pkgmw.OptionalAuth(testSecret) r.With(optAuth).Get("/api/v1/videos", vh.List) r.With(optAuth).Get("/api/v1/videos/{id}", vh.Get) r.Patch("/internal/videos/{id}/status", vh.UpdateStatus) diff --git a/services/metadata/internal/metrics/metrics.go b/services/metadata/internal/metrics/metrics.go deleted file mode 100644 index fd56e8a..0000000 --- a/services/metadata/internal/metrics/metrics.go +++ /dev/null @@ -1,36 +0,0 @@ -package metrics - -import ( - "net/http" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var ( - RabbitMQQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "rabbitmq_queue_depth", - Help: "RabbitMQ queue depth", - }) - VodCacheHit = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "vod_cache_hit", - Help: "VOD cache hits", - }) - UploadBytes = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "upload_bytes", - Help: "Uploaded bytes", - }) - FfmpegDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "ffmpeg_duration_seconds", - Help: "FFmpeg duration", - Buckets: prometheus.DefBuckets, - }) -) - -func init() { - prometheus.MustRegister(RabbitMQQueueDepth, VodCacheHit, UploadBytes, FfmpegDuration) -} - -func Handler() http.Handler { - return promhttp.Handler() -} diff --git a/services/metadata/internal/middleware/auth.go b/services/metadata/internal/middleware/auth.go deleted file mode 100644 index c73229f..0000000 --- a/services/metadata/internal/middleware/auth.go +++ /dev/null @@ -1,96 +0,0 @@ -// Package middleware provides HTTP middleware for metadata service. -package middleware - -import ( - "context" - "net/http" - "strings" - - "github.com/golang-jwt/jwt/v5" -) - -type ctxKey string - -const UserIDKey ctxKey = "user_id" - -func AuthMiddleware(secret string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := r.Header.Get("Authorization") - if h == "" || !strings.HasPrefix(h, "Bearer ") { - http.Error(w, `{"error":"missing token"}`, http.StatusUnauthorized) - return - } - tokenStr := strings.TrimPrefix(h, "Bearer ") - token, err := jwt.Parse(tokenStr, func(t *jwt.Token) (interface{}, error) { - return []byte(secret), nil - }, jwt.WithValidMethods([]string{"HS256"}), jwt.WithExpirationRequired()) - if err != nil || !token.Valid { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - sub, _ := claims["sub"].(string) - if sub == "" { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - // тип токена обязан быть access (issue #53: обязательные exp и type) - if typ, _ := claims["type"].(string); typ != "access" { - http.Error(w, `{"error":"invalid token type"}`, http.StatusUnauthorized) - return - } - ctx := context.WithValue(r.Context(), UserIDKey, sub) - next.ServeHTTP(w, r.WithContext(ctx)) - }) - } -} - -func UserIDFromCtx(ctx context.Context) string { - v, _ := ctx.Value(UserIDKey).(string) - return v -} - -// OptionalAuth — как Auth, но не требует токен: валидный Bearer кладёт sub в -// контекст, отсутствие/невалидность токена — анонимный доступ (публичные GET -// не должны падать 401 при истёкшем access-токене, issue #44). Владелец видит -// свои private/unlisted видео только через собственный JWT — заголовок -// X-User-ID не доверенный, т.к. metadata опубликована наружу. -func OptionalAuth(secret string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := r.Header.Get("Authorization") - if !strings.HasPrefix(h, "Bearer ") { - next.ServeHTTP(w, r) - return - } - token, err := jwt.Parse(strings.TrimPrefix(h, "Bearer "), func(t *jwt.Token) (interface{}, error) { - return []byte(secret), nil - }, jwt.WithValidMethods([]string{"HS256"}), jwt.WithExpirationRequired()) - if err != nil || !token.Valid { - next.ServeHTTP(w, r) - return - } - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - next.ServeHTTP(w, r) - return - } - // refresh-токен и токен без type не являются идентичностью (issue #53) - if typ, _ := claims["type"].(string); typ != "access" { - next.ServeHTTP(w, r) - return - } - sub, _ := claims["sub"].(string) - if sub == "" { - next.ServeHTTP(w, r) - return - } - next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserIDKey, sub))) - }) - } -} diff --git a/services/metadata/internal/middleware/auth_test.go b/services/metadata/internal/middleware/auth_test.go deleted file mode 100644 index f9e3fc0..0000000 --- a/services/metadata/internal/middleware/auth_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Package middleware tests — T2 stepwise auth checks. -package middleware - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/golang-jwt/jwt/v5" -) - -func mustToken(secret, sub string) string { - t := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ - "sub": sub, - "type": "access", - "exp": time.Now().Add(time.Hour).Unix(), - }) - s, _ := t.SignedString([]byte(secret)) - return s -} - -// issue #53: token without exp or type=access must be rejected. -func TestAuthRejectsTokenWithoutExpOrType(t *testing.T) { - secret := "test-secret" - noExp := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"sub": "u1", "type": "access"}) - noType := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"sub": "u1", "exp": time.Now().Add(time.Hour).Unix()}) - for name, tok := range map[string]*jwt.Token{"no_exp": noExp, "no_type": noType} { - s, _ := tok.SignedString([]byte(secret)) - h := AuthMiddleware(secret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("should not call") })) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer "+s) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("%s: want 401 got %d", name, w.Code) - } - } -} - -func TestAuthMiddlewareOK(t *testing.T) { - secret := "test-secret-32chars-for-unit-tests-!!" - called := false - next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - called = true - if UserIDFromCtx(r.Context()) != "u1" { - t.Fatalf("want u1 got %s", UserIDFromCtx(r.Context())) - } - w.WriteHeader(200) - }) - h := AuthMiddleware(secret)(next) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer "+mustToken(secret, "u1")) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 200 || !called { - t.Fatalf("want 200 called true got %d %v", w.Code, called) - } -} - -func TestAuthMiddlewareMissing(t *testing.T) { - h := AuthMiddleware("s")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("should not call") })) - req := httptest.NewRequest("GET", "/", nil) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("want 401 got %d", w.Code) - } -} - -func TestAuthMiddlewareInvalid(t *testing.T) { - h := AuthMiddleware("s")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("should not call") })) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer invalid.token.here") - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("want 401 got %d", w.Code) - } -} diff --git a/services/metadata/internal/middleware/internal.go b/services/metadata/internal/middleware/internal.go index 752f401..5b29971 100644 --- a/services/metadata/internal/middleware/internal.go +++ b/services/metadata/internal/middleware/internal.go @@ -1,3 +1,4 @@ +// Package middleware provides metadata-specific middleware (internal token auth). package middleware import ( diff --git a/services/metadata/internal/middleware/logger.go b/services/metadata/internal/middleware/logger.go deleted file mode 100644 index d87debe..0000000 --- a/services/metadata/internal/middleware/logger.go +++ /dev/null @@ -1,48 +0,0 @@ -package middleware - -import ( - "net/http" - "time" - - "github.com/rs/zerolog/log" -) - -// RequestLogger — structured logging via zerolog, includes trace_id -func RequestLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - ww := &respWriter{ResponseWriter: w, status: http.StatusOK} - next.ServeHTTP(ww, r) - dur := time.Since(start) - reqID := r.Header.Get("X-Request-ID") - if reqID == "" { - reqID = r.Header.Get("X-Request-Id") - } - if reqID == "" { - reqID = r.Header.Get("X-Correlation-ID") - } - ev := log.Info() - if ww.status >= 500 { - ev = log.Error() - } else if ww.status >= 400 { - ev = log.Warn() - } - ev.Str("method", r.Method). - Str("path", r.URL.Path). - Int("status", ww.status). - Dur("duration", dur). - Str("trace_id", reqID). - Str("service", "metadata"). - Msg("request") - }) -} - -type respWriter struct { - http.ResponseWriter - status int -} - -func (w *respWriter) WriteHeader(code int) { - w.status = code - w.ResponseWriter.WriteHeader(code) -} diff --git a/services/metadata/internal/model/video.go b/services/metadata/internal/model/video.go index b5d50ad..d307d08 100644 --- a/services/metadata/internal/model/video.go +++ b/services/metadata/internal/model/video.go @@ -1,3 +1,4 @@ +// Package model defines video metadata domain types and visibility enum. package model import "time" diff --git a/services/metadata/internal/repository/video.go b/services/metadata/internal/repository/video.go index d2999e8..44038a8 100644 --- a/services/metadata/internal/repository/video.go +++ b/services/metadata/internal/repository/video.go @@ -64,7 +64,9 @@ func (r *VideoRepo) GetByID(ctx context.Context, id string) (*model.Video, error // List returns videos visible to viewerID: everyone sees public, the owner // also sees their own private/unlisted (issue #44). Empty viewerID = anonymous. func (r *VideoRepo) List(ctx context.Context, limit, offset int, viewerID string) ([]model.Video, error) { - q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE v.visibility='public' OR ($3 <> '' AND v.owner_id=$3) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` + // $3 участвует и в текстовом сравнении ($3 <> ''), и в сравнении с uuid-колонкой: + // без ::uuid Postgres выводит тип $3 как text и падает с SQLSTATE 42883 (uuid = text) + q := `SELECT v.id, v.owner_id, u.email, v.title, v.description, v.duration, v.status, COALESCE(v.visibility::text,'public'), v.thumbnail_s3_key, v.created_at, v.updated_at FROM videos v LEFT JOIN users u ON u.id=v.owner_id WHERE v.visibility='public' OR ($3 <> '' AND v.owner_id=$3::uuid) ORDER BY v.created_at DESC LIMIT $1 OFFSET $2` rows, err := r.pool.Query(ctx, q, limit, offset, viewerID) if err != nil { return nil, err diff --git a/services/metadata/internal/storage/minio.go b/services/metadata/internal/storage/minio.go index 7ee3a7e..5b17f8b 100644 --- a/services/metadata/internal/storage/minio.go +++ b/services/metadata/internal/storage/minio.go @@ -1,3 +1,4 @@ +// Package storage wraps MinIO access for metadata (delete, presign init). package storage import ( diff --git a/services/transcoder/README.md b/services/transcoder/README.md index 52db1c5..32afe0c 100644 --- a/services/transcoder/README.md +++ b/services/transcoder/README.md @@ -46,6 +46,8 @@ FFMPEG_THREADS=2 FFMPEG_PRESET=veryfast python -m app.consumer ## Лимиты - Последовательный рендеринг (было 3× parallel → OOM), `timeout 900`, `fps` из probe (≤30 preserve, >30 cap 30), `-threads 2 -preset veryfast -maxrate 1.10×` +- Pipe-режим: при ошибке стрима/таймауте ffmpeg SIGKILL-ится и реапится (`_kill_and_reap`, issue #51) — без зомби +- Heartbeat: `_HeartbeatKeeper` (daemon-поток, `process_data_events` каждые 30с) держит pika-соединение живым во время длинных транскодов (issue #51) - `shutil.disk_usage` чек, `SIGTERM` graceful, `deploy/docker-compose.yml:179` + `prod.yml:44` limits `cpus 2 / mem 4G` - `.env.example:30` `FFMPEG_THREADS`/`FFMPEG_PRESET`, `make dev-transcoder` теперь `python -m app.consumer` (celery legacy `make dev-transcoder-celery`) diff --git a/services/transcoder/app/consumer.py b/services/transcoder/app/consumer.py index a1acf02..40444a5 100644 --- a/services/transcoder/app/consumer.py +++ b/services/transcoder/app/consumer.py @@ -5,7 +5,9 @@ import signal import subprocess import tempfile +import threading import time +from typing import Any import pika # type: ignore[import-untyped] from pika.exceptions import AMQPConnectionError # type: ignore[import-untyped] @@ -432,6 +434,19 @@ def transcode_one( pass +def _kill_and_reap(proc: subprocess.Popen) -> None: + """SIGKILL an ffmpeg subprocess and reap it so it cannot linger as a zombie (issue #51). + + Safe to call on an already-exited process (skips kill, still waits to reap). + """ + try: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=10) + except Exception as e: + log.error("failed to kill/reap ffmpeg pid=%s: %s", proc.pid, e) + + def transcode_one_pipe( get_object_stream, audio_path: str | None, @@ -518,40 +533,56 @@ def transcode_one_pipe( proc = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + stdout: bytes | None = None + stderr: bytes | None = None try: - # stream MinIO object to ffmpeg stdin in chunks - chunk_size = 256 * 1024 - # get_object_stream may be response object with stream() or read() - if hasattr(get_object_stream, "stream"): - # minio get_object returns HTTPResponse with stream() - assert proc.stdin is not None - for chunk in get_object_stream.stream(chunk_size): - if chunk: + try: + # stream MinIO object to ffmpeg stdin in chunks + chunk_size = 256 * 1024 + # get_object_stream may be response object with stream() or read() + if hasattr(get_object_stream, "stream"): + # minio get_object returns HTTPResponse with stream() + assert proc.stdin is not None + for chunk in get_object_stream.stream(chunk_size): + if chunk: + proc.stdin.write(chunk) + proc.stdin.close() + elif hasattr(get_object_stream, "read"): + assert proc.stdin is not None + while True: + chunk = get_object_stream.read(chunk_size) + if not chunk: + break proc.stdin.write(chunk) - proc.stdin.close() - elif hasattr(get_object_stream, "read"): - assert proc.stdin is not None - while True: - chunk = get_object_stream.read(chunk_size) - if not chunk: - break - proc.stdin.write(chunk) - proc.stdin.close() - else: - # iterable - assert proc.stdin is not None - for chunk in get_object_stream: - proc.stdin.write(chunk) - proc.stdin.close() - stdout, stderr = proc.communicate(timeout=900) + proc.stdin.close() + else: + # iterable + assert proc.stdin is not None + for chunk in get_object_stream: + proc.stdin.write(chunk) + proc.stdin.close() + stdout, stderr = proc.communicate(timeout=900) + except subprocess.TimeoutExpired: + # issue #51: ffmpeg outlived its budget — kill it, collect output, re-raise + log.warning("ffmpeg pipe timed out after 900s, killing pid=%s", proc.pid) + proc.kill() + stdout, stderr = proc.communicate() + raise subprocess.TimeoutExpired(cmd, 900, output=stdout, stderr=stderr) + except Exception: + # issue #51: stream error mid-feed (or communicate failure) — never leave ffmpeg running + _kill_and_reap(proc) + raise + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, cmd, output=stdout, stderr=stderr) if METRICS_AVAILABLE and ffmpeg_duration_metric is not None: try: ffmpeg_duration_metric.labels(quality=f"{height}p").observe(time.time() - start) except Exception: pass - if proc.returncode != 0: - raise subprocess.CalledProcessError(proc.returncode, cmd, output=stdout, stderr=stderr) finally: + # belt-and-braces (issue #51): reap anything that survived the paths above — no zombies + if proc.poll() is None: + _kill_and_reap(proc) try: if proc.stdin and not proc.stdin.closed: proc.stdin.close() @@ -870,6 +901,57 @@ def _get_status(video_id: str) -> str | None: return None +class _HeartbeatKeeper: + """Keep the pika BlockingConnection alive during long-blocking transcodes (issue #51). + + BlockingConnection only processes heartbeat frames inside + process_data_events(); a transcode blocks that loop for minutes, so the + broker closes the connection (600s heartbeat vs 900s x 3 ffmpeg runs). + A daemon thread pokes the connection every `interval` seconds. + + Pika calls in handle_message are allowed only after `quiesced` is True + (ревью фазы 17, M2): __exit__ ждёт остановки потока; если поток застрял + внутри process_data_events, ack/publish откладывать нельзя — + BlockingConnection не потокобезопасен, и сообщение будет доставлено + повторно после потери соединения. + """ + + def __init__(self, connection: Any, interval: float = 30.0): + self._conn = connection + self._interval = interval + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self.quiesced = False + + def __enter__(self) -> "_HeartbeatKeeper": + self._thread = threading.Thread(target=self._run, daemon=True, name="pika-heartbeat") + self._thread.start() + return self + + def _run(self) -> None: + while not self._stop.wait(self._interval): + try: + self._conn.process_data_events(time_limit=0) + except Exception as e: + log.warning("heartbeat poke failed: %s", e) + return # main thread will observe the real AMQP error + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: Any, + ) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2) + self.quiesced = not self._thread.is_alive() + else: + self.quiesced = True + if not self.quiesced: + log.error("heartbeat thread did not stop in time — pika calls are unsafe") + + def handle_message(ch, method, properties, body): """Ack only after process_message succeeds (including the ready metadata update). @@ -887,7 +969,17 @@ def handle_message(ch, method, properties, body): except Exception: retry_count = 0 try: - process_message(body) + # issue #51: process_message blocks the pika dispatch loop for minutes + # (ffmpeg 900s x 3) — poke the connection from a daemon thread so the + # broker does not close it on heartbeat timeout. + keeper = _HeartbeatKeeper(ch.connection) + with keeper: + process_message(body) + if not keeper.quiesced: + # ревью фазы 17 (M2): поток ещё гоняет process_data_events — любые + # pika-вызовы небезопасны; не ackаем, брокер доставит повторно + log.error("heartbeat keeper not quiesced — skipping ack, message will be redelivered") + return ch.basic_ack(delivery_tag=method.delivery_tag) except Exception as e: log.exception("process failed: %s", e) @@ -949,6 +1041,8 @@ def main(): params = pika.URLParameters(RABBITMQ_URL) # Phase 10 fix: long transcoding (2GB ~5min) blocks heartbeat thread → broker closes connection (104). # Default heartbeat 60s is too short; set to 600s (10min) to cover 5-6GB files. For larger files Phase 11 will use chunked. + # Issue #51: 600s still loses to ffmpeg 900s x 3 — _HeartbeatKeeper pokes the + # connection from a daemon thread while process_message runs. params.heartbeat = 600 params.blocked_connection_timeout = 300 while not _shutdown: diff --git a/services/transcoder/tests/test_consumer.py b/services/transcoder/tests/test_consumer.py index 16dc336..a8d97ed 100644 --- a/services/transcoder/tests/test_consumer.py +++ b/services/transcoder/tests/test_consumer.py @@ -1,4 +1,6 @@ import json +import subprocess +import time from unittest.mock import MagicMock, patch import pytest @@ -383,6 +385,34 @@ def test_handle_message_acks_after_success(): ch.basic_nack.assert_not_called() +def test_handle_message_skips_ack_when_keeper_not_quiesced(): + # ревью фазы 17 (M2): пока heartbeat-поток не остановлен, pika-вызовы + # небезопасны (BlockingConnection не потокобезопасен) — ack пропускаем, + # брокер доставит сообщение повторно после потери соединения + ch, method = _handle_message_channel() + body = json.dumps({"video_id": "vid-1", "s3_key": "raw/vid-1/original.mp4"}).encode() + + class StuckKeeper: + def __init__(self, connection): + self.quiesced = False + + def __enter__(self): + return self + + def __exit__(self, *exc): + return None + + with ( + patch("app.consumer.process_message") as mock_proc, + patch("app.consumer._HeartbeatKeeper", StuckKeeper), + ): + cons.handle_message(ch, method, MagicMock(headers={}), body) + mock_proc.assert_called_once_with(body) + ch.basic_ack.assert_not_called() + ch.basic_publish.assert_not_called() + ch.basic_nack.assert_not_called() + + def test_handle_message_metadata_failure_retries_and_does_not_ack_silently(): # issue #50 verify: metadata update failed -> retry-queue republish, not a plain ack ch, method = _handle_message_channel() @@ -441,3 +471,122 @@ def test_handle_message_moves_to_dlq_after_max_retries(): ch.basic_nack.assert_called_once_with(delivery_tag=7, requeue=False) ch.basic_ack.assert_not_called() ch.basic_publish.assert_not_called() + + +class _FakeProc: + """Minimal Popen stand-in with tracked kill/wait/communicate (issue #51). + + Behaves like a real process: poll() returns None while alive, kill() makes + it exit (SIGKILL), wait()/communicate() reap it. + """ + + def __init__(self, returncode: int = 0, communicate_raises: Exception | None = None): + self.stdin = MagicMock() + self.pid = 4242 + self.returncode: int | None = None + self._exit_code = returncode + self._communicate_raises = communicate_raises + self.kill_calls = 0 + self.wait_calls = 0 + self.communicate_calls = 0 + + def poll(self) -> int | None: + return self.returncode + + def kill(self) -> None: + self.kill_calls += 1 + self.returncode = -9 + + def wait(self, timeout: float | None = None) -> int: + self.wait_calls += 1 + assert self.returncode is not None, "wait() on a live process would block" + return self.returncode + + def communicate(self, timeout: float | None = None) -> tuple[bytes, bytes]: + self.communicate_calls += 1 + if self._communicate_raises is not None and self.returncode is None: + raise self._communicate_raises + if self.returncode is None: + self.returncode = self._exit_code + return b"out", b"err" + + +class _BrokenStream: + """MinIO-like stream that dies mid-feed with an error.""" + + def __init__(self): + self.closed = False + + def stream(self, chunk_size: int): + yield b"chunk" + raise OSError("stream broken") + + def close(self) -> None: + self.closed = True + + +def test_transcode_one_pipe_success_does_not_kill_ffmpeg(): + proc = _FakeProc(returncode=0) + stream = MagicMock() + with patch("app.consumer.subprocess.Popen", return_value=proc): + cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) + assert proc.kill_calls == 0 + assert proc.wait_calls == 0 + proc.stdin.close.assert_called_once() + stream.close.assert_called_once() + + +def test_transcode_one_pipe_stream_error_kills_and_reaps_ffmpeg(): + # issue #51 verify: stream failure mid-feed -> ffmpeg killed AND waited/reaped, no zombie + proc = _FakeProc() + stream = _BrokenStream() + with patch("app.consumer.subprocess.Popen", return_value=proc): + with pytest.raises(OSError, match="stream broken"): + cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) + assert proc.kill_calls == 1 + assert proc.wait_calls == 1 + assert proc.returncode == -9 # reaped + assert stream.closed + + +def test_transcode_one_pipe_timeout_kills_and_reaps_ffmpeg(): + # issue #51: communicate(timeout=900) fires -> kill, collect output via + # communicate() again (reaps), and propagate TimeoutExpired + proc = _FakeProc(communicate_raises=subprocess.TimeoutExpired("ffmpeg", 900)) + stream = MagicMock() + with patch("app.consumer.subprocess.Popen", return_value=proc): + with pytest.raises(subprocess.TimeoutExpired): + cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) + assert proc.kill_calls == 1 + assert proc.communicate_calls == 2 # retry after kill to drain output and reap + assert proc.returncode == -9 + + +def test_transcode_one_pipe_nonzero_exit_raises_and_skips_kill(): + # ffmpeg already exited on its own -> only CalledProcessError, no kill needed + proc = _FakeProc(returncode=1) + stream = MagicMock() + with patch("app.consumer.subprocess.Popen", return_value=proc): + with pytest.raises(subprocess.CalledProcessError): + cons.transcode_one_pipe(stream, None, "/tmp/out.mp4", 1280, 720, 800) + assert proc.kill_calls == 0 + assert proc.wait_calls == 0 + stream.close.assert_called_once() + + +def test_heartbeat_keeper_pokes_connection_periodically(): + # issue #51: keeper must drive process_data_events while the transcode blocks + conn = MagicMock() + with cons._HeartbeatKeeper(conn, interval=0.05): + time.sleep(0.12) + assert conn.process_data_events.call_count >= 1 + assert conn.process_data_events.call_args[1]["time_limit"] == 0 + + +def test_heartbeat_keeper_swallows_poke_errors_and_stops(): + # a dead connection must not crash the worker thread — main loop handles it + conn = MagicMock() + conn.process_data_events.side_effect = Exception("connection closed") + with cons._HeartbeatKeeper(conn, interval=0.02): + time.sleep(0.08) # must not raise + assert conn.process_data_events.call_count == 1 # stopped after the first failure diff --git a/services/upload/Dockerfile b/services/upload/Dockerfile index b1a552b..654eb6d 100644 --- a/services/upload/Dockerfile +++ b/services/upload/Dockerfile @@ -1,8 +1,14 @@ +# Issue #63: build context = repo root (см. deploy/docker-compose.yml) — +# рядом копируем общий модуль pkg/. go.work в образ не попадает: сборка +# идёт per-module через replace flowix/pkg => ../../pkg. FROM golang:1.27-alpine AS builder WORKDIR /app -COPY go.mod go.sum* ./ -RUN go mod download || true -COPY . . +COPY pkg/go.mod pkg/go.sum pkg/ +COPY services/upload/go.mod services/upload/go.sum services/upload/ +RUN cd /app/pkg && go mod download && cd /app/services/upload && go mod download +COPY pkg/ pkg/ +COPY services/upload/ services/upload/ +WORKDIR /app/services/upload RUN CGO_ENABLED=0 go build -o /server ./cmd/server FROM alpine:3.19 diff --git a/services/upload/README.md b/services/upload/README.md index f579158..600e165 100644 --- a/services/upload/README.md +++ b/services/upload/README.md @@ -36,3 +36,7 @@ go vet ./services/upload/... ## Zed IDE Тот же `.zed/settings.json:1` → Go `format_on_save` + `organizeImports`. + +## Общий Go-модуль pkg (issue #63) + +AuthMiddleware/OptionalAuth, RequestLogger, метрики Prometheus, `httputil.WriteJSON` и запуск сервера (таймауты Read/Write/Idle + graceful shutdown по SIGTERM/SIGINT) живут в общем модуле `pkg/` (`flowix/pkg`, подключён через `go.work` и `replace` в `go.mod`). Приватные копии в сервисе удалены — правки общих хелперов делай в `pkg/`, а не в сервисе. Таймауты и грейс shutdown: `pkg/httpserver` (env `SHUTDOWN_GRACE` переопределяет грейс, см. `stop_grace_period` в `deploy/docker-compose.yml`). diff --git a/services/upload/cmd/server/main.go b/services/upload/cmd/server/main.go index bdf88e6..90b5999 100644 --- a/services/upload/cmd/server/main.go +++ b/services/upload/cmd/server/main.go @@ -13,19 +13,18 @@ import ( "net/http" "os" "strconv" - "strings" + pkghttp "flowix/pkg/httpserver" + pkglogger "flowix/pkg/logger" + "flowix/pkg/metrics" + pkgmw "flowix/pkg/middleware" _ "flowix/upload/docs" "flowix/upload/internal/client" "flowix/upload/internal/handler" - "flowix/upload/internal/metrics" - mw "flowix/upload/internal/middleware" "flowix/upload/internal/queue" "flowix/upload/internal/storage" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/rs/zerolog" - zlog "github.com/rs/zerolog/log" httpSwagger "github.com/swaggo/http-swagger" ) @@ -55,18 +54,8 @@ func main() { minioSecret := os.Getenv("MINIO_SECRET_KEY") secure, _ := strconv.ParseBool(os.Getenv("MINIO_SECURE")) - if strings.ToLower(os.Getenv("LOG_FORMAT")) == "console" || os.Getenv("ENV") == "dev" { - zlog.Logger = zlog.Output(zerolog.ConsoleWriter{Out: os.Stdout}) - } else { - zerolog.TimeFieldFormat = zerolog.TimeFormatUnix - } - zerolog.SetGlobalLevel(zerolog.InfoLevel) - if lvl := os.Getenv("LOG_LEVEL"); lvl != "" { - if l, err := zerolog.ParseLevel(lvl); err == nil { - zerolog.SetGlobalLevel(l) - } - } - logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + // zerolog console in dev, json in prod — общий bootstrap (issue #63) + logger := pkglogger.Setup("upload") store, err := storage.NewMinioClient(minioEndpoint, minioAccess, minioSecret, bucket, secure) if err != nil { @@ -84,7 +73,7 @@ func main() { rh := handler.NewResumableHandler(store, metaCl) r := chi.NewRouter() - r.Use(middleware.RequestID, middleware.Recoverer, mw.RequestLogger) + r.Use(middleware.RequestID, middleware.Recoverer, pkgmw.RequestLogger("upload")) r.Get("/health", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -97,7 +86,7 @@ func main() { }) r.Group(func(r chi.Router) { - r.Use(mw.AuthMiddleware(jwtSecret)) + r.Use(pkgmw.AuthMiddleware(jwtSecret)) r.Post("/api/v1/videos/upload", uh.Upload) r.Post("/api/v1/videos/presign", ph.Presign) r.Post("/api/v1/videos/{id}/complete", ph.Complete) @@ -107,7 +96,10 @@ func main() { }) logger.Info().Str("port", port).Str("bucket", bucket).Str("metadata", metadataURL).Msg("upload starting") - if err := http.ListenAndServe(":"+port, r); err != nil { + // Streaming(): тела до 5 ГБ не влезают в Read/Write timeout — лимитируем + // только заголовки; Run делает graceful shutdown, in-flight загрузки + // не обрываются (issue #63). + if err := pkghttp.Run(":"+port, r, pkghttp.Streaming()); err != nil { log.Fatal(err) } } diff --git a/services/upload/go.mod b/services/upload/go.mod index 93e8133..2679fe4 100644 --- a/services/upload/go.mod +++ b/services/upload/go.mod @@ -1,16 +1,14 @@ module flowix/upload -go 1.23.0 - -toolchain go1.23.12 +go 1.25.0 require ( + flowix/pkg v0.0.0-00010101000000-000000000000 github.com/go-chi/chi/v5 v5.2.1 - github.com/golang-jwt/jwt/v5 v5.2.2 - github.com/minio/minio-go/v7 v7.0.70 - github.com/prometheus/client_golang v1.22.0 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/minio/minio-go/v7 v7.3.0 github.com/rabbitmq/amqp091-go v1.10.0 - github.com/rs/zerolog v1.33.0 + github.com/rs/zerolog v1.35.1 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.4 ) @@ -24,27 +22,35 @@ require ( github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/spec v0.20.6 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/goccy/go-json v0.10.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rs/xid v1.5.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.48.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +replace flowix/pkg => ../../pkg diff --git a/services/upload/go.sum b/services/upload/go.sum index 0078194..314b23e 100644 --- a/services/upload/go.sum +++ b/services/upload/go.sum @@ -4,7 +4,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -23,22 +22,21 @@ github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -52,20 +50,21 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= -github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU= +github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= @@ -78,58 +77,69 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= -github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc= github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/services/upload/internal/client/metadata.go b/services/upload/internal/client/metadata.go index cef5ab9..db80b51 100644 --- a/services/upload/internal/client/metadata.go +++ b/services/upload/internal/client/metadata.go @@ -1,3 +1,4 @@ +// Package client wraps HTTP calls from upload to the metadata service. package client import ( @@ -6,8 +7,13 @@ import ( "fmt" "net/http" "net/url" + "time" ) +// metadataClientTimeout — исходящие вызовы в metadata (create/owner) — +// маленькие JSON, без таймаута клиент висел вечно (issue #63). +const metadataClientTimeout = 10 * time.Second + type MetadataClient struct { baseURL string internalToken string @@ -15,7 +21,7 @@ type MetadataClient struct { } func NewMetadataClient(baseURL, internalToken string) *MetadataClient { - return &MetadataClient{baseURL: baseURL, internalToken: internalToken, client: &http.Client{}} + return &MetadataClient{baseURL: baseURL, internalToken: internalToken, client: &http.Client{Timeout: metadataClientTimeout}} } type CreateVideoResponse struct { diff --git a/services/upload/internal/handler/ownership.go b/services/upload/internal/handler/ownership.go index 69a9d50..8458b0c 100644 --- a/services/upload/internal/handler/ownership.go +++ b/services/upload/internal/handler/ownership.go @@ -1,3 +1,5 @@ +// Package handler implements HTTP handlers of the upload service +// (upload, presign, resumable, ownership checks). package handler import ( diff --git a/services/upload/internal/handler/presign.go b/services/upload/internal/handler/presign.go index 840db50..3bed9e8 100644 --- a/services/upload/internal/handler/presign.go +++ b/services/upload/internal/handler/presign.go @@ -9,8 +9,8 @@ import ( "strings" "time" - "flowix/upload/internal/metrics" - mw "flowix/upload/internal/middleware" + "flowix/pkg/metrics" + mw "flowix/pkg/middleware" "github.com/go-chi/chi/v5" ) diff --git a/services/upload/internal/handler/presign_test.go b/services/upload/internal/handler/presign_test.go index 57bcf8a..f4858d9 100644 --- a/services/upload/internal/handler/presign_test.go +++ b/services/upload/internal/handler/presign_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - mw "flowix/upload/internal/middleware" + mw "flowix/pkg/middleware" "github.com/go-chi/chi/v5" ) diff --git a/services/upload/internal/handler/resumable.go b/services/upload/internal/handler/resumable.go index 3e3bb82..e099eaa 100644 --- a/services/upload/internal/handler/resumable.go +++ b/services/upload/internal/handler/resumable.go @@ -15,7 +15,7 @@ import ( "github.com/go-chi/chi/v5" - mw "flowix/upload/internal/middleware" + mw "flowix/pkg/middleware" ) // ResumableStorage extends PresignStorage with size and object access. diff --git a/services/upload/internal/handler/resumable_test.go b/services/upload/internal/handler/resumable_test.go index d6da1c5..18aa31b 100644 --- a/services/upload/internal/handler/resumable_test.go +++ b/services/upload/internal/handler/resumable_test.go @@ -8,7 +8,7 @@ import ( "net/http/httptest" "testing" - mw "flowix/upload/internal/middleware" + mw "flowix/pkg/middleware" "github.com/go-chi/chi/v5" ) diff --git a/services/upload/internal/handler/upload.go b/services/upload/internal/handler/upload.go index cbfccfd..1864a84 100644 --- a/services/upload/internal/handler/upload.go +++ b/services/upload/internal/handler/upload.go @@ -11,11 +11,12 @@ import ( "strconv" "strings" - "flowix/upload/internal/metrics" - mw "flowix/upload/internal/middleware" + "flowix/pkg/metrics" + mw "flowix/pkg/middleware" ) -// UploadHandler dependencies as interfaces — T3 stepwise fakes without MinIO/RabbitMQ. +// Storage abstracts the object store used by upload handlers — stepwise +// fakes in tests work without MinIO/RabbitMQ. type Storage interface { PutObject(ctx context.Context, key string, reader io.Reader, size int64, contentType string) error } diff --git a/services/upload/internal/handler/upload_test.go b/services/upload/internal/handler/upload_test.go index 643aebc..8049731 100644 --- a/services/upload/internal/handler/upload_test.go +++ b/services/upload/internal/handler/upload_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - mw "flowix/upload/internal/middleware" + mw "flowix/pkg/middleware" "github.com/golang-jwt/jwt/v5" ) diff --git a/services/upload/internal/metrics/metrics.go b/services/upload/internal/metrics/metrics.go deleted file mode 100644 index 2fecd00..0000000 --- a/services/upload/internal/metrics/metrics.go +++ /dev/null @@ -1,36 +0,0 @@ -package metrics - -import ( - "net/http" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var ( - UploadBytes = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "upload_bytes", - Help: "Total uploaded bytes", - }) - RabbitMQQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "rabbitmq_queue_depth", - Help: "RabbitMQ queue depth", - }) - VodCacheHit = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "vod_cache_hit", - Help: "VOD cache hits", - }) - FfmpegDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "ffmpeg_duration_seconds", - Help: "FFmpeg duration", - Buckets: prometheus.DefBuckets, - }) -) - -func init() { - prometheus.MustRegister(UploadBytes, RabbitMQQueueDepth, VodCacheHit, FfmpegDuration) -} - -func Handler() http.Handler { - return promhttp.Handler() -} diff --git a/services/upload/internal/middleware/auth.go b/services/upload/internal/middleware/auth.go deleted file mode 100644 index 29ad493..0000000 --- a/services/upload/internal/middleware/auth.go +++ /dev/null @@ -1,56 +0,0 @@ -// Package middleware provides JWT auth for upload service. -package middleware - -import ( - "context" - "net/http" - "strings" - - "github.com/golang-jwt/jwt/v5" -) - -type ctxKey string - -const UserIDKey ctxKey = "user_id" - -func AuthMiddleware(secret string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := r.Header.Get("Authorization") - if h == "" || !strings.HasPrefix(h, "Bearer ") { - http.Error(w, `{"error":"missing token"}`, http.StatusUnauthorized) - return - } - tokenStr := strings.TrimPrefix(h, "Bearer ") - token, err := jwt.Parse(tokenStr, func(t *jwt.Token) (interface{}, error) { - return []byte(secret), nil - }, jwt.WithValidMethods([]string{"HS256"}), jwt.WithExpirationRequired()) - if err != nil || !token.Valid { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - sub, _ := claims["sub"].(string) - if sub == "" { - http.Error(w, `{"error":"invalid token"}`, http.StatusUnauthorized) - return - } - // тип токена обязан быть access (issue #53: обязательные exp и type) - if typ, _ := claims["type"].(string); typ != "access" { - http.Error(w, `{"error":"invalid token type"}`, http.StatusUnauthorized) - return - } - ctx := context.WithValue(r.Context(), UserIDKey, sub) - next.ServeHTTP(w, r.WithContext(ctx)) - }) - } -} - -func UserIDFromCtx(ctx context.Context) string { - v, _ := ctx.Value(UserIDKey).(string) - return v -} diff --git a/services/upload/internal/middleware/auth_test.go b/services/upload/internal/middleware/auth_test.go deleted file mode 100644 index fca93f3..0000000 --- a/services/upload/internal/middleware/auth_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Package middleware tests — T3 upload auth. -package middleware - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/golang-jwt/jwt/v5" -) - -func mustToken(secret, sub string) string { - t := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ - "sub": sub, - "type": "access", - "exp": time.Now().Add(time.Hour).Unix(), - }) - s, _ := t.SignedString([]byte(secret)) - return s -} - -// issue #53: token without exp or type=access must be rejected. -func TestAuthRejectsTokenWithoutExpOrType(t *testing.T) { - secret := "test-secret" - noExp := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"sub": "u1", "type": "access"}) - noType := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"sub": "u1", "exp": time.Now().Add(time.Hour).Unix()}) - for name, tok := range map[string]*jwt.Token{"no_exp": noExp, "no_type": noType} { - s, _ := tok.SignedString([]byte(secret)) - h := AuthMiddleware(secret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("should not call") })) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer "+s) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("%s: want 401 got %d", name, w.Code) - } - } -} - -func TestAuthOK(t *testing.T) { - secret := "test-secret-32chars-for-upload-tests!!" - next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if UserIDFromCtx(r.Context()) != "u1" { - t.Fatalf("want u1 got %s", UserIDFromCtx(r.Context())) - } - w.WriteHeader(200) - }) - h := AuthMiddleware(secret)(next) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer "+mustToken(secret, "u1")) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 200 { - t.Fatalf("want 200 got %d", w.Code) - } -} - -func TestAuthMissing(t *testing.T) { - h := AuthMiddleware("s")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("no call") })) - req := httptest.NewRequest("GET", "/", nil) - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("want 401 got %d", w.Code) - } -} - -func TestAuthInvalid(t *testing.T) { - h := AuthMiddleware("s")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Fatal("no call") })) - req := httptest.NewRequest("GET", "/", nil) - req.Header.Set("Authorization", "Bearer bad.token") - w := httptest.NewRecorder() - h.ServeHTTP(w, req) - if w.Code != 401 { - t.Fatalf("want 401 got %d", w.Code) - } -} diff --git a/services/upload/internal/middleware/logger.go b/services/upload/internal/middleware/logger.go deleted file mode 100644 index b2a8a43..0000000 --- a/services/upload/internal/middleware/logger.go +++ /dev/null @@ -1,48 +0,0 @@ -package middleware - -import ( - "net/http" - "time" - - "github.com/rs/zerolog/log" -) - -// RequestLogger — structured logging via zerolog, includes trace_id -func RequestLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - ww := &respWriter{ResponseWriter: w, status: http.StatusOK} - next.ServeHTTP(ww, r) - dur := time.Since(start) - reqID := r.Header.Get("X-Request-ID") - if reqID == "" { - reqID = r.Header.Get("X-Request-Id") - } - if reqID == "" { - reqID = r.Header.Get("X-Correlation-ID") - } - ev := log.Info() - if ww.status >= 500 { - ev = log.Error() - } else if ww.status >= 400 { - ev = log.Warn() - } - ev.Str("method", r.Method). - Str("path", r.URL.Path). - Int("status", ww.status). - Dur("duration", dur). - Str("trace_id", reqID). - Str("service", "upload"). - Msg("request") - }) -} - -type respWriter struct { - http.ResponseWriter - status int -} - -func (w *respWriter) WriteHeader(code int) { - w.status = code - w.ResponseWriter.WriteHeader(code) -} diff --git a/services/upload/internal/queue/publisher.go b/services/upload/internal/queue/publisher.go index a700eec..071d1c0 100644 --- a/services/upload/internal/queue/publisher.go +++ b/services/upload/internal/queue/publisher.go @@ -1,3 +1,4 @@ +// Package queue publishes transcoding events to RabbitMQ from the upload service. package queue import ( diff --git a/services/upload/internal/storage/minio.go b/services/upload/internal/storage/minio.go index 1bb182f..4eb1ccc 100644 --- a/services/upload/internal/storage/minio.go +++ b/services/upload/internal/storage/minio.go @@ -1,3 +1,4 @@ +// Package storage implements object storage backends for upload/metadata (MinIO). package storage import ( @@ -59,9 +60,9 @@ func (m *MinioClient) PresignedPutObject(ctx context.Context, key string, expire return u.String(), nil } -// PresignedPutObjectWithURL returns presigned URL for external access. -// Previously we rewrote host after signing, which broke AWS SigV4 (host is signed). -// Now we generate the signature with the public host directly if provided. +// PresignedPutObjectExternal returns a presigned PUT URL for external +// (browser) access. Previously we rewrote the host after signing, which broke +// AWS SigV4 (host is signed); now we sign with the public host directly. func (m *MinioClient) PresignedPutObjectExternal(ctx context.Context, key string, expires time.Duration, publicEndpoint string) (string, error) { if publicEndpoint == "" { return m.PresignedPutObject(ctx, key, expires)