docker-compose.yml

services:
  telegram-bot-api:
    image: aiogram/telegram-bot-api:latest
    container_name: telegram-bot-api
    restart: always
    environment:
      TELEGRAM_API_ID: "${TELEGRAM_API_ID}"
      TELEGRAM_API_HASH: "${TELEGRAM_API_HASH}"
      TELEGRAM_LOCAL: "1"
    volumes:
      - telegram-bot-api-data:/var/lib/telegram-bot-api
    ports:
      - "127.0.0.1:8666:8081"
    healthcheck:
      test: ["CMD-SHELL", "ss -tlnp | grep -q ':8081' || netstat -tlnp | grep -q ':8081' || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 30s

  redis:
    image: redis:7-alpine
    container_name: scoutbot-redis
    ports:
      - "6222:6379"
    volumes:
      - redis_data:/data
    # Redis configuration with memory management
    command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru --save 900 1 --save 300 10
    restart: unless-stopped
    # Memory limits for Redis
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.5'
        reservations:
          memory: 128M
          cpus: '0.25'
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 3s
      retries: 3

  pentaract:
    build:
      context: ./pentaract-api
      dockerfile: Dockerfile
    container_name: pentaract
    restart: always
    environment:
      - PYTHONUNBUFFERED=1
      - PENTARACT_EMAIL=${PENTARACT_EMAIL}
      - PENTARACT_PASSWORD=${PENTARACT_PASSWORD}
      - PENTARACT_USERNAME=pablo
    ports:
      - "8547:8547"
    volumes:
      - pentaract_data:/data
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8547/api/health || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 20s
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'

  scoutbot:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scoutbot
    user: "101:101"  # Align with telegram-bot-api UID/GID for file access
    ports:
      - "8916:8916"
    # Memory and resource limits optimized for lower consumption
    deploy:
      resources:
        limits:
          memory: 2G  # Reduced from 4G (optimized)
          cpus: '0.75'  # Reduced from 1.0 (optimized)
        reservations:
          memory: 512M  # Reduced from 1G (optimized)
          cpus: '0.25'  # Reduced from 0.5 (optimized)
    environment:
      - NODE_ENV=production
      - BOT_TOKEN=${BOT_TOKEN}
      - DATABASE_URL=file:/app/data/production.db
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - PORT=8916
      - HOST=0.0.0.0
      - ENVIRONMENT=production
      - LOG_LEVEL=${LOG_LEVEL:-info}
      # Timezone Configuration
      - TZ=${TZ:-UTC}
      # Reddit OAuth API credentials
      - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID}
      - REDDIT_CLIENT_SECRET=${REDDIT_CLIENT_SECRET}
      - REDDIT_USERNAME=${REDDIT_USERNAME}
      - REDDIT_PASSWORD=${REDDIT_PASSWORD}
      # Feature flags for Reddit
      - USE_REDDIT_API=${USE_REDDIT_API:-true}
      - USE_REDDIT_JSON_FALLBACK=${USE_REDDIT_JSON_FALLBACK:-true}
      - FEATURE_INSTAGRAM=${FEATURE_INSTAGRAM:-false}
      # Access Control
      - ALLOWED_USER_ID=${ALLOWED_USER_ID}
      # Resilience System Configuration
      - TELEGRAM_RESILIENCE_ENABLED=${TELEGRAM_RESILIENCE_ENABLED:-true}
      - TELEGRAM_MAX_RETRIES=${TELEGRAM_MAX_RETRIES:-10}
      - TELEGRAM_BASE_DELAY=${TELEGRAM_BASE_DELAY:-1000}
      - TELEGRAM_MAX_DELAY=${TELEGRAM_MAX_DELAY:-60000}
      - TELEGRAM_CIRCUIT_BREAKER_THRESHOLD=${TELEGRAM_CIRCUIT_BREAKER_THRESHOLD:-5}
      - TELEGRAM_CIRCUIT_BREAKER_TIMEOUT=${TELEGRAM_CIRCUIT_BREAKER_TIMEOUT:-300000}
      # Message Queue Configuration
      - MESSAGE_QUEUE_ENABLED=${MESSAGE_QUEUE_ENABLED:-true}
      - MESSAGE_QUEUE_MAX_SIZE=${MESSAGE_QUEUE_MAX_SIZE:-1000}
      - MESSAGE_QUEUE_BATCH_SIZE=${MESSAGE_QUEUE_BATCH_SIZE:-20}
      - MESSAGE_QUEUE_PROCESSING_INTERVAL=${MESSAGE_QUEUE_PROCESSING_INTERVAL:-5000}
      - MESSAGE_QUEUE_MESSAGE_TTL=${MESSAGE_QUEUE_MESSAGE_TTL:-3600000}
      # Health Monitoring
      - HEALTH_CHECK_INTERVAL=${HEALTH_CHECK_INTERVAL:-30000}
      - ALERT_THRESHOLD_ERROR_RATE=${ALERT_THRESHOLD_ERROR_RATE:-0.1}
      - ALERT_THRESHOLD_DOWNTIME_MINUTES=${ALERT_THRESHOLD_DOWNTIME_MINUTES:-15}
      - ALERT_THRESHOLD_QUEUE_SIZE=${ALERT_THRESHOLD_QUEUE_SIZE:-500}
      # Job Cleanup Configuration
      - JOB_CLEANUP_ENABLED=${JOB_CLEANUP_ENABLED:-true}
      - JOB_CLEANUP_INTERVAL_MINUTES=${JOB_CLEANUP_INTERVAL_MINUTES:-30}
      - JOB_CLEANUP_THOROUGH_INTERVAL_HOURS=${JOB_CLEANUP_THOROUGH_INTERVAL_HOURS:-2}
      - JOB_CLEANUP_ORPHANED_THRESHOLD=${JOB_CLEANUP_ORPHANED_THRESHOLD:-10}
      # Anti-Blocking System Configuration
      - ANTI_BLOCK_ENABLED=${ANTI_BLOCK_ENABLED:-true}
      - ANTI_BLOCK_MIN_DELAY=${ANTI_BLOCK_MIN_DELAY:-5.0}
      - ANTI_BLOCK_MAX_DELAY=${ANTI_BLOCK_MAX_DELAY:-300.0}
      - ANTI_BLOCK_CIRCUIT_BREAKER_THRESHOLD=${ANTI_BLOCK_CIRCUIT_BREAKER_THRESHOLD:-5}
      # Video Download Configuration
      - ENABLE_FFMPEG=${ENABLE_FFMPEG:-true}
      - ENABLE_ARIA2=${ENABLE_ARIA2:-false}
      - AUDIO_FORMAT=${AUDIO_FORMAT:-m4a}
      # Media Toolbox Configuration
      - ENABLE_CLIP=${ENABLE_CLIP:-true}
      - ENABLE_GIF=${ENABLE_GIF:-true}
      - MAX_GIF_SIZE=${MAX_GIF_SIZE:-10}
      - GIF_DURATION_LIMIT=${GIF_DURATION_LIMIT:-15}
      # Job System Configuration
      - JOB_QUEUE_BACKEND=${JOB_QUEUE_BACKEND:-apscheduler}
      - JOB_PERSISTENCE_ENABLED=${JOB_PERSISTENCE_ENABLED:-true}
      - JOB_STATUS_TRACKING=${JOB_STATUS_TRACKING:-true}
      # ImageMagick Configuration
      - ENABLE_IMAGEMAGICK=${ENABLE_IMAGEMAGICK:-true}
      - ENABLE_STICKERS=${ENABLE_STICKERS:-true}
      - ENABLE_MEMES=${ENABLE_MEMES:-true}
      # OCR Configuration
      - ENABLE_OCR=${ENABLE_OCR:-false}
      - TESSERACT_LANG=${TESSERACT_LANG:-por+eng}
      # Webhook Configuration
      - USE_WEBHOOK=${USE_WEBHOOK:-false}
      - WEBHOOK_URL=${WEBHOOK_URL:-}
      - WEBHOOK_SECRET=${WEBHOOK_SECRET:-}
      - WEBHOOK_PORT=${WEBHOOK_PORT:-8916}
      # Telegram Bot API Server Configuration
      - TELEGRAM_API_ID=${TELEGRAM_API_ID}
      - TELEGRAM_API_HASH=${TELEGRAM_API_HASH}
      - TELEGRAM_BOT_API_SERVER_URL=${TELEGRAM_BOT_API_SERVER_URL}
      - TELEGRAM_USE_LOCAL_API=${TELEGRAM_USE_LOCAL_API:-false}
      - TELEGRAM_BOT_API_DATA_PATH=/var/lib/telegram-bot-api
      # Spotify Configuration
      - SPOTIFY_CLIENT_ID=${SPOTIFY_CLIENT_ID}
      - SPOTIFY_CLIENT_SECRET=${SPOTIFY_CLIENT_SECRET}
      - SPOTIFY_AUDIO_FORMAT=${SPOTIFY_AUDIO_FORMAT:-mp3}
      - SPOTIFY_BITRATE=${SPOTIFY_BITRATE:-128k}
      - SPOTIFY_AUDIO_PROVIDERS=${SPOTIFY_AUDIO_PROVIDERS:-youtube-music,youtube}
      - SPOTIFY_LYRICS_PROVIDERS=${SPOTIFY_LYRICS_PROVIDERS:-genius,musixmatch,azlyrics}
      - SPOTIFY_ENABLED=${SPOTIFY_ENABLED:-true}
      # Pentaract Storage Configuration
      - PENTARACT_ENABLED=${PENTARACT_ENABLED:-false}
      - PENTARACT_API_URL=${PENTARACT_API_URL:-http://localhost:8547/api}
      - PENTARACT_EMAIL=${PENTARACT_EMAIL}
      - PENTARACT_PASSWORD=${PENTARACT_PASSWORD}
      - PENTARACT_UPLOAD_THRESHOLD=${PENTARACT_UPLOAD_THRESHOLD:-50}
      - PENTARACT_AUTO_CLEANUP=${PENTARACT_AUTO_CLEANUP:-true}
      - PENTARACT_CLEANUP_INTERVAL=${PENTARACT_CLEANUP_INTERVAL:-30}
      - PENTARACT_MAX_CONCURRENT_UPLOADS=${PENTARACT_MAX_CONCURRENT_UPLOADS:-3}
      - PENTARACT_TIMEOUT=${PENTARACT_TIMEOUT:-30}
      - PENTARACT_RETRY_ATTEMPTS=${PENTARACT_RETRY_ATTEMPTS:-3}
      # YouTube Authentication Configuration (yt-dlp unified)
      - YTDLP_AUTH_MODE=${YTDLP_AUTH_MODE:-both}
      - YTDLP_COOKIES_FILE=${YTDLP_COOKIES_FILE:-/secrets/youtube-cookies.txt}
      - YTDLP_COOKIES_FROM_BROWSER=${YTDLP_COOKIES_FROM_BROWSER:-}
      - YTDLP_BROWSER_PROFILE=${YTDLP_BROWSER_PROFILE:-default}
      - YTDLP_FIREFOX_CONTAINER=${YTDLP_FIREFOX_CONTAINER:-none}
      - YTDLP_PO_TOKEN=${YTDLP_PO_TOKEN:-}
      - YTDLP_CHROMIUM_PATH=${YTDLP_CHROMIUM_PATH:-/usr/bin/chromium}
      - YTDLP_PLAYER_CLIENTS=${YTDLP_PLAYER_CLIENTS:-default,mweb}
      - YTDLP_SLEEP_INTERVAL=${YTDLP_SLEEP_INTERVAL:-6}
      - YTDLP_MAX_SLEEP_INTERVAL=${YTDLP_MAX_SLEEP_INTERVAL:-10}
      - YTDLP_RETRIES=${YTDLP_RETRIES:-5}
      # Ensure Node.js is available for yt-dlp signature solving
      - PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin
    volumes:
      - app_data:/app/data
      - backups_data:/app/backups
      - tmp_data:/tmp  # Temporary files for downloads
      - telegram-bot-api-data:/var/lib/telegram-bot-api:ro  # Access to Bot API files (read-only, files are copied to /tmp for processing)
      - ./secrets:/secrets  # Secrets directory (cookies, tokens, etc.) - read-write for cookie updates
    depends_on:
      redis:
        condition: service_healthy
      telegram-bot-api:
        condition: service_healthy
      pentaract:
        condition: service_healthy
    restart: always
    # Enhanced health check with optimized intervals
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8916/health"]
      interval: 60s  # Increased from 30s (optimized)
      timeout: 10s
      retries: 10
      start_period: 60s
    # Prevent OOM kills
    oom_kill_disable: false
    # Faster restart on failure
    stop_grace_period: 10s

volumes:
  redis_data:
    driver: local
  app_data:
    driver: local
  backups_data:
    driver: local
  tmp_data:
    driver: local
  telegram-bot-api-data:
    driver: local
  pentaract_data:
    driver: local

networks:
  default:
    name: scoutbot_network
    driver: bridge

# ============================================================================
# DATA PERSISTENCE AND VOLUME MANAGEMENT
# ============================================================================
#
# IMPORTANT: All data is persisted in Docker named volumes that survive
# container rebuilds, restarts, and removals. Your data is SAFE when using
# normal deployment commands.
#
# ----------------------------------------------------------------------------
# VOLUMES EXPLANATION
# ----------------------------------------------------------------------------
#
# The following volumes are used to persist data:
#
# 1. app_data:/app/data
#    - Stores the SQLite database (production.db)
#    - Contains all application data (feeds, chats, user settings, etc.)
#    - CRITICAL: This volume contains your database - never remove it!
#
# 2. backups_data:/app/backups
#    - Stores automatic and manual backups
#    - Used for backup/restore operations
#
# 3. tmp_data:/tmp
#    - Temporary files for downloads and processing
#    - Can be safely cleared, but persists between restarts for efficiency
#
# 4. redis_data:/data
#    - Redis persistent data (AOF files)
#    - Contains cached data and job queue state
#
# 5. telegram-bot-api-data:/var/lib/telegram-bot-api
#    - Telegram Bot API server data
#    - Downloaded files and session data
#
# 6. pentaract_data:/data
#    - Pentaract storage service data
#    - Uploaded files and metadata
#
# ----------------------------------------------------------------------------
# DEPLOYMENT COMMANDS
# ----------------------------------------------------------------------------
#
# SAFE - Updates preserving all data (RECOMMENDED):
#   docker compose up -d --build
#
#   This command:
#   - Rebuilds containers with latest code
#   - Preserves ALL volumes and data
#   - Applies database migrations automatically
#   - Your data is 100% safe
#
# DANGEROUS - Removes ALL data (use with caution!):
#   docker-compose down -v && docker-compose up -d --build
#
#   This command:
#   - Removes all volumes (-v flag)
#   - Deletes ALL databases and data
#   - Creates fresh empty database
#   - ONLY use when you want to start completely fresh
#
# ----------------------------------------------------------------------------
# BACKUP AND RESTORE
# ----------------------------------------------------------------------------
#
# Manual backup (recommended before major updates):
#   docker exec scoutbot python -c "from app.database import database; import shutil; shutil.copy('/app/data/production.db', '/app/backups/production.db.backup')"
#
# List volumes:
#   docker volume ls | grep scoutbot
#
# Inspect volume location:
#   docker volume inspect scoutbot_app_data
#
# Backup entire volume (advanced):
#   docker run --rm -v scoutbot_app_data:/data -v $(pwd):/backup alpine tar czf /backup/app_data_backup.tar.gz /data
#
# Restore volume (advanced):
#   docker run --rm -v scoutbot_app_data:/data -v $(pwd):/backup alpine tar xzf /backup/app_data_backup.tar.gz -C /data
#
# ----------------------------------------------------------------------------
# VERIFICATION
# ----------------------------------------------------------------------------
#
# Verify data persistence after rebuild:
#   docker exec scoutbot ls -lh /app/data/
#   docker exec scoutbot sqlite3 /app/data/production.db "SELECT COUNT(*) FROM usersettings;"
#
# Check volume usage:
#   docker system df -v | grep scoutbot
#
# ----------------------------------------------------------------------------
# MIGRATION SAFETY
# ----------------------------------------------------------------------------
#
# Database migrations are automatically applied on startup:
# - Migrations are validated before and after execution
# - Database integrity is checked automatically
# - Failed migrations are logged but don't break startup
# - All migrations are idempotent (safe to run multiple times)
#
# The migration system will:
# - Detect missing columns in existing tables
# - Add missing columns with appropriate defaults
# - Validate that migrations succeeded
# - Preserve all existing data
#
# ----------------------------------------------------------------------------
# TROUBLESHOOTING
# ----------------------------------------------------------------------------
#
# If you suspect data loss:
# 1. Check volume exists: docker volume ls | grep scoutbot
# 2. Verify database file: docker exec scoutbot ls -lh /app/data/production.db
# 3. Check logs: docker logs scoutbot | grep -i "migration\|integrity\|database"
# 4. Inspect volume: docker volume inspect scoutbot_app_data
#
# If database seems corrupted:
# 1. Check integrity: docker exec scoutbot sqlite3 /app/data/production.db "PRAGMA integrity_check;"
# 2. Review migration logs: docker logs scoutbot | grep -i migration
# 3. Restore from backup if available
#
# ============================================================================