diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 48bb72437..741c28c35 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -44,10 +44,11 @@ jobs: echo "HUGGINGFACE_ACCESS_TOKEN=${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}" >> $GITHUB_ENV echo "RELEASE_VERSION=${GITHUB_REF##refs/heads/}" | sed 's/\//-/g' >> $GITHUB_ENV - - name: Build and push the images to Docker Hub + - name: Build and push the base image to Docker Hub uses: docker/bake-action@v2 with: push: true + targets: base set: | *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} diff --git a/Dockerfile b/Dockerfile index 2d7e08e3e..b007ea764 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ WORKDIR / RUN uv pip install runpod requests websocket-client # Add application code and scripts -ADD src/start.sh handler.py test_input.json ./ +ADD src/start.sh src/network_volume.py handler.py test_input.json ./ RUN chmod +x /start.sh # Add script to install custom nodes diff --git a/docs/configuration.md b/docs/configuration.md index 8a7c58691..25bed9e70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,9 +12,10 @@ This document outlines the environment variables available for configuring the ` ## Logging Configuration -| Environment Variable | Description | Default | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `COMFY_LOG_LEVEL` | Controls ComfyUI's internal logging verbosity. Options: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Use `DEBUG` for troubleshooting, `INFO` for production. | `DEBUG` | +| Environment Variable | Description | Default | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `COMFY_LOG_LEVEL` | Controls ComfyUI's internal logging verbosity. Options: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Use `DEBUG` for troubleshooting, `INFO` for production. | `DEBUG` | +| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volumes & Model Paths](network-volumes.md). | `false` | ## Debugging Configuration @@ -24,8 +25,6 @@ This document outlines the environment variables available for configuring the ` | `WEBSOCKET_RECONNECT_DELAY_S` | Delay in seconds between websocket reconnection attempts. | `3` | | `WEBSOCKET_TRACE` | Enable low-level websocket frame tracing for protocol debugging. Set to `true` only when diagnosing connection issues. | `false` | -> [!TIP] > **For troubleshooting:** Set `COMFY_LOG_LEVEL=DEBUG` to get detailed logs when ComfyUI crashes or behaves unexpectedly. This helps identify the exact point of failure in your workflows. - ## AWS S3 Upload Configuration Configure these variables **only** if you want the worker to upload generated images directly to an AWS S3 bucket. If these are not set, images will be returned as base64-encoded strings in the API response. diff --git a/docs/customization.md b/docs/customization.md index b75839b51..715791e49 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -2,7 +2,9 @@ This guide covers methods for adding your own models, custom nodes, and static input files into a custom `worker-comfyui`. -> [!TIP] > **Looking for the easiest way to deploy custom workflows?** +> [!TIP] +> +> **Looking for the easiest way to deploy custom workflows?** > > [ComfyUI-to-API](https://comfy.getrunpod.io) automatically generates a custom Dockerfile and GitHub repository from your ComfyUI workflow, eliminating the manual setup described below. See the [ComfyUI-to-API Documentation](https://docs.runpod.io/community-solutions/comfyui-to-api/overview) for details. > @@ -90,20 +92,21 @@ Using a Network Volume is primarily useful if you want to manage **models** sepa 1. **Create a Network Volume**: - Follow the [RunPod Network Volumes guide](https://docs.runpod.io/pods/storage/create-network-volumes) to create a volume in the same region as your endpoint. 2. **Populate the Volume with Models**: - - Use one of the methods described in the RunPod guide (e.g., temporary Pod + `wget`, direct upload) to place your model files into the correct ComfyUI directory structure **within the volume**. The root of the volume corresponds to `/workspace` inside the container. + - Use one of the methods described in the RunPod guide (e.g., temporary Pod + `wget`, direct upload, or the S3-compatible API) to place your model files into the correct ComfyUI directory structure **within the volume**. + - For **serverless endpoints**, the network volume is mounted at `/runpod-volume`, and ComfyUI expects models under `/runpod-volume/models/...`. See [Network Volumes & Model Paths](network-volumes.md) for the exact structure and debugging tips. ```bash - # Example structure inside the Network Volume: - # /models/checkpoints/your_model.safetensors - # /models/loras/your_lora.pt - # /models/vae/your_vae.safetensors + # Example structure inside the Network Volume (serverless worker view): + # /runpod-volume/models/checkpoints/your_model.safetensors + # /runpod-volume/models/loras/your_lora.pt + # /runpod-volume/models/vae/your_vae.safetensors ``` - - **Important:** Ensure models are placed in the correct subdirectories (e.g., checkpoints in `models/checkpoints`, LoRAs in `models/loras`). + - **Important:** Ensure models are placed in the correct subdirectories (e.g., checkpoints in `models/checkpoints`, LoRAs in `models/loras`). If models are not detected, enable `NETWORK_VOLUME_DEBUG` as described in [Network Volumes & Model Paths](network-volumes.md). 3. **Configure Your Endpoint**: - Use the Network Volume in your endpoint configuration: - Either create a new endpoint or update an existing one (see [Deployment Guide](deployment.md)). - In the endpoint configuration, under `Advanced > Select Network Volume`, select your Network Volume. -**Note:** - -- When a Network Volume is correctly attached, ComfyUI running inside the worker container will automatically detect and load models from the standard directories (`/workspace/models/...`) within that volume. -- This method is **not suitable for installing custom nodes**; use the Custom Dockerfile method for that. +> [!NOTE] +> +> - When a Network Volume is correctly attached, ComfyUI running inside the worker container will automatically detect and load models from the standard directories (`/runpod-volume/models/...`) within that volume (for serverless workers). For directory mapping details and troubleshooting, see [Network Volumes & Model Paths](network-volumes.md). +> - This method is **not suitable for installing custom nodes**; use the Custom Dockerfile method for that. diff --git a/docs/deployment.md b/docs/deployment.md index 210728b0a..7e020eb81 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -16,7 +16,7 @@ This is the simplest method if the official images meet your needs. - Container Registry Credentials: Leave as default (images are public). - Container Disk: Adjust based on the chosen image tag, see [GPU Recommendations](#gpu-recommendations). - (optional) Environment Variables: Configure S3 or other settings (see [Configuration Guide](configuration.md)). - - Note: If you don't configure S3, images are returned as base64. For persistent storage across jobs without S3, consider using a [Network Volume](customization.md#method-2-network-volume-alternative-for-models). + - Note: If you don't configure S3, images are returned as base64. For persistent storage across jobs without S3, consider using a [Network Volume](customization.md#method-2-network-volume-alternative-for-models). If models on your network volume are not being detected, see [Network Volumes & Model Paths](network-volumes.md) for troubleshooting steps. - Click on `Save Template` ### Create your endpoint @@ -32,7 +32,7 @@ This is the simplest method if the official images meet your needs. - Idle Timeout: `5` (Default is usually fine, adjust if needed). - Flash Boot: `enabled` (Recommended for faster worker startup). - Select Template: `worker-comfyui` (or the name you gave your template). - - (optional) Advanced: If you are using a Network Volume, select it under `Select Network Volume`. See the [Customization Guide](customization.md#method-2-network-volume-alternative-for-models). + - (optional) Advanced: If you are using a Network Volume, select it under `Select Network Volume`. See the [Customization Guide](customization.md#method-2-network-volume-alternative-for-models). For detailed model path layout and debugging tips, see [Network Volumes & Model Paths](network-volumes.md). - Click `deploy` - Your endpoint will be created. You can click on it to view the dashboard and find its ID. diff --git a/docs/network-volumes.md b/docs/network-volumes.md new file mode 100644 index 000000000..c9e634ce5 --- /dev/null +++ b/docs/network-volumes.md @@ -0,0 +1,147 @@ +# Network Volumes & Model Paths + +This document explains how to use RunPod **Network Volumes** with `worker-comfyui`, how model paths are resolved inside the container, and how to debug cases where models are not detected. + +> **Scope** +> +> These instructions apply to **serverless endpoints** using this worker. Pods mount network volumes at `/workspace` by default, while serverless workers see them at `/runpod-volume`. + +## Directory Mapping + +For **serverless endpoints**: + +- Network volume root is mounted at: `/runpod-volume` +- ComfyUI models are expected under: `/runpod-volume/models/...` + +For **Pods**: + +- Network volume root is mounted at: `/workspace` +- Equivalent ComfyUI model path: `/workspace/models/...` + +If you use the S3-compatible API, the same paths map as: + +- Serverless: `/runpod-volume/my-folder/file.txt` +- Pod: `/workspace/my-folder/file.txt` +- S3 API: `s3:///my-folder/file.txt` + +## Expected Directory Structure + +Models must be placed in the following structure on your network volume: + +```text +/runpod-volume/ +└── models/ + ├── checkpoints/ # Stable Diffusion checkpoints (.safetensors, .ckpt) + ├── loras/ # LoRA files (.safetensors, .pt) + ├── vae/ # VAE models (.safetensors, .pt) + ├── clip/ # CLIP models (.safetensors, .pt) + ├── clip_vision/ # CLIP Vision models + ├── controlnet/ # ControlNet models (.safetensors, .pt) + ├── embeddings/ # Textual inversion embeddings (.safetensors, .pt) + ├── upscale_models/ # Upscaling models (.safetensors, .pt) + ├── unet/ # UNet models + └── configs/ # Model configs (.yaml, .json) +``` + +> **Note** +> +> Only create the subdirectories you actually need; empty or missing folders are fine. + +## Supported File Extensions + +ComfyUI only recognizes files with specific extensions when scanning model directories. + +| Model Type | Supported Extensions | +| -------------- | ------------------------------------------- | +| Checkpoints | `.safetensors`, `.ckpt`, `.pt`, `.pth`, `.bin` | +| LoRAs | `.safetensors`, `.pt` | +| VAE | `.safetensors`, `.pt`, `.bin` | +| CLIP | `.safetensors`, `.pt`, `.bin` | +| ControlNet | `.safetensors`, `.pt`, `.pth`, `.bin` | +| Embeddings | `.safetensors`, `.pt`, `.bin` | +| Upscale Models | `.safetensors`, `.pt`, `.pth` | + +Files with other extensions (for example `.txt`, `.zip`) are **ignored** by ComfyUI’s model discovery. + +## Common Issues + +- **Wrong root directory** + - Models placed directly under `/runpod-volume/checkpoints/...` instead of `/runpod-volume/models/checkpoints/...`. +- **Incorrect extensions** + - Files named without one of the supported extensions are skipped. +- **Empty directories** + - No actual model files present in `models/checkpoints` (or other folders). +- **Volume not attached** + - Endpoint created without selecting a network volume under **Advanced → Select Network Volume**. + +If any of the above is true, ComfyUI will silently fail to discover models from the network volume. + +## Debugging with `NETWORK_VOLUME_DEBUG` + +The worker exposes an opt‑in debug mode controlled via the `NETWORK_VOLUME_DEBUG` environment variable. + +### When to Use + +Enable this when: + +- Models on your network volume are not appearing in ComfyUI +- You suspect the directory structure or file extensions are wrong +- You want to quickly verify what the worker can actually see on `/runpod-volume` + +### How to Enable + +1. Go to your serverless **Endpoint → Manage → Edit**. +2. Under **Environment Variables**, add: + + - `NETWORK_VOLUME_DEBUG=true` + +3. Save and wait for workers to restart (or scale to zero and back up). +4. Send any request to your endpoint (even a minimal one) to trigger the diagnostics. + +### Reading the Diagnostics + +When enabled, each request prints a detailed report to the worker logs, for example: + +```text +====================================================================== +NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true) +====================================================================== + +[1] Checking extra_model_paths.yaml configuration... + ✓ FOUND: /comfyui/extra_model_paths.yaml + +[2] Checking network volume mount at /runpod-volume... + ✓ MOUNTED: /runpod-volume + +[3] Checking directory structure... + ✓ FOUND: /runpod-volume/models + +[4] Scanning model directories... + + checkpoints/: + - my-model.safetensors (6.5 GB) + + loras/: + - style-lora.safetensors (144.2 MB) + +[5] Summary + ✓ Models found on network volume! +====================================================================== +``` + +If there is a problem, the diagnostics will instead highlight it, for example: + +- Missing `models/` directory +- No valid model files in any subdirectory +- Files present but ignored due to wrong extensions + +### Disabling Debug Mode + +Once you have resolved your issue, disable diagnostics to keep logs clean: + +- Remove the `NETWORK_VOLUME_DEBUG` environment variable, **or** +- Set `NETWORK_VOLUME_DEBUG=false` + +This returns the worker to normal behavior without extra log noise. + + diff --git a/handler.py b/handler.py index e966744fd..65c8390b0 100644 --- a/handler.py +++ b/handler.py @@ -13,6 +13,18 @@ import tempfile import socket import traceback +import logging + +from network_volume import ( + is_network_volume_debug_enabled, + run_network_volume_diagnostics, +) + +# --------------------------------------------------------------------------- +# Logging setup +# --------------------------------------------------------------------------- +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) # Time to wait between API check attempts in milliseconds COMFY_API_AVAILABLE_INTERVAL_MS = 50 @@ -502,6 +514,12 @@ def handler(job): Returns: dict: A dictionary containing either an error message or a success status with generated images. """ + # --------------------------------------------------------------------------- + # Network Volume Diagnostics (opt-in via NETWORK_VOLUME_DEBUG=true) + # --------------------------------------------------------------------------- + if is_network_volume_debug_enabled(): + run_network_volume_diagnostics() + job_input = job["input"] job_id = job["id"] diff --git a/src/network_volume.py b/src/network_volume.py new file mode 100644 index 000000000..ec6d34739 --- /dev/null +++ b/src/network_volume.py @@ -0,0 +1,153 @@ +""" +Network Volume diagnostics for worker-comfyui. + +This module provides tools to debug network volume model path issues. +Enable diagnostics by setting NETWORK_VOLUME_DEBUG=true environment variable. +""" + +import os + +# Expected model types and their file extensions +MODEL_TYPES = { + "checkpoints": [".safetensors", ".ckpt", ".pt", ".pth", ".bin"], + "clip": [".safetensors", ".pt", ".bin"], + "clip_vision": [".safetensors", ".pt", ".bin"], + "configs": [".yaml", ".json"], + "controlnet": [".safetensors", ".pt", ".pth", ".bin"], + "embeddings": [".safetensors", ".pt", ".bin"], + "loras": [".safetensors", ".pt"], + "upscale_models": [".safetensors", ".pt", ".pth"], + "vae": [".safetensors", ".pt", ".bin"], + "unet": [".safetensors", ".pt", ".bin"], +} + + +def is_network_volume_debug_enabled(): + """Check if network volume debug mode is enabled via environment variable.""" + return os.environ.get("NETWORK_VOLUME_DEBUG", "false").lower() == "true" + + +def run_network_volume_diagnostics(): + """ + Run comprehensive network volume diagnostics and print helpful output. + Only runs when NETWORK_VOLUME_DEBUG=true environment variable is set. + """ + print("=" * 70) + print("NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true)") + print("=" * 70) + + # Check extra_model_paths.yaml + extra_model_paths_file = "/comfyui/extra_model_paths.yaml" + print("\n[1] Checking extra_model_paths.yaml configuration...") + if os.path.isfile(extra_model_paths_file): + print(f" ✓ FOUND: {extra_model_paths_file}") + with open(extra_model_paths_file, "r") as f: + content = f.read() + print("\n Configuration content:") + for line in content.split("\n"): + print(f" {line}") + else: + print(f" ✗ NOT FOUND: {extra_model_paths_file}") + print( + " This file is required for ComfyUI to find models on the network volume." + ) + + # Check network volume mount + runpod_volume = "/runpod-volume" + print(f"\n[2] Checking network volume mount at {runpod_volume}...") + if os.path.isdir(runpod_volume): + print(f" ✓ MOUNTED: {runpod_volume}") + else: + print(f" ✗ NOT MOUNTED: {runpod_volume}") + print( + " Make sure you have attached a network volume to your serverless endpoint." + ) + print("=" * 70) + return + + # Check directory structure + print("\n[3] Checking directory structure...") + models_dir = os.path.join(runpod_volume, "models") + if os.path.isdir(models_dir): + print(f" ✓ FOUND: {models_dir}") + else: + print(f" ✗ NOT FOUND: {models_dir}") + print("\n ⚠️ PROBLEM: The 'models' directory does not exist!") + print(" You need to create the following structure on your network volume:") + print_expected_structure() + print("=" * 70) + return + + # List model directories and their contents + print("\n[4] Scanning model directories...") + found_any_models = False + + for model_type, extensions in MODEL_TYPES.items(): + model_path = os.path.join(models_dir, model_type) + if os.path.isdir(model_path): + files = [] + try: + for f in os.listdir(model_path): + file_path = os.path.join(model_path, f) + if os.path.isfile(file_path): + # Check if file has valid extension + ext = os.path.splitext(f)[1].lower() + if ext in extensions: + size = os.path.getsize(file_path) + size_str = format_size(size) + files.append(f"{f} ({size_str})") + found_any_models = True + else: + files.append(f"{f} (⚠️ ignored - invalid extension)") + except Exception as e: + print(f" {model_type}/: Error reading directory - {e}") + continue + + if files: + print(f"\n {model_type}/:") + for f in files: + print(f" - {f}") + else: + print(f"\n {model_type}/: (empty)") + else: + print(f"\n {model_type}/: (directory not found)") + + # Summary + print("\n[5] Summary") + if found_any_models: + print(" ✓ Models found on network volume!") + print(" ComfyUI should be able to load these models.") + else: + print(" ⚠️ No valid model files found on network volume!") + print("\n Make sure your models have the correct file extensions:") + print(" - Checkpoints: .safetensors, .ckpt, .pt, .pth, .bin") + print(" - LoRAs: .safetensors, .pt") + print(" - VAE: .safetensors, .pt, .bin") + print(" - etc.") + + print_expected_structure() + print("=" * 70) + + +def print_expected_structure(): + """Print the expected directory structure for the network volume.""" + print("\n Expected directory structure:") + print(" /runpod-volume/") + print(" └── models/") + print(" ├── checkpoints/ <- Put your .safetensors/.ckpt models here") + print(" ├── loras/ <- Put your LoRA files here") + print(" ├── vae/ <- Put your VAE files here") + print(" ├── clip/ <- Put your CLIP models here") + print(" ├── controlnet/ <- Put your ControlNet models here") + print(" ├── embeddings/ <- Put your embedding files here") + print(" └── upscale_models/ <- Put your upscale models here") + + +def format_size(size_bytes): + """Format bytes into human-readable size.""" + for unit in ["B", "KB", "MB", "GB"]: + if size_bytes < 1024: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024 + return f"{size_bytes:.1f} TB" +