From ab1648eff109ca1db082b5efcb8b820151d74e53 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Mon, 1 Dec 2025 16:29:34 +0100 Subject: [PATCH 1/9] Add diagnostic logging for network volume debugging - Check if extra_model_paths.yaml exists at /comfyui/ - Print yaml content to logs - Check if /runpod-volume is mounted - List contents of /runpod-volume and /runpod-volume/models This helps debug why models on network volumes might not be detected. --- src/start.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/start.sh b/src/start.sh index 15d38ad51..3536055ac 100644 --- a/src/start.sh +++ b/src/start.sh @@ -7,6 +7,30 @@ export LD_PRELOAD="${TCMALLOC}" # Ensure ComfyUI-Manager runs in offline network mode inside the container comfy-manager-set-mode offline || echo "worker-comfyui - Could not set ComfyUI-Manager network_mode" >&2 +# Diagnostic: Check extra_model_paths.yaml +if [ -f /comfyui/extra_model_paths.yaml ]; then + echo "worker-comfyui: extra_model_paths.yaml found at /comfyui/" + echo "worker-comfyui: extra_model_paths.yaml content:" + cat /comfyui/extra_model_paths.yaml +else + echo "worker-comfyui: WARNING - extra_model_paths.yaml NOT found at /comfyui/" +fi + +# Diagnostic: Check network volume +if [ -d /runpod-volume ]; then + echo "worker-comfyui: Network volume mounted at /runpod-volume" + echo "worker-comfyui: /runpod-volume contents:" + ls -la /runpod-volume/ + if [ -d /runpod-volume/models ]; then + echo "worker-comfyui: /runpod-volume/models contents:" + ls -la /runpod-volume/models/ + else + echo "worker-comfyui: WARNING - /runpod-volume/models does NOT exist" + fi +else + echo "worker-comfyui: INFO - /runpod-volume does not exist (no network volume attached)" +fi + echo "worker-comfyui: Starting ComfyUI" # Allow operators to tweak verbosity; default is DEBUG. From 52149aa9704b0194a510decbc1b98467317f8a55 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Mon, 1 Dec 2025 16:33:53 +0100 Subject: [PATCH 2/9] Dev workflow: only build base target for faster iteration --- .github/workflows/dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 48bb72437..741c28c35 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -44,10 +44,11 @@ jobs: echo "HUGGINGFACE_ACCESS_TOKEN=${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}" >> $GITHUB_ENV echo "RELEASE_VERSION=${GITHUB_REF##refs/heads/}" | sed 's/\//-/g' >> $GITHUB_ENV - - name: Build and push the images to Docker Hub + - name: Build and push the base image to Docker Hub uses: docker/bake-action@v2 with: push: true + targets: base set: | *.args.DOCKERHUB_REPO=${{ env.DOCKERHUB_REPO }} *.args.DOCKERHUB_IMG=${{ env.DOCKERHUB_IMG }} From 9168350208bde5ece32029983198aa982b7bba1e Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Mon, 1 Dec 2025 19:50:15 +0100 Subject: [PATCH 3/9] fix(handler): add python diagnostic logging for network volume debugging - add logging to check if extra_model_paths.yaml exists and print content - add logging to check if /runpod-volume is mounted and list contents - replaces bash echo statements that were not captured by runpod serverless this diagnostic output will appear in runpod logs to help debug why models on network volumes are not being detected by comfyui. --- handler.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/handler.py b/handler.py index e966744fd..9dcc2bdb1 100644 --- a/handler.py +++ b/handler.py @@ -13,6 +13,38 @@ import tempfile import socket import traceback +import logging + +# --------------------------------------------------------------------------- +# Diagnostic logging for network volume debugging +# --------------------------------------------------------------------------- +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Check extra_model_paths.yaml +extra_model_paths_file = "/comfyui/extra_model_paths.yaml" +if os.path.isfile(extra_model_paths_file): + logger.info(f"extra_model_paths.yaml found at {extra_model_paths_file}") + with open(extra_model_paths_file, "r") as f: + logger.info(f"extra_model_paths.yaml content:\n{f.read()}") +else: + logger.warning(f"extra_model_paths.yaml NOT found at {extra_model_paths_file}") + +# Check network volume mount +runpod_volume = "/runpod-volume" +if os.path.isdir(runpod_volume): + logger.info(f"Network volume mounted at {runpod_volume}") + try: + contents = os.listdir(runpod_volume) + logger.info(f"{runpod_volume} contents: {contents}") + models_dir = os.path.join(runpod_volume, "models") + if os.path.isdir(models_dir): + models_contents = os.listdir(models_dir) + logger.info(f"{models_dir} contents: {models_contents}") + except Exception as e: + logger.error(f"Error listing {runpod_volume}: {e}") +else: + logger.warning(f"Network volume NOT mounted at {runpod_volume}") # Time to wait between API check attempts in milliseconds COMFY_API_AVAILABLE_INTERVAL_MS = 50 From 31dde9cacace3c567f321a8a356ad1acbcc69cbb Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 07:59:52 +0100 Subject: [PATCH 4/9] fix(handler): move diagnostics into handler function for log capture - diagnostics now run on every request instead of module import time - use print() instead of logger for better stdout capture - this ensures diagnostic output appears in runpod serverless logs --- handler.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/handler.py b/handler.py index 9dcc2bdb1..2480834d7 100644 --- a/handler.py +++ b/handler.py @@ -534,6 +534,34 @@ def handler(job): Returns: dict: A dictionary containing either an error message or a success status with generated images. """ + # --------------------------------------------------------------------------- + # Diagnostic output for network volume debugging (runs on every request) + # --------------------------------------------------------------------------- + print("--- worker-comfyui: Network Volume Diagnostics ---") + extra_model_paths_file = "/comfyui/extra_model_paths.yaml" + if os.path.isfile(extra_model_paths_file): + print(f"extra_model_paths.yaml: FOUND at {extra_model_paths_file}") + with open(extra_model_paths_file, "r") as f: + print(f"extra_model_paths.yaml content:\n{f.read()}") + else: + print(f"extra_model_paths.yaml: NOT FOUND at {extra_model_paths_file}") + + runpod_volume = "/runpod-volume" + if os.path.isdir(runpod_volume): + print(f"Network volume: MOUNTED at {runpod_volume}") + try: + contents = os.listdir(runpod_volume) + print(f"{runpod_volume} contents: {contents}") + models_dir = os.path.join(runpod_volume, "models") + if os.path.isdir(models_dir): + models_contents = os.listdir(models_dir) + print(f"{models_dir} contents: {models_contents}") + except Exception as e: + print(f"Error listing {runpod_volume}: {e}") + else: + print(f"Network volume: NOT MOUNTED at {runpod_volume}") + print("--- End Diagnostics ---") + job_input = job["input"] job_id = job["id"] From dd8caec037dd360eaf06194eb675d1a1383770e5 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 09:34:52 +0100 Subject: [PATCH 5/9] feat(debug): add opt-in NETWORK_VOLUME_DEBUG for model path troubleshooting - Add NETWORK_VOLUME_DEBUG environment variable (default: false) - Create comprehensive diagnostic function that shows: - Configuration file status - Network volume mount status - Directory structure validation - Model files found (with size and extension validation) - Expected structure guidance when issues found - Document network volume configuration in docs/configuration.md - Expected directory structure - Supported file extensions by model type - Step-by-step debugging instructions - Common issues and solutions Resolves user reports of models not being detected on network volumes. Root cause: user configuration issues (wrong directory structure or missing file extensions), not a bug in worker-comfyui. --- docs/configuration.md | 84 +++++++++++++++++- handler.py | 194 +++++++++++++++++++++++++++++++----------- 2 files changed, 227 insertions(+), 51 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8a7c58691..6b627d0d2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,8 +23,90 @@ This document outlines the environment variables available for configuring the ` | `WEBSOCKET_RECONNECT_ATTEMPTS` | Number of websocket reconnection attempts when connection drops during job execution. | `5` | | `WEBSOCKET_RECONNECT_DELAY_S` | Delay in seconds between websocket reconnection attempts. | `3` | | `WEBSOCKET_TRACE` | Enable low-level websocket frame tracing for protocol debugging. Set to `true` only when diagnosing connection issues. | `false` | +| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volume Configuration](#network-volume-configuration) below. | `false` | -> [!TIP] > **For troubleshooting:** Set `COMFY_LOG_LEVEL=DEBUG` to get detailed logs when ComfyUI crashes or behaves unexpectedly. This helps identify the exact point of failure in your workflows. +> [!TIP] +> **For troubleshooting:** Set `COMFY_LOG_LEVEL=DEBUG` to get detailed logs when ComfyUI crashes or behaves unexpectedly. This helps identify the exact point of failure in your workflows. + +## Network Volume Configuration + +When using a RunPod network volume to store your models, the worker expects a specific directory structure. If ComfyUI is not finding your models, enable diagnostics by setting `NETWORK_VOLUME_DEBUG=true`. + +### Expected Directory Structure + +Models must be placed in the following structure on your network volume: + +``` +/runpod-volume/ +└── models/ + ├── checkpoints/ # Stable Diffusion checkpoints (.safetensors, .ckpt) + ├── loras/ # LoRA files (.safetensors, .pt) + ├── vae/ # VAE models (.safetensors, .pt) + ├── clip/ # CLIP models (.safetensors, .pt) + ├── clip_vision/ # CLIP Vision models + ├── controlnet/ # ControlNet models (.safetensors, .pt) + ├── embeddings/ # Textual inversion embeddings (.safetensors, .pt) + ├── upscale_models/ # Upscaling models (.safetensors, .pt) + ├── unet/ # UNet models + └── configs/ # Model configs (.yaml, .json) +``` + +### Supported File Extensions + +ComfyUI only recognizes files with specific extensions: + +| Model Type | Supported Extensions | +| ---------------- | ----------------------------------- | +| Checkpoints | `.safetensors`, `.ckpt`, `.pt`, `.pth`, `.bin` | +| LoRAs | `.safetensors`, `.pt` | +| VAE | `.safetensors`, `.pt`, `.bin` | +| CLIP | `.safetensors`, `.pt`, `.bin` | +| ControlNet | `.safetensors`, `.pt`, `.pth`, `.bin` | +| Embeddings | `.safetensors`, `.pt`, `.bin` | +| Upscale Models | `.safetensors`, `.pt`, `.pth` | + +> [!WARNING] +> **Common Issues:** +> - Models placed directly in `/runpod-volume/checkpoints/` instead of `/runpod-volume/models/checkpoints/` will not be found. +> - Files with incorrect extensions (e.g., `.txt`, `.zip`) will be ignored. +> - Empty directories or missing subdirectories are fine—only create the folders you need. + +### Debugging Network Volume Issues + +1. **Enable diagnostics** by adding `NETWORK_VOLUME_DEBUG=true` to your endpoint's environment variables. + +2. **Send a test request** to your endpoint (any request will trigger the diagnostics). + +3. **Check the worker logs** in the RunPod console. You'll see detailed output like: + +``` +====================================================================== +NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true) +====================================================================== + +[1] Checking extra_model_paths.yaml configuration... + ✓ FOUND: /comfyui/extra_model_paths.yaml + +[2] Checking network volume mount at /runpod-volume... + ✓ MOUNTED: /runpod-volume + +[3] Checking directory structure... + ✓ FOUND: /runpod-volume/models + +[4] Scanning model directories... + + checkpoints/: + - my-model.safetensors (6.5 GB) + + loras/: + - style-lora.safetensors (144.2 MB) + +[5] Summary + ✓ Models found on network volume! +====================================================================== +``` + +4. **Disable diagnostics** once your issue is resolved by removing the environment variable or setting it to `false`. ## AWS S3 Upload Configuration diff --git a/handler.py b/handler.py index 2480834d7..b9e8e9cda 100644 --- a/handler.py +++ b/handler.py @@ -16,35 +16,151 @@ import logging # --------------------------------------------------------------------------- -# Diagnostic logging for network volume debugging +# Logging setup # --------------------------------------------------------------------------- logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -# Check extra_model_paths.yaml -extra_model_paths_file = "/comfyui/extra_model_paths.yaml" -if os.path.isfile(extra_model_paths_file): - logger.info(f"extra_model_paths.yaml found at {extra_model_paths_file}") - with open(extra_model_paths_file, "r") as f: - logger.info(f"extra_model_paths.yaml content:\n{f.read()}") -else: - logger.warning(f"extra_model_paths.yaml NOT found at {extra_model_paths_file}") - -# Check network volume mount -runpod_volume = "/runpod-volume" -if os.path.isdir(runpod_volume): - logger.info(f"Network volume mounted at {runpod_volume}") - try: - contents = os.listdir(runpod_volume) - logger.info(f"{runpod_volume} contents: {contents}") - models_dir = os.path.join(runpod_volume, "models") - if os.path.isdir(models_dir): - models_contents = os.listdir(models_dir) - logger.info(f"{models_dir} contents: {models_contents}") - except Exception as e: - logger.error(f"Error listing {runpod_volume}: {e}") -else: - logger.warning(f"Network volume NOT mounted at {runpod_volume}") +# --------------------------------------------------------------------------- +# Network Volume Debug Mode (opt-in via environment variable) +# Set NETWORK_VOLUME_DEBUG=true to enable detailed diagnostics +# --------------------------------------------------------------------------- +NETWORK_VOLUME_DEBUG = os.environ.get("NETWORK_VOLUME_DEBUG", "false").lower() == "true" + +# Expected model types and their file extensions +MODEL_TYPES = { + "checkpoints": [".safetensors", ".ckpt", ".pt", ".pth", ".bin"], + "clip": [".safetensors", ".pt", ".bin"], + "clip_vision": [".safetensors", ".pt", ".bin"], + "configs": [".yaml", ".json"], + "controlnet": [".safetensors", ".pt", ".pth", ".bin"], + "embeddings": [".safetensors", ".pt", ".bin"], + "loras": [".safetensors", ".pt"], + "upscale_models": [".safetensors", ".pt", ".pth"], + "vae": [".safetensors", ".pt", ".bin"], + "unet": [".safetensors", ".pt", ".bin"], +} + + +def run_network_volume_diagnostics(): + """ + Run comprehensive network volume diagnostics and print helpful output. + Only runs when NETWORK_VOLUME_DEBUG=true environment variable is set. + """ + print("=" * 70) + print("NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true)") + print("=" * 70) + + # Check extra_model_paths.yaml + extra_model_paths_file = "/comfyui/extra_model_paths.yaml" + print("\n[1] Checking extra_model_paths.yaml configuration...") + if os.path.isfile(extra_model_paths_file): + print(f" ✓ FOUND: {extra_model_paths_file}") + with open(extra_model_paths_file, "r") as f: + content = f.read() + print(f"\n Configuration content:") + for line in content.split("\n"): + print(f" {line}") + else: + print(f" ✗ NOT FOUND: {extra_model_paths_file}") + print(" This file is required for ComfyUI to find models on the network volume.") + + # Check network volume mount + runpod_volume = "/runpod-volume" + print(f"\n[2] Checking network volume mount at {runpod_volume}...") + if os.path.isdir(runpod_volume): + print(f" ✓ MOUNTED: {runpod_volume}") + else: + print(f" ✗ NOT MOUNTED: {runpod_volume}") + print(" Make sure you have attached a network volume to your serverless endpoint.") + print("=" * 70) + return + + # Check directory structure + print(f"\n[3] Checking directory structure...") + models_dir = os.path.join(runpod_volume, "models") + if os.path.isdir(models_dir): + print(f" ✓ FOUND: {models_dir}") + else: + print(f" ✗ NOT FOUND: {models_dir}") + print("\n ⚠️ PROBLEM: The 'models' directory does not exist!") + print(" You need to create the following structure on your network volume:") + print_expected_structure() + print("=" * 70) + return + + # List model directories and their contents + print(f"\n[4] Scanning model directories...") + found_any_models = False + + for model_type, extensions in MODEL_TYPES.items(): + model_path = os.path.join(models_dir, model_type) + if os.path.isdir(model_path): + files = [] + try: + for f in os.listdir(model_path): + file_path = os.path.join(model_path, f) + if os.path.isfile(file_path): + # Check if file has valid extension + ext = os.path.splitext(f)[1].lower() + if ext in extensions: + size = os.path.getsize(file_path) + size_str = format_size(size) + files.append(f"{f} ({size_str})") + found_any_models = True + else: + files.append(f"{f} (⚠️ ignored - invalid extension)") + except Exception as e: + print(f" {model_type}/: Error reading directory - {e}") + continue + + if files: + print(f"\n {model_type}/:") + for f in files: + print(f" - {f}") + else: + print(f"\n {model_type}/: (empty)") + else: + print(f"\n {model_type}/: (directory not found)") + + # Summary + print(f"\n[5] Summary") + if found_any_models: + print(" ✓ Models found on network volume!") + print(" ComfyUI should be able to load these models.") + else: + print(" ⚠️ No valid model files found on network volume!") + print("\n Make sure your models have the correct file extensions:") + print(" - Checkpoints: .safetensors, .ckpt, .pt, .pth, .bin") + print(" - LoRAs: .safetensors, .pt") + print(" - VAE: .safetensors, .pt, .bin") + print(" - etc.") + + print_expected_structure() + print("=" * 70) + + +def print_expected_structure(): + """Print the expected directory structure for the network volume.""" + print("\n Expected directory structure:") + print(" /runpod-volume/") + print(" └── models/") + print(" ├── checkpoints/ <- Put your .safetensors/.ckpt models here") + print(" ├── loras/ <- Put your LoRA files here") + print(" ├── vae/ <- Put your VAE files here") + print(" ├── clip/ <- Put your CLIP models here") + print(" ├── controlnet/ <- Put your ControlNet models here") + print(" ├── embeddings/ <- Put your embedding files here") + print(" └── upscale_models/ <- Put your upscale models here") + + +def format_size(size_bytes): + """Format bytes into human-readable size.""" + for unit in ["B", "KB", "MB", "GB"]: + if size_bytes < 1024: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024 + return f"{size_bytes:.1f} TB" # Time to wait between API check attempts in milliseconds COMFY_API_AVAILABLE_INTERVAL_MS = 50 @@ -535,32 +651,10 @@ def handler(job): dict: A dictionary containing either an error message or a success status with generated images. """ # --------------------------------------------------------------------------- - # Diagnostic output for network volume debugging (runs on every request) + # Network Volume Diagnostics (opt-in via NETWORK_VOLUME_DEBUG=true) # --------------------------------------------------------------------------- - print("--- worker-comfyui: Network Volume Diagnostics ---") - extra_model_paths_file = "/comfyui/extra_model_paths.yaml" - if os.path.isfile(extra_model_paths_file): - print(f"extra_model_paths.yaml: FOUND at {extra_model_paths_file}") - with open(extra_model_paths_file, "r") as f: - print(f"extra_model_paths.yaml content:\n{f.read()}") - else: - print(f"extra_model_paths.yaml: NOT FOUND at {extra_model_paths_file}") - - runpod_volume = "/runpod-volume" - if os.path.isdir(runpod_volume): - print(f"Network volume: MOUNTED at {runpod_volume}") - try: - contents = os.listdir(runpod_volume) - print(f"{runpod_volume} contents: {contents}") - models_dir = os.path.join(runpod_volume, "models") - if os.path.isdir(models_dir): - models_contents = os.listdir(models_dir) - print(f"{models_dir} contents: {models_contents}") - except Exception as e: - print(f"Error listing {runpod_volume}: {e}") - else: - print(f"Network volume: NOT MOUNTED at {runpod_volume}") - print("--- End Diagnostics ---") + if NETWORK_VOLUME_DEBUG: + run_network_volume_diagnostics() job_input = job["input"] job_id = job["id"] From 1f813ea50546c8dca4099814001070c9de460e78 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 09:43:31 +0100 Subject: [PATCH 6/9] style: apply formatting to network volume diagnostics --- handler.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/handler.py b/handler.py index b9e8e9cda..f2b10a1ad 100644 --- a/handler.py +++ b/handler.py @@ -50,7 +50,7 @@ def run_network_volume_diagnostics(): print("=" * 70) print("NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true)") print("=" * 70) - + # Check extra_model_paths.yaml extra_model_paths_file = "/comfyui/extra_model_paths.yaml" print("\n[1] Checking extra_model_paths.yaml configuration...") @@ -63,8 +63,10 @@ def run_network_volume_diagnostics(): print(f" {line}") else: print(f" ✗ NOT FOUND: {extra_model_paths_file}") - print(" This file is required for ComfyUI to find models on the network volume.") - + print( + " This file is required for ComfyUI to find models on the network volume." + ) + # Check network volume mount runpod_volume = "/runpod-volume" print(f"\n[2] Checking network volume mount at {runpod_volume}...") @@ -72,10 +74,12 @@ def run_network_volume_diagnostics(): print(f" ✓ MOUNTED: {runpod_volume}") else: print(f" ✗ NOT MOUNTED: {runpod_volume}") - print(" Make sure you have attached a network volume to your serverless endpoint.") + print( + " Make sure you have attached a network volume to your serverless endpoint." + ) print("=" * 70) return - + # Check directory structure print(f"\n[3] Checking directory structure...") models_dir = os.path.join(runpod_volume, "models") @@ -88,11 +92,11 @@ def run_network_volume_diagnostics(): print_expected_structure() print("=" * 70) return - + # List model directories and their contents print(f"\n[4] Scanning model directories...") found_any_models = False - + for model_type, extensions in MODEL_TYPES.items(): model_path = os.path.join(models_dir, model_type) if os.path.isdir(model_path): @@ -113,7 +117,7 @@ def run_network_volume_diagnostics(): except Exception as e: print(f" {model_type}/: Error reading directory - {e}") continue - + if files: print(f"\n {model_type}/:") for f in files: @@ -122,7 +126,7 @@ def run_network_volume_diagnostics(): print(f"\n {model_type}/: (empty)") else: print(f"\n {model_type}/: (directory not found)") - + # Summary print(f"\n[5] Summary") if found_any_models: @@ -135,7 +139,7 @@ def run_network_volume_diagnostics(): print(" - LoRAs: .safetensors, .pt") print(" - VAE: .safetensors, .pt, .bin") print(" - etc.") - + print_expected_structure() print("=" * 70) @@ -162,6 +166,7 @@ def format_size(size_bytes): size_bytes /= 1024 return f"{size_bytes:.1f} TB" + # Time to wait between API check attempts in milliseconds COMFY_API_AVAILABLE_INTERVAL_MS = 50 # Maximum number of API check attempts From 8b5b2d1a2f212d6f27c0683e506afd1982d5c424 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 09:46:39 +0100 Subject: [PATCH 7/9] docs: move network volume troubleshooting to dedicated guide --- docs/configuration.md | 82 +--------------------- docs/deployment.md | 4 +- docs/network-volumes.md | 147 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 83 deletions(-) create mode 100644 docs/network-volumes.md diff --git a/docs/configuration.md b/docs/configuration.md index 6b627d0d2..70be42666 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,91 +23,11 @@ This document outlines the environment variables available for configuring the ` | `WEBSOCKET_RECONNECT_ATTEMPTS` | Number of websocket reconnection attempts when connection drops during job execution. | `5` | | `WEBSOCKET_RECONNECT_DELAY_S` | Delay in seconds between websocket reconnection attempts. | `3` | | `WEBSOCKET_TRACE` | Enable low-level websocket frame tracing for protocol debugging. Set to `true` only when diagnosing connection issues. | `false` | -| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volume Configuration](#network-volume-configuration) below. | `false` | +| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volumes & Model Paths](network-volumes.md). | `false` | > [!TIP] > **For troubleshooting:** Set `COMFY_LOG_LEVEL=DEBUG` to get detailed logs when ComfyUI crashes or behaves unexpectedly. This helps identify the exact point of failure in your workflows. -## Network Volume Configuration - -When using a RunPod network volume to store your models, the worker expects a specific directory structure. If ComfyUI is not finding your models, enable diagnostics by setting `NETWORK_VOLUME_DEBUG=true`. - -### Expected Directory Structure - -Models must be placed in the following structure on your network volume: - -``` -/runpod-volume/ -└── models/ - ├── checkpoints/ # Stable Diffusion checkpoints (.safetensors, .ckpt) - ├── loras/ # LoRA files (.safetensors, .pt) - ├── vae/ # VAE models (.safetensors, .pt) - ├── clip/ # CLIP models (.safetensors, .pt) - ├── clip_vision/ # CLIP Vision models - ├── controlnet/ # ControlNet models (.safetensors, .pt) - ├── embeddings/ # Textual inversion embeddings (.safetensors, .pt) - ├── upscale_models/ # Upscaling models (.safetensors, .pt) - ├── unet/ # UNet models - └── configs/ # Model configs (.yaml, .json) -``` - -### Supported File Extensions - -ComfyUI only recognizes files with specific extensions: - -| Model Type | Supported Extensions | -| ---------------- | ----------------------------------- | -| Checkpoints | `.safetensors`, `.ckpt`, `.pt`, `.pth`, `.bin` | -| LoRAs | `.safetensors`, `.pt` | -| VAE | `.safetensors`, `.pt`, `.bin` | -| CLIP | `.safetensors`, `.pt`, `.bin` | -| ControlNet | `.safetensors`, `.pt`, `.pth`, `.bin` | -| Embeddings | `.safetensors`, `.pt`, `.bin` | -| Upscale Models | `.safetensors`, `.pt`, `.pth` | - -> [!WARNING] -> **Common Issues:** -> - Models placed directly in `/runpod-volume/checkpoints/` instead of `/runpod-volume/models/checkpoints/` will not be found. -> - Files with incorrect extensions (e.g., `.txt`, `.zip`) will be ignored. -> - Empty directories or missing subdirectories are fine—only create the folders you need. - -### Debugging Network Volume Issues - -1. **Enable diagnostics** by adding `NETWORK_VOLUME_DEBUG=true` to your endpoint's environment variables. - -2. **Send a test request** to your endpoint (any request will trigger the diagnostics). - -3. **Check the worker logs** in the RunPod console. You'll see detailed output like: - -``` -====================================================================== -NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true) -====================================================================== - -[1] Checking extra_model_paths.yaml configuration... - ✓ FOUND: /comfyui/extra_model_paths.yaml - -[2] Checking network volume mount at /runpod-volume... - ✓ MOUNTED: /runpod-volume - -[3] Checking directory structure... - ✓ FOUND: /runpod-volume/models - -[4] Scanning model directories... - - checkpoints/: - - my-model.safetensors (6.5 GB) - - loras/: - - style-lora.safetensors (144.2 MB) - -[5] Summary - ✓ Models found on network volume! -====================================================================== -``` - -4. **Disable diagnostics** once your issue is resolved by removing the environment variable or setting it to `false`. - ## AWS S3 Upload Configuration Configure these variables **only** if you want the worker to upload generated images directly to an AWS S3 bucket. If these are not set, images will be returned as base64-encoded strings in the API response. diff --git a/docs/deployment.md b/docs/deployment.md index 210728b0a..7e020eb81 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -16,7 +16,7 @@ This is the simplest method if the official images meet your needs. - Container Registry Credentials: Leave as default (images are public). - Container Disk: Adjust based on the chosen image tag, see [GPU Recommendations](#gpu-recommendations). - (optional) Environment Variables: Configure S3 or other settings (see [Configuration Guide](configuration.md)). - - Note: If you don't configure S3, images are returned as base64. For persistent storage across jobs without S3, consider using a [Network Volume](customization.md#method-2-network-volume-alternative-for-models). + - Note: If you don't configure S3, images are returned as base64. For persistent storage across jobs without S3, consider using a [Network Volume](customization.md#method-2-network-volume-alternative-for-models). If models on your network volume are not being detected, see [Network Volumes & Model Paths](network-volumes.md) for troubleshooting steps. - Click on `Save Template` ### Create your endpoint @@ -32,7 +32,7 @@ This is the simplest method if the official images meet your needs. - Idle Timeout: `5` (Default is usually fine, adjust if needed). - Flash Boot: `enabled` (Recommended for faster worker startup). - Select Template: `worker-comfyui` (or the name you gave your template). - - (optional) Advanced: If you are using a Network Volume, select it under `Select Network Volume`. See the [Customization Guide](customization.md#method-2-network-volume-alternative-for-models). + - (optional) Advanced: If you are using a Network Volume, select it under `Select Network Volume`. See the [Customization Guide](customization.md#method-2-network-volume-alternative-for-models). For detailed model path layout and debugging tips, see [Network Volumes & Model Paths](network-volumes.md). - Click `deploy` - Your endpoint will be created. You can click on it to view the dashboard and find its ID. diff --git a/docs/network-volumes.md b/docs/network-volumes.md new file mode 100644 index 000000000..c9e634ce5 --- /dev/null +++ b/docs/network-volumes.md @@ -0,0 +1,147 @@ +# Network Volumes & Model Paths + +This document explains how to use RunPod **Network Volumes** with `worker-comfyui`, how model paths are resolved inside the container, and how to debug cases where models are not detected. + +> **Scope** +> +> These instructions apply to **serverless endpoints** using this worker. Pods mount network volumes at `/workspace` by default, while serverless workers see them at `/runpod-volume`. + +## Directory Mapping + +For **serverless endpoints**: + +- Network volume root is mounted at: `/runpod-volume` +- ComfyUI models are expected under: `/runpod-volume/models/...` + +For **Pods**: + +- Network volume root is mounted at: `/workspace` +- Equivalent ComfyUI model path: `/workspace/models/...` + +If you use the S3-compatible API, the same paths map as: + +- Serverless: `/runpod-volume/my-folder/file.txt` +- Pod: `/workspace/my-folder/file.txt` +- S3 API: `s3:///my-folder/file.txt` + +## Expected Directory Structure + +Models must be placed in the following structure on your network volume: + +```text +/runpod-volume/ +└── models/ + ├── checkpoints/ # Stable Diffusion checkpoints (.safetensors, .ckpt) + ├── loras/ # LoRA files (.safetensors, .pt) + ├── vae/ # VAE models (.safetensors, .pt) + ├── clip/ # CLIP models (.safetensors, .pt) + ├── clip_vision/ # CLIP Vision models + ├── controlnet/ # ControlNet models (.safetensors, .pt) + ├── embeddings/ # Textual inversion embeddings (.safetensors, .pt) + ├── upscale_models/ # Upscaling models (.safetensors, .pt) + ├── unet/ # UNet models + └── configs/ # Model configs (.yaml, .json) +``` + +> **Note** +> +> Only create the subdirectories you actually need; empty or missing folders are fine. + +## Supported File Extensions + +ComfyUI only recognizes files with specific extensions when scanning model directories. + +| Model Type | Supported Extensions | +| -------------- | ------------------------------------------- | +| Checkpoints | `.safetensors`, `.ckpt`, `.pt`, `.pth`, `.bin` | +| LoRAs | `.safetensors`, `.pt` | +| VAE | `.safetensors`, `.pt`, `.bin` | +| CLIP | `.safetensors`, `.pt`, `.bin` | +| ControlNet | `.safetensors`, `.pt`, `.pth`, `.bin` | +| Embeddings | `.safetensors`, `.pt`, `.bin` | +| Upscale Models | `.safetensors`, `.pt`, `.pth` | + +Files with other extensions (for example `.txt`, `.zip`) are **ignored** by ComfyUI’s model discovery. + +## Common Issues + +- **Wrong root directory** + - Models placed directly under `/runpod-volume/checkpoints/...` instead of `/runpod-volume/models/checkpoints/...`. +- **Incorrect extensions** + - Files named without one of the supported extensions are skipped. +- **Empty directories** + - No actual model files present in `models/checkpoints` (or other folders). +- **Volume not attached** + - Endpoint created without selecting a network volume under **Advanced → Select Network Volume**. + +If any of the above is true, ComfyUI will silently fail to discover models from the network volume. + +## Debugging with `NETWORK_VOLUME_DEBUG` + +The worker exposes an opt‑in debug mode controlled via the `NETWORK_VOLUME_DEBUG` environment variable. + +### When to Use + +Enable this when: + +- Models on your network volume are not appearing in ComfyUI +- You suspect the directory structure or file extensions are wrong +- You want to quickly verify what the worker can actually see on `/runpod-volume` + +### How to Enable + +1. Go to your serverless **Endpoint → Manage → Edit**. +2. Under **Environment Variables**, add: + + - `NETWORK_VOLUME_DEBUG=true` + +3. Save and wait for workers to restart (or scale to zero and back up). +4. Send any request to your endpoint (even a minimal one) to trigger the diagnostics. + +### Reading the Diagnostics + +When enabled, each request prints a detailed report to the worker logs, for example: + +```text +====================================================================== +NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true) +====================================================================== + +[1] Checking extra_model_paths.yaml configuration... + ✓ FOUND: /comfyui/extra_model_paths.yaml + +[2] Checking network volume mount at /runpod-volume... + ✓ MOUNTED: /runpod-volume + +[3] Checking directory structure... + ✓ FOUND: /runpod-volume/models + +[4] Scanning model directories... + + checkpoints/: + - my-model.safetensors (6.5 GB) + + loras/: + - style-lora.safetensors (144.2 MB) + +[5] Summary + ✓ Models found on network volume! +====================================================================== +``` + +If there is a problem, the diagnostics will instead highlight it, for example: + +- Missing `models/` directory +- No valid model files in any subdirectory +- Files present but ignored due to wrong extensions + +### Disabling Debug Mode + +Once you have resolved your issue, disable diagnostics to keep logs clean: + +- Remove the `NETWORK_VOLUME_DEBUG` environment variable, **or** +- Set `NETWORK_VOLUME_DEBUG=false` + +This returns the worker to normal behavior without extra log noise. + + From 1339a8cbb0bcc79a5678774ed079fd5b667c3738 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 09:57:00 +0100 Subject: [PATCH 8/9] docs(config): reorganize network volume debug and fix path documentation - move NETWORK_VOLUME_DEBUG from debugging to logging configuration table - remove redundant TIP block about COMFY_LOG_LEVEL - fix customization.md tip block syntax to proper github admonition format - correct network volume paths in customization guide: - clarify /runpod-volume mount point for serverless endpoints - update example paths to show /runpod-volume/models/ structure - fix note block syntax to proper github admonition format - add links to network-volumes.md for detailed structure and debugging - mention s3-compatible api as upload option --- docs/configuration.md | 11 ++++------- docs/customization.md | 25 ++++++++++++++----------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 70be42666..25bed9e70 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,9 +12,10 @@ This document outlines the environment variables available for configuring the ` ## Logging Configuration -| Environment Variable | Description | Default | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `COMFY_LOG_LEVEL` | Controls ComfyUI's internal logging verbosity. Options: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Use `DEBUG` for troubleshooting, `INFO` for production. | `DEBUG` | +| Environment Variable | Description | Default | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `COMFY_LOG_LEVEL` | Controls ComfyUI's internal logging verbosity. Options: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Use `DEBUG` for troubleshooting, `INFO` for production. | `DEBUG` | +| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volumes & Model Paths](network-volumes.md). | `false` | ## Debugging Configuration @@ -23,10 +24,6 @@ This document outlines the environment variables available for configuring the ` | `WEBSOCKET_RECONNECT_ATTEMPTS` | Number of websocket reconnection attempts when connection drops during job execution. | `5` | | `WEBSOCKET_RECONNECT_DELAY_S` | Delay in seconds between websocket reconnection attempts. | `3` | | `WEBSOCKET_TRACE` | Enable low-level websocket frame tracing for protocol debugging. Set to `true` only when diagnosing connection issues. | `false` | -| `NETWORK_VOLUME_DEBUG` | Enable detailed network volume diagnostics in worker logs. Useful for debugging model path issues. See [Network Volumes & Model Paths](network-volumes.md). | `false` | - -> [!TIP] -> **For troubleshooting:** Set `COMFY_LOG_LEVEL=DEBUG` to get detailed logs when ComfyUI crashes or behaves unexpectedly. This helps identify the exact point of failure in your workflows. ## AWS S3 Upload Configuration diff --git a/docs/customization.md b/docs/customization.md index b75839b51..715791e49 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -2,7 +2,9 @@ This guide covers methods for adding your own models, custom nodes, and static input files into a custom `worker-comfyui`. -> [!TIP] > **Looking for the easiest way to deploy custom workflows?** +> [!TIP] +> +> **Looking for the easiest way to deploy custom workflows?** > > [ComfyUI-to-API](https://comfy.getrunpod.io) automatically generates a custom Dockerfile and GitHub repository from your ComfyUI workflow, eliminating the manual setup described below. See the [ComfyUI-to-API Documentation](https://docs.runpod.io/community-solutions/comfyui-to-api/overview) for details. > @@ -90,20 +92,21 @@ Using a Network Volume is primarily useful if you want to manage **models** sepa 1. **Create a Network Volume**: - Follow the [RunPod Network Volumes guide](https://docs.runpod.io/pods/storage/create-network-volumes) to create a volume in the same region as your endpoint. 2. **Populate the Volume with Models**: - - Use one of the methods described in the RunPod guide (e.g., temporary Pod + `wget`, direct upload) to place your model files into the correct ComfyUI directory structure **within the volume**. The root of the volume corresponds to `/workspace` inside the container. + - Use one of the methods described in the RunPod guide (e.g., temporary Pod + `wget`, direct upload, or the S3-compatible API) to place your model files into the correct ComfyUI directory structure **within the volume**. + - For **serverless endpoints**, the network volume is mounted at `/runpod-volume`, and ComfyUI expects models under `/runpod-volume/models/...`. See [Network Volumes & Model Paths](network-volumes.md) for the exact structure and debugging tips. ```bash - # Example structure inside the Network Volume: - # /models/checkpoints/your_model.safetensors - # /models/loras/your_lora.pt - # /models/vae/your_vae.safetensors + # Example structure inside the Network Volume (serverless worker view): + # /runpod-volume/models/checkpoints/your_model.safetensors + # /runpod-volume/models/loras/your_lora.pt + # /runpod-volume/models/vae/your_vae.safetensors ``` - - **Important:** Ensure models are placed in the correct subdirectories (e.g., checkpoints in `models/checkpoints`, LoRAs in `models/loras`). + - **Important:** Ensure models are placed in the correct subdirectories (e.g., checkpoints in `models/checkpoints`, LoRAs in `models/loras`). If models are not detected, enable `NETWORK_VOLUME_DEBUG` as described in [Network Volumes & Model Paths](network-volumes.md). 3. **Configure Your Endpoint**: - Use the Network Volume in your endpoint configuration: - Either create a new endpoint or update an existing one (see [Deployment Guide](deployment.md)). - In the endpoint configuration, under `Advanced > Select Network Volume`, select your Network Volume. -**Note:** - -- When a Network Volume is correctly attached, ComfyUI running inside the worker container will automatically detect and load models from the standard directories (`/workspace/models/...`) within that volume. -- This method is **not suitable for installing custom nodes**; use the Custom Dockerfile method for that. +> [!NOTE] +> +> - When a Network Volume is correctly attached, ComfyUI running inside the worker container will automatically detect and load models from the standard directories (`/runpod-volume/models/...`) within that volume (for serverless workers). For directory mapping details and troubleshooting, see [Network Volumes & Model Paths](network-volumes.md). +> - This method is **not suitable for installing custom nodes**; use the Custom Dockerfile method for that. From 6b0fb4d89aaa3dbee978be577e2ef14e2fb77ba4 Mon Sep 17 00:00:00 2001 From: Tim Pietrusky Date: Wed, 3 Dec 2025 10:10:23 +0100 Subject: [PATCH 9/9] refactor: extract network volume diagnostics to separate module - move network volume diagnostic functions from handler.py to src/network_volume.py - remove obsolete bash diagnostics from src/start.sh (replaced by python diagnostics) - update handler.py to import diagnostics from network_volume module - update dockerfile to include src/network_volume.py in image build - reduces handler.py size by ~150 lines for better maintainability --- Dockerfile | 2 +- handler.py | 153 ++---------------------------------------- src/network_volume.py | 153 ++++++++++++++++++++++++++++++++++++++++++ src/start.sh | 24 ------- 4 files changed, 160 insertions(+), 172 deletions(-) create mode 100644 src/network_volume.py diff --git a/Dockerfile b/Dockerfile index 2d7e08e3e..b007ea764 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ WORKDIR / RUN uv pip install runpod requests websocket-client # Add application code and scripts -ADD src/start.sh handler.py test_input.json ./ +ADD src/start.sh src/network_volume.py handler.py test_input.json ./ RUN chmod +x /start.sh # Add script to install custom nodes diff --git a/handler.py b/handler.py index f2b10a1ad..65c8390b0 100644 --- a/handler.py +++ b/handler.py @@ -15,158 +15,17 @@ import traceback import logging +from network_volume import ( + is_network_volume_debug_enabled, + run_network_volume_diagnostics, +) + # --------------------------------------------------------------------------- # Logging setup # --------------------------------------------------------------------------- logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Network Volume Debug Mode (opt-in via environment variable) -# Set NETWORK_VOLUME_DEBUG=true to enable detailed diagnostics -# --------------------------------------------------------------------------- -NETWORK_VOLUME_DEBUG = os.environ.get("NETWORK_VOLUME_DEBUG", "false").lower() == "true" - -# Expected model types and their file extensions -MODEL_TYPES = { - "checkpoints": [".safetensors", ".ckpt", ".pt", ".pth", ".bin"], - "clip": [".safetensors", ".pt", ".bin"], - "clip_vision": [".safetensors", ".pt", ".bin"], - "configs": [".yaml", ".json"], - "controlnet": [".safetensors", ".pt", ".pth", ".bin"], - "embeddings": [".safetensors", ".pt", ".bin"], - "loras": [".safetensors", ".pt"], - "upscale_models": [".safetensors", ".pt", ".pth"], - "vae": [".safetensors", ".pt", ".bin"], - "unet": [".safetensors", ".pt", ".bin"], -} - - -def run_network_volume_diagnostics(): - """ - Run comprehensive network volume diagnostics and print helpful output. - Only runs when NETWORK_VOLUME_DEBUG=true environment variable is set. - """ - print("=" * 70) - print("NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true)") - print("=" * 70) - - # Check extra_model_paths.yaml - extra_model_paths_file = "/comfyui/extra_model_paths.yaml" - print("\n[1] Checking extra_model_paths.yaml configuration...") - if os.path.isfile(extra_model_paths_file): - print(f" ✓ FOUND: {extra_model_paths_file}") - with open(extra_model_paths_file, "r") as f: - content = f.read() - print(f"\n Configuration content:") - for line in content.split("\n"): - print(f" {line}") - else: - print(f" ✗ NOT FOUND: {extra_model_paths_file}") - print( - " This file is required for ComfyUI to find models on the network volume." - ) - - # Check network volume mount - runpod_volume = "/runpod-volume" - print(f"\n[2] Checking network volume mount at {runpod_volume}...") - if os.path.isdir(runpod_volume): - print(f" ✓ MOUNTED: {runpod_volume}") - else: - print(f" ✗ NOT MOUNTED: {runpod_volume}") - print( - " Make sure you have attached a network volume to your serverless endpoint." - ) - print("=" * 70) - return - - # Check directory structure - print(f"\n[3] Checking directory structure...") - models_dir = os.path.join(runpod_volume, "models") - if os.path.isdir(models_dir): - print(f" ✓ FOUND: {models_dir}") - else: - print(f" ✗ NOT FOUND: {models_dir}") - print("\n ⚠️ PROBLEM: The 'models' directory does not exist!") - print(" You need to create the following structure on your network volume:") - print_expected_structure() - print("=" * 70) - return - - # List model directories and their contents - print(f"\n[4] Scanning model directories...") - found_any_models = False - - for model_type, extensions in MODEL_TYPES.items(): - model_path = os.path.join(models_dir, model_type) - if os.path.isdir(model_path): - files = [] - try: - for f in os.listdir(model_path): - file_path = os.path.join(model_path, f) - if os.path.isfile(file_path): - # Check if file has valid extension - ext = os.path.splitext(f)[1].lower() - if ext in extensions: - size = os.path.getsize(file_path) - size_str = format_size(size) - files.append(f"{f} ({size_str})") - found_any_models = True - else: - files.append(f"{f} (⚠️ ignored - invalid extension)") - except Exception as e: - print(f" {model_type}/: Error reading directory - {e}") - continue - - if files: - print(f"\n {model_type}/:") - for f in files: - print(f" - {f}") - else: - print(f"\n {model_type}/: (empty)") - else: - print(f"\n {model_type}/: (directory not found)") - - # Summary - print(f"\n[5] Summary") - if found_any_models: - print(" ✓ Models found on network volume!") - print(" ComfyUI should be able to load these models.") - else: - print(" ⚠️ No valid model files found on network volume!") - print("\n Make sure your models have the correct file extensions:") - print(" - Checkpoints: .safetensors, .ckpt, .pt, .pth, .bin") - print(" - LoRAs: .safetensors, .pt") - print(" - VAE: .safetensors, .pt, .bin") - print(" - etc.") - - print_expected_structure() - print("=" * 70) - - -def print_expected_structure(): - """Print the expected directory structure for the network volume.""" - print("\n Expected directory structure:") - print(" /runpod-volume/") - print(" └── models/") - print(" ├── checkpoints/ <- Put your .safetensors/.ckpt models here") - print(" ├── loras/ <- Put your LoRA files here") - print(" ├── vae/ <- Put your VAE files here") - print(" ├── clip/ <- Put your CLIP models here") - print(" ├── controlnet/ <- Put your ControlNet models here") - print(" ├── embeddings/ <- Put your embedding files here") - print(" └── upscale_models/ <- Put your upscale models here") - - -def format_size(size_bytes): - """Format bytes into human-readable size.""" - for unit in ["B", "KB", "MB", "GB"]: - if size_bytes < 1024: - return f"{size_bytes:.1f} {unit}" - size_bytes /= 1024 - return f"{size_bytes:.1f} TB" - - # Time to wait between API check attempts in milliseconds COMFY_API_AVAILABLE_INTERVAL_MS = 50 # Maximum number of API check attempts @@ -658,7 +517,7 @@ def handler(job): # --------------------------------------------------------------------------- # Network Volume Diagnostics (opt-in via NETWORK_VOLUME_DEBUG=true) # --------------------------------------------------------------------------- - if NETWORK_VOLUME_DEBUG: + if is_network_volume_debug_enabled(): run_network_volume_diagnostics() job_input = job["input"] diff --git a/src/network_volume.py b/src/network_volume.py new file mode 100644 index 000000000..ec6d34739 --- /dev/null +++ b/src/network_volume.py @@ -0,0 +1,153 @@ +""" +Network Volume diagnostics for worker-comfyui. + +This module provides tools to debug network volume model path issues. +Enable diagnostics by setting NETWORK_VOLUME_DEBUG=true environment variable. +""" + +import os + +# Expected model types and their file extensions +MODEL_TYPES = { + "checkpoints": [".safetensors", ".ckpt", ".pt", ".pth", ".bin"], + "clip": [".safetensors", ".pt", ".bin"], + "clip_vision": [".safetensors", ".pt", ".bin"], + "configs": [".yaml", ".json"], + "controlnet": [".safetensors", ".pt", ".pth", ".bin"], + "embeddings": [".safetensors", ".pt", ".bin"], + "loras": [".safetensors", ".pt"], + "upscale_models": [".safetensors", ".pt", ".pth"], + "vae": [".safetensors", ".pt", ".bin"], + "unet": [".safetensors", ".pt", ".bin"], +} + + +def is_network_volume_debug_enabled(): + """Check if network volume debug mode is enabled via environment variable.""" + return os.environ.get("NETWORK_VOLUME_DEBUG", "false").lower() == "true" + + +def run_network_volume_diagnostics(): + """ + Run comprehensive network volume diagnostics and print helpful output. + Only runs when NETWORK_VOLUME_DEBUG=true environment variable is set. + """ + print("=" * 70) + print("NETWORK VOLUME DIAGNOSTICS (NETWORK_VOLUME_DEBUG=true)") + print("=" * 70) + + # Check extra_model_paths.yaml + extra_model_paths_file = "/comfyui/extra_model_paths.yaml" + print("\n[1] Checking extra_model_paths.yaml configuration...") + if os.path.isfile(extra_model_paths_file): + print(f" ✓ FOUND: {extra_model_paths_file}") + with open(extra_model_paths_file, "r") as f: + content = f.read() + print("\n Configuration content:") + for line in content.split("\n"): + print(f" {line}") + else: + print(f" ✗ NOT FOUND: {extra_model_paths_file}") + print( + " This file is required for ComfyUI to find models on the network volume." + ) + + # Check network volume mount + runpod_volume = "/runpod-volume" + print(f"\n[2] Checking network volume mount at {runpod_volume}...") + if os.path.isdir(runpod_volume): + print(f" ✓ MOUNTED: {runpod_volume}") + else: + print(f" ✗ NOT MOUNTED: {runpod_volume}") + print( + " Make sure you have attached a network volume to your serverless endpoint." + ) + print("=" * 70) + return + + # Check directory structure + print("\n[3] Checking directory structure...") + models_dir = os.path.join(runpod_volume, "models") + if os.path.isdir(models_dir): + print(f" ✓ FOUND: {models_dir}") + else: + print(f" ✗ NOT FOUND: {models_dir}") + print("\n ⚠️ PROBLEM: The 'models' directory does not exist!") + print(" You need to create the following structure on your network volume:") + print_expected_structure() + print("=" * 70) + return + + # List model directories and their contents + print("\n[4] Scanning model directories...") + found_any_models = False + + for model_type, extensions in MODEL_TYPES.items(): + model_path = os.path.join(models_dir, model_type) + if os.path.isdir(model_path): + files = [] + try: + for f in os.listdir(model_path): + file_path = os.path.join(model_path, f) + if os.path.isfile(file_path): + # Check if file has valid extension + ext = os.path.splitext(f)[1].lower() + if ext in extensions: + size = os.path.getsize(file_path) + size_str = format_size(size) + files.append(f"{f} ({size_str})") + found_any_models = True + else: + files.append(f"{f} (⚠️ ignored - invalid extension)") + except Exception as e: + print(f" {model_type}/: Error reading directory - {e}") + continue + + if files: + print(f"\n {model_type}/:") + for f in files: + print(f" - {f}") + else: + print(f"\n {model_type}/: (empty)") + else: + print(f"\n {model_type}/: (directory not found)") + + # Summary + print("\n[5] Summary") + if found_any_models: + print(" ✓ Models found on network volume!") + print(" ComfyUI should be able to load these models.") + else: + print(" ⚠️ No valid model files found on network volume!") + print("\n Make sure your models have the correct file extensions:") + print(" - Checkpoints: .safetensors, .ckpt, .pt, .pth, .bin") + print(" - LoRAs: .safetensors, .pt") + print(" - VAE: .safetensors, .pt, .bin") + print(" - etc.") + + print_expected_structure() + print("=" * 70) + + +def print_expected_structure(): + """Print the expected directory structure for the network volume.""" + print("\n Expected directory structure:") + print(" /runpod-volume/") + print(" └── models/") + print(" ├── checkpoints/ <- Put your .safetensors/.ckpt models here") + print(" ├── loras/ <- Put your LoRA files here") + print(" ├── vae/ <- Put your VAE files here") + print(" ├── clip/ <- Put your CLIP models here") + print(" ├── controlnet/ <- Put your ControlNet models here") + print(" ├── embeddings/ <- Put your embedding files here") + print(" └── upscale_models/ <- Put your upscale models here") + + +def format_size(size_bytes): + """Format bytes into human-readable size.""" + for unit in ["B", "KB", "MB", "GB"]: + if size_bytes < 1024: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024 + return f"{size_bytes:.1f} TB" + diff --git a/src/start.sh b/src/start.sh index 3536055ac..15d38ad51 100644 --- a/src/start.sh +++ b/src/start.sh @@ -7,30 +7,6 @@ export LD_PRELOAD="${TCMALLOC}" # Ensure ComfyUI-Manager runs in offline network mode inside the container comfy-manager-set-mode offline || echo "worker-comfyui - Could not set ComfyUI-Manager network_mode" >&2 -# Diagnostic: Check extra_model_paths.yaml -if [ -f /comfyui/extra_model_paths.yaml ]; then - echo "worker-comfyui: extra_model_paths.yaml found at /comfyui/" - echo "worker-comfyui: extra_model_paths.yaml content:" - cat /comfyui/extra_model_paths.yaml -else - echo "worker-comfyui: WARNING - extra_model_paths.yaml NOT found at /comfyui/" -fi - -# Diagnostic: Check network volume -if [ -d /runpod-volume ]; then - echo "worker-comfyui: Network volume mounted at /runpod-volume" - echo "worker-comfyui: /runpod-volume contents:" - ls -la /runpod-volume/ - if [ -d /runpod-volume/models ]; then - echo "worker-comfyui: /runpod-volume/models contents:" - ls -la /runpod-volume/models/ - else - echo "worker-comfyui: WARNING - /runpod-volume/models does NOT exist" - fi -else - echo "worker-comfyui: INFO - /runpod-volume does not exist (no network volume attached)" -fi - echo "worker-comfyui: Starting ComfyUI" # Allow operators to tweak verbosity; default is DEBUG.