diff --git a/.changeset/dr-1170-startup-crash.md b/.changeset/dr-1170-startup-crash.md new file mode 100644 index 000000000..7d19c56a2 --- /dev/null +++ b/.changeset/dr-1170-startup-crash.md @@ -0,0 +1,5 @@ +--- +"worker-comfyui": patch +--- + +Fix ComfyUI startup crash that surfaced as `ComfyUI server (127.0.0.1:8188) not reachable after multiple retries`. ComfyUI's full runtime dependency set is now installed into the same virtualenv that `start.sh` launches ComfyUI with, so the server starts reliably. Also pins `transformers<5` / `huggingface-hub<1`, adds a build-time smoke test that fails the build if ComfyUI can't start, and the GPU pre-flight now launches a real kernel for a clear error on a kernel/arch mismatch. diff --git a/Dockerfile b/Dockerfile index e4fcf7d23..c772d61b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,32 @@ RUN if [ "$ENABLE_PYTORCH_UPGRADE" = "true" ]; then \ uv pip install --force-reinstall torch torchvision torchaudio --index-url ${PYTORCH_INDEX_URL}; \ fi +# comfy-cli installs ComfyUI into its own workspace venv (/comfyui/.venv), but +# start.sh launches ComfyUI with /opt/venv's python. That mismatch leaves the +# launch venv missing ComfyUI's runtime deps (e.g. sqlalchemy, pulled in by +# ComfyUI's asset DB), so ComfyUI crashes at startup and surfaces as the +# misleading "ComfyUI server (127.0.0.1:8188) not reachable" error. Mirror +# ComfyUI's full dependency set (core + custom nodes) into /opt/venv so the +# launch venv is complete. Root-cause fix for DR-1170. +# +# The transformers/huggingface-hub pin is part of the SAME step on purpose: +# ComfyUI declares transformers>=4.50.3 and huggingface-hub with NO upper bound, +# so a fresh install can pull transformers 5.x / huggingface-hub 1.x whose +# breaking API changes also crash ComfyUI at startup. Pinning them in the same +# RUN downgrades within one layer, so the unwanted versions aren't left behind +# bloating the image. +RUN uv pip install -r /comfyui/requirements.txt \ + && for r in /comfyui/custom_nodes/*/requirements.txt; do \ + [ -f "$r" ] && uv pip install -r "$r" || true; \ + done \ + && uv pip install "transformers>=4.50.3,<5" "huggingface-hub<1.0" + +# Build-time smoke test: actually start ComfyUI (imports the full node graph) so +# a startup-breaking dependency is caught HERE, at build time, instead of as a +# runtime "server not reachable" failure on a live worker. Runs on CPU — no GPU +# needed to exercise the import graph. +RUN cd /comfyui && timeout 300 python main.py --quick-test-for-ci --cpu + # Change working directory to ComfyUI WORKDIR /comfyui diff --git a/src/start.sh b/src/start.sh index c490b0db1..4d10703d9 100644 --- a/src/start.sh +++ b/src/start.sh @@ -34,15 +34,24 @@ import torch try: torch.cuda.init() name = torch.cuda.get_device_name(0) - print(f'OK: {name}') + cap = torch.cuda.get_device_capability(0) + # Launch a real kernel. The driver-only calls above succeed even when this + # PyTorch build has no compiled kernels for the GPU architecture (e.g. an + # older torch on a newer GPU). Without this, the worker boots, ComfyUI dies + # on the first GPU op, and it surfaces as the misleading 'server not + # reachable' error instead of a clear cause here. + _ = (torch.zeros(8, device='cuda') + 1).sum().item() + torch.cuda.synchronize() + print(f'OK: {name} (sm_{cap[0]}{cap[1]}), torch {torch.__version__}, cuda {torch.version.cuda}') except Exception as e: print(f'FAIL: {e}') exit(1) " 2>&1); then - echo "worker-comfyui: GPU is not available. PyTorch CUDA init failed:" + echo "worker-comfyui: GPU is not available or incompatible with this PyTorch build:" echo "worker-comfyui: $GPU_CHECK" - echo "worker-comfyui: This usually means the GPU on this machine is not properly initialized." - echo "worker-comfyui: Please contact RunPod support and report this machine." + echo "worker-comfyui: A 'no kernel image is available' error means this torch build" + echo "worker-comfyui: lacks kernels for this GPU. Otherwise the GPU may not be" + echo "worker-comfyui: properly initialized — please contact RunPod support." exit 1 fi echo "worker-comfyui: GPU available — $GPU_CHECK"