feat(cli): add deterministic task preflight checks

docs/v6/reference/cli.mdx

@@ -44,7 +44,8 @@ hud serve env.py -p 9000
 
 ### `hud deploy`
 
-Build **and** publish to HUD infra in one step. The environment's name comes
+Build **and** publish to HUD infra in one step. This is the CLI's environment
+build path. The environment's name comes
 from the `Environment(...)` declaration in code; deploying the same name again
 rebuilds that environment.
 
@@ -110,19 +111,24 @@ For a platform taskset, pass its name or id directly: `hud eval "My Tasks" claud
 
 ## Run a packaged image
 
-`hud task start` / `hud task grade` attach to an env already serving locally (e.g. inside a built image, or alongside `hud serve`), or load one from source with `--source`. `hud task list` always reads from source (default `.`) - it doesn't attach.
+`hud task start` / `hud task grade` attach to an env already serving locally (e.g. inside a built image, or alongside `hud serve`), or load one from source with `--source`. Both verify that the live environment exposes the requested task before running it.
 
 ```bash
 hud task list                          # what tasks are exposed
+hud task list --env browser-v2         # boot a deployed env and read its live manifest
 hud task start fix_bug                 # -> the prompt (stdout)
 hud task grade fix_bug --answer "..."  # -> the reward (stdout)
+hud task grade fix_bug --dry-run       # start + empty-answer grade, no agent rollout
+hud task grade task/fix_bug --env env/browser-v2 --dry-run  # check a deployed env
 ```
 
+`--dry-run` reports the environment, task, grader, and reward-shape phases separately. A valid low reward passes: the check verifies that grading works and returns finite values in the supported range, not that an empty answer solves the task.
+
 | Command | Key options |
 |---------|-------------|
-| `hud task start <task>` | `--source`/`-s`, `--args` (JSON), `--url`/`-u`, `--out`/`-o` |
-| `hud task grade <task>` | `--answer`, `--answer-file`, `--source`, `--args`, `--url`, `--out` |
-| `hud task list` | `--source`/`-s` |
+| `hud task start <task>` | `--source`/`-s`, `--args` (JSON), `--url`/`-u`, `--env`, `--out`/`-o` |
+| `hud task grade <task>` | `--answer`, `--answer-file`, `--dry-run`, `--timeout`, `--source`, `--args`, `--url`, `--env`, `--out` |
+| `hud task list` | `--source`/`-s`, `--env`, `--url`/`-u` |
 
 ## Platform
 
@@ -131,6 +137,10 @@ hud sync tasks my-taskset      # publish tasks as a named taskset
 hud sync env                   # sync environment metadata
 ```
 
+Before uploading tasks, `hud sync tasks` validates the local source and rejects
+references to deployed environments or task ids that do not exist. `hud deploy`
+runs the same source validation before starting a build.
+
 External benchmark formats can be adapted into runnable `Taskset`s through the
 experimental [Harbor integration](/v6/experimental/harbor).
 

hud/cli/eval.py

@@ -5,7 +5,6 @@
 
 from __future__ import annotations
 
-import ast
 import asyncio
 import logging
 import os
@@ -23,6 +22,7 @@
 from rich import box
 from rich.table import Table
 
+from hud.cli.task_runtime import spawn_target as _spawn_target
 from hud.cli.utils.api import require_api_key
 from hud.cli.utils.config import parse_key_value
 from hud.settings import settings
@@ -686,49 +686,6 @@ def _build_agent(cfg: EvalConfig) -> Any:
     return cast("Any", cfg.agent_type.cls)(config=config)
 
 
-def _python_defines_environment(path: Path) -> bool:
-    """Return True when ``path`` constructs a v6 :class:`~hud.environment.Environment`."""
-    try:
-        tree = ast.parse(path.read_text(encoding="utf-8"))
-    except (OSError, SyntaxError):
-        return False
-    for node in ast.walk(tree):
-        if not isinstance(node, ast.Call):
-            continue
-        callee = node.func
-        callee_name = (
-            callee.id
-            if isinstance(callee, ast.Name)
-            else callee.attr
-            if isinstance(callee, ast.Attribute)
-            else None
-        )
-        if callee_name == "Environment":
-            return True
-    return False
-
-
-def _spawn_target(source: Path) -> Path:
-    """The path the ``SubprocessRuntime`` provider serves.
-
-    Directories and env-defining ``.py`` files are served as-is. Task-only
-    sources (``tasks.py`` importing from ``env.py``) resolve to a sibling
-    ``env.py`` or the containing directory. JSON/JSONL data files use the
-    surrounding directory (the env source lives next to the tasks file).
-    """
-    resolved = source.resolve()
-    if resolved.is_dir():
-        return resolved
-    if resolved.suffix != ".py":
-        return resolved.parent
-    if _python_defines_environment(resolved):
-        return resolved
-    env_py = resolved.parent / "env.py"
-    if env_py.is_file():
-        return env_py
-    return resolved.parent
-
-
 def _resolve_placement(cfg: EvalConfig, source_path: Path | None, taskset: Any) -> Any:
     """Map the config's ``runtime`` onto a placement for ``Taskset.run``.
 

hud/cli/sync.py

@@ -7,6 +7,7 @@
 import logging
 from pathlib import Path
 from typing import Any
+from uuid import UUID
 
 import typer
 
@@ -22,6 +23,7 @@
 from hud.eval.sync import diff, resolve_taskset_id, upload_taskset
 from hud.utils.exceptions import HudException, HudRequestError
 from hud.utils.hud_console import HUDConsole
+from hud.utils.naming import normalize_environment_name
 from hud.utils.platform import PlatformClient
 
 LOGGER = logging.getLogger(__name__)
@@ -175,6 +177,89 @@ def _warn_on_linked_environment_mismatch(
         )
 
 
+def _resolve_registry_environment_detail(
+    platform: PlatformClient,
+    ref: str,
+) -> RegistryEnvironment:
+    lookup_ref = ref.removeprefix("environment/").removeprefix("env/")
+    normalized_ref = normalize_environment_name(lookup_ref)
+    try:
+        UUID(lookup_ref)
+        resolution_ref = lookup_ref
+    except ValueError:
+        resolution_ref = normalized_ref
+    matches = [
+        env
+        for env in resolve_registry_environments(platform, resolution_ref)
+        if env.id == lookup_ref or normalize_environment_name(env.name) == normalized_ref
+    ]
+    if not matches:
+        raise ValueError(f"environment {ref!r} is not deployed")
+    if len(matches) > 1:
+        raise ValueError(f"environment name {ref!r} is ambiguous")
+    registry_env = get_registry_environment(platform, matches[0].id)
+    if registry_env is None:
+        raise ValueError(f"environment {ref!r} is not deployed")
+    return registry_env
+
+
+def _validate_task_manifests(
+    taskset: Taskset,
+    platform: PlatformClient,
+    console: HUDConsole,
+) -> None:
+    """Fail before upload when a deployed environment cannot expose a task."""
+    errors: list[str] = []
+    for env_name in sorted(taskset.environment_names()):
+        try:
+            registry_env = _resolve_registry_environment_detail(platform, env_name)
+        except ValueError as exc:
+            errors.append(str(exc))
+            continue
+        except HudException as exc:
+            errors.append(f"could not validate environment {env_name!r}: {exc}")
+            continue
+        manifest = registry_env.manifest
+        raw_tasks = manifest.get("tasks") if manifest is not None else None
+        if not isinstance(raw_tasks, list):
+            errors.append(f"environment {env_name!r} has no successful build manifest")
+            continue
+        exposed = {
+            task["id"]
+            for task in raw_tasks
+            if isinstance(task, dict) and isinstance(task.get("id"), str)
+        }
+        required = {task.id for task in taskset if task.env == env_name}
+        required.update(
+            task.verifier.id
+            for task in taskset
+            if task.verifier is not None and task.verifier.env == env_name
+        )
+        missing = sorted(required - exposed)
+        if missing:
+            errors.append(f"environment {env_name!r} does not expose task(s): {', '.join(missing)}")
+
+    if errors:
+        console.error("Task validation failed:")
+        for error in errors:
+            console.error(f"  {error}")
+        console.hint("Deploy the environment or fix the task ids before syncing.")
+        raise typer.Exit(1)
+    console.success("Task environment manifests validated")
+
+
+def _validate_source(source: str, console: HUDConsole) -> None:
+    errors = [
+        issue for issue in EnvironmentSource.open(source).validate() if issue.severity == "error"
+    ]
+    if not errors:
+        return
+    console.error("Source validation failed:")
+    for issue in errors:
+        console.error(f"  {issue.message} ({issue.file})")
+    raise typer.Exit(1)
+
+
 def _fetch_remote_taskset(
     platform: PlatformClient,
     target_ref: str,
@@ -313,6 +398,8 @@ def sync_tasks_command(
         exclude=exclude,
         console=hud_console,
     )
+    _validate_source(source, hud_console)
+    _validate_task_manifests(local_taskset, platform, hud_console)
     _warn_on_linked_environment_mismatch(local_taskset, platform, hud_console)
 
     # Creating a new taskset is only allowed when targeting an explicit name