diff --git a/.ai-context/COMMANDS.md b/.ai-context/COMMANDS.md index 4a2526e5..cd185606 100644 --- a/.ai-context/COMMANDS.md +++ b/.ai-context/COMMANDS.md @@ -57,13 +57,13 @@ the v1.x compatibility window; new automation should use `--debug-wrapper`. - `basectl docs` - open the Base documentation home page on GitHub. - `basectl projects list` - list Base-managed projects discovered in the workspace. -- `basectl workspace ` - +- `basectl workspace ` - inspect workspace status, checks, diagnostics, read-only first-day onboarding, and local agent-handoff readiness; explicitly clone expected repositories from a manifest; initialize a workspace from a workspace configuration repo; explicitly sync a local manifest from a configured canonical source; apply repo configuration across a workspace; or run local - project setup across eligible repositories. + project setup and declared tests across eligible repositories. - `workspace status`, `workspace check`, `workspace doctor`, `workspace onboarding`, and `workspace agent-brief` support `--format json`; `workspace update` also supports stable `--format json` output documented in @@ -101,6 +101,11 @@ the v1.x compatibility window; new automation should use `--debug-wrapper`. without `--dry-run`, it skips ineligible repositories, continues after per-repo failures, and reports setup/skipped/failed counts. `--yes` forwards confirmation to each delegated setup command. + - `workspace test` runs declared project test commands serially in manifest + order. `--projects name[,name...]` selects a subset, `--fail-fast` stops + after the first failure, and `--format json` emits per-project results and + passed/failed/skipped counts. Projects without a declared test command are + skipped. - `workspace update --dry-run` previews the ordered Git pull plan; without `--dry-run`, it continues after per-repo failures and reports updated/unchanged/skipped/failed counts. JSON dry runs report `planned` diff --git a/cli/bash/commands/basectl/basectl.sh b/cli/bash/commands/basectl/basectl.sh index 77b97de0..ce68485a 100644 --- a/cli/bash/commands/basectl/basectl.sh +++ b/cli/bash/commands/basectl/basectl.sh @@ -68,8 +68,8 @@ Compatibility: Deprecated alias for the corresponding lifecycle command with --ci. Workspace and repositories: - workspace [options] - Show workspace status, onboarding, agent readiness, checks, or explicit workspace mutations. + workspace [options] + Show workspace status, onboarding, agent readiness, checks, declared project tests, or explicit workspace mutations. repo [options] Create, clone, check, and configure repository baselines and guidance. gh [options] diff --git a/cli/bash/commands/basectl/subcommands/workspace.sh b/cli/bash/commands/basectl/subcommands/workspace.sh index 5e73c5c5..17135b3e 100644 --- a/cli/bash/commands/basectl/subcommands/workspace.sh +++ b/cli/bash/commands/basectl/subcommands/workspace.sh @@ -175,6 +175,28 @@ Set up eligible repositories in a workspace manifest in manifest order. EOF } +base_workspace_test_usage() { + cat <<'EOF' +Usage: + basectl workspace test [options] + +Options: + --workspace Workspace directory to test. Defaults to workspace.root, then BASE_HOME's parent. + --manifest Local workspace manifest describing expected repositories. + Overrides workspace.manifest from ~/.base.d/config.yaml. + --projects + Test only the selected workspace projects, in manifest order. + --fail-fast Stop after the first project failure. + --format + Output text or structured JSON. Defaults to text. + -v Enable DEBUG logging for this subcommand. + -h, --help Show this help text. + +Run each selected project's declared test command serially. +Projects without a declared test command are skipped. +EOF +} + base_workspace_subcommand_usage() { case "${1:-}" in status|check|doctor) @@ -204,10 +226,13 @@ base_workspace_subcommand_usage() { setup) base_workspace_setup_usage ;; + test) + base_workspace_test_usage + ;; *) cat <<'EOF' Usage: - basectl workspace [options] + basectl workspace [options] Commands: status Show workspace status. Supports --format text|csv|tsv|yaml|json. @@ -221,6 +246,7 @@ Commands: init Initialize a workspace from a workspace configuration repository. configure Apply repo configure across workspace repositories. setup Set up eligible workspace repositories in manifest order. + test Run declared project tests serially across selected repositories. Run `basectl workspace --help` for command-specific options. EOF @@ -244,7 +270,7 @@ base_workspace_subcommand_main() { base_workspace_subcommand_usage return 0 ;; - status|check|doctor|onboarding|agent-brief|clone|pull|update|init|configure|setup) + status|check|doctor|onboarding|agent-brief|clone|pull|update|init|configure|setup|test) shift ;; *) diff --git a/cli/bash/commands/basectl/tests/completions.bats b/cli/bash/commands/basectl/tests/completions.bats index 35ad7302..172f4601 100644 --- a/cli/bash/commands/basectl/tests/completions.bats +++ b/cli/bash/commands/basectl/tests/completions.bats @@ -173,6 +173,10 @@ EOF COMP_CWORD=3; \ _base_basectl_completion; \ printf "workspace_setup_options=%s\n" "${COMPREPLY[*]}"; \ + COMP_WORDS=(basectl workspace test --); \ + COMP_CWORD=3; \ + _base_basectl_completion; \ + printf "workspace_test_options=%s\n" "${COMPREPLY[*]}"; \ COMP_WORDS=(basectl onboard --); \ COMP_CWORD=2; \ _base_basectl_completion; \ @@ -311,6 +315,7 @@ EOF [[ "$output" == *"workspace_init_options=--owner --path --workspace --manifest --include-optional --dry-run"* ]] [[ "$output" == *"workspace_configure_options=--workspace --manifest --dry-run"* ]] [[ "$output" == *"workspace_setup_options=--workspace --manifest --dry-run --yes"* ]] + [[ "$output" == *"workspace_test_options=--workspace --manifest --projects --fail-fast --format"* ]] [[ "$output" == *"onboard_options=--profile --dry-run --yes --allow-project-ide-mutations --no-profile"* ]] [[ "$output" == *"onboard_projects=base demo"* ]] [[ "$output" == *"onboard_profiles=dev sre ai linux-lab dev,sre dev,ai dev,linux-lab sre,ai sre,linux-lab ai,linux-lab dev,sre,ai dev,sre,linux-lab dev,ai,linux-lab sre,ai,linux-lab dev,sre,ai,linux-lab"* ]] @@ -451,6 +456,6 @@ EOF printf "options=%s\n" "${COMPREPLY[*]}"' [ "$status" -eq 0 ] - [[ "$output" == *"commands=status check doctor onboarding agent-brief clone pull update init configure setup"* ]] + [[ "$output" == *"commands=status check doctor onboarding agent-brief clone pull update init configure setup test"* ]] [[ "$output" == *"options=--workspace --manifest --dry-run --apply --yes"* ]] } diff --git a/cli/bash/commands/basectl/tests/help.bats b/cli/bash/commands/basectl/tests/help.bats index ecdb11cc..383d5759 100644 --- a/cli/bash/commands/basectl/tests/help.bats +++ b/cli/bash/commands/basectl/tests/help.bats @@ -32,7 +32,7 @@ load ./basectl_helpers.bash [[ "$output" == *"demo [project] [options]"* ]] [[ "$output" == *"update [project] [options]"* ]] [[ "$output" == *"projects list [options]"* ]] - [[ "$output" == *"workspace [options]"* ]] + [[ "$output" == *"workspace [options]"* ]] [[ "$output" == *"Invoking \`basectl\` with no command starts a Base runtime shell"* ]] [[ "$output" == *"--version"* ]] [[ "$output" == *"Wrapper options:"* ]] @@ -98,7 +98,7 @@ load ./basectl_helpers.bash grep -Fqx ' docs [options]' <<<"$output" grep -Fqx ' logs [options]' <<<"$output" grep -Fqx ' history [options]' <<<"$output" - grep -Fqx ' workspace [options]' <<<"$output" + grep -Fqx ' workspace [options]' <<<"$output" grep -Fqx ' trust [project] [options]' <<<"$output" [[ "$output" != *"-b DIR"* ]] [[ "$output" != *"Force install"* ]] @@ -117,7 +117,7 @@ load ./basectl_helpers.bash [ "$status" -eq 0 ] [[ "$output" == *"Usage:"* ]] - [[ "$output" == *"basectl workspace [options]"* ]] + [[ "$output" == *"basectl workspace [options]"* ]] [[ "$output" != *"Usage: basectl [options] [args...]"* ]] run_basectl help release @@ -250,7 +250,7 @@ load ./basectl_helpers.bash @test "AI command context includes current clone and update surfaces" { local commands_file="$BASE_REPO_ROOT/.ai-context/COMMANDS.md" - grep -Fqx -- "- \`basectl workspace \` -" "$commands_file" + grep -Fqx -- "- \`basectl workspace \` -" "$commands_file" grep -Fqx -- " - \`workspace clone\` mutates repository checkouts only when invoked directly;" "$commands_file" grep -Fqx -- "- \`basectl repo \` -" "$commands_file" grep -Fqx -- "- \`basectl update [project]\` - update Base or a named project using the" "$commands_file" diff --git a/cli/bash/commands/basectl/tests/workspace.bats b/cli/bash/commands/basectl/tests/workspace.bats index e42e486b..cdd176f4 100644 --- a/cli/bash/commands/basectl/tests/workspace.bats +++ b/cli/bash/commands/basectl/tests/workspace.bats @@ -475,6 +475,17 @@ EOF [[ "$output" == *"ordered workspace setup plan"* ]] [[ "$output" != *"--format"* ]] + run_basectl workspace test --help + + [ "$status" -eq 0 ] + [[ "$output" == *"basectl workspace test [options]"* ]] + [[ "$output" == *"--workspace "* ]] + [[ "$output" == *"--manifest "* ]] + [[ "$output" == *"--projects "* ]] + [[ "$output" == *"--fail-fast"* ]] + [[ "$output" == *"--format "* ]] + [[ "$output" == *"declared test command"* ]] + run_basectl workspace init --help [ "$status" -eq 0 ] @@ -490,7 +501,7 @@ EOF run_basectl workspace help [ "$status" -eq 0 ] - [[ "$output" == *"basectl workspace [options]"* ]] + [[ "$output" == *"basectl workspace [options]"* ]] [[ "$output" != *"Project virtual environment Python was not found"* ]] } diff --git a/cli/python/base_projects/engine.py b/cli/python/base_projects/engine.py index b04f582e..31ba22b5 100644 --- a/cli/python/base_projects/engine.py +++ b/cli/python/base_projects/engine.py @@ -53,6 +53,7 @@ from base_projects.workspace_init import workspace_init_command from base_projects.workspace_pull_command import workspace_pull_command from base_projects.workspace_setup import workspace_setup_from_options +from base_projects.workspace_test import workspace_test_from_options from base_projects.workspace_update import workspace_update_from_options from base_projects.workspace_onboarding import workspace_onboarding_summary from base_projects.workspace_report_json import workspace_check_to_json @@ -122,6 +123,16 @@ def main(argv: list[str] | None = None) -> int: ) @base_cli.option("--apply", is_flag=True, help="Apply workspace configure changes after the plan is shown.") @base_cli.option("--yes", is_flag=True, help="Approve workspace setup or configure changes that require confirmation.") +@base_cli.option( + "--projects", + "workspace_projects", + help="Comma-separated workspace project names to test.", +) +@base_cli.option( + "--fail-fast", + is_flag=True, + help="Stop workspace testing after the first project failure.", +) @base_cli.option( "--test-preflight", is_flag=True, @@ -143,6 +154,8 @@ def run( dry_run: bool, apply: bool, yes: bool, + workspace_projects: str | None, + fail_fast: bool, test_preflight: bool, ) -> int: try: @@ -162,6 +175,8 @@ def run( dry_run=dry_run, apply=apply, yes=yes, + workspace_projects=workspace_projects, + fail_fast=fail_fast, test_preflight=test_preflight, ), project_command_actions(), @@ -184,6 +199,7 @@ def project_command_actions() -> ProjectCommandActions: workspace_init=workspace_init_project_command, workspace_configure=workspace_configure_from_options, workspace_setup=workspace_setup_from_options, + workspace_test=workspace_test_from_options, workspace_update=workspace_update_from_options, current_project=current_project_command, manifest_project=manifest_project_command, diff --git a/cli/python/base_projects/project_dispatch.py b/cli/python/base_projects/project_dispatch.py index 1c2357c0..c351f168 100644 --- a/cli/python/base_projects/project_dispatch.py +++ b/cli/python/base_projects/project_dispatch.py @@ -14,6 +14,7 @@ class WorkspaceCommandOptions: project_name: str | None = None workspace_manifest: str | None = None workspace_repos: str | None = None + workspace_projects: str | None = None workspace_manifest_source: str | None = None workspace_config_path: str | None = None workspace_owner: str | None = None @@ -21,6 +22,7 @@ class WorkspaceCommandOptions: dry_run: bool = False apply: bool = False yes: bool = False + fail_fast: bool = False test_preflight: bool = False @@ -38,6 +40,7 @@ class ProjectCommandActions: workspace_init: Callable[[base_cli.Context, str, WorkspaceCommandOptions], int] workspace_configure: Callable[[base_cli.Context, WorkspaceCommandOptions], int] workspace_setup: Callable[[base_cli.Context, WorkspaceCommandOptions], int] + workspace_test: Callable[[base_cli.Context, WorkspaceCommandOptions], int] current_project: Callable[[base_cli.Context, str], int] manifest_project: Callable[[base_cli.Context, str | None, str], int] resolve_project: Callable[[base_cli.Context, str | None, str | None, str], int] @@ -217,6 +220,16 @@ def _handle_setup( return actions.workspace_setup(ctx, options) +def _handle_test( + ctx: base_cli.Context, + arguments: tuple[str, ...], + options: WorkspaceCommandOptions, + actions: ProjectCommandActions, +) -> int: + require_argument_count("test", arguments, 0, 0) + return actions.workspace_test(ctx, options) + + def _handle_current( ctx: base_cli.Context, arguments: tuple[str, ...], @@ -339,6 +352,7 @@ def _handle_build_target_list( "clone", "configure", "setup", + "test", "init", "test-command", "demo-script", @@ -364,6 +378,7 @@ def _handle_build_target_list( "init": _handle_init, "configure": _handle_configure, "setup": _handle_setup, + "test": _handle_test, "current": _handle_current, "manifest": _handle_manifest, "resolve": _handle_resolve, diff --git a/cli/python/base_projects/tests/test_engine.py b/cli/python/base_projects/tests/test_engine.py index 9a4d0602..23eba2dd 100644 --- a/cli/python/base_projects/tests/test_engine.py +++ b/cli/python/base_projects/tests/test_engine.py @@ -168,7 +168,7 @@ def invoke_engine( user_config: str | None = None, extra_env: dict[str, str] | None = None, ) -> tuple[int, str, str]: - workspace_commands = {"status", "check", "doctor", "onboarding", "agent-brief", "run-commands"} + workspace_commands = {"status", "check", "doctor", "onboarding", "agent-brief", "run-commands", "test"} stdout = TerminalStringIO() if any(argument in workspace_commands for argument in args) else io.StringIO() stderr = io.StringIO() if user_config is not None: diff --git a/cli/python/base_projects/tests/test_workspace_test.py b/cli/python/base_projects/tests/test_workspace_test.py new file mode 100644 index 00000000..953c8491 --- /dev/null +++ b/cli/python/base_projects/tests/test_workspace_test.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from unittest import mock + +from base_projects import engine + + +def write_workspace_manifest(path: Path, repositories: tuple[str, ...]) -> None: + path.write_text( + "\n".join( + [ + "schema_version: 1", + "workspace:", + " name: test-workspace", + "repos:", + *[f" - name: {repository}" for repository in repositories], + "", + ] + ), + encoding="utf-8", + ) + + +def write_test_manifest(project_root: Path, name: str) -> None: + project_root.mkdir(parents=True) + (project_root / "base_manifest.yaml").write_text( + f"project:\n name: {name}\ntest:\n command: ./run-tests.sh\nartifacts: []\n", + encoding="utf-8", + ) + + +def write_shell_only_manifest(project_root: Path, name: str) -> None: + project_root.mkdir(parents=True) + (project_root / "base_manifest.yaml").write_text( + f"project:\n name: {name}\nartifacts: []\n", + encoding="utf-8", + ) + + +def write_fake_basectl(base_home: Path) -> Path: + basectl = base_home / "bin" / "basectl" + basectl.parent.mkdir(parents=True) + basectl.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + basectl.chmod(0o755) + return basectl + + +def invoke_workspace_test( + args: list[str], + base_home: Path, + home: Path, +) -> tuple[int, str, str]: + stdout = StringIO() + stderr = StringIO() + env = { + "HOME": str(home), + "BASE_HOME": str(base_home), + "BASE_PROJECT": "", + "BASE_PROJECT_MANIFEST": "", + } + with mock.patch.dict(os.environ, env): + with redirect_stdout(stdout), redirect_stderr(stderr): + status = engine.main(args) + return status, stdout.getvalue(), stderr.getvalue() + + +class WorkspaceTestCommandTests(unittest.TestCase): + def test_workspace_test_runs_all_projects_and_reports_json_results(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "beta")) + write_test_manifest(workspace / "alpha", "alpha") + write_test_manifest(workspace / "beta", "beta") + completed = subprocess.CompletedProcess([], 0, "alpha output\n", "") + + with mock.patch("base_projects.workspace_test.subprocess.run", return_value=completed) as run: + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest), "--format", "json"], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 0) + self.assertEqual(stderr, "") + self.assertEqual(payload["counts"], {"passed": 2, "failed": 0, "skipped": 0}) + self.assertEqual(payload["selected_projects"], ["alpha", "beta"]) + self.assertEqual([project["status"] for project in payload["projects"]], ["passed", "passed"]) + self.assertEqual(run.call_count, 2) + self.assertEqual(run.call_args_list[0].kwargs["cwd"], (workspace / "alpha").resolve()) + self.assertEqual(run.call_args_list[1].kwargs["cwd"], (workspace / "beta").resolve()) + self.assertEqual(run.call_args_list[0].args[0][-2:], ["--project", "alpha"]) + + def test_workspace_test_continues_after_a_project_failure(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "beta", "gamma")) + for project in ("alpha", "beta", "gamma"): + write_test_manifest(workspace / project, project) + results = [ + subprocess.CompletedProcess([], 0, "", ""), + subprocess.CompletedProcess([], 7, "", "beta failed\n"), + subprocess.CompletedProcess([], 0, "", ""), + ] + + with mock.patch("base_projects.workspace_test.subprocess.run", side_effect=results): + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest), "--format", "json"], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(payload["counts"], {"passed": 2, "failed": 1, "skipped": 0}) + self.assertEqual(payload["projects"][1]["exit_code"], 7) + self.assertEqual(payload["projects"][1]["stderr"], "beta failed\n") + self.assertEqual(stderr, "") + + def test_workspace_test_fail_fast_skips_remaining_projects(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "beta", "gamma")) + for project in ("alpha", "beta", "gamma"): + write_test_manifest(workspace / project, project) + + with mock.patch( + "base_projects.workspace_test.subprocess.run", + return_value=subprocess.CompletedProcess([], 3, "", "failed\n"), + ) as run: + status, stdout, stderr = invoke_workspace_test( + [ + "test", + "--workspace", + str(workspace), + "--manifest", + str(manifest), + "--fail-fast", + "--format", + "json", + ], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(run.call_count, 1) + self.assertEqual(payload["counts"], {"passed": 0, "failed": 1, "skipped": 2}) + self.assertEqual(payload["projects"][1]["status"], "skipped") + self.assertIn("--fail-fast", payload["projects"][1]["detail"]) + self.assertEqual(stderr, "") + + def test_workspace_test_projects_filter_runs_only_selected_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "beta")) + write_test_manifest(workspace / "alpha", "alpha") + write_test_manifest(workspace / "beta", "beta") + + with mock.patch( + "base_projects.workspace_test.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, "", ""), + ) as run: + status, stdout, stderr = invoke_workspace_test( + [ + "test", + "--workspace", + str(workspace), + "--manifest", + str(manifest), + "--projects", + "beta", + "--format", + "json", + ], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 0) + self.assertEqual(stderr, "") + self.assertEqual(run.call_count, 1) + self.assertEqual(payload["selected_projects"], ["beta"]) + self.assertEqual(payload["projects"][0]["project"], "beta") + + def test_workspace_test_skips_project_without_test_command(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "shell-only")) + write_test_manifest(workspace / "alpha", "alpha") + write_shell_only_manifest(workspace / "shell-only", "shell-only") + + with mock.patch( + "base_projects.workspace_test.subprocess.run", + return_value=subprocess.CompletedProcess([], 0, "", ""), + ) as run: + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest), "--format", "json"], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 0) + self.assertEqual(stderr, "") + self.assertEqual(run.call_count, 1) + self.assertEqual(payload["counts"], {"passed": 1, "failed": 0, "skipped": 1}) + self.assertEqual(payload["projects"][1]["status"], "skipped") + self.assertIn("does not declare a test command", payload["projects"][1]["detail"]) diff --git a/cli/python/base_projects/workspace_test.py b/cli/python/base_projects/workspace_test.py new file mode 100644 index 00000000..0a57c513 --- /dev/null +++ b/cli/python/base_projects/workspace_test.py @@ -0,0 +1,424 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +import base_cli +from base_projects import workspace_context +from base_projects.workspace_context import resolve_workspace_manifest +from base_projects.workspace_manifest import WorkspaceManifest +from base_projects.workspace_manifest import WorkspaceManifestError +from base_projects.workspace_manifest import WorkspaceManifestRepo +from base_projects.workspace_scanner import ProjectDiscoveryError +from base_setup.manifest import read_manifest +from base_setup.manifest_loader import ManifestError + + +WorkspaceTestAction = Literal["test", "skip"] +WorkspaceTestStatus = Literal["passed", "failed", "skipped"] +WORKSPACE_TEST_TIMEOUT_SECONDS = 1800 + + +@dataclass(frozen=True) +class WorkspaceTestTarget: + name: str + root: Path + manifest_path: Path | None + project_name: str | None + action: WorkspaceTestAction + reason: str | None = None + required: bool = True + fatal: bool = False + + +@dataclass(frozen=True) +class WorkspaceTestResult: + status: WorkspaceTestStatus + detail: str | None = None + exit_code: int | None = None + stdout: str = "" + stderr: str = "" + + +@dataclass(frozen=True) +class WorkspaceTestCounts: + passed: int = 0 + failed: int = 0 + skipped: int = 0 + + +class WorkspaceTestSelectionError(ValueError): + """Raised when --projects does not identify a valid manifest selection.""" + + +def workspace_test_from_options(ctx: base_cli.Context, options: Any) -> int: + output_format = str(getattr(options, "output_format", "text") or "text").lower() + if output_format not in {"text", "json"}: + ctx.log.error("Unsupported output format '%s'. Expected: text or json.", options.output_format) + return base_cli.ExitCode.USAGE_ERROR + if getattr(options, "dry_run", False): + ctx.log.error("Workspace test does not support --dry-run.") + return base_cli.ExitCode.USAGE_ERROR + + try: + workspace_root = workspace_context.resolve_workspace_root(ctx, options.workspace) + manifest = resolve_workspace_manifest( + workspace_context.effective_workspace_manifest(ctx, options.workspace_manifest) + ) + except (ProjectDiscoveryError, WorkspaceManifestError) as exc: + ctx.log.error(str(exc)) + return base_cli.ExitCode.FAILURE + + if manifest is None: + ctx.log.error( + "Workspace test requires a configured or explicit workspace manifest. " + "Pass --manifest or configure workspace.manifest." + ) + return base_cli.ExitCode.FAILURE + + targets = workspace_test_targets(workspace_root, manifest) + try: + targets = select_workspace_test_targets(targets, getattr(options, "workspace_projects", None)) + except WorkspaceTestSelectionError as exc: + ctx.log.error(str(exc)) + return base_cli.ExitCode.USAGE_ERROR + + return workspace_test_command( + ctx, + workspace_root, + manifest, + targets, + fail_fast=bool(getattr(options, "fail_fast", False)), + output_format=output_format, + ) + + +def workspace_test_targets( + workspace_root: Path, + workspace_manifest: WorkspaceManifest, +) -> tuple[WorkspaceTestTarget, ...]: + return tuple( + workspace_test_manifest_target(workspace_root, repo) + for repo in workspace_manifest.repos + ) + + +def workspace_test_manifest_target( + workspace_root: Path, + repo: WorkspaceManifestRepo, +) -> WorkspaceTestTarget: + try: + root = workspace_context.resolve_workspace_repo_root(workspace_root, repo.name) + except workspace_context.WorkspacePathOutsideRootError as exc: + return WorkspaceTestTarget( + name=repo.name, + root=workspace_root / repo.name, + manifest_path=None, + project_name=None, + action="skip", + reason=str(exc), + required=repo.required, + fatal=True, + ) + + manifest_path = root / "base_manifest.yaml" + if not root.is_dir(): + return WorkspaceTestTarget( + name=repo.name, + root=root, + manifest_path=None, + project_name=None, + action="skip", + reason=f"repository is missing at '{root}'", + required=repo.required, + fatal=repo.required, + ) + if not manifest_path.is_file(): + return WorkspaceTestTarget( + name=repo.name, + root=root, + manifest_path=None, + project_name=None, + action="skip", + reason="repository does not contain base_manifest.yaml", + required=repo.required, + ) + + try: + manifest = read_manifest(manifest_path) + except ManifestError as exc: + return WorkspaceTestTarget( + name=repo.name, + root=root, + manifest_path=manifest_path.resolve(), + project_name=None, + action="skip", + reason=f"base_manifest.yaml is invalid: {exc}", + required=repo.required, + fatal=repo.required, + ) + + if manifest.test is None: + return WorkspaceTestTarget( + name=repo.name, + root=root, + manifest_path=manifest_path.resolve(), + project_name=manifest.project_name, + action="skip", + reason="project does not declare a test command", + required=repo.required, + ) + + return WorkspaceTestTarget( + name=repo.name, + root=root, + manifest_path=manifest_path.resolve(), + project_name=manifest.project_name, + action="test", + required=repo.required, + ) + + +def select_workspace_test_targets( + targets: tuple[WorkspaceTestTarget, ...], + project_selector: str | None, +) -> tuple[WorkspaceTestTarget, ...]: + if project_selector is None: + return targets + + requested_names: list[str] = [] + for raw_name in project_selector.split(","): + name = raw_name.strip() + if not name: + raise WorkspaceTestSelectionError("--projects must not contain an empty project name.") + if name in requested_names: + raise WorkspaceTestSelectionError(f"--projects contains duplicate project '{name}'.") + requested_names.append(name) + + available_names = { + name + for target in targets + for name in (target.name, target.project_name) + if name is not None + } + unknown_names = [name for name in requested_names if name not in available_names] + if unknown_names: + available = ", ".join( + target.project_name or target.name + for target in targets + ) + unknown = ", ".join(unknown_names) + raise WorkspaceTestSelectionError( + f"--projects contains unknown project name(s): {unknown}. Available projects: {available}." + ) + + requested = set(requested_names) + return tuple( + target + for target in targets + if target.name in requested or target.project_name in requested + ) + + +# pylint: disable=too-many-arguments +def workspace_test_command( + ctx: base_cli.Context, + workspace_root: Path, + workspace_manifest: WorkspaceManifest, + targets: tuple[WorkspaceTestTarget, ...], + *, + fail_fast: bool, + output_format: str, +) -> int: + if ctx.application_home is None: + ctx.log.error("BASE_HOME is required to execute workspace tests.") + return base_cli.ExitCode.FAILURE + + basectl = ctx.application_home / "bin" / "basectl" + if not basectl.is_file() or not os.access(basectl, os.X_OK): + ctx.log.error("Base CLI '%s' is missing or is not executable.", basectl) + return base_cli.ExitCode.FAILURE + + if output_format == "text": + print_workspace_test_header(workspace_root, workspace_manifest, targets) + + counts = WorkspaceTestCounts() + results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]] = [] + stopped = False + for target in targets: + if stopped: + result = WorkspaceTestResult("skipped", "not run because --fail-fast stopped after a failure") + elif target.action == "skip": + status: WorkspaceTestStatus = "failed" if target.fatal else "skipped" + result = WorkspaceTestResult(status, target.reason) + else: + result = execute_workspace_test_target(ctx, basectl, workspace_root, target) + + results.append((target, result)) + counts = update_workspace_test_counts(counts, result) + if output_format == "text": + print_workspace_test_result(target, result) + if result.status == "failed" and fail_fast: + stopped = True + + if output_format == "json": + render_workspace_test_json( + workspace_root, + workspace_manifest, + targets, + results, + counts, + fail_fast=fail_fast, + ) + else: + print_workspace_test_summary(counts) + return base_cli.ExitCode.FAILURE if counts.failed else base_cli.ExitCode.SUCCESS + + +def execute_workspace_test_target( + ctx: base_cli.Context, + basectl: Path, + workspace_root: Path, + target: WorkspaceTestTarget, +) -> WorkspaceTestResult: + if target.project_name is None: + return WorkspaceTestResult("failed", "test target is missing project routing metadata") + + command = [ + str(basectl), + "test", + "--workspace", + str(workspace_root), + "--project", + target.project_name, + ] + env = os.environ.copy() + env["BASE_HOME"] = str(ctx.application_home) + for variable in ("BASE_PROJECT", "BASE_PROJECT_ROOT", "BASE_PROJECT_MANIFEST", "BASE_PROJECT_VENV_DIR"): + env.pop(variable, None) + + try: + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + cwd=target.root, + env=env, + timeout=WORKSPACE_TEST_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + return WorkspaceTestResult( + "failed", + f"timed out after {WORKSPACE_TEST_TIMEOUT_SECONDS} seconds", + ) + except OSError as exc: + return WorkspaceTestResult("failed", f"could not run test command: {exc}") + + if result.returncode == 0: + return WorkspaceTestResult("passed", stdout=result.stdout or "", stderr=result.stderr or "") + return WorkspaceTestResult( + "failed", + f"test command exited with status {result.returncode}", + exit_code=result.returncode, + stdout=result.stdout or "", + stderr=result.stderr or "", + ) + + +def update_workspace_test_counts( + counts: WorkspaceTestCounts, + result: WorkspaceTestResult, +) -> WorkspaceTestCounts: + if result.status == "passed": + return WorkspaceTestCounts(counts.passed + 1, counts.failed, counts.skipped) + if result.status == "failed": + return WorkspaceTestCounts(counts.passed, counts.failed + 1, counts.skipped) + return WorkspaceTestCounts(counts.passed, counts.failed, counts.skipped + 1) + + +def print_workspace_test_header( + workspace_root: Path, + workspace_manifest: WorkspaceManifest, + targets: tuple[WorkspaceTestTarget, ...], +) -> None: + print(f"Workspace test: {workspace_root} ({len(targets)} projects)") + print(f"Workspace manifest: {workspace_manifest.path} ({workspace_manifest.name})") + + +def print_workspace_test_result(target: WorkspaceTestTarget, result: WorkspaceTestResult) -> None: + project = target.project_name or target.name + if result.status == "passed": + print(f"PASS project '{project}' at '{target.root}'.") + elif result.status == "failed": + print(f"FAIL project '{project}' at '{target.root}': {result.detail}.") + else: + print(f"SKIP project '{project}' at '{target.root}': {result.detail}.") + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + + +def print_workspace_test_summary(counts: WorkspaceTestCounts) -> None: + print( + "Workspace test completed: " + f"passed={counts.passed} failed={counts.failed} skipped={counts.skipped}." + ) + + +# pylint: disable=too-many-arguments +def render_workspace_test_json( + workspace_root: Path, + workspace_manifest: WorkspaceManifest, + targets: tuple[WorkspaceTestTarget, ...], + results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]], + counts: WorkspaceTestCounts, + *, + fail_fast: bool, +) -> None: + payload: dict[str, Any] = { + "schema_version": 1, + "workspace": str(workspace_root), + "workspace_manifest": { + "path": str(workspace_manifest.path), + "name": workspace_manifest.name, + "schema_version": workspace_manifest.schema_version, + }, + "fail_fast": fail_fast, + "selected_projects": [target.project_name or target.name for target in targets], + "project_count": len(targets), + "projects": [workspace_test_project_to_json(target, result) for target, result in results], + "counts": { + "passed": counts.passed, + "failed": counts.failed, + "skipped": counts.skipped, + }, + } + base_cli.render_document(payload, requested_format="json") + + +def workspace_test_project_to_json( + target: WorkspaceTestTarget, + result: WorkspaceTestResult, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "project": target.project_name, + "repository": target.name, + "path": str(target.root), + "required": target.required, + "status": result.status, + } + if result.detail is not None: + payload["detail"] = result.detail + if result.exit_code is not None: + payload["exit_code"] = result.exit_code + if result.stdout: + payload["stdout"] = result.stdout + if result.stderr: + payload["stderr"] = result.stderr + return payload diff --git a/docs/command-reference.md b/docs/command-reference.md index 3cb287e7..bc2275ec 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -214,6 +214,7 @@ manifest trust. | `basectl workspace init ` | Initialize a workspace from a workspace configuration repository, update local workspace config, and optionally materialize member repositories. | `--owner `, `--path `, `--workspace `, `--manifest `, `--include-optional`, `--dry-run` | | `basectl workspace configure` | Preview the existing `repo configure` repair path by default across discovered Base-managed workspace repositories or an explicit workspace manifest. Use `--apply` to authorize changes; interactive runs prompt unless `--yes` is supplied. Skips missing, non-Base-managed, or non-GitHub repos and continues after per-repo failures. | `--workspace `, `--manifest `, `--dry-run`, `--apply`, `--yes` | | `basectl workspace setup` | Set up eligible repositories from a workspace manifest in manifest order by delegating to each repository's local `basectl setup` command. Skips ineligible repositories, continues after per-repo failures, and reports setup/skipped/failed counts. | `--workspace `, `--manifest `, `--dry-run`, `--yes` | +| `basectl workspace test` | Run declared project test commands serially across workspace-manifest repositories. Projects without a declared test command are skipped; failures are reported per project and `--fail-fast` stops after the first failure. | `--workspace `, `--manifest `, `--projects `, `--fail-fast`, `--format ` | ## Repository And GitHub Workflow @@ -287,7 +288,7 @@ Current implemented commands include: - `basectl update-profile` - `basectl update` - `basectl projects list` -- `basectl workspace ` +- `basectl workspace ` - `basectl trust status [project]` - `basectl trust ` - `basectl repo init ` diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md index b896cd17..c8ae59be 100644 --- a/docs/stability-tiers.md +++ b/docs/stability-tiers.md @@ -40,6 +40,10 @@ compatibility surfaces, such as Dev Containers and Nix/devenv planning reports, are stable as commands but may keep individual classification details experimental while those ecosystems mature. +`basectl workspace test` is Experimental. Its serial execution, selection, +failure, skip, and JSON-report details may evolve while workspace-wide test +orchestration matures. + ## JSON And Schema-Versioned Output Base JSON intended for automation must carry a `schema_version` when the payload diff --git a/lib/shell/completions/basectl_completion.sh b/lib/shell/completions/basectl_completion.sh index bea7302c..a9ccd522 100644 --- a/lib/shell/completions/basectl_completion.sh +++ b/lib/shell/completions/basectl_completion.sh @@ -652,7 +652,7 @@ _base_basectl_completion() { ;; workspace) if ((COMP_CWORD == 2)); then - _base_basectl_completion_compgen "status check doctor onboarding agent-brief clone pull update init configure setup" "$cur" + _base_basectl_completion_compgen "status check doctor onboarding agent-brief clone pull update init configure setup test" "$cur" else case "${COMP_WORDS[2]:-}" in status|check|doctor) @@ -679,6 +679,9 @@ _base_basectl_completion() { setup) _base_basectl_completion_compgen "--workspace --manifest --dry-run --yes -v -h --help" "$cur" ;; + test) + _base_basectl_completion_compgen "--workspace --manifest --projects --fail-fast --format -v -h --help" "$cur" + ;; esac fi ;; diff --git a/lib/shell/completions/basectl_completion.zsh b/lib/shell/completions/basectl_completion.zsh index 7e1556e8..5055f257 100644 --- a/lib/shell/completions/basectl_completion.zsh +++ b/lib/shell/completions/basectl_completion.zsh @@ -541,14 +541,14 @@ _base_basectl_completion() { workspace) case "${words[3]:-}" in status|check|doctor|onboarding|agent-brief) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--workspace[Workspace directory to scan]:path:_files' \ '--manifest[Local workspace manifest]:path:_files' \ '--format[Output format]:format:(text csv tsv yaml json)' \ '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; clone) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--workspace[Workspace directory to scan]:path:_files' \ '--manifest[Local workspace manifest]:path:_files' \ '--include-optional[Include optional manifest repositories when cloning]' \ @@ -556,14 +556,14 @@ _base_basectl_completion() { '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; pull) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--source[Canonical workspace manifest source]:url-or-path:' \ '--manifest[Local workspace manifest]:path:_files' \ '--dry-run[Show planned workspace pull work without writing]' \ '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; update) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--workspace[Workspace directory to update]:path:_files' \ '--manifest[Local workspace manifest]:path:_files' \ '--repos[Comma-separated manifest repository names]:repository-names:' \ @@ -572,7 +572,7 @@ _base_basectl_completion() { '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; init) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '3:workspace source:' \ '--owner[GitHub owner for short workspace repository names]:owner:' \ '--path[Workspace configuration repository checkout path]:path:_files' \ @@ -583,7 +583,7 @@ _base_basectl_completion() { '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; configure) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--workspace[Workspace directory to configure]:path:_files' \ '--manifest[Local workspace manifest]:path:_files' \ '--dry-run[Show planned workspace configuration without applying repo changes]' \ @@ -592,15 +592,24 @@ _base_basectl_completion() { '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; setup) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' \ + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ '--workspace[Workspace directory to prepare]:path:_files' \ '--manifest[Local workspace manifest]:path:_files' \ '--dry-run[Show the ordered workspace setup plan without writing]' \ '--yes[Apply setup changes that require confirmation]' \ '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' ;; + test) + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' \ + '--workspace[Workspace directory to test]:path:_files' \ + '--manifest[Local workspace manifest]:path:_files' \ + '--projects[Comma-separated workspace project names]:project-names:' \ + '--fail-fast[Stop after the first project failure]' \ + '--format[Output format]:format:(text json)' \ + '-v[Enable DEBUG logging]' '(-h --help)'{-h,--help}'[Show help text]' + ;; *) - _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup)' + _arguments '2:workspace command:(status check doctor onboarding agent-brief clone pull update init configure setup test)' ;; esac ;;