From 859bbb2312752ed533dd54221a257613470af64d Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:22:34 +0530 Subject: [PATCH 01/11] test: demonstrate isolated trust and consent boundaries --- .ai-context/overview.md | 6 +++ .github/workflows/scenarios.yml | 61 ++++++++++++++++++++++ CHANGELOG.md | 3 ++ README.md | 5 ++ docs/base-capability-matrix.md | 2 +- docs/trust-scenarios.md | 65 +++++++++++++++++++++++ tests/scenario_harness_test.py | 41 +++++++++++++++ tests/scenarios/fixture.py | 89 ++++++++++++++++++++++++++++++++ tests/scenarios/ide_guard.py | 36 +++++++++++++ tests/scenarios/trust.py | 91 +++++++++++++++++++++++++++++++++ tests/validate.sh | 1 + 11 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/scenarios.yml create mode 100644 docs/trust-scenarios.md create mode 100644 tests/scenario_harness_test.py create mode 100644 tests/scenarios/fixture.py create mode 100644 tests/scenarios/ide_guard.py create mode 100644 tests/scenarios/trust.py diff --git a/.ai-context/overview.md b/.ai-context/overview.md index 13b23a8..8984c65 100644 --- a/.ai-context/overview.md +++ b/.ai-context/overview.md @@ -1,5 +1,11 @@ # base-demo Overview +Disposable trust scenarios live in `tests/scenarios/trust.py` and +`docs/trust-scenarios.md`. The stable v1.9 lane does not claim full historical +revocation or static runtime inspection; those assertions use a separately +pinned v1.10 candidate. Fixture homes and IDE delegate spies prevent learner +state mutation. Scenario success does not certify host readiness. + The release BOM gate reuses Base's governed contract and requires both platform jobs plus source-provider evidence at the exact release commit. Static pins and historical release:// records are not passing compatibility proof; see diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml new file mode 100644 index 0000000..d0deb80 --- /dev/null +++ b/.github/workflows/scenarios.yml @@ -0,0 +1,61 @@ +name: Isolated Base scenarios + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scenarios-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + trust: + name: Trust scenarios (${{ matrix.lane }}) + runs-on: macos-14 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - lane: stable-v1.9 + base: ac8d294421e1bfc14afa8c6a2a12f1affb5268ee + options: "" + - lane: advisory-v1.10-candidate + base: 5f316aeddc3680b92bd209fcfe652eac020d02d0 + options: --candidate + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base + ref: ${{ matrix.base }} + path: .dependencies/base + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base-cli + ref: 8a93d22156ba75a99965f7c355f867acba630069 + path: .dependencies/base-cli + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base-bash-libs + ref: 36fec50c446dcea8c521a1ba3e7fee2394f169c0 + path: .dependencies/base-bash-libs + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.13' + - name: Install fixture interpreter dependencies + run: python -m pip install ./.dependencies/base-cli PyYAML tomli + - name: Verify isolated trust and consent contracts + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/trust.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS diff --git a/CHANGELOG.md b/CHANGELOG.md index d5fcd71..9d6471c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ promotes this section into a dated version heading before tagging. - Aligned current source with the supported Base 1.9.0, base-cli 0.4.3 and base-bash-libs 2.1.0 input contract, with exact macOS/Ubuntu CI revisions and live final-candidate evidence binding outside the tracked tree. +- Added isolated stable/candidate trust scenarios for denial, invalidation, + revocation, runtime verification and independent IDE consent, with explicit + Base v1.9 historical-revocation limitations. - Made release BOM publication fail closed on incomplete participants, stale pins, inconsistent platforms, and missing exact-commit hosted evidence. diff --git a/README.md b/README.md index d4b71a1..6512581 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ blocked Base capabilities is maintained in the [Base capability and evidence matrix](docs/base-capability-matrix.md). Use it during Base release reviews and when deciding whether a new base-demo scenario has executable evidence. +The [isolated trust and consent scenarios](docs/trust-scenarios.md) demonstrate +denial, approval, invalidation and revocation without changing your trust store +or IDE settings. They explicitly separate Base v1.9 behavior from the stronger +implemented v1.10 candidate contracts. + The external tooling direction is tracked in [Tooling Test Bed](docs/tooling-testbed.md). That matrix separates active baseline tools from optional wrappers, reference-only examples, and future Base diff --git a/docs/base-capability-matrix.md b/docs/base-capability-matrix.md index 85a7409..688de4c 100644 --- a/docs/base-capability-matrix.md +++ b/docs/base-capability-matrix.md @@ -24,7 +24,7 @@ useful for development, but does not by itself establish released compatibility. | Linux and WSL2 read-only support boundary | `1.9.0` | The supported commands and explicit native-Windows boundary are in [`README.md`](../README.md) and [`docs/contracts.md`](contracts.md); CI's Ubuntu path is in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Ubuntu/Debian and WSL2 use setup, dev-profile, check, and doctor/read-only paths; native Windows is excluded | Demonstrated | Base + base-demo | Keep platform claims tied to a hosted or repository-local check. | | Base `1.9.0` released-compatibility pin and full Go/live-HTTP evidence | `1.9.0` | Structured pins in [supported inputs](../.release/supported-dependencies.json), verified by `bin/base-demo-dependencies`; full-language and live-HTTP gates run in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Exact macOS 14 full demo and Ubuntu 24.04 setup/read-only scope; no Linux full-demo claim | Demonstrated | Base + base-demo | Bind final-candidate runs during #303; later dependency changes require fresh proof. | | Workspace scenarios planned for the Base `1.10.0` train | `1.10.0` (planned) | The workspace manifest and current boundary are [`workspace.yaml.example`](../workspace.yaml.example) and [`README.md`](../README.md); the new scenario evidence is tracked by [base-demo#297](https://github.com/basefoundry/base-demo/issues/297) | Planned release work; no `1.10.0` claim is made by the current `main` branch | Blocked upstream | Base + base-demo | Implement #297 after the Base `1.10.0` workspace contract is fixed. | -| Trust and consent scenarios planned for the Base `1.10.0` train | `1.10.0` (planned) | Current trust documentation and CI ordering are in [`README.md`](../README.md), [`docs/contracts.md`](contracts.md), and [`.github/workflows/tests.yml`](../.github/workflows/tests.yml); the additional scenario is tracked by [base-demo#298](https://github.com/basefoundry/base-demo/issues/298) | Planned release work; current evidence remains the `1.9.0` boundary | Blocked upstream | Base + base-demo | Implement #298 after the Base trust/consent contract is stable. | +| Trust lifecycle and separate runtime/IDE consent | `1.9.0` baseline; exact `1.10.0` candidate | `tests/scenarios/trust.py` and [trust scenario guide](trust-scenarios.md), executed by [isolated scenario CI](../.github/workflows/scenarios.yml) | macOS fixture lane; full historical revocation and runtime inspection require the pinned candidate | Demonstrated | Base + base-demo | Refresh exact candidate evidence before release; do not attribute candidate-only guarantees to v1.9.0. | | Optional `test.requirements` and uninstall guidance | `1.9.0` | The manifest test entry and contributor setup guidance are [`base_manifest.yaml`](../base_manifest.yaml) and [`README.md`](../README.md) | Optional metadata is not required for the baseline demo contract | Intentionally omitted | base-demo | Revisit when Base publishes a stable user-facing contract and a concrete scenario. | | Native Windows support | `1.11.0` (planned) | The current non-goal is recorded in [`README.md`](../README.md); the staged Base work is tracked by [base-demo#302](https://github.com/basefoundry/base-demo/issues/302) and Base [#2215](https://github.com/basefoundry/base/issues/2215) | Native Windows is not shipped; Git Bash and WSL2 do not count as native Windows evidence | Blocked upstream | Base + base-demo | Wait for the PowerShell-first Base contract and hosted Windows evidence. | | First-class Base Docker-service contract | Future | The existing Compose fixture is documented in [`docs/tooling-testbed.md`](tooling-testbed.md) and [`infra/compose.yaml`](../infra/compose.yaml); adoption remains tracked by [base-demo#163](https://github.com/basefoundry/base-demo/issues/163) and Base [#124](https://github.com/basefoundry/base/issues/124) | Compose is a repository fixture; it does not establish a Base Docker-service contract | Blocked upstream | Base + base-demo | Do not make the future Base command a required demo dependency before Base publishes it. | diff --git a/docs/trust-scenarios.md b/docs/trust-scenarios.md new file mode 100644 index 0000000..e19ebe8 --- /dev/null +++ b/docs/trust-scenarios.md @@ -0,0 +1,65 @@ +# Trust and consent, in disposable fixtures + +Run the scenario against clean, exact-commit provider checkouts and an existing +Python interpreter with Base's dependencies. It creates its own home, workspace, +cache and trust store, and cleans them on success or failure. It never edits a +learner's real approval store or IDE settings. The project is deliberately a +tiny shell fixture; uv remains the dependency owner for the normal demo. + +```bash +python3 tests/scenarios/trust.py \ + --base /path/to/base-v1.9.0 \ + --base-commit ac8d294421e1bfc14afa8c6a2a12f1affb5268ee \ + --base-cli /path/to/base-cli-v0.4.3 \ + --bash-libs /path/to/base-bash-libs-v2.1.0 \ + --python /path/to/base-compatible-venv/bin/python +``` + +The stable lane asserts inspection before trust, denied execution (exit 1), +explicit approval, successful execution (exit 0), manifest-change invalidation, +and revocation of a single reviewed approval. Expected failures are assertions, +not errors the learner needs to repair. Each completed boundary prints `PASS`. +Unchanged external scripts are **not** bound by command approval; this is not a +sandbox or a guarantee that every referenced executable is safe. + +## Candidate-only guarantees + +**Base v1.9.0 does not guarantee complete historical-approval revocation.** +Do not infer that the stable example proves cleanup of every previously reviewed +manifest version. Full historical revocation and static-by-default runtime +inspection are demonstrated only against the implemented v1.10 candidate. +The stable lane prints this boundary and does not invoke candidate-only flags. + +The separate advisory lane uses exact Base commit +`5f316aeddc3680b92bd209fcfe652eac020d02d0`. Run the same command with that +checkout/commit and add `--candidate`. A published v1.10 release is not required, +but a moving branch is not an acceptable substitute for the recorded revision. +Both lanes run in [isolated scenario CI](../.github/workflows/scenarios.yml). +Candidate success does not replace stable compatibility evidence. + +The candidate additionally approves multiple manifest versions, revokes them, +and proves that reverting to an older manifest does not restore execution. +It uses a harmless marker-producing interpreter fixture to prove static +inspection does not execute runtime code—even after command approval—and that +`--verify-project-runtime` explicitly permits the probe. + +| Consent | What it permits | What it does not permit | +| --- | --- | --- | +| `trust allow` | Reviewed manifest command execution | Runtime inspection or IDE mutation | +| `--verify-project-runtime` | Project/runtime probes for that check or doctor invocation | Saved command approval or IDE mutation | +| `--allow-project-ide-mutations` | Applying a reviewed project-originated IDE plan | Command approval or runtime verification | +| `--yes` | Ordinary confirmation handling | Any of the independent approvals above | + +IDE behavior is tested through public dry-run previews and an isolated consent +guard test whose mutation delegates are spies. No application install, extension +install or IDE user-settings write is performed. The guard rejects mutation +without its own consent before reaching any delegate. + +Host prerequisite findings remain real and may make `check` return 1 on a +partially configured machine. The scenario asserts the project findings and the +JSON aggregate exit contract separately; a passing scenario is **not** a claim +that the host is ready. Local temporary paths are redacted from assertion +diagnostics. `tests/scenario_harness_test.py` verifies cleanup and redaction. + +For the authoritative boundaries, see Base's +[command-trust policy](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/manifest-command-trust.md). diff --git a/tests/scenario_harness_test.py b/tests/scenario_harness_test.py new file mode 100644 index 0000000..e286c83 --- /dev/null +++ b/tests/scenario_harness_test.py @@ -0,0 +1,41 @@ +"""Verify isolation, redaction and failure cleanup without external providers.""" +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest + +sys.path.insert(0, str(Path(__file__).parent / "scenarios")) +from fixture import Fixture + + +class HarnessTests(unittest.TestCase): + def args(self): + return SimpleNamespace(python=Path(sys.executable), base=Path("/unused"), + base_cli=Path("/unused-cli"), bash_libs=Path("/unused-libs")) + + def test_failure_cleans_disposable_home(self): + root = None + with self.assertRaisesRegex(RuntimeError, "expected failure"): + with Fixture(self.args()) as fixture: + root = fixture.root + self.assertEqual(Path(fixture.env["HOME"]), fixture.home) + self.assertNotIn("GH_TOKEN", fixture.env) + self.assertNotIn("BASH_ENV", fixture.env) + raise RuntimeError("expected failure") + self.assertFalse(root.exists()) + + def test_cli_failures_redact_fixture_root(self): + with Fixture(self.args()) as fixture: + fixture.args.base = fixture.root / "provider" + launcher = fixture.args.base / "bin/basectl" + launcher.parent.mkdir(parents=True) + launcher.write_text('#!/bin/sh\nprintf "%s\\n" "$HOME"\nexit 1\n') + launcher.chmod(0o755) + with self.assertRaises(AssertionError) as failure: + fixture.run("check", "--manifest", fixture.root / "project.yaml") + self.assertNotIn(str(fixture.root), str(failure.exception)) + self.assertIn("", str(failure.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scenarios/fixture.py b/tests/scenarios/fixture.py new file mode 100644 index 0000000..8b5084c --- /dev/null +++ b/tests/scenarios/fixture.py @@ -0,0 +1,89 @@ +"""Disposable public-CLI scenario support; never inherit learner Base state.""" +from __future__ import annotations + +import argparse +import json +import os +import shlex +from pathlib import Path +import subprocess +import tempfile + + +def arguments(description): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--base-commit", required=True) + parser.add_argument("--base-cli", type=Path, required=True) + parser.add_argument("--bash-libs", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True, help="existing Base-compatible Python interpreter") + parser.add_argument("--candidate", action="store_true") + args = parser.parse_args() + for name, path, expected in ( + ("base", args.base, args.base_commit), + ("base-cli", args.base_cli, "8a93d22156ba75a99965f7c355f867acba630069"), + ("base-bash-libs", args.bash_libs, "36fec50c446dcea8c521a1ba3e7fee2394f169c0"), + ): + actual = subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + dirty = subprocess.check_output(["git", "-C", str(path), "status", "--porcelain"], text=True) + if actual != expected or dirty: + parser.error(f"{name} must be a clean exact-commit checkout") + if not args.candidate and args.base_commit != "ac8d294421e1bfc14afa8c6a2a12f1affb5268ee": + parser.error("stable lane requires supported Base v1.9.0; pass --candidate for separately recorded source") + return args + + +class Fixture: + def __init__(self, args): + self.args = args + self.temporary = tempfile.TemporaryDirectory(prefix="base-demo-scenario-") + self.root = Path(self.temporary.name).resolve() + self.home = self.root / "home" + self.workspace = self.root / "workspace" + self.home.mkdir() + self.workspace.mkdir() + # A proxy selects the existing interpreter without writing to its venv. + venv = self.home / ".base.d/base/.venv" + (venv / "bin").mkdir(parents=True) + proxy = venv / "bin/python" + proxy.write_text("#!/bin/sh\nexec " + shlex.quote(str(args.python.absolute())) + ' "$@"\n') + proxy.chmod(0o755) + (venv / "pyvenv.cfg").write_text("scenario = isolated\n") + self.env = { + "HOME": str(self.home), "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "BASE_HOME": str(args.base.resolve()), "BASE_SETUP_VENV_DIR": str(venv), + "BASE_CLI_SOURCE_DIR": str(args.base_cli.resolve() / "lib/python"), + "BASE_BASH_LIBS_DIR": str(args.bash_libs.resolve() / "lib/bash"), + "BASE_CACHE_DIR": str(self.root / "cache"), "BASE_SETUP_NOTIFY": "false", + "XDG_CONFIG_HOME": str(self.home / ".config"), "XDG_CACHE_HOME": str(self.root / "xdg-cache"), + "TMPDIR": str(self.root), "NO_COLOR": "1", "TERM": "dumb", + } + + def __enter__(self): + return self + + def __exit__(self, *_): + self.temporary.cleanup() + + def project(self, name="demo", extra=""): + root = self.workspace / name + root.mkdir(exist_ok=True) + command = 'printf "executed\\n" > "$BASE_PROJECT_ROOT/executed"' + (root / "base_manifest.yaml").write_text( + f"project:\n name: {name}\nartifacts: []\ncommands:\n hello: {json.dumps(command)}\n" + extra + ) + return root + + def run(self, *args, expected=0, cwd=None): + result = subprocess.run([str(self.args.base.resolve() / "bin/basectl"), *map(str, args)], + env=self.env, cwd=cwd or self.workspace, capture_output=True, text=True, timeout=90) + allowed = expected if isinstance(expected, tuple) else (expected,) + if result.returncode not in allowed: + # Only redacted fixture-local diagnostics are emitted on failure. + output = (result.stdout + result.stderr).replace(str(self.root), "") + command = str(args).replace(str(self.root), "") + raise AssertionError(f"{command}: expected exit {expected}, got {result.returncode}\n{output}") + return result + + def no_ide_settings(self): + assert not list(self.home.rglob("settings.json")), "fixture unexpectedly wrote IDE settings" diff --git a/tests/scenarios/ide_guard.py b/tests/scenarios/ide_guard.py new file mode 100644 index 0000000..88c797f --- /dev/null +++ b/tests/scenarios/ide_guard.py @@ -0,0 +1,36 @@ +"""Exercise the IDE consent guard with all mutation delegates replaced by spies.""" +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from base_cli_adapters.config import UserConfig, UserIdeConfig +from base_setup.errors import ArtifactError +from base_setup.manifest import BaseManifest, IdeConfig +from base_setup import setup_reconcile + + +def main(): + context = SimpleNamespace(log=Mock(), yes=True, + user_config=UserConfig(raw={}, ide=UserIdeConfig(enabled=None, preferences={}))) + defaults = BaseManifest(path=Path("defaults.yaml"), project_name="defaults", brewfile=None, artifacts=()) + project = BaseManifest(path=Path("project.yaml"), project_name="fixture", brewfile=None, artifacts=(), + ide={"vscode": IdeConfig(install=False, extensions=(), settings={"editor.formatOnSave": True})}) + names = ("reconcile_brewfile", "reconcile_mise", "reconcile_ide_installs", "reconcile_ide_extensions", + "reconcile_ide_settings", "reconcile_uv_project", "reconcile_artifacts") + delegates = {name: Mock() for name in names} + with patch.multiple(setup_reconcile, **delegates): + try: + setup_reconcile.reconcile_manifest(context, defaults, project, dry_run=False) + except ArtifactError as error: + assert "--allow-project-ide-mutations" in str(error) + assert "--yes" in str(error) + else: + raise AssertionError("IDE mutation was not denied") + assert not any(spy.called for spy in delegates.values()) + setup_reconcile.reconcile_manifest(context, defaults, project, dry_run=True) + assert delegates["reconcile_ide_settings"].call_args.kwargs["dry_run"] is True + print("PASS: IDE mutation guard denies without its own consent; preview delegates are dry-run only") + + +if __name__ == "__main__": + main() diff --git a/tests/scenarios/trust.py b/tests/scenarios/trust.py new file mode 100644 index 0000000..0f44271 --- /dev/null +++ b/tests/scenarios/trust.py @@ -0,0 +1,91 @@ +"""Assert command approval, invalidation, revoke and independent consent.""" +from __future__ import annotations + +import json +from pathlib import Path +import shlex +import subprocess + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + with Fixture(args) as fixture: + project = fixture.project() + manifest = project / "base_manifest.yaml" + original = manifest.read_text() + workspace = ("--workspace", fixture.workspace) + fixture.run("run", "demo", "--list", *workspace) + denied = fixture.run("run", "demo", "hello", *workspace, expected=1) + assert "trust" in (denied.stdout + denied.stderr).lower() + assert not (project / "executed").exists() + fixture.run("trust", "allow", "demo", *workspace) + fixture.run("run", "demo", "hello", *workspace) + assert (project / "executed").read_text() == "executed\n" + (project / "executed").unlink() + manifest.write_text(original + "\n# changed after review\n") + fixture.run("run", "demo", "hello", *workspace, expected=1) + assert not (project / "executed").exists() + if args.candidate: + # Historical multi-approval cleanup is the implemented v1.10 + # contract, not a guarantee of the older stable release. + fixture.run("trust", "allow", "demo", *workspace) + fixture.run("trust", "revoke", "demo", *workspace) + fixture.run("run", "demo", "hello", *workspace, expected=1) + # Reverting the manifest must not resurrect the older approval. + manifest.write_text(original) + fixture.run("run", "demo", "hello", *workspace, expected=1) + scope = "historical approvals" if args.candidate else "single reviewed approval" + print(f"PASS: inspection -> denial -> approval -> execution -> invalidation -> revoke ({scope})") + + # Preview only: do not authorize or apply any IDE mutation. + manifest.write_text(original + "ide:\n vscode:\n settings:\n editor.formatOnSave: true\n") + preview = fixture.run("setup", "demo", "--manifest", manifest, "--dry-run", "--yes") + assert "IDE" in (preview.stdout + preview.stderr) or "settings" in (preview.stdout + preview.stderr) + fixture.run("run", "demo", "hello", *workspace, expected=1) + fixture.no_ide_settings() + print("PASS: --yes plus setup preview does not approve manifest commands or write IDE settings") + env = dict(fixture.env) + env["PYTHONPATH"] = str(args.base_cli.resolve() / "lib/python") + ":" + str(args.base.resolve() / "cli/python") + guard = subprocess.run([str(args.python.absolute()), str(Path(__file__).with_name("ide_guard.py"))], + env=env, cwd=fixture.root, capture_output=True, text=True, timeout=30) + assert guard.returncode == 0, (guard.stdout + guard.stderr).replace(str(fixture.root), "") + print(guard.stdout.strip()) + fixture.no_ide_settings() + + if args.candidate: + help_result = fixture.run("check", "--help") + assert "--verify-project-runtime" in help_result.stdout, "candidate lacks runtime verification contract" + manifest.write_text(original + "python: {}\n") + runtime = project / ".venv/bin/python" + runtime.parent.mkdir(parents=True) + marker = project / "runtime-probed" + runtime.write_text("#!/bin/sh\n" + f"printf probe >> {shlex.quote(str(marker))}\n" + + f"exec {shlex.quote(str(args.python.absolute()))} \"$@\"\n") + runtime.chmod(0o755) + (project / ".venv/pyvenv.cfg").write_text("scenario = project-runtime\n") + fixture.run("trust", "allow", "demo", *workspace) + static = fixture.run("check", "--ci", "--manifest", manifest, "--format", "json", expected=(0, 1)) + assert "unverified" in static.stdout + payload = json.loads(static.stdout) + assert payload["project_checks"]["status"] == "warn" + # Host prerequisites remain real diagnostics, separate from this + # disposable project contract. Assert their aggregate exit exactly. + assert static.returncode == (1 if payload["status"] == "error" else 0) + assert not marker.exists(), "saved command approval unexpectedly permitted a runtime probe" + verified = fixture.run("check", "--ci", "--manifest", manifest, "--verify-project-runtime", "--format", "json", + expected=(0, 1)) + payload = json.loads(verified.stdout) + assert payload["project_checks"]["status"] != "error", payload["project_checks"] + assert verified.returncode == (1 if payload["status"] == "error" else 0) + assert marker.exists(), "explicit verification did not probe the runtime" + fixture.no_ide_settings() + print("PASS: saved command approval does not grant runtime inspection; explicit verification probes only the fixture") + else: + print("BOUNDARY: full historical revocation and --verify-project-runtime require the implemented v1.10 candidate") + print(f"PASS: isolated trust scenario at Base {args.base_commit}; fixture cleanup on success and failure") + + +if __name__ == "__main__": + main() diff --git a/tests/validate.sh b/tests/validate.sh index 50d5832..4d16621 100755 --- a/tests/validate.sh +++ b/tests/validate.sh @@ -168,6 +168,7 @@ fi python3 tests/release_bom_test.py || exit 1 python3 tests/dependency_inputs_test.py || exit 1 +python3 tests/scenario_harness_test.py || exit 1 if ! bash tests/public_install_test.sh; then printf 'The exact public bootstrap input failed its contract tests.\n' >&2 From 8182487c412f038700b3e031cf791c6744fceefc Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:33:53 +0530 Subject: [PATCH 02/11] ci: provision supported Bash for isolated scenarios --- .github/workflows/scenarios.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml index d0deb80..ec10ecf 100644 --- a/.github/workflows/scenarios.yml +++ b/.github/workflows/scenarios.yml @@ -50,6 +50,8 @@ jobs: python-version: '3.13' - name: Install fixture interpreter dependencies run: python -m pip install ./.dependencies/base-cli PyYAML tomli + - name: Install supported Bash for Base + run: brew install bash - name: Verify isolated trust and consent contracts env: BASE_SCENARIO_COMMIT: ${{ matrix.base }} From 4453a05ec4d93523bccd6babd881dc6122a4af7e Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:32:18 +0530 Subject: [PATCH 03/11] test: demonstrate workspace selection and targeted recovery --- .ai-context/overview.md | 4 + .github/workflows/scenarios.yml | 11 ++- CHANGELOG.md | 3 + README.md | 2 + docs/base-capability-matrix.md | 2 +- docs/workspace-scenarios.md | 50 ++++++++++++ tests/scenarios/workspace.py | 130 ++++++++++++++++++++++++++++++++ 7 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 docs/workspace-scenarios.md create mode 100644 tests/scenarios/workspace.py diff --git a/.ai-context/overview.md b/.ai-context/overview.md index 8984c65..0f598cb 100644 --- a/.ai-context/overview.md +++ b/.ai-context/overview.md @@ -125,6 +125,10 @@ change stream, and its three existing validation job IDs remain stable. ## Quick Loop +`tests/scenarios/workspace.py` is the isolated multi-peer consumer fixture. +Stable 1.9 covers reports; exact-candidate 1.10 covers selection, aggregate +failure/skip, aliases, undeclared inventory and checkout-bound next actions. + ```bash basectl setup base-demo basectl activate base-demo diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml index ec10ecf..7091480 100644 --- a/.github/workflows/scenarios.yml +++ b/.github/workflows/scenarios.yml @@ -15,7 +15,7 @@ concurrency: jobs: trust: - name: Trust scenarios (${{ matrix.lane }}) + name: Trust and workspace scenarios (${{ matrix.lane }}) runs-on: macos-14 timeout-minutes: 15 strategy: @@ -61,3 +61,12 @@ jobs: --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ --python "$(command -v python)" $BASE_SCENARIO_OPTIONS + - name: Verify isolated workspace contracts + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/workspace.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6471c..2b58f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ promotes this section into a dated version heading before tagging. - Aligned current source with the supported Base 1.9.0, base-cli 0.4.3 and base-bash-libs 2.1.0 input contract, with exact macOS/Ubuntu CI revisions and live final-candidate evidence binding outside the tracked tree. +- Added disposable workspace selection, failure/skip, fail-fast and targeted + recovery scenarios, with separate stable and exact-candidate boundaries. + - Added isolated stable/candidate trust scenarios for denial, invalidation, revocation, runtime verification and independent IDE consent, with explicit Base v1.9 historical-revocation limitations. diff --git a/README.md b/README.md index 6512581..442647f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ blocked Base capabilities is maintained in the [Base capability and evidence matrix](docs/base-capability-matrix.md). Use it during Base release reviews and when deciding whether a new base-demo scenario has executable evidence. +The [workspace scenarios](docs/workspace-scenarios.md) prove selected execution, +failure aggregation and checkout-bound recovery in disposable peers. The [isolated trust and consent scenarios](docs/trust-scenarios.md) demonstrate denial, approval, invalidation and revocation without changing your trust store or IDE settings. They explicitly separate Base v1.9 behavior from the stronger diff --git a/docs/base-capability-matrix.md b/docs/base-capability-matrix.md index 688de4c..7c9be51 100644 --- a/docs/base-capability-matrix.md +++ b/docs/base-capability-matrix.md @@ -23,7 +23,7 @@ useful for development, but does not by itself establish released compatibility. | Representative build, test, service, environment, and non-interactive demo loop | `1.9.0` | The manifest targets in [`base_manifest.yaml`](../base_manifest.yaml), baseline gate in [`tests/validate.sh`](../tests/validate.sh), and focused suites in [`tests/services_test.bats`](../tests/services_test.bats), [`tests/environments_test.bats`](../tests/environments_test.bats), and [`tests/demo_test.bats`](../tests/demo_test.bats) | Full project loop is macOS; Ubuntu/Debian validates Base setup and project health only | Demonstrated | base-demo | Add executable evidence before calling a new service or command demonstrated. | | Linux and WSL2 read-only support boundary | `1.9.0` | The supported commands and explicit native-Windows boundary are in [`README.md`](../README.md) and [`docs/contracts.md`](contracts.md); CI's Ubuntu path is in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Ubuntu/Debian and WSL2 use setup, dev-profile, check, and doctor/read-only paths; native Windows is excluded | Demonstrated | Base + base-demo | Keep platform claims tied to a hosted or repository-local check. | | Base `1.9.0` released-compatibility pin and full Go/live-HTTP evidence | `1.9.0` | Structured pins in [supported inputs](../.release/supported-dependencies.json), verified by `bin/base-demo-dependencies`; full-language and live-HTTP gates run in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Exact macOS 14 full demo and Ubuntu 24.04 setup/read-only scope; no Linux full-demo claim | Demonstrated | Base + base-demo | Bind final-candidate runs during #303; later dependency changes require fresh proof. | -| Workspace scenarios planned for the Base `1.10.0` train | `1.10.0` (planned) | The workspace manifest and current boundary are [`workspace.yaml.example`](../workspace.yaml.example) and [`README.md`](../README.md); the new scenario evidence is tracked by [base-demo#297](https://github.com/basefoundry/base-demo/issues/297) | Planned release work; no `1.10.0` claim is made by the current `main` branch | Blocked upstream | Base + base-demo | Implement #297 after the Base `1.10.0` workspace contract is fixed. | +| Workspace inventory, selected tests and targeted recovery | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI | macOS fixture lane; selection and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. | | Trust lifecycle and separate runtime/IDE consent | `1.9.0` baseline; exact `1.10.0` candidate | `tests/scenarios/trust.py` and [trust scenario guide](trust-scenarios.md), executed by [isolated scenario CI](../.github/workflows/scenarios.yml) | macOS fixture lane; full historical revocation and runtime inspection require the pinned candidate | Demonstrated | Base + base-demo | Refresh exact candidate evidence before release; do not attribute candidate-only guarantees to v1.9.0. | | Optional `test.requirements` and uninstall guidance | `1.9.0` | The manifest test entry and contributor setup guidance are [`base_manifest.yaml`](../base_manifest.yaml) and [`README.md`](../README.md) | Optional metadata is not required for the baseline demo contract | Intentionally omitted | base-demo | Revisit when Base publishes a stable user-facing contract and a concrete scenario. | | Native Windows support | `1.11.0` (planned) | The current non-goal is recorded in [`README.md`](../README.md); the staged Base work is tracked by [base-demo#302](https://github.com/basefoundry/base-demo/issues/302) and Base [#2215](https://github.com/basefoundry/base/issues/2215) | Native Windows is not shipped; Git Bash and WSL2 do not count as native Windows evidence | Blocked upstream | Base + base-demo | Wait for the PowerShell-first Base contract and hosted Windows evidence. | diff --git a/docs/workspace-scenarios.md b/docs/workspace-scenarios.md new file mode 100644 index 0000000..a98fc00 --- /dev/null +++ b/docs/workspace-scenarios.md @@ -0,0 +1,50 @@ +# Workspace validation and targeted recovery + +This compact scenario creates disposable healthy, failing, untrusted, no-test, +missing-required and undeclared peers. It uses the same isolated HOME, cache, +provider checks and cleanup as the [trust scenario](trust-scenarios.md). No +clone, setup, repository initialization or learner-state mutation is performed. +Generated recovery commands are inspected; only a reviewed fixture-local trust +command is executed. Expected failures are assertions, not repair requests. + +Run with clean exact-provider checkouts and an existing Base-compatible Python: + +```bash +python3 tests/scenarios/workspace.py \ + --base /path/to/base-v1.9.0 \ + --base-commit ac8d294421e1bfc14afa8c6a2a12f1affb5268ee \ + --base-cli /path/to/base-cli-v0.4.3 \ + --bash-libs /path/to/base-bash-libs-v2.1.0 \ + --python /path/to/base-compatible-venv/bin/python +``` + +Base 1.9 checks the existing status, onboarding and agent-brief reports, their +workspace identity, missing-peer reporting and read-only behavior. It prints +an explicit supported-version boundary without invoking newer flags. + +For the implemented 1.10 candidate, select its checkout and exact commit +`5f316aeddc3680b92bd209fcfe652eac020d02d0`, and add `--candidate`. +[Scenario CI](../.github/workflows/scenarios.yml) runs both lanes. This is +advisory candidate evidence, not a stable-release compatibility claim. + +The candidate asserts: + +- Onboarding actions are ordered clone, setup, trust, verify; final verification + retains the workspace and workspace-manifest identity. Agent-brief actions + instead follow repository inventory order and target each exact checkout. +- Undeclared peers are inventoried but not included in declared workspace tests. +- A generated digest-bound trust command targets its original workspace even + when invoked from another same-named project. Changed manifest bytes reject + that saved command with exit 2. +- `workspace test --projects healthy` executes only that selected checkout. + Selection filters declaration order; the comma-list does not reorder tests. +- A failing command and untrusted command are failures, no test command is a + skip, and missing required peers fail the aggregate. Aggregate failure exits + 1; successful selection exits 0. `--fail-fast` skips later selected peers. +- Two selected aliases of one manifest are rejected (exit 2); selecting one + alias runs the intended checkout once. + +Each completed assertion group prints `PASS`; fixtures are removed on success +or exception. There is no second application stack and no host-readiness claim. +See the exact candidate's [workspace contract](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/workspace-manifest.md) +for the authoritative API. diff --git a/tests/scenarios/workspace.py b/tests/scenarios/workspace.py new file mode 100644 index 0000000..0e28f9e --- /dev/null +++ b/tests/scenarios/workspace.py @@ -0,0 +1,130 @@ +"""Assert disposable workspace inventory, checkout targeting and selected tests.""" +from __future__ import annotations + +import json +import shlex + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + with Fixture(args) as fixture: + roots = {} + for name in ("healthy", "failing", "untrusted", "no-test", "undeclared"): + root = fixture.project(name, "python: {}\n" if name == "no-test" else "") + roots[name] = root + if name != "no-test": + command = 'printf "tested\\n" >> "$BASE_PROJECT_ROOT/tested"' + if name == "failing": + command += "; exit 7" + with (root / "base_manifest.yaml").open("a") as stream: + stream.write("test:\n command: " + json.dumps(command) + "\n") + manifest = fixture.root / "team workspace.yaml" + manifest.write_text("schema_version: 1\nworkspace:\n name: scenario\nrepos:\n" + " - name: healthy\n - name: failing\n - name: untrusted\n" + " - name: no-test\n - name: missing\n" + " url: https://github.com/example/missing.git\n") + options = ("--workspace", fixture.workspace, "--manifest", manifest) + for command in ("status", "onboarding", "agent-brief"): + result = fixture.run("workspace", command, *options, "--format", "json", expected=(0, 1)) + payload = json.loads(result.stdout) + assert payload["workspace"] == str(fixture.workspace) + assert payload["workspace_manifest"]["path"] == str(manifest) + assert "missing" in result.stdout + assert not list(fixture.workspace.rglob("tested")), "read-only inventory executed a test" + if not args.candidate: + print("PASS: stable workspace status, onboarding and agent-brief are read-only") + print("BOUNDARY: selected workspace tests, expanded inventory and targeted next_actions require the v1.10 candidate") + return + + assert "test" in fixture.run("workspace", "--help").stdout + reports = {} + for command in ("onboarding", "agent-brief"): + reports[command] = json.loads(fixture.run("workspace", command, *options, "--format", "json").stdout) + actions = reports[command]["next_actions"] + assert [a["order"] for a in actions] == list(range(1, len(actions) + 1)) + actions = reports["onboarding"]["next_actions"] + assert [a["description"] for a in actions] == [ + "Clone missing repos", "Set up unconfigured projects", "Trust new manifests", + "Review runtimes and verify workspace health", + ] + commands = [c for a in actions for c in a["commands"]] + verify = shlex.split(commands[-1]) + assert verify[verify.index("--workspace") + 1] == str(fixture.workspace) + assert verify[verify.index("--manifest") + 1] == str(manifest) + actions = reports["agent-brief"]["next_actions"] + names = ["healthy", "failing", "untrusted", "no-test", "missing", "undeclared"] + assert [a["description"] for a in actions] == [f"Prepare repository {name}" for name in names] + for name, action in zip(names, actions): + assert action["commands"] + assert all(str(fixture.workspace / name) in shlex.split(c) for c in action["commands"]) + inventory = {r["repository"]: r for r in reports["agent-brief"]["repositories"]} + assert not inventory["undeclared"]["expected"] + assert inventory["missing"]["required"] and inventory["missing"]["discovery_status"] == "missing" + + # The generated, digest-bound trust command must retain its workspace + # even when invoked from another checkout with the same project name. + other = fixture.root / "other workspace" / "healthy" + other.mkdir(parents=True) + (other / "base_manifest.yaml").write_text((roots["healthy"] / "base_manifest.yaml").read_text()) + entry = next(r for r in reports["onboarding"]["repositories"] if r["repository"] == "healthy") + words = shlex.split(entry["trust_command"]) + assert words[:3] == ["basectl", "trust", "allow"] + assert words[words.index("--workspace") + 1] == str(fixture.workspace) + fixture.run(*words[1:], cwd=other) + selected = fixture.run("trust", "status", "healthy", "--workspace", fixture.workspace) + unselected = fixture.run("trust", "status", "healthy", "--workspace", other.parent) + assert "\tallowed\t" in selected.stdout and "\tblocked\t" in unselected.stdout + with (roots["healthy"] / "base_manifest.yaml").open("a") as stream: + stream.write("\n# changed after recovery guidance\n") + stale = fixture.run(*words[1:], cwd=other, expected=2) + assert "SHA-256" in stale.stdout + stale.stderr + for name in ("healthy", "failing"): + fixture.run("trust", "allow", name, "--workspace", fixture.workspace) + print("PASS: ordered recovery actions bind the reviewed checkout and reject stale manifest evidence") + + # Selection is a filter, not an execution-order override. Put the + # failing peer first in declaration order for the fail-fast example. + manifest.write_text(manifest.read_text().replace( + " - name: healthy\n - name: failing\n", " - name: failing\n - name: healthy\n")) + + def tests(selection=None, fail_fast=False, expected=0): + extra = ("--projects", selection) if selection else () + if fail_fast: + extra += ("--fail-fast",) + return json.loads(fixture.run("workspace", "test", *options, *extra, + "--format", "json", expected=expected).stdout) + + result = tests("healthy") + assert result["counts"] == {"passed": 1, "failed": 0, "skipped": 0} + assert result["projects"][0]["path"] == str(roots["healthy"]) + assert (roots["healthy"] / "tested").read_text() == "tested\n" + assert len(list(fixture.root.rglob("tested"))) == 1 + (roots["healthy"] / "tested").unlink() + result = tests("failing,untrusted,no-test", expected=1) + assert result["counts"] == {"passed": 0, "failed": 2, "skipped": 1} + assert not (roots["untrusted"] / "tested").exists() + result = tests("failing,healthy", fail_fast=True, expected=1) + assert result["counts"] == {"passed": 0, "failed": 1, "skipped": 1} + assert not (roots["healthy"] / "tested").exists() + result = tests(expected=1) + assert result["counts"] == {"passed": 1, "failed": 3, "skipped": 1} + assert not (roots["undeclared"] / "tested").exists() + assert not (other / "tested").exists() + print("PASS: selected execution, failure, untrusted denial, skip, fail-fast and missing-required aggregate exit") + + alias = fixture.workspace / "alias" + alias.symlink_to(roots["healthy"], target_is_directory=True) + with manifest.open("a") as stream: + stream.write(" - name: alias\n") + fixture.run("workspace", "test", *options, "--projects", "healthy,alias", expected=2) + result = tests("alias") + assert result["counts"]["passed"] == 1 + assert result["projects"][0]["repository"] == "alias" + fixture.no_ide_settings() + print(f"PASS: alias selection cannot double-run one manifest; workspace scenario at Base {args.base_commit}") + + +if __name__ == "__main__": + main() From 8eb5c2d18ef0df8e5a48bc2bdb647ab49fa9405e Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:40:08 +0530 Subject: [PATCH 04/11] ci: include Base runtime artifact dependencies in fixture Python --- .github/workflows/scenarios.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml index ec10ecf..c5d4af3 100644 --- a/.github/workflows/scenarios.yml +++ b/.github/workflows/scenarios.yml @@ -49,7 +49,7 @@ jobs: with: python-version: '3.13' - name: Install fixture interpreter dependencies - run: python -m pip install ./.dependencies/base-cli PyYAML tomli + run: python -m pip install ./.dependencies/base-cli click PyYAML tomli - name: Install supported Bash for Base run: brew install bash - name: Verify isolated trust and consent contracts From cf4c7260380ad5bf82aa26153e3a093161259c13 Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:36:34 +0530 Subject: [PATCH 05/11] docs: guide evaluator adopter and contributor first success --- .ai-context/overview.md | 7 ++- .github/workflows/scenarios.yml | 9 +++ README.md | 18 ++++++ docs/first-success.md | 108 ++++++++++++++++++++++++++++++++ tests/journeys_docs_test.py | 37 +++++++++++ tests/scenarios/journeys.py | 37 +++++++++++ tests/validate.sh | 1 + 7 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 docs/first-success.md create mode 100644 tests/journeys_docs_test.py create mode 100644 tests/scenarios/journeys.py diff --git a/.ai-context/overview.md b/.ai-context/overview.md index 0f598cb..3b5c790 100644 --- a/.ai-context/overview.md +++ b/.ai-context/overview.md @@ -25,7 +25,7 @@ binds both to the annotated tag target, checks their SHA-256 manifests, and publishes those exact verified assets after the read-only validation job passes. It includes the Base project shape plus a reduced-scale representative -environment: a `base_manifest.yaml` that declares every current Base contract, +environment: a `base_manifest.yaml` that declares a curated representative subset of Base contracts, runnable commands, a Python CLI that uses `base_cli.App`, an interactive demo script, validation tests, multiple language services, common build tools, one Dockerized service, one React/Vite UI, local databases and cache through @@ -128,6 +128,11 @@ change stream, and its three existing validation job IDs remain stable. `tests/scenarios/workspace.py` is the isolated multi-peer consumer fixture. Stable 1.9 covers reports; exact-candidate 1.10 covers selection, aggregate failure/skip, aliases, undeclared inventory and checkout-bound next actions. +Start at `docs/first-success.md`: evaluator, adopting-project maintainer and +demo contributor have separate prerequisites, completion criteria and handoff +artifacts. The full README command map is a reference, not an unattended setup +script. Current source uses `.release/supported-dependencies.json`; the interim +public bootstrap still consumes historical Base 1.8/demo 0.1 releases. ```bash basectl setup base-demo diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml index 6481768..a3a4652 100644 --- a/.github/workflows/scenarios.yml +++ b/.github/workflows/scenarios.yml @@ -70,3 +70,12 @@ jobs: --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ --python "$(command -v python)" $BASE_SCENARIO_OPTIONS + - name: Rehearse evaluator first success + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/journeys.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS diff --git a/README.md b/README.md index 442647f..9ab34f5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,18 @@ Reference Base-managed project and representative demo environment. +## Start with your goal + +- **Evaluate Base:** [inspect, run one command, and export a handoff](docs/first-success.md#evaluate-base). +- **Adopt Base in your project:** [map one real validation command](docs/first-success.md#adopt-base-in-a-project). +- **Contribute to this demo:** [validate an issue-backed change](docs/first-success.md#contribute-to-base-demo). + +Each path states prerequisites, a completion check, and one safe recovery. +This demo is a **curated representative subset**, not every Base contract. +Current-source paths use the [supported inputs](.release/supported-dependencies.json); +the historical-release Quick Start below installs older versions until v0.2.0 +publication. Neither path implies native Windows or a full Linux demo. + This repository is the public reference project for Base-managed repositories. It demonstrates Base on a compact but credible project shape: small enough to inspect in one sitting, but substantial enough to represent the tools and @@ -159,6 +171,12 @@ toolchain is unavailable, and requires live HTTP execution markers for all four API services. That smoke lane uses loopback listeners and does not require Docker Compose. +## Complete command reference + +Use this inventory after a [first-success path](docs/first-success.md), not as +an unattended script. Review the manifest before the explicit trust command; +setup, approval and command execution can change local state. + ```bash basectl projects list basectl setup base-demo # macOS only diff --git a/docs/first-success.md b/docs/first-success.md new file mode 100644 index 0000000..2815ec0 --- /dev/null +++ b/docs/first-success.md @@ -0,0 +1,108 @@ +# Three short paths to first success + +This is a curated representative subset of Base, not an exhaustive framework +certification. The [capability matrix](base-capability-matrix.md) separates +demonstrated contracts, intentional omissions and future work. + +For these **current-source** paths, use the exact stable versions in +[supported inputs](../.release/supported-dependencies.json): Base 1.9.0, +base-cli 0.4.3 and base-bash-libs 2.1.0. Start in your reviewed base-demo checkout +with Base already configured and its `basectl` on PATH. Full setup, activation, +build, test and demo paths require macOS. Bash must be 4.2 or newer. Contributors +need the toolchains declared by the manifest/mise configuration; setup may +install tools and change the project environment, so preview it first. + +Need Base first? Follow the canonical [adopter golden path](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/adopter-golden-path.md) +for install and consent decisions, then select the stable inputs above. That +document is pinned for reference, not an instruction to substitute its candidate +for the stable runtime. The README's checksum-verified [Quick Start](../README.md#quick-start) +is a separate historical Base 1.8/demo 0.1 route until v0.2.0 is published; do not +mix its results with current-source evidence or reset a divergent checkout. + +Ubuntu/Debian (including WSL2 on its native filesystem) supports Base setup and +the CI-safe read-only project-health path, not the full demo loop. Native Windows +is not supported by these journeys. Candidate 1.10 examples are explicitly +separate in the [workspace](workspace-scenarios.md) and [trust](trust-scenarios.md) +scenario guides; their success is not stable-release evidence. + +## Evaluate Base + +**Prerequisite:** macOS, the configured stable Base runtime above, and a reviewed +source checkout. No project build or service startup is needed for this path. + +```bash +basectl run --list +basectl test --dry-run +basectl trust status base-demo --workspace "$(dirname "$PWD")" +# Inspect base_manifest.yaml and src/hello.sh before approving this checkout. +basectl trust allow base-demo --workspace "$(dirname "$PWD")" +BASE_DEMO_ENV=baseline basectl run hello +basectl export-context --format markdown --print +``` + +**Done:** the command prints `hello from base-demo`, `BASE_PROJECT=base-demo` +and `BASE_DEMO_ENV=baseline`; context export produces the handoff Markdown. +This proves discovery/routing and one reviewed command, not full environment +readiness. Review the exported content before sharing it. + +**Safe failure/recovery:** if execution says the manifest is untrusted, stop and +review the named manifest and command. Use the explicit `trust allow` only after +review; changing manifest bytes invalidates approval. Do not approve to suppress +an unexplained error. `--yes` does not grant command trust or IDE consent. + +## Adopt Base in a project + +**Prerequisite:** complete the evaluator path, then use a disposable branch or +new project under a separate workspace. Choose one existing, harmless project +validation command; do not copy the demo's entire toolchain or service graph. + +Follow [the adopter golden path's project recipe](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/adopter-golden-path.md#adopt-an-external-style-project) +to declare that command in your own manifest, preview setup, inspect command +surfaces, approve the reviewed digest, and run your project's test. Keep command +approval, runtime inspection and IDE mutation as separate decisions. The demo's +`test: mise: validate` is an example, not a requirement for adopters. + +**Done:** your declared test passes in the intended checkout and a handoff PR +contains the manifest, test output, version/provider identities and remaining +warnings. Use [JSON quickstart](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/json-output-quickstart.md) +for machine-readable evidence; check both JSON status and process exit. + +**Safe failure/recovery:** preview setup with `basectl setup --dry-run` before +applying it. A missing prerequisite or stale environment is a diagnostic, not +permission to grant more trust. Use [first-run troubleshooting](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/first-run-troubleshooting.md) +to repair the specific finding and rerun validation. Review any proposed IDE or +shell-profile changes separately. This internal rehearsal is not independent +external-adoption evidence. + +## Contribute to base-demo + +**Prerequisite:** macOS, an issue-backed dedicated worktree per [AGENTS.md](../AGENTS.md), +the supported provider inputs, and the repository's declared development tools. +Use [developer mode](../README.md#quick-start) for peer checkouts; it does not +pull or switch branches. `uv` owns locked Python dependencies; source-provider +testing is an explicit separate CI lane, never a silent replacement. + +```bash +basectl setup --dry-run +# Apply reviewed setup separately; do not approve unexpected IDE changes. +basectl repo check . --agent-ready +./tests/validate.sh +git diff --check +``` + +**Done:** focused tests and the validator pass, and the issue-linked PR has +passing hosted checks. The full hosted lane requires Go/Java and all four live +HTTP services; a local optional-tool skip is not equivalent evidence. Handoff +the PR plus check links and any explicitly unresolved local limitations. + +**Safe failure/recovery:** an unset `BASE_DEMO_ENV` intentionally produces a +health finding. On macOS activation sets `baseline`; for a one-command read-only +probe use `BASE_DEMO_ENV=baseline basectl check --ci --format json`. That marker +does not select a service environment or fix other readiness findings. Inspect +each remaining finding; do not declare success from the marker alone. Services +use `services --env dev` separately; staging/prod are non-operational examples. + +For release work, use the canonical [downstream release smoke test](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/downstream-release-smoke-test.md) +and the demo's [release policy](release.md). A contributor PR is not permission +to tag or publish. The [complete command reference](../README.md#complete-command-reference) +remains available after these short paths. diff --git a/tests/journeys_docs_test.py b/tests/journeys_docs_test.py new file mode 100644 index 0000000..31b9f10 --- /dev/null +++ b/tests/journeys_docs_test.py @@ -0,0 +1,37 @@ +"""Keep short entry points and AI guidance aligned without pretending to run setup.""" +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class JourneyDocsTests(unittest.TestCase): + def test_entry_points_precede_inventory(self): + readme = (ROOT / "README.md").read_text() + self.assertLess(readme.index("## Start with your goal"), readme.index("## Complete command reference")) + guide = (ROOT / "docs/first-success.md").read_text() + for title in ("Evaluate Base", "Adopt Base in a project", "Contribute to base-demo"): + section = guide.split("## " + title + "\n", 1)[1].split("\n## ", 1)[0] + for marker in ("**Prerequisite:", "**Done:", "**Safe failure/recovery:"): + self.assertIn(marker, section) + for page in ("adopter-golden-path", "first-run-troubleshooting", "json-output-quickstart", + "downstream-release-smoke-test"): + self.assertIn(f"/docs/{page}.md", guide) + for target in re.findall(r"\]\(([^)]+)\)", guide): + if not target.startswith("https://"): + self.assertTrue((ROOT / "docs" / target.split("#")[0]).exists(), target) + + def test_scope_and_health_marker_agree(self): + overview = (ROOT / ".ai-context/overview.md").read_text() + guide = (ROOT / "docs/first-success.md").read_text() + self.assertNotIn("declares every current Base contract", overview) + for text in (overview, guide): + self.assertIn("curated representative subset", text) + self.assertIn("BASE_DEMO_ENV", text) + self.assertIn("baseline", text) + self.assertIn("services --env dev", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scenarios/journeys.py b/tests/scenarios/journeys.py new file mode 100644 index 0000000..f09bbf2 --- /dev/null +++ b/tests/scenarios/journeys.py @@ -0,0 +1,37 @@ +"""Rehearse the evaluator command sequence in a disposable minimal project.""" +from pathlib import Path +import shutil + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + source = Path(__file__).resolve().parents[2] + with Fixture(args) as fixture: + root = fixture.project("base-demo") + (root / "base_manifest.yaml").write_text( + "project:\n name: base-demo\nartifacts: []\n" + "commands:\n hello: ./src/hello.sh\ntest:\n command: ./src/hello.sh\n") + (root / "src").mkdir() + shutil.copy2(source / "src/hello.sh", root / "src/hello.sh") + (root / ".ai-context").mkdir() + (root / ".ai-context/overview.md").write_text("# Disposable evaluator handoff\n") + fixture.run("run", "--list", cwd=root) + fixture.run("test", "--dry-run", cwd=root) + fixture.run("trust", "status", "base-demo", "--workspace", fixture.workspace, cwd=root) + fixture.run("run", "hello", cwd=root, expected=1) + fixture.run("trust", "allow", "base-demo", "--workspace", fixture.workspace, cwd=root) + fixture.env["BASE_DEMO_ENV"] = "baseline" + output = fixture.run("run", "hello", cwd=root).stdout + for expected in ("hello from base-demo", "BASE_PROJECT=base-demo", "BASE_DEMO_ENV=baseline"): + assert expected in output + handoff = fixture.run("export-context", "--format", "markdown", "--print", cwd=root).stdout + assert "Disposable evaluator handoff" in handoff + fixture.no_ide_settings() + print("PASS: evaluator inspection, safe denial/recovery, actual hello script and context handoff") + print("SCOPE: minimal project rehearsal; full setup/toolchain validation remains the hosted demo gate") + + +if __name__ == "__main__": + main() diff --git a/tests/validate.sh b/tests/validate.sh index 4d16621..5314883 100755 --- a/tests/validate.sh +++ b/tests/validate.sh @@ -169,6 +169,7 @@ fi python3 tests/release_bom_test.py || exit 1 python3 tests/dependency_inputs_test.py || exit 1 python3 tests/scenario_harness_test.py || exit 1 +python3 tests/journeys_docs_test.py || exit 1 if ! bash tests/public_install_test.sh; then printf 'The exact public bootstrap input failed its contract tests.\n' >&2 From eba798074ca4b7ebb9ba27ceacb4ea543cfca5e3 Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:39:19 +0530 Subject: [PATCH 06/11] docs: bind evaluator commands to the selected workspace --- docs/first-success.md | 8 ++++---- tests/scenarios/journeys.py | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/first-success.md b/docs/first-success.md index 2815ec0..8786c7b 100644 --- a/docs/first-success.md +++ b/docs/first-success.md @@ -31,13 +31,13 @@ scenario guides; their success is not stable-release evidence. source checkout. No project build or service startup is needed for this path. ```bash -basectl run --list -basectl test --dry-run +basectl run base-demo --list --workspace "$(dirname "$PWD")" +basectl test base-demo --dry-run --workspace "$(dirname "$PWD")" basectl trust status base-demo --workspace "$(dirname "$PWD")" # Inspect base_manifest.yaml and src/hello.sh before approving this checkout. basectl trust allow base-demo --workspace "$(dirname "$PWD")" -BASE_DEMO_ENV=baseline basectl run hello -basectl export-context --format markdown --print +BASE_DEMO_ENV=baseline basectl run base-demo hello --workspace "$(dirname "$PWD")" +basectl export-context base-demo --workspace "$(dirname "$PWD")" --format markdown --print ``` **Done:** the command prints `hello from base-demo`, `BASE_PROJECT=base-demo` diff --git a/tests/scenarios/journeys.py b/tests/scenarios/journeys.py index f09bbf2..c51b977 100644 --- a/tests/scenarios/journeys.py +++ b/tests/scenarios/journeys.py @@ -17,16 +17,17 @@ def main(): shutil.copy2(source / "src/hello.sh", root / "src/hello.sh") (root / ".ai-context").mkdir() (root / ".ai-context/overview.md").write_text("# Disposable evaluator handoff\n") - fixture.run("run", "--list", cwd=root) - fixture.run("test", "--dry-run", cwd=root) + workspace = ("--workspace", fixture.workspace) + fixture.run("run", "base-demo", "--list", *workspace, cwd=root) + fixture.run("test", "base-demo", "--dry-run", *workspace, cwd=root) fixture.run("trust", "status", "base-demo", "--workspace", fixture.workspace, cwd=root) - fixture.run("run", "hello", cwd=root, expected=1) + fixture.run("run", "base-demo", "hello", *workspace, cwd=root, expected=1) fixture.run("trust", "allow", "base-demo", "--workspace", fixture.workspace, cwd=root) fixture.env["BASE_DEMO_ENV"] = "baseline" - output = fixture.run("run", "hello", cwd=root).stdout + output = fixture.run("run", "base-demo", "hello", *workspace, cwd=root).stdout for expected in ("hello from base-demo", "BASE_PROJECT=base-demo", "BASE_DEMO_ENV=baseline"): assert expected in output - handoff = fixture.run("export-context", "--format", "markdown", "--print", cwd=root).stdout + handoff = fixture.run("export-context", "base-demo", *workspace, "--format", "markdown", "--print", cwd=root).stdout assert "Disposable evaluator handoff" in handoff fixture.no_ide_settings() print("PASS: evaluator inspection, safe denial/recovery, actual hello script and context handoff") From 577ee3de6f92fb942f8ee8a92a74c8152e3ea874 Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:42:18 +0530 Subject: [PATCH 07/11] docs(release): prepare v0.2.0 rehearsal and owner handoff --- docs/release.md | 12 +++- docs/v0.2.0-readiness.md | 117 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 docs/v0.2.0-readiness.md diff --git a/docs/release.md b/docs/release.md index 8c6a662..2cd99c1 100644 --- a/docs/release.md +++ b/docs/release.md @@ -119,6 +119,10 @@ silently accepted by the release path. ## Release procedure +For the current train, use the [v0.2.0 owner handoff](v0.2.0-readiness.md). +Implementation/merge authority does not grant tag/publication authority or +waive the minor-release bake and independent-review decision. + 1. Keep post-release work under `## [Unreleased]` in `CHANGELOG.md`. 2. In a release PR, choose the next SemVer version, update `VERSION` and all governed metadata, promote `Unreleased` into a dated version section, and @@ -127,8 +131,12 @@ silently accepted by the release path. 3. Generate or update the demo component row with `bin/base-demo-release-bom-row`, update `.release/release-bom.json` from the coordinated release inputs, then run `bin/base-demo-release-check`, - `bin/base-demo-release-bom-check`, `mise run validate`, and the normal hosted - pull-request checks. Do not try to pin the final merge SHA in this commit. + `bin/base-demo-dependencies --check`, `mise run validate`, and the normal + hosted pull-request checks. The prepared `not_tested` BOM is not publication + proof and must fail the strict BOM gate. After merge, bind a successful + exact-commit compatibility run with `base-demo-release-finalize --evidence-run` + and check the resulting external BOM as shown in the owner handoff. Do not + try to pin the final merge SHA in this commit. 4. After the release PR is merged to `main`, create an annotated tag from the clean merge commit: `git tag -a vX.Y.Z -m "base-demo vX.Y.Z"`. 5. Push the tag. The read-only `verify` job in the `Release Demo` workflow diff --git a/docs/v0.2.0-readiness.md b/docs/v0.2.0-readiness.md new file mode 100644 index 0000000..d84384f --- /dev/null +++ b/docs/v0.2.0-readiness.md @@ -0,0 +1,117 @@ +# v0.2.0 release-owner handoff + +Status: **preparation only; not approved for publication**. +The core implementation train does not create a version tag, publish assets, +waive independent review, or start a bake window on the owner's behalf. +[#303](https://github.com/basefoundry/base-demo/issues/303) remains open until +the publication and downloaded-asset checks below are complete. + +## Intended scope and inputs + +The release includes the repaired immutable bootstrap, fail-closed governed +BOM/evidence gate, single supported dependency-input record, disposable trust +and workspace scenarios, and short evaluator/adopter/contributor paths. +Record the final disposition of #291–#300 and #256 in #303; do not infer that an +issue is complete from this document. Optional refactoring (#301), native +Windows parity (#302), and future Base Docker-service adoption (#163) are not +release prerequisites. + +Stable compatibility uses `.release/supported-dependencies.json`: Base 1.9.0 +`ac8d294421e1bfc14afa8c6a2a12f1affb5268ee`, base-cli 0.4.3 +`8a93d22156ba75a99965f7c355f867acba630069`, and base-bash-libs 2.1.0 +`36fec50c446dcea8c521a1ba3e7fee2394f169c0`. The exact Base 1.10 candidate +`5f316aeddc3680b92bd209fcfe652eac020d02d0` has a **separate advisory scenario +lane**. Neither candidate success nor PR-head success substitutes for the +stable final-main-commit evidence required by the publication gate. + +Base #2289's shared updater is repaired. Base #2256's scheduled-update +credentials are a separate operational concern; merging this train neither +provisions those credentials nor proves a scheduled bot update succeeded. + +## Owner decisions still required + +This is a minor release with high-impact installer, trust and BOM boundaries. +Apply Base's [stabilization and independent-review policy](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/release-stabilization-policy.md): +the default minor-release bake is **three full working days**, starting from a +named reviewed candidate commit (an equivalent reviewed commit may replace an +RC tag). Record explicit UTC start/end times and any findings in #303. + +The owner must name an independent reviewer or record a waiver naming the +unavailable reviewer, justification, residual risk and follow-up review date. +Automated CI and agent-authored review do not establish independent human +review. No waiver or shortened bake is implied by this handoff. + +## Final candidate checklist + +1. Land the scoped implementation PRs. Prepare a separate reviewed version + change for 0.2.0: `VERSION`, Python/uv project metadata, frontend package and + lock metadata, README release strip, dated changelog, installer release ref, + and prepared BOM self-version/tag/API identity. Keep valid placeholder self + commits and `not_tested` evidence in tracked prepared input; do not pretend + that a tracked file can identify its own eventual merge SHA. +2. Keep the interim public bootstrap until verified v0.2.0 assets actually + exist. Do not link an unavailable asset or claim that the historical + v0.1.0 installer has the new safety properties. Version/readme publication + wording must be reviewed together; `base-demo-release-check` enforces the + governed release strip, not asset availability. +3. Record the resulting **full merge commit** and exact successful `tests.yml` + push/manual run. It must include `validate` on macos-14, `validate-ubuntu` on + ubuntu-24.04, and `validate-base-cli-source`; full macOS validation must + execute Go/Java and all four live HTTP services. Record both scenario lane + results separately at that same demo commit. Local Xcode-license failures + or optional-tool skips are not equivalent hosted evidence. +4. Use the read-only artifact rehearsal below. Rehearse a fresh install in an + isolated home/workspace against the exact output; verify reused matching + checkouts, dirty/divergent rejection and checksum mismatch. The installer + deliberately does not upgrade a divergent checkout in place. Review any + upgrade manually and rerun the exact-pin installer; do not reset user work. + Unit/mock installer fixtures supplement, but do not replace, this final + artifact install/upgrade rehearsal. +5. Complete the bake and review/waiver record. Resolve or explicitly disposition + every finding. Only then request/record human permission to create and push + the annotated v0.2.0 tag. No existing published tag or asset may be replaced. + +## Read-only artifact rehearsal + +From the clean, reviewed final candidate checkout, set `candidate_commit` to +its full SHA and `evidence_run` to the successful exact-commit run ID. These +commands write only a new external temporary artifact directory; they do not +tag, install, or publish. Authentication needs read access to Actions evidence. + +```bash +test "$(git rev-parse HEAD)" = "$candidate_commit" +test -z "$(git status --porcelain)" +bin/base-demo-release-check +bin/base-demo-dependencies --check --verify-refs +rehearsal_dir="$(mktemp -d)" +bin/base-demo-release-finalize --commit "$candidate_commit" \ + --evidence-run "$evidence_run" --output-dir "$rehearsal_dir" +bin/base-demo-release-finalize --commit "$candidate_commit" \ + --evidence-run "$evidence_run" --verify-dir "$rehearsal_dir" +BASE_DEMO_RELEASE_BOM_PATH="$rehearsal_dir/release-bom.json" \ + BASE_DEMO_RELEASE_BOM_EXPECTED_COMMIT="$candidate_commit" \ + bin/base-demo-release-bom-check +(cd "$rehearsal_dir" && shasum -a 256 -c release-bom.sha256 install.sh.sha256) +BASE_DEMO_TEST_BOOTSTRAP="$rehearsal_dir/install.sh" bats tests/install_test.bats +``` + +The finalizer verifies live server-owned run, job, platform, commit and input +identity **before writing assets** when `--evidence-run` is supplied. Omitting +that flag is only a self-identity fixture operation, not compatibility proof. +The strict BOM publication checker is expected to reject the tracked +`not_tested` input. Validate the finalized external BOM instead. + +Because the published version is still 0.1.0 during core-train preparation, +any rehearsal at that stage tests mechanics only. It must not be published as +another 0.1.0 release or called the final 0.2.0 candidate. + +## After separately authorized publication + +Download `release-bom.json`, `release-bom.sha256`, `install.sh` and +`install.sh.sha256` from the new release into a fresh directory. Independently +verify checksums, annotated tag target, finalized self-identity and exact +provider inputs. Re-run the strict BOM gate and inspect the downloaded +installer pins. Update the README public URL/digest and consumed versions to +those actual assets, run `tests/public_install_test.sh`, and rehearse that exact +public command. Record the immutable demo identity for the Base-owned +compatibility BOM. Only then close #303 and mark the release complete. From 45cd69aba563611ec27eb7f2e187d6fc5f864cdd Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:43:54 +0530 Subject: [PATCH 08/11] ci: match exact Base bootstrap artifact versions --- .github/workflows/scenarios.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml index c5d4af3..feaf707 100644 --- a/.github/workflows/scenarios.yml +++ b/.github/workflows/scenarios.yml @@ -49,7 +49,8 @@ jobs: with: python-version: '3.13' - name: Install fixture interpreter dependencies - run: python -m pip install ./.dependencies/base-cli click PyYAML tomli + # Match both exact Base revisions' lib/base/default_manifest.yaml. + run: python -m pip install ./.dependencies/base-cli click==8.4.1 PyYAML==6.0.3 tomli==2.4.1 - name: Install supported Bash for Base run: brew install bash - name: Verify isolated trust and consent contracts From cf1b8b231efb7831368dd8f593db2d80c2b06203 Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:45:41 +0530 Subject: [PATCH 09/11] test(release): separate prepared self pins from published tag proof --- docs/v0.2.0-readiness.md | 3 +++ tests/release_test.bats | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/v0.2.0-readiness.md b/docs/v0.2.0-readiness.md index d84384f..c4da946 100644 --- a/docs/v0.2.0-readiness.md +++ b/docs/v0.2.0-readiness.md @@ -49,6 +49,9 @@ review. No waiver or shortened bake is implied by this handoff. and prepared BOM self-version/tag/API identity. Keep valid placeholder self commits and `not_tested` evidence in tracked prepared input; do not pretend that a tracked file can identify its own eventual merge SHA. + Prepared installer tests compare self pins to the prepared BOM, not a + not-yet-created demo tag. Released Base pins still resolve live; final demo + tag identity is enforced separately by provenance and finalized evidence. 2. Keep the interim public bootstrap until verified v0.2.0 assets actually exist. Do not link an unavailable asset or claim that the historical v0.1.0 installer has the new safety properties. Version/readme publication diff --git a/tests/release_test.bats b/tests/release_test.bats index 3f9733a..d3c9b60 100644 --- a/tests/release_test.bats +++ b/tests/release_test.bats @@ -182,7 +182,7 @@ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] [ ! -e "$TEST_TMPDIR/invalid/release-bom.json" ] } -@test "install release pins resolve refs to their target commits" { +@test "prepared installer self pin matches BOM while released Base ref resolves exactly" { project_ref="$(sed -n 's/^PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" project_pin="$(sed -n 's/^PROJECT_RELEASE_COMMIT="${PROJECT_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" base_ref="$(sed -n 's/^BASE_RELEASE_REF="${BASE_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" @@ -193,9 +193,21 @@ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] [ -n "$base_ref" ] [ -n "$base_pin" ] - run resolve_release_commit "$TEST_ROOT" "${PROJECT_REPO_URL:-https://github.com/basefoundry/base-demo.git}" "$project_ref" + # The next demo tag does not exist during its version PR, and the tracked + # self pin cannot name its own eventual merge SHA. Do not demand a published + # tag here. Finalized artifact identity is checked by provenance + live BOM + # evidence after merge; this test only checks the prepared input contract. + run python3 -c ' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +bom = json.loads((root / ".release/release-bom.json").read_text()) +tag = "v" + (root / "VERSION").read_text().strip() +assert sys.argv[2] == tag == bom["release"]["tag"] +assert sys.argv[3] == bom["release"]["commit"] +rows = [r for r in bom["components"] if r["repository"] == "basefoundry/base-demo"] +assert len(rows) == 1 and rows[0]["tag"] == tag and rows[0]["commit"] == sys.argv[3] +' "$TEST_ROOT" "$project_ref" "$project_pin" [ "$status" -eq 0 ] - [ "$output" = "$project_pin" ] run resolve_release_commit "$TEST_ROOT/../base" "${BASE_REPO_URL:-https://github.com/basefoundry/base.git}" "$base_ref" [ "$status" -eq 0 ] From 2f099b1ff5905c3c0f4b44115375ac9966e0588f Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:46:41 +0530 Subject: [PATCH 10/11] test(release): prove next-version preparation remains uncertified --- tests/release_bom_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/release_bom_test.py b/tests/release_bom_test.py index e7ead12..6d5c963 100644 --- a/tests/release_bom_test.py +++ b/tests/release_bom_test.py @@ -155,6 +155,26 @@ def test_finalizer_binds_only_verified_evidence(self): build(root, "a" * 40, "123") self.assertEqual(path.read_bytes(), original) + def test_next_version_can_be_prepared_without_certifying_or_publishing_it(self): + build = runpy.run_path(str(ROOT / "bin/base-demo-release-finalize"))["build_assets"] + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / ".release").mkdir() + (root / "VERSION").write_text("0.2.0\n") + (root / "install.sh").write_bytes((ROOT / "install.sh").read_bytes()) + self.bom["release"].update(version="0.2.0", tag="v0.2.0") + self.bom["components"][0].update(version="0.2.0", tag="v0.2.0") + for row in self.bom["components"] + self.bom["combinations"]: + row.update(result="not_tested", evidence="pending") + (root / ".release/release-bom.json").write_text(json.dumps(self.bom)) + tag, content, installer, _ = build(root, "a" * 40) + self.assertEqual(tag, "v0.2.0") + self.assertIn(b'PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-v0.2.0}"', installer) + self.assertTrue(all(r["result"] == "not_tested" for r in json.loads(content)["components"])) + with self.assertRaises(ValueError): + demo_bom.check(json.loads(content), self.inputs, "0.2.0", "a" * 40) + self.assertFalse((root / ".git").exists()) + if __name__ == "__main__": unittest.main() From a4f2fab05082026619e598252c84604f0c5f6537 Mon Sep 17 00:00:00 2001 From: codeforester <22363102+codeforester@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:54:52 +0530 Subject: [PATCH 11/11] test(release): derive installer fixture tags from selected inputs --- tests/install_test.bats | 5 +++-- tests/release_test.bats | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/install_test.bats b/tests/install_test.bats index eb58d7c..d6cab9b 100644 --- a/tests/install_test.bats +++ b/tests/install_test.bats @@ -10,6 +10,7 @@ setup() { TEST_GIT_LOG="$TEST_TMPDIR/git.log" TEST_BASE_COMMIT="$(sed -n 's/^BASE_RELEASE_COMMIT="${BASE_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" TEST_PROJECT_COMMIT="$(sed -n 's/^PROJECT_RELEASE_COMMIT="${PROJECT_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" + TEST_PROJECT_REF="$(sed -n 's/^PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" TEST_BASE_REF="$(sed -n 's/^BASE_RELEASE_REF="${BASE_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" mkdir -p "$TEST_FAKE_BIN" @@ -185,10 +186,10 @@ installer_sha256() { [ "$status" -eq 0 ] [[ "$output" == *"Installing pinned Base release '$TEST_BASE_REF'"* ]] - [[ "$output" == *"Cloning pinned base-demo release 'v0.1.0'"* ]] + [[ "$output" == *"Cloning pinned base-demo release '$TEST_PROJECT_REF'"* ]] [[ "$output" == *"Verified pinned Base commit $TEST_BASE_COMMIT"* ]] [[ "$output" == *"Verified pinned base-demo commit $TEST_PROJECT_COMMIT"* ]] - grep -Fq -- "clone --depth 1 --branch v0.1.0 https://github.com/basefoundry/base-demo.git" "$TEST_GIT_LOG" + grep -Fq -- "clone --depth 1 --branch $TEST_PROJECT_REF https://github.com/basefoundry/base-demo.git" "$TEST_GIT_LOG" [ -f "$TEST_MARKER" ] } diff --git a/tests/release_test.bats b/tests/release_test.bats index d3c9b60..b3f7826 100644 --- a/tests/release_test.bats +++ b/tests/release_test.bats @@ -2,6 +2,7 @@ setup() { TEST_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd -P)" + TEST_TAG="v$(< "$TEST_ROOT/VERSION")" TEST_TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/base-demo-release-test.XXXXXX")" TEST_REPO="$TEST_TMPDIR/repo" mkdir -p "$TEST_REPO" @@ -107,13 +108,13 @@ resolve_release_commit() { git -C "$TEST_REPO" add VERSION install.sh .release/release-bom.json git -C "$TEST_REPO" commit -q -m "prepare release inputs" target_commit="$(git -C "$TEST_REPO" rev-parse HEAD)" - git -C "$TEST_REPO" tag -a v0.1.0 -m "base-demo v0.1.0" "$target_commit" + git -C "$TEST_REPO" tag -a "$TEST_TAG" -m "base-demo $TEST_TAG" "$target_commit" output_dir="$TEST_TMPDIR/finalized" run "$TEST_ROOT/bin/base-demo-release-provenance" \ --repo "$TEST_REPO" \ --main-ref main \ - v0.1.0 \ + "$TEST_TAG" \ "$target_commit" [ "$status" -eq 0 ] @@ -123,7 +124,7 @@ resolve_release_commit() { --output-dir "$output_dir" [ "$status" -eq 0 ] - [[ "$output" == *"v0.1.0"* ]] + [[ "$output" == *"$TEST_TAG"* ]] run python3 -c ' import json import sys @@ -138,7 +139,7 @@ rows = [ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] ' "$output_dir/release-bom.json" "$target_commit" [ "$status" -eq 0 ] - grep -Fq 'PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-v0.1.0}"' "$output_dir/install.sh" + grep -Fq "PROJECT_RELEASE_REF=\"\${PROJECT_RELEASE_REF:-$TEST_TAG}\"" "$output_dir/install.sh" grep -Fq "PROJECT_RELEASE_COMMIT=\"\${PROJECT_RELEASE_COMMIT:-$target_commit}\"" "$output_dir/install.sh" [ -x "$output_dir/install.sh" ] [ "$(shasum -a 256 "$TEST_REPO/install.sh" | awk '{print $1}')" = "$original_installer_sha" ]