diff --git a/.ai-context/overview.md b/.ai-context/overview.md index 13b23a8..3b5c790 100644 --- a/.ai-context/overview.md +++ b/.ai-context/overview.md @@ -1,5 +1,11 @@ # base-demo Overview +Disposable trust scenarios live in `tests/scenarios/trust.py` and +`docs/trust-scenarios.md`. The stable v1.9 lane does not claim full historical +revocation or static runtime inspection; those assertions use a separately +pinned v1.10 candidate. Fixture homes and IDE delegate spies prevent learner +state mutation. Scenario success does not certify host readiness. + The release BOM gate reuses Base's governed contract and requires both platform jobs plus source-provider evidence at the exact release commit. Static pins and historical release:// records are not passing compatibility proof; see @@ -19,7 +25,7 @@ binds both to the annotated tag target, checks their SHA-256 manifests, and publishes those exact verified assets after the read-only validation job passes. It includes the Base project shape plus a reduced-scale representative -environment: a `base_manifest.yaml` that declares every current Base contract, +environment: a `base_manifest.yaml` that declares a curated representative subset of Base contracts, runnable commands, a Python CLI that uses `base_cli.App`, an interactive demo script, validation tests, multiple language services, common build tools, one Dockerized service, one React/Vite UI, local databases and cache through @@ -119,6 +125,15 @@ change stream, and its three existing validation job IDs remain stable. ## Quick Loop +`tests/scenarios/workspace.py` is the isolated multi-peer consumer fixture. +Stable 1.9 covers reports; exact-candidate 1.10 covers selection, aggregate +failure/skip, aliases, undeclared inventory and checkout-bound next actions. +Start at `docs/first-success.md`: evaluator, adopting-project maintainer and +demo contributor have separate prerequisites, completion criteria and handoff +artifacts. The full README command map is a reference, not an unattended setup +script. Current source uses `.release/supported-dependencies.json`; the interim +public bootstrap still consumes historical Base 1.8/demo 0.1 releases. + ```bash basectl setup base-demo basectl activate base-demo diff --git a/.github/workflows/scenarios.yml b/.github/workflows/scenarios.yml new file mode 100644 index 0000000..e7e5ccb --- /dev/null +++ b/.github/workflows/scenarios.yml @@ -0,0 +1,82 @@ +name: Isolated Base scenarios + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scenarios-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + trust: + name: Trust and workspace scenarios (${{ matrix.lane }}) + runs-on: macos-14 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - lane: stable-v1.9 + base: ac8d294421e1bfc14afa8c6a2a12f1affb5268ee + options: "" + - lane: advisory-v1.10-candidate + base: 5f316aeddc3680b92bd209fcfe652eac020d02d0 + options: --candidate + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base + ref: ${{ matrix.base }} + path: .dependencies/base + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base-cli + ref: 8a93d22156ba75a99965f7c355f867acba630069 + path: .dependencies/base-cli + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + repository: basefoundry/base-bash-libs + ref: 36fec50c446dcea8c521a1ba3e7fee2394f169c0 + path: .dependencies/base-bash-libs + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.13' + - name: Install fixture interpreter dependencies + # Match both exact Base revisions' lib/base/default_manifest.yaml. + run: python -m pip install ./.dependencies/base-cli click==8.4.1 PyYAML==6.0.3 tomli==2.4.1 + - name: Install supported Bash for Base + run: brew install bash + - name: Verify isolated trust and consent contracts + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/trust.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS + - name: Verify isolated workspace contracts + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/workspace.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS + - name: Rehearse evaluator first success + env: + BASE_SCENARIO_COMMIT: ${{ matrix.base }} + BASE_SCENARIO_OPTIONS: ${{ matrix.options }} + run: | + python tests/scenarios/journeys.py \ + --base .dependencies/base --base-commit "$BASE_SCENARIO_COMMIT" \ + --base-cli .dependencies/base-cli --bash-libs .dependencies/base-bash-libs \ + --python "$(command -v python)" $BASE_SCENARIO_OPTIONS diff --git a/CHANGELOG.md b/CHANGELOG.md index d5fcd71..2b58f96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ promotes this section into a dated version heading before tagging. - Aligned current source with the supported Base 1.9.0, base-cli 0.4.3 and base-bash-libs 2.1.0 input contract, with exact macOS/Ubuntu CI revisions and live final-candidate evidence binding outside the tracked tree. +- Added disposable workspace selection, failure/skip, fail-fast and targeted + recovery scenarios, with separate stable and exact-candidate boundaries. + +- Added isolated stable/candidate trust scenarios for denial, invalidation, + revocation, runtime verification and independent IDE consent, with explicit + Base v1.9 historical-revocation limitations. - Made release BOM publication fail closed on incomplete participants, stale pins, inconsistent platforms, and missing exact-commit hosted evidence. diff --git a/README.md b/README.md index d4b71a1..9ab34f5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,18 @@ Reference Base-managed project and representative demo environment. +## Start with your goal + +- **Evaluate Base:** [inspect, run one command, and export a handoff](docs/first-success.md#evaluate-base). +- **Adopt Base in your project:** [map one real validation command](docs/first-success.md#adopt-base-in-a-project). +- **Contribute to this demo:** [validate an issue-backed change](docs/first-success.md#contribute-to-base-demo). + +Each path states prerequisites, a completion check, and one safe recovery. +This demo is a **curated representative subset**, not every Base contract. +Current-source paths use the [supported inputs](.release/supported-dependencies.json); +the historical-release Quick Start below installs older versions until v0.2.0 +publication. Neither path implies native Windows or a full Linux demo. + This repository is the public reference project for Base-managed repositories. It demonstrates Base on a compact but credible project shape: small enough to inspect in one sitting, but substantial enough to represent the tools and @@ -29,6 +41,13 @@ blocked Base capabilities is maintained in the [Base capability and evidence matrix](docs/base-capability-matrix.md). Use it during Base release reviews and when deciding whether a new base-demo scenario has executable evidence. +The [workspace scenarios](docs/workspace-scenarios.md) prove selected execution, +failure aggregation and checkout-bound recovery in disposable peers. +The [isolated trust and consent scenarios](docs/trust-scenarios.md) demonstrate +denial, approval, invalidation and revocation without changing your trust store +or IDE settings. They explicitly separate Base v1.9 behavior from the stronger +implemented v1.10 candidate contracts. + The external tooling direction is tracked in [Tooling Test Bed](docs/tooling-testbed.md). That matrix separates active baseline tools from optional wrappers, reference-only examples, and future Base @@ -152,6 +171,12 @@ toolchain is unavailable, and requires live HTTP execution markers for all four API services. That smoke lane uses loopback listeners and does not require Docker Compose. +## Complete command reference + +Use this inventory after a [first-success path](docs/first-success.md), not as +an unattended script. Review the manifest before the explicit trust command; +setup, approval and command execution can change local state. + ```bash basectl projects list basectl setup base-demo # macOS only diff --git a/docs/base-capability-matrix.md b/docs/base-capability-matrix.md index 85a7409..7c9be51 100644 --- a/docs/base-capability-matrix.md +++ b/docs/base-capability-matrix.md @@ -23,8 +23,8 @@ useful for development, but does not by itself establish released compatibility. | Representative build, test, service, environment, and non-interactive demo loop | `1.9.0` | The manifest targets in [`base_manifest.yaml`](../base_manifest.yaml), baseline gate in [`tests/validate.sh`](../tests/validate.sh), and focused suites in [`tests/services_test.bats`](../tests/services_test.bats), [`tests/environments_test.bats`](../tests/environments_test.bats), and [`tests/demo_test.bats`](../tests/demo_test.bats) | Full project loop is macOS; Ubuntu/Debian validates Base setup and project health only | Demonstrated | base-demo | Add executable evidence before calling a new service or command demonstrated. | | Linux and WSL2 read-only support boundary | `1.9.0` | The supported commands and explicit native-Windows boundary are in [`README.md`](../README.md) and [`docs/contracts.md`](contracts.md); CI's Ubuntu path is in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Ubuntu/Debian and WSL2 use setup, dev-profile, check, and doctor/read-only paths; native Windows is excluded | Demonstrated | Base + base-demo | Keep platform claims tied to a hosted or repository-local check. | | Base `1.9.0` released-compatibility pin and full Go/live-HTTP evidence | `1.9.0` | Structured pins in [supported inputs](../.release/supported-dependencies.json), verified by `bin/base-demo-dependencies`; full-language and live-HTTP gates run in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Exact macOS 14 full demo and Ubuntu 24.04 setup/read-only scope; no Linux full-demo claim | Demonstrated | Base + base-demo | Bind final-candidate runs during #303; later dependency changes require fresh proof. | -| Workspace scenarios planned for the Base `1.10.0` train | `1.10.0` (planned) | The workspace manifest and current boundary are [`workspace.yaml.example`](../workspace.yaml.example) and [`README.md`](../README.md); the new scenario evidence is tracked by [base-demo#297](https://github.com/basefoundry/base-demo/issues/297) | Planned release work; no `1.10.0` claim is made by the current `main` branch | Blocked upstream | Base + base-demo | Implement #297 after the Base `1.10.0` workspace contract is fixed. | -| Trust and consent scenarios planned for the Base `1.10.0` train | `1.10.0` (planned) | Current trust documentation and CI ordering are in [`README.md`](../README.md), [`docs/contracts.md`](contracts.md), and [`.github/workflows/tests.yml`](../.github/workflows/tests.yml); the additional scenario is tracked by [base-demo#298](https://github.com/basefoundry/base-demo/issues/298) | Planned release work; current evidence remains the `1.9.0` boundary | Blocked upstream | Base + base-demo | Implement #298 after the Base trust/consent contract is stable. | +| Workspace inventory, selected tests and targeted recovery | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI | macOS fixture lane; selection and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. | +| Trust lifecycle and separate runtime/IDE consent | `1.9.0` baseline; exact `1.10.0` candidate | `tests/scenarios/trust.py` and [trust scenario guide](trust-scenarios.md), executed by [isolated scenario CI](../.github/workflows/scenarios.yml) | macOS fixture lane; full historical revocation and runtime inspection require the pinned candidate | Demonstrated | Base + base-demo | Refresh exact candidate evidence before release; do not attribute candidate-only guarantees to v1.9.0. | | Optional `test.requirements` and uninstall guidance | `1.9.0` | The manifest test entry and contributor setup guidance are [`base_manifest.yaml`](../base_manifest.yaml) and [`README.md`](../README.md) | Optional metadata is not required for the baseline demo contract | Intentionally omitted | base-demo | Revisit when Base publishes a stable user-facing contract and a concrete scenario. | | Native Windows support | `1.11.0` (planned) | The current non-goal is recorded in [`README.md`](../README.md); the staged Base work is tracked by [base-demo#302](https://github.com/basefoundry/base-demo/issues/302) and Base [#2215](https://github.com/basefoundry/base/issues/2215) | Native Windows is not shipped; Git Bash and WSL2 do not count as native Windows evidence | Blocked upstream | Base + base-demo | Wait for the PowerShell-first Base contract and hosted Windows evidence. | | First-class Base Docker-service contract | Future | The existing Compose fixture is documented in [`docs/tooling-testbed.md`](tooling-testbed.md) and [`infra/compose.yaml`](../infra/compose.yaml); adoption remains tracked by [base-demo#163](https://github.com/basefoundry/base-demo/issues/163) and Base [#124](https://github.com/basefoundry/base/issues/124) | Compose is a repository fixture; it does not establish a Base Docker-service contract | Blocked upstream | Base + base-demo | Do not make the future Base command a required demo dependency before Base publishes it. | diff --git a/docs/first-success.md b/docs/first-success.md new file mode 100644 index 0000000..8786c7b --- /dev/null +++ b/docs/first-success.md @@ -0,0 +1,108 @@ +# Three short paths to first success + +This is a curated representative subset of Base, not an exhaustive framework +certification. The [capability matrix](base-capability-matrix.md) separates +demonstrated contracts, intentional omissions and future work. + +For these **current-source** paths, use the exact stable versions in +[supported inputs](../.release/supported-dependencies.json): Base 1.9.0, +base-cli 0.4.3 and base-bash-libs 2.1.0. Start in your reviewed base-demo checkout +with Base already configured and its `basectl` on PATH. Full setup, activation, +build, test and demo paths require macOS. Bash must be 4.2 or newer. Contributors +need the toolchains declared by the manifest/mise configuration; setup may +install tools and change the project environment, so preview it first. + +Need Base first? Follow the canonical [adopter golden path](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/adopter-golden-path.md) +for install and consent decisions, then select the stable inputs above. That +document is pinned for reference, not an instruction to substitute its candidate +for the stable runtime. The README's checksum-verified [Quick Start](../README.md#quick-start) +is a separate historical Base 1.8/demo 0.1 route until v0.2.0 is published; do not +mix its results with current-source evidence or reset a divergent checkout. + +Ubuntu/Debian (including WSL2 on its native filesystem) supports Base setup and +the CI-safe read-only project-health path, not the full demo loop. Native Windows +is not supported by these journeys. Candidate 1.10 examples are explicitly +separate in the [workspace](workspace-scenarios.md) and [trust](trust-scenarios.md) +scenario guides; their success is not stable-release evidence. + +## Evaluate Base + +**Prerequisite:** macOS, the configured stable Base runtime above, and a reviewed +source checkout. No project build or service startup is needed for this path. + +```bash +basectl run base-demo --list --workspace "$(dirname "$PWD")" +basectl test base-demo --dry-run --workspace "$(dirname "$PWD")" +basectl trust status base-demo --workspace "$(dirname "$PWD")" +# Inspect base_manifest.yaml and src/hello.sh before approving this checkout. +basectl trust allow base-demo --workspace "$(dirname "$PWD")" +BASE_DEMO_ENV=baseline basectl run base-demo hello --workspace "$(dirname "$PWD")" +basectl export-context base-demo --workspace "$(dirname "$PWD")" --format markdown --print +``` + +**Done:** the command prints `hello from base-demo`, `BASE_PROJECT=base-demo` +and `BASE_DEMO_ENV=baseline`; context export produces the handoff Markdown. +This proves discovery/routing and one reviewed command, not full environment +readiness. Review the exported content before sharing it. + +**Safe failure/recovery:** if execution says the manifest is untrusted, stop and +review the named manifest and command. Use the explicit `trust allow` only after +review; changing manifest bytes invalidates approval. Do not approve to suppress +an unexplained error. `--yes` does not grant command trust or IDE consent. + +## Adopt Base in a project + +**Prerequisite:** complete the evaluator path, then use a disposable branch or +new project under a separate workspace. Choose one existing, harmless project +validation command; do not copy the demo's entire toolchain or service graph. + +Follow [the adopter golden path's project recipe](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/adopter-golden-path.md#adopt-an-external-style-project) +to declare that command in your own manifest, preview setup, inspect command +surfaces, approve the reviewed digest, and run your project's test. Keep command +approval, runtime inspection and IDE mutation as separate decisions. The demo's +`test: mise: validate` is an example, not a requirement for adopters. + +**Done:** your declared test passes in the intended checkout and a handoff PR +contains the manifest, test output, version/provider identities and remaining +warnings. Use [JSON quickstart](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/json-output-quickstart.md) +for machine-readable evidence; check both JSON status and process exit. + +**Safe failure/recovery:** preview setup with `basectl setup --dry-run` before +applying it. A missing prerequisite or stale environment is a diagnostic, not +permission to grant more trust. Use [first-run troubleshooting](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/first-run-troubleshooting.md) +to repair the specific finding and rerun validation. Review any proposed IDE or +shell-profile changes separately. This internal rehearsal is not independent +external-adoption evidence. + +## Contribute to base-demo + +**Prerequisite:** macOS, an issue-backed dedicated worktree per [AGENTS.md](../AGENTS.md), +the supported provider inputs, and the repository's declared development tools. +Use [developer mode](../README.md#quick-start) for peer checkouts; it does not +pull or switch branches. `uv` owns locked Python dependencies; source-provider +testing is an explicit separate CI lane, never a silent replacement. + +```bash +basectl setup --dry-run +# Apply reviewed setup separately; do not approve unexpected IDE changes. +basectl repo check . --agent-ready +./tests/validate.sh +git diff --check +``` + +**Done:** focused tests and the validator pass, and the issue-linked PR has +passing hosted checks. The full hosted lane requires Go/Java and all four live +HTTP services; a local optional-tool skip is not equivalent evidence. Handoff +the PR plus check links and any explicitly unresolved local limitations. + +**Safe failure/recovery:** an unset `BASE_DEMO_ENV` intentionally produces a +health finding. On macOS activation sets `baseline`; for a one-command read-only +probe use `BASE_DEMO_ENV=baseline basectl check --ci --format json`. That marker +does not select a service environment or fix other readiness findings. Inspect +each remaining finding; do not declare success from the marker alone. Services +use `services --env dev` separately; staging/prod are non-operational examples. + +For release work, use the canonical [downstream release smoke test](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/downstream-release-smoke-test.md) +and the demo's [release policy](release.md). A contributor PR is not permission +to tag or publish. The [complete command reference](../README.md#complete-command-reference) +remains available after these short paths. diff --git a/docs/release.md b/docs/release.md index 8c6a662..2cd99c1 100644 --- a/docs/release.md +++ b/docs/release.md @@ -119,6 +119,10 @@ silently accepted by the release path. ## Release procedure +For the current train, use the [v0.2.0 owner handoff](v0.2.0-readiness.md). +Implementation/merge authority does not grant tag/publication authority or +waive the minor-release bake and independent-review decision. + 1. Keep post-release work under `## [Unreleased]` in `CHANGELOG.md`. 2. In a release PR, choose the next SemVer version, update `VERSION` and all governed metadata, promote `Unreleased` into a dated version section, and @@ -127,8 +131,12 @@ silently accepted by the release path. 3. Generate or update the demo component row with `bin/base-demo-release-bom-row`, update `.release/release-bom.json` from the coordinated release inputs, then run `bin/base-demo-release-check`, - `bin/base-demo-release-bom-check`, `mise run validate`, and the normal hosted - pull-request checks. Do not try to pin the final merge SHA in this commit. + `bin/base-demo-dependencies --check`, `mise run validate`, and the normal + hosted pull-request checks. The prepared `not_tested` BOM is not publication + proof and must fail the strict BOM gate. After merge, bind a successful + exact-commit compatibility run with `base-demo-release-finalize --evidence-run` + and check the resulting external BOM as shown in the owner handoff. Do not + try to pin the final merge SHA in this commit. 4. After the release PR is merged to `main`, create an annotated tag from the clean merge commit: `git tag -a vX.Y.Z -m "base-demo vX.Y.Z"`. 5. Push the tag. The read-only `verify` job in the `Release Demo` workflow diff --git a/docs/trust-scenarios.md b/docs/trust-scenarios.md new file mode 100644 index 0000000..e19ebe8 --- /dev/null +++ b/docs/trust-scenarios.md @@ -0,0 +1,65 @@ +# Trust and consent, in disposable fixtures + +Run the scenario against clean, exact-commit provider checkouts and an existing +Python interpreter with Base's dependencies. It creates its own home, workspace, +cache and trust store, and cleans them on success or failure. It never edits a +learner's real approval store or IDE settings. The project is deliberately a +tiny shell fixture; uv remains the dependency owner for the normal demo. + +```bash +python3 tests/scenarios/trust.py \ + --base /path/to/base-v1.9.0 \ + --base-commit ac8d294421e1bfc14afa8c6a2a12f1affb5268ee \ + --base-cli /path/to/base-cli-v0.4.3 \ + --bash-libs /path/to/base-bash-libs-v2.1.0 \ + --python /path/to/base-compatible-venv/bin/python +``` + +The stable lane asserts inspection before trust, denied execution (exit 1), +explicit approval, successful execution (exit 0), manifest-change invalidation, +and revocation of a single reviewed approval. Expected failures are assertions, +not errors the learner needs to repair. Each completed boundary prints `PASS`. +Unchanged external scripts are **not** bound by command approval; this is not a +sandbox or a guarantee that every referenced executable is safe. + +## Candidate-only guarantees + +**Base v1.9.0 does not guarantee complete historical-approval revocation.** +Do not infer that the stable example proves cleanup of every previously reviewed +manifest version. Full historical revocation and static-by-default runtime +inspection are demonstrated only against the implemented v1.10 candidate. +The stable lane prints this boundary and does not invoke candidate-only flags. + +The separate advisory lane uses exact Base commit +`5f316aeddc3680b92bd209fcfe652eac020d02d0`. Run the same command with that +checkout/commit and add `--candidate`. A published v1.10 release is not required, +but a moving branch is not an acceptable substitute for the recorded revision. +Both lanes run in [isolated scenario CI](../.github/workflows/scenarios.yml). +Candidate success does not replace stable compatibility evidence. + +The candidate additionally approves multiple manifest versions, revokes them, +and proves that reverting to an older manifest does not restore execution. +It uses a harmless marker-producing interpreter fixture to prove static +inspection does not execute runtime code—even after command approval—and that +`--verify-project-runtime` explicitly permits the probe. + +| Consent | What it permits | What it does not permit | +| --- | --- | --- | +| `trust allow` | Reviewed manifest command execution | Runtime inspection or IDE mutation | +| `--verify-project-runtime` | Project/runtime probes for that check or doctor invocation | Saved command approval or IDE mutation | +| `--allow-project-ide-mutations` | Applying a reviewed project-originated IDE plan | Command approval or runtime verification | +| `--yes` | Ordinary confirmation handling | Any of the independent approvals above | + +IDE behavior is tested through public dry-run previews and an isolated consent +guard test whose mutation delegates are spies. No application install, extension +install or IDE user-settings write is performed. The guard rejects mutation +without its own consent before reaching any delegate. + +Host prerequisite findings remain real and may make `check` return 1 on a +partially configured machine. The scenario asserts the project findings and the +JSON aggregate exit contract separately; a passing scenario is **not** a claim +that the host is ready. Local temporary paths are redacted from assertion +diagnostics. `tests/scenario_harness_test.py` verifies cleanup and redaction. + +For the authoritative boundaries, see Base's +[command-trust policy](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/manifest-command-trust.md). diff --git a/docs/v0.2.0-readiness.md b/docs/v0.2.0-readiness.md new file mode 100644 index 0000000..c4da946 --- /dev/null +++ b/docs/v0.2.0-readiness.md @@ -0,0 +1,120 @@ +# v0.2.0 release-owner handoff + +Status: **preparation only; not approved for publication**. +The core implementation train does not create a version tag, publish assets, +waive independent review, or start a bake window on the owner's behalf. +[#303](https://github.com/basefoundry/base-demo/issues/303) remains open until +the publication and downloaded-asset checks below are complete. + +## Intended scope and inputs + +The release includes the repaired immutable bootstrap, fail-closed governed +BOM/evidence gate, single supported dependency-input record, disposable trust +and workspace scenarios, and short evaluator/adopter/contributor paths. +Record the final disposition of #291–#300 and #256 in #303; do not infer that an +issue is complete from this document. Optional refactoring (#301), native +Windows parity (#302), and future Base Docker-service adoption (#163) are not +release prerequisites. + +Stable compatibility uses `.release/supported-dependencies.json`: Base 1.9.0 +`ac8d294421e1bfc14afa8c6a2a12f1affb5268ee`, base-cli 0.4.3 +`8a93d22156ba75a99965f7c355f867acba630069`, and base-bash-libs 2.1.0 +`36fec50c446dcea8c521a1ba3e7fee2394f169c0`. The exact Base 1.10 candidate +`5f316aeddc3680b92bd209fcfe652eac020d02d0` has a **separate advisory scenario +lane**. Neither candidate success nor PR-head success substitutes for the +stable final-main-commit evidence required by the publication gate. + +Base #2289's shared updater is repaired. Base #2256's scheduled-update +credentials are a separate operational concern; merging this train neither +provisions those credentials nor proves a scheduled bot update succeeded. + +## Owner decisions still required + +This is a minor release with high-impact installer, trust and BOM boundaries. +Apply Base's [stabilization and independent-review policy](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/release-stabilization-policy.md): +the default minor-release bake is **three full working days**, starting from a +named reviewed candidate commit (an equivalent reviewed commit may replace an +RC tag). Record explicit UTC start/end times and any findings in #303. + +The owner must name an independent reviewer or record a waiver naming the +unavailable reviewer, justification, residual risk and follow-up review date. +Automated CI and agent-authored review do not establish independent human +review. No waiver or shortened bake is implied by this handoff. + +## Final candidate checklist + +1. Land the scoped implementation PRs. Prepare a separate reviewed version + change for 0.2.0: `VERSION`, Python/uv project metadata, frontend package and + lock metadata, README release strip, dated changelog, installer release ref, + and prepared BOM self-version/tag/API identity. Keep valid placeholder self + commits and `not_tested` evidence in tracked prepared input; do not pretend + that a tracked file can identify its own eventual merge SHA. + Prepared installer tests compare self pins to the prepared BOM, not a + not-yet-created demo tag. Released Base pins still resolve live; final demo + tag identity is enforced separately by provenance and finalized evidence. +2. Keep the interim public bootstrap until verified v0.2.0 assets actually + exist. Do not link an unavailable asset or claim that the historical + v0.1.0 installer has the new safety properties. Version/readme publication + wording must be reviewed together; `base-demo-release-check` enforces the + governed release strip, not asset availability. +3. Record the resulting **full merge commit** and exact successful `tests.yml` + push/manual run. It must include `validate` on macos-14, `validate-ubuntu` on + ubuntu-24.04, and `validate-base-cli-source`; full macOS validation must + execute Go/Java and all four live HTTP services. Record both scenario lane + results separately at that same demo commit. Local Xcode-license failures + or optional-tool skips are not equivalent hosted evidence. +4. Use the read-only artifact rehearsal below. Rehearse a fresh install in an + isolated home/workspace against the exact output; verify reused matching + checkouts, dirty/divergent rejection and checksum mismatch. The installer + deliberately does not upgrade a divergent checkout in place. Review any + upgrade manually and rerun the exact-pin installer; do not reset user work. + Unit/mock installer fixtures supplement, but do not replace, this final + artifact install/upgrade rehearsal. +5. Complete the bake and review/waiver record. Resolve or explicitly disposition + every finding. Only then request/record human permission to create and push + the annotated v0.2.0 tag. No existing published tag or asset may be replaced. + +## Read-only artifact rehearsal + +From the clean, reviewed final candidate checkout, set `candidate_commit` to +its full SHA and `evidence_run` to the successful exact-commit run ID. These +commands write only a new external temporary artifact directory; they do not +tag, install, or publish. Authentication needs read access to Actions evidence. + +```bash +test "$(git rev-parse HEAD)" = "$candidate_commit" +test -z "$(git status --porcelain)" +bin/base-demo-release-check +bin/base-demo-dependencies --check --verify-refs +rehearsal_dir="$(mktemp -d)" +bin/base-demo-release-finalize --commit "$candidate_commit" \ + --evidence-run "$evidence_run" --output-dir "$rehearsal_dir" +bin/base-demo-release-finalize --commit "$candidate_commit" \ + --evidence-run "$evidence_run" --verify-dir "$rehearsal_dir" +BASE_DEMO_RELEASE_BOM_PATH="$rehearsal_dir/release-bom.json" \ + BASE_DEMO_RELEASE_BOM_EXPECTED_COMMIT="$candidate_commit" \ + bin/base-demo-release-bom-check +(cd "$rehearsal_dir" && shasum -a 256 -c release-bom.sha256 install.sh.sha256) +BASE_DEMO_TEST_BOOTSTRAP="$rehearsal_dir/install.sh" bats tests/install_test.bats +``` + +The finalizer verifies live server-owned run, job, platform, commit and input +identity **before writing assets** when `--evidence-run` is supplied. Omitting +that flag is only a self-identity fixture operation, not compatibility proof. +The strict BOM publication checker is expected to reject the tracked +`not_tested` input. Validate the finalized external BOM instead. + +Because the published version is still 0.1.0 during core-train preparation, +any rehearsal at that stage tests mechanics only. It must not be published as +another 0.1.0 release or called the final 0.2.0 candidate. + +## After separately authorized publication + +Download `release-bom.json`, `release-bom.sha256`, `install.sh` and +`install.sh.sha256` from the new release into a fresh directory. Independently +verify checksums, annotated tag target, finalized self-identity and exact +provider inputs. Re-run the strict BOM gate and inspect the downloaded +installer pins. Update the README public URL/digest and consumed versions to +those actual assets, run `tests/public_install_test.sh`, and rehearse that exact +public command. Record the immutable demo identity for the Base-owned +compatibility BOM. Only then close #303 and mark the release complete. diff --git a/docs/workspace-scenarios.md b/docs/workspace-scenarios.md new file mode 100644 index 0000000..a98fc00 --- /dev/null +++ b/docs/workspace-scenarios.md @@ -0,0 +1,50 @@ +# Workspace validation and targeted recovery + +This compact scenario creates disposable healthy, failing, untrusted, no-test, +missing-required and undeclared peers. It uses the same isolated HOME, cache, +provider checks and cleanup as the [trust scenario](trust-scenarios.md). No +clone, setup, repository initialization or learner-state mutation is performed. +Generated recovery commands are inspected; only a reviewed fixture-local trust +command is executed. Expected failures are assertions, not repair requests. + +Run with clean exact-provider checkouts and an existing Base-compatible Python: + +```bash +python3 tests/scenarios/workspace.py \ + --base /path/to/base-v1.9.0 \ + --base-commit ac8d294421e1bfc14afa8c6a2a12f1affb5268ee \ + --base-cli /path/to/base-cli-v0.4.3 \ + --bash-libs /path/to/base-bash-libs-v2.1.0 \ + --python /path/to/base-compatible-venv/bin/python +``` + +Base 1.9 checks the existing status, onboarding and agent-brief reports, their +workspace identity, missing-peer reporting and read-only behavior. It prints +an explicit supported-version boundary without invoking newer flags. + +For the implemented 1.10 candidate, select its checkout and exact commit +`5f316aeddc3680b92bd209fcfe652eac020d02d0`, and add `--candidate`. +[Scenario CI](../.github/workflows/scenarios.yml) runs both lanes. This is +advisory candidate evidence, not a stable-release compatibility claim. + +The candidate asserts: + +- Onboarding actions are ordered clone, setup, trust, verify; final verification + retains the workspace and workspace-manifest identity. Agent-brief actions + instead follow repository inventory order and target each exact checkout. +- Undeclared peers are inventoried but not included in declared workspace tests. +- A generated digest-bound trust command targets its original workspace even + when invoked from another same-named project. Changed manifest bytes reject + that saved command with exit 2. +- `workspace test --projects healthy` executes only that selected checkout. + Selection filters declaration order; the comma-list does not reorder tests. +- A failing command and untrusted command are failures, no test command is a + skip, and missing required peers fail the aggregate. Aggregate failure exits + 1; successful selection exits 0. `--fail-fast` skips later selected peers. +- Two selected aliases of one manifest are rejected (exit 2); selecting one + alias runs the intended checkout once. + +Each completed assertion group prints `PASS`; fixtures are removed on success +or exception. There is no second application stack and no host-readiness claim. +See the exact candidate's [workspace contract](https://github.com/basefoundry/base/blob/5f316aeddc3680b92bd209fcfe652eac020d02d0/docs/workspace-manifest.md) +for the authoritative API. diff --git a/tests/install_test.bats b/tests/install_test.bats index eb58d7c..d6cab9b 100644 --- a/tests/install_test.bats +++ b/tests/install_test.bats @@ -10,6 +10,7 @@ setup() { TEST_GIT_LOG="$TEST_TMPDIR/git.log" TEST_BASE_COMMIT="$(sed -n 's/^BASE_RELEASE_COMMIT="${BASE_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" TEST_PROJECT_COMMIT="$(sed -n 's/^PROJECT_RELEASE_COMMIT="${PROJECT_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" + TEST_PROJECT_REF="$(sed -n 's/^PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" TEST_BASE_REF="$(sed -n 's/^BASE_RELEASE_REF="${BASE_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_BOOTSTRAP")" mkdir -p "$TEST_FAKE_BIN" @@ -185,10 +186,10 @@ installer_sha256() { [ "$status" -eq 0 ] [[ "$output" == *"Installing pinned Base release '$TEST_BASE_REF'"* ]] - [[ "$output" == *"Cloning pinned base-demo release 'v0.1.0'"* ]] + [[ "$output" == *"Cloning pinned base-demo release '$TEST_PROJECT_REF'"* ]] [[ "$output" == *"Verified pinned Base commit $TEST_BASE_COMMIT"* ]] [[ "$output" == *"Verified pinned base-demo commit $TEST_PROJECT_COMMIT"* ]] - grep -Fq -- "clone --depth 1 --branch v0.1.0 https://github.com/basefoundry/base-demo.git" "$TEST_GIT_LOG" + grep -Fq -- "clone --depth 1 --branch $TEST_PROJECT_REF https://github.com/basefoundry/base-demo.git" "$TEST_GIT_LOG" [ -f "$TEST_MARKER" ] } diff --git a/tests/journeys_docs_test.py b/tests/journeys_docs_test.py new file mode 100644 index 0000000..31b9f10 --- /dev/null +++ b/tests/journeys_docs_test.py @@ -0,0 +1,37 @@ +"""Keep short entry points and AI guidance aligned without pretending to run setup.""" +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class JourneyDocsTests(unittest.TestCase): + def test_entry_points_precede_inventory(self): + readme = (ROOT / "README.md").read_text() + self.assertLess(readme.index("## Start with your goal"), readme.index("## Complete command reference")) + guide = (ROOT / "docs/first-success.md").read_text() + for title in ("Evaluate Base", "Adopt Base in a project", "Contribute to base-demo"): + section = guide.split("## " + title + "\n", 1)[1].split("\n## ", 1)[0] + for marker in ("**Prerequisite:", "**Done:", "**Safe failure/recovery:"): + self.assertIn(marker, section) + for page in ("adopter-golden-path", "first-run-troubleshooting", "json-output-quickstart", + "downstream-release-smoke-test"): + self.assertIn(f"/docs/{page}.md", guide) + for target in re.findall(r"\]\(([^)]+)\)", guide): + if not target.startswith("https://"): + self.assertTrue((ROOT / "docs" / target.split("#")[0]).exists(), target) + + def test_scope_and_health_marker_agree(self): + overview = (ROOT / ".ai-context/overview.md").read_text() + guide = (ROOT / "docs/first-success.md").read_text() + self.assertNotIn("declares every current Base contract", overview) + for text in (overview, guide): + self.assertIn("curated representative subset", text) + self.assertIn("BASE_DEMO_ENV", text) + self.assertIn("baseline", text) + self.assertIn("services --env dev", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/release_bom_test.py b/tests/release_bom_test.py index e7ead12..6d5c963 100644 --- a/tests/release_bom_test.py +++ b/tests/release_bom_test.py @@ -155,6 +155,26 @@ def test_finalizer_binds_only_verified_evidence(self): build(root, "a" * 40, "123") self.assertEqual(path.read_bytes(), original) + def test_next_version_can_be_prepared_without_certifying_or_publishing_it(self): + build = runpy.run_path(str(ROOT / "bin/base-demo-release-finalize"))["build_assets"] + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / ".release").mkdir() + (root / "VERSION").write_text("0.2.0\n") + (root / "install.sh").write_bytes((ROOT / "install.sh").read_bytes()) + self.bom["release"].update(version="0.2.0", tag="v0.2.0") + self.bom["components"][0].update(version="0.2.0", tag="v0.2.0") + for row in self.bom["components"] + self.bom["combinations"]: + row.update(result="not_tested", evidence="pending") + (root / ".release/release-bom.json").write_text(json.dumps(self.bom)) + tag, content, installer, _ = build(root, "a" * 40) + self.assertEqual(tag, "v0.2.0") + self.assertIn(b'PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-v0.2.0}"', installer) + self.assertTrue(all(r["result"] == "not_tested" for r in json.loads(content)["components"])) + with self.assertRaises(ValueError): + demo_bom.check(json.loads(content), self.inputs, "0.2.0", "a" * 40) + self.assertFalse((root / ".git").exists()) + if __name__ == "__main__": unittest.main() diff --git a/tests/release_test.bats b/tests/release_test.bats index 3f9733a..b3f7826 100644 --- a/tests/release_test.bats +++ b/tests/release_test.bats @@ -2,6 +2,7 @@ setup() { TEST_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd -P)" + TEST_TAG="v$(< "$TEST_ROOT/VERSION")" TEST_TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/base-demo-release-test.XXXXXX")" TEST_REPO="$TEST_TMPDIR/repo" mkdir -p "$TEST_REPO" @@ -107,13 +108,13 @@ resolve_release_commit() { git -C "$TEST_REPO" add VERSION install.sh .release/release-bom.json git -C "$TEST_REPO" commit -q -m "prepare release inputs" target_commit="$(git -C "$TEST_REPO" rev-parse HEAD)" - git -C "$TEST_REPO" tag -a v0.1.0 -m "base-demo v0.1.0" "$target_commit" + git -C "$TEST_REPO" tag -a "$TEST_TAG" -m "base-demo $TEST_TAG" "$target_commit" output_dir="$TEST_TMPDIR/finalized" run "$TEST_ROOT/bin/base-demo-release-provenance" \ --repo "$TEST_REPO" \ --main-ref main \ - v0.1.0 \ + "$TEST_TAG" \ "$target_commit" [ "$status" -eq 0 ] @@ -123,7 +124,7 @@ resolve_release_commit() { --output-dir "$output_dir" [ "$status" -eq 0 ] - [[ "$output" == *"v0.1.0"* ]] + [[ "$output" == *"$TEST_TAG"* ]] run python3 -c ' import json import sys @@ -138,7 +139,7 @@ rows = [ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] ' "$output_dir/release-bom.json" "$target_commit" [ "$status" -eq 0 ] - grep -Fq 'PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-v0.1.0}"' "$output_dir/install.sh" + grep -Fq "PROJECT_RELEASE_REF=\"\${PROJECT_RELEASE_REF:-$TEST_TAG}\"" "$output_dir/install.sh" grep -Fq "PROJECT_RELEASE_COMMIT=\"\${PROJECT_RELEASE_COMMIT:-$target_commit}\"" "$output_dir/install.sh" [ -x "$output_dir/install.sh" ] [ "$(shasum -a 256 "$TEST_REPO/install.sh" | awk '{print $1}')" = "$original_installer_sha" ] @@ -182,7 +183,7 @@ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] [ ! -e "$TEST_TMPDIR/invalid/release-bom.json" ] } -@test "install release pins resolve refs to their target commits" { +@test "prepared installer self pin matches BOM while released Base ref resolves exactly" { project_ref="$(sed -n 's/^PROJECT_RELEASE_REF="${PROJECT_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" project_pin="$(sed -n 's/^PROJECT_RELEASE_COMMIT="${PROJECT_RELEASE_COMMIT:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" base_ref="$(sed -n 's/^BASE_RELEASE_REF="${BASE_RELEASE_REF:-\([^}]*\)}"$/\1/p' "$TEST_ROOT/install.sh")" @@ -193,9 +194,21 @@ assert len(rows) == 1 and rows[0]["commit"] == sys.argv[2] [ -n "$base_ref" ] [ -n "$base_pin" ] - run resolve_release_commit "$TEST_ROOT" "${PROJECT_REPO_URL:-https://github.com/basefoundry/base-demo.git}" "$project_ref" + # The next demo tag does not exist during its version PR, and the tracked + # self pin cannot name its own eventual merge SHA. Do not demand a published + # tag here. Finalized artifact identity is checked by provenance + live BOM + # evidence after merge; this test only checks the prepared input contract. + run python3 -c ' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +bom = json.loads((root / ".release/release-bom.json").read_text()) +tag = "v" + (root / "VERSION").read_text().strip() +assert sys.argv[2] == tag == bom["release"]["tag"] +assert sys.argv[3] == bom["release"]["commit"] +rows = [r for r in bom["components"] if r["repository"] == "basefoundry/base-demo"] +assert len(rows) == 1 and rows[0]["tag"] == tag and rows[0]["commit"] == sys.argv[3] +' "$TEST_ROOT" "$project_ref" "$project_pin" [ "$status" -eq 0 ] - [ "$output" = "$project_pin" ] run resolve_release_commit "$TEST_ROOT/../base" "${BASE_REPO_URL:-https://github.com/basefoundry/base.git}" "$base_ref" [ "$status" -eq 0 ] diff --git a/tests/scenario_harness_test.py b/tests/scenario_harness_test.py new file mode 100644 index 0000000..e286c83 --- /dev/null +++ b/tests/scenario_harness_test.py @@ -0,0 +1,41 @@ +"""Verify isolation, redaction and failure cleanup without external providers.""" +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest + +sys.path.insert(0, str(Path(__file__).parent / "scenarios")) +from fixture import Fixture + + +class HarnessTests(unittest.TestCase): + def args(self): + return SimpleNamespace(python=Path(sys.executable), base=Path("/unused"), + base_cli=Path("/unused-cli"), bash_libs=Path("/unused-libs")) + + def test_failure_cleans_disposable_home(self): + root = None + with self.assertRaisesRegex(RuntimeError, "expected failure"): + with Fixture(self.args()) as fixture: + root = fixture.root + self.assertEqual(Path(fixture.env["HOME"]), fixture.home) + self.assertNotIn("GH_TOKEN", fixture.env) + self.assertNotIn("BASH_ENV", fixture.env) + raise RuntimeError("expected failure") + self.assertFalse(root.exists()) + + def test_cli_failures_redact_fixture_root(self): + with Fixture(self.args()) as fixture: + fixture.args.base = fixture.root / "provider" + launcher = fixture.args.base / "bin/basectl" + launcher.parent.mkdir(parents=True) + launcher.write_text('#!/bin/sh\nprintf "%s\\n" "$HOME"\nexit 1\n') + launcher.chmod(0o755) + with self.assertRaises(AssertionError) as failure: + fixture.run("check", "--manifest", fixture.root / "project.yaml") + self.assertNotIn(str(fixture.root), str(failure.exception)) + self.assertIn("", str(failure.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scenarios/fixture.py b/tests/scenarios/fixture.py new file mode 100644 index 0000000..8b5084c --- /dev/null +++ b/tests/scenarios/fixture.py @@ -0,0 +1,89 @@ +"""Disposable public-CLI scenario support; never inherit learner Base state.""" +from __future__ import annotations + +import argparse +import json +import os +import shlex +from pathlib import Path +import subprocess +import tempfile + + +def arguments(description): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--base-commit", required=True) + parser.add_argument("--base-cli", type=Path, required=True) + parser.add_argument("--bash-libs", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True, help="existing Base-compatible Python interpreter") + parser.add_argument("--candidate", action="store_true") + args = parser.parse_args() + for name, path, expected in ( + ("base", args.base, args.base_commit), + ("base-cli", args.base_cli, "8a93d22156ba75a99965f7c355f867acba630069"), + ("base-bash-libs", args.bash_libs, "36fec50c446dcea8c521a1ba3e7fee2394f169c0"), + ): + actual = subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + dirty = subprocess.check_output(["git", "-C", str(path), "status", "--porcelain"], text=True) + if actual != expected or dirty: + parser.error(f"{name} must be a clean exact-commit checkout") + if not args.candidate and args.base_commit != "ac8d294421e1bfc14afa8c6a2a12f1affb5268ee": + parser.error("stable lane requires supported Base v1.9.0; pass --candidate for separately recorded source") + return args + + +class Fixture: + def __init__(self, args): + self.args = args + self.temporary = tempfile.TemporaryDirectory(prefix="base-demo-scenario-") + self.root = Path(self.temporary.name).resolve() + self.home = self.root / "home" + self.workspace = self.root / "workspace" + self.home.mkdir() + self.workspace.mkdir() + # A proxy selects the existing interpreter without writing to its venv. + venv = self.home / ".base.d/base/.venv" + (venv / "bin").mkdir(parents=True) + proxy = venv / "bin/python" + proxy.write_text("#!/bin/sh\nexec " + shlex.quote(str(args.python.absolute())) + ' "$@"\n') + proxy.chmod(0o755) + (venv / "pyvenv.cfg").write_text("scenario = isolated\n") + self.env = { + "HOME": str(self.home), "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "BASE_HOME": str(args.base.resolve()), "BASE_SETUP_VENV_DIR": str(venv), + "BASE_CLI_SOURCE_DIR": str(args.base_cli.resolve() / "lib/python"), + "BASE_BASH_LIBS_DIR": str(args.bash_libs.resolve() / "lib/bash"), + "BASE_CACHE_DIR": str(self.root / "cache"), "BASE_SETUP_NOTIFY": "false", + "XDG_CONFIG_HOME": str(self.home / ".config"), "XDG_CACHE_HOME": str(self.root / "xdg-cache"), + "TMPDIR": str(self.root), "NO_COLOR": "1", "TERM": "dumb", + } + + def __enter__(self): + return self + + def __exit__(self, *_): + self.temporary.cleanup() + + def project(self, name="demo", extra=""): + root = self.workspace / name + root.mkdir(exist_ok=True) + command = 'printf "executed\\n" > "$BASE_PROJECT_ROOT/executed"' + (root / "base_manifest.yaml").write_text( + f"project:\n name: {name}\nartifacts: []\ncommands:\n hello: {json.dumps(command)}\n" + extra + ) + return root + + def run(self, *args, expected=0, cwd=None): + result = subprocess.run([str(self.args.base.resolve() / "bin/basectl"), *map(str, args)], + env=self.env, cwd=cwd or self.workspace, capture_output=True, text=True, timeout=90) + allowed = expected if isinstance(expected, tuple) else (expected,) + if result.returncode not in allowed: + # Only redacted fixture-local diagnostics are emitted on failure. + output = (result.stdout + result.stderr).replace(str(self.root), "") + command = str(args).replace(str(self.root), "") + raise AssertionError(f"{command}: expected exit {expected}, got {result.returncode}\n{output}") + return result + + def no_ide_settings(self): + assert not list(self.home.rglob("settings.json")), "fixture unexpectedly wrote IDE settings" diff --git a/tests/scenarios/ide_guard.py b/tests/scenarios/ide_guard.py new file mode 100644 index 0000000..88c797f --- /dev/null +++ b/tests/scenarios/ide_guard.py @@ -0,0 +1,36 @@ +"""Exercise the IDE consent guard with all mutation delegates replaced by spies.""" +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from base_cli_adapters.config import UserConfig, UserIdeConfig +from base_setup.errors import ArtifactError +from base_setup.manifest import BaseManifest, IdeConfig +from base_setup import setup_reconcile + + +def main(): + context = SimpleNamespace(log=Mock(), yes=True, + user_config=UserConfig(raw={}, ide=UserIdeConfig(enabled=None, preferences={}))) + defaults = BaseManifest(path=Path("defaults.yaml"), project_name="defaults", brewfile=None, artifacts=()) + project = BaseManifest(path=Path("project.yaml"), project_name="fixture", brewfile=None, artifacts=(), + ide={"vscode": IdeConfig(install=False, extensions=(), settings={"editor.formatOnSave": True})}) + names = ("reconcile_brewfile", "reconcile_mise", "reconcile_ide_installs", "reconcile_ide_extensions", + "reconcile_ide_settings", "reconcile_uv_project", "reconcile_artifacts") + delegates = {name: Mock() for name in names} + with patch.multiple(setup_reconcile, **delegates): + try: + setup_reconcile.reconcile_manifest(context, defaults, project, dry_run=False) + except ArtifactError as error: + assert "--allow-project-ide-mutations" in str(error) + assert "--yes" in str(error) + else: + raise AssertionError("IDE mutation was not denied") + assert not any(spy.called for spy in delegates.values()) + setup_reconcile.reconcile_manifest(context, defaults, project, dry_run=True) + assert delegates["reconcile_ide_settings"].call_args.kwargs["dry_run"] is True + print("PASS: IDE mutation guard denies without its own consent; preview delegates are dry-run only") + + +if __name__ == "__main__": + main() diff --git a/tests/scenarios/journeys.py b/tests/scenarios/journeys.py new file mode 100644 index 0000000..c51b977 --- /dev/null +++ b/tests/scenarios/journeys.py @@ -0,0 +1,38 @@ +"""Rehearse the evaluator command sequence in a disposable minimal project.""" +from pathlib import Path +import shutil + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + source = Path(__file__).resolve().parents[2] + with Fixture(args) as fixture: + root = fixture.project("base-demo") + (root / "base_manifest.yaml").write_text( + "project:\n name: base-demo\nartifacts: []\n" + "commands:\n hello: ./src/hello.sh\ntest:\n command: ./src/hello.sh\n") + (root / "src").mkdir() + shutil.copy2(source / "src/hello.sh", root / "src/hello.sh") + (root / ".ai-context").mkdir() + (root / ".ai-context/overview.md").write_text("# Disposable evaluator handoff\n") + workspace = ("--workspace", fixture.workspace) + fixture.run("run", "base-demo", "--list", *workspace, cwd=root) + fixture.run("test", "base-demo", "--dry-run", *workspace, cwd=root) + fixture.run("trust", "status", "base-demo", "--workspace", fixture.workspace, cwd=root) + fixture.run("run", "base-demo", "hello", *workspace, cwd=root, expected=1) + fixture.run("trust", "allow", "base-demo", "--workspace", fixture.workspace, cwd=root) + fixture.env["BASE_DEMO_ENV"] = "baseline" + output = fixture.run("run", "base-demo", "hello", *workspace, cwd=root).stdout + for expected in ("hello from base-demo", "BASE_PROJECT=base-demo", "BASE_DEMO_ENV=baseline"): + assert expected in output + handoff = fixture.run("export-context", "base-demo", *workspace, "--format", "markdown", "--print", cwd=root).stdout + assert "Disposable evaluator handoff" in handoff + fixture.no_ide_settings() + print("PASS: evaluator inspection, safe denial/recovery, actual hello script and context handoff") + print("SCOPE: minimal project rehearsal; full setup/toolchain validation remains the hosted demo gate") + + +if __name__ == "__main__": + main() diff --git a/tests/scenarios/trust.py b/tests/scenarios/trust.py new file mode 100644 index 0000000..0f44271 --- /dev/null +++ b/tests/scenarios/trust.py @@ -0,0 +1,91 @@ +"""Assert command approval, invalidation, revoke and independent consent.""" +from __future__ import annotations + +import json +from pathlib import Path +import shlex +import subprocess + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + with Fixture(args) as fixture: + project = fixture.project() + manifest = project / "base_manifest.yaml" + original = manifest.read_text() + workspace = ("--workspace", fixture.workspace) + fixture.run("run", "demo", "--list", *workspace) + denied = fixture.run("run", "demo", "hello", *workspace, expected=1) + assert "trust" in (denied.stdout + denied.stderr).lower() + assert not (project / "executed").exists() + fixture.run("trust", "allow", "demo", *workspace) + fixture.run("run", "demo", "hello", *workspace) + assert (project / "executed").read_text() == "executed\n" + (project / "executed").unlink() + manifest.write_text(original + "\n# changed after review\n") + fixture.run("run", "demo", "hello", *workspace, expected=1) + assert not (project / "executed").exists() + if args.candidate: + # Historical multi-approval cleanup is the implemented v1.10 + # contract, not a guarantee of the older stable release. + fixture.run("trust", "allow", "demo", *workspace) + fixture.run("trust", "revoke", "demo", *workspace) + fixture.run("run", "demo", "hello", *workspace, expected=1) + # Reverting the manifest must not resurrect the older approval. + manifest.write_text(original) + fixture.run("run", "demo", "hello", *workspace, expected=1) + scope = "historical approvals" if args.candidate else "single reviewed approval" + print(f"PASS: inspection -> denial -> approval -> execution -> invalidation -> revoke ({scope})") + + # Preview only: do not authorize or apply any IDE mutation. + manifest.write_text(original + "ide:\n vscode:\n settings:\n editor.formatOnSave: true\n") + preview = fixture.run("setup", "demo", "--manifest", manifest, "--dry-run", "--yes") + assert "IDE" in (preview.stdout + preview.stderr) or "settings" in (preview.stdout + preview.stderr) + fixture.run("run", "demo", "hello", *workspace, expected=1) + fixture.no_ide_settings() + print("PASS: --yes plus setup preview does not approve manifest commands or write IDE settings") + env = dict(fixture.env) + env["PYTHONPATH"] = str(args.base_cli.resolve() / "lib/python") + ":" + str(args.base.resolve() / "cli/python") + guard = subprocess.run([str(args.python.absolute()), str(Path(__file__).with_name("ide_guard.py"))], + env=env, cwd=fixture.root, capture_output=True, text=True, timeout=30) + assert guard.returncode == 0, (guard.stdout + guard.stderr).replace(str(fixture.root), "") + print(guard.stdout.strip()) + fixture.no_ide_settings() + + if args.candidate: + help_result = fixture.run("check", "--help") + assert "--verify-project-runtime" in help_result.stdout, "candidate lacks runtime verification contract" + manifest.write_text(original + "python: {}\n") + runtime = project / ".venv/bin/python" + runtime.parent.mkdir(parents=True) + marker = project / "runtime-probed" + runtime.write_text("#!/bin/sh\n" + f"printf probe >> {shlex.quote(str(marker))}\n" + + f"exec {shlex.quote(str(args.python.absolute()))} \"$@\"\n") + runtime.chmod(0o755) + (project / ".venv/pyvenv.cfg").write_text("scenario = project-runtime\n") + fixture.run("trust", "allow", "demo", *workspace) + static = fixture.run("check", "--ci", "--manifest", manifest, "--format", "json", expected=(0, 1)) + assert "unverified" in static.stdout + payload = json.loads(static.stdout) + assert payload["project_checks"]["status"] == "warn" + # Host prerequisites remain real diagnostics, separate from this + # disposable project contract. Assert their aggregate exit exactly. + assert static.returncode == (1 if payload["status"] == "error" else 0) + assert not marker.exists(), "saved command approval unexpectedly permitted a runtime probe" + verified = fixture.run("check", "--ci", "--manifest", manifest, "--verify-project-runtime", "--format", "json", + expected=(0, 1)) + payload = json.loads(verified.stdout) + assert payload["project_checks"]["status"] != "error", payload["project_checks"] + assert verified.returncode == (1 if payload["status"] == "error" else 0) + assert marker.exists(), "explicit verification did not probe the runtime" + fixture.no_ide_settings() + print("PASS: saved command approval does not grant runtime inspection; explicit verification probes only the fixture") + else: + print("BOUNDARY: full historical revocation and --verify-project-runtime require the implemented v1.10 candidate") + print(f"PASS: isolated trust scenario at Base {args.base_commit}; fixture cleanup on success and failure") + + +if __name__ == "__main__": + main() diff --git a/tests/scenarios/workspace.py b/tests/scenarios/workspace.py new file mode 100644 index 0000000..0e28f9e --- /dev/null +++ b/tests/scenarios/workspace.py @@ -0,0 +1,130 @@ +"""Assert disposable workspace inventory, checkout targeting and selected tests.""" +from __future__ import annotations + +import json +import shlex + +from fixture import Fixture, arguments + + +def main(): + args = arguments(__doc__) + with Fixture(args) as fixture: + roots = {} + for name in ("healthy", "failing", "untrusted", "no-test", "undeclared"): + root = fixture.project(name, "python: {}\n" if name == "no-test" else "") + roots[name] = root + if name != "no-test": + command = 'printf "tested\\n" >> "$BASE_PROJECT_ROOT/tested"' + if name == "failing": + command += "; exit 7" + with (root / "base_manifest.yaml").open("a") as stream: + stream.write("test:\n command: " + json.dumps(command) + "\n") + manifest = fixture.root / "team workspace.yaml" + manifest.write_text("schema_version: 1\nworkspace:\n name: scenario\nrepos:\n" + " - name: healthy\n - name: failing\n - name: untrusted\n" + " - name: no-test\n - name: missing\n" + " url: https://github.com/example/missing.git\n") + options = ("--workspace", fixture.workspace, "--manifest", manifest) + for command in ("status", "onboarding", "agent-brief"): + result = fixture.run("workspace", command, *options, "--format", "json", expected=(0, 1)) + payload = json.loads(result.stdout) + assert payload["workspace"] == str(fixture.workspace) + assert payload["workspace_manifest"]["path"] == str(manifest) + assert "missing" in result.stdout + assert not list(fixture.workspace.rglob("tested")), "read-only inventory executed a test" + if not args.candidate: + print("PASS: stable workspace status, onboarding and agent-brief are read-only") + print("BOUNDARY: selected workspace tests, expanded inventory and targeted next_actions require the v1.10 candidate") + return + + assert "test" in fixture.run("workspace", "--help").stdout + reports = {} + for command in ("onboarding", "agent-brief"): + reports[command] = json.loads(fixture.run("workspace", command, *options, "--format", "json").stdout) + actions = reports[command]["next_actions"] + assert [a["order"] for a in actions] == list(range(1, len(actions) + 1)) + actions = reports["onboarding"]["next_actions"] + assert [a["description"] for a in actions] == [ + "Clone missing repos", "Set up unconfigured projects", "Trust new manifests", + "Review runtimes and verify workspace health", + ] + commands = [c for a in actions for c in a["commands"]] + verify = shlex.split(commands[-1]) + assert verify[verify.index("--workspace") + 1] == str(fixture.workspace) + assert verify[verify.index("--manifest") + 1] == str(manifest) + actions = reports["agent-brief"]["next_actions"] + names = ["healthy", "failing", "untrusted", "no-test", "missing", "undeclared"] + assert [a["description"] for a in actions] == [f"Prepare repository {name}" for name in names] + for name, action in zip(names, actions): + assert action["commands"] + assert all(str(fixture.workspace / name) in shlex.split(c) for c in action["commands"]) + inventory = {r["repository"]: r for r in reports["agent-brief"]["repositories"]} + assert not inventory["undeclared"]["expected"] + assert inventory["missing"]["required"] and inventory["missing"]["discovery_status"] == "missing" + + # The generated, digest-bound trust command must retain its workspace + # even when invoked from another checkout with the same project name. + other = fixture.root / "other workspace" / "healthy" + other.mkdir(parents=True) + (other / "base_manifest.yaml").write_text((roots["healthy"] / "base_manifest.yaml").read_text()) + entry = next(r for r in reports["onboarding"]["repositories"] if r["repository"] == "healthy") + words = shlex.split(entry["trust_command"]) + assert words[:3] == ["basectl", "trust", "allow"] + assert words[words.index("--workspace") + 1] == str(fixture.workspace) + fixture.run(*words[1:], cwd=other) + selected = fixture.run("trust", "status", "healthy", "--workspace", fixture.workspace) + unselected = fixture.run("trust", "status", "healthy", "--workspace", other.parent) + assert "\tallowed\t" in selected.stdout and "\tblocked\t" in unselected.stdout + with (roots["healthy"] / "base_manifest.yaml").open("a") as stream: + stream.write("\n# changed after recovery guidance\n") + stale = fixture.run(*words[1:], cwd=other, expected=2) + assert "SHA-256" in stale.stdout + stale.stderr + for name in ("healthy", "failing"): + fixture.run("trust", "allow", name, "--workspace", fixture.workspace) + print("PASS: ordered recovery actions bind the reviewed checkout and reject stale manifest evidence") + + # Selection is a filter, not an execution-order override. Put the + # failing peer first in declaration order for the fail-fast example. + manifest.write_text(manifest.read_text().replace( + " - name: healthy\n - name: failing\n", " - name: failing\n - name: healthy\n")) + + def tests(selection=None, fail_fast=False, expected=0): + extra = ("--projects", selection) if selection else () + if fail_fast: + extra += ("--fail-fast",) + return json.loads(fixture.run("workspace", "test", *options, *extra, + "--format", "json", expected=expected).stdout) + + result = tests("healthy") + assert result["counts"] == {"passed": 1, "failed": 0, "skipped": 0} + assert result["projects"][0]["path"] == str(roots["healthy"]) + assert (roots["healthy"] / "tested").read_text() == "tested\n" + assert len(list(fixture.root.rglob("tested"))) == 1 + (roots["healthy"] / "tested").unlink() + result = tests("failing,untrusted,no-test", expected=1) + assert result["counts"] == {"passed": 0, "failed": 2, "skipped": 1} + assert not (roots["untrusted"] / "tested").exists() + result = tests("failing,healthy", fail_fast=True, expected=1) + assert result["counts"] == {"passed": 0, "failed": 1, "skipped": 1} + assert not (roots["healthy"] / "tested").exists() + result = tests(expected=1) + assert result["counts"] == {"passed": 1, "failed": 3, "skipped": 1} + assert not (roots["undeclared"] / "tested").exists() + assert not (other / "tested").exists() + print("PASS: selected execution, failure, untrusted denial, skip, fail-fast and missing-required aggregate exit") + + alias = fixture.workspace / "alias" + alias.symlink_to(roots["healthy"], target_is_directory=True) + with manifest.open("a") as stream: + stream.write(" - name: alias\n") + fixture.run("workspace", "test", *options, "--projects", "healthy,alias", expected=2) + result = tests("alias") + assert result["counts"]["passed"] == 1 + assert result["projects"][0]["repository"] == "alias" + fixture.no_ide_settings() + print(f"PASS: alias selection cannot double-run one manifest; workspace scenario at Base {args.base_commit}") + + +if __name__ == "__main__": + main() diff --git a/tests/validate.sh b/tests/validate.sh index 50d5832..5314883 100755 --- a/tests/validate.sh +++ b/tests/validate.sh @@ -168,6 +168,8 @@ fi python3 tests/release_bom_test.py || exit 1 python3 tests/dependency_inputs_test.py || exit 1 +python3 tests/scenario_harness_test.py || exit 1 +python3 tests/journeys_docs_test.py || exit 1 if ! bash tests/public_install_test.sh; then printf 'The exact public bootstrap input failed its contract tests.\n' >&2