From 5e0e8beddcbe0eed7f6dfc7ba5951f93810560af Mon Sep 17 00:00:00 2001 From: David J Parrott Date: Fri, 21 Aug 2026 19:24:50 -0400 Subject: [PATCH] fix(cluster-tool): verify the Solana .so at deploy The harness never builds the Solana program and `target/` is gitignored, so nothing tied the binary on disk to the sources beside it. A `git checkout` or rebase moved the sources while the `.so` stayed put, and the stale binary deployed silently. It did not fail as a build-provenance problem. The validator loaded it, the program mis-executed at instruction entry, and the run showed `consumed 427 of 200000 compute units` / `Access violation writing 1 bytes at address 0x32` during the outpost's init-PDAs step -- which reads as a program bug, and was diagnosed as one. `anchor-build-strict.mjs` now stamps a manifest beside each emitted `.so` (arch, byte length, sha256, and the build checkout's `git describe`). `assertProgramSoFile` checks the binary against it and fails at startup naming the mismatch. Both halves of that check are load-bearing. The sha alone proves only that the `.so` and the manifest agree, and the two are written together into the same gitignored directory -- a branch switch leaves the pair intact and mutually consistent, so a sha-only check passes on exactly the incident that motivated it. `sourceDescribe` closes that gap by comparing the build's checkout against the current one. A DIRTY build is reported, not rejected: two different dirty trees at one commit describe identically, so the stamp can only mark the binary unverifiable. Refusing to run would block ordinary local iteration for a guarantee it cannot give either way. Verification lives at `runStart` -- the one path that actually loads the binary on this host -- and NOT in the shared `resolvePrograms`, which also feeds the `start.sh` renderer. That renderer emits a script which runs later and often elsewhere: `create-external-config` clones a tree whose wire-solana was never built here, so checking against this checkout would reject a valid deployment payload while proving nothing about the machine that will run it. The remediation hint moves from `anchor build` to `npm run build:programs`, since only the wrapper writes the manifest. --- .../processes/SolanaValidatorProcessSteps.ts | 14 ++ .../tools/solana/SolanaOutpostProgramTool.ts | 169 +++++++++++++++++- .../SolanaValidatorProcessSteps.test.ts | 92 ++++++++++ .../solana/SolanaOutpostProgramTool.test.ts | 144 +++++++++++++++ 4 files changed, 417 insertions(+), 2 deletions(-) diff --git a/packages/cluster-tool/src/orchestration/steps/processes/SolanaValidatorProcessSteps.ts b/packages/cluster-tool/src/orchestration/steps/processes/SolanaValidatorProcessSteps.ts index a5a151062..96dd09b8e 100644 --- a/packages/cluster-tool/src/orchestration/steps/processes/SolanaValidatorProcessSteps.ts +++ b/packages/cluster-tool/src/orchestration/steps/processes/SolanaValidatorProcessSteps.ts @@ -38,6 +38,11 @@ export namespace SolanaValidatorProcessSteps { signal.throwIfAborted() if (ctx.processManager.get(SolanaValidatorProcess.ProcessLabel) != null) return + // THIS is the deploy: the validator loads the binary at genesis via + // `--upgradeable-program`, so whatever sits at that path is what executes + // on chain. Verify it against the recorded build BEFORE launching. The + // shared `resolvePrograms` deliberately does NOT — see its JSDoc. + SolanaOutpostProgramTool.assertProgramSoFile(ctx.config.solanaPath) const validator = await SolanaValidatorProcess.create(ctx.processManager, { address: ctx.config.bind.solana.address, rpcPort: ctx.config.bind.solana.ports.http, @@ -64,6 +69,15 @@ export namespace SolanaValidatorProcessSteps { * the OPP admin ops require. `createDeployerKeypair` is create-or-load, so * calling it from either path yields the identical identity. * + * `soFile` is the PATH ONLY — this resolves what the argv must say, never + * whether the binary on disk is the one the recorded build emitted. That + * check belongs to whoever actually loads it, which is {@link runStart} + * alone: the renderer emits a script that runs LATER, and often on ANOTHER + * host — `create-external-config` clones a tree whose `wire-solana` was + * never built here, so verifying against THIS checkout would reject a + * perfectly good deployment payload while proving nothing about the host + * that will run it. + * * @param config - The resolved cluster config. * @returns The validator's program list. */ diff --git a/packages/cluster-tool/src/tools/solana/SolanaOutpostProgramTool.ts b/packages/cluster-tool/src/tools/solana/SolanaOutpostProgramTool.ts index 9275a8219..bde362268 100644 --- a/packages/cluster-tool/src/tools/solana/SolanaOutpostProgramTool.ts +++ b/packages/cluster-tool/src/tools/solana/SolanaOutpostProgramTool.ts @@ -11,10 +11,16 @@ */ import Assert from "node:assert" +import { execFileSync } from "node:child_process" +import Crypto from "node:crypto" import Fs from "node:fs" import Path from "node:path" +import { Either } from "@3fv/prelude-ts" import type * as anchor from "@coral-xyz/anchor" import { Keypair, PublicKey } from "@solana/web3.js" +import { getLogger, NestedError } from "@wireio/shared" + +const log = getLogger(__filename) export namespace SolanaOutpostProgramTool { /** @@ -40,9 +46,77 @@ export namespace SolanaOutpostProgramTool { * 6000-6056 the daemons surface would be missing). */ export const ProgramIdlSubpath = "target/idl/liqsol_core.json" - /** Remediation hint appended to every missing-artifact assertion. */ + /** + * Subpath (under `wire-solana`) of the build manifest + * `scripts/build/anchor-build-strict.mjs` writes beside the compiled `.so`s. + * It records the SBPF arch the build targeted and each emitted binary's + * sha256 — what {@link assertProgramSoFile} checks the deployed `.so` against. + */ + export const BuildManifestSubpath = "target/deploy/wire-build-manifest.json" + /** + * Remediation hint appended to every missing-artifact assertion. + * + * `npm run build:programs`, NOT a bare `anchor build`: the wrapper is what + * fails the build on an SBF frame overflow, emits each `.so` at the arch + * `Anchor.toml`'s pinned toolchain implies, and writes + * {@link BuildManifestSubpath}. + */ export const BuildRemediationHint = - "(run 'anchor build && node scripts/opp/patch-idl-errors.js' in wire-solana)" + "(run 'npm run build:programs && node scripts/opp/patch-idl-errors.js' in wire-solana)" + + /** One program's entry in the wire-solana build manifest. */ + export interface BuildManifestProgram { + /** Repo-relative path of the emitted binary. */ + programBinaryPath: string + /** Byte length of the emitted binary. */ + programBinaryLength: number + /** Hex sha256 of the emitted binary. */ + programBinarySha256: string + } + + /** The build manifest emitted alongside the compiled `.so`s. */ + export interface BuildManifest { + /** Manifest format version. */ + schemaVersion: number + /** SBPF arch the `.so`s were built for (`v0`…`v3`). */ + arch: string + /** + * `git describe --tags --always --dirty` of the checkout the binaries were + * built from — see {@link assertProgramSoFile} for why the sha pair alone + * is not sufficient. + */ + sourceDescribe: string + /** Per-program entries, keyed by the program's snake_case name. */ + programs: Record + } + + /** Marker `git describe --dirty` appends when tracked files are modified. */ + export const DirtyDescribeSuffix = "-dirty" + + /** + * `git describe --tags --always --dirty` of a checkout — the same value the + * build records, so the two are directly comparable. + * + * @param repositoryPath - Repo root to describe. + * @returns The describe string, e.g. `devnet-v1.5.2-237-g12f95d37-dirty`. + */ + export function describeCheckout(repositoryPath: string): string { + return Either.try(() => + execFileSync("git", ["describe", "--tags", "--always", "--dirty"], { + cwd: repositoryPath, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"] + }) + ) + .ifLeft(error => { + throw new NestedError( + "SolanaOutpostProgramTool: could not describe the wire-solana checkout", + { cause: error, context: { repositoryPath } } + ) + }) + .getOrThrow() + .trim() + } /** Absolute path of the committed program keypair under `solanaPath`. */ export function programKeypairFile(solanaPath: string): string { @@ -59,6 +133,97 @@ export namespace SolanaOutpostProgramTool { return Path.join(solanaPath, ProgramIdlSubpath) } + /** Absolute path of the build manifest under `solanaPath`. */ + export function buildManifestFile(solanaPath: string): string { + return Path.join(solanaPath, BuildManifestSubpath) + } + + /** Parse the wire-solana build manifest; throws when the file is absent. */ + export function readBuildManifest(solanaPath: string): BuildManifest { + const manifestFile = buildManifestFile(solanaPath) + Assert.ok( + Fs.existsSync(manifestFile), + `SolanaOutpostProgramTool: build manifest missing: ${manifestFile} ${BuildRemediationHint}` + ) + return JSON.parse(Fs.readFileSync(manifestFile, "utf8")) as BuildManifest + } + + /** + * Absolute path of the compiled `.so`, PROVEN to be the binary the recorded + * build emitted — called by `SolanaValidatorProcessSteps.runStart`, the one + * path that actually loads the binary on THIS host. The `start.sh` renderer + * takes the unverified {@link programSoFile} instead, because the script it + * emits runs later and often elsewhere; see that step's JSDoc. + * + * The validator is launched with `--upgradeable-program `, so + * whatever sits at that path IS what executes on chain. Existence alone is + * not enough: nothing in the harness rebuilds the program, and `target/` is + * git-ignored, so a `git checkout`/rebase moves the sources while the `.so` + * stays put and a stale binary from another branch deploys silently. + * + * Comparing the file's sha256 against the manifest the build wrote turns that + * into a startup error naming the mismatch. A wrong binary otherwise fails as + * undefined behavior at instruction entry — observed 2026-08-21 as + * `consumed 427 of 200000 compute units` / `Access violation writing 1 bytes + * at address 0x32` during the outpost's init-PDAs step, which reads as a + * program bug rather than a build-provenance one. + * + * The sha pair alone is NOT sufficient, because the `.so` and the manifest + * are written together and both live in gitignored `target/`: a branch switch + * leaves the pair behind intact and mutually consistent, so a sha-only check + * passes while the sources have moved. `sourceDescribe` closes that — the + * build stamps its `git describe` and this compares it against the current + * checkout. A DIRTY build is reported rather than rejected: two different + * dirty trees at one commit describe identically, so the stamp marks the + * binary unverifiable instead of pretending otherwise. + * + * @param solanaPath - The `wire-solana` repo root. + * @returns Absolute path of the verified `.so`. + */ + export function assertProgramSoFile(solanaPath: string): string { + const soFile = programSoFile(solanaPath) + Assert.ok( + Fs.existsSync(soFile), + `SolanaOutpostProgramTool: ${ProgramName} .so missing: ${soFile} ${BuildRemediationHint}` + ) + + // Structural integrity first (is this manifest about this binary?), then + // provenance (was that build made from these sources?) — so a malformed + // manifest reports itself rather than surfacing as a checkout mismatch. + const { arch, sourceDescribe, programs } = readBuildManifest(solanaPath), + recorded = programs?.[ProgramName] + Assert.ok( + recorded != null, + `SolanaOutpostProgramTool: build manifest has no ${ProgramName} entry: ` + + `${buildManifestFile(solanaPath)} ${BuildRemediationHint}` + ) + + const actual = Crypto.createHash("sha256") + .update(Fs.readFileSync(soFile)) + .digest("hex") + Assert.ok( + actual === recorded.programBinarySha256, + `SolanaOutpostProgramTool: ${ProgramName} .so does not match the recorded ` + + `SBPF ${arch} build — ${soFile} is sha256 ${actual}, manifest records ` + + `${recorded.programBinarySha256}. The binary on disk was NOT produced by ` + + `that build ${BuildRemediationHint}` + ) + + const currentDescribe = describeCheckout(solanaPath) + Assert.ok( + sourceDescribe === currentDescribe, + `SolanaOutpostProgramTool: ${ProgramName} was built from a different checkout — ` + + `manifest records "${sourceDescribe}", ${solanaPath} is now "${currentDescribe}". ` + + `The binary predates the current sources ${BuildRemediationHint}` + ) + if (sourceDescribe.endsWith(DirtyDescribeSuffix)) + log.warn( + `${ProgramName} was built from a DIRTY checkout (${sourceDescribe}) — ` + + `its provenance cannot be verified beyond the commit` + ) + return soFile + } + /** * Program id derived from the committed program keypair, or `null` when the * keypair file is absent (tolerant path — callers that can proceed without diff --git a/packages/cluster-tool/tests/orchestration/steps/processes/SolanaValidatorProcessSteps.test.ts b/packages/cluster-tool/tests/orchestration/steps/processes/SolanaValidatorProcessSteps.test.ts index 92d4b9b95..7051af688 100644 --- a/packages/cluster-tool/tests/orchestration/steps/processes/SolanaValidatorProcessSteps.test.ts +++ b/packages/cluster-tool/tests/orchestration/steps/processes/SolanaValidatorProcessSteps.test.ts @@ -1,7 +1,46 @@ +import Fs from "node:fs" +import Os from "node:os" +import Path from "node:path" +import { Keypair } from "@solana/web3.js" +import { + ProcessManager, + SolanaValidatorProcess +} from "@wireio/cluster-tool/cluster/processes" import { Steps } from "@wireio/cluster-tool/orchestration" import { Report } from "@wireio/cluster-tool/report" +import { SolanaOutpostProgramTool } from "@wireio/cluster-tool/tools/solana" +import { fixtureContext } from "../../../config/clusterBuildContextFixture.js" describe("Steps.processes.solanaValidator", () => { + /** + * One cluster root for the whole file — `ProcessManager.setClusterPath` may + * be set ONCE per process, so every context here names the SAME root. + */ + let dir: string + beforeAll(() => { + dir = Fs.mkdtempSync(Path.join(Os.tmpdir(), "solana-validator-steps-")) + ProcessManager.setClusterPath(dir) + }) + afterAll(() => { + Fs.rmSync(dir, { recursive: true, force: true }) + }) + + /** + * A wire-solana root carrying ONLY the committed program keypair — no `.so`, + * no build manifest. That is exactly the shape a cloned/never-built tree has, + * and the keypair is present because it is committed to the repo. + */ + function newUnbuiltSolanaPath(prefix: string): string { + const solanaPath = Fs.mkdtempSync(Path.join(Os.tmpdir(), prefix)), + keypairFile = SolanaOutpostProgramTool.programKeypairFile(solanaPath) + Fs.mkdirSync(Path.dirname(keypairFile), { recursive: true }) + Fs.writeFileSync( + keypairFile, + JSON.stringify([...Keypair.generate().secretKey]) + ) + return solanaPath + } + it("start builds an input-less step with a runner", () => { const step = Steps.processes.solanaValidator.planStart( Report.Actor.SolanaOutpost, @@ -13,4 +52,57 @@ describe("Steps.processes.solanaValidator", () => { expect(step.input).toBeNull() expect(typeof step.runner).toBe("function") }) + + describe("resolvePrograms", () => { + it("resolves the .so PATH without requiring a built wire-solana tree", () => { + // The `start.sh` renderer resolves this for a tree it will never launch + // itself — `create-external-config` clones a cluster whose wire-solana + // was never built here. Demanding a verified binary would fail that + // pipeline on a payload destined for another host entirely. + const solanaPath = newUnbuiltSolanaPath("solana-unbuilt-") + try { + const config = fixtureContext({ + clusterPath: dir, + dataPath: Path.join(dir, "data"), + solanaPath + }).config, + [program] = Steps.processes.solanaValidator.resolvePrograms(config) + expect(program.name).toBe(SolanaOutpostProgramTool.ProgramName) + expect(program.soFile).toBe( + SolanaOutpostProgramTool.programSoFile(solanaPath) + ) + expect(Fs.existsSync(program.soFile)).toBe(false) + } finally { + Fs.rmSync(solanaPath, { recursive: true, force: true }) + } + }) + }) + + describe("runStart", () => { + it("REJECTS an unverified binary before launching the validator", async () => { + // runStart IS the deploy — it loads the .so at genesis, so the recorded + // build has to match here even though the renderer above tolerates its + // absence. + const solanaPath = newUnbuiltSolanaPath("solana-unbuilt-start-") + try { + const ctx = fixtureContext({ + clusterPath: dir, + dataPath: Path.join(dir, "data"), + solanaPath + }) + await expect( + Steps.processes.solanaValidator.runStart( + ctx, + null, + new AbortController().signal + ) + ).rejects.toThrow(/liqsol_core \.so missing.*build:programs/s) + expect( + ctx.processManager.get(SolanaValidatorProcess.ProcessLabel) + ).toBeNull() + } finally { + Fs.rmSync(solanaPath, { recursive: true, force: true }) + } + }) + }) }) diff --git a/packages/cluster-tool/tests/tools/solana/SolanaOutpostProgramTool.test.ts b/packages/cluster-tool/tests/tools/solana/SolanaOutpostProgramTool.test.ts index 35f24c267..0666e9a9e 100644 --- a/packages/cluster-tool/tests/tools/solana/SolanaOutpostProgramTool.test.ts +++ b/packages/cluster-tool/tests/tools/solana/SolanaOutpostProgramTool.test.ts @@ -1,3 +1,5 @@ +import { execFileSync } from "node:child_process" +import Crypto from "node:crypto" import Fs from "node:fs" import Os from "node:os" import Path from "node:path" @@ -68,6 +70,148 @@ describe("SolanaOutpostProgramTool", () => { } }) + describe("assertProgramSoFile", () => { + /** Write a `.so` plus a manifest recording `recordedBytes` for it. */ + function writeProgramArtifacts( + root: string, + soBytes: Buffer, + recordedBytes: Buffer = soBytes, + sourceDescribe: string = SolanaOutpostProgramTool.describeCheckout(root) + ): void { + Fs.mkdirSync(Path.join(root, "target", "deploy"), { recursive: true }) + Fs.writeFileSync(SolanaOutpostProgramTool.programSoFile(root), soBytes) + Fs.writeFileSync( + SolanaOutpostProgramTool.buildManifestFile(root), + JSON.stringify({ + schemaVersion: 1, + arch: "v3", + sourceDescribe, + programs: { + [SolanaOutpostProgramTool.ProgramName]: { + programBinaryPath: SolanaOutpostProgramTool.ProgramSoSubpath, + programBinaryLength: recordedBytes.length, + programBinarySha256: Crypto.createHash("sha256") + .update(recordedBytes) + .digest("hex") + } + } + }) + ) + } + + /** + * Run `body` against a throwaway wire-solana root that is a real git repo + * with one commit — `describeCheckout` shells out to git, so the fixture + * has to be describable. + */ + function withRoot(prefix: string, body: (root: string) => void): void { + const root = Fs.mkdtempSync(Path.join(Os.tmpdir(), prefix)) + try { + const git = (...args: string[]) => + execFileSync("git", args, { cwd: root, stdio: "ignore" }) + git("init", "--quiet") + git("config", "user.email", "harness@wire.test") + git("config", "user.name", "harness") + Fs.writeFileSync(Path.join(root, "Anchor.toml"), "[toolchain]\n") + git("add", "-A") + git("commit", "--quiet", "-m", "fixture") + body(root) + } finally { + Fs.rmSync(root, { recursive: true, force: true }) + } + } + + it("returns the .so path when it matches the recorded build", () => { + withRoot("solana-outpost-so-ok-", root => { + writeProgramArtifacts(root, Buffer.from("compiled-liqsol-core")) + expect(SolanaOutpostProgramTool.assertProgramSoFile(root)).toBe( + SolanaOutpostProgramTool.programSoFile(root) + ) + expect(SolanaOutpostProgramTool.readBuildManifest(root).arch).toBe("v3") + }) + }) + + it("REJECTS a .so whose sha256 differs from the manifest (the stale-binary case)", () => { + withRoot("solana-outpost-so-stale-", root => { + writeProgramArtifacts( + root, + Buffer.from("binary-from-another-branch"), + Buffer.from("binary-the-build-emitted") + ) + expect(() => SolanaOutpostProgramTool.assertProgramSoFile(root)).toThrow( + /does not match the recorded SBPF v3 build.*build:programs/s + ) + }) + }) + + it("REJECTS a binary built from a different checkout (the branch-switch case)", () => { + withRoot("solana-outpost-so-checkout-", root => { + // The .so and its manifest agree with EACH OTHER — only the checkout + // they were built from has moved on, which a sha-only check misses. + writeProgramArtifacts( + root, + Buffer.from("compiled-liqsol-core"), + Buffer.from("compiled-liqsol-core"), + "devnet-v1.5.2-100-gdeadbee" + ) + expect(() => SolanaOutpostProgramTool.assertProgramSoFile(root)).toThrow( + /built from a different checkout.*devnet-v1\.5\.2-100-gdeadbee/s + ) + }) + }) + + it("accepts a dirty build but reports it as unverifiable", () => { + withRoot("solana-outpost-so-dirty-", root => { + // Modify a TRACKED file so the checkout really describes as dirty. + Fs.appendFileSync(Path.join(root, "Anchor.toml"), "# edited\n") + const dirty = SolanaOutpostProgramTool.describeCheckout(root) + expect(dirty).toMatch( + new RegExp(`${SolanaOutpostProgramTool.DirtyDescribeSuffix}$`) + ) + + writeProgramArtifacts(root, Buffer.from("dirty-build"), undefined, dirty) + expect(SolanaOutpostProgramTool.assertProgramSoFile(root)).toBe( + SolanaOutpostProgramTool.programSoFile(root) + ) + }) + }) + + it("throws when the checkout cannot be described", () => { + const notARepo = Fs.mkdtempSync( + Path.join(Os.tmpdir(), "solana-outpost-nogit-") + ) + try { + expect(() => + SolanaOutpostProgramTool.describeCheckout(notARepo) + ).toThrow(/could not describe the wire-solana checkout/s) + } finally { + Fs.rmSync(notARepo, { recursive: true, force: true }) + } + }) + + it("throws when the .so, the manifest, or its program entry is absent", () => { + withRoot("solana-outpost-so-missing-", root => { + expect(() => SolanaOutpostProgramTool.assertProgramSoFile(root)).toThrow( + /\.so missing.*build:programs/s + ) + + Fs.mkdirSync(Path.join(root, "target", "deploy"), { recursive: true }) + Fs.writeFileSync(SolanaOutpostProgramTool.programSoFile(root), "so") + expect(() => SolanaOutpostProgramTool.assertProgramSoFile(root)).toThrow( + /build manifest missing.*build:programs/s + ) + + Fs.writeFileSync( + SolanaOutpostProgramTool.buildManifestFile(root), + JSON.stringify({ schemaVersion: 1, arch: "v3", programs: {} }) + ) + expect(() => SolanaOutpostProgramTool.assertProgramSoFile(root)).toThrow( + /build manifest has no liqsol_core entry/s + ) + }) + }) + }) + it("throws on a malformed IDL file", () => { const brokenPath = Fs.mkdtempSync(Path.join(Os.tmpdir(), "solana-outpost-broken-")) try {