diff --git a/CLAUDE.md b/CLAUDE.md index 27a5728c2..73be38042 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,7 @@ pnpm workspaces (no nx/turbo/lerna). All packages under `packages/`: |---------|---------| | `cluster-tool` (`@wireio/cluster-tool`) | THE core library: orchestration engine (PhaseGroup → Phase → Step → Report), process managers, chain clients, config/bind resolution, Steps palette, flow substrate (`FlowCLI`/`FlowScenario`), CLI | | `cluster-tool-shared` (`@wireio/cluster-tool-shared`) | Zod schema-first persisted shapes (`ClusterConfig`, `BindConfig`, `ClusterState`, `SignatureProviderConfig`, `ExternalOutpostConfig`, `ExternalClusterConfig`, `ChainTokenAmount`) behind the generic `SchemaCodec` (validate-both-ends serialize/deserialize) | -| `flow-*` (13 packages) | One scenario each — standalone executables built on `FlowCLI.create(Scenario).run()`; batch-operator lifecycle (slashing/termination), collateral, reserves, emissions soak, node-owner NFT, yield distribution, and the six swap variants | +| `flow-*` (14 packages) | One scenario each — standalone executables built on `FlowCLI.create(Scenario).run()`; batch-operator lifecycle (slashing/termination), collateral, reserves, emissions soak, node-owner NFT, yield distribution, the six functional swap variants, and the manual-only 10-actor swap/epoch stress diagnostic | | `debugging-shared` / `debugging-server` / `debugging-client-shared` / `debugging-client-tool` / `debugging-client-tool-tui` | OPP debugging surface: shared types + storage paths, ingest server, RPC client, CLI, TUI | | `test-app-server` | Fixture app server used by debugging tests | diff --git a/README.md b/README.md index d713c8af1..181ca9ec4 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ pnpm workspace (no nx/turbo/lerna); everything lives under `packages/`. | `cluster-tool` | `@wireio/cluster-tool` | Core harness: process managers, chain clients, bootstrap, **`wire-cluster-tool` CLI** | | `flow-operator-collateral-deposit` | `@wireio/test-flow-operator-collateral-deposit` | Node-operator collateral deposit + withdraw remit | | `flow-swap-with-underwriting` | `@wireio/test-flow-swap-with-underwriting` | Bidirectional SWAP (ETH ↔ SOL) with underwriting | +| `flow-swap-epoch-stress` | `@wireio/test-flow-swap-epoch-stress` | Manual 10-actor ETH → SOL swap/epoch stress diagnostic (21 producers + 21 batch operators) | | `flow-swap-non-native-tokens` | `@wireio/test-flow-swap-non-native-tokens` | SWAP of non-native tokens (USDC / USDT / LIQ) | | `flow-swap-variance-revert` | `@wireio/test-flow-swap-variance-revert` | Swap variance-tolerance revert | | `flow-batch-operator-termination` | `@wireio/test-flow-batch-operator-termination` | Batch-operator termination via delivery underperformance | @@ -216,6 +217,13 @@ and every live run is paired with the heartbeat monitor (see ./scripts/run-flow.mjs swap --wire-build-path … --ethereum-path … --solana-path … ``` +The long-running `flow-swap-epoch-stress` scenario is manual-only and excluded +from the default E2E suite. It launches 10 simultaneous Ethereum → Solana swaps +against 21 producer accounts and 21 batch operators, then records payout, +UWREQ, Solana memory-log, and post-load epoch-liveness evidence. Run it locally +with the canonical runner and heartbeat monitor shown in the package +[README](packages/flow-swap-epoch-stress/README.md). + Each `--wire-build-path` / `--ethereum-path` / `--solana-path` flag falls back to its env var (`WIRE_BUILD_PATH` / `WIRE_ETH_PATH` / `WIRE_SOLANA_PATH`); one of the two is required. `--cluster-path` (env `WIRE_CLUSTER_PATH`) is optional — omit it diff --git a/packages/flow-swap-epoch-stress/README.md b/packages/flow-swap-epoch-stress/README.md new file mode 100644 index 000000000..00b96b213 --- /dev/null +++ b/packages/flow-swap-epoch-stress/README.md @@ -0,0 +1,45 @@ +# flow-swap-epoch-stress + +Manual-only FlowScenario that reproduces the swap load which exposed the +Solana terminal/epoch-stall failure. It intentionally multiplies the existing +Ethereum-to-Solana half of `flow-swap-with-underwriting`: + +- 10 distinct prefunded Ethereum actors submit `requestSwap` concurrently. +- Each actor targets a different Solana recipient. +- The cluster bootstraps 21 producer accounts and 21 batch operators. +- One real underwriter is bonded on Ethereum and Solana. +- The flow verifies all 10 destination payouts, all 10 confirmed UWREQ rows, + three post-load epoch advances, and recent Solana outpost logs for + memory/heap errors. + +Every transaction is its own Step. The generated Markdown, HTML, and CSV +reports contain the parameters, transaction/RPC evidence, payout observations, +UWREQ statuses, epoch baseline/result, and relevant Solana log evidence. + +## Run locally + +Build the required sibling repositories, then use the canonical runner and +heartbeat monitor in separate terminals: + +```bash +node scripts/run-flow.mjs flow-swap-epoch-stress \ + --cluster-path /tmp/wire-flow-swap-epoch-stress \ + --wire-build-path ../wire-sysio/build/debug \ + --ethereum-path ../wire-ethereum \ + --solana-path ../wire-solana +``` + +```bash +node scripts/flow-heartbeat-monitor.mjs \ + --cluster-path /tmp/wire-flow-swap-epoch-stress +``` + +This flow can take at least 138 minutes and is excluded from the default E2E +suite. Run it on demand when validating Solana terminal, validator, OPP, or +epoch-liveness changes. Reports are written beneath +`/tmp/wire-flow-swap-epoch-stress/reports/`; cluster logs and OPP debugging +artifacts remain in the same cluster directory. + +The separate `wire-cluster-tool readiness` command remains the read-only tool +for inspecting an already-running SIM2 cluster; this flow owns and stresses a +fresh local cluster for reproducible debugging. diff --git a/packages/flow-swap-epoch-stress/package.json b/packages/flow-swap-epoch-stress/package.json new file mode 100644 index 000000000..65b17a93c --- /dev/null +++ b/packages/flow-swap-epoch-stress/package.json @@ -0,0 +1,22 @@ +{ + "name": "@wireio/test-flow-swap-epoch-stress", + "version": "0.1.0", + "private": true, + "type": "commonjs", + "description": "Manual FlowScenario: concurrent Ethereum to Solana swaps with epoch-stall diagnostics", + "scripts": { + "build": "tsc -b tsconfig.json", + "test": "node lib/index.js", + "clean": "rm -rf lib" + }, + "dependencies": { + "@solana/web3.js": "^1.98.0", + "@wireio/cluster-tool": "workspace:*", + "@wireio/sdk-core": "^1.0.17", + "@wireio/shared": "^1.0.17", + "ethers": "^6.14.0" + }, + "devDependencies": { + "@types/node": "25.5.0" + } +} diff --git a/packages/flow-swap-epoch-stress/src/SwapEpochStressScenario.ts b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenario.ts new file mode 100644 index 000000000..8ce5af048 --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenario.ts @@ -0,0 +1,257 @@ +import Assert from "node:assert" +import type { ClusterConfig } from "@wireio/cluster-tool-shared" +import { SysioContracts } from "@wireio/sdk-core" +import type { Logger } from "@wireio/shared" +import { + ClusterBuildFailureMode, + ClusterBuildPhase, + ClusterBuildPhaseGroup, + Constants as ClusterConstants, + FlowScenario, + Report, + SwapScenarioContext, + WireReserveTool, + WireUnderwriterTool, + matchesProtoEnum, + pollUntil, + verifyStep, + type ClusterBuild, + type ClusterBuildOptions +} from "@wireio/cluster-tool" +import { SwapEpochStressScenarioConstants as Constants } from "./SwapEpochStressScenarioConstants.js" +import { + StressBaselineEpochKey, + StressBaselineUwreqIdsKey, + StressTargetAmountKey +} from "./SwapEpochStressScenarioOutputs.js" +import { SwapEpochStressScenarioSteps as StressSteps } from "./steps/index.js" + +const { SysioContractName, SysioOpregOperatorstatus } = SysioContracts +const { Actor } = Report + +/** Manual, report-first reproduction for the Solana terminal/epoch-stall load. */ +export class SwapEpochStressScenario extends FlowScenario { + readonly name = "flow-swap-epoch-stress" + readonly description = + "Ten concurrent Ethereum→Solana swaps across 21 producers and 21 batch operators, with settlement and epoch-liveness diagnostics" + + override readonly defaults: ClusterBuildOptions = { + enableMockReserves: true, + epochDurationSec: Constants.EpochDurationSec, + producerCount: Constants.ProducerCount, + batchOperatorCount: Constants.BatchOperatorCount, + underwriterCount: Constants.UnderwriterCount, + requiredUnderwriterCollateral: [ + { + chainCode: Constants.EthereumChainCode, + tokenCode: Constants.EthereumTokenCode, + minimumBond: Constants.UnderwriterMinimumBond + }, + { + chainCode: Constants.SolanaChainCode, + tokenCode: Constants.SolanaTokenCode, + minimumBond: Constants.UnderwriterMinimumBond + } + ] + } + + override createContext( + config: ClusterConfig, + log: Logger + ): SwapScenarioContext { + return new SwapScenarioContext(config, log) + } + + plan(cluster: ClusterBuild): void { + const firstUnderwriter = ClusterConstants.underwriterLabel(0) + const requestOptions = { timeoutMs: Constants.RequestStepTimeoutMs } + const settlementOptions = { + timeoutMs: Constants.SettlementDeadlineMs + Constants.PollDeadlineBufferMs + } + const diagnosticsOptions = { + timeoutMs: + Constants.postLoadEpochDeadlineMs() + + Constants.PollDeadlineBufferMs + + 120_000 + } + + WireUnderwriterTool.planCollateralDeposit( + cluster, + "UnderwriterCollateral", + "Bond the underwriter on both outposts before applying swap load", + requestOptions, + [firstUnderwriter], + cluster.context.config.underwriterCollateral ?? + WireUnderwriterTool.load(null, Constants.UnderwriterCount) + ) + + ClusterBuildPhase.create( + cluster, + "StressActors", + "Create ten distinct Ethereum senders and ten distinct Solana recipients", + Array.from({ length: Constants.ActorCount }, (_, actorIndex) => + StressSteps.planProvisionActor( + Actor.User, + `stress-actor-${actorIndex + 1}`, + `create Ethereum sender ${actorIndex + 1} and Solana recipient ${actorIndex + 1}`, + {}, + actorIndex, + Constants.EthereumHdIndexBase + actorIndex + ) + ), + { + parallelize: true, + failureMode: ClusterBuildFailureMode.collect + } + ) + + ClusterBuildPhase.create( + cluster, + "PreLoadGate", + "Require an active underwriter, healthy reserves, and capture the pre-load baseline" + ).push( + verifyStep( + Actor.Underwriter, + "underwriter-active", + `${firstUnderwriter} is ACTIVE before swaps are submitted`, + async ctx => { + const account = ctx.keyStore.assertOperator(firstUnderwriter).account + await pollUntil( + `${firstUnderwriter} ACTIVE`, + async () => { + const { rows } = await ctx.wire + .getSysioContract(SysioContractName.opreg) + .tables.operators.query({ limit: 100 }) + const row = rows.find(operator => operator.account === account) + return ( + row != null && + matchesProtoEnum( + row.status, + SysioOpregOperatorstatus, + SysioOpregOperatorstatus.OPERATOR_STATUS_ACTIVE + ) + ) + }, + Constants.underwriterActiveDeadlineMs(), + Constants.LongPollIntervalMs + ) + }, + { + timeoutMs: + Constants.underwriterActiveDeadlineMs() + + Constants.PollDeadlineBufferMs + } + ), + verifyStep( + Actor.Sysio, + "capture-pre-load-baseline", + "verify reserves, compute the live quote, and snapshot epoch/UWREQ IDs", + async ctx => { + await ctx.reserveBook( + Constants.EthereumChainCode, + Constants.EthereumTokenCode, + Constants.PrimaryReserveCode + ) + await ctx.reserveBook( + Constants.SolanaChainCode, + Constants.SolanaTokenCode, + Constants.PrimaryReserveCode + ) + const targetAmount = await WireReserveTool.swapquote(ctx.wire, { + from: { + chainCode: Constants.EthereumChainCode, + tokenCode: Constants.EthereumTokenCode, + reserveCode: Constants.PrimaryReserveCode + }, + fromAmount: Constants.SourceEthereumWei / Constants.WeiPerDepotUnit, + to: { + chainCode: Constants.SolanaChainCode, + tokenCode: Constants.SolanaTokenCode, + reserveCode: Constants.PrimaryReserveCode + } + }) + Assert.ok(targetAmount > 0n, "ETH→SOL live swap quote is zero") + const { rows: epochRows } = await ctx.wire.getEpochState() + Assert.ok(epochRows[0], "sysio.epoch::epochstate is empty") + const { rows: requestRows } = await ctx.wire + .getSysioContract(SysioContractName.uwrit) + .tables.uwreqs.query({ limit: 256 }) + const baselineEpoch = Number(epochRows[0].current_epoch_index) + const baselineUwreqIds = requestRows.map(request => + Number(request.id) + ) + ctx.outputs + .set(StressTargetAmountKey, targetAmount) + .set(StressBaselineEpochKey, baselineEpoch) + .set(StressBaselineUwreqIdsKey, baselineUwreqIds) + Report.StepExtraRecorder.note("pre-load baseline captured", { + targetAmount, + baselineEpoch, + baselineUwreqIds + }) + } + ) + ) + + const stress = ClusterBuildPhaseGroup.create( + cluster, + "ConcurrentSwapStress", + "Apply simultaneous swaps, collect every result, and always run terminal diagnostics", + { failureMode: ClusterBuildFailureMode.collect } + ) + + ClusterBuildPhase.create( + stress, + "ConcurrentRequests", + "Launch ten Ethereum→Solana requestSwap transactions at the same time", + Array.from({ length: Constants.ActorCount }, (_, actorIndex) => + StressSteps.planRequestSwap( + Actor.User, + `request-swap-${actorIndex + 1}`, + `Ethereum actor ${actorIndex + 1} swaps to Solana recipient ${actorIndex + 1}`, + requestOptions, + actorIndex + ) + ), + { + parallelize: true, + failureMode: ClusterBuildFailureMode.collect + } + ) + + ClusterBuildPhase.create( + stress, + "DestinationPayouts", + "Verify all ten distinct Solana recipients receive their remit payouts", + Array.from({ length: Constants.ActorCount }, (_, actorIndex) => + StressSteps.planVerifyPayout( + Actor.SolanaOutpost, + `verify-payout-${actorIndex + 1}`, + `Solana recipient ${actorIndex + 1} receives the expected target amount`, + settlementOptions, + actorIndex + ) + ), + { + parallelize: true, + failureMode: ClusterBuildFailureMode.collect + } + ) + + ClusterBuildPhase.create( + stress, + "TerminalDiagnostics", + "Prove all UWREQs confirmed, the epoch remains live, and Solana logs are clean", + [ + StressSteps.planTerminalDiagnostics( + Actor.Sysio, + "post-load-epoch-and-solana-health", + "require three post-load epoch advances and report UWREQ/Solana memory evidence", + diagnosticsOptions, + Constants.ActorCount + ) + ], + { failureMode: ClusterBuildFailureMode.collect } + ) + } +} diff --git a/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioConstants.ts b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioConstants.ts new file mode 100644 index 000000000..92a36e175 --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioConstants.ts @@ -0,0 +1,50 @@ +import { SlugName } from "@wireio/sdk-core" +import { ProtocolTiming } from "@wireio/cluster-tool" + +/** Fixed reproduction shape requested for the manual swap/epoch stress flow. */ +export namespace SwapEpochStressScenarioConstants { + export const ActorCount = 10 + export const ProducerCount = 21 + export const BatchOperatorCount = 21 + export const UnderwriterCount = 1 + export const EpochDurationSec = 60 + export const EthereumHdIndexBase = 32 + + /** 0.01 ETH per actor; ten requests consume about 1% of the seeded book. */ + export const SourceEthereumWei = 10_000_000_000_000_000n + export const WeiPerDepotUnit = 10n ** 9n + export const ToleranceBps = 500 + export const UnderwriterMinimumBond = 1_000_000_000 + + export const EthereumChainCode = SlugName.from("ETHEREUM") + export const SolanaChainCode = SlugName.from("SOLANA") + export const EthereumTokenCode = SlugName.from("ETH") + export const SolanaTokenCode = SlugName.from("SOL") + export const PrimaryReserveCode = SlugName.from("PRIMARY") + export const ReserveManagerContractName = "ReserveManager" + + export const LongPollIntervalMs = 3_000 + export const PollDeadlineBufferMs = 30_000 + export const RequestStepTimeoutMs = 60_000 + export const UnderwriterActiveEpochBudget = 9 + export const RequiredPostLoadEpochAdvances = 3 + export const PostLoadEpochBudget = 6 + export const SettlementDeadlineMs = ProtocolTiming.DoubleHopBudgetMs + export const MsPerSecond = 1_000 + + export function underwriterActiveDeadlineMs(): number { + return ( + ProtocolTiming.effectiveEpochSec(EpochDurationSec) * + UnderwriterActiveEpochBudget * + MsPerSecond + ) + } + + export function postLoadEpochDeadlineMs(): number { + return ( + ProtocolTiming.effectiveEpochSec(EpochDurationSec) * + PostLoadEpochBudget * + MsPerSecond + ) + } +} diff --git a/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioOutputs.ts b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioOutputs.ts new file mode 100644 index 000000000..428e21b7a --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/SwapEpochStressScenarioOutputs.ts @@ -0,0 +1,34 @@ +import type { HDNodeWallet } from "ethers" +import type { Keypair } from "@solana/web3.js" +import { outputKey, type OutputKey } from "@wireio/cluster-tool" + +export interface SwapEpochStressActorOutput { + readonly actorIndex: number + readonly ethereumWallet: HDNodeWallet + readonly solanaKeypair: Keypair + readonly solanaBalanceBefore: number +} + +export function stressActorOutputKey( + actorIndex: number +): OutputKey { + return outputKey( + `swapEpochStress.actor.${actorIndex}`, + `swap epoch stress actor ${actorIndex}` + ) +} + +export const StressTargetAmountKey: OutputKey = outputKey( + "swapEpochStress.targetAmount", + "shared ETH to SOL target amount from the pre-load live quote" +) + +export const StressBaselineEpochKey: OutputKey = outputKey( + "swapEpochStress.baselineEpoch", + "WIRE epoch immediately before concurrent swap submission" +) + +export const StressBaselineUwreqIdsKey: OutputKey = outputKey( + "swapEpochStress.baselineUwreqIds", + "underwrite request IDs present before concurrent swap submission" +) diff --git a/packages/flow-swap-epoch-stress/src/index.ts b/packages/flow-swap-epoch-stress/src/index.ts new file mode 100644 index 000000000..96cd6a7d6 --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/index.ts @@ -0,0 +1,9 @@ +import { FlowCLI } from "@wireio/cluster-tool" +import { SwapEpochStressScenario } from "./SwapEpochStressScenario.js" + +async function main(): Promise { + const report = await FlowCLI.create(SwapEpochStressScenario).run() + process.exit(report.succeeded ? 0 : 1) +} + +void main() diff --git a/packages/flow-swap-epoch-stress/src/steps/SwapEpochStressScenarioSteps.ts b/packages/flow-swap-epoch-stress/src/steps/SwapEpochStressScenarioSteps.ts new file mode 100644 index 000000000..82027fcd6 --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/steps/SwapEpochStressScenarioSteps.ts @@ -0,0 +1,375 @@ +import Assert from "node:assert" +import { ethers } from "ethers" +import { Keypair } from "@solana/web3.js" +import { SysioContracts } from "@wireio/sdk-core" +import { + ClusterBuildStep, + ClusterConfigProvider, + EthereumCollateralTool, + EthereumOutpostBootstrapper, + Report, + SolanaCollateralTool, + contractView, + matchesProtoEnum, + pollUntil, + requestEthereumSwap, + slugValue, + type ClusterBuildStepOptions, + type ReserveManagerRequestSwapContract, + type StepInput, + type SwapScenarioContext +} from "@wireio/cluster-tool" +import { SwapEpochStressScenarioConstants as Constants } from "../SwapEpochStressScenarioConstants.js" +import { + StressBaselineEpochKey, + StressBaselineUwreqIdsKey, + StressTargetAmountKey, + stressActorOutputKey +} from "../SwapEpochStressScenarioOutputs.js" + +const { SysioContractName, SysioUwritUnderwriterequeststatus } = SysioContracts +type Uwreq = SysioContracts.SysioUwritUwRequestTType + +function isStressRoute(request: Uwreq): boolean { + return ( + slugValue(request.src_chain_code) === Constants.EthereumChainCode && + slugValue(request.dst_chain_code) === Constants.SolanaChainCode + ) +} + +async function currentEpoch(ctx: SwapScenarioContext): Promise { + const { rows } = await ctx.wire.getEpochState() + Assert.ok(rows[0], "sysio.epoch::epochstate is empty") + return Number(rows[0].current_epoch_index) +} + +async function stressRequests(ctx: SwapScenarioContext): Promise { + const baseline = new Set(ctx.outputs.assert(StressBaselineUwreqIdsKey)) + const { rows } = await ctx.wire + .getSysioContract(SysioContractName.uwrit) + .tables.uwreqs.query({ limit: 256 }) + return rows.filter(row => !baseline.has(Number(row.id)) && isStressRoute(row)) +} + +export namespace SwapEpochStressScenarioSteps { + export interface ProvisionActorInput extends StepInput { + readonly kind: "SwapEpochStressScenarioSteps.ProvisionActorInput" + readonly actorIndex: number + readonly ethereumHdIndex: number + } + + export function planProvisionActor( + actor: Report.Actor, + name: string, + description: string, + options: ClusterBuildStepOptions, + actorIndex: number, + ethereumHdIndex: number + ): ClusterBuildStep { + return ClusterBuildStep.create( + actor, + name, + description, + options, + { + kind: "SwapEpochStressScenarioSteps.ProvisionActorInput", + actorIndex, + ethereumHdIndex + }, + runProvisionActor + ) + } + + export async function runProvisionActor( + ctx: SwapScenarioContext, + input: ProvisionActorInput, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + const derivation = `${EthereumOutpostBootstrapper.DerivationPath}${input.ethereumHdIndex}` + const ethereumWallet = ethers.HDNodeWallet.fromMnemonic( + ethers.Mnemonic.fromPhrase(EthereumOutpostBootstrapper.AnvilMnemonic), + derivation + ).connect(ctx.ethereum.provider) + const solanaKeypair = Keypair.generate() + const solanaBalanceBefore = await ctx.solana.getLamports( + solanaKeypair.publicKey + ) + Report.StepExtraRecorder.note("created distinct stress actor", { + actorIndex: input.actorIndex, + ethereumHdIndex: input.ethereumHdIndex, + ethereumAddress: ethereumWallet.address, + solanaRecipient: solanaKeypair.publicKey.toBase58(), + solanaBalanceBefore + }) + ctx.outputs.set(stressActorOutputKey(input.actorIndex), { + actorIndex: input.actorIndex, + ethereumWallet, + solanaKeypair, + solanaBalanceBefore + }) + } + + export interface RequestSwapInput extends StepInput { + readonly kind: "SwapEpochStressScenarioSteps.RequestSwapInput" + readonly actorIndex: number + } + + export function planRequestSwap( + actor: Report.Actor, + name: string, + description: string, + options: ClusterBuildStepOptions, + actorIndex: number + ): ClusterBuildStep { + return ClusterBuildStep.create( + actor, + name, + description, + options, + { kind: "SwapEpochStressScenarioSteps.RequestSwapInput", actorIndex }, + runRequestSwap + ) + } + + export async function runRequestSwap( + ctx: SwapScenarioContext, + input: RequestSwapInput, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + const stressActor = ctx.outputs.assert( + stressActorOutputKey(input.actorIndex) + ) + const targetAmount = ctx.outputs.assert(StressTargetAmountKey) + const reserveManager = loadReserveManager(ctx, stressActor.ethereumWallet) + const result = await requestEthereumSwap(reserveManager, { + sourceTokenCode: BigInt(Constants.EthereumTokenCode), + sourceReserveCode: BigInt(Constants.PrimaryReserveCode), + sourceAmountWei: Constants.SourceEthereumWei, + targetChainCode: BigInt(Constants.SolanaChainCode), + targetTokenCode: BigInt(Constants.SolanaTokenCode), + targetReserveCode: BigInt(Constants.PrimaryReserveCode), + targetRecipient: stressActor.solanaKeypair.publicKey.toBytes(), + targetAmount, + targetToleranceBps: Constants.ToleranceBps + }) + Assert.ok( + result.transactionHash, + `actor ${input.actorIndex} has no tx hash` + ) + Report.StepExtraRecorder.note("concurrent swap request submitted", { + actorIndex: input.actorIndex, + source: stressActor.ethereumWallet.address, + recipient: stressActor.solanaKeypair.publicKey.toBase58(), + transactionHash: result.transactionHash, + blockNumber: result.blockNumber, + sourceAmountWei: Constants.SourceEthereumWei, + targetAmount + }) + } + + export interface VerifyPayoutInput extends StepInput { + readonly kind: "SwapEpochStressScenarioSteps.VerifyPayoutInput" + readonly actorIndex: number + } + + export function planVerifyPayout( + actor: Report.Actor, + name: string, + description: string, + options: ClusterBuildStepOptions, + actorIndex: number + ): ClusterBuildStep { + return ClusterBuildStep.create( + actor, + name, + description, + options, + { kind: "SwapEpochStressScenarioSteps.VerifyPayoutInput", actorIndex }, + runVerifyPayout + ) + } + + export async function runVerifyPayout( + ctx: SwapScenarioContext, + input: VerifyPayoutInput, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + const stressActor = ctx.outputs.assert( + stressActorOutputKey(input.actorIndex) + ) + const targetAmount = ctx.outputs.assert(StressTargetAmountKey) + Assert.ok( + targetAmount <= BigInt(Number.MAX_SAFE_INTEGER), + "target amount exceeds JavaScript's exact integer range" + ) + const expectedMinimum = + stressActor.solanaBalanceBefore + Number(targetAmount) + let observed = stressActor.solanaBalanceBefore + await pollUntil( + `actor ${input.actorIndex} SOL payout`, + async () => { + signal.throwIfAborted() + observed = await ctx.solana.getLamports( + stressActor.solanaKeypair.publicKey + ) + return observed >= expectedMinimum + }, + Constants.SettlementDeadlineMs, + Constants.LongPollIntervalMs + ) + Report.StepExtraRecorder.note("destination payout observed", { + actorIndex: input.actorIndex, + recipient: stressActor.solanaKeypair.publicKey.toBase58(), + before: stressActor.solanaBalanceBefore, + observed, + expectedMinimum + }) + } + + export interface DiagnosticInput extends StepInput { + readonly kind: "SwapEpochStressScenarioSteps.DiagnosticInput" + readonly expectedRequestCount: number + } + + export function planTerminalDiagnostics( + actor: Report.Actor, + name: string, + description: string, + options: ClusterBuildStepOptions, + expectedRequestCount: number + ): ClusterBuildStep { + return ClusterBuildStep.create( + actor, + name, + description, + options, + { + kind: "SwapEpochStressScenarioSteps.DiagnosticInput", + expectedRequestCount + }, + runTerminalDiagnostics + ) + } + + export async function runTerminalDiagnostics( + ctx: SwapScenarioContext, + input: DiagnosticInput, + signal: AbortSignal + ): Promise { + signal.throwIfAborted() + const baselineEpoch = ctx.outputs.assert(StressBaselineEpochKey) + const requiredEpoch = + baselineEpoch + Constants.RequiredPostLoadEpochAdvances + let observedEpoch = await currentEpoch(ctx) + let epochPollError: unknown + try { + await pollUntil( + `WIRE epoch advances from ${baselineEpoch} to at least ${requiredEpoch}`, + async () => { + signal.throwIfAborted() + observedEpoch = await currentEpoch(ctx) + return observedEpoch >= requiredEpoch + }, + Constants.postLoadEpochDeadlineMs(), + Constants.LongPollIntervalMs + ) + } catch (error) { + epochPollError = error + } + + const requests = await stressRequests(ctx) + const confirmed = requests.filter(request => + matchesProtoEnum( + request.status, + SysioUwritUnderwriterequeststatus, + SysioUwritUnderwriterequeststatus.UNDERWRITE_REQUEST_STATUS_CONFIRMED + ) + ) + const firstActor = ctx.outputs.assert(stressActorOutputKey(0)) + const program = SolanaCollateralTool.loadOppOutpostProgram( + ctx, + firstActor.solanaKeypair + ) + let programLogs: string[] = [] + let programLogReadError: unknown + try { + programLogs = ( + await ctx.solana.getProgramLogs(program.programId, 100) + ).flat() + } catch (error) { + programLogReadError = error + } + const memoryErrors = programLogs.filter(line => + /memory allocation failed|out of memory|heap(?:[ -]space)? violation/i.test( + line + ) + ) + + Report.StepExtraRecorder.note("post-load terminal diagnostics", { + baselineEpoch, + observedEpoch, + requiredEpoch, + expectedRequestCount: input.expectedRequestCount, + observedRequestCount: requests.length, + confirmedRequestCount: confirmed.length, + requestIds: requests.map(request => Number(request.id)), + requestStatuses: requests.map(request => String(request.status)), + epochPollError: + epochPollError instanceof Error + ? epochPollError.message + : String(epochPollError ?? ""), + solanaMemoryErrors: memoryErrors, + solanaProgramLogReadError: + programLogReadError instanceof Error + ? programLogReadError.message + : String(programLogReadError ?? "") + }) + + Assert.strictEqual( + epochPollError, + undefined, + `WIRE epoch failed to advance from ${baselineEpoch} to ${requiredEpoch}` + ) + Assert.strictEqual( + requests.length, + input.expectedRequestCount, + "unexpected number of new ETH→SOL underwrite requests" + ) + Assert.strictEqual( + confirmed.length, + input.expectedRequestCount, + "not every stress underwrite request reached CONFIRMED" + ) + Assert.deepStrictEqual( + memoryErrors, + [], + "Solana outpost logs contain memory/heap failure evidence" + ) + Assert.strictEqual( + programLogReadError, + undefined, + "failed to read recent Solana outpost program logs" + ) + } + + function loadReserveManager( + ctx: SwapScenarioContext, + wallet: ethers.Signer + ): ReserveManagerRequestSwapContract { + const address = EthereumCollateralTool.loadOutpostAddresses( + ClusterConfigProvider.ethereumDeploymentsPath(ctx.config) + )[Constants.ReserveManagerContractName] + Assert.ok( + address != null && /^0x[0-9a-fA-F]{40}$/.test(address), + `${Constants.ReserveManagerContractName} missing from outpost-addrs.json` + ) + const abi = EthereumCollateralTool.loadOutpostAbi( + ctx.config.ethereumPath, + Constants.ReserveManagerContractName + ) + return contractView(address, abi, wallet) + } +} diff --git a/packages/flow-swap-epoch-stress/src/steps/index.ts b/packages/flow-swap-epoch-stress/src/steps/index.ts new file mode 100644 index 000000000..e6b2e3d56 --- /dev/null +++ b/packages/flow-swap-epoch-stress/src/steps/index.ts @@ -0,0 +1 @@ +export * from "./SwapEpochStressScenarioSteps.js" diff --git a/packages/flow-swap-epoch-stress/tsconfig.json b/packages/flow-swap-epoch-stress/tsconfig.json new file mode 100644 index 000000000..55ae8166e --- /dev/null +++ b/packages/flow-swap-epoch-stress/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../etc/tsconfig/tsconfig.base.jest.json", + "compilerOptions": {}, + "include": [], + "references": [ + { "path": "../cluster-tool/tsconfig.json" }, + { "path": "./tsconfig.src.json" } + ] +} diff --git a/packages/flow-swap-epoch-stress/tsconfig.src.json b/packages/flow-swap-epoch-stress/tsconfig.src.json new file mode 100644 index 000000000..3be54c59c --- /dev/null +++ b/packages/flow-swap-epoch-stress/tsconfig.src.json @@ -0,0 +1,16 @@ +{ + "extends": "../../etc/tsconfig/tsconfig.base.jest.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib", + "paths": { + "@wireio/cluster-tool": ["../cluster-tool/src"], + "@wireio/cluster-tool-shared": ["../cluster-tool-shared/src"] + } + }, + "include": ["src"], + "references": [ + { "path": "../cluster-tool-shared/tsconfig.json" }, + { "path": "../cluster-tool/tsconfig.json" } + ] +} diff --git a/tsconfig.json b/tsconfig.json index 9af85855a..727b019ae 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,7 @@ { "path": "./packages/cluster-tool/tsconfig.json" }, { "path": "./packages/flow-operator-collateral-deposit/tsconfig.json" }, { "path": "./packages/flow-swap-with-underwriting/tsconfig.json" }, + { "path": "./packages/flow-swap-epoch-stress/tsconfig.json" }, { "path": "./packages/flow-batch-operator-termination/tsconfig.json" }, { "path": "./packages/flow-batch-operator-slashing/tsconfig.json" }, { "path": "./packages/flow-swap-variance-revert/tsconfig.json" },