diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5566edb..622e655 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,3 +77,37 @@ jobs: - uses: dtolnay/rust-toolchain@1.93 - uses: Swatinem/rust-cache@v2 - run: cargo test --workspace --all-features + + publisher-parity: + name: Real publisher parity + runs-on: ubuntu-latest + env: + STELAE_PARITY_NEW_REVISION: ${{ github.sha }} + STELAE_PARITY_REPORT: ${{ github.workspace }}/target/publisher-parity.json + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@1.93 + - uses: Swatinem/rust-cache@v2 + - run: cargo test --locked -p stelae-cardano --test publisher_parity -- --ignored --nocapture + - uses: actions/upload-artifact@v4 + if: always() + with: + name: publisher-parity-report + path: target/publisher-parity.json + if-no-files-found: warn + + registry-parity: + name: Local OCI fault and restart parity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@1.93 + - uses: Swatinem/rust-cache@v2 + # These tests deliberately spawn disposable, authenticated registry:2 + # containers. Keeping the ignored selector here makes Docker a visible + # CI requirement instead of silently treating the registry gate as run. + - run: cargo test --locked -p stelae --all-features --test oci -- --ignored --nocapture diff --git a/Cargo.lock b/Cargo.lock index 3734e09..a1a3d96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4644,13 +4644,18 @@ dependencies = [ "dolos", "dolos-cardano", "dolos-core", + "dolos-flatfiles", "dolos-mithril", "dolos-snapshot", "dolos-testing", + "hex", "itertools 0.14.0", + "libc", "pallas", + "rustls", "serde", "serde_json", + "sha2 0.10.9", "stelae", "stelae-driver", "tempfile", @@ -4659,6 +4664,7 @@ dependencies = [ "tokio-util", "toml 0.8.23", "tracing", + "zstd-safe", ] [[package]] diff --git a/deny.toml b/deny.toml index 03e5f85..6100205 100644 --- a/deny.toml +++ b/deny.toml @@ -28,7 +28,7 @@ deny = [ { crate = "dolos-cardano", wrappers = ["stelae-cardano", "dolos", "dolos-snapshot", "dolos-testing"], reason = "Dolos belongs only below the Cardano integration" }, { crate = "dolos-core", wrappers = ["stelae-cardano", "dolos", "dolos-cardano", "dolos-snapshot", "dolos-mithril", "dolos-fjall", "dolos-redb3", "dolos-testing"], reason = "Dolos belongs only below the Cardano integration" }, { crate = "dolos-fjall", wrappers = ["dolos", "dolos-testing"], reason = "Dolos belongs only below the Cardano integration" }, - { crate = "dolos-flatfiles", wrappers = ["dolos-fjall"], reason = "Dolos belongs only below the Cardano integration" }, + { crate = "dolos-flatfiles", wrappers = ["stelae-cardano", "dolos-fjall"], reason = "Dolos belongs only below the Cardano integration" }, { crate = "dolos-minibf", reason = "Dolos belongs only below the Cardano integration" }, { crate = "dolos-minikupo", reason = "Dolos belongs only below the Cardano integration" }, { crate = "dolos-mithril", wrappers = ["stelae-cardano"], reason = "Dolos belongs only below the Cardano integration" }, diff --git a/docs/publisher-parity.md b/docs/publisher-parity.md new file mode 100644 index 0000000..f787efc --- /dev/null +++ b/docs/publisher-parity.md @@ -0,0 +1,104 @@ +# Publisher host parity + +This is the executable gate for replacing the Dolos-hosted publisher with +`stelae-publisher`. It compares application hosts without changing the Stelae +wire format or the Dolos consumer. + +## Pinned identities + +- old host: Dolos `1ae4e91c18a9e1456a3612af402d7b9b97546d30` + (`dolos snapshot publish` and its headless snapshot facade); +- new host base: Stelae `47b38a4`, the merge of publisher-host PR 3; +- protocol/profile: Stelae 0.2.0 and `io.txpipe.dolos.cardano` v1; +- real fixture: `stelae-cardano/tests/fixtures/preview-epoch-0`, identified + and explained by its README; +- expected first-boundary identity: + `sha256:6232659be34afdf24f56784b9ba3db3ccee70cd7476708652b8ba0381bea0027`. + +The fixture contains 4,958 public Preview blocks. It starts at origin, crosses +the exact slot-86,400 epoch boundary, and ends at slot 99,140. Both hosts replay +the same immutable bodies into separate fresh stores. Replay stops at the +boundary, as the production backfill loop does, so publication observes an +equivalent checkpoint and identical predecessor/history inputs. + +## Automated report + +Run the real replay, directory/OCI publication, and consumer restore gate with +Docker available: + +```sh +STELAE_PARITY_NEW_REVISION="$(git rev-parse HEAD)" \ +STELAE_PARITY_REPORT="$(pwd)/target/publisher-parity.json" \ +cargo test --locked -p stelae-cardano --test publisher_parity \ + -- --ignored --nocapture +``` + +The JSON report identifies both hosts and all pins, records pass/fail checks, +and measures replay/publish wall time, process peak RSS, artifact size, +transfer, and scratch use. Directory publication moves zero network bytes and +uses no OCI scratch, so both values are explicitly zero rather than omitted. +The report also records each host's first-push layer and byte transfer counters +from separate repositories in a disposable `registry:2`. CI runs the command +in the `Real publisher parity` job and uploads `publisher-parity-report`. + +The test requires all of the following before it writes a passing report: + +- identical boundary cursor; +- identical layer descriptors, diffIds, retained epoch-1 dump, and canonical + inscription bytes; +- identical inscription digest; +- actual old/new publication to separate OCI repositories, including equal + transfer results, dry-run/no-op policy, and forced reproduction; +- successful directory and OCI restore of the new-host artifact through the + unchanged Dolos profile consumer; +- reproduction of the same identity from the restored cursor/state/history. + +The checked run on 2026-09-13 passed in a debug build. Old/new replay took +2.166/2.013 seconds and directory publication took 1.050/1.025 seconds. Both +directory artifacts were 2,003,124 bytes. Peak process RSS after each host was +416,694,272/421,937,152 bytes. Each OCI host uploaded 1,953,824 layer bytes, +skipped 6,288 bytes already present, and produced a 1,960,112-byte compressed +artifact with the expected identity. These tiny-fixture numbers are regression +evidence only; they make no mainnet throughput or memory claim. + +## Local OCI gate + +The host parity gate above uses an anonymous loopback registry so both immutable +old-host and new-host code paths publish the real fixture under the same simple +transport policy. The deeper transport suite uses disposable, authenticated +`registry:2` containers and separate repository names. It covers first push +and pull, subsequent missing- +blob-only upload, layer reuse and verified reuse refusal, forced concurrent and +serial agreement, corrupt and wrong-layer refusal, interruption before manifest +publication, restart with the previous `latest` intact, staging cleanup, +credential refusal, and bounded upload/download memory. + +```sh +cargo test --locked -p stelae --all-features --test oci \ + -- --ignored --nocapture +``` + +The tests are marked ignored because they require Docker, and the dedicated +`Local OCI fault and restart parity` CI job explicitly selects them. A run on +2026-09-13 passed all 16 Docker-backed cases. The same run observed a 49,495,816 +byte uncompressed transport fixture with 4,251,854 peak bytes held while +uploading and 1,255,355 while pulling. + +Host policy and failure boundaries remain covered by the workspace suite: +dry-run/no-op/require-new and predecessor-gap decisions, transient retry and +cancellation, publish-pending-before-advance, replay finalization after failure, +fail-closed latest restore, explicit genesis initialization, corrupt or missing +input refusal, and layer-journal/checkpoint resume. The real-fixture host test +and transport fault tests both exercise an actual registry; narrower unit tests +exercise the orchestration seams around it. + +## Scope and interpretation + +All repositories are loopback/disposable and no production registry is named. +The old source is required only by immutable git revision, not by a future +Dolos release branch. The real fixture and genesis inputs live here by content +identity, so removal of the old Dolos command does not remove the evidence. + +This gate establishes code-path parity for packaging. Production canary, soak, +and mainnet performance remain separate operations and must not be inferred +from this fixture. diff --git a/docs/publisher.md b/docs/publisher.md index 93eb8e0..a6075ef 100644 --- a/docs/publisher.md +++ b/docs/publisher.md @@ -131,3 +131,6 @@ This increment does not ship an image, change a Helm chart, deploy, publish to a production registry, or claim full production parity. Packaging, operational cutover, differential live-registry evidence and later pin cleanup remain the following publisher-pipeline steps. + +The executable old/new replay, publication, recovery, resource, and local OCI +evidence is documented in [publisher-parity.md](publisher-parity.md). diff --git a/stelae-cardano/Cargo.toml b/stelae-cardano/Cargo.toml index d9e6b24..b086dad 100644 --- a/stelae-cardano/Cargo.toml +++ b/stelae-cardano/Cargo.toml @@ -27,6 +27,12 @@ tokio-util = { version = "0.7.11", features = ["rt"] } tracing = "0.1.37" [dev-dependencies] +dolos-flatfiles = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } dolos-testing = { git = "https://github.com/txpipe/dolos", rev = "1ae4e91c18a9e1456a3612af402d7b9b97546d30" } +hex = "0.4.3" +libc = "0.2.186" +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +sha2 = "0.10.9" tempfile = "3.20.0" toml = "0.8.13" +zstd-safe = "7.2.4" diff --git a/stelae-cardano/src/publisher.rs b/stelae-cardano/src/publisher.rs index aa6b291..532f6c8 100644 --- a/stelae-cardano/src/publisher.rs +++ b/stelae-cardano/src/publisher.rs @@ -207,6 +207,7 @@ pub enum PublishOutcome { identity: String, built: usize, reused: usize, + transfer: stelae::oci::Transfer, }, } @@ -279,6 +280,7 @@ pub fn publish_once( identity: published.identity.to_string(), built: published.layers_built, reused: published.layers_reused, + transfer: published.transfer, } } } diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/000000.segment b/stelae-cardano/tests/fixtures/preview-epoch-0/000000.segment new file mode 100644 index 0000000..e36faae Binary files /dev/null and b/stelae-cardano/tests/fixtures/preview-epoch-0/000000.segment differ diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/README.md b/stelae-cardano/tests/fixtures/preview-epoch-0/README.md new file mode 100644 index 0000000..58155fb --- /dev/null +++ b/stelae-cardano/tests/fixtures/preview-epoch-0/README.md @@ -0,0 +1,39 @@ +# Preview epoch-zero replay fixture + +This fixture is the first 4,958 public Cardano Preview blocks, from origin +through the first epoch transition. It is enough to make both publisher hosts +execute real Byron replay and the epoch-boundary state transition without a +network dependency. + +The blocks were acquired from `relay.cnode-m1.demeter.run:3002` on 2026-09-13 +with Dolos revision `1ae4e91c18a9e1456a3612af402d7b9b97546d30`. The bounded +capture was interrupted after its first 1,000-block request and resumed with a +4,000-block request; intersection overlap left 4,958 canonical blocks. +`000000.segment` is the Dolos v4 flat-file +format: one independently checksummed zstd frame per raw block, using the +dictionary pinned by that Dolos revision. No store indexes or mutable state are +part of the fixture. + +`byron.json`, `shelley.json`, `alonzo.json`, and `conway.json` are the public +Preview genesis inputs supplied by that same revision's `dolos init`. The test +loads the equivalent embedded genesis from the pinned dependency and verifies +the file identities recorded below, so deleting the old publisher command in a +later Dolos release does not delete either the fixture or its inputs. + +Identities: + +- segment sha256: + `728c5aa9c7c78c4221bdee765f3cd109b630cb00266ab824f6db2bfa43276b4d` +- blocks: 4,958 +- first: slot 0, block 0, + `268ae601af8f9214804735910a3301881fbe0eec9936db7d1fb9fc39e93d1e37` +- boundary: slot 86,400, block 4,320, + `4a9761ddc291b0c352d1712b624759132936a07b3e02d1d3bdaaf17b9abfe683` +- final: slot 99,140, block 4,957, + `da6efb517e112a8439820d505286964bcc65ddd073b560a8f04d1c0902f1125f` +- old host: Dolos `1ae4e91c18a9e1456a3612af402d7b9b97546d30` +- protocol/profile: Stelae 0.2.0, `io.txpipe.dolos.cardano` v1 +- first-publication predecessor/history: none / empty + +The fixture contains only public chain protocol data and public network +configuration. It contains no credentials, registry state, or production data. diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/alonzo.json b/stelae-cardano/tests/fixtures/preview-epoch-0/alonzo.json new file mode 100644 index 0000000..3621f2d --- /dev/null +++ b/stelae-cardano/tests/fixtures/preview-epoch-0/alonzo.json @@ -0,0 +1,196 @@ +{ + "lovelacePerUTxOWord": 34482, + "executionPrices": { + "prSteps": + { + "numerator" : 721, + "denominator" : 10000000 + }, + "prMem": + { + "numerator" : 577, + "denominator" : 10000 + } + }, + "maxTxExUnits": { + "exUnitsMem": 10000000, + "exUnitsSteps": 10000000000 + }, + "maxBlockExUnits": { + "exUnitsMem": 50000000, + "exUnitsSteps": 40000000000 + }, + "maxValueSize": 5000, + "collateralPercentage": 150, + "maxCollateralInputs": 3, + "costModels": { + "PlutusV1": { + "sha2_256-memory-arguments": 4, + "equalsString-cpu-arguments-constant": 1000, + "cekDelayCost-exBudgetMemory": 100, + "lessThanEqualsByteString-cpu-arguments-intercept": 103599, + "divideInteger-memory-arguments-minimum": 1, + "appendByteString-cpu-arguments-slope": 621, + "blake2b-cpu-arguments-slope": 29175, + "iData-cpu-arguments": 150000, + "encodeUtf8-cpu-arguments-slope": 1000, + "unBData-cpu-arguments": 150000, + "multiplyInteger-cpu-arguments-intercept": 61516, + "cekConstCost-exBudgetMemory": 100, + "nullList-cpu-arguments": 150000, + "equalsString-cpu-arguments-intercept": 150000, + "trace-cpu-arguments": 150000, + "mkNilData-memory-arguments": 32, + "lengthOfByteString-cpu-arguments": 150000, + "cekBuiltinCost-exBudgetCPU": 29773, + "bData-cpu-arguments": 150000, + "subtractInteger-cpu-arguments-slope": 0, + "unIData-cpu-arguments": 150000, + "consByteString-memory-arguments-intercept": 0, + "divideInteger-memory-arguments-slope": 1, + "divideInteger-cpu-arguments-model-arguments-slope": 118, + "listData-cpu-arguments": 150000, + "headList-cpu-arguments": 150000, + "chooseData-memory-arguments": 32, + "equalsInteger-cpu-arguments-intercept": 136542, + "sha3_256-cpu-arguments-slope": 82363, + "sliceByteString-cpu-arguments-slope": 5000, + "unMapData-cpu-arguments": 150000, + "lessThanInteger-cpu-arguments-intercept": 179690, + "mkCons-cpu-arguments": 150000, + "appendString-memory-arguments-intercept": 0, + "modInteger-cpu-arguments-model-arguments-slope": 118, + "ifThenElse-cpu-arguments": 1, + "mkNilPairData-cpu-arguments": 150000, + "lessThanEqualsInteger-cpu-arguments-intercept": 145276, + "addInteger-memory-arguments-slope": 1, + "chooseList-memory-arguments": 32, + "constrData-memory-arguments": 32, + "decodeUtf8-cpu-arguments-intercept": 150000, + "equalsData-memory-arguments": 1, + "subtractInteger-memory-arguments-slope": 1, + "appendByteString-memory-arguments-intercept": 0, + "lengthOfByteString-memory-arguments": 4, + "headList-memory-arguments": 32, + "listData-memory-arguments": 32, + "consByteString-cpu-arguments-intercept": 150000, + "unIData-memory-arguments": 32, + "remainderInteger-memory-arguments-minimum": 1, + "bData-memory-arguments": 32, + "lessThanByteString-cpu-arguments-slope": 248, + "encodeUtf8-memory-arguments-intercept": 0, + "cekStartupCost-exBudgetCPU": 100, + "multiplyInteger-memory-arguments-intercept": 0, + "unListData-memory-arguments": 32, + "remainderInteger-cpu-arguments-model-arguments-slope": 118, + "cekVarCost-exBudgetCPU": 29773, + "remainderInteger-memory-arguments-slope": 1, + "cekForceCost-exBudgetCPU": 29773, + "sha2_256-cpu-arguments-slope": 29175, + "equalsInteger-memory-arguments": 1, + "indexByteString-memory-arguments": 1, + "addInteger-memory-arguments-intercept": 1, + "chooseUnit-cpu-arguments": 150000, + "sndPair-cpu-arguments": 150000, + "cekLamCost-exBudgetCPU": 29773, + "fstPair-cpu-arguments": 150000, + "quotientInteger-memory-arguments-minimum": 1, + "decodeUtf8-cpu-arguments-slope": 1000, + "lessThanInteger-memory-arguments": 1, + "lessThanEqualsInteger-cpu-arguments-slope": 1366, + "fstPair-memory-arguments": 32, + "modInteger-memory-arguments-intercept": 0, + "unConstrData-cpu-arguments": 150000, + "lessThanEqualsInteger-memory-arguments": 1, + "chooseUnit-memory-arguments": 32, + "sndPair-memory-arguments": 32, + "addInteger-cpu-arguments-intercept": 197209, + "decodeUtf8-memory-arguments-slope": 8, + "equalsData-cpu-arguments-intercept": 150000, + "mapData-cpu-arguments": 150000, + "mkPairData-cpu-arguments": 150000, + "quotientInteger-cpu-arguments-constant": 148000, + "consByteString-memory-arguments-slope": 1, + "cekVarCost-exBudgetMemory": 100, + "indexByteString-cpu-arguments": 150000, + "unListData-cpu-arguments": 150000, + "equalsInteger-cpu-arguments-slope": 1326, + "cekStartupCost-exBudgetMemory": 100, + "subtractInteger-cpu-arguments-intercept": 197209, + "divideInteger-cpu-arguments-model-arguments-intercept": 425507, + "divideInteger-memory-arguments-intercept": 0, + "cekForceCost-exBudgetMemory": 100, + "blake2b-cpu-arguments-intercept": 2477736, + "remainderInteger-cpu-arguments-constant": 148000, + "tailList-cpu-arguments": 150000, + "encodeUtf8-cpu-arguments-intercept": 150000, + "equalsString-cpu-arguments-slope": 1000, + "lessThanByteString-memory-arguments": 1, + "multiplyInteger-cpu-arguments-slope": 11218, + "appendByteString-cpu-arguments-intercept": 396231, + "lessThanEqualsByteString-cpu-arguments-slope": 248, + "modInteger-memory-arguments-slope": 1, + "addInteger-cpu-arguments-slope": 0, + "equalsData-cpu-arguments-slope": 10000, + "decodeUtf8-memory-arguments-intercept": 0, + "chooseList-cpu-arguments": 150000, + "constrData-cpu-arguments": 150000, + "equalsByteString-memory-arguments": 1, + "cekApplyCost-exBudgetCPU": 29773, + "quotientInteger-memory-arguments-slope": 1, + "verifySignature-cpu-arguments-intercept": 3345831, + "unMapData-memory-arguments": 32, + "mkCons-memory-arguments": 32, + "sliceByteString-memory-arguments-slope": 1, + "sha3_256-memory-arguments": 4, + "ifThenElse-memory-arguments": 1, + "mkNilPairData-memory-arguments": 32, + "equalsByteString-cpu-arguments-slope": 247, + "appendString-cpu-arguments-intercept": 150000, + "quotientInteger-cpu-arguments-model-arguments-slope": 118, + "cekApplyCost-exBudgetMemory": 100, + "equalsString-memory-arguments": 1, + "multiplyInteger-memory-arguments-slope": 1, + "cekBuiltinCost-exBudgetMemory": 100, + "remainderInteger-memory-arguments-intercept": 0, + "sha2_256-cpu-arguments-intercept": 2477736, + "remainderInteger-cpu-arguments-model-arguments-intercept": 425507, + "lessThanEqualsByteString-memory-arguments": 1, + "tailList-memory-arguments": 32, + "mkNilData-cpu-arguments": 150000, + "chooseData-cpu-arguments": 150000, + "unBData-memory-arguments": 32, + "blake2b-memory-arguments": 4, + "iData-memory-arguments": 32, + "nullList-memory-arguments": 32, + "cekDelayCost-exBudgetCPU": 29773, + "subtractInteger-memory-arguments-intercept": 1, + "lessThanByteString-cpu-arguments-intercept": 103599, + "consByteString-cpu-arguments-slope": 1000, + "appendByteString-memory-arguments-slope": 1, + "trace-memory-arguments": 32, + "divideInteger-cpu-arguments-constant": 148000, + "cekConstCost-exBudgetCPU": 29773, + "encodeUtf8-memory-arguments-slope": 8, + "quotientInteger-cpu-arguments-model-arguments-intercept": 425507, + "mapData-memory-arguments": 32, + "appendString-cpu-arguments-slope": 1000, + "modInteger-cpu-arguments-constant": 148000, + "verifySignature-cpu-arguments-slope": 1, + "unConstrData-memory-arguments": 32, + "quotientInteger-memory-arguments-intercept": 0, + "equalsByteString-cpu-arguments-constant": 150000, + "sliceByteString-memory-arguments-intercept": 0, + "mkPairData-memory-arguments": 32, + "equalsByteString-cpu-arguments-intercept": 112536, + "appendString-memory-arguments-slope": 1, + "lessThanInteger-cpu-arguments-slope": 497, + "modInteger-cpu-arguments-model-arguments-intercept": 425507, + "modInteger-memory-arguments-minimum": 1, + "sha3_256-cpu-arguments-intercept": 0, + "verifySignature-memory-arguments": 1, + "cekLamCost-exBudgetMemory": 100, + "sliceByteString-cpu-arguments-intercept": 150000 + } + } +} \ No newline at end of file diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/byron.json b/stelae-cardano/tests/fixtures/preview-epoch-0/byron.json new file mode 100644 index 0000000..6236376 --- /dev/null +++ b/stelae-cardano/tests/fixtures/preview-epoch-0/byron.json @@ -0,0 +1,91 @@ +{ + "bootStakeholders": { + "021e737009040bf7f1e7b1bcc148f29d748d4a6b561902c95e4a9f36": 1, + "0bc82ced9544980b9ffe7f64b1538bbda6804a5cc32c8035485e184b": 1, + "18ed9844deef98cf9ba8b39791dede0538d2d2fa79bf67ef37dcc826": 1, + "66cfa84ad0ee5ca8586244c8393007cf3d9622d77cfa03fd4f35065b": 1, + "76c4d6c68c0ef81ae364411a84e52ce66089ed006ca29adfc0227901": 1, + "8cc6b89fec65cc83d34b7bab2e6494db631d8476a86625767dd0c2a0": 1, + "e90060fdc085ac9f63cdb3b32ba1d84e0f7eb98561687b213b4c8770": 1 + }, + "heavyDelegation": { + "021e737009040bf7f1e7b1bcc148f29d748d4a6b561902c95e4a9f36": { + "omega": 0, + "issuerPk": "6hSFCotivD08t02n43RMiaF9LzwtYVrFMu/WX6ShfEsxfdXFL5Y6c+DwHSZOCywU0RJz5er2icIO03UytC9NTg==", + "delegatePk": "JEnSVQTPGriTx1+lAMkKhCNsMBDNPGw+NiEvNPh4ui6IdvxrO+WkQPTy5U865XB4VFvi/zb7d+H1bilnztQNBg==", + "cert": "558952d17442e8cc73f0c7dd606e329b38ed2ec0c1f83fe2567d28b21ef2223d2d23640cd0531f75832b50e519631c48643fcfaa7168851645dce07b90d87f0e" + }, + "0bc82ced9544980b9ffe7f64b1538bbda6804a5cc32c8035485e184b": { + "omega": 0, + "issuerPk": "MJ7IskKU8GKk0Eeg3zhfSOK1DDVXOMHD2V/zhEpODUtL9YB0Y7sXnbZfg3+Df05hskP5Jz+dZvdC6DH/dP9jmQ==", + "delegatePk": "hwO7NJL7LfAk5e/QG61FKcdORoK60tvprE3063Muh4EQKrWA6l7t23B2GziK8D0hRO0j5W1Gzpn8WW69XLIlKA==", + "cert": "2bccf50d0c3cbb03dd29cfba817e8ba615db3d7722b41b264ad08722e548cfe83d069b29d13e490823d7519ecdd9940ea49573f6027056c4bd58da1adf75020e" + }, + "18ed9844deef98cf9ba8b39791dede0538d2d2fa79bf67ef37dcc826": { + "omega": 0, + "issuerPk": "pXbW4Jak8maeuWiosvrurykKnqDSHswUjroonSDS3fTnWS+BKe+vjT4zZJNKhQ33KbagiHVJ5CJUNggfsCtG2g==", + "delegatePk": "rbJAZp3kWCUvp8dnLR6qsgpGU+qKAFow4NHYKWiKCkfm1qFCFONob50N1IbNWCGWAhg38ZPTvBazTasjsfj6yQ==", + "cert": "89e1638e31fd3d402cecb897ba773d8c2c11c2d3cff2462b266e21461539b1a4fe8fb528e159b9af473799b51e49aa5b5816a88f10c484aa7cef7ad12850830a" + }, + "66cfa84ad0ee5ca8586244c8393007cf3d9622d77cfa03fd4f35065b": { + "omega": 0, + "issuerPk": "/LGZjmmcAMRisP7Rf454GM2QUKgj2aAyqE+iQo2PIEhcistFOlT+idtbLTceZAnQcwwPJDtTcNi+EnPQyscZOg==", + "delegatePk": "rinFUiKKCPPFY0ULEKn1SPRgLVmOS3jdTXDtrxK6VI1I11G3uBS1Olxi0mQSN3kf+B3hm/xHkuUDVNaSXNiBeQ==", + "cert": "3e7f30bb68c5bc4d23c2a730ac154a188a1fd45aac3f438efd380303171443d2ca4f50e5a1ff66b40ae3da64697f2599956ae06c21b73fa828b8c0dc9fb27302" + }, + "76c4d6c68c0ef81ae364411a84e52ce66089ed006ca29adfc0227901": { + "omega": 0, + "issuerPk": "9EE85tTLdSSR4T1Xoy6n9wr6jlbavCdfp9oQKusskO3DSSyNqRYS7QzYQ96j/WnphUey63082YkKijMfF9A4eA==", + "delegatePk": "dvyHDkXg8LFtb0K6Sitl8OGSEZPvfCVQYLDR6Au6t6/ROvlerMKQ8uri4fG7hQQzbHKtdKWgv94t+zuFJTQ1fw==", + "cert": "5ec0ed46ae7e575bdb089f1bceca3b2689b13a7162fe08578fe60ba64607fffaa507412a97652c3c81cc0ef93ff404cf809a628ae19faba1a035fca0505c1d04" + }, + "8cc6b89fec65cc83d34b7bab2e6494db631d8476a86625767dd0c2a0": { + "omega": 0, + "issuerPk": "Hr5S5PAxf9HSB4FzmtZzaFcXrNrctrI5XUrDrnCkOUTX6rhbtOMkXU3sWVDOvU6LNSSr3/Ws2+iCYZIr7LmTWg==", + "delegatePk": "FaLH2b5H/XS31YRnm98N6fP4Etx6m+GbniVAXMwOp8KhYXPKBJBsX/EjIy3pSkvRBhGCjsycB0yrDxWMi5ZsIQ==", + "cert": "10f06304cceb42071605ebba67b308c7568e5e6fe0d773c58f7e8c13bc8d8a340f70a4fd5e1b4a1c1db1de5c7646802bbc929d6c82d7adb8a77cb6ad77eac50a" + }, + "e90060fdc085ac9f63cdb3b32ba1d84e0f7eb98561687b213b4c8770": { + "omega": 0, + "issuerPk": "B2R+VXzy3c8bxncdOpQ2Z/tblxRNQO8AXQ0OsJDQvZYnLeGQcLD78kyYLpi3nfuS4SfnLar23NV4yiEVwaw+Yw==", + "delegatePk": "nACHGIBacymrKwn07iW/a5ZKJCPZ2cKQqeXw3ivR7WOYVUuufWhZlCoUTZ7rtBqoDaexblUQwkC7hA7AmNA3FA==", + "cert": "b5440daa05f7fae557df46e4f1b7c5802b86f465daad1137e315abf6e72f1c877207276abb8dcba86e18e42d39b34c2f0fa82ba2919944cdc8e2e5264baa450b" + } + }, + "startTime": 1666656000, + "nonAvvmBalances": { + "FHnt4NL7yPXjpZtYj1YUiX9QYYUZGXDT9gA2PJXQFkTSMx3EgawXK5BUrCHdhe2": "0", + "FHnt4NL7yPXk7D87qAWEmfnL7wSQ9AzBU2mjZt3eM48NSCbygxgzAU6vCGiRZEW": "0", + "FHnt4NL7yPXpazQsTdJ3Gp1twQUo4N5rrgGbRNSzchjchPiApc1k4CvqDMcdd7H": "0", + "FHnt4NL7yPXtNo1wLCLZyGTMfAvB14h8onafiYkM7B69ZwvGgXeUyQWfi7FPrif": "0", + "FHnt4NL7yPXtmi4mAjD43V3NB3shDs1gCuHNcMLPsRWjaw1b2yRV2xad8S8V6aq": "0", + "FHnt4NL7yPXvDWHa8bVs73UEUdJd64VxWXSFNqetECtYfTd9TtJguJ14Lu3feth": "30000000000000000", + "FHnt4NL7yPXvNSRpCYydjRr7koQCrsTtkovk5uYMimgqMJX2DyrEEBqiXaTd8rG": "0", + "FHnt4NL7yPY9rTvdsCeyRnsbzp4bN7XdmAZeU5PzA1qR2asYmN6CsdxJw4YoDjG": "0" + }, + "blockVersionData": { + "scriptVersion": 0, + "slotDuration": "20000", + "maxBlockSize": "2000000", + "maxHeaderSize": "2000000", + "maxTxSize": "4096", + "maxProposalSize": "700", + "mpcThd": "20000000000000", + "heavyDelThd": "300000000000", + "updateVoteThd": "1000000000000", + "updateProposalThd": "100000000000000", + "updateImplicit": "10000", + "softforkRule": { + "initThd": "900000000000000", + "minThd": "600000000000000", + "thdDecrement": "50000000000000" + }, + "txFeePolicy": { + "summand": "155381000000000", + "multiplier": "43946000000" + }, + "unlockStakeEpoch": "18446744073709551615" + }, + "protocolConsts": { "k": 432, "protocolMagic": 2 }, + "avvmDistr": {} +} diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/conway.json b/stelae-cardano/tests/fixtures/preview-epoch-0/conway.json new file mode 100644 index 0000000..cad84ab --- /dev/null +++ b/stelae-cardano/tests/fixtures/preview-epoch-0/conway.json @@ -0,0 +1,297 @@ +{ + "poolVotingThresholds": { + "committeeNormal": 0.51, + "committeeNoConfidence": 0.51, + "hardForkInitiation": 0.51, + "motionNoConfidence": 0.51, + "ppSecurityGroup": 0.51 + }, + "dRepVotingThresholds": { + "motionNoConfidence": 0.67, + "committeeNormal": 0.67, + "committeeNoConfidence": 0.6, + "updateToConstitution": 0.75, + "hardForkInitiation": 0.6, + "ppNetworkGroup": 0.67, + "ppEconomicGroup": 0.67, + "ppTechnicalGroup": 0.67, + "ppGovGroup": 0.75, + "treasuryWithdrawal": 0.67 + }, + "committeeMinSize": 0, + "committeeMaxTermLength": 365, + "govActionLifetime": 30, + "govActionDeposit": 100000000000, + "dRepDeposit": 500000000, + "dRepActivity": 20, + "minFeeRefScriptCostPerByte": 15, + "plutusV3CostModel": [ + 100788, + 420, + 1, + 1, + 1000, + 173, + 0, + 1, + 1000, + 59957, + 4, + 1, + 11183, + 32, + 201305, + 8356, + 4, + 16000, + 100, + 16000, + 100, + 16000, + 100, + 16000, + 100, + 16000, + 100, + 16000, + 100, + 100, + 100, + 16000, + 100, + 94375, + 32, + 132994, + 32, + 61462, + 4, + 72010, + 178, + 0, + 1, + 22151, + 32, + 91189, + 769, + 4, + 2, + 85848, + 123203, + 7305, + -900, + 1716, + 549, + 57, + 85848, + 0, + 1, + 1, + 1000, + 42921, + 4, + 2, + 24548, + 29498, + 38, + 1, + 898148, + 27279, + 1, + 51775, + 558, + 1, + 39184, + 1000, + 60594, + 1, + 141895, + 32, + 83150, + 32, + 15299, + 32, + 76049, + 1, + 13169, + 4, + 22100, + 10, + 28999, + 74, + 1, + 28999, + 74, + 1, + 43285, + 552, + 1, + 44749, + 541, + 1, + 33852, + 32, + 68246, + 32, + 72362, + 32, + 7243, + 32, + 7391, + 32, + 11546, + 32, + 85848, + 123203, + 7305, + -900, + 1716, + 549, + 57, + 85848, + 0, + 1, + 90434, + 519, + 0, + 1, + 74433, + 32, + 85848, + 123203, + 7305, + -900, + 1716, + 549, + 57, + 85848, + 0, + 1, + 1, + 85848, + 123203, + 7305, + -900, + 1716, + 549, + 57, + 85848, + 0, + 1, + 955506, + 213312, + 0, + 2, + 270652, + 22588, + 4, + 1457325, + 64566, + 4, + 20467, + 1, + 4, + 0, + 141992, + 32, + 100788, + 420, + 1, + 1, + 81663, + 32, + 59498, + 32, + 20142, + 32, + 24588, + 32, + 20744, + 32, + 25933, + 32, + 24623, + 32, + 43053543, + 10, + 53384111, + 14333, + 10, + 43574283, + 26308, + 10, + 16000, + 100, + 16000, + 100, + 962335, + 18, + 2780678, + 6, + 442008, + 1, + 52538055, + 3756, + 18, + 267929, + 18, + 76433006, + 8868, + 18, + 52948122, + 18, + 1995836, + 36, + 3227919, + 12, + 901022, + 1, + 166917843, + 4307, + 36, + 284546, + 36, + 158221314, + 26549, + 36, + 74698472, + 36, + 333849714, + 1, + 254006273, + 72, + 2174038, + 72, + 2261318, + 64571, + 4, + 207616, + 8310, + 4, + 1293828, + 28716, + 63, + 0, + 1, + 1006041, + 43623, + 251, + 0, + 1 + ], + "constitution": { + "anchor": { + "dataHash": "ca41a91f399259bcefe57f9858e91f6d00e1a38d6d9c63d4052914ea7bd70cb2", + "url": "ipfs://bafkreifnwj6zpu3ixa4siz2lndqybyc5wnnt3jkwyutci4e2tmbnj3xrdm" + }, + "script": "fa24fb305126805cf2164c161d852a0e7330cf988f1fe558cf7d4a64" + }, + "committee": { + "members": { + "scriptHash-ff9babf23fef3f54ec29132c07a8e23807d7b395b143ecd8ff79f4c7": 1000 + }, + "threshold": { + "numerator": 2, + "denominator": 3 + } + } +} \ No newline at end of file diff --git a/stelae-cardano/tests/fixtures/preview-epoch-0/shelley.json b/stelae-cardano/tests/fixtures/preview-epoch-0/shelley.json new file mode 100644 index 0000000..20abba3 --- /dev/null +++ b/stelae-cardano/tests/fixtures/preview-epoch-0/shelley.json @@ -0,0 +1,68 @@ +{ + "activeSlotsCoeff": 0.05, + "epochLength": 86400, + "genDelegs": { + "12b0f443d02861948a0fce9541916b014e8402984c7b83ad70a834ce": { + "delegate": "7c54a168c731f2f44ced620f3cca7c2bd90731cab223d5167aa994e6", + "vrf": "62d546a35e1be66a2b06e29558ef33f4222f1c466adbb59b52d800964d4e60ec" + }, + "3df542796a64e399b60c74acfbdb5afa1e114532fa36b46d6368ef3a": { + "delegate": "c44bc2f3cc7e98c0f227aa399e4035c33c0d775a0985875fff488e20", + "vrf": "4f9d334decadff6eba258b2df8ae1f02580a2628bce47ae7d957e1acd3f42a3c" + }, + "93fd5083ff20e7ab5570948831730073143bea5a5d5539852ed45889": { + "delegate": "82a02922f10105566b70366b07c758c8134fa91b3d8ae697dfa5e8e0", + "vrf": "8a57e94a9b4c65ec575f35d41edb1df399fa30fdf10775389f5d1ef670ca3f9f" + }, + "a86cab3ea72eabb2e8aafbbf4abbd2ba5bdfd04eea26a39b126a78e4": { + "delegate": "10257f6d3bae913514bdc96c9170b3166bf6838cca95736b0e418426", + "vrf": "1b54aad6b013145a0fc74bb5c2aa368ebaf3999e88637d78e09706d0cc29874a" + }, + "b799804a28885bd49c0e1b99d8b3b26de0fac17a5cf651ecf0c872f0": { + "delegate": "ebe606e22d932d51be2c1ce87e7d7e4c9a7d1f7df4a5535c29e23d22", + "vrf": "b3fc06a1f8ee69ff23185d9af453503be8b15b2652e1f9fb7c3ded6797a2d6f9" + }, + "d125812d6ab973a2c152a0525b7fd32d36ff13555a427966a9cac9b1": { + "delegate": "e302198135fb5b00bfe0b9b5623426f7cf03179ab7ba75f945d5b79b", + "vrf": "b45ca2ed95f92248fa0322ce1fc9f815a5a5aa2f21f1adc2c42c4dccfc7ba631" + }, + "ef27651990a26449a40767d5e06cdef1670a3f3ff4b951d385b51787": { + "delegate": "0e0b11e80d958732e587585d30978d683a061831d1b753878f549d05", + "vrf": "b860ec844f6cd476c4fabb4aa1ca72d5c74d82f3835aed3c9515a35b6e048719" + } + }, + "initialFunds": {}, + "maxKESEvolutions": 62, + "maxLovelaceSupply": 45000000000000000, + "networkId": "Testnet", + "networkMagic": 2, + "protocolParams": { + "protocolVersion": { + "minor": 0, + "major": 6 + }, + "decentralisationParam": 1, + "eMax": 18, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "maxTxSize": 16384, + "maxBlockBodySize": 65536, + "maxBlockHeaderSize": 1100, + "minFeeA": 44, + "minFeeB": 155381, + "minUTxOValue": 1000000, + "poolDeposit": 500000000, + "minPoolCost": 340000000, + "keyDeposit": 2000000, + "nOpt": 150, + "rho": 0.003, + "tau": 0.20, + "a0": 0.3 + }, + "securityParam": 432, + "slotLength": 1, + "slotsPerKESPeriod": 129600, + "systemStart": "2022-10-25T00:00:00Z", + "updateQuorum": 5 +} diff --git a/stelae-cardano/tests/publisher_parity.rs b/stelae-cardano/tests/publisher_parity.rs new file mode 100644 index 0000000..e6865b9 --- /dev/null +++ b/stelae-cardano/tests/publisher_parity.rs @@ -0,0 +1,670 @@ +//! Executable old/new-host parity over pinned public Preview chain data. +//! +//! The fixture is deliberately not synthetic: it is the first 4,958 blocks +//! obtained from the public Preview relay. Both hosts replay it into +//! independent stores and stop on the first epoch boundary before publishing. +//! The test is ignored in the ordinary unit-test lane because it is a +//! multi-minute replay; the dedicated parity CI lane runs it explicitly. + +use std::{ + fs, + io::{Read as _, Write as _}, + net::TcpStream, + path::Path, + process::Command, + sync::Arc, + time::{Duration, Instant}, +}; + +use dolos::engine::{BulkReplaySession, ReplayWorkspace}; +use dolos_core::{config::RootConfig, Genesis, ReplayProgress}; +use dolos_flatfiles::{BlockLocation, FlatFileStore}; +use dolos_snapshot::{ + facade::SnapshotSource, + planning, + publisher::{Next as OldNext, Publisher as OldPublisher, RepositoryPublish as OldPublish}, + registry::{Auth, Point as RepositoryPoint, Published, Repository, SnapshotRepository, Tuning}, +}; +use pallas::ledger::traverse::MultiEraBlock; +use stelae::progress::Observer; +use stelae_cardano::{ + initialize, + publisher::{publish_once, Destination, PublishOutcome}, + Point, RestoreSource, Selection, +}; + +const OLD_HOST_REVISION: &str = "1ae4e91c18a9e1456a3612af402d7b9b97546d30"; +const FIXTURE_SHA256: &str = "728c5aa9c7c78c4221bdee765f3cd109b630cb00266ab824f6db2bfa43276b4d"; +const FIXTURE_BLOCKS: usize = 4_958; +const PREVIEW_EPOCH_LENGTH: u64 = 86_400; +const BOUNDARY_HASH: &str = "4a9761ddc291b0c352d1712b624759132936a07b3e02d1d3bdaaf17b9abfe683"; +const LAST_HASH: &str = "da6efb517e112a8439820d505286964bcc65ddd073b560a8f04d1c0902f1125f"; + +type AnyError = Box; + +struct Node { + root: tempfile::TempDir, + config: RootConfig, +} + +impl Node { + fn new() -> Self { + let root = tempfile::tempdir().unwrap(); + let document = format!( + r#" + [upstream] + peer_address = "unused.invalid:3001" + + [storage] + version = "v4" + path = {} + + [genesis] + byron_path = "unused" + shelley_path = "unused" + alonzo_path = "unused" + conway_path = "unused" + + [snapshot] + state_epochs = [1] + + [chain] + type = "cardano" + magic = 2 + is_testnet = true + "#, + toml::Value::String(root.path().join("data").display().to_string()), + ); + Self { + root, + config: toml::from_str(&document).unwrap(), + } + } + + fn stele(&self, name: &str) -> std::path::PathBuf { + self.root.path().join(name) + } +} + +fn fixture_path() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/preview-epoch-0/000000.segment") +} + +fn fixture_genesis() -> Genesis { + let root = fixture_path().parent().unwrap().to_owned(); + Genesis::from_file_paths( + root.join("byron.json"), + root.join("shelley.json"), + root.join("alonzo.json"), + root.join("conway.json"), + Some(6), + ) + .unwrap() +} + +fn read_fixture() -> Vec>> { + let path = fixture_path(); + for (name, expected) in [ + ( + "byron.json", + "101650cc8ccbd020f6ceae29bcee2aed5cbdb0bebaa26288d132a9dc3299d9cd", + ), + ( + "shelley.json", + "c5ccb45161676718a8c08b1362ec1ef2cee516fd123aecafacf0f3e4625a746a", + ), + ( + "alonzo.json", + "3511a116de9496a84f956ac50c90dbe92e95d0fbee7b2d7134fb4d407efb9d0e", + ), + ( + "conway.json", + "51ae5b86ae59f872462f22c5034b37d8f8b8561bb7a7ad5c6d9b45502ca65b9e", + ), + ] { + assert_eq!( + sha256(&fs::read(path.with_file_name(name)).unwrap()), + expected + ); + } + let bytes = fs::read(&path).unwrap(); + assert_eq!(sha256(&bytes), FIXTURE_SHA256, "fixture identity drifted"); + let store = FlatFileStore::new(path.parent().unwrap()).unwrap(); + let mut blocks = Vec::new(); + let mut offset = 0usize; + while offset < bytes.len() { + let length = zstd_safe::find_frame_compressed_size(&bytes[offset..]).unwrap(); + let body = store + .read(&BlockLocation { + segment_id: 0, + offset: offset as u64, + length: length as u32, + }) + .unwrap(); + blocks.push(Arc::new(body)); + offset += length; + } + assert_eq!(offset, bytes.len()); + assert_eq!(blocks.len(), FIXTURE_BLOCKS); + let first = MultiEraBlock::decode(blocks[0].as_slice()).unwrap(); + let last = MultiEraBlock::decode(blocks.last().unwrap().as_slice()).unwrap(); + let boundary = blocks + .iter() + .map(|body| MultiEraBlock::decode(body.as_slice()).unwrap()) + .find(|block| block.slot() >= PREVIEW_EPOCH_LENGTH) + .unwrap(); + assert_eq!((first.slot(), first.number()), (0, 0)); + assert_eq!((boundary.slot(), boundary.number()), (86_400, 4_320)); + assert_eq!(boundary.hash().to_string(), BOUNDARY_HASH); + assert_eq!((last.slot(), last.number()), (99_140, 4_957)); + assert_eq!(last.hash().to_string(), LAST_HASH); + blocks +} + +fn sha256(bytes: &[u8]) -> String { + use sha2::{Digest as _, Sha256}; + hex::encode(Sha256::digest(bytes)) +} + +fn replay(node: &Node, blocks: Vec>>) -> (u64, u128) { + let started = Instant::now(); + let genesis = Arc::new(fixture_genesis()); + let mut replay = BulkReplaySession::open(&node.config, genesis, Some(1)).unwrap(); + let ReplayProgress::Boundary { position } = replay.import_blocks(blocks).unwrap() else { + panic!("real fixture did not stop at the first Preview epoch boundary") + }; + assert!(position.slot() >= PREVIEW_EPOCH_LENGTH); + replay.finish().unwrap(); + (position.slot(), started.elapsed().as_millis()) +} + +fn old_host_publish(node: &Node, output: &Path) -> (String, u128) { + let started = Instant::now(); + let genesis = Arc::new(fixture_genesis()); + let workspace = ReplayWorkspace::open(&node.config, genesis.clone()).unwrap(); + let result = (|| { + let snapshot = workspace.snapshot(); + let retained = planning::retained_epochs(&node.config)?; + let plan = snapshot.selected_plan( + u64::from(genesis.network_magic()), + retained, + Selection::default(), + )?; + let inscription = snapshot.publish_directory(output, &plan, &Observer::silent())?; + Ok::<_, dolos_snapshot::Error>(inscription.digest()?.to_string()) + })(); + workspace.finish().unwrap(); + (result.unwrap(), started.elapsed().as_millis()) +} + +fn new_host_publish(node: &Node, output: &Path) -> (String, u128) { + let started = Instant::now(); + let genesis = fixture_genesis(); + let (_, outcome) = publish_once( + &node.config, + &genesis, + Selection::default(), + Destination::Directory(output.to_owned()), + false, + &Observer::silent(), + ) + .unwrap(); + let PublishOutcome::Directory { identity, .. } = outcome else { + panic!("new host did not publish a directory stele") + }; + (identity, started.elapsed().as_millis()) +} + +fn old_host_publish_repository( + node: &Node, + repository: &Repository, + rebuild: bool, + dry_run: bool, + require_new: bool, +) -> Result, AnyError> { + let genesis = Arc::new(fixture_genesis()); + let workspace = ReplayWorkspace::open(&node.config, genesis.clone())?; + let result: Result, dolos_snapshot::Error> = (|| { + let snapshot = workspace.snapshot(); + let plan = snapshot.selected_plan( + u64::from(genesis.network_magic()), + planning::retained_epochs(&node.config)?, + Selection::default(), + )?; + let settings = OldPublish { + repo: repository, + insecure: true, + scratch_dir: None, + rebuild, + dry_run, + require_new, + tuning: Tuning::default(), + }; + let publisher = OldPublisher::open(&node.config, &settings, Auth::Anonymous)?; + match OldNext::read(publisher.standing(&plan)?, plan.sequence, require_new)? { + OldNext::Nothing(_) => Ok(None), + OldNext::First | OldNext::After { .. } => { + publisher.preflight()?; + if dry_run { + let _ = snapshot.preview(&publisher, &plan)?; + Ok(None) + } else { + Ok(Some(snapshot.publish( + &publisher, + &plan, + &Observer::silent(), + )?)) + } + } + } + })(); + let finish = workspace + .finish() + .map_err(|error| -> AnyError { Box::new(error) }); + match ( + result.map_err(|error| -> AnyError { Box::new(error) }), + finish, + ) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error), + (Err(operation), Err(finish)) => Err(format!( + "publication failed ({operation}); closing stores also failed ({finish})" + ) + .into()), + } +} + +fn new_host_publish_repository( + node: &Node, + repository: &Repository, + rebuild: bool, + dry_run: bool, + require_new: bool, +) -> Result { + let genesis = fixture_genesis(); + let (_, outcome) = publish_once( + &node.config, + &genesis, + Selection::default(), + Destination::Repository { + repository: repository.clone(), + insecure: true, + scratch_dir: None, + rebuild, + concurrency: None, + verify_carried: false, + require_new, + }, + dry_run, + &Observer::silent(), + )?; + Ok(outcome) +} + +struct LocalRegistry { + container: String, + address: String, +} + +impl LocalRegistry { + fn spawn() -> Self { + static CRYPTO: std::sync::Once = std::sync::Once::new(); + CRYPTO.call_once(|| { + rustls::crypto::ring::default_provider() + .install_default() + .expect("nothing installed a rustls provider first"); + }); + + let image = + std::env::var("STELAE_TEST_REGISTRY_IMAGE").unwrap_or_else(|_| "registry:2".to_owned()); + let run = Command::new("docker") + .args([ + "run", + "--detach", + "--rm", + "--publish", + "127.0.0.1::5000", + &image, + ]) + .output() + .expect("docker is required to run publisher parity"); + assert!( + run.status.success(), + "docker run {image}: {}", + String::from_utf8_lossy(&run.stderr) + ); + let container = String::from_utf8(run.stdout).unwrap().trim().to_owned(); + let ports = Command::new("docker") + .args(["port", &container, "5000/tcp"]) + .output() + .expect("docker port"); + let mapped = String::from_utf8(ports.stdout).unwrap(); + let port = mapped + .lines() + .find_map(|line| line.rsplit(':').next()) + .and_then(|port| port.trim().parse::().ok()) + .unwrap_or_else(|| panic!("no published port in {mapped:?}")); + let fixture = Self { + container, + address: format!("127.0.0.1:{port}"), + }; + fixture.wait_until_ready(); + fixture + } + + fn repository(&self, name: &str) -> Repository { + format!("oci://{}/{name}", self.address).parse().unwrap() + } + + fn wait_until_ready(&self) { + for _ in 0..300 { + if let Ok(mut stream) = TcpStream::connect(&self.address) { + let request = format!( + "GET /v2/ HTTP/1.1\r\nHost: {}\r\nConnection: close\r\n\r\n", + self.address + ); + if stream.write_all(request.as_bytes()).is_ok() { + let mut response = String::new(); + if stream.read_to_string(&mut response).is_ok() + && response.starts_with("HTTP/1.1 200") + { + return; + } + } + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!("registry never answered on {}", self.address); + } +} + +impl Drop for LocalRegistry { + fn drop(&mut self) { + let _ = Command::new("docker") + .args(["rm", "--force", &self.container]) + .output(); + } +} + +fn inspect_repository(repository: &Repository, scratch: &Path) -> (String, Vec, u64) { + let repository = SnapshotRepository::open( + repository, + true, + Auth::Anonymous, + scratch.to_owned(), + Tuning::default(), + ) + .unwrap(); + let inspected = repository.inspect(RepositoryPoint::Latest).unwrap(); + ( + inspected.identity.to_string(), + inspected.inscription.canonicalize().unwrap(), + inspected.total_compressed, + ) +} + +fn restore_with_unchanged_consumer(source: &Path) -> Node { + let node = Node::new(); + let genesis = fixture_genesis(); + let outcome = initialize::run( + &node.config, + &genesis, + &initialize::Options { + source: RestoreSource::Dir(source.to_owned()), + point: Point::Latest, + insecure: false, + scratch_dir: None, + skip_space_check: false, + resume: false, + allow_genesis: false, + }, + &Observer::silent(), + ) + .unwrap(); + assert!(matches!( + outcome, + initialize::Outcome::Restored { sequence: 1 } + )); + node +} + +fn restore_repository_with_unchanged_consumer(source: Repository) -> Node { + let node = Node::new(); + let genesis = fixture_genesis(); + let outcome = initialize::run( + &node.config, + &genesis, + &initialize::Options { + source: RestoreSource::Repo(source), + point: RepositoryPoint::Latest, + insecure: true, + scratch_dir: None, + skip_space_check: false, + resume: false, + allow_genesis: false, + }, + &Observer::silent(), + ) + .unwrap(); + assert!(matches!( + outcome, + initialize::Outcome::Restored { sequence: 1 } + )); + node +} + +fn directory_bytes(path: &Path) -> u64 { + if !path.exists() { + return 0; + } + fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().path()) + .map(|path| { + if path.is_dir() { + directory_bytes(&path) + } else { + path.metadata().unwrap().len() + } + }) + .sum() +} + +#[cfg(unix)] +fn peak_rss_bytes() -> u64 { + let mut usage = std::mem::MaybeUninit::::zeroed(); + // SAFETY: getrusage initializes the supplied rusage on a zero return. + let rc = unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }; + assert_eq!(rc, 0, "getrusage failed"); + let rss = unsafe { usage.assume_init() }.ru_maxrss as u64; + if cfg!(target_os = "macos") { + rss + } else { + rss * 1024 + } +} + +#[cfg(not(unix))] +fn peak_rss_bytes() -> u64 { + 0 +} + +#[test] +#[ignore = "replays 4,958 real Preview blocks; run by the parity CI lane"] +fn old_and_new_hosts_replay_publish_and_restore_the_same_real_boundary() { + let fixture = read_fixture(); + let old = Node::new(); + let new = Node::new(); + let (old_slot, old_replay_ms) = replay(&old, fixture.clone()); + let (new_slot, new_replay_ms) = replay(&new, fixture); + assert_eq!(old_slot, new_slot); + + let old_dir = old.stele("old-host-stele"); + let new_dir = new.stele("new-host-stele"); + let (old_identity, old_publish_ms) = old_host_publish(&old, &old_dir); + let old_peak_rss_bytes = peak_rss_bytes(); + let (new_identity, new_publish_ms) = new_host_publish(&new, &new_dir); + let new_peak_rss_bytes = peak_rss_bytes(); + assert_eq!(old_identity, new_identity); + assert_eq!( + fs::read(old_dir.join("inscription.json")).unwrap(), + fs::read(new_dir.join("inscription.json")).unwrap() + ); + + let restored = restore_with_unchanged_consumer(&new_dir); + let restored_dir = restored.stele("restored-stele"); + let (restored_identity, _) = old_host_publish(&restored, &restored_dir); + assert_eq!(new_identity, restored_identity); + + let registry = LocalRegistry::spawn(); + let old_repository = registry.repository("parity/old-host"); + let new_repository = registry.repository("parity/new-host"); + + assert!( + old_host_publish_repository(&old, &old_repository, false, true, false) + .unwrap() + .is_none() + ); + assert!(matches!( + new_host_publish_repository(&new, &new_repository, false, true, false).unwrap(), + PublishOutcome::DryRun { sequence: 1 } + )); + + let old_published = old_host_publish_repository(&old, &old_repository, false, false, false) + .unwrap() + .expect("the old host should make the first publication"); + let PublishOutcome::Repository { + sequence: 1, + identity: new_registry_identity, + built: new_built, + reused: new_reused, + transfer: new_transfer, + } = new_host_publish_repository(&new, &new_repository, false, false, false).unwrap() + else { + panic!("the new host should make the first publication") + }; + assert_eq!(old_published.identity.to_string(), new_registry_identity); + assert_eq!(old_published.layers_built, new_built); + assert_eq!(old_published.layers_reused, new_reused); + assert_eq!(old_published.transfer, new_transfer); + + let old_inspected = inspect_repository(&old_repository, &old.stele("inspect-scratch")); + let new_inspected = inspect_repository(&new_repository, &new.stele("inspect-scratch")); + assert_eq!(old_inspected, new_inspected); + assert_eq!(old_inspected.0, new_identity); + + assert!( + old_host_publish_repository(&old, &old_repository, false, false, false,) + .unwrap() + .is_none() + ); + assert!(matches!( + new_host_publish_repository(&new, &new_repository, false, false, false).unwrap(), + PublishOutcome::Nothing(_) + )); + assert!(old_host_publish_repository(&old, &old_repository, false, false, true).is_err()); + assert!(new_host_publish_repository(&new, &new_repository, false, false, true).is_err()); + + let rebuilt_old_repository = registry.repository("parity/old-host-rebuild"); + let rebuilt_new_repository = registry.repository("parity/new-host-rebuild"); + let rebuilt_old = + old_host_publish_repository(&old, &rebuilt_old_repository, true, false, false) + .unwrap() + .expect("the old host should force a fresh reproduction"); + let PublishOutcome::Repository { + sequence: 1, + identity: rebuilt_new_identity, + built: rebuilt_new_built, + reused: 0, + .. + } = new_host_publish_repository(&new, &rebuilt_new_repository, true, false, false).unwrap() + else { + panic!("the new host should force a fresh reproduction") + }; + assert_eq!(rebuilt_old.identity.to_string(), rebuilt_new_identity); + assert_eq!(rebuilt_new_identity, new_registry_identity); + assert_eq!(rebuilt_old.layers_built, rebuilt_new_built); + assert_eq!(rebuilt_old.layers_reused, 0); + + let restored_registry = restore_repository_with_unchanged_consumer(new_repository.clone()); + let restored_registry_dir = restored_registry.stele("restored-registry-stele"); + let (restored_registry_identity, _) = + old_host_publish(&restored_registry, &restored_registry_dir); + assert_eq!(new_registry_identity, restored_registry_identity); + + let report = serde_json::json!({ + "schema": 1, + "outcome": "pass", + "hosts": { + "old": {"revision": OLD_HOST_REVISION, "kind": "dolos snapshot publish"}, + "new": { + "revision": option_env!("STELAE_PARITY_NEW_REVISION").unwrap_or("worktree"), + "kind": "stelae-publisher" + } + }, + "fixture": { + "network": "preview", + "block_count": FIXTURE_BLOCKS, + "sha256": FIXTURE_SHA256, + "boundary_slot": old_slot + }, + "expected": {"sequence": 1, "identity": new_identity}, + "checks": [ + {"name": "real-boundary replay", "outcome": "pass"}, + {"name": "source position", "outcome": "pass"}, + {"name": "layer diffIds and canonical inscription", "outcome": "pass"}, + {"name": "old/new host OCI publication", "outcome": "pass"}, + {"name": "dry-run, no-op and force-rebuild agreement", "outcome": "pass"}, + {"name": "unchanged Dolos directory and OCI consumer restore", "outcome": "pass"} + ], + "registry": { + "kind": "disposable registry:2", + "repositories": "separate", + "identity": new_registry_identity, + "compressed_bytes": old_inspected.2, + "old_transfer": { + "layers_uploaded": old_published.transfer.layers_uploaded, + "layers_skipped": old_published.transfer.layers_skipped, + "layers_reused": old_published.transfer.layers_reused, + "bytes_uploaded": old_published.transfer.bytes_uploaded, + "bytes_skipped": old_published.transfer.bytes_skipped, + "bytes_reused": old_published.transfer.bytes_reused + }, + "new_transfer": { + "layers_uploaded": new_transfer.layers_uploaded, + "layers_skipped": new_transfer.layers_skipped, + "layers_reused": new_transfer.layers_reused, + "bytes_uploaded": new_transfer.bytes_uploaded, + "bytes_skipped": new_transfer.bytes_skipped, + "bytes_reused": new_transfer.bytes_reused + } + }, + "resources": { + "old": { + "replay_ms": old_replay_ms, + "publish_ms": old_publish_ms, + "peak_rss_bytes": old_peak_rss_bytes, + "artifact_bytes": directory_bytes(&old_dir), + "transfer_bytes": 0, + "scratch_bytes": 0 + }, + "new": { + "replay_ms": new_replay_ms, + "publish_ms": new_publish_ms, + "peak_rss_bytes": new_peak_rss_bytes, + "artifact_bytes": directory_bytes(&new_dir), + "transfer_bytes": 0, + "scratch_bytes": 0 + } + } + }); + let rendered = serde_json::to_string_pretty(&report).unwrap(); + eprintln!("{rendered}"); + if let Some(path) = std::env::var_os("STELAE_PARITY_REPORT") { + let path = std::path::PathBuf::from(path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, format!("{rendered}\n")).unwrap(); + } +} diff --git a/stelae-publisher/src/main.rs b/stelae-publisher/src/main.rs index a64a05d..5b3716a 100644 --- a/stelae-publisher/src/main.rs +++ b/stelae-publisher/src/main.rs @@ -335,8 +335,12 @@ fn run_publish( identity, built, reused, + transfer, } => { - println!("wrote sequence {sequence}: {identity} ({built} built, {reused} reused)") + println!( + "wrote sequence {sequence}: {identity} ({built} built, {reused} reused; {} bytes uploaded, {} skipped, {} reused)", + transfer.bytes_uploaded, transfer.bytes_skipped, transfer.bytes_reused + ) } } Ok(())