diff --git a/.github/workflows/check-mainnet-config.yml b/.github/workflows/check-mainnet-config.yml index 9b3c7ebf98a..4f4eb657b9b 100644 --- a/.github/workflows/check-mainnet-config.yml +++ b/.github/workflows/check-mainnet-config.yml @@ -43,7 +43,6 @@ jobs: 'mainnet-byron-genesis.json' 'mainnet-checkpoints.json' 'mainnet-config.json' - 'mainnet-config-legacy.json' 'mainnet-peer-snapshot.json' 'mainnet-shelley-genesis.json' 'mainnet-topology.json' diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json deleted file mode 100644 index f4bc557037e..00000000000 --- a/configuration/cardano/mainnet-config-legacy.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "AlonzoGenesisFile": "mainnet-alonzo-genesis.json", - "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", - "ByronGenesisFile": "mainnet-byron-genesis.json", - "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb", - "CheckpointsFile": "mainnet-checkpoints.json", - "CheckpointsFileHash": "3e6dee5bae7acc6d870187e72674b37c929be8c66e62a552cf6a876b1af31ade", - "ConsensusMode": "PraosMode", - "ConwayGenesisFile": "mainnet-conway-genesis.json", - "ConwayGenesisHash": "15a199f895e461ec0ffc6dd4e4028af28a492ab4e806d39cb674c88f7643ef62", - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 0, - "LedgerDB": { - "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, - "QueryBatchSize": 100000, - "SnapshotInterval": 4320 - }, - "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresNoMagic", - "ShelleyGenesisFile": "mainnet-shelley-genesis.json", - "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81", - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": true, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalConnectionManager": true, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": true, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Info", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ] -} diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json index b587e72e99d..5027e9dd27a 100644 --- a/configuration/cardano/mainnet-config.json +++ b/configuration/cardano/mainnet-config.json @@ -13,12 +13,14 @@ "LastKnownBlockVersion-Minor": 0, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 4320 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 4320 + } }, "MaxKnownMajorProtocolVersion": 2, - "MinNodeVersion": "10.7.0", + "MinNodeVersion": "11.1.0", "Protocol": "Cardano", "RequiresNetworkMagic": "RequiresNoMagic", "ShelleyGenesisFile": "mainnet-shelley-genesis.json", @@ -107,14 +109,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + } } diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml index 86f805d4e8e..51c99d46458 100644 --- a/configuration/cardano/mainnet-config.yaml +++ b/configuration/cardano/mainnet-config.yaml @@ -79,39 +79,44 @@ ConsensusMode: PraosMode # Additional configuration options can be found at: # https://ouroboros-consensus.cardano.intersectmbo.org/docs/for-developers/utxo-hd/migrating -LedgerDB: - # The time interval between snapshots, in seconds. - SnapshotInterval: 4320 - - # The number of disk snapshots to keep. - NumOfDiskSnapshots: 2 +LedgerDB: # When querying the store for a big range of UTxOs (such as with # QueryUTxOByAddress), the store will be read in batches of this size. QueryBatchSize: 100000 # The backend can either be in memory with `V2InMemory` or on disk with - # `V1LMDB`. + # `V2LSM`. Backend: V2InMemory -##### Version Information ##### + # Instead of an object with individual options, a predefined snapshot + # policy can be selected by name, e.g. `Snapshots: Mithril`. + Snapshots: + # The snapshot interval in slots. + SnapshotInterval: 4320 -# Min is currently 10.7.0 due to change of config bundled peer-snapshot -# version. -MinNodeVersion: "10.7.0" + # Start taking the snaphots at a slot offset. + # SlotOffset = 172800; -##### Logging configuration ##### + # A minimum duration between snapshots, in seconds (used to avoid excessive snapshots while syncing). + # Default is 10 minutes. + # RateLimit = 600; -# Enable or disable logging overall -TurnOnLogging: True + # Randomised snapshot delay range, in seconds. + # Both Min and Max need to be specified, otherwise the default delay of (5min, 10min) will be used. + # MinDelay = 300; + # MaxDelay = 600; -# Enable the collection of various OS metrics such as memory and CPU use. -# These metrics are traced in the context name: 'cardano.node.metrics' and can -# be directed to the logs or monitoring backends. -TurnOnLogMetrics: True + # The number of disk snapshots to keep. + NumOfDiskSnapshots: 2 -# Use the modern tracing system instead of the legacy tracing system. -UseTraceDispatcher: True +##### Version Information ##### + +# Min is currently 11.1.0 due to removal of legacy tracing system and +# introduction of deterministic snapshots. +MinNodeVersion: "11.1.0" + +##### Logging configuration ##### # Match the metrics prefix of the legacy tracing system to minimize breaking # changes. @@ -255,17 +260,6 @@ TraceOptions: Mempool.SyncNotNeeded: severity: Silence -# Required by the legacy tracing system, this key is still required for -# cardano-node to start. -minSeverity: Critical - -# Required by some legacy tests which may otherwise fail to start. -defaultBackends: [] -defaultScribes: [] -options: {} -setupBackends: [] -setupScribes: [] - # Set or unset the mempool capacity override in number of bytes. # # This is intended for testing, and for low-resource machines to run with a smaller mempool. diff --git a/configuration/cardano/testnet-template-config-legacy.json b/configuration/cardano/testnet-template-config-legacy.json deleted file mode 100644 index 1d58efae3f3..00000000000 --- a/configuration/cardano/testnet-template-config-legacy.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "AlonzoGenesisFile": "alonzo-genesis.json", - "ApplicationName": "cardano-sl", - "ApplicationVersion": 0, - "ByronGenesisFile": "byron-genesis.json", - "ConwayGenesisFile": "conway-genesis.json", - "DijkstraGenesisFile": "dijkstra-genesis.json", - "ExperimentalHardForksEnabled": false, - "ExperimentalProtocolsEnabled": true, - "LastKnownBlockVersion-Alt": 0, - "LastKnownBlockVersion-Major": 3, - "LastKnownBlockVersion-Minor": 1, - "LedgerDB": { - "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, - "QueryBatchSize": 100000, - "SnapshotInterval": 216 - }, - "MaxConcurrencyDeadline": 4, - "MaxKnownMajorProtocolVersion": 2, - "PBftSignatureThreshold": 1.1, - "Protocol": "Cardano", - "RequiresNetworkMagic": "RequiresMagic", - "ShelleyGenesisFile": "shelley-genesis.json", - "TestAllegraHardForkAtEpoch": 0, - "TestAlonzoHardForkAtEpoch": 0, - "TestMaryHardForkAtEpoch": 0, - "TestShelleyHardForkAtEpoch": 0, - "TraceAcceptPolicy": true, - "TraceBlockFetchClient": false, - "TraceBlockFetchDecisions": false, - "TraceBlockFetchProtocol": false, - "TraceBlockFetchProtocolSerialised": false, - "TraceBlockFetchServer": false, - "TraceChainDb": true, - "TraceChainSyncBlockServer": false, - "TraceChainSyncClient": false, - "TraceChainSyncHeaderServer": false, - "TraceChainSyncProtocol": false, - "TraceConnectionManager": true, - "TraceDNSResolver": true, - "TraceDNSSubscription": true, - "TraceDiffusionInitialization": true, - "TraceErrorPolicy": true, - "TraceForge": true, - "TraceHandshake": false, - "TraceInboundGovernor": true, - "TraceIpSubscription": true, - "TraceLedgerPeers": true, - "TraceLocalChainSyncProtocol": false, - "TraceLocalErrorPolicy": true, - "TraceLocalHandshake": false, - "TraceLocalRootPeers": true, - "TraceLocalTxSubmissionProtocol": false, - "TraceLocalTxSubmissionServer": false, - "TraceMempool": false, - "TraceMux": false, - "TracePeerSelection": true, - "TracePeerSelectionActions": true, - "TracePublicRootPeers": true, - "TraceServer": true, - "TraceTxInbound": false, - "TraceTxOutbound": false, - "TraceTxSubmissionProtocol": false, - "TracingVerbosity": "NormalVerbosity", - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": false, - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "cardano" - ] - ], - "hasEKG": 12788, - "hasPrometheus": [ - "127.0.0.1", - 12798 - ], - "minSeverity": "Debug", - "options": { - "mapBackends": { - "cardano.node.metrics": [ - "EKGViewBK" - ], - "cardano.node.resources": [ - "EKGViewBK" - ] - }, - "mapSubtrace": { - "cardano.node.metrics": { - "subtrace": "Neutral" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "cardano" - } - ] -} diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json index 0ab850d4bad..720f362803d 100644 --- a/configuration/cardano/testnet-template-config.json +++ b/configuration/cardano/testnet-template-config.json @@ -13,9 +13,11 @@ "LastKnownBlockVersion-Minor": 1, "LedgerDB": { "Backend": "V2InMemory", - "NumOfDiskSnapshots": 2, "QueryBatchSize": 100000, - "SnapshotInterval": 216 + "Snapshots": { + "NumOfDiskSnapshots": 2, + "SnapshotInterval": 216 + } }, "MaxConcurrencyDeadline": 4, "MaxKnownMajorProtocolVersion": 2, @@ -111,14 +113,5 @@ "Startup.DiffusionInit": { "severity": "Info" } - }, - "TurnOnLogMetrics": true, - "TurnOnLogging": true, - "UseTraceDispatcher": true, - "defaultBackends": [], - "defaultScribes": [], - "minSeverity": "Critical", - "options": {}, - "setupBackends": [], - "setupScribes": [] + } } diff --git a/configuration/cardano/update-config-files.sh b/configuration/cardano/update-config-files.sh index 28ca99ed46b..f7693918c8b 100755 --- a/configuration/cardano/update-config-files.sh +++ b/configuration/cardano/update-config-files.sh @@ -30,7 +30,6 @@ copyCfg "mainnet-alonzo-genesis.json" copyCfg "mainnet-byron-genesis.json" copyCfg "mainnet-checkpoints.json" copyCfg "mainnet-config.json" -copyCfg "mainnet-config-legacy.json" copyCfg "mainnet-conway-genesis.json" copyCfg "mainnet-peer-snapshot.json" copyCfg "mainnet-shelley-genesis.json" @@ -40,7 +39,6 @@ copyCfg "mainnet-topology.json" copyTmplCfg "alonzo.json" copyTmplCfg "byron.json" copyTmplCfg "config.json" -copyTmplCfg "config-legacy.json" copyTmplCfg "conway.json" copyTmplCfg "dijkstra.json" copyTmplCfg "shelley.json" diff --git a/flake.lock b/flake.lock index 2f38f596a20..412d023b1fc 100644 --- a/flake.lock +++ b/flake.lock @@ -142,6 +142,21 @@ "type": "github" } }, + "crane": { + "locked": { + "lastModified": 1776635034, + "narHash": "sha256-OEOJrT3ZfwbChzODfIH4GzlNTtOFuZFWPtW7jIeR8xU=", + "owner": "ipetkov", + "repo": "crane", + "rev": "dc7496d8ea6e526b1254b55d09b966e94673750f", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "customConfig": { "locked": { "lastModified": 1630400035, @@ -206,6 +221,24 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1667395993, @@ -604,15 +637,16 @@ "sodium": "sodium" }, "locked": { - "lastModified": 1777941182, - "narHash": "sha256-FX3+8GIrB2z4akmcYTStELDKVJWgqy9yFt0mxwpU3Qc=", + "lastModified": 1782080462, + "narHash": "sha256-jHlPlNk/3PKEx+A++IiWGX3jQtTYzsIM0iTub/VOn2g=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "9de00113c11ba8cac908a63acf34b193cda7475b", + "rev": "d7af9d6d6cd3efc91a23772e52fc24a01117f798", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "node-11.1", "repo": "iohk-nix", "type": "github" } @@ -634,6 +668,29 @@ "type": "github" } }, + "mithril": { + "inputs": { + "crane": "crane", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1776926918, + "narHash": "sha256-muV2LpheC4OZsU1ipL9j6TmjGufMpGrXzvon+eWahgg=", + "owner": "input-output-hk", + "repo": "mithril", + "rev": "2478748ea9771baed8181ef0938c78f79ed60760", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "ref": "refs/tags/2617.0", + "repo": "mithril", + "type": "github" + } + }, "nixlib": { "locked": { "lastModified": 1667696192, @@ -649,6 +706,22 @@ "type": "github" } }, + "nixpkgs": { + "locked": { + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-2305": { "locked": { "lastModified": 1705033721, @@ -729,6 +802,21 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1774748309, + "narHash": "sha256-+U7gF3qxzwD5TZuANzZPeJTZRHS29OFQgkQ2kiTJBIQ=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "333c4e0545a6da976206c74db8773a1645b5870a", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs-unstable": { "locked": { "lastModified": 1759070547, @@ -773,6 +861,7 @@ "haskellNix": "haskellNix", "incl": "incl", "iohkNix": "iohkNix", + "mithril": "mithril", "nixpkgs": [ "haskellNix", "nixpkgs-unstable" @@ -780,6 +869,27 @@ "utils": "utils" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1776654897, + "narHash": "sha256-Vqi4AiJVCcBGn/RmBtRCgyH5rCxqm/w0xV9diJWF1Ic=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "25d75be8139815a53560745fa060909777495105", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, "secp256k1": { "flake": false, "locked": { @@ -845,6 +955,27 @@ "type": "github" } }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "mithril", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775636079, + "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "utils": { "inputs": { "systems": "systems" diff --git a/flake.nix b/flake.nix index 9d2e94b87f8..a8959cce99c 100644 --- a/flake.nix +++ b/flake.nix @@ -48,13 +48,17 @@ incl.url = "github:divnix/incl"; iohkNix = { - url = "github:input-output-hk/iohk-nix"; + url = "github:input-output-hk/iohk-nix/node-11.1"; inputs.nixpkgs.follows = "nixpkgs"; }; nixpkgs.follows = "haskellNix/nixpkgs-unstable"; utils.url = "github:numtide/flake-utils"; + + # Mithril signer is required as a release artifact constitutent. + # Use explicit ref tag path to ensure we get exactly what we expect. + mithril.url = "github:input-output-hk/mithril?ref=refs/tags/2617.0"; }; outputs = { @@ -63,6 +67,7 @@ haskellNix, incl, iohkNix, + mithril, nixpkgs, self, utils, @@ -70,7 +75,7 @@ } @ input: let inherit (builtins) elem match; inherit (nixpkgs) lib; - inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optionals recursiveUpdate; + inherit (lib) collect getAttr genAttrs filterAttrs hasPrefix head isDerivation mapAttrs optionalAttrs optional optionals recursiveUpdate; inherit (utils.lib) eachSystem flattenTree; inherit (iohkNix.lib) prefixNamesWith; removeRecurse = lib.filterAttrsRecursive (n: _: n != "recurseForDerivations"); @@ -364,9 +369,11 @@ inherit pkgs; inherit (exes.cardano-node.identifier) version; platform = "linux"; - exes = collect isDerivation ( - filterAttrs (n: _: elem n releaseBins) projectExes - ); + exes = + collect isDerivation ( + filterAttrs (n: _: elem n releaseBins) projectExes + ) + ++ optional (system == "x86_64-linux") mithril.packages.${system}.mithril-signer; }; internal.roots.project = muslProject.roots; variants = mapAttrs (_: v: removeAttrs v.musl ["variants"]) ciJobsVariants; diff --git a/nix/binary-release.nix b/nix/binary-release.nix index 8a5e68bb5cc..5f4a7bbb79b 100644 --- a/nix/binary-release.nix +++ b/nix/binary-release.nix @@ -41,11 +41,6 @@ let (builtins.toJSON (env.nodeConfig // genesisAttrs)); - nodeConfigLegacy= pkgs.writeText - "config-legacy.json" - (builtins.toJSON - (env.nodeConfigLegacy // genesisAttrs)); - submitApiConfig = pkgs.writeText "submit-api-config.json" (builtins.toJSON env.submitApiConfig); @@ -68,7 +63,6 @@ let '' mkdir -p "share/${name}" jq . < "${nodeConfig}" > share/${name}/config.json - jq . < "${nodeConfigLegacy}" > share/${name}/config-legacy.json jq . < "${submitApiConfig}" > share/${name}/submit-api-config.json jq . < "${tracerConfig}" > share/${name}/tracer-config.json jq . < "${peerSnapshot}" > share/${name}/peer-snapshot.json diff --git a/nix/docker/README.md b/nix/docker/README.md index 902fe87ac80..78f0a6a7a89 100644 --- a/nix/docker/README.md +++ b/nix/docker/README.md @@ -216,13 +216,13 @@ state types as needed, without relying on host level tooling or full chain ledger replays. An example follows to convert preprod ledger state in a named docker volume -from a memory based ledger snapshot to an LMDB snapshot when node is not +from a memory based ledger snapshot to an LSM snapshot when node is not already running: ``` docker run -v preprod-data:/data --rm -it --entrypoint=bash ghcr.io/intersectmbo/cardano-node:dev -c ' mv /data/db/ledger /data/db/ledger-old \ - && mkdir -p /data/db/ledger \ - && snapshot-converter --mem-in /data/db/ledger-old/20807240 --lmdb-out /data/db/ledger/20807240 --config /opt/cardano/config/preprod/config.json + && mkdir -p /data/db/ledger /data/db/lsm \ + && snapshot-converter --input-mem /data/db/ledger-old/20807240 --output-lsm-snapshot /data/db/ledger/20807240 --output-lsm-database /data/db/lsm --config /opt/cardano/config/preprod/config.json ' ``` @@ -232,26 +232,6 @@ otherwise ledger replay from genesis will re-occur. For more info, see the [UTxO Migration Guide](https://ouroboros-consensus.cardano.intersectmbo.org/docs/references/miscellaneous/utxo-hd/migrating/). -## Legacy Tracing System -Cardano-node now defaults to using the new tracing system. The legacy tracing -system is deprecated and will be removed in a future node version. While still -available, the legacy tracing system can be used by following the example -above in "custom" mode whereby config is passed, and in this case, the config -passed is the legacy style configuration. - -Legacy default configuration files are also available within the image at paths: -`/opt/cardano/config/$NETWORK/config-legacy.json` - -An example of legacy tracing system usage is: -``` -docker run \ - -v preprod-data:/data \ - -e CARDANO_CONFIG="/opt/cardano/config/preprod/config-legacy.json" \ - -e CARDANO_TOPOLOGY="/opt/cardano/config/preprod/topology.json" \ - ghcr.io/intersectmbo/cardano-node:dev \ - run -``` - # Cardano Submit API Image Operation ## Scripts Mode diff --git a/nix/docker/default.nix b/nix/docker/default.nix index 3805b4f85ce..cde340be66e 100644 --- a/nix/docker/default.nix +++ b/nix/docker/default.nix @@ -118,7 +118,7 @@ let done # Adjust genesis file, config refs - for i in config config-legacy db-sync-config; do + for i in config db-sync-config; do if [ -f "$out/config/$ENV/$i.json" ]; then sed -i "s|\"$ENV-|\"|g" "$out/config/$ENV/$i.json" fi diff --git a/nix/haskell.nix b/nix/haskell.nix index d086e60638f..13ba8ff8e2f 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -54,7 +54,6 @@ let # These programs will be available inside the nix-shell. nativeBuildInputs = with pkgs.pkgsBuildBuild; [ alejandra - lmdb nix-prefetch-git pkg-config git @@ -216,7 +215,6 @@ let mainnetConfigFiles = [ "configuration/cardano/mainnet-config.yaml" "configuration/cardano/mainnet-config.json" - "configuration/cardano/mainnet-config-legacy.json" "configuration/cardano/mainnet-byron-genesis.json" "configuration/cardano/mainnet-shelley-genesis.json" "configuration/cardano/mainnet-alonzo-genesis.json" diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index 33860e28d54..9bcc7d5d47a 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -5,7 +5,7 @@ with lib; with builtins; let - inherit (types) attrs attrsOf bool either enum functionTo int listOf package nullOr str; + inherit (types) attrs attrsOf bool either enum functionTo int listOf package path nullOr str; cfg = config.services.cardano-node; envConfig = cfg.environments.${cfg.environment}; @@ -32,16 +32,6 @@ let peerSnapshotFile = cfg.peerSnapshotFile i; }; - oldTopology = i: { - Producers = concatMap (g: map (a: { - addr = a.address; - inherit (a) port; - valency = a.valency or 1; - }) g.accessPoints) ( - cfg.producers ++ (cfg.instanceProducers i) ++ cfg.publicProducers ++ (cfg.instancePublicProducers i) - ); - }; - assertNewTopology = i: let checkEval = tryEval ( @@ -58,7 +48,7 @@ let selectTopology = i: if cfg.topology != null then cfg.topology - else toFile "topology.json" (toJSON (if (cfg.useNewTopology != false) then assertNewTopology i else oldTopology i)); + else toFile "topology.json" (toJSON (assertNewTopology i)); topology = i: if cfg.useSystemdReload @@ -72,43 +62,27 @@ let // (mapAttrs' (era: epoch: nameValuePair "Test${era}HardForkAtEpoch" epoch ) cfg.forceHardForks) - // (optionalAttrs (cfg.useNewTopology != false) ( - { - MaxConcurrencyBulkSync = 2; - } // optionalAttrs (cfg.useNewTopology == true) { - # Starting with node 10.6.0, p2p is the only network - # operating mode and EnableP2P becomes a no-op and is not - # declared by default. - # - # Older node versions which still require an explicit - # declaration can set useNewTopology true. - EnableP2P = true; - } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { - TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; - } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { - TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; - } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { - TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; - } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { - TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; - }) - ) + // { + MaxConcurrencyBulkSync = 2; + } // optionalAttrs (cfg.targetNumberOfRootPeers != null) { + TargetNumberOfRootPeers = cfg.targetNumberOfRootPeers; + } // optionalAttrs (cfg.targetNumberOfKnownPeers != null) { + TargetNumberOfKnownPeers = cfg.targetNumberOfKnownPeers; + } // optionalAttrs (cfg.targetNumberOfEstablishedPeers != null) { + TargetNumberOfEstablishedPeers = cfg.targetNumberOfEstablishedPeers; + } // optionalAttrs (cfg.targetNumberOfActivePeers != null) { + TargetNumberOfActivePeers = cfg.targetNumberOfActivePeers; + } ) cfg.extraNodeConfig; baseInstanceConfig = i: - baseConfig - // optionalAttrs (cfg.withUtxoHdLsmt i){ - LedgerDB = { - Backend = "V2LSM"; - LSMDatabasePath = cfg.lsmDatabasePath i; - }; - } - // optionalAttrs (cfg.withUtxoHdLmdb i){ - LedgerDB = { - Backend = "V1LMDB"; - LiveTablesPath = cfg.lmdbDatabasePath i; - }; + recursiveUpdate + baseConfig (optionalAttrs (cfg.withUtxoHdLsmt i) { + LedgerDB = { + Backend = "V2LSM"; + LSMDatabasePath = cfg.lsmDatabasePath i; }; + }); in i: let instanceConfig = recursiveUpdate (baseInstanceConfig i) (cfg.extraNodeInstanceConfig i); nodeConfigFile = if (cfg.nodeConfigFile != null) then cfg.nodeConfigFile @@ -413,16 +387,6 @@ in { description = ''The node database path, for each instance.''; }; - lmdbDatabasePath = mkOption { - type = funcToOr nullOrStr; - default = null; - apply = x : if lib.isFunction x then x else if x == null then _: null else _: x; - description = '' - A node UTxO-HD on-disk LMDB path for performant disk I/O, for each instance. - This could point to a direct-access SSD, with a specifically created journal-less file system and optimized mount options. - ''; - }; - lsmDatabasePath = mkOption { type = funcToOr nullOrStr; default = null; @@ -635,31 +599,6 @@ in { ''; }; - useNewTopology = mkOption { - type = nullOr bool; - default = cfg.nodeConfig.EnableP2P or null; - description = '' - Use new, p2p and ledger peers compatible topology. - - The useNewTopology option is deprecated and will be removed in the - future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only - be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the - useNewTopology option will be removed. - - For node version < 10.6.0, useNewTopology will need to be explicitly - declared true or false to behave accordingly. If left null while - also using the auto-generated p2p topology, node will fail to start. - - For node version >= 10.6.0, useNewTopology should be left as null - until the option is removed after the Dijkstra hard fork. If - explicitly declared true, node will continue to work, but if declared - false while using the auto-generated legacy topology, node will fail to - start. - ''; - }; - useLegacyTracing = mkOption { type = bool; default = false; @@ -802,16 +741,6 @@ in { ''; }; - withUtxoHdLmdb = mkOption { - type = funcToOr bool; - default = false; - apply = x: if lib.isFunction x then x else _: x; - description = '' - On a UTxO-HD enabled node, the in-memory backend is the default. - This activates the on-disk backend (LMDB) instead. - ''; - }; - withUtxoHdLsmt = mkOption { type = funcToOr bool; default = false; @@ -877,12 +806,9 @@ in { # # Mainnet does not yet require it, but declaring it will also # facilitate testing. - if (cfg.useNewTopology != false) - then - if cfg.useSystemdReload - then "peer-snapshot-${toString i}.json" - else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)) - else null; + if cfg.useSystemdReload + then "peer-snapshot-${toString i}.json" + else toFile "peer-snapshot.json" (toJSON (envConfig.peerSnapshot)); example = i: "/etc/cardano-node/peer-snapshot-${toString i}.json"; apply = x: if lib.isFunction x then x else _: x; description = '' @@ -906,7 +832,6 @@ in { }; config = mkIf cfg.enable ( let - lmdbPaths = filter (x: x != null) (map (e: cfg.lmdbDatabasePath e) (genList trivial.id cfg.instances)); lsmPaths = filter (x: x != null) (map (e: cfg.lsmDatabasePath e) (genList trivial.id cfg.instances)); genInstanceConf = f: listToAttrs (if cfg.instances > 1 then genList (i: let n = "cardano-node-${toString i}"; in nameValuePair n (f n i)) cfg.instances @@ -926,7 +851,7 @@ in { (acc: i: recursiveUpdate acc {"cardano-node/topology-${toString i}.json".source = selectTopology i;}) {} (range 0 (cfg.instances - 1))) ) - (mkIf ((cfg.useNewTopology != false) && cfg.useSystemdReload) + (mkIf cfg.useSystemdReload (foldl' (acc: i: recursiveUpdate acc ( optionalAttrs (cfg.peerSnapshotFile i != null) { @@ -950,12 +875,12 @@ in { wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; partOf = mkIf (cfg.instances > 1) ["cardano-node.service"]; - reloadTriggers = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) [ (selectTopology i) ]; + reloadTriggers = mkIf cfg.useSystemdReload [ (selectTopology i) ]; script = mkScript cfg i; serviceConfig = { User = "cardano-node"; Group = "cardano-node"; - ExecReload = mkIf (cfg.useSystemdReload && (cfg.useNewTopology != false)) "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecReload = mkIf cfg.useSystemdReload "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "always"; RuntimeDirectory = mkIf (!cfg.systemdSocketActivation) (removePrefix cfg.runDirBase (runtimeDir i)); @@ -1033,21 +958,13 @@ in { ''; } { - assertion = !(cfg.systemdSocketActivation && (cfg.useNewTopology != false)); + assertion = !cfg.systemdSocketActivation; message = "Systemd socket activation cannot be used with p2p topology due to a systemd socket re-use issue."; } - { - assertion = (length lmdbPaths) == (length (lists.unique lmdbPaths)); - message = "When configuring multiple LMDB enabled nodes on one instance, lmdbDatabasePath must be unique."; - } { assertion = (length lsmPaths) == (length (lists.unique lsmPaths)); message = "When configuring multiple LSM enabled nodes on one instance, lsmDatabasePath must be unique."; } - { - assertion = all (i: !(cfg.withUtxoHdLmdb i && cfg.withUtxoHdLsmt i)) (genList trivial.id cfg.instances); - message = "Each instance can only declare either withUtxoHdLmdb or withUtxoHdLsmt"; - } { assertion = count (o: o != null) (with cfg; [ (tracerSocketPathAccept i) @@ -1058,13 +975,6 @@ in { message = "Only one option of services.cardano-node.tracerSocket(PathAccept|PathConnect|NetworkAccept|NetworkConnect) can be declared."; } ]; - - warnings = optional (cfg.useNewTopology != null) '' - The useNewTopology option is deprecated and will be removed in the future. As of cardano-node 10.6.0, this option should remain null. - For older node versions, a bool value can be set, but this will only be supported until the Dijkstra hard fork at which point all - cardano-node versions will be compelled to upgrade and the useNewTopology option will be removed. See the services.cardano-node.useNewTopology - option description for further details. - ''; } ]); } diff --git a/nix/nixos/cardano-submit-api-service.nix b/nix/nixos/cardano-submit-api-service.nix index fcebf68ca94..36c3bb83456 100644 --- a/nix/nixos/cardano-submit-api-service.nix +++ b/nix/nixos/cardano-submit-api-service.nix @@ -1,88 +1,138 @@ -{ config, lib, pkgs, ... }: +# This service exposes an http port, and connects to a cardano-node over a UNIX socket +{ + config, + lib, + pkgs, + ... +}: let + inherit (builtins) fromJSON readFile; + inherit (cfg.cardanoNodePackages) cardanoLib; -# notes: -# this service exposes an http port, and connects to a cardano-node over a UNIX socket -let cfg = config.services.cardano-submit-api; - inherit (cfg.cardanoNodePackages) cardanoLib; - envConfig = cfg.environment; in { options = { services.cardano-submit-api = { - enable = lib.mkEnableOption "enable the cardano-submit-api api"; - script = lib.mkOption { - internal = true; - type = lib.types.package; - }; - package = lib.mkOption { - type = lib.types.package; - default = cfg.cardanoNodePackages.cardano-submit-api; + enable = lib.mkEnableOption "Enable the cardano-submit-api api"; + + cardanoNodePackages = lib.mkOption { + type = lib.types.attrs; + default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; + defaultText = "cardano-node packages"; + description = '' + The cardano-node packages and library that should be used. + Main usage is sharing optimization to reduce eval time when services + are instantiated multiple times. + ''; }; - port = lib.mkOption { - type = lib.types.port; - default = 8090; + + config = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cardanoLib.defaultSubmitApiConfig; + description = "Tracing configuration passed to submit-api."; }; - listenAddress = lib.mkOption { - type = lib.types.str; - default = "127.0.0.1"; + + environment = lib.mkOption { + type = lib.types.nullOr lib.types.attrs; + default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + description = "Cardano environment attrset for the selected network."; }; - socketPath = lib.mkOption { - type = lib.types.nullOr lib.types.path; + + group = lib.mkOption { + type = lib.types.nullOr lib.types.str; default = null; description = '' - cardano node socket path. If set, the entrypoint - takes this value over CARDANO_NODE_SOCKET_PATH env - variable. + Optional supplementary group added to the service's dynamic user, + typically the cardano-node socket group, so cardano-submit-api can + access the node socket at CARDANO_NODE_SOCKET_PATH. ''; }; - config = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cardanoLib.defaultExplorerLogConfig; + + listenAddress = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "Host address submit-api binds to."; }; + network = lib.mkOption { type = lib.types.nullOr lib.types.str; - description = "network name"; + description = "Network name."; default = null; }; - environment = lib.mkOption { - type = lib.types.nullOr lib.types.attrs; - default = cfg.cardanoNodePackages.cardanoLib.environments.${cfg.network}; + + package = lib.mkOption { + type = lib.types.package; + default = cfg.cardanoNodePackages.cardano-submit-api; + description = "The cardano-submit-api package to run."; }; - cardanoNodePackages = lib.mkOption { - type = lib.types.attrs; - default = pkgs.cardanoNodePackages or (import ../. {}).cardanoNodePackages; - defaultText = "cardano-node packages"; + + port = lib.mkOption { + type = lib.types.port; + default = 8090; + description = "HTTP port submit-api listens on."; + }; + + script = lib.mkOption { + internal = true; + type = lib.types.package; + description = "Generated cardano-submit-api launch script (internal)."; + }; + + socketPath = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; description = '' - The cardano-node packages and library that should be used. - Main usage is sharing optimization: - reduce eval time when service is instantiated multiple times. + The cardano node socket path. If set, the entrypoint takes this value + over CARDANO_NODE_SOCKET_PATH env variable. ''; }; }; }; config = let envNodeCfg = cfg.environment.nodeConfig; - shelleyGenesisParams = __fromJSON (__readFile envNodeCfg.ShelleyGenesisFile); - envFlag = if cfg.network == "mainnet" then "--mainnet" else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; - in lib.mkIf cfg.enable { - services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' - ${if (cfg.socketPath == null) then ''if [ -z "$CARDANO_NODE_SOCKET_PATH" ] - then - echo "You must set \$CARDANO_NODE_SOCKET_PATH" - exit 1 - fi'' else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\""} - exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ - --port ${toString cfg.port} \ - --listen-address ${cfg.listenAddress} \ - --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} - ''; - systemd.services.cardano-submit-api = { - serviceConfig = { - ExecStart = config.services.cardano-submit-api.script; - DynamicUser = true; + shelleyGenesisParams = fromJSON (readFile envNodeCfg.ShelleyGenesisFile); + envFlag = + if cfg.network == "mainnet" + then "--mainnet" + else "--testnet-magic ${toString shelleyGenesisParams.networkMagic}"; + in + lib.mkIf cfg.enable { + services.cardano-submit-api.script = pkgs.writeShellScript "cardano-submit-api" '' + ${ + if (cfg.socketPath == null) + then '' if [ -z "$CARDANO_NODE_SOCKET_PATH" ] + then + echo "You must set \$CARDANO_NODE_SOCKET_PATH" + exit 1 + fi'' + else "export \"CARDANO_NODE_SOCKET_PATH=${cfg.socketPath}\"" + } + exec ${cfg.package}/bin/cardano-submit-api --socket-path "$CARDANO_NODE_SOCKET_PATH" ${envFlag} \ + --port ${toString cfg.port} \ + --listen-address ${cfg.listenAddress} \ + --config ${builtins.toFile "submit-api.json" (builtins.toJSON cfg.config)} + ''; + systemd.services.cardano-submit-api = { + serviceConfig = + { + ExecStart = config.services.cardano-submit-api.script; + DynamicUser = true; + + # The api connects to the node over a UNIX socket that only becomes + # available once the node has started; `after` orders startup but does + # not wait for the socket. Default to restarting until it is reachable + # rather than failing permanently on a fresh boot. These are mkDefault + # so a consumer can impose a bounded restart + start-limit policy that + # lets persistent failures surface as a failed unit for alerting. + Restart = lib.mkDefault "always"; + RestartSec = lib.mkDefault 1; + } + // lib.optionalAttrs (cfg.group != null) { + # A DynamicUser is not otherwise a member of the node socket group, so + # without this it cannot open CARDANO_NODE_SOCKET_PATH. + SupplementaryGroups = [cfg.group]; + }; + wantedBy = ["multi-user.target"]; + after = ["cardano-node.service"]; }; - wantedBy = [ "multi-user.target" ]; - after = [ "cardano-node.service" ]; }; - }; } diff --git a/nix/workbench/backend/nomad/cloud.sh b/nix/workbench/backend/nomad/cloud.sh index f99cadf919c..d46f619a044 100644 --- a/nix/workbench/backend/nomad/cloud.sh +++ b/nix/workbench/backend/nomad/cloud.sh @@ -701,13 +701,13 @@ allocate-run-nomadcloud() { read -p "Hit enter to continue ..." fi fi - # Clean, only producers, the "host_volumes" if being used for LMDB/LSMT. + # Clean, only producers, the "host_volumes" if being used for LSMT. # We do this for each producer instead of for all producer at once because # even if modules have the same name from a Nomad perspective, in each # client the real path is defined in Nomad's config file and may differ! # It's "slow" (fetches individual client configs), done only if necessary. if test "${node_name}" != "explorer" \ - && jqtest '.node.utxo_lmdb or .node.utxo_lsmt' "${dir}"/profile.json \ + && jqtest '.node.utxo_lsmt' "${dir}"/profile.json \ && jqtest '(.cluster.nomad.host_volumes.producer | length) > 0' "${dir}"/profile.json then # Iterate over the profile's Nomad "host_volumes" array by key/index. diff --git a/nix/workbench/service/nodes.nix b/nix/workbench/service/nodes.nix index 22e5a5f85cd..aa2442287b7 100644 --- a/nix/workbench/service/nodes.nix +++ b/nix/workbench/service/nodes.nix @@ -73,11 +73,9 @@ let topology = "topology.json"; nodeConfigFile = "config.json"; - # Allow for local clusters to have multiple LMDB directories in the same physical ssd_directory; + # Allow for local clusters to have multiple on-disk (LSM-tree) directories in the same physical ssd_directory; # non-block producers (like the explorer node) keep using the in-memory backend - withUtxoHdLmdb = profile.node.utxo_lmdb && isProducer; withUtxoHdLsmt = profile.node.utxo_lsmt && isProducer; - lmdbDatabasePath = liveTablesPath i; lsmDatabasePath = liveTablesPath i; ## Combine: @@ -130,12 +128,6 @@ let Backend = "V2LSM"; LSMDatabasePath = liveTablesPath i; }; - } // optionalAttrs (profile.node.utxo_lmdb && isProducer) - { - LedgerDB = { - Backend = "V1LMDB"; - LiveTablesPath = liveTablesPath i; - }; }) (if __hasAttr "preset" profile && profile.preset != null ## It's either an undisturbed preset, diff --git a/scripts/lite/mainnet-legacy-tracing.sh b/scripts/lite/mainnet-legacy-tracing.sh deleted file mode 100755 index 3fbe4b3d2ae..00000000000 --- a/scripts/lite/mainnet-legacy-tracing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# This script connects a node to mainnet - -ROOT="$(realpath "$(dirname "$0")/../..")" -configuration="${ROOT}/configuration/cardano" - -data_dir=mainnetsingle -mkdir -p "${data_dir}" -db_dir="${data_dir}/db/node" -mkdir -p "${db_dir}" -socket_dir="${data_dir}/socket" -mkdir -p "${socket_dir}" - -# Launch a node -cabal run exe:cardano-node -- run \ - --config "${configuration}/mainnet-config-legacy.json" \ - --topology "${configuration}/mainnet-topology.json" \ - --database-path "${db_dir}" \ - --socket-path "${socket_dir}/node-1-socket" \ - --host-addr "0.0.0.0" \ - --port "3001" - - - -function cleanup() -{ - for child in $(jobs -p); do - echo kill "$child" && kill "$child" - done -} - -trap cleanup EXIT