fix: Remove possible divergence

changes/node/changed/cnight-observation-fetch-determinism.md

@@ -0,0 +1,24 @@
+#node #runtime #security
+# Make cNIGHT observation inherent generation deterministic and unskippable
+
+The cNIGHT inherent provider used a row-count over-fetch (`tx_capacity * factor`)
+as a one-shot SQL `LIMIT`, then treated "fewer than `tx_capacity` distinct txs
+returned" as "range complete" and advanced the Cardano cursor to the tip —
+without checking whether the query was truncated by its limit. A range holding
+more matching rows than the limit across fewer txs (many UTXOs per tx) was
+silently skipped, and a node that fetched more rows derived a different inherent
+→ `check_inherent` rejection (fork/liveness) plus corrupted mint/burn accounting.
+
+Fix: one deterministic path for both the in-memory cache and the db fallback.
+Fetch the complete range (`bulk_pull` now reports whether its limit was hit) and
+truncate whole-transaction to the runtime envelope (`tx_capacity *
+UTXO_PER_TX_OVERESTIMATE`). The cursor reaches the tip only on a proven-complete
+fetch; otherwise it stops at the last fully-observed tx. With no fetch-size input
+left, every node derives identical inherents.
+
+`UTXO_PER_TX_OVERESTIMATE` moves to `midnight-primitives-cnight-observation` as
+the single source shared by runtime and node. Byte-identical to the prior 64x
+path on benign history, so finalized blocks replay unchanged.
+
+PR: <link to PR>
+Issue: <link to Github Issue, if applicable>

ledger/src/lib.rs

@@ -154,6 +154,129 @@ pub fn drop_all_default_storage() {
 	ledger_9::storage::drop_default_storage_if_exists();
 }
 
+/// Genesis bootstrap that dispatches on the ledger version embedded in the
+/// genesis state, instead of assuming the newest.
+///
+/// All ledger versions are compiled in, but a genesis blob is serialized with
+/// exactly one version's `LedgerState`, and its deserializer rejects any other
+/// version's tag. Hardcoding the latest (as each ledger bump historically did)
+/// makes the node unable to bootstrap a chain still on an older genesis — e.g.
+/// live mainnet, whose genesis is immutable. These wrappers read the blob's tag
+/// and route to the matching module. This is *not* state migration: each chain
+/// keeps running its own ledger version; we only stop forcing the newest at the
+/// genesis boundary.
+#[cfg(feature = "std")]
+pub mod genesis_version {
+	/// Ledger storage version a tagged genesis blob was produced with.
+	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+	pub enum LedgerVersion {
+		/// ledger 7
+		V7,
+		/// ledger 8
+		V8,
+		/// ledger 9
+		V9,
+	}
+
+	/// Detect a genesis blob's ledger version by matching its leading tag
+	/// against each compiled-in module's own `LedgerState` tag. Newest first.
+	/// `None` if no known version matches (corrupt or future-version blob).
+	pub fn detect(genesis_state: &[u8]) -> Option<LedgerVersion> {
+		let head = &genesis_state[..genesis_state.len().min(64)];
+		let has = |tag: &str| head.windows(tag.len()).any(|w| w == tag.as_bytes());
+		if has(&super::ledger_9::storage::genesis_ledger_state_tag()) {
+			Some(LedgerVersion::V9)
+		} else if has(&super::ledger_8::storage::genesis_ledger_state_tag()) {
+			Some(LedgerVersion::V8)
+		} else if has(&super::ledger_7::storage::genesis_ledger_state_tag()) {
+			Some(LedgerVersion::V7)
+		} else {
+			None
+		}
+	}
+
+	fn unknown_version(genesis_state: &[u8]) -> String {
+		format!(
+			"unrecognised ledger genesis version tag in {}-byte genesis blob (known: {}, {}, {})",
+			genesis_state.len(),
+			super::ledger_9::storage::genesis_ledger_state_tag(),
+			super::ledger_8::storage::genesis_ledger_state_tag(),
+			super::ledger_7::storage::genesis_ledger_state_tag(),
+		)
+	}
+
+	/// Version-dispatched [`get_root`](super::ledger_9::storage::get_root).
+	pub fn get_root(genesis_state: &[u8], network_id: Option<&str>) -> Result<Vec<u8>, String> {
+		match detect(genesis_state) {
+			Some(LedgerVersion::V9) => super::ledger_9::storage::get_root(genesis_state, network_id)
+				.map_err(|e| e.to_string()),
+			Some(LedgerVersion::V8) => super::ledger_8::storage::get_root(genesis_state, network_id)
+				.map_err(|e| e.to_string()),
+			Some(LedgerVersion::V7) => super::ledger_7::storage::get_root(genesis_state, network_id)
+				.map_err(|e| e.to_string()),
+			None => Err(unknown_version(genesis_state)),
+		}
+	}
+
+	/// Version-dispatched separate-db genesis init.
+	pub fn init_storage_paritydb_separate<P: AsRef<std::path::Path>>(
+		dir: P,
+		genesis_state: &[u8],
+		cache_size: usize,
+	) -> Vec<u8> {
+		match detect(genesis_state) {
+			Some(LedgerVersion::V9) => super::ledger_9::storage::init_storage_paritydb_separate(
+				dir,
+				genesis_state,
+				cache_size,
+			),
+			Some(LedgerVersion::V8) => super::ledger_8::storage::init_storage_paritydb_separate(
+				dir,
+				genesis_state,
+				cache_size,
+			),
+			Some(LedgerVersion::V7) => super::ledger_7::storage::init_storage_paritydb_separate(
+				dir,
+				genesis_state,
+				cache_size,
+			),
+			None => panic!("{}", unknown_version(genesis_state)),
+		}
+	}
+
+	/// Version-dispatched unified-db genesis init.
+	pub fn init_storage_paritydb_unified<
+		D: std::ops::Deref<Target = parity_db::Db> + Default + Send + Sync + 'static,
+		const COLUMN_OFFSET: u8,
+	>(
+		db_instance: D,
+		genesis_state: &[u8],
+		cache_size: usize,
+	) -> Vec<u8> {
+		match detect(genesis_state) {
+			Some(LedgerVersion::V9) =>
+				super::ledger_9::storage::init_storage_paritydb_unified::<D, COLUMN_OFFSET>(
+					db_instance,
+					genesis_state,
+					cache_size,
+				),
+			Some(LedgerVersion::V8) =>
+				super::ledger_8::storage::init_storage_paritydb_unified::<D, COLUMN_OFFSET>(
+					db_instance,
+					genesis_state,
+					cache_size,
+				),
+			Some(LedgerVersion::V7) =>
+				super::ledger_7::storage::init_storage_paritydb_unified::<D, COLUMN_OFFSET>(
+					db_instance,
+					genesis_state,
+					cache_size,
+				),
+			None => panic!("{}", unknown_version(genesis_state)),
+		}
+	}
+}
+
 mod common;
 
 pub mod types {

ledger/src/versions/common/storage.rs

@@ -74,6 +74,17 @@ impl core::fmt::Display for GetRootError {
 	}
 }
 
+/// The `LedgerState` serialization tag for this ledger version, e.g.
+/// `ledger-state[v13]` (ledger 8) or `ledger-state[v16]` (ledger 9). A tagged
+/// genesis blob carries this verbatim, so it lets the bootstrap pick the right
+/// deserializer before touching the body (see [`crate::genesis_version`]).
+#[cfg(feature = "std")]
+pub fn genesis_ledger_state_tag() -> std::borrow::Cow<'static, str> {
+	use super::ledger_storage_local::DefaultDB;
+	use super::midnight_serialize_local::Tagged;
+	<super::mn_ledger_local::structure::LedgerState<DefaultDB> as Tagged>::tag()
+}
+
 pub fn get_root(state: &[u8], network_id: Option<&str>) -> Result<Vec<u8>, GetRootError> {
 	// Get empty state key
 	use super::api::Ledger;

node/src/backend/custom_parity_db.rs

@@ -109,15 +109,15 @@ pub fn open<H: Clone + AsRef<[u8]>>(
 
 	match storage_config.separation {
 		StorageSeparation::Separate => {
-			midnight_node_ledger::ledger_9::storage::init_storage_paritydb_separate(
+			midnight_node_ledger::genesis_version::init_storage_paritydb_separate(
 				&storage_config.db_path,
 				&storage_config.genesis_state,
 				storage_config.cache_size,
 			);
 			Ok((OwnedDb(db), LedgerStorageDb::SeparateDb(storage_config.db_path.clone())))
 		},
 		StorageSeparation::Unified => {
-			midnight_node_ledger::ledger_9::storage::init_storage_paritydb_unified::<
+			midnight_node_ledger::genesis_version::init_storage_paritydb_unified::<
 				_,
 				NUM_COLUMNS_POLKADOT,
 			>(OwnedDb(db.clone()), &storage_config.genesis_state, storage_config.cache_size);

node/src/chain_spec/mod.rs

@@ -56,7 +56,7 @@ pub enum ChainSpecInitError {
 	Missing(String),
 	ParseError(String),
 	Serialization(String),
-	GenesisStateError(midnight_node_ledger::ledger_9::storage::GetRootError),
+	GenesisStateError(String),
 }
 
 impl fmt::Display for ChainSpecInitError {
@@ -268,7 +268,7 @@ fn genesis_config<T: MidnightNetwork>(genesis: T) -> Result<serde_json::Value, C
 		midnight: MidnightConfig {
 			_config: Default::default(),
 			network_id: genesis.network_id(),
-			genesis_state_key: midnight_node_ledger::ledger_9::storage::get_root(
+			genesis_state_key: midnight_node_ledger::genesis_version::get_root(
 				genesis.genesis_state(),
 				Some(&genesis.network_id()),
 			)

node/src/genesis/creation/cnight_genesis.rs

@@ -2,7 +2,7 @@ use alloc::collections::{BTreeMap, BTreeSet};
 use frame_support::inherent::ProvideInherent;
 use midnight_primitives_cnight_observation::{
 	CNightAddresses, CardanoPosition, CardanoRewardAddressBytes, DustPublicKeyBytes,
-	INHERENT_IDENTIFIER, ObservedUtxos, TimestampUnixMillis,
+	INHERENT_IDENTIFIER, ObservedUtxos, TimestampUnixMillis, UTXO_PER_TX_OVERESTIMATE,
 };
 use midnight_primitives_mainchain_follower::{
 	MidnightCNightObservationDataSource, MidnightObservationTokenMovement, ObservedUtxo,
@@ -22,9 +22,8 @@ use serde_json;
 use tokio::{fs::File, io::AsyncWriteExt};
 
 const TX_CAPACITY: usize = 1000;
-// Genesis is one-shot and not consensus-validated, so we use a generous over-fetch
-// factor to ensure we never split a transaction across a paged query.
-const UTXO_OVERESTIMATE: usize = TX_CAPACITY * 64;
+// Genesis is one-shot, but the data source applies the same envelope as consensus.
+const MAX_UTXOS: usize = TX_CAPACITY * UTXO_PER_TX_OVERESTIMATE as usize;
 
 #[derive(Debug, thiserror::Error)]
 pub enum CNightGenesisError {
@@ -111,7 +110,7 @@ pub async fn generate_cnight_genesis(
 				&current_position,
 				cardano_tip.clone(),
 				TX_CAPACITY,
-				UTXO_OVERESTIMATE,
+				MAX_UTXOS,
 			)
 			.await
 			.map_err(CNightGenesisError::UtxoQueryError)?;

pallets/cnight-observation/src/lib.rs

@@ -68,15 +68,9 @@ pub enum UtxoActionType {
 pub const INITIAL_CARDANO_BLOCK_WINDOW_SIZE: u32 = 1000;
 pub const DEFAULT_CARDANO_TX_CAPACITY_PER_BLOCK: u32 = 200;
 
-/// Runtime acceptance envelope: upper bound on the UTXO-to-TX ratio that
-/// `process_tokens` and the worst-case weight will accept per inherent.
-///
-/// This is intentionally *wider* than the IDP's actual fetch factor (which the
-/// node binary picks per `CNightObservationApi` version — 4x at v2+, 64x at v1).
-/// The runtime must keep accepting the legacy 64x envelope so that v1 binaries
-/// pairing with a v2 runtime during the upgrade window can still have their
-/// inherents verified. Do not lower this to match the IDP fetch factor.
-pub const UTXO_PER_TX_OVERESTIMATE: u32 = 64;
+/// Acceptance envelope for `process_tokens` and worst-case weight. Defined in
+/// the primitives crate so the IDP and runtime share one value (see there).
+pub use midnight_primitives_cnight_observation::UTXO_PER_TX_OVERESTIMATE;
 
 /// Upper bound on UTXO count per block, used for worst-case weight declaration.
 pub const MAX_UTXO_COUNT: u32 = DEFAULT_CARDANO_TX_CAPACITY_PER_BLOCK * UTXO_PER_TX_OVERESTIMATE;

primitives/cnight-observation/src/lib.rs

@@ -40,6 +40,12 @@ pub const CNIGHT_POLICY_ID_LENGTH: u32 = 28;
 /// Cardano native-asset name maximum length in bytes.
 pub const CARDANO_ASSET_NAME_MAX_LENGTH: u32 = 32;
 
+/// UTXO acceptance envelope per block, as a multiple of tx capacity. Shared by
+/// the runtime (`process_tokens` bound) and the node IDP (inherent truncation
+/// cap); they must use one value or nodes disagree, hence this lives here rather
+/// than in the pallet.
+pub const UTXO_PER_TX_OVERESTIMATE: u32 = 64;
+
 #[derive(
 	Encode,
 	Decode,
@@ -421,10 +427,9 @@ impl PartialOrd for ObservedUtxoHeader {
 }
 
 decl_runtime_apis! {
-	// v2 marks the consensus-affecting reduction of the cNight db-sync over-fetch
-	// factor from 64x to 4x. Node binaries gate the multiplier on this version so
-	// the change only takes effect at the runtime upgrade boundary; mixing old and
-	// new binaries against the same runtime version stays consensus-equivalent.
+	// v2 once gated a node-side db-sync over-fetch multiplier; the node now
+	// fetches the whole range instead, so nothing is gated on it. Retained for
+	// compatibility.
 	#[api_version(2)]
 	pub trait CNightObservationApi {
 		/// Get the contract address on Cardano which emits registration mappings in utxo datums