diff --git a/CHANGELOG.md b/CHANGELOG.md index 85fa15d6637495..d774c279636661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Release channels have their own copy of this changelog: ## [2.2.0] - Unreleased * Breaking: + * Blockstore Index column format change + * The Blockstore Index column format has been updated. The column format written in v2.2 is compatible with v2.1, but incompatible with v2.0 and older. * Snapshot format change * The snapshot format has been modified to implement SIMD-215. Since only adjacent versions are guaranteed to maintain snapshot compatibility, this means snapshots created with v2.2 are compatible with v2.1 and incompatible with v2.0 and older. * Changes diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 625368b94c7cff..8371e18112c4da 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -5372,7 +5372,7 @@ pub mod tests { shred::{max_ticks_per_n_shreds, ShredFlags, LEGACY_SHRED_DATA_CAPACITY}, }, assert_matches::assert_matches, - bincode::serialize, + bincode::{serialize, Options}, crossbeam_channel::unbounded, rand::{seq::SliceRandom, thread_rng}, solana_account_decoder::parse_token::UiTokenAmount, @@ -5840,6 +5840,36 @@ pub mod tests { test_insert_data_shreds_slots(true); } + #[test] + fn test_index_fallback_deserialize() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let mut rng = rand::thread_rng(); + let slot = rng.gen_range(0..100); + let bincode = bincode::DefaultOptions::new() + .reject_trailing_bytes() + .with_fixint_encoding(); + + let data = 0..rng.gen_range(100..MAX_DATA_SHREDS_PER_SLOT as u64); + let coding = 0..rng.gen_range(100..MAX_DATA_SHREDS_PER_SLOT as u64); + let mut fallback = IndexFallback::new(slot); + for (d, c) in data.clone().zip(coding.clone()) { + fallback.data_mut().insert(d); + fallback.coding_mut().insert(c); + } + + blockstore + .index_cf + .put_bytes(slot, &bincode.serialize(&fallback).unwrap()) + .unwrap(); + + let current = blockstore.index_cf.get(slot).unwrap().unwrap(); + for (d, c) in data.zip(coding) { + assert!(current.data().contains(d)); + assert!(current.coding().contains(c)); + } + } + /* #[test] pub fn test_iteration_order() { diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index cab7656448f2a1..5effc529470ccd 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1267,7 +1267,7 @@ impl TypedColumn for columns::Index { match index { Ok(index) => Ok(index), Err(_) => { - let index: blockstore_meta::IndexV2 = config.deserialize(data)?; + let index: blockstore_meta::IndexFallback = config.deserialize(data)?; Ok(index.into()) } } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 60d156b1642c00..b895a7bf6c3037 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -107,12 +107,20 @@ mod serde_compat { } } +pub type Index = IndexV2; +pub type ShredIndex = ShredIndexV2; +/// We currently support falling back to the previous format for migration purposes. +/// +/// See https://github.com/anza-xyz/agave/issues/3570. +pub type IndexFallback = IndexV1; +pub type ShredIndexFallback = ShredIndexV1; + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] /// Index recording presence/absence of shreds -pub struct Index { +pub struct IndexV1 { pub slot: Slot, - data: ShredIndex, - coding: ShredIndex, + data: ShredIndexV1, + coding: ShredIndexV1, } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] @@ -122,9 +130,9 @@ pub struct IndexV2 { coding: ShredIndexV2, } -impl From for Index { +impl From for IndexV1 { fn from(index: IndexV2) -> Self { - Index { + IndexV1 { slot: index.slot, data: index.data.into(), coding: index.coding.into(), @@ -132,8 +140,8 @@ impl From for Index { } } -impl From for IndexV2 { - fn from(index: Index) -> Self { +impl From for IndexV2 { + fn from(index: IndexV1) -> Self { IndexV2 { slot: index.slot, data: index.data.into(), @@ -143,7 +151,7 @@ impl From for IndexV2 { } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] -pub struct ShredIndex { +pub struct ShredIndexV1 { /// Map representing presence/absence of shreds index: BTreeSet, } @@ -251,7 +259,7 @@ pub struct FrozenHashStatus { impl Index { pub(crate) fn new(slot: Slot) -> Self { - Index { + Self { slot, data: ShredIndex::default(), coding: ShredIndex::default(), @@ -273,11 +281,39 @@ impl Index { } } +#[cfg(test)] +#[allow(unused)] +impl IndexFallback { + pub(crate) fn new(slot: Slot) -> Self { + Self { + slot, + data: ShredIndexFallback::default(), + coding: ShredIndexFallback::default(), + } + } + + pub fn data(&self) -> &ShredIndexFallback { + &self.data + } + pub fn coding(&self) -> &ShredIndexFallback { + &self.coding + } + + pub(crate) fn data_mut(&mut self) -> &mut ShredIndexFallback { + &mut self.data + } + pub(crate) fn coding_mut(&mut self) -> &mut ShredIndexFallback { + &mut self.coding + } +} + /// Superseded by [`ShredIndexV2`]. /// /// TODO: Remove this once new [`ShredIndexV2`] is fully rolled out /// and no longer relies on it for fallback. -impl ShredIndex { +#[cfg(test)] +#[allow(unused)] +impl ShredIndexV1 { pub fn num_shreds(&self) -> usize { self.index.len() } @@ -297,7 +333,6 @@ impl ShredIndex { self.index.insert(index); } - #[cfg(test)] fn remove(&mut self, index: u64) { self.index.remove(&index); } @@ -498,23 +533,23 @@ impl FromIterator for ShredIndexV2 { } } -impl FromIterator for ShredIndex { +impl FromIterator for ShredIndexV1 { fn from_iter>(iter: T) -> Self { - ShredIndex { + ShredIndexV1 { index: iter.into_iter().collect(), } } } -impl From for ShredIndexV2 { - fn from(value: ShredIndex) -> Self { +impl From for ShredIndexV2 { + fn from(value: ShredIndexV1) -> Self { value.index.into_iter().collect() } } -impl From for ShredIndex { +impl From for ShredIndexV1 { fn from(value: ShredIndexV2) -> Self { - ShredIndex { + ShredIndexV1 { index: value.iter().collect(), } } @@ -922,7 +957,7 @@ mod test { shreds in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), range in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64) ) { - let mut legacy = ShredIndex::default(); + let mut legacy = ShredIndexV1::default(); let mut v2 = ShredIndexV2::default(); for i in shreds { @@ -942,7 +977,7 @@ mod test { ); assert_eq!(ShredIndexV2::from(legacy.clone()), v2.clone()); - assert_eq!(ShredIndex::from(v2), legacy); + assert_eq!(ShredIndexV1::from(v2), legacy); } /// Property: [`Index`] cannot be deserialized from [`IndexV2`]. @@ -965,7 +1000,7 @@ mod test { slot, }; let config = bincode::DefaultOptions::new().with_fixint_encoding().reject_trailing_bytes(); - let legacy = config.deserialize::(&config.serialize(&index).unwrap()); + let legacy = config.deserialize::(&config.serialize(&index).unwrap()); prop_assert!(legacy.is_err()); } @@ -983,7 +1018,7 @@ mod test { data_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), slot in 0..u64::MAX ) { - let index = Index { + let index = IndexV1 { coding: coding_indices.into_iter().collect(), data: data_indices.into_iter().collect(), slot,