From c7d0aea5f433b3871f9549cf2d466511def57eec Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 2 Jul 2021 04:44:38 +0000 Subject: [PATCH] Make set roots an iterator (#18357) (#18377) (cherry picked from commit 0eca92de18a7b3e6c5aece76c6e6f2c9bd89dc6f) Co-authored-by: carllin --- core/src/banking_stage.rs | 2 +- core/src/consensus.rs | 4 +-- core/src/optimistic_confirmation_verifier.rs | 2 +- core/src/replay_stage.rs | 2 +- ledger-tool/src/main.rs | 10 +++--- ledger/src/ancestor_iterator.rs | 2 +- ledger/src/blockstore.rs | 38 +++++++++++--------- ledger/src/blockstore_processor.rs | 26 ++++++++------ ledger/src/next_slots_iterator.rs | 2 +- ledger/src/rooted_slot_iterator.rs | 10 +++--- rpc/src/rpc.rs | 6 ++-- 11 files changed, 58 insertions(+), 46 deletions(-) diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index f71770396a..43416184e6 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -2380,7 +2380,7 @@ mod tests { let shreds = entries_to_test_shreds(entries, bank.slot(), 0, true, 0); blockstore.insert_shreds(shreds, None, false).unwrap(); - blockstore.set_roots(&[bank.slot()]).unwrap(); + blockstore.set_roots(std::iter::once(&bank.slot())).unwrap(); let (transaction_status_sender, transaction_status_receiver) = unbounded(); let transaction_status_service = TransactionStatusService::new( diff --git a/core/src/consensus.rs b/core/src/consensus.rs index e482e0232d..8df061e1e7 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -1337,7 +1337,7 @@ pub fn reconcile_blockstore_roots_with_tower( "Reconciling slots as root based on tower root: {:?} ({}..{}) ", new_roots, tower_root, last_blockstore_root ); - blockstore.set_roots(&new_roots)?; + blockstore.set_roots(new_roots.iter())?; } else { // This indicates we're in bad state; but still don't panic here. // That's because we might have a chance of recovering properly with @@ -3177,7 +3177,7 @@ pub mod test { blockstore.insert_shreds(shreds, None, false).unwrap(); let (shreds, _) = make_slot_entries(4, 1, 42); blockstore.insert_shreds(shreds, None, false).unwrap(); - blockstore.set_roots(&[3]).unwrap(); + blockstore.set_roots(std::iter::once(&3)).unwrap(); assert!(!blockstore.is_root(0)); assert!(!blockstore.is_root(1)); assert!(blockstore.is_root(3)); diff --git a/core/src/optimistic_confirmation_verifier.rs b/core/src/optimistic_confirmation_verifier.rs index c5445df46d..5e53abb342 100644 --- a/core/src/optimistic_confirmation_verifier.rs +++ b/core/src/optimistic_confirmation_verifier.rs @@ -316,7 +316,7 @@ mod test { assert!(optimistic_confirmation_verifier.unchecked_slots.is_empty()); // If we know set the root in blockstore, should return nothing - blockstore.set_roots(&[1, 3]).unwrap(); + blockstore.set_roots(vec![1, 3].iter()).unwrap(); optimistic_confirmation_verifier.add_new_optimistic_confirmed_slots(optimistic_slots); assert!(optimistic_confirmation_verifier .verify_for_unrooted_optimistic_slots(&bank7, &blockstore) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 91a6dbf6ef..3af2c59b26 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1337,7 +1337,7 @@ impl ReplayStage { // get dropped. leader_schedule_cache.set_root(rooted_banks.last().unwrap()); blockstore - .set_roots(&rooted_slots) + .set_roots(rooted_slots.iter()) .expect("Ledger set roots failed"); let highest_confirmed_root = Some( block_commitment_cache diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index d764e78157..9af42305ea 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -2917,10 +2917,12 @@ fn main() { eprintln!("{} slots to be rooted", roots_to_fix.len()); for chunk in roots_to_fix.chunks(100) { eprintln!("{:?}", chunk); - blockstore.set_roots(&roots_to_fix).unwrap_or_else(|err| { - eprintln!("Unable to set roots {:?}: {}", roots_to_fix, err); - exit(1); - }); + blockstore + .set_roots(roots_to_fix.iter()) + .unwrap_or_else(|err| { + eprintln!("Unable to set roots {:?}: {}", roots_to_fix, err); + exit(1); + }); } } else { println!( diff --git a/ledger/src/ancestor_iterator.rs b/ledger/src/ancestor_iterator.rs index 6c8099ce98..333974cc05 100644 --- a/ledger/src/ancestor_iterator.rs +++ b/ledger/src/ancestor_iterator.rs @@ -60,7 +60,7 @@ mod tests { fn test_ancestor_iterator() { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(&[0]).unwrap(); + blockstore.set_roots(std::iter::once(&0)).unwrap(); let ticks_per_slot = 5; /* diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 81892338d0..57599bcb6f 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2941,9 +2941,11 @@ impl Blockstore { } } - pub fn set_roots(&self, rooted_slots: &[u64]) -> Result<()> { + pub fn set_roots<'a>(&self, rooted_slots: impl Iterator) -> Result<()> { let mut write_batch = self.db.batch()?; + let mut max_new_rooted_slot = 0; for slot in rooted_slots { + max_new_rooted_slot = std::cmp::max(max_new_rooted_slot, *slot); write_batch.put::(*slot, &true)?; } @@ -2953,7 +2955,7 @@ impl Blockstore { if *last_root == std::u64::MAX { *last_root = 0; } - *last_root = cmp::max(*rooted_slots.iter().max().unwrap(), *last_root); + *last_root = cmp::max(max_new_rooted_slot, *last_root); Ok(()) } @@ -3108,7 +3110,7 @@ impl Blockstore { return Ok(()); } trace!("{:?}", chunk); - self.set_roots(chunk)?; + self.set_roots(chunk.iter())?; } } else { debug!( @@ -3630,7 +3632,7 @@ pub fn create_new_ledger( assert!(shreds.last().unwrap().last_in_slot()); blockstore.insert_shreds(shreds, None, false)?; - blockstore.set_roots(&[0])?; + blockstore.set_roots(std::iter::once(&0))?; // Explicitly close the blockstore before we create the archived genesis file drop(blockstore); @@ -5881,7 +5883,7 @@ pub mod tests { let chained_slots = vec![0, 2, 4, 7, 12, 15]; assert_eq!(blockstore.last_root(), 0); - blockstore.set_roots(&chained_slots).unwrap(); + blockstore.set_roots(chained_slots.iter()).unwrap(); assert_eq!(blockstore.last_root(), 15); @@ -5898,7 +5900,7 @@ pub mod tests { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); let roots = vec![2, 4, 7, 12, 15]; - blockstore.set_roots(&roots).unwrap(); + blockstore.set_roots(roots.iter()).unwrap(); for i in 0..20 { if i < 2 || roots.contains(&i) || i > 15 { @@ -6067,7 +6069,7 @@ pub mod tests { let last_root = 100; { let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(&[last_root]).unwrap(); + blockstore.set_roots(std::iter::once(&last_root)).unwrap(); // Insert will fail, slot < root blockstore @@ -6096,7 +6098,9 @@ pub mod tests { ledger.insert_shreds(shreds, None, false).unwrap(); ledger.insert_shreds(more_shreds, None, false).unwrap(); ledger.insert_shreds(unrooted_shreds, None, false).unwrap(); - ledger.set_roots(&[slot - 1, slot, slot + 1]).unwrap(); + ledger + .set_roots(vec![slot - 1, slot, slot + 1].iter()) + .unwrap(); let parent_meta = SlotMeta { parent_slot: std::u64::MAX, @@ -6651,7 +6655,7 @@ pub mod tests { let meta3 = SlotMeta::new(3, 2); blockstore.meta_cf.put(3, &meta3).unwrap(); - blockstore.set_roots(&[0, 2]).unwrap(); + blockstore.set_roots(vec![0, 2].iter()).unwrap(); // Initialize index 0, including: // signature2 in non-root and root, @@ -6836,7 +6840,7 @@ pub mod tests { let meta3 = SlotMeta::new(3, 2); blockstore.meta_cf.put(3, &meta3).unwrap(); - blockstore.set_roots(&[0, 1, 2, 3]).unwrap(); + blockstore.set_roots(vec![0, 1, 2, 3].iter()).unwrap(); let lowest_cleanup_slot = 1; let lowest_available_slot = lowest_cleanup_slot + 1; @@ -6974,7 +6978,7 @@ pub mod tests { let ledger_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&ledger_path).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap(); - blockstore.set_roots(&[slot - 1, slot]).unwrap(); + blockstore.set_roots(vec![slot - 1, slot].iter()).unwrap(); let expected_transactions: Vec = entries .iter() @@ -7162,7 +7166,7 @@ pub mod tests { fn test_empty_transaction_status() { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(&[0]).unwrap(); + blockstore.set_roots(std::iter::once(&0)).unwrap(); assert_eq!( blockstore .get_rooted_transaction(Signature::default()) @@ -7208,7 +7212,7 @@ pub mod tests { ) .unwrap(); } - blockstore.set_roots(&[slot0, slot1]).unwrap(); + blockstore.set_roots(vec![slot0, slot1].iter()).unwrap(); let all0 = blockstore .get_confirmed_signatures_for_address(address0, 0, 50) @@ -7301,7 +7305,7 @@ pub mod tests { ) .unwrap(); } - blockstore.set_roots(&[21, 22, 23, 24]).unwrap(); + blockstore.set_roots(vec![21, 22, 23, 24].iter()).unwrap(); let mut past_slot = 0; for (slot, _) in blockstore.find_address_signatures(address0, 1, 25).unwrap() { assert!(slot >= past_slot); @@ -7373,7 +7377,7 @@ pub mod tests { ) .unwrap(); } - blockstore.set_roots(&[slot1]).unwrap(); + blockstore.set_roots(std::iter::once(&slot1)).unwrap(); let slot1_signatures = blockstore .find_address_signatures_for_slot(address0, 1) @@ -7479,7 +7483,9 @@ pub mod tests { } // Leave one slot unrooted to test only returns confirmed signatures - blockstore.set_roots(&[1, 2, 4, 5, 6, 7, 8]).unwrap(); + blockstore + .set_roots(vec![1, 2, 4, 5, 6, 7, 8].iter()) + .unwrap(); let highest_confirmed_root = 8; // Fetch all rooted signatures for address 0 at once... diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index ce77215bd8..8a6a2cdbd8 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -485,7 +485,7 @@ fn do_process_blockstore_from_root( // ensure start_slot is rooted for correct replay if blockstore.is_primary_access() { blockstore - .set_roots(&[start_slot]) + .set_roots(std::iter::once(&start_slot)) .expect("Couldn't set root slot on startup"); } else if !blockstore.is_root(start_slot) { panic!("starting slot isn't root and can't update due to being secondary blockstore access: {}", start_slot); @@ -1030,13 +1030,13 @@ fn load_frozen_forks( if new_root_bank.slot() == *root { break; } // Found the last root in the chain, yay! assert!(new_root_bank.slot() > *root); - rooted_slots.push(new_root_bank.slot()); + rooted_slots.push((new_root_bank.slot(), new_root_bank.hash())); // As noted, the cluster confirmed root should be descended from // our last root; therefore parent should be set new_root_bank = new_root_bank.parent().unwrap(); } inc_new_counter_info!("load_frozen_forks-cluster-confirmed-root", rooted_slots.len()); - blockstore.set_roots(&rooted_slots).expect("Blockstore::set_roots should succeed"); + blockstore.set_roots(rooted_slots.iter().map(|(slot, _hash)| slot)).expect("Blockstore::set_roots should succeed"); Some(cluster_root_bank) } else { None @@ -1640,7 +1640,7 @@ pub mod tests { info!("last_fork1_entry.hash: {:?}", last_fork1_entry_hash); info!("last_fork2_entry.hash: {:?}", last_fork2_entry_hash); - blockstore.set_roots(&[0, 1, 4]).unwrap(); + blockstore.set_roots(vec![0, 1, 4].iter()).unwrap(); let opts = ProcessOptions { poh_verify: true, @@ -1720,7 +1720,7 @@ pub mod tests { info!("last_fork1_entry.hash: {:?}", last_fork1_entry_hash); info!("last_fork2_entry.hash: {:?}", last_fork2_entry_hash); - blockstore.set_roots(&[0, 1]).unwrap(); + blockstore.set_roots(vec![0, 1].iter()).unwrap(); let opts = ProcessOptions { poh_verify: true, @@ -1930,11 +1930,13 @@ pub mod tests { } // Set a root on the last slot of the last confirmed epoch - let rooted_slots: Vec<_> = (0..=last_slot).collect(); - blockstore.set_roots(&rooted_slots).unwrap(); + let rooted_slots: Vec = (0..=last_slot).collect(); + blockstore.set_roots(rooted_slots.iter()).unwrap(); // Set a root on the next slot of the confirmed epoch - blockstore.set_roots(&[last_slot + 1]).unwrap(); + blockstore + .set_roots(std::iter::once(&(last_slot + 1))) + .unwrap(); // Check that we can properly restart the ledger / leader scheduler doesn't fail let opts = ProcessOptions { @@ -2860,7 +2862,7 @@ pub mod tests { genesis_config.ticks_per_slot, genesis_config.hash(), ); - blockstore.set_roots(&[0, 1]).unwrap(); + blockstore.set_roots(vec![0, 1].iter()).unwrap(); // Specify halting at slot 0 let opts = ProcessOptions { @@ -2911,7 +2913,7 @@ pub mod tests { last_hash = fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, i + 1, i, last_hash); } - blockstore.set_roots(&[3, 5]).unwrap(); + blockstore.set_roots(vec![3, 5].iter()).unwrap(); // Set up bank1 let bank0 = Arc::new(Bank::new(&genesis_config)); @@ -3367,7 +3369,9 @@ pub mod tests { blockstore.add_tree(forks, false, true, ticks_per_slot, genesis_config.hash()); if let Some(blockstore_root) = blockstore_root { - blockstore.set_roots(&[blockstore_root]).unwrap(); + blockstore + .set_roots(std::iter::once(&blockstore_root)) + .unwrap(); } let opts = ProcessOptions { diff --git a/ledger/src/next_slots_iterator.rs b/ledger/src/next_slots_iterator.rs index 945cdfab33..f3c67a6cf0 100644 --- a/ledger/src/next_slots_iterator.rs +++ b/ledger/src/next_slots_iterator.rs @@ -43,7 +43,7 @@ mod tests { fn test_next_slots_iterator() { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(&[0]).unwrap(); + blockstore.set_roots(std::iter::once(&0)).unwrap(); let ticks_per_slot = 5; /* Build a blockstore in the ledger with the following fork structure: diff --git a/ledger/src/rooted_slot_iterator.rs b/ledger/src/rooted_slot_iterator.rs index 9bd3b77b73..cdc6f1c46e 100644 --- a/ledger/src/rooted_slot_iterator.rs +++ b/ledger/src/rooted_slot_iterator.rs @@ -84,7 +84,7 @@ mod tests { fn test_rooted_slot_iterator() { let blockstore_path = get_tmp_ledger_path!(); let blockstore = Blockstore::open(&blockstore_path).unwrap(); - blockstore.set_roots(&[0]).unwrap(); + blockstore.set_roots(std::iter::once(&0)).unwrap(); let ticks_per_slot = 5; /* Build a blockstore in the ledger with the following fork structure: @@ -131,7 +131,7 @@ mod tests { fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 4, fork_point, fork_hash); // Set a root - blockstore.set_roots(&[1, 2, 3]).unwrap(); + blockstore.set_roots(vec![1, 2, 3].iter()).unwrap(); // Trying to get an iterator on a different fork will error assert!(RootedSlotIterator::new(4, &blockstore).is_err()); @@ -196,11 +196,11 @@ mod tests { } // Set roots - blockstore.set_roots(&[0, 1, 2, 3]).unwrap(); + blockstore.set_roots(vec![0, 1, 2, 3].iter()).unwrap(); // Create one post-skip slot at 10, simulating starting from a snapshot // at 10 - blockstore.set_roots(&[10]).unwrap(); + blockstore.set_roots(std::iter::once(&10)).unwrap(); // Try to get an iterator from before the skip. The post-skip slot // should not return a SlotMeta let result: Vec<_> = RootedSlotIterator::new(3, &blockstore) @@ -214,7 +214,7 @@ mod tests { fill_blockstore_slot_with_ticks(&blockstore, ticks_per_slot, 11, 10, Hash::default()); // Set roots - blockstore.set_roots(&[11]).unwrap(); + blockstore.set_roots(std::iter::once(&11)).unwrap(); let result: Vec<_> = RootedSlotIterator::new(0, &blockstore) .unwrap() diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index f476b14fa5..a61051a211 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -3860,7 +3860,7 @@ pub fn create_test_transactions_and_populate_blockstore( 0, ); blockstore.insert_shreds(shreds, None, false).unwrap(); - blockstore.set_roots(&[slot]).unwrap(); + blockstore.set_roots(std::iter::once(&slot)).unwrap(); let (transaction_status_sender, transaction_status_receiver) = crossbeam_channel::unbounded(); let (replay_vote_sender, _replay_vote_receiver) = crossbeam_channel::unbounded(); @@ -4024,7 +4024,7 @@ pub mod tests { let parent = if i > 0 { roots[i - 1] } else { 0 }; fill_blockstore_slot_with_ticks(&blockstore, 5, *root, parent, Hash::default()); } - blockstore.set_roots(&roots).unwrap(); + blockstore.set_roots(roots.iter()).unwrap(); let new_bank = Bank::new_from_parent( &parent_bank, parent_bank.collector_id(), @@ -6694,7 +6694,7 @@ pub mod tests { let bank = Arc::new(Bank::default()); let ledger_path = get_tmp_ledger_path!(); let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap()); - blockstore.set_roots(&[0, 1]).unwrap(); + blockstore.set_roots(vec![0, 1].iter()).unwrap(); // Build BlockCommitmentCache with rooted slots let mut cache0 = BlockCommitment::default(); cache0.increase_rooted_stake(50);