diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index d6dd61d62d..161c4a4726 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -10,6 +10,7 @@ use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES}; use solana_runtime::{ accounts_db, snapshot_package::{AccountsPackage, AccountsPackagePre, AccountsPackageReceiver}, + snapshot_utils::SnapshotArchiveInfoGetter, }; use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey}; use std::collections::{HashMap, HashSet}; @@ -125,19 +126,19 @@ impl AccountsHashVerifier { fault_injection_rate_slots: u64, snapshot_interval_slots: u64, ) { - let hash = accounts_package.hash; + let hash = *accounts_package.hash(); if fault_injection_rate_slots != 0 - && accounts_package.slot % fault_injection_rate_slots == 0 + && accounts_package.slot() % fault_injection_rate_slots == 0 { // For testing, publish an invalid hash to gossip. use rand::{thread_rng, Rng}; use solana_sdk::hash::extend_and_hash; - warn!("inserting fault at slot: {}", accounts_package.slot); + warn!("inserting fault at slot: {}", accounts_package.slot()); let rand = thread_rng().gen_range(0, 10); let hash = extend_and_hash(&hash, &[rand]); - hashes.push((accounts_package.slot, hash)); + hashes.push((accounts_package.slot(), hash)); } else { - hashes.push((accounts_package.slot, hash)); + hashes.push((accounts_package.slot(), hash)); } while hashes.len() > MAX_SNAPSHOT_HASHES { @@ -281,18 +282,26 @@ mod tests { let exit = Arc::new(AtomicBool::new(false)); let mut hashes = vec![]; for i in 0..MAX_SNAPSHOT_HASHES + 1 { + let slot = 100 + i as u64; + let block_height = 100 + i as u64; + let slot_deltas = vec![]; let snapshot_links = TempDir::new().unwrap(); - let accounts_package = AccountsPackage { - hash: hash(&[i as u8]), - block_height: 100 + i as u64, - slot: 100 + i as u64, - slot_deltas: vec![], + let storages = vec![]; + let snapshot_archive_path = PathBuf::from("."); + let hash = hash(&[i as u8]); + let archive_format = ArchiveFormat::TarBzip2; + let snapshot_version = SnapshotVersion::default(); + let accounts_package = AccountsPackage::new( + slot, + block_height, + slot_deltas, snapshot_links, - tar_output_file: PathBuf::from("."), - storages: vec![], - archive_format: ArchiveFormat::TarBzip2, - snapshot_version: SnapshotVersion::default(), - }; + storages, + snapshot_archive_path, + hash, + archive_format, + snapshot_version, + ); AccountsHashVerifier::process_accounts_package( accounts_package, diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index d85fd44c9e..a1e4ff5df5 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -1,5 +1,8 @@ use solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES}; -use solana_runtime::{snapshot_package::AccountsPackage, snapshot_utils}; +use solana_runtime::{ + snapshot_package::AccountsPackage, + snapshot_utils::{self, SnapshotArchiveInfoGetter}, +}; use solana_sdk::{clock::Slot, hash::Hash}; use std::{ sync::{ @@ -48,7 +51,7 @@ impl SnapshotPackagerService { ) { warn!("Failed to create snapshot archive: {}", err); } else { - hashes.push((snapshot_package.slot, snapshot_package.hash)); + hashes.push((snapshot_package.slot(), *snapshot_package.hash())); while hashes.len() > MAX_SNAPSHOT_HASHES { hashes.remove(0); } diff --git a/core/src/validator.rs b/core/src/validator.rs index 30d70a1ad4..37ff7ac7ae 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -61,6 +61,7 @@ use solana_runtime::{ commitment::BlockCommitmentCache, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_config::SnapshotConfig, + snapshot_utils::{self, SnapshotArchiveInfoGetter}, }; use solana_sdk::{ clock::Slot, @@ -1194,7 +1195,7 @@ fn new_banks_from_ledger( ); leader_schedule_cache.set_root(&bank_forks.root_bank()); - let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive( + let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive( ledger_path, &bank_forks.root_bank(), None, @@ -1207,7 +1208,10 @@ fn new_banks_from_ledger( error!("Unable to create snapshot: {}", err); abort(); }); - info!("created snapshot: {}", archive_file.display()); + info!( + "created snapshot: {}", + full_snapshot_archive_info.path().display() + ); } let tower = post_process_restored_tower( diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 704144fae7..7be3eedf9f 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -161,6 +161,16 @@ mod tests { let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); let check_hash_calculation = false; + let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path( + snapshot_package_output_path.to_path_buf(), + old_last_bank.slot(), + &old_last_bank.get_accounts_hash(), + ArchiveFormat::TarBzip2, + ); + let full_snapshot_archive_info = + snapshot_utils::FullSnapshotArchiveInfo::new_from_path(full_snapshot_archive_path) + .unwrap(); + let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives( account_paths, &[], @@ -169,14 +179,8 @@ mod tests { .as_ref() .unwrap() .snapshot_path, - snapshot_utils::build_full_snapshot_archive_path( - snapshot_package_output_path.to_path_buf(), - old_last_bank.slot(), - &old_last_bank.get_accounts_hash(), - ArchiveFormat::TarBzip2, - ), + &full_snapshot_archive_info, None, - ArchiveFormat::TarBzip2, old_genesis_config, None, None, @@ -716,9 +720,8 @@ mod tests { ) .unwrap(); - restore_from_incremental_snapshot_and_check_banks_are_equal( + restore_from_snapshots_and_check_banks_are_equal( &bank, - last_full_snapshot_slot.unwrap(), &snapshot_test_config.snapshot_config, snapshot_test_config.accounts_dir.path().to_path_buf(), &snapshot_test_config.genesis_config_info.genesis_config, @@ -783,57 +786,17 @@ mod tests { Ok(()) } - fn restore_from_incremental_snapshot_and_check_banks_are_equal( + fn restore_from_snapshots_and_check_banks_are_equal( bank: &Bank, - last_full_snapshot_slot: Slot, snapshot_config: &SnapshotConfig, accounts_dir: PathBuf, genesis_config: &GenesisConfig, ) -> snapshot_utils::Result<()> { - let ( - full_snapshot_archive_slot, - (incremental_snapshot_archive_base_slot, incremental_snapshot_archive_slot), - deserialized_bank, - ) = restore_from_incremental_snapshot(snapshot_config, accounts_dir, genesis_config)?; - - assert_eq!( - full_snapshot_archive_slot, - incremental_snapshot_archive_base_slot - ); - assert_eq!(full_snapshot_archive_slot, last_full_snapshot_slot); - assert_eq!(incremental_snapshot_archive_slot, bank.slot(),); - assert_eq!(*bank, deserialized_bank); - - Ok(()) - } - - fn restore_from_incremental_snapshot( - snapshot_config: &SnapshotConfig, - accounts_dir: PathBuf, - genesis_config: &GenesisConfig, - ) -> snapshot_utils::Result<(Slot, (Slot, Slot), Bank)> { - let full_snapshot_archive_info = snapshot_utils::get_highest_full_snapshot_archive_info( + let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives( + &snapshot_config.snapshot_path, &snapshot_config.snapshot_package_output_path, - ) - .ok_or_else(|| Error::new(ErrorKind::Other, "no full snapshot"))?; - - let incremental_snapshot_archive_info = - snapshot_utils::get_highest_incremental_snapshot_archive_info( - &snapshot_config.snapshot_package_output_path, - *full_snapshot_archive_info.slot(), - ) - .ok_or_else(|| Error::new(ErrorKind::Other, "no incremental snapshot"))?; - - info!("Restoring bank from full snapshot slot: {}, and incremental snapshot slot: {} (with base slot: {})", - full_snapshot_archive_info.slot(), incremental_snapshot_archive_info.slot(), incremental_snapshot_archive_info.base_slot()); - - let (deserialized_bank, _) = snapshot_utils::bank_from_snapshot_archives( &[accounts_dir], &[], - &snapshot_config.snapshot_path, - full_snapshot_archive_info.path(), - Some(incremental_snapshot_archive_info.path()), - snapshot_config.archive_format, genesis_config, None, None, @@ -846,13 +809,8 @@ mod tests { false, )?; - Ok(( - *full_snapshot_archive_info.slot(), - ( - *incremental_snapshot_archive_info.base_slot(), - *incremental_snapshot_archive_info.slot(), - ), - deserialized_bank, - )) + assert_eq!(bank, &deserialized_bank); + + Ok(()) } } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 1a07319962..a806e728a6 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -31,7 +31,8 @@ use solana_runtime::{ hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_config::SnapshotConfig, snapshot_utils::{ - self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + self, ArchiveFormat, SnapshotArchiveInfoGetter, SnapshotVersion, + DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }; use solana_sdk::{ @@ -2199,7 +2200,7 @@ fn main() { bank.slot(), ); - let archive_file = snapshot_utils::bank_to_full_snapshot_archive( + let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive( ledger_path, &bank, Some(snapshot_version), @@ -2217,7 +2218,7 @@ fn main() { "Successfully created snapshot for slot {}, hash {}: {}", bank.slot(), bank.hash(), - archive_file.display(), + full_snapshot_archive_info.path().display(), ); println!( "Shred version: {}", diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 12653f2800..05087408aa 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -8,11 +8,7 @@ use crate::{ }; use log::*; use solana_entry::entry::VerifyRecyclers; -use solana_runtime::{ - bank_forks::BankForks, - snapshot_config::SnapshotConfig, - snapshot_utils::{self, FullSnapshotArchiveInfo}, -}; +use solana_runtime::{bank_forks::BankForks, snapshot_config::SnapshotConfig, snapshot_utils}; use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash}; use std::{fs, path::PathBuf, process, result}; @@ -44,19 +40,19 @@ pub fn load( transaction_status_sender: Option<&TransactionStatusSender>, cache_block_meta_sender: Option<&CacheBlockMetaSender>, ) -> LoadResult { - if let Some(snapshot_config) = snapshot_config.as_ref() { + if let Some(snapshot_config) = snapshot_config { info!( - "Initializing snapshot path: {:?}", - snapshot_config.snapshot_path + "Initializing snapshot path: {}", + snapshot_config.snapshot_path.display() ); let _ = fs::remove_dir_all(&snapshot_config.snapshot_path); fs::create_dir_all(&snapshot_config.snapshot_path) .expect("Couldn't create snapshot directory"); - if let Some(full_snapshot_archive_info) = - snapshot_utils::get_highest_full_snapshot_archive_info( - &snapshot_config.snapshot_package_output_path, - ) + if snapshot_utils::get_highest_full_snapshot_archive_info( + &snapshot_config.snapshot_package_output_path, + ) + .is_some() { return load_from_snapshot( genesis_config, @@ -67,7 +63,6 @@ pub fn load( process_options, transaction_status_sender, cache_block_meta_sender, - &full_snapshot_archive_info, ); } else { info!("No snapshot package available; will load from genesis"); @@ -115,26 +110,18 @@ fn load_from_snapshot( process_options: ProcessOptions, transaction_status_sender: Option<&TransactionStatusSender>, cache_block_meta_sender: Option<&CacheBlockMetaSender>, - full_snapshot_archive_info: &FullSnapshotArchiveInfo, ) -> LoadResult { - info!( - "Loading snapshot package: {:?}", - full_snapshot_archive_info.path() - ); - // Fail hard here if snapshot fails to load, don't silently continue if account_paths.is_empty() { error!("Account paths not present when booting from snapshot"); process::exit(1); } - let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives( + let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives( + &snapshot_config.snapshot_path, + &snapshot_config.snapshot_package_output_path, &account_paths, &process_options.frozen_accounts, - &snapshot_config.snapshot_path, - full_snapshot_archive_info.path(), - None, - *full_snapshot_archive_info.archive_format(), genesis_config, process_options.debug_keys.clone(), Some(&crate::builtins::get(process_options.bpf_jit)), @@ -147,30 +134,14 @@ fn load_from_snapshot( process_options.verify_index, ) .expect("Load from snapshot failed"); - if let Some(shrink_paths) = shrink_paths { - deserialized_bank.set_shrink_paths(shrink_paths); - } let deserialized_bank_slot_and_hash = ( deserialized_bank.slot(), deserialized_bank.get_accounts_hash(), ); - if deserialized_bank_slot_and_hash - != ( - *full_snapshot_archive_info.slot(), - *full_snapshot_archive_info.hash(), - ) - { - error!( - "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}", - ( - full_snapshot_archive_info.slot(), - full_snapshot_archive_info.hash() - ), - deserialized_bank_slot_and_hash - ); - process::exit(1); + if let Some(shrink_paths) = shrink_paths { + deserialized_bank.set_shrink_paths(shrink_paths); } to_loadresult( diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index e78bf2e98f..f92d49d582 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -42,7 +42,7 @@ use { }, solana_runtime::{ snapshot_config::SnapshotConfig, - snapshot_utils::{self, ArchiveFormat}, + snapshot_utils::{self, ArchiveFormat, SnapshotArchiveInfoGetter}, }, solana_sdk::{ account::AccountSharedData, @@ -1855,12 +1855,12 @@ fn test_snapshots_blockstore_floor() { .snapshot_archives_dir .path() .to_path_buf(), - *archive_info.slot(), + archive_info.slot(), archive_info.hash(), ArchiveFormat::TarBzip2, ); fs::hard_link(archive_info.path(), &validator_archive_path).unwrap(); - let slot_floor = *archive_info.slot(); + let slot_floor = archive_info.slot(); // Start up a new node from a snapshot let validator_stake = 5; @@ -3411,11 +3411,11 @@ fn wait_for_next_snapshot( "full snapshot for slot {} exists", full_snapshot_archive_info.slot() ); - if *full_snapshot_archive_info.slot() >= last_slot { + if full_snapshot_archive_info.slot() >= last_slot { return ( full_snapshot_archive_info.path().clone(), ( - *full_snapshot_archive_info.slot(), + full_snapshot_archive_info.slot(), *full_snapshot_archive_info.hash(), ), ); diff --git a/replica-node/src/replica_node.rs b/replica-node/src/replica_node.rs index 0b97a9086b..b8b4bddac6 100644 --- a/replica-node/src/replica_node.rs +++ b/replica-node/src/replica_node.rs @@ -115,9 +115,8 @@ fn initialize_from_snapshot( &replica_config.account_paths, &[], &snapshot_config.snapshot_path, - archive_info.path(), + &archive_info, None, - *archive_info.archive_format(), genesis_config, process_options.debug_keys.clone(), None, diff --git a/replica-node/src/replica_util.rs b/replica-node/src/replica_util.rs index ba7f8dc5e4..faad3887a2 100644 --- a/replica-node/src/replica_util.rs +++ b/replica-node/src/replica_util.rs @@ -6,7 +6,7 @@ use { contact_info::ContactInfo, gossip_service::GossipService, }, - solana_runtime::snapshot_utils, + solana_runtime::snapshot_utils::{self, SnapshotArchiveInfoGetter}, solana_sdk::{ clock::Slot, hash::Hash, @@ -112,7 +112,7 @@ fn get_rpc_peer_node( let mut highest_snapshot_info: Option<(Slot, Hash)> = snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map( |snapshot_archive_info| { - (*snapshot_archive_info.slot(), *snapshot_archive_info.hash()) + (snapshot_archive_info.slot(), *snapshot_archive_info.hash()) }, ); let eligible_rpc_peers = { diff --git a/replica-node/tests/local_replica.rs b/replica-node/tests/local_replica.rs index 078d17d0d1..316e0cf293 100644 --- a/replica-node/tests/local_replica.rs +++ b/replica-node/tests/local_replica.rs @@ -17,7 +17,7 @@ use { solana_runtime::{ accounts_index::AccountSecondaryIndexes, snapshot_config::SnapshotConfig, - snapshot_utils::{self, ArchiveFormat}, + snapshot_utils::{self, ArchiveFormat, SnapshotArchiveInfoGetter}, }, solana_sdk::{ client::SyncClient, @@ -68,11 +68,11 @@ fn wait_for_next_snapshot( "full snapshot for slot {} exists", full_snapshot_archive_info.slot() ); - if *full_snapshot_archive_info.slot() >= last_slot { + if full_snapshot_archive_info.slot() >= last_slot { return ( full_snapshot_archive_info.path().clone(), ( - *full_snapshot_archive_info.slot(), + full_snapshot_archive_info.slot(), *full_snapshot_archive_info.hash(), ), ); diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index a4c89ca92e..2f2b0bd26f 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -26,8 +26,10 @@ use { solana_metrics::inc_new_counter_info, solana_poh::poh_recorder::PohRecorder, solana_runtime::{ - bank_forks::BankForks, commitment::BlockCommitmentCache, snapshot_config::SnapshotConfig, - snapshot_utils, + bank_forks::BankForks, + commitment::BlockCommitmentCache, + snapshot_config::SnapshotConfig, + snapshot_utils::{self, SnapshotArchiveInfoGetter}, }, solana_sdk::{ exit::Exit, genesis_config::DEFAULT_GENESIS_DOWNLOAD_PATH, hash::Hash, diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index ec5d2a547a..44fa9d62a8 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,6 +1,6 @@ use crate::snapshot_utils::{ - ArchiveFormat, BankSnapshotInfo, Result, SnapshotVersion, TMP_FULL_SNAPSHOT_PREFIX, - TMP_INCREMENTAL_SNAPSHOT_PREFIX, + ArchiveFormat, BankSnapshotInfo, Result, SnapshotArchiveInfo, SnapshotArchiveInfoGetter, + SnapshotVersion, TMP_FULL_SNAPSHOT_PREFIX, TMP_INCREMENTAL_SNAPSHOT_PREFIX, }; use crate::{ accounts_db::SnapshotStorages, @@ -209,14 +209,11 @@ impl AccountsPackagePre { } pub struct AccountsPackage { - pub slot: Slot, + pub snapshot_archive_info: SnapshotArchiveInfo, pub block_height: Slot, pub slot_deltas: Vec, pub snapshot_links: TempDir, pub storages: SnapshotStorages, - pub tar_output_file: PathBuf, - pub hash: Hash, - pub archive_format: ArchiveFormat, pub snapshot_version: SnapshotVersion, } @@ -227,21 +224,29 @@ impl AccountsPackage { slot_deltas: Vec, snapshot_links: TempDir, storages: SnapshotStorages, - tar_output_file: PathBuf, + snapshot_archive_path: PathBuf, hash: Hash, archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, ) -> Self { Self { - slot, + snapshot_archive_info: SnapshotArchiveInfo { + path: snapshot_archive_path, + slot, + hash, + archive_format, + }, block_height, slot_deltas, snapshot_links, storages, - tar_output_file, - hash, - archive_format, snapshot_version, } } } + +impl SnapshotArchiveInfoGetter for AccountsPackage { + fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo { + &self.snapshot_archive_info + } +} diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 44e5e79705..53e83d9450 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -39,33 +39,54 @@ use { thiserror::Error, }; +/// Trait to query the snapshot archive information +pub trait SnapshotArchiveInfoGetter { + fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo; + + fn path(&self) -> &PathBuf { + &self.snapshot_archive_info().path + } + + fn slot(&self) -> Slot { + self.snapshot_archive_info().slot + } + + fn hash(&self) -> &Hash { + &self.snapshot_archive_info().hash + } + + fn archive_format(&self) -> ArchiveFormat { + self.snapshot_archive_info().archive_format + } +} + /// Common information about a snapshot archive -#[derive(PartialEq, Eq, Debug)] -struct SnapshotArchiveInfo { +#[derive(PartialEq, Eq, Debug, Clone)] +pub struct SnapshotArchiveInfo { /// Path to the snapshot archive file - path: PathBuf, + pub path: PathBuf, /// Slot that the snapshot was made - slot: Slot, + pub slot: Slot, /// Hash of the accounts at this slot - hash: Hash, + pub hash: Hash, /// Archive format for the snapshot file - archive_format: ArchiveFormat, + pub archive_format: ArchiveFormat, } /// Information about a full snapshot archive: its path, slot, hash, and archive format -#[derive(PartialEq, Eq, Debug)] +#[derive(PartialEq, Eq, Debug, Clone)] pub struct FullSnapshotArchiveInfo(SnapshotArchiveInfo); impl FullSnapshotArchiveInfo { /// Parse the path to a full snapshot archive and return a new `FullSnapshotArchiveInfo` - fn new_from_path(path: PathBuf) -> Result { + pub fn new_from_path(path: PathBuf) -> Result { let filename = path_to_file_name_str(path.as_path())?; let (slot, hash, archive_format) = parse_full_snapshot_archive_filename(filename)?; - Ok(Self(SnapshotArchiveInfo { + Ok(Self::new(SnapshotArchiveInfo { path, slot, hash, @@ -73,20 +94,14 @@ impl FullSnapshotArchiveInfo { })) } - pub fn path(&self) -> &PathBuf { - &self.0.path + fn new(snapshot_archive_info: SnapshotArchiveInfo) -> Self { + Self(snapshot_archive_info) } +} - pub fn slot(&self) -> &Slot { - &self.0.slot - } - - pub fn hash(&self) -> &Hash { - &self.0.hash - } - - pub fn archive_format(&self) -> &ArchiveFormat { - &self.0.archive_format +impl SnapshotArchiveInfoGetter for FullSnapshotArchiveInfo { + fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo { + &self.0 } } @@ -99,12 +114,12 @@ impl PartialOrd for FullSnapshotArchiveInfo { // Order `FullSnapshotArchiveInfo` by slot (ascending), which practially is sorting chronologically impl Ord for FullSnapshotArchiveInfo { fn cmp(&self, other: &Self) -> Ordering { - self.slot().cmp(other.slot()) + self.slot().cmp(&other.slot()) } } /// Information about an incremental snapshot archive: its path, slot, base slot, hash, and archive format -#[derive(PartialEq, Eq, Debug)] +#[derive(PartialEq, Eq, Debug, Clone)] pub struct IncrementalSnapshotArchiveInfo { /// The slot that the incremental snapshot was based from. This is the same as the full /// snapshot slot used when making the incremental snapshot. @@ -117,40 +132,37 @@ pub struct IncrementalSnapshotArchiveInfo { impl IncrementalSnapshotArchiveInfo { /// Parse the path to an incremental snapshot archive and return a new `IncrementalSnapshotArchiveInfo` - fn new_from_path(path: PathBuf) -> Result { + pub fn new_from_path(path: PathBuf) -> Result { let filename = path_to_file_name_str(path.as_path())?; let (base_slot, slot, hash, archive_format) = parse_incremental_snapshot_archive_filename(filename)?; - Ok(Self { + Ok(Self::new( base_slot, - inner: SnapshotArchiveInfo { + SnapshotArchiveInfo { path, slot, hash, archive_format, }, - }) + )) } - pub fn path(&self) -> &PathBuf { - &self.inner.path + fn new(base_slot: Slot, snapshot_archive_info: SnapshotArchiveInfo) -> Self { + Self { + base_slot, + inner: snapshot_archive_info, + } } - pub fn base_slot(&self) -> &Slot { - &self.base_slot + pub fn base_slot(&self) -> Slot { + self.base_slot } +} - pub fn slot(&self) -> &Slot { - &self.inner.slot - } - - fn _hash(&self) -> &Hash { - &self.inner.hash - } - - fn _archive_format(&self) -> &ArchiveFormat { - &self.inner.archive_format +impl SnapshotArchiveInfoGetter for IncrementalSnapshotArchiveInfo { + fn snapshot_archive_info(&self) -> &SnapshotArchiveInfo { + &self.inner } } @@ -165,8 +177,8 @@ impl PartialOrd for IncrementalSnapshotArchiveInfo { impl Ord for IncrementalSnapshotArchiveInfo { fn cmp(&self, other: &Self) -> Ordering { self.base_slot() - .cmp(other.base_slot()) - .then(self.slot().cmp(other.slot())) + .cmp(&other.base_slot()) + .then(self.slot().cmp(&other.slot())) } } @@ -324,6 +336,12 @@ pub enum SnapshotError { #[error("snapshots are incompatible: full snapshot slot ({0}) and incremental snapshot base slot ({1}) do not match")] MismatchedBaseSlot(Slot, Slot), + + #[error("no snapshot archives to load from")] + NoSnapshotArchives, + + #[error("snapshot has mismatch: deserialized bank: {:?}, snapshot archive info: {:?}", .0, .1)] + MismatchedSlotHash((Slot, Hash), (Slot, Hash)), } pub type Result = std::result::Result; @@ -368,11 +386,11 @@ pub fn archive_snapshot_package( ) -> Result<()> { info!( "Generating snapshot archive for slot {}", - snapshot_package.slot + snapshot_package.slot() ); serialize_status_cache( - snapshot_package.slot, + snapshot_package.slot(), &snapshot_package.slot_deltas, &snapshot_package .snapshot_links @@ -382,7 +400,7 @@ pub fn archive_snapshot_package( let mut timer = Measure::start("snapshot_package-package_snapshots"); let tar_dir = snapshot_package - .tar_output_file + .path() .parent() .expect("Tar output path is invalid"); @@ -393,7 +411,8 @@ pub fn archive_snapshot_package( let staging_dir = tempfile::Builder::new() .prefix(&format!( "{}{}-", - TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot + TMP_FULL_SNAPSHOT_PREFIX, + snapshot_package.slot() )) .tempdir_in(tar_dir) .map_err(|e| SnapshotError::IoWithSource(e, "create archive tempdir"))?; @@ -439,12 +458,14 @@ pub fn archive_snapshot_package( .map_err(|e| SnapshotError::IoWithSource(e, "write version file"))?; } - let file_ext = get_archive_ext(snapshot_package.archive_format); + let file_ext = get_archive_ext(snapshot_package.archive_format()); // Tar the staging directory into the archive at `archive_path` let archive_path = tar_dir.join(format!( "{}{}.{}", - TMP_FULL_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext + TMP_FULL_SNAPSHOT_PREFIX, + snapshot_package.slot(), + file_ext )); { @@ -460,7 +481,7 @@ pub fn archive_snapshot_package( Ok(()) }; - match snapshot_package.archive_format { + match snapshot_package.archive_format() { ArchiveFormat::TarBzip2 => { let mut encoder = bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best()); @@ -487,7 +508,7 @@ pub fn archive_snapshot_package( // Atomically move the archive into position for other validators to find let metadata = fs::metadata(&archive_path) .map_err(|e| SnapshotError::IoWithSource(e, "archive path stat"))?; - fs::rename(&archive_path, &snapshot_package.tar_output_file) + fs::rename(&archive_path, &snapshot_package.path()) .map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?; purge_old_snapshot_archives(tar_dir, maximum_snapshots_to_retain); @@ -495,14 +516,14 @@ pub fn archive_snapshot_package( timer.stop(); info!( "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}", - snapshot_package.tar_output_file, - snapshot_package.slot, + snapshot_package.path(), + snapshot_package.slot(), timer.as_ms(), metadata.len() ); datapoint_info!( "snapshot-package", - ("slot", snapshot_package.slot, i64), + ("slot", snapshot_package.slot(), i64), ("duration_ms", timer.as_ms(), i64), ("size", metadata.len(), i64) ); @@ -829,13 +850,12 @@ const PARALLEL_UNTAR_READERS_DEFAULT: usize = 4; /// Rebuild bank from snapshot archives. Handles either just a full snapshot, or both a full /// snapshot and an incremental snapshot. #[allow(clippy::too_many_arguments)] -pub fn bank_from_snapshot_archives

( +pub fn bank_from_snapshot_archives( account_paths: &[PathBuf], frozen_account_pubkeys: &[Pubkey], - snapshots_dir: &Path, - full_snapshot_archive_path: P, - incremental_snapshot_archive_path: Option

, - archive_format: ArchiveFormat, + snapshots_dir: impl AsRef, + full_snapshot_archive_info: &FullSnapshotArchiveInfo, + incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>, genesis_config: &GenesisConfig, debug_keys: Option>>, additional_builtins: Option<&Builtins>, @@ -846,39 +866,36 @@ pub fn bank_from_snapshot_archives

( test_hash_calculation: bool, accounts_db_skip_shrink: bool, verify_index: bool, -) -> Result<(Bank, BankFromArchiveTimings)> -where - P: AsRef + std::marker::Sync, -{ +) -> Result<(Bank, BankFromArchiveTimings)> { + check_are_snapshots_compatible( + full_snapshot_archive_info, + incremental_snapshot_archive_info, + )?; + let parallel_divisions = std::cmp::min( PARALLEL_UNTAR_READERS_DEFAULT, std::cmp::max(1, num_cpus::get() / 4), ); let unarchived_full_snapshot = unarchive_snapshot( - snapshots_dir, + &snapshots_dir, TMP_FULL_SNAPSHOT_PREFIX, - &full_snapshot_archive_path, + full_snapshot_archive_info.path(), "snapshot untar", account_paths, - archive_format, + full_snapshot_archive_info.archive_format(), parallel_divisions, )?; let mut unarchived_incremental_snapshot = - if let Some(incremental_snapshot_archive_path) = incremental_snapshot_archive_path { - check_are_snapshots_compatible( - &full_snapshot_archive_path, - &incremental_snapshot_archive_path, - )?; - + if let Some(incremental_snapshot_archive_info) = incremental_snapshot_archive_info { let unarchived_incremental_snapshot = unarchive_snapshot( - snapshots_dir, + &snapshots_dir, TMP_INCREMENTAL_SNAPSHOT_PREFIX, - &incremental_snapshot_archive_path, + incremental_snapshot_archive_info.path(), "incremental snapshot untar", account_paths, - archive_format, + incremental_snapshot_archive_info.archive_format(), parallel_divisions, )?; Some(unarchived_incremental_snapshot) @@ -936,6 +953,98 @@ where Ok((bank, timings)) } +/// Rebuild bank from snapshot archives. This function searches `snapshot_archives_dir` for the +/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank. +#[allow(clippy::too_many_arguments)] +pub fn bank_from_latest_snapshot_archives( + snapshots_dir: impl AsRef, + snapshot_archives_dir: impl AsRef, + account_paths: &[PathBuf], + frozen_account_pubkeys: &[Pubkey], + genesis_config: &GenesisConfig, + debug_keys: Option>>, + additional_builtins: Option<&Builtins>, + account_secondary_indexes: AccountSecondaryIndexes, + accounts_db_caching_enabled: bool, + limit_load_slot_count_from_snapshot: Option, + shrink_ratio: AccountShrinkThreshold, + test_hash_calculation: bool, + accounts_db_skip_shrink: bool, + verify_index: bool, +) -> Result<(Bank, BankFromArchiveTimings)> { + let full_snapshot_archive_info = get_highest_full_snapshot_archive_info(&snapshot_archives_dir) + .ok_or(SnapshotError::NoSnapshotArchives)?; + + let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info( + &snapshot_archives_dir, + full_snapshot_archive_info.slot(), + ); + + info!( + "Loading bank from full snapshot: {}, and incremental snapshot: {:?}", + full_snapshot_archive_info.path().display(), + incremental_snapshot_archive_info + .as_ref() + .map( + |incremental_snapshot_archive_info| incremental_snapshot_archive_info + .path() + .display() + ) + ); + + let (bank, timings) = bank_from_snapshot_archives( + account_paths, + frozen_account_pubkeys, + snapshots_dir.as_ref(), + &full_snapshot_archive_info, + incremental_snapshot_archive_info.as_ref(), + genesis_config, + debug_keys, + additional_builtins, + account_secondary_indexes, + accounts_db_caching_enabled, + limit_load_slot_count_from_snapshot, + shrink_ratio, + test_hash_calculation, + accounts_db_skip_shrink, + verify_index, + )?; + + verify_bank_against_expected_slot_hash( + &bank, + incremental_snapshot_archive_info.as_ref().map_or( + full_snapshot_archive_info.slot(), + |incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot(), + ), + incremental_snapshot_archive_info.as_ref().map_or( + *full_snapshot_archive_info.hash(), + |incremental_snapshot_archive_info| *incremental_snapshot_archive_info.hash(), + ), + )?; + + Ok((bank, timings)) +} + +/// Check to make sure the deserialized bank's slot and hash matches the snapshot archive's slot +/// and hash +fn verify_bank_against_expected_slot_hash( + bank: &Bank, + expected_slot: Slot, + expected_hash: Hash, +) -> Result<()> { + let bank_slot = bank.slot(); + let bank_hash = bank.get_accounts_hash(); + + if bank_slot != expected_slot || bank_hash != expected_hash { + return Err(SnapshotError::MismatchedSlotHash( + (bank_slot, bank_hash), + (expected_slot, expected_hash), + )); + } + + Ok(()) +} + /// Perform the common tasks when unarchiving a snapshot. Handles creating the temporary /// directories, untaring, reading the version file, and then returning those fields plus the /// unpacked append vec map. @@ -987,31 +1096,26 @@ where }) } -/// Check if an incremental snapshot is compatible with a full snapshot. This function parses the -/// paths to see if the incremental snapshot's base slot is the same as the full snapshot's slot. -/// Return an error if they are incompatible (or if the paths cannot be parsed), otherwise return a -/// tuple of the full snapshot slot and the incremental snapshot slot. -fn check_are_snapshots_compatible

( - full_snapshot_archive_path: P, - incremental_snapshot_archive_path: P, -) -> Result<()> -where - P: AsRef, -{ - let full_snapshot_filename = path_to_file_name_str(full_snapshot_archive_path.as_ref())?; - let (full_snapshot_slot, _, _) = parse_full_snapshot_archive_filename(full_snapshot_filename)?; +/// Check if an incremental snapshot is compatible with a full snapshot. This is done by checking +/// if the incremental snapshot's base slot is the same as the full snapshot's slot. +fn check_are_snapshots_compatible( + full_snapshot_archive_info: &FullSnapshotArchiveInfo, + incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>, +) -> Result<()> { + if incremental_snapshot_archive_info.is_none() { + return Ok(()); + } - let incremental_snapshot_filename = - path_to_file_name_str(incremental_snapshot_archive_path.as_ref())?; - let (incremental_snapshot_base_slot, _, _, _) = - parse_incremental_snapshot_archive_filename(incremental_snapshot_filename)?; + let incremental_snapshot_archive_info = incremental_snapshot_archive_info.unwrap(); - (full_snapshot_slot == incremental_snapshot_base_slot) + (full_snapshot_archive_info.slot() == incremental_snapshot_archive_info.base_slot()) .then(|| ()) - .ok_or(SnapshotError::MismatchedBaseSlot( - full_snapshot_slot, - incremental_snapshot_base_slot, - )) + .ok_or_else(|| { + SnapshotError::MismatchedBaseSlot( + full_snapshot_archive_info.slot(), + incremental_snapshot_archive_info.base_slot(), + ) + }) } /// Get the `&str` from a `&Path` @@ -1189,7 +1293,7 @@ where P: AsRef, { get_highest_full_snapshot_archive_info(snapshot_archives_dir) - .map(|full_snapshot_archive_info| *full_snapshot_archive_info.slot()) + .map(|full_snapshot_archive_info| full_snapshot_archive_info.slot()) } /// Get the highest slot of the incremental snapshot archives in a directory, for a given full @@ -1199,7 +1303,7 @@ pub fn get_highest_incremental_snapshot_archive_slot>( full_snapshot_slot: Slot, ) -> Option { get_highest_incremental_snapshot_archive_info(snapshot_archives_dir, full_snapshot_slot) - .map(|incremental_snapshot_archive_info| *incremental_snapshot_archive_info.slot()) + .map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot()) } /// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory @@ -1230,7 +1334,7 @@ where get_incremental_snapshot_archives(snapshot_archives_dir) .into_iter() .filter(|incremental_snapshot_archive_info| { - *incremental_snapshot_archive_info.base_slot() == full_snapshot_slot + incremental_snapshot_archive_info.base_slot() == full_snapshot_slot }) .collect::>(); incremental_snapshot_archives.sort_unstable(); @@ -1272,7 +1376,7 @@ where get_highest_full_snapshot_archive_slot(&snapshot_archives_dir).unwrap_or(Slot::MAX); get_incremental_snapshot_archives(&snapshot_archives_dir) .iter() - .filter(|archive_info| *archive_info.base_slot() < last_full_snapshot_slot) + .filter(|archive_info| archive_info.base_slot() < last_full_snapshot_slot) .for_each(|old_archive| { trace!( "Purging old incremental snapshot archive: {}", @@ -1598,15 +1702,15 @@ pub fn snapshot_bank( /// /// Requires: /// - `bank` is complete -pub fn bank_to_full_snapshot_archive, Q: AsRef>( - snapshots_dir: P, +pub fn bank_to_full_snapshot_archive( + snapshots_dir: impl AsRef, bank: &Bank, snapshot_version: Option, - snapshot_package_output_path: Q, + snapshot_package_output_path: impl AsRef, archive_format: ArchiveFormat, thread_pool: Option<&ThreadPool>, maximum_snapshots_to_retain: usize, -) -> Result { +) -> Result { let snapshot_version = snapshot_version.unwrap_or_default(); assert!(bank.is_complete()); @@ -1639,16 +1743,16 @@ pub fn bank_to_full_snapshot_archive, Q: AsRef>( /// Requires: /// - `bank` is complete /// - `bank`'s slot is greater than `full_snapshot_slot` -pub fn bank_to_incremental_snapshot_archive, Q: AsRef>( - snapshots_dir: P, +pub fn bank_to_incremental_snapshot_archive( + snapshots_dir: impl AsRef, bank: &Bank, full_snapshot_slot: Slot, snapshot_version: Option, - snapshot_package_output_path: Q, + snapshot_package_output_path: impl AsRef, archive_format: ArchiveFormat, thread_pool: Option<&ThreadPool>, maximum_snapshots_to_retain: usize, -) -> Result { +) -> Result { let snapshot_version = snapshot_version.unwrap_or_default(); assert!(bank.is_complete()); @@ -1688,7 +1792,7 @@ pub fn package_process_and_archive_full_snapshot( snapshot_version: SnapshotVersion, thread_pool: Option<&ThreadPool>, maximum_snapshots_to_retain: usize, -) -> Result { +) -> Result { let package = AccountsPackagePre::new_full_snapshot_package( bank, bank_snapshot_info, @@ -1701,12 +1805,14 @@ pub fn package_process_and_archive_full_snapshot( None, )?; - process_and_archive_snapshot_package_pre( + let package = process_and_archive_snapshot_package_pre( package, thread_pool, None, maximum_snapshots_to_retain, - ) + )?; + + Ok(FullSnapshotArchiveInfo::new(package.snapshot_archive_info)) } /// Helper function to hold shared code to package, process, and archive incremental snapshots @@ -1722,7 +1828,7 @@ pub fn package_process_and_archive_incremental_snapshot( snapshot_version: SnapshotVersion, thread_pool: Option<&ThreadPool>, maximum_snapshots_to_retain: usize, -) -> Result { +) -> Result { let package = AccountsPackagePre::new_incremental_snapshot_package( bank, incremental_snapshot_base_slot, @@ -1736,12 +1842,17 @@ pub fn package_process_and_archive_incremental_snapshot( None, )?; - process_and_archive_snapshot_package_pre( + let package = process_and_archive_snapshot_package_pre( package, thread_pool, Some(incremental_snapshot_base_slot), maximum_snapshots_to_retain, - ) + )?; + + Ok(IncrementalSnapshotArchiveInfo::new( + incremental_snapshot_base_slot, + package.snapshot_archive_info, + )) } /// Helper function to hold shared code to process and archive snapshot packages @@ -1750,13 +1861,13 @@ fn process_and_archive_snapshot_package_pre( thread_pool: Option<&ThreadPool>, incremental_snapshot_base_slot: Option, maximum_snapshots_to_retain: usize, -) -> Result { +) -> Result { let package = process_accounts_package_pre(package_pre, thread_pool, incremental_snapshot_base_slot); archive_snapshot_package(&package, maximum_snapshots_to_retain)?; - Ok(package.tar_output_file) + Ok(package) } pub fn process_accounts_package_pre( @@ -1789,7 +1900,7 @@ pub fn process_accounts_package_pre( ("calculate_hash", time.as_us(), i64), ); - let tar_output_file = match incremental_snapshot_base_slot { + let snapshot_archive_path = match incremental_snapshot_base_slot { None => build_full_snapshot_archive_path( accounts_package.snapshot_output_dir, accounts_package.slot, @@ -1811,7 +1922,7 @@ pub fn process_accounts_package_pre( accounts_package.slot_deltas, accounts_package.snapshot_links, accounts_package.storages, - tar_output_file, + snapshot_archive_path, hash, accounts_package.archive_format, accounts_package.snapshot_version, @@ -2095,25 +2206,40 @@ mod tests { let slot2: Slot = 5678; let slot3: Slot = 999_999; - assert!(check_are_snapshots_compatible( - &format!("/dir/snapshot-{}-{}.tar", slot1, Hash::new_unique()), - &format!( + let full_snapshot_archive_info = FullSnapshotArchiveInfo::new_from_path(PathBuf::from( + format!("/dir/snapshot-{}-{}.tar", slot1, Hash::new_unique()), + )) + .unwrap(); + + assert!(check_are_snapshots_compatible(&full_snapshot_archive_info, None,).is_ok()); + + let incremental_snapshot_archive_info = + IncrementalSnapshotArchiveInfo::new_from_path(PathBuf::from(format!( "/dir/incremental-snapshot-{}-{}-{}.tar", slot1, slot2, Hash::new_unique() - ), + ))) + .unwrap(); + + assert!(check_are_snapshots_compatible( + &full_snapshot_archive_info, + Some(&incremental_snapshot_archive_info) ) .is_ok()); - assert!(check_are_snapshots_compatible( - &format!("/dir/snapshot-{}-{}.tar", slot1, Hash::new_unique()), - &format!( + let incremental_snapshot_archive_info = + IncrementalSnapshotArchiveInfo::new_from_path(PathBuf::from(format!( "/dir/incremental-snapshot-{}-{}-{}.tar", slot2, slot3, Hash::new_unique() - ), + ))) + .unwrap(); + + assert!(check_are_snapshots_compatible( + &full_snapshot_archive_info, + Some(&incremental_snapshot_archive_info) ) .is_err()); } @@ -2387,14 +2513,14 @@ mod tests { maximum_snapshots_to_retain + 1 ); assert_eq!( - *full_snapshot_archives.first().unwrap().slot(), + full_snapshot_archives.first().unwrap().slot(), starting_slot ); - assert_eq!(*full_snapshot_archives.last().unwrap().slot(), slot); + assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot); for (i, full_snapshot_archive) in full_snapshot_archives.iter().skip(1).rev().enumerate() { - assert_eq!(*full_snapshot_archive.slot(), slot - i as Slot); + assert_eq!(full_snapshot_archive.slot(), slot - i as Slot); } } } @@ -2425,7 +2551,7 @@ mod tests { get_incremental_snapshot_archives(snapshot_archives_dir.path()); assert_eq!(remaining_incremental_snapshot_archives.len(), 4); for archive in &remaining_incremental_snapshot_archives { - assert_eq!(*archive.base_slot(), 200); + assert_eq!(archive.base_slot(), 200); } } @@ -2471,8 +2597,8 @@ mod tests { let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::Tar; - let snapshot_archive_path = bank_to_full_snapshot_archive( - snapshots_dir.path(), + let snapshot_archive_info = bank_to_full_snapshot_archive( + &snapshots_dir, &original_bank, None, snapshot_archives_dir.path(), @@ -2486,9 +2612,8 @@ mod tests { &[PathBuf::from(accounts_dir.path())], &[], snapshots_dir.path(), - &snapshot_archive_path, + &snapshot_archive_info, None, - snapshot_archive_format, &genesis_config, None, None, @@ -2562,7 +2687,7 @@ mod tests { let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::TarGzip; - let full_snapshot_archive_path = bank_to_full_snapshot_archive( + let full_snapshot_archive_info = bank_to_full_snapshot_archive( snapshots_dir.path(), &bank4, None, @@ -2577,9 +2702,8 @@ mod tests { &[PathBuf::from(accounts_dir.path())], &[], snapshots_dir.path(), - &full_snapshot_archive_path, + &full_snapshot_archive_info, None, - snapshot_archive_format, &genesis_config, None, None, @@ -2639,7 +2763,7 @@ mod tests { let snapshot_archive_format = ArchiveFormat::TarZstd; let full_snapshot_slot = slot; - let full_snapshot_archive_path = bank_to_full_snapshot_archive( + let full_snapshot_archive_info = bank_to_full_snapshot_archive( snapshots_dir.path(), &bank1, None, @@ -2671,7 +2795,7 @@ mod tests { bank4.register_tick(&Hash::new_unique()); } - let incremental_snapshot_archive_path = bank_to_incremental_snapshot_archive( + let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive( snapshots_dir.path(), &bank4, full_snapshot_slot, @@ -2687,9 +2811,8 @@ mod tests { &[PathBuf::from(accounts_dir.path())], &[], snapshots_dir.path(), - &full_snapshot_archive_path, - Some(&incremental_snapshot_archive_path), - snapshot_archive_format, + &full_snapshot_archive_info, + Some(&incremental_snapshot_archive_info), &genesis_config, None, None, @@ -2705,4 +2828,102 @@ mod tests { assert_eq!(*bank4, roundtrip_bank); } + + /// Test rebuilding bank from the latest snapshot archives + #[test] + fn test_bank_from_latest_snapshot_archives() { + solana_logger::setup(); + let collector = Pubkey::new_unique(); + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + + let (genesis_config, mint_keypair) = create_genesis_config(1_000_000); + let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); + bank0.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + bank0.transfer(2, &mint_keypair, &key2.pubkey()).unwrap(); + bank0.transfer(3, &mint_keypair, &key3.pubkey()).unwrap(); + while !bank0.is_complete() { + bank0.register_tick(&Hash::new_unique()); + } + + let slot = 1; + let bank1 = Arc::new(Bank::new_from_parent(&bank0, &collector, slot)); + bank1.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + bank1.transfer(2, &mint_keypair, &key2.pubkey()).unwrap(); + bank1.transfer(3, &mint_keypair, &key3.pubkey()).unwrap(); + while !bank1.is_complete() { + bank1.register_tick(&Hash::new_unique()); + } + + let accounts_dir = tempfile::TempDir::new().unwrap(); + let snapshots_dir = tempfile::TempDir::new().unwrap(); + let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let snapshot_archive_format = ArchiveFormat::Tar; + + let full_snapshot_slot = slot; + bank_to_full_snapshot_archive( + &snapshots_dir, + &bank1, + None, + &snapshot_archives_dir, + snapshot_archive_format, + None, + std::usize::MAX, + ) + .unwrap(); + + let slot = slot + 1; + let bank2 = Arc::new(Bank::new_from_parent(&bank1, &collector, slot)); + bank2.transfer(1, &mint_keypair, &key1.pubkey()).unwrap(); + while !bank2.is_complete() { + bank2.register_tick(&Hash::new_unique()); + } + + let slot = slot + 1; + let bank3 = Arc::new(Bank::new_from_parent(&bank2, &collector, slot)); + bank3.transfer(2, &mint_keypair, &key2.pubkey()).unwrap(); + while !bank3.is_complete() { + bank3.register_tick(&Hash::new_unique()); + } + + let slot = slot + 1; + let bank4 = Arc::new(Bank::new_from_parent(&bank3, &collector, slot)); + bank4.transfer(3, &mint_keypair, &key3.pubkey()).unwrap(); + while !bank4.is_complete() { + bank4.register_tick(&Hash::new_unique()); + } + + bank_to_incremental_snapshot_archive( + &snapshots_dir, + &bank4, + full_snapshot_slot, + None, + &snapshot_archives_dir, + snapshot_archive_format, + None, + std::usize::MAX, + ) + .unwrap(); + + let (deserialized_bank, _) = bank_from_latest_snapshot_archives( + &snapshots_dir, + &snapshot_archives_dir, + &[accounts_dir.as_ref().to_path_buf()], + &[], + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + false, + false, + ) + .unwrap(); + + assert_eq!(deserialized_bank, *bank4); + } } diff --git a/validator/src/main.rs b/validator/src/main.rs index 719404ef86..927e78b03c 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -49,7 +49,8 @@ use { hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, snapshot_config::SnapshotConfig, snapshot_utils::{ - self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + self, ArchiveFormat, SnapshotArchiveInfoGetter, SnapshotVersion, + DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }, solana_sdk::{ @@ -482,7 +483,7 @@ fn get_rpc_node( let mut highest_snapshot_hash: Option<(Slot, Hash)> = snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map( |snapshot_archive_info| { - (*snapshot_archive_info.slot(), *snapshot_archive_info.hash()) + (snapshot_archive_info.slot(), *snapshot_archive_info.hash()) }, ); let eligible_rpc_peers = if snapshot_not_required {