gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add CLI options and runtime support for selection of output snapshot version. (#10536)

Browse Source
This commit is contained in:
Kristofer Peterson
2020-06-19 06:38:37 +01:00
committed by GitHub
parent b172b3369e
commit 6d81eede93
10 changed files with 228 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/accounts_hash_verifier.rs
View File

@@ -176,6 +176,7 @@ mod tests {
use crate::cluster_info::make_accounts_hashes_message; use crate::cluster_info::make_accounts_hashes_message;
use crate::contact_info::ContactInfo; use crate::contact_info::ContactInfo;
use solana_runtime::bank_forks::CompressionType; use solana_runtime::bank_forks::CompressionType;
use solana_runtime::snapshot_utils::SnapshotVersion;
use solana_sdk::{ use solana_sdk::{
hash::hash, hash::hash,
signature::{Keypair, Signer}, signature::{Keypair, Signer},
@@ -238,6 +239,7 @@ mod tests {
tar_output_file: PathBuf::from("."), tar_output_file: PathBuf::from("."),
storages: vec![], storages: vec![],
compression: CompressionType::Bzip2, compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
}; };
AccountsHashVerifier::process_accounts_package( AccountsHashVerifier::process_accounts_package(

5
core/src/rpc_service.rs
View File

@@ -353,7 +353,9 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo}, genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path, get_tmp_ledger_path,
}; };
use solana_runtime::{bank::Bank, bank_forks::CompressionType}; use solana_runtime::{
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
};
use solana_sdk::signature::Signer; use solana_sdk::signature::Signer;
use std::net::{IpAddr, Ipv4Addr}; use std::net::{IpAddr, Ipv4Addr};
@@ -445,6 +447,7 @@ mod tests {
snapshot_package_output_path: PathBuf::from("/"), snapshot_package_output_path: PathBuf::from("/"),
snapshot_path: PathBuf::from("/"), snapshot_path: PathBuf::from("/"),
compression: CompressionType::Bzip2, compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
}), }),
bank_forks, bank_forks,
RpcHealth::stub(), RpcHealth::stub(),

3
core/src/snapshot_packager_service.rs
View File

@@ -83,7 +83,7 @@ mod tests {
bank::BankSlotDelta, bank::BankSlotDelta,
bank_forks::CompressionType, bank_forks::CompressionType,
snapshot_package::AccountsPackage, snapshot_package::AccountsPackage,
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME}, snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
}; };
use solana_sdk::hash::Hash; use solana_sdk::hash::Hash;
use std::{ use std::{
@@ -174,6 +174,7 @@ mod tests {
output_tar_path.clone(), output_tar_path.clone(),
Hash::default(), Hash::default(),
CompressionType::Bzip2, CompressionType::Bzip2,
SnapshotVersion::default(),
); );
// Make tarball from packageable snapshot // Make tarball from packageable snapshot

150
core/tests/bank_forks.rs
View File

@@ -1,5 +1,36 @@
// Long-running bank_forks tests // Long-running bank_forks tests
macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
($x:ident) => {
#[allow(non_snake_case)]
mod $x {
use super::*;
const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
#[test]
fn test_bank_forks_status_cache_snapshot_n() {
run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_bank_forks_snapshot_n() {
run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_concurrent_snapshot_packaging() {
run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
}
#[test]
fn test_slots_to_snapshot() {
run_test_slots_to_snapshot(SNAPSHOT_VERSION)
}
}
};
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use bincode::serialize_into; use bincode::serialize_into;
@@ -13,6 +44,7 @@ mod tests {
bank_forks::{BankForks, CompressionType, SnapshotConfig}, bank_forks::{BankForks, CompressionType, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo}, genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils, snapshot_utils,
snapshot_utils::SnapshotVersion,
status_cache::MAX_CACHE_ENTRIES, status_cache::MAX_CACHE_ENTRIES,
}; };
use solana_sdk::{ use solana_sdk::{
@@ -26,6 +58,9 @@ mod tests {
use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc}; use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
use tempfile::TempDir; use tempfile::TempDir;
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
struct SnapshotTestConfig { struct SnapshotTestConfig {
accounts_dir: TempDir, accounts_dir: TempDir,
snapshot_dir: TempDir, snapshot_dir: TempDir,
@@ -35,7 +70,11 @@ mod tests {
genesis_config_info: GenesisConfigInfo, genesis_config_info: GenesisConfigInfo,
} }
fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig { impl SnapshotTestConfig {
fn new(
snapshot_version: SnapshotVersion,
snapshot_interval_slots: u64,
) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap(); let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap(); let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap(); let snapshot_output_path = TempDir::new().unwrap();
@@ -54,6 +93,7 @@ mod tests {
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()), snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2, compression: CompressionType::Bzip2,
snapshot_version,
}; };
bank_forks.set_snapshot_config(Some(snapshot_config.clone())); bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig { SnapshotTestConfig {
@@ -65,11 +105,12 @@ mod tests {
genesis_config_info, genesis_config_info,
} }
} }
}
fn restore_from_snapshot( fn restore_from_snapshot(
old_bank_forks: &BankForks, old_bank_forks: &BankForks,
old_last_slot: Slot, old_last_slot: Slot,
account_paths: Vec<PathBuf>, account_paths: &[PathBuf],
) { ) {
let (snapshot_path, snapshot_package_output_path) = old_bank_forks let (snapshot_path, snapshot_package_output_path) = old_bank_forks
.snapshot_config .snapshot_config
@@ -115,17 +156,20 @@ mod tests {
// also marks each bank as root and generates snapshots // also marks each bank as root and generates snapshots
// finally tries to restore from the last bank's snapshot and compares the restored bank to the // finally tries to restore from the last bank's snapshot and compares the restored bank to the
// `last_slot` bank // `last_slot` bank
fn run_bank_forks_snapshot_n<F>(last_slot: Slot, f: F, set_root_interval: u64) fn run_bank_forks_snapshot_n<F>(
where snapshot_version: SnapshotVersion,
last_slot: Slot,
f: F,
set_root_interval: u64,
) where
F: Fn(&mut Bank, &Keypair), F: Fn(&mut Bank, &Keypair),
{ {
solana_logger::setup(); solana_logger::setup();
// Set up snapshotting config // Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1); let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir; let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let (s, _r) = channel(); let (s, _r) = channel();
@@ -143,36 +187,33 @@ mod tests {
} }
// Generate a snapshot package for last bank // Generate a snapshot package for last bank
let last_bank = bank_forks.get(last_slot).unwrap(); let last_bank = bank_forks.get(last_slot).unwrap();
let storages: Vec<_> = last_bank.get_snapshot_storages(); let snapshot_config = &snapshot_test_config.snapshot_config;
let slot_snapshot_paths = let snapshot_path = &snapshot_config.snapshot_path;
snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path); let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
.pop()
.expect("no snapshots found in path");
let snapshot_package = snapshot_utils::package_snapshot( let snapshot_package = snapshot_utils::package_snapshot(
last_bank, last_bank,
slot_snapshot_paths &last_slot_snapshot_path,
.last() snapshot_path,
.expect("no snapshots found in path"),
&snapshot_config.snapshot_path,
&last_bank.src.roots(), &last_bank.src.roots(),
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_package_output_path,
storages, last_bank.get_snapshot_storages(),
CompressionType::Bzip2, CompressionType::Bzip2,
snapshot_version,
) )
.unwrap(); .unwrap();
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap(); snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
restore_from_snapshot( restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
bank_forks,
last_slot,
vec![accounts_dir.path().to_path_buf()],
);
} }
#[test] fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
fn test_bank_forks_snapshot_n() {
// create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots // create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots
// and restores correctly // and restores correctly
run_bank_forks_snapshot_n( run_bank_forks_snapshot_n(
snapshot_version,
4, 4,
|bank, mint_keypair| { |bank, mint_keypair| {
let key1 = Keypair::new().pubkey(); let key1 = Keypair::new().pubkey();
@@ -203,24 +244,25 @@ mod tests {
} }
} }
#[test] fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
fn test_concurrent_snapshot_packaging() {
solana_logger::setup(); solana_logger::setup();
// Set up snapshotting config // Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1); let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir; let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshots_dir = &snapshot_test_config.snapshot_dir; let snapshots_dir = &snapshot_test_config.snapshot_dir;
let snapshot_config = &snapshot_test_config.snapshot_config; let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
// Take snapshot of zeroth bank // Take snapshot of zeroth bank
let bank0 = bank_forks.get(0).unwrap(); let bank0 = bank_forks.get(0).unwrap();
let storages: Vec<_> = bank0.get_snapshot_storages(); let storages = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap(); snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
// Set up snapshotting channels // Set up snapshotting channels
let (sender, receiver) = channel(); let (sender, receiver) = channel();
@@ -270,7 +312,7 @@ mod tests {
if slot == saved_slot as u64 { if slot == saved_slot as u64 {
let options = CopyOptions::new(); let options = CopyOptions::new();
fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap(); fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap();
let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path) let snapshot_paths = fs::read_dir(snapshot_path)
.unwrap() .unwrap()
.filter_map(|entry| { .filter_map(|entry| {
let e = entry.unwrap(); let e = entry.unwrap();
@@ -282,17 +324,13 @@ mod tests {
.unwrap_or(None) .unwrap_or(None)
}) })
.sorted() .sorted()
.collect(); .last()
// only save off the snapshot of this slot, we don't need the others.
fs_extra::dir::copy(
&snapshot_paths.last().unwrap(),
&saved_snapshots_dir,
&options,
)
.unwrap(); .unwrap();
// only save off the snapshot of this slot, we don't need the others.
fs_extra::dir::copy(&snapshot_paths, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
&snapshot_config.snapshot_package_output_path, snapshot_package_output_path,
&(slot, accounts_hash), &(slot, accounts_hash),
&CompressionType::Bzip2, &CompressionType::Bzip2,
)); ));
@@ -302,11 +340,12 @@ mod tests {
// Purge all the outdated snapshots, including the ones needed to generate the package // Purge all the outdated snapshots, including the ones needed to generate the package
// currently sitting in the channel // currently sitting in the channel
bank_forks.purge_old_snapshots(); bank_forks.purge_old_snapshots();
let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir);
snapshot_paths.sort();
assert_eq!( assert_eq!(
snapshot_paths.iter().map(|path| path.slot).collect_vec(), snapshot_utils::get_snapshot_paths(&snapshots_dir)
(3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec() .into_iter()
.map(|path| path.slot)
.eq(3..=MAX_CACHE_ENTRIES as u64 + 2),
true
); );
// Create a SnapshotPackagerService to create tarballs from all the pending // Create a SnapshotPackagerService to create tarballs from all the pending
@@ -336,13 +375,12 @@ mod tests {
// before we compare, stick an empty status_cache in this dir so that the package comparison works // before we compare, stick an empty status_cache in this dir so that the package comparison works
// This is needed since the status_cache is added by the packager and is not collected from // This is needed since the status_cache is added by the packager and is not collected from
// the source dir for snapshots // the source dir for snapshots
let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
snapshot_utils::serialize_snapshot_data_file( snapshot_utils::serialize_snapshot_data_file(
&saved_snapshots_dir &saved_snapshots_dir
.path() .path()
.join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME), .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
|stream| { |stream| {
serialize_into(stream, &dummy_slot_deltas)?; serialize_into(stream, &[] as &[BankSlotDelta])?;
Ok(()) Ok(())
}, },
) )
@@ -358,16 +396,17 @@ mod tests {
); );
} }
#[test] fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
fn test_slots_to_snapshot() {
solana_logger::setup(); solana_logger::setup();
let num_set_roots = MAX_CACHE_ENTRIES * 2; let num_set_roots = MAX_CACHE_ENTRIES * 2;
for add_root_interval in &[1, 3, 9] { for add_root_interval in &[1, 3, 9] {
let (snapshot_sender, _snapshot_receiver) = channel(); let (snapshot_sender, _snapshot_receiver) = channel();
// Make sure this test never clears bank.slots_since_snapshot // Make sure this test never clears bank.slots_since_snapshot
let mut snapshot_test_config = let mut snapshot_test_config = SnapshotTestConfig::new(
setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64); snapshot_version,
(*add_root_interval * num_set_roots * 2) as u64,
);
let mut current_bank = snapshot_test_config.bank_forks[0].clone(); let mut current_bank = snapshot_test_config.bank_forks[0].clone();
let snapshot_sender = Some(snapshot_sender); let snapshot_sender = Some(snapshot_sender);
for _ in 0..num_set_roots { for _ in 0..num_set_roots {
@@ -386,21 +425,23 @@ mod tests {
} }
let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1; let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
let expected_slots_to_snapshot = (num_old_slots as u64 let expected_slots_to_snapshot =
..=num_set_roots as u64 * *add_root_interval as u64) num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
.collect_vec();
let rooted_bank = snapshot_test_config let slots_to_snapshot = snapshot_test_config
.bank_forks .bank_forks
.get(snapshot_test_config.bank_forks.root()) .get(snapshot_test_config.bank_forks.root())
.unwrap(); .unwrap()
let slots_to_snapshot = rooted_bank.src.roots(); .src
assert_eq!(slots_to_snapshot, expected_slots_to_snapshot); .roots();
assert_eq!(
slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot),
true
);
} }
} }
#[test] fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
fn test_bank_forks_status_cache_snapshot_n() {
// create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time // create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
// this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves // this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
// ahead. Also tests the status_cache purge and status cache snapshotting. // ahead. Also tests the status_cache purge and status cache snapshotting.
@@ -409,6 +450,7 @@ mod tests {
let key2 = Keypair::new().pubkey(); let key2 = Keypair::new().pubkey();
for set_root_interval in &[1, 4] { for set_root_interval in &[1, 4] {
run_bank_forks_snapshot_n( run_bank_forks_snapshot_n(
snapshot_version,
(MAX_CACHE_ENTRIES * 2 + 1) as u64, (MAX_CACHE_ENTRIES * 2 + 1) as u64,
|bank, mint_keypair| { |bank, mint_keypair| {
let tx = system_transaction::transfer( let tx = system_transaction::transfer(

32
ledger-tool/src/main.rs
View File

@@ -3,7 +3,7 @@ use clap::{
ArgMatches, SubCommand, ArgMatches, SubCommand,
}; };
use serde_json::json; use serde_json::json;
use solana_clap_utils::input_validators::is_slot; use solana_clap_utils::input_validators::{is_parsable, is_slot};
use solana_ledger::{ use solana_ledger::{
bank_forks_utils, bank_forks_utils,
blockstore::Blockstore, blockstore::Blockstore,
@@ -16,6 +16,7 @@ use solana_runtime::{
bank_forks::{BankForks, CompressionType, SnapshotConfig}, bank_forks::{BankForks, CompressionType, SnapshotConfig},
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
snapshot_utils, snapshot_utils,
snapshot_utils::SnapshotVersion,
}; };
use solana_sdk::{ use solana_sdk::{
clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey, clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
@@ -574,6 +575,7 @@ fn load_bank_forks(
snapshot_package_output_path: ledger_path.clone(), snapshot_package_output_path: ledger_path.clone(),
snapshot_path, snapshot_path,
compression: CompressionType::Bzip2, compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
}) })
}; };
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@@ -654,7 +656,13 @@ fn main() {
.takes_value(true) .takes_value(true)
.default_value(&default_genesis_archive_unpacked_size) .default_value(&default_genesis_archive_unpacked_size)
.help("maximum total uncompressed size of unpacked genesis archive"); .help("maximum total uncompressed size of unpacked genesis archive");
let snapshot_version_arg = Arg::with_name("snapshot_version")
.long("snapshot-version")
.value_name("SNAPSHOT_VERSION")
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version");
let matches = App::new(crate_name!()) let matches = App::new(crate_name!())
.about(crate_description!()) .about(crate_description!())
.version(solana_version::version!()) .version(solana_version::version!())
@@ -775,6 +783,7 @@ fn main() {
.arg(&account_paths_arg) .arg(&account_paths_arg)
.arg(&hard_forks_arg) .arg(&hard_forks_arg)
.arg(&max_genesis_archive_unpacked_size_arg) .arg(&max_genesis_archive_unpacked_size_arg)
.arg(&snapshot_version_arg)
.arg( .arg(
Arg::with_name("snapshot_slot") Arg::with_name("snapshot_slot")
.index(1) .index(1)
@@ -1049,7 +1058,15 @@ fn main() {
let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot); let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot);
let output_directory = value_t_or_exit!(arg_matches, "output_directory", String); let output_directory = value_t_or_exit!(arg_matches, "output_directory", String);
let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok(); let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
let snapshot_version =
arg_matches
.value_of("snapshot_version")
.map_or(SnapshotVersion::default(), |s| {
s.parse::<SnapshotVersion>().unwrap_or_else(|e| {
eprintln!("Error: {}", e);
exit(1)
})
});
let process_options = ProcessOptions { let process_options = ProcessOptions {
dev_halt_at_slot: Some(snapshot_slot), dev_halt_at_slot: Some(snapshot_slot),
new_hard_forks: hardforks_of(arg_matches, "hard_forks"), new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
@@ -1083,7 +1100,11 @@ fn main() {
bank bank
}; };
println!("Creating a snapshot of slot {}", bank.slot()); println!(
"Creating a version {} snapshot of slot {}",
snapshot_version,
bank.slot(),
);
assert!(bank.is_complete()); assert!(bank.is_complete());
bank.squash(); bank.squash();
bank.clean_accounts(); bank.clean_accounts();
@@ -1095,7 +1116,7 @@ fn main() {
}); });
let storages: Vec<_> = bank.get_snapshot_storages(); let storages: Vec<_> = bank.get_snapshot_storages();
snapshot_utils::add_snapshot(&temp_dir, &bank, &storages) snapshot_utils::add_snapshot(&temp_dir, &bank, &storages, snapshot_version)
.and_then(|slot_snapshot_paths| { .and_then(|slot_snapshot_paths| {
snapshot_utils::package_snapshot( snapshot_utils::package_snapshot(
&bank, &bank,
@@ -1105,6 +1126,7 @@ fn main() {
output_directory, output_directory,
storages, storages,
CompressionType::Bzip2, CompressionType::Bzip2,
snapshot_version,
) )
}) })
.and_then(|package| { .and_then(|package| {

1
local-cluster/tests/local_cluster.rs
View File

@@ -1210,6 +1210,7 @@ fn setup_snapshot_validator_config(
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()), snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2, compression: CompressionType::Bzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(),
}; };
// Create the account paths // Create the account paths

13
runtime/src/bank_forks.rs
View File

@@ -16,6 +16,8 @@ use std::{
}; };
use thiserror::Error; use thiserror::Error;
pub use crate::snapshot_utils::SnapshotVersion;
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub enum CompressionType { pub enum CompressionType {
Bzip2, Bzip2,
@@ -36,6 +38,9 @@ pub struct SnapshotConfig {
pub snapshot_path: PathBuf, pub snapshot_path: PathBuf,
pub compression: CompressionType, pub compression: CompressionType,
// Snapshot version to generate
pub snapshot_version: SnapshotVersion,
} }
#[derive(Error, Debug)] #[derive(Error, Debug)]
@@ -302,7 +307,12 @@ impl BankForks {
let storages: Vec<_> = bank.get_snapshot_storages(); let storages: Vec<_> = bank.get_snapshot_storages();
let mut add_snapshot_time = Measure::start("add-snapshot-ms"); let mut add_snapshot_time = Measure::start("add-snapshot-ms");
snapshot_utils::add_snapshot(&config.snapshot_path, &bank, &storages)?; snapshot_utils::add_snapshot(
&config.snapshot_path,
&bank,
&storages,
config.snapshot_version,
)?;
add_snapshot_time.stop(); add_snapshot_time.stop();
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize); inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
@@ -321,6 +331,7 @@ impl BankForks {
&config.snapshot_package_output_path, &config.snapshot_package_output_path,
storages, storages,
config.compression.clone(), config.compression.clone(),
config.snapshot_version,
)?; )?;
accounts_package_sender.send(package)?; accounts_package_sender.send(package)?;

4
runtime/src/snapshot_package.rs
View File

@@ -1,4 +1,5 @@
use crate::bank_forks::CompressionType; use crate::bank_forks::CompressionType;
use crate::snapshot_utils::SnapshotVersion;
use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
use solana_sdk::clock::Slot; use solana_sdk::clock::Slot;
use solana_sdk::hash::Hash; use solana_sdk::hash::Hash;
@@ -22,6 +23,7 @@ pub struct AccountsPackage {
pub tar_output_file: PathBuf, pub tar_output_file: PathBuf,
pub hash: Hash, pub hash: Hash,
pub compression: CompressionType, pub compression: CompressionType,
pub snapshot_version: SnapshotVersion,
} }
impl AccountsPackage { impl AccountsPackage {
@@ -34,6 +36,7 @@ impl AccountsPackage {
tar_output_file: PathBuf, tar_output_file: PathBuf,
hash: Hash, hash: Hash,
compression: CompressionType, compression: CompressionType,
snapshot_version: SnapshotVersion,
) -> Self { ) -> Self {
Self { Self {
root, root,
@@ -44,6 +47,7 @@ impl AccountsPackage {
tar_output_file, tar_output_file,
hash, hash,
compression, compression,
snapshot_version,
} }
} }
} }

39
runtime/src/snapshot_utils.rs
View File

@@ -37,7 +37,7 @@ pub const TAR_VERSION_FILE: &str = "version";
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
const VERSION_STRING_V1_1_0: &str = "1.1.0"; const VERSION_STRING_V1_1_0: &str = "1.1.0";
const VERSION_STRING_V1_2_0: &str = "1.2.0"; const VERSION_STRING_V1_2_0: &str = "1.2.0";
const OUTPUT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
#[derive(Copy, Clone, Eq, PartialEq, Debug)] #[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub enum SnapshotVersion { pub enum SnapshotVersion {
@@ -45,6 +45,18 @@ pub enum SnapshotVersion {
V1_2_0, V1_2_0,
} }
impl Default for SnapshotVersion {
fn default() -> Self {
DEFAULT_SNAPSHOT_VERSION
}
}
impl fmt::Display for SnapshotVersion {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(From::from(*self))
}
}
impl From<SnapshotVersion> for &'static str { impl From<SnapshotVersion> for &'static str {
fn from(snapshot_version: SnapshotVersion) -> &'static str { fn from(snapshot_version: SnapshotVersion) -> &'static str {
match snapshot_version { match snapshot_version {
@@ -58,6 +70,15 @@ impl FromStr for SnapshotVersion {
type Err = &'static str; type Err = &'static str;
fn from_str(version_string: &str) -> std::result::Result<Self, Self::Err> { fn from_str(version_string: &str) -> std::result::Result<Self, Self::Err> {
// Remove leading 'v' or 'V' from slice
let version_string = if version_string
.get(..1)
.map_or(false, |s| s.eq_ignore_ascii_case("v"))
{
&version_string[1..]
} else {
version_string
};
match version_string { match version_string {
VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0), VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0),
VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0), VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0),
@@ -134,6 +155,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_package_output_path: P, snapshot_package_output_path: P,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
compression: CompressionType, compression: CompressionType,
snapshot_version: SnapshotVersion,
) -> Result<AccountsPackage> { ) -> Result<AccountsPackage> {
// Hard link all the snapshots we need for this package // Hard link all the snapshots we need for this package
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?; let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
@@ -164,6 +186,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_package_output_file, snapshot_package_output_file,
bank.get_accounts_hash(), bank.get_accounts_hash(),
compression, compression,
snapshot_version,
); );
Ok(package) Ok(package)
@@ -234,7 +257,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
// Write version file // Write version file
{ {
let mut f = std::fs::File::create(staging_version_file)?; let mut f = std::fs::File::create(staging_version_file)?;
f.write_all(OUTPUT_SNAPSHOT_VERSION.as_str().as_bytes())?; f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
} }
let (compression_option, file_ext) = get_compression_ext(&snapshot_package.compression); let (compression_option, file_ext) = get_compression_ext(&snapshot_package.compression);
@@ -417,6 +440,7 @@ pub fn add_snapshot<P: AsRef<Path>>(
snapshot_path: P, snapshot_path: P,
bank: &Bank, bank: &Bank,
snapshot_storages: &[SnapshotStorage], snapshot_storages: &[SnapshotStorage],
snapshot_version: SnapshotVersion,
) -> Result<SlotSnapshotPaths> { ) -> Result<SlotSnapshotPaths> {
let slot = bank.slot(); let slot = bank.slot();
// snapshot_path/slot // snapshot_path/slot
@@ -432,13 +456,12 @@ pub fn add_snapshot<P: AsRef<Path>>(
let mut bank_serialize = Measure::start("bank-serialize-ms"); let mut bank_serialize = Measure::start("bank-serialize-ms");
let bank_snapshot_serializer = move |stream: &mut BufWriter<File>| -> Result<()> { let bank_snapshot_serializer = move |stream: &mut BufWriter<File>| -> Result<()> {
let serde_style = match snapshot_version {
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
};
serialize_into(stream.by_ref(), bank)?; serialize_into(stream.by_ref(), bank)?;
bankrc_to_stream( bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?;
SerdeStyle::NEWER,
stream.by_ref(),
&bank.rc,
snapshot_storages,
)?;
Ok(()) Ok(())
}; };
let consumed_size = let consumed_size =

25
validator/src/main.rs
View File

@@ -6,7 +6,9 @@ use log::*;
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use solana_clap_utils::{ use solana_clap_utils::{
input_parsers::{keypair_of, keypairs_of, pubkey_of}, input_parsers::{keypair_of, keypairs_of, pubkey_of},
input_validators::{is_keypair_or_ask_keyword, is_pubkey, is_pubkey_or_keypair, is_slot}, input_validators::{
is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot,
},
keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
}; };
use solana_client::rpc_client::RpcClient; use solana_client::rpc_client::RpcClient;
@@ -23,7 +25,7 @@ use solana_core::{
use solana_download_utils::{download_genesis_if_missing, download_snapshot}; use solana_download_utils::{download_genesis_if_missing, download_snapshot};
use solana_perf::recycler::enable_recycler_warming; use solana_perf::recycler::enable_recycler_warming;
use solana_runtime::{ use solana_runtime::{
bank_forks::{CompressionType, SnapshotConfig}, bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion},
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
}; };
use solana_sdk::{ use solana_sdk::{
@@ -694,6 +696,15 @@ pub fn main() {
.default_value("100") .default_value("100")
.help("Number of slots between generating accounts hash."), .help("Number of slots between generating accounts hash."),
) )
.arg(
Arg::with_name("snapshot_version")
.long("snapshot-version")
.value_name("SNAPSHOT_VERSION")
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version"),
)
.arg( .arg(
Arg::with_name("limit_ledger_size") Arg::with_name("limit_ledger_size")
.long("limit-ledger-size") .long("limit-ledger-size")
@@ -969,6 +980,15 @@ pub fn main() {
_ => panic!("Compression type not recognized: {}", compression_str), _ => panic!("Compression type not recognized: {}", compression_str),
} }
} }
let snapshot_version =
matches
.value_of("snapshot_version")
.map_or(SnapshotVersion::default(), |s| {
s.parse::<SnapshotVersion>().unwrap_or_else(|err| {
eprintln!("Error: {}", err);
exit(1)
})
});
validator_config.snapshot_config = Some(SnapshotConfig { validator_config.snapshot_config = Some(SnapshotConfig {
snapshot_interval_slots: if snapshot_interval_slots > 0 { snapshot_interval_slots: if snapshot_interval_slots > 0 {
snapshot_interval_slots snapshot_interval_slots
@@ -978,6 +998,7 @@ pub fn main() {
snapshot_path, snapshot_path,
snapshot_package_output_path: ledger_path.clone(), snapshot_package_output_path: ledger_path.clone(),
compression: snapshot_compression, compression: snapshot_compression,
snapshot_version,
}); });
validator_config.accounts_hash_interval_slots = validator_config.accounts_hash_interval_slots =