gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add CLI options and runtime support for selection of output snapshot version. (#10536)

Browse Source
This commit is contained in:
Kristofer Peterson
2020-06-19 06:38:37 +01:00
committed by GitHub
parent b172b3369e
commit 6d81eede93
10 changed files with 228 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/accounts_hash_verifier.rs
View File

@@ -176,6 +176,7 @@ mod tests {
use crate::cluster_info::make_accounts_hashes_message;
use crate::contact_info::ContactInfo;
use solana_runtime::bank_forks::CompressionType;
use solana_runtime::snapshot_utils::SnapshotVersion;
use solana_sdk::{
hash::hash,
signature::{Keypair, Signer},
@@ -238,6 +239,7 @@ mod tests {
tar_output_file: PathBuf::from("."),
storages: vec![],
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
};
AccountsHashVerifier::process_accounts_package(

5
core/src/rpc_service.rs
View File

@@ -353,7 +353,9 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
};
use solana_runtime::{bank::Bank, bank_forks::CompressionType};
use solana_runtime::{
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
};
use solana_sdk::signature::Signer;
use std::net::{IpAddr, Ipv4Addr};
@@ -445,6 +447,7 @@ mod tests {
snapshot_package_output_path: PathBuf::from("/"),
snapshot_path: PathBuf::from("/"),
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
}),
bank_forks,
RpcHealth::stub(),

3
core/src/snapshot_packager_service.rs
View File

@@ -83,7 +83,7 @@ mod tests {
bank::BankSlotDelta,
bank_forks::CompressionType,
snapshot_package::AccountsPackage,
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
};
use solana_sdk::hash::Hash;
use std::{
@@ -174,6 +174,7 @@ mod tests {
output_tar_path.clone(),
Hash::default(),
CompressionType::Bzip2,
SnapshotVersion::default(),
);
// Make tarball from packageable snapshot

202
core/tests/bank_forks.rs
View File

@@ -1,5 +1,36 @@
// Long-running bank_forks tests
macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
($x:ident) => {
#[allow(non_snake_case)]
mod $x {
use super::*;
const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
#[test]
fn test_bank_forks_status_cache_snapshot_n() {
run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_bank_forks_snapshot_n() {
run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_concurrent_snapshot_packaging() {
run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
}
#[test]
fn test_slots_to_snapshot() {
run_test_slots_to_snapshot(SNAPSHOT_VERSION)
}
}
};
}
#[cfg(test)]
mod tests {
use bincode::serialize_into;
@@ -13,6 +44,7 @@ mod tests {
bank_forks::{BankForks, CompressionType, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils,
snapshot_utils::SnapshotVersion,
status_cache::MAX_CACHE_ENTRIES,
};
use solana_sdk::{
@@ -26,6 +58,9 @@ mod tests {
use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
use tempfile::TempDir;
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
struct SnapshotTestConfig {
accounts_dir: TempDir,
snapshot_dir: TempDir,
@@ -35,41 +70,47 @@ mod tests {
genesis_config_info: GenesisConfigInfo,
}
fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap();
let genesis_config_info = create_genesis_config(10_000);
let bank0 = Bank::new_with_paths(
&genesis_config_info.genesis_config,
vec![accounts_dir.path().to_path_buf()],
&[],
);
bank0.freeze();
let mut bank_forks = BankForks::new(bank0);
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
impl SnapshotTestConfig {
fn new(
snapshot_version: SnapshotVersion,
snapshot_interval_slots: u64,
) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap();
let genesis_config_info = create_genesis_config(10_000);
let bank0 = Bank::new_with_paths(
&genesis_config_info.genesis_config,
vec![accounts_dir.path().to_path_buf()],
&[],
);
bank0.freeze();
let mut bank_forks = BankForks::new(bank0);
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
accounts_dir,
snapshot_dir,
_snapshot_output_path: snapshot_output_path,
snapshot_config,
bank_forks,
genesis_config_info,
let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2,
snapshot_version,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
accounts_dir,
snapshot_dir,
_snapshot_output_path: snapshot_output_path,
snapshot_config,
bank_forks,
genesis_config_info,
}
}
}
fn restore_from_snapshot(
old_bank_forks: &BankForks,
old_last_slot: Slot,
account_paths: Vec<PathBuf>,
account_paths: &[PathBuf],
) {
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
.snapshot_config
@@ -115,17 +156,20 @@ mod tests {
// also marks each bank as root and generates snapshots
// finally tries to restore from the last bank's snapshot and compares the restored bank to the
// `last_slot` bank
fn run_bank_forks_snapshot_n<F>(last_slot: Slot, f: F, set_root_interval: u64)
where
fn run_bank_forks_snapshot_n<F>(
snapshot_version: SnapshotVersion,
last_slot: Slot,
f: F,
set_root_interval: u64,
) where
F: Fn(&mut Bank, &Keypair),
{
solana_logger::setup();
// Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1);
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let (s, _r) = channel();
@@ -143,36 +187,33 @@ mod tests {
}
// Generate a snapshot package for last bank
let last_bank = bank_forks.get(last_slot).unwrap();
let storages: Vec<_> = last_bank.get_snapshot_storages();
let slot_snapshot_paths =
snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path);
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
.pop()
.expect("no snapshots found in path");
let snapshot_package = snapshot_utils::package_snapshot(
last_bank,
slot_snapshot_paths
.last()
.expect("no snapshots found in path"),
&snapshot_config.snapshot_path,
&last_slot_snapshot_path,
snapshot_path,
&last_bank.src.roots(),
&snapshot_config.snapshot_package_output_path,
storages,
last_bank.get_snapshot_storages(),
CompressionType::Bzip2,
snapshot_version,
)
.unwrap();
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
restore_from_snapshot(
bank_forks,
last_slot,
vec![accounts_dir.path().to_path_buf()],
);
restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
}
#[test]
fn test_bank_forks_snapshot_n() {
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
// create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots
// and restores correctly
run_bank_forks_snapshot_n(
snapshot_version,
4,
|bank, mint_keypair| {
let key1 = Keypair::new().pubkey();
@@ -203,24 +244,25 @@ mod tests {
}
}
#[test]
fn test_concurrent_snapshot_packaging() {
fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
solana_logger::setup();
// Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1);
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshots_dir = &snapshot_test_config.snapshot_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
// Take snapshot of zeroth bank
let bank0 = bank_forks.get(0).unwrap();
let storages: Vec<_> = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap();
let storages = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
// Set up snapshotting channels
let (sender, receiver) = channel();
@@ -270,7 +312,7 @@ mod tests {
if slot == saved_slot as u64 {
let options = CopyOptions::new();
fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap();
let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path)
let snapshot_paths = fs::read_dir(snapshot_path)
.unwrap()
.filter_map(|entry| {
let e = entry.unwrap();
@@ -282,17 +324,13 @@ mod tests {
.unwrap_or(None)
})
.sorted()
.collect();
.last()
.unwrap();
// only save off the snapshot of this slot, we don't need the others.
fs_extra::dir::copy(
&snapshot_paths.last().unwrap(),
&saved_snapshots_dir,
&options,
)
.unwrap();
fs_extra::dir::copy(&snapshot_paths, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
&snapshot_config.snapshot_package_output_path,
snapshot_package_output_path,
&(slot, accounts_hash),
&CompressionType::Bzip2,
));
@@ -302,11 +340,12 @@ mod tests {
// Purge all the outdated snapshots, including the ones needed to generate the package
// currently sitting in the channel
bank_forks.purge_old_snapshots();
let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir);
snapshot_paths.sort();
assert_eq!(
snapshot_paths.iter().map(|path| path.slot).collect_vec(),
(3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec()
snapshot_utils::get_snapshot_paths(&snapshots_dir)
.into_iter()
.map(|path| path.slot)
.eq(3..=MAX_CACHE_ENTRIES as u64 + 2),
true
);
// Create a SnapshotPackagerService to create tarballs from all the pending
@@ -336,13 +375,12 @@ mod tests {
// before we compare, stick an empty status_cache in this dir so that the package comparison works
// This is needed since the status_cache is added by the packager and is not collected from
// the source dir for snapshots
let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
snapshot_utils::serialize_snapshot_data_file(
&saved_snapshots_dir
.path()
.join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
|stream| {
serialize_into(stream, &dummy_slot_deltas)?;
serialize_into(stream, &[] as &[BankSlotDelta])?;
Ok(())
},
)
@@ -358,16 +396,17 @@ mod tests {
);
}
#[test]
fn test_slots_to_snapshot() {
fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
solana_logger::setup();
let num_set_roots = MAX_CACHE_ENTRIES * 2;
for add_root_interval in &[1, 3, 9] {
let (snapshot_sender, _snapshot_receiver) = channel();
// Make sure this test never clears bank.slots_since_snapshot
let mut snapshot_test_config =
setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64);
let mut snapshot_test_config = SnapshotTestConfig::new(
snapshot_version,
(*add_root_interval * num_set_roots * 2) as u64,
);
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
let snapshot_sender = Some(snapshot_sender);
for _ in 0..num_set_roots {
@@ -386,21 +425,23 @@ mod tests {
}
let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
let expected_slots_to_snapshot = (num_old_slots as u64
..=num_set_roots as u64 * *add_root_interval as u64)
.collect_vec();
let expected_slots_to_snapshot =
num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
let rooted_bank = snapshot_test_config
let slots_to_snapshot = snapshot_test_config
.bank_forks
.get(snapshot_test_config.bank_forks.root())
.unwrap();
let slots_to_snapshot = rooted_bank.src.roots();
assert_eq!(slots_to_snapshot, expected_slots_to_snapshot);
.unwrap()
.src
.roots();
assert_eq!(
slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot),
true
);
}
}
#[test]
fn test_bank_forks_status_cache_snapshot_n() {
fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
// create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
// this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
// ahead. Also tests the status_cache purge and status cache snapshotting.
@@ -409,6 +450,7 @@ mod tests {
let key2 = Keypair::new().pubkey();
for set_root_interval in &[1, 4] {
run_bank_forks_snapshot_n(
snapshot_version,
(MAX_CACHE_ENTRIES * 2 + 1) as u64,
|bank, mint_keypair| {
let tx = system_transaction::transfer(