gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue #17008 -- make snapshot archives to hold on to configurable. (#17158)

Browse Source 
* purge_old_snapshot_archives is changed to take an extra argument 'maximum_snapshots_to_retain' to control the max number of latest snapshot archives to retain. Note the oldest snapshot is always retained as before and is not subjected to this new options.
* The validator and ledger-tool executables are modified with a CLI argument --maximum-snapshots-to-retain. And the options are propagated down the call chains. Their corresponding shell scripts were changed accordingly.
* SnapshotConfig is modified to have an extra field for the maximum_snapshots_to_retain
* Unit tests are developed to cover purge_old_snapshot_archives
This commit is contained in:
Lijun Wang
2021-05-12 10:32:27 -07:00
committed by GitHub
parent e3d722bb42
commit 9c42a89a43
13 changed files with 149 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/rpc_service.rs
View File

@ -482,7 +482,10 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
};
use solana_runtime::{bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion};
use solana_runtime::{
bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
};
use solana_sdk::{genesis_config::ClusterType, signature::Signer};
use std::io::Write;
use std::net::{IpAddr, Ipv4Addr};
@ -581,6 +584,7 @@ mod tests {
snapshot_path: PathBuf::from("/"),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
}),
bank_forks,
RpcHealth::stub(),

14
core/src/snapshot_packager_service.rs
View File

@ -22,6 +22,7 @@ impl SnapshotPackagerService {
starting_snapshot_hash: Option<(Slot, Hash)>,
exit: &Arc<AtomicBool>,
cluster_info: &Arc<ClusterInfo>,
maximum_snapshots_to_retain: usize,
) -> Self {
let exit = exit.clone();
let cluster_info = cluster_info.clone();
@ -41,9 +42,10 @@ impl SnapshotPackagerService {
let snapshot_package = pending_snapshot_package.lock().unwrap().take();
if let Some(snapshot_package) = snapshot_package {
if let Err(err) =
snapshot_utils::archive_snapshot_package(&snapshot_package)
{
if let Err(err) = snapshot_utils::archive_snapshot_package(
&snapshot_package,
maximum_snapshots_to_retain,
) {
warn!("Failed to create snapshot archive: {}", err);
} else {
hashes.push((snapshot_package.slot, snapshot_package.hash));
@ -173,7 +175,11 @@ mod tests {
);
// Make tarball from packageable snapshot
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
snapshot_utils::archive_snapshot_package(
&snapshot_package,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
)
.unwrap();
// before we compare, stick an empty status_cache in this dir so that the package comparison works
// This is needed since the status_cache is added by the packager and is not collected from

2
core/src/test_validator.rs
View File

@ -12,6 +12,7 @@ use {
bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
genesis_utils::create_genesis_config_with_leader_ex,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
},
solana_sdk::{
account::{Account, AccountSharedData},
@ -492,6 +493,7 @@ impl TestValidator {
snapshot_package_output_path: ledger_path.to_path_buf(),
archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
}),
enforce_ulimit_nofile: false,
warp_slot: config.warp_slot,

2
core/src/validator.rs
View File

@ -626,6 +626,7 @@ impl Validator {
snapshot_hash,
&exit,
&cluster_info,
snapshot_config.maximum_snapshots_to_retain,
);
(
Some(snapshot_packager_service),
@ -1166,6 +1167,7 @@ fn new_banks_from_ledger(
&snapshot_config.snapshot_package_output_path,
snapshot_config.archive_format,
Some(&bank_forks.root_bank().get_thread_pool()),
snapshot_config.maximum_snapshots_to_retain,
)
.unwrap_or_else(|err| {
error!("Unable to create snapshot: {}", err);

10
core/tests/snapshots.rs
View File

@ -52,7 +52,7 @@ mod tests {
bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils,
snapshot_utils::SnapshotVersion,
snapshot_utils::{SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
status_cache::MAX_CACHE_ENTRIES,
};
use solana_sdk::{
@ -120,6 +120,7 @@ mod tests {
snapshot_path: PathBuf::from(snapshot_dir.path()),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version,
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
@ -248,7 +249,11 @@ mod tests {
snapshot_package,
Some(&last_bank.get_thread_pool()),
);
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
snapshot_utils::archive_snapshot_package(
&snapshot_package,
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
)
.unwrap();
// Restore bank from snapshot
let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()];
@ -442,6 +447,7 @@ mod tests {
None,
&exit,
&cluster_info,
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
);
let thread_pool = accounts_db::make_min_priority_thread_pool();