gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Plumb maximum_incremental_snapshot_archives_to_retain (#19640)

Browse Source
This commit is contained in:
Brooks Prumo
2021-09-06 18:01:56 -05:00
committed by GitHub
parent 37fe5139cc
commit 9d9482b9d8
9 changed files with 69 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
runtime/src/snapshot_utils.rs
View File

@ -45,12 +45,12 @@ use {
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
pub const MAX_BANK_SNAPSHOTS: usize = 8; // Save some snapshots but not too many
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
const VERSION_STRING_V1_2_0: &str = "1.2.0";
const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
pub(crate) const TMP_BANK_SNAPSHOT_PREFIX: &str = "tmp-bank-snapshot-";
pub const TMP_SNAPSHOT_ARCHIVE_PREFIX: &str = "tmp-snapshot-archive-";
pub const MAX_BANK_SNAPSHOTS_TO_RETAIN: usize = 8; // Save some bank snapshots but not too many
pub const DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN: usize = 2;
pub const DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN: usize = 4;
pub const FULL_SNAPSHOT_ARCHIVE_FILENAME_REGEX: &str = r"^snapshot-(?P<slot>[[:digit:]]+)-(?P<hash>[[:alnum:]]+)\.(?P<ext>tar|tar\.bz2|tar\.zst|tar\.gz)$";
@ -242,7 +242,8 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
/// Make a snapshot archive out of the snapshot package
pub fn archive_snapshot_package(
snapshot_package: &SnapshotPackage,
maximum_snapshot_archives_to_retain: usize,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<()> {
info!(
"Generating snapshot archive for slot {}",
@ -374,8 +375,8 @@ pub fn archive_snapshot_package(
purge_old_snapshot_archives(
tar_dir,
maximum_snapshot_archives_to_retain,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain,
);
timer.stop();
@ -1552,12 +1553,12 @@ where
bank_snapshot_infos
.into_iter()
.rev()
.skip(MAX_BANK_SNAPSHOTS)
.skip(MAX_BANK_SNAPSHOTS_TO_RETAIN)
.for_each(|bank_snapshot_info| {
let r = remove_bank_snapshot(bank_snapshot_info.slot, &bank_snapshots_dir);
if r.is_err() {
warn!(
"Couldn't remove snapshot at: {}",
"Couldn't remove bank snapshot at: {}",
bank_snapshot_info.snapshot_path.display()
);
}
@ -1630,7 +1631,8 @@ pub fn bank_to_full_snapshot_archive(
snapshot_version: Option<SnapshotVersion>,
snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat,
maximum_snapshots_to_retain: usize,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<FullSnapshotArchiveInfo> {
let snapshot_version = snapshot_version.unwrap_or_default();
@ -1654,7 +1656,8 @@ pub fn bank_to_full_snapshot_archive(
snapshot_storages,
archive_format,
snapshot_version,
maximum_snapshots_to_retain,
maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain,
)
}
@ -1671,7 +1674,8 @@ pub fn bank_to_incremental_snapshot_archive(
snapshot_version: Option<SnapshotVersion>,
snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat,
maximum_snapshots_to_retain: usize,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<IncrementalSnapshotArchiveInfo> {
let snapshot_version = snapshot_version.unwrap_or_default();
@ -1697,7 +1701,8 @@ pub fn bank_to_incremental_snapshot_archive(
snapshot_storages,
archive_format,
snapshot_version,
maximum_snapshots_to_retain,
maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain,
)
}
@ -1710,7 +1715,8 @@ pub fn package_and_archive_full_snapshot(
snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion,
maximum_snapshots_to_retain: usize,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<FullSnapshotArchiveInfo> {
let accounts_package = AccountsPackage::new(
bank,
@ -1726,7 +1732,11 @@ pub fn package_and_archive_full_snapshot(
)?;
let snapshot_package = SnapshotPackage::from(accounts_package);
archive_snapshot_package(&snapshot_package, maximum_snapshots_to_retain)?;
archive_snapshot_package(
&snapshot_package,
maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain,
)?;
Ok(FullSnapshotArchiveInfo::new(
snapshot_package.snapshot_archive_info,
@ -1744,7 +1754,8 @@ pub fn package_and_archive_incremental_snapshot(
snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion,
maximum_snapshots_to_retain: usize,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<IncrementalSnapshotArchiveInfo> {
let accounts_package = AccountsPackage::new(
bank,
@ -1762,7 +1773,11 @@ pub fn package_and_archive_incremental_snapshot(
)?;
let snapshot_package = SnapshotPackage::from(accounts_package);
archive_snapshot_package(&snapshot_package, maximum_snapshots_to_retain)?;
archive_snapshot_package(
&snapshot_package,
maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain,
)?;
Ok(IncrementalSnapshotArchiveInfo::new(
incremental_snapshot_base_slot,
@ -2544,7 +2559,8 @@ mod tests {
None,
snapshot_archives_dir.path(),
snapshot_archive_format,
1,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2634,7 +2650,8 @@ mod tests {
None,
snapshot_archives_dir.path(),
snapshot_archive_format,
std::usize::MAX,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2710,7 +2727,8 @@ mod tests {
None,
snapshot_archives_dir.path(),
snapshot_archive_format,
std::usize::MAX,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2742,7 +2760,8 @@ mod tests {
None,
snapshot_archives_dir.path(),
snapshot_archive_format,
std::usize::MAX,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2808,7 +2827,8 @@ mod tests {
None,
&snapshot_archives_dir,
snapshot_archive_format,
std::usize::MAX,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2840,7 +2860,8 @@ mod tests {
None,
&snapshot_archives_dir,
snapshot_archive_format,
std::usize::MAX,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2939,6 +2960,7 @@ mod tests {
snapshot_archives_dir.path(),
snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -2980,6 +3002,7 @@ mod tests {
snapshot_archives_dir.path(),
snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
let (deserialized_bank, _) = bank_from_snapshot_archives(
@ -3040,6 +3063,7 @@ mod tests {
snapshot_archives_dir.path(),
snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();