From 5e25ee5ebe0d7c5128329761289b5b96e1d9b8a3 Mon Sep 17 00:00:00 2001 From: Brooks Prumo Date: Fri, 3 Sep 2021 15:21:32 -0500 Subject: [PATCH] Add maximum_incremental_snapshot_archives_to_retain to SnapshotConfig (#19612) --- core/src/accounts_hash_verifier.rs | 1 + core/src/test_validator.rs | 3 +++ core/tests/snapshots.rs | 3 +++ ledger-tool/src/main.rs | 3 +++ local-cluster/tests/local_cluster.rs | 2 ++ replica-node/src/replica_node.rs | 2 ++ replica-node/tests/local_replica.rs | 2 ++ rpc/src/rpc_service.rs | 3 +++ runtime/src/snapshot_config.rs | 4 ++++ runtime/src/snapshot_utils.rs | 1 + validator/src/main.rs | 3 +++ 11 files changed, 27 insertions(+) diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index d6d9baf739..0679721a59 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -343,6 +343,7 @@ mod tests { archive_format: ArchiveFormat::Tar, snapshot_version: SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: usize::MAX, + maximum_incremental_snapshot_archives_to_retain: usize::MAX, }; for i in 0..MAX_SNAPSHOT_HASHES + 1 { let accounts_package = AccountsPackage { diff --git a/core/src/test_validator.rs b/core/src/test_validator.rs index c43f7efc10..220195abd0 100644 --- a/core/src/test_validator.rs +++ b/core/src/test_validator.rs @@ -18,6 +18,7 @@ use { snapshot_config::SnapshotConfig, snapshot_utils::{ ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }, solana_sdk::{ @@ -529,6 +530,8 @@ impl TestValidator { snapshot_version: SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }), enforce_ulimit_nofile: false, warp_slot: config.warp_slot, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index f66389f5c9..289b6d1361 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -71,6 +71,7 @@ mod tests { }, snapshot_utils::{ self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, status_cache::MAX_CACHE_ENTRIES, }; @@ -149,6 +150,8 @@ mod tests { snapshot_version, maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }; bank_forks.set_snapshot_config(Some(snapshot_config.clone())); SnapshotTestConfig { diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 0d0f78a404..67e27375e7 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -34,6 +34,7 @@ use solana_runtime::{ snapshot_config::SnapshotConfig, snapshot_utils::{ self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }; use solana_sdk::{ @@ -719,6 +720,8 @@ fn load_bank_forks( archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }) }; let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 13eaa2f694..e5272f97df 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -3536,6 +3536,8 @@ fn setup_snapshot_validator_config( snapshot_version: snapshot_utils::SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }; // Create the account paths diff --git a/replica-node/src/replica_node.rs b/replica-node/src/replica_node.rs index a9705c4b61..871a7f4644 100644 --- a/replica-node/src/replica_node.rs +++ b/replica-node/src/replica_node.rs @@ -270,6 +270,8 @@ impl ReplicaNode { snapshot_version: snapshot_utils::SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }; let bank_info = diff --git a/replica-node/tests/local_replica.rs b/replica-node/tests/local_replica.rs index e06d94c1b4..eaf9727fe3 100644 --- a/replica-node/tests/local_replica.rs +++ b/replica-node/tests/local_replica.rs @@ -129,6 +129,8 @@ fn setup_snapshot_validator_config( snapshot_version: snapshot_utils::SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }; // Create the account paths diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index ded9638846..4abbb28575 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -506,6 +506,7 @@ mod tests { bank::Bank, snapshot_utils::{ ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }, solana_sdk::{ @@ -623,6 +624,8 @@ mod tests { snapshot_version: SnapshotVersion::default(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain: + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }), bank_forks, RpcHealth::stub(), diff --git a/runtime/src/snapshot_config.rs b/runtime/src/snapshot_config.rs index 3e06780ee2..e644751c32 100644 --- a/runtime/src/snapshot_config.rs +++ b/runtime/src/snapshot_config.rs @@ -26,4 +26,8 @@ pub struct SnapshotConfig { /// Maximum number of full snapshot archives to retain pub maximum_full_snapshot_archives_to_retain: usize, + + /// Maximum number of incremental snapshot archives to retain + /// NOTE: Incremental snapshots will only be kept for the latest full snapshot + pub maximum_incremental_snapshot_archives_to_retain: usize, } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index b04c1e6ae5..5e0379c58c 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -52,6 +52,7 @@ const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; pub(crate) const TMP_BANK_SNAPSHOT_PREFIX: &str = "tmp-bank-snapshot-"; pub const TMP_SNAPSHOT_ARCHIVE_PREFIX: &str = "tmp-snapshot-archive-"; pub const DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN: usize = 2; +pub const DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN: usize = 100; pub const FULL_SNAPSHOT_ARCHIVE_FILENAME_REGEX: &str = r"^snapshot-(?P[[:digit:]]+)-(?P[[:alnum:]]+)\.(?Ptar|tar\.bz2|tar\.zst|tar\.gz)$"; pub const INCREMENTAL_SNAPSHOT_ARCHIVE_FILENAME_REGEX: &str = r"^incremental-snapshot-(?P[[:digit:]]+)-(?P[[:digit:]]+)-(?P[[:alnum:]]+)\.(?Ptar|tar\.bz2|tar\.zst|tar\.gz)$"; diff --git a/validator/src/main.rs b/validator/src/main.rs index 498413ef60..f83b11b113 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -53,6 +53,7 @@ use { snapshot_config::SnapshotConfig, snapshot_utils::{ self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }, solana_sdk::{ @@ -2691,6 +2692,8 @@ pub fn main() { archive_format, snapshot_version, maximum_full_snapshot_archives_to_retain, + maximum_incremental_snapshot_archives_to_retain: + DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }); validator_config.accounts_hash_interval_slots =