From 7aa5f6b8332516347282d0e36ceb8ae020c02ddb Mon Sep 17 00:00:00 2001 From: Brooks Prumo Date: Fri, 10 Sep 2021 15:59:26 -0500 Subject: [PATCH] Add CLI args for incremental snapshots (#19694) Add `--incremental-snapshots` flag to enable incremental snapshots. This will allow setting `--full-snapshot-interval-slots` and `--incremental-snapshot-interval-slots`. Also added `--maximum-incremental-snapshots-to-retain`. Co-authored-by: Michael Vines --- core/src/validator.rs | 64 +++++++++++++--- runtime/src/snapshot_utils.rs | 3 +- validator/src/main.rs | 134 +++++++++++++++++++++++++--------- 3 files changed, 154 insertions(+), 47 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index 2699a1407a..7e0c3172ca 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -640,6 +640,7 @@ impl Validator { if let Some(snapshot_config) = config.snapshot_config.clone() { if !is_snapshot_config_valid( snapshot_config.full_snapshot_archive_interval_slots, + snapshot_config.incremental_snapshot_archive_interval_slots, config.accounts_hash_interval_slots, ) { error!("Snapshot config is invalid"); @@ -1661,12 +1662,26 @@ fn cleanup_accounts_path(account_path: &std::path::Path) { } pub fn is_snapshot_config_valid( - snapshot_interval_slots: u64, - accounts_hash_interval_slots: u64, + full_snapshot_interval_slots: Slot, + incremental_snapshot_interval_slots: Slot, + accounts_hash_interval_slots: Slot, ) -> bool { - snapshot_interval_slots == 0 - || (snapshot_interval_slots >= accounts_hash_interval_slots - && snapshot_interval_slots % accounts_hash_interval_slots == 0) + // if full snapshot interval is MAX, that means snapshots are turned off, so yes, valid + if full_snapshot_interval_slots == Slot::MAX { + return true; + } + + let is_incremental_config_valid = if incremental_snapshot_interval_slots == Slot::MAX { + true + } else { + incremental_snapshot_interval_slots >= accounts_hash_interval_slots + && incremental_snapshot_interval_slots % accounts_hash_interval_slots == 0 + && full_snapshot_interval_slots > incremental_snapshot_interval_slots + }; + + full_snapshot_interval_slots >= accounts_hash_interval_slots + && full_snapshot_interval_slots % accounts_hash_interval_slots == 0 + && is_incremental_config_valid } #[cfg(test)] @@ -1871,11 +1886,40 @@ mod tests { #[test] fn test_interval_check() { - assert!(is_snapshot_config_valid(0, 100)); - assert!(!is_snapshot_config_valid(1, 100)); - assert!(!is_snapshot_config_valid(230, 100)); - assert!(is_snapshot_config_valid(500, 100)); - assert!(is_snapshot_config_valid(5, 5)); + assert!(is_snapshot_config_valid(300, 200, 100)); + + let default_accounts_hash_interval = + snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS; + assert!(is_snapshot_config_valid( + snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + default_accounts_hash_interval, + )); + + assert!(is_snapshot_config_valid( + Slot::MAX, + snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + default_accounts_hash_interval + )); + assert!(is_snapshot_config_valid( + snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + Slot::MAX, + default_accounts_hash_interval + )); + assert!(is_snapshot_config_valid( + snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + Slot::MAX, + default_accounts_hash_interval + )); + + assert!(!is_snapshot_config_valid(0, 100, 100)); + assert!(!is_snapshot_config_valid(100, 0, 100)); + assert!(!is_snapshot_config_valid(42, 100, 100)); + assert!(!is_snapshot_config_valid(100, 42, 100)); + assert!(!is_snapshot_config_valid(100, 100, 100)); + assert!(!is_snapshot_config_valid(100, 200, 100)); + assert!(!is_snapshot_config_valid(444, 200, 100)); + assert!(!is_snapshot_config_valid(400, 222, 100)); } #[test] diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index a81e37e918..7354d3fb5e 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -44,7 +44,8 @@ use { }; pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; - +pub const DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 100_000; +pub const DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 100; const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB const VERSION_STRING_V1_2_0: &str = "1.2.0"; const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; diff --git a/validator/src/main.rs b/validator/src/main.rs index 1f75e9676b..c3849d688a 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -50,7 +50,9 @@ use { snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig, snapshot_utils::{ - self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, + self, ArchiveFormat, SnapshotVersion, DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, + DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, }, }, @@ -1072,7 +1074,14 @@ pub fn main() { .to_string(); let default_rpc_threads = num_cpus::get().to_string(); let default_accountsdb_repl_threads = num_cpus::get().to_string(); - let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string(); + let default_maximum_full_snapshot_archives_to_retain = + &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string(); + let default_maximum_incremental_snapshot_archives_to_retain = + &DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string(); + let default_full_snapshot_archive_interval_slots = + &DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS.to_string(); + let default_incremental_snapshot_archive_interval_slots = + &DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS.to_string(); let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string(); let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string(); let default_accounts_shrink_optimize_total_space = @@ -1415,21 +1424,48 @@ pub fn main() { download from other validators"), ) .arg( - Arg::with_name("snapshot_interval_slots") - .long("snapshot-interval-slots") - .value_name("SNAPSHOT_INTERVAL_SLOTS") + Arg::with_name("incremental_snapshots") + .long("incremental-snapshots") + .takes_value(false) + .long_help("Enable incremental snapshots by setting this flag. \ + When enabled, --snapshot-interval-slots will set the \ + incremental snapshot interval. To set the full snapshot \ + interval, use --full-snapshot-interval-slots.") + ) + .arg( + Arg::with_name("incremental_snapshot_interval_slots") + .long("incremental-snapshot-interval-slots") + .alias("snapshot-interval-slots") + .value_name("NUMBER") .takes_value(true) - .default_value("100") + .default_value(default_incremental_snapshot_archive_interval_slots) .help("Number of slots between generating snapshots, \ 0 to disable snapshots"), ) .arg( - Arg::with_name("maximum_snapshots_to_retain") - .long("maximum-snapshots-to-retain") - .value_name("MAXIMUM_SNAPSHOTS_TO_RETAIN") + Arg::with_name("full_snapshot_interval_slots") + .long("full-snapshot-interval-slots") + .value_name("NUMBER") .takes_value(true) - .default_value(default_max_snapshot_to_retain) - .help("The maximum number of snapshots to hold on to when purging older snapshots.") + .default_value(default_full_snapshot_archive_interval_slots) + .help("Number of slots between generating full snapshots") + ) + .arg( + Arg::with_name("maximum_full_snapshots_to_retain") + .long("maximum-full-snapshots-to-retain") + .alias("maximum-snapshots-to-retain") + .value_name("NUMBER") + .takes_value(true) + .default_value(default_maximum_full_snapshot_archives_to_retain) + .help("The maximum number of full snapshot archives to hold on to when purging older snapshots.") + ) + .arg( + Arg::with_name("maximum_incremental_snapshots_to_retain") + .long("maximum-incremental-snapshots-to-retain") + .value_name("NUMBER") + .takes_value(true) + .default_value(default_maximum_incremental_snapshot_archives_to_retain) + .help("The maximum number of incremental snapshot archives to hold on to when purging older snapshots.") ) .arg( Arg::with_name("minimal_snapshot_download_speed") @@ -1464,12 +1500,20 @@ pub fn main() { .help("Skip the check for PoH speed."), ) .arg( - Arg::with_name("accounts_hash_interval_slots") - .long("accounts-hash-slots") - .value_name("ACCOUNTS_HASH_INTERVAL_SLOTS") + Arg::with_name("accounts-hash-interval-slots") + .long("accounts-hash-interval-slots") + .value_name("NUMBER") .takes_value(true) .default_value("100") - .help("Number of slots between generating accounts hash."), + .help("Number of slots between generating accounts hash.") + .validator(|val| { + if val.eq("0") { + Err(String::from("Accounts hash interval cannot be zero")) + } + else { + Ok(()) + } + }), ) .arg( Arg::with_name("snapshot_version") @@ -2637,10 +2681,11 @@ pub fn main() { .collect() }); - let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", u64); let maximum_local_snapshot_age = value_t_or_exit!(matches, "maximum_local_snapshot_age", u64); let maximum_full_snapshot_archives_to_retain = - value_t_or_exit!(matches, "maximum_snapshots_to_retain", usize); + value_t_or_exit!(matches, "maximum_full_snapshots_to_retain", usize); + let maximum_incremental_snapshot_archives_to_retain = + value_t_or_exit!(matches, "maximum_incremental_snapshots_to_retain", usize); let minimal_snapshot_download_speed = value_t_or_exit!(matches, "minimal_snapshot_download_speed", f32); let maximum_snapshot_download_abort = @@ -2680,36 +2725,53 @@ pub fn main() { exit(1) }) }); - validator_config.snapshot_config = Some(SnapshotConfig { - full_snapshot_archive_interval_slots: if snapshot_interval_slots > 0 { - snapshot_interval_slots + + let incremental_snapshot_interval_slots = + value_t_or_exit!(matches, "incremental_snapshot_interval_slots", u64); + let (full_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots) = + if incremental_snapshot_interval_slots > 0 { + if matches.is_present("incremental_snapshots") { + ( + value_t_or_exit!(matches, "full_snapshot_interval_slots", u64), + incremental_snapshot_interval_slots, + ) + } else { + (incremental_snapshot_interval_slots, Slot::MAX) + } } else { - std::u64::MAX - }, - incremental_snapshot_archive_interval_slots: Slot::MAX, + (Slot::MAX, Slot::MAX) + }; + + validator_config.snapshot_config = Some(SnapshotConfig { + full_snapshot_archive_interval_slots, + incremental_snapshot_archive_interval_slots, bank_snapshots_dir, snapshot_archives_dir: snapshot_archives_dir.clone(), archive_format, snapshot_version, maximum_full_snapshot_archives_to_retain, - maximum_incremental_snapshot_archives_to_retain: - DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, + maximum_incremental_snapshot_archives_to_retain, }); validator_config.accounts_hash_interval_slots = - value_t_or_exit!(matches, "accounts_hash_interval_slots", u64); - if validator_config.accounts_hash_interval_slots == 0 { - eprintln!("Accounts hash interval should not be 0."); - exit(1); - } + value_t_or_exit!(matches, "accounts-hash-interval-slots", u64); if !is_snapshot_config_valid( - snapshot_interval_slots, + full_snapshot_archive_interval_slots, + incremental_snapshot_archive_interval_slots, validator_config.accounts_hash_interval_slots, ) { - eprintln!("Invalid snapshot interval provided ({}), must be a multiple of accounts_hash_interval_slots ({})", - snapshot_interval_slots, - validator_config.accounts_hash_interval_slots, - ); + eprintln!("Invalid snapshot configuration provided: snapshot intervals are incompatible. \ + \n\t- full snapshot interval MUST be a multiple of accounts hash interval (if enabled) \ + \n\t- incremental snapshot interval MUST be a multiple of accounts hash interval (if enabled) \ + \n\t- full snapshot interval MUST be larger than incremental snapshot interval (if enabled) \ + \nSnapshot configuration values: \ + \n\tfull snapshot interval: {} \ + \n\tincremental snapshot interval: {} \ + \n\taccounts hash interval: {}", + if full_snapshot_archive_interval_slots == Slot::MAX { "disabled".to_string() } else { full_snapshot_archive_interval_slots.to_string() }, + if incremental_snapshot_archive_interval_slots == Slot::MAX { "disabled".to_string() } else { incremental_snapshot_archive_interval_slots.to_string() }, + validator_config.accounts_hash_interval_slots); + exit(1); } @@ -2856,7 +2918,7 @@ pub fn main() { solana_metrics::set_panic_hook("validator"); solana_entry::entry::init_poh(); - solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir); + snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir); let identity_keypair = Arc::new(identity_keypair);