gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add CLI args for incremental snapshots (#19694)

Browse Source 
Add `--incremental-snapshots` flag to enable incremental snapshots.
This will allow setting `--full-snapshot-interval-slots` and
`--incremental-snapshot-interval-slots`.

Also added `--maximum-incremental-snapshots-to-retain`.

Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
Brooks Prumo
2021-09-10 15:59:26 -05:00
committed by GitHub
parent 2de2c6ddd7
commit 7aa5f6b833
3 changed files with 154 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
core/src/validator.rs
View File

@ -640,6 +640,7 @@ impl Validator {
if let Some(snapshot_config) = config.snapshot_config.clone() { if let Some(snapshot_config) = config.snapshot_config.clone() {
if !is_snapshot_config_valid( if !is_snapshot_config_valid(
snapshot_config.full_snapshot_archive_interval_slots, snapshot_config.full_snapshot_archive_interval_slots,
snapshot_config.incremental_snapshot_archive_interval_slots,
config.accounts_hash_interval_slots, config.accounts_hash_interval_slots,
) { ) {
error!("Snapshot config is invalid"); error!("Snapshot config is invalid");
@ -1661,12 +1662,26 @@ fn cleanup_accounts_path(account_path: &std::path::Path) {
} }
pub fn is_snapshot_config_valid( pub fn is_snapshot_config_valid(
snapshot_interval_slots: u64, full_snapshot_interval_slots: Slot,
accounts_hash_interval_slots: u64, incremental_snapshot_interval_slots: Slot,
accounts_hash_interval_slots: Slot,
) -> bool { ) -> bool {
snapshot_interval_slots == 0 // if full snapshot interval is MAX, that means snapshots are turned off, so yes, valid
|| (snapshot_interval_slots >= accounts_hash_interval_slots if full_snapshot_interval_slots == Slot::MAX {
&& snapshot_interval_slots % accounts_hash_interval_slots == 0) return true;
}
let is_incremental_config_valid = if incremental_snapshot_interval_slots == Slot::MAX {
true
} else {
incremental_snapshot_interval_slots >= accounts_hash_interval_slots
&& incremental_snapshot_interval_slots % accounts_hash_interval_slots == 0
&& full_snapshot_interval_slots > incremental_snapshot_interval_slots
};
full_snapshot_interval_slots >= accounts_hash_interval_slots
&& full_snapshot_interval_slots % accounts_hash_interval_slots == 0
&& is_incremental_config_valid
} }
#[cfg(test)] #[cfg(test)]
@ -1871,11 +1886,40 @@ mod tests {
#[test] #[test]
fn test_interval_check() { fn test_interval_check() {
assert!(is_snapshot_config_valid(0, 100)); assert!(is_snapshot_config_valid(300, 200, 100));
assert!(!is_snapshot_config_valid(1, 100));
assert!(!is_snapshot_config_valid(230, 100)); let default_accounts_hash_interval =
assert!(is_snapshot_config_valid(500, 100)); snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS;
assert!(is_snapshot_config_valid(5, 5)); assert!(is_snapshot_config_valid(
snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
default_accounts_hash_interval,
));
assert!(is_snapshot_config_valid(
Slot::MAX,
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
default_accounts_hash_interval
));
assert!(is_snapshot_config_valid(
snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
Slot::MAX,
default_accounts_hash_interval
));
assert!(is_snapshot_config_valid(
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
Slot::MAX,
default_accounts_hash_interval
));
assert!(!is_snapshot_config_valid(0, 100, 100));
assert!(!is_snapshot_config_valid(100, 0, 100));
assert!(!is_snapshot_config_valid(42, 100, 100));
assert!(!is_snapshot_config_valid(100, 42, 100));
assert!(!is_snapshot_config_valid(100, 100, 100));
assert!(!is_snapshot_config_valid(100, 200, 100));
assert!(!is_snapshot_config_valid(444, 200, 100));
assert!(!is_snapshot_config_valid(400, 222, 100));
} }
#[test] #[test]

3
runtime/src/snapshot_utils.rs
View File

@ -44,7 +44,8 @@ use {
}; };
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
pub const DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 100_000;
pub const DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = 100;
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
const VERSION_STRING_V1_2_0: &str = "1.2.0"; const VERSION_STRING_V1_2_0: &str = "1.2.0";
const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;

134
validator/src/main.rs
View File

@ -50,7 +50,9 @@ use {
snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_utils::{ snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, self, ArchiveFormat, SnapshotVersion, DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
}, },
}, },
@ -1072,7 +1074,14 @@ pub fn main() {
.to_string(); .to_string();
let default_rpc_threads = num_cpus::get().to_string(); let default_rpc_threads = num_cpus::get().to_string();
let default_accountsdb_repl_threads = num_cpus::get().to_string(); let default_accountsdb_repl_threads = num_cpus::get().to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string(); let default_maximum_full_snapshot_archives_to_retain =
&DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
let default_maximum_incremental_snapshot_archives_to_retain =
&DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
let default_full_snapshot_archive_interval_slots =
&DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS.to_string();
let default_incremental_snapshot_archive_interval_slots =
&DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS.to_string();
let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string(); let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string();
let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string(); let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string();
let default_accounts_shrink_optimize_total_space = let default_accounts_shrink_optimize_total_space =
@ -1415,21 +1424,48 @@ pub fn main() {
download from other validators"), download from other validators"),
) )
.arg( .arg(
Arg::with_name("snapshot_interval_slots") Arg::with_name("incremental_snapshots")
.long("snapshot-interval-slots") .long("incremental-snapshots")
.value_name("SNAPSHOT_INTERVAL_SLOTS") .takes_value(false)
.long_help("Enable incremental snapshots by setting this flag. \
When enabled, --snapshot-interval-slots will set the \
incremental snapshot interval. To set the full snapshot \
interval, use --full-snapshot-interval-slots.")
)
.arg(
Arg::with_name("incremental_snapshot_interval_slots")
.long("incremental-snapshot-interval-slots")
.alias("snapshot-interval-slots")
.value_name("NUMBER")
.takes_value(true) .takes_value(true)
.default_value("100") .default_value(default_incremental_snapshot_archive_interval_slots)
.help("Number of slots between generating snapshots, \ .help("Number of slots between generating snapshots, \
0 to disable snapshots"), 0 to disable snapshots"),
) )
.arg( .arg(
Arg::with_name("maximum_snapshots_to_retain") Arg::with_name("full_snapshot_interval_slots")
.long("maximum-snapshots-to-retain") .long("full-snapshot-interval-slots")
.value_name("MAXIMUM_SNAPSHOTS_TO_RETAIN") .value_name("NUMBER")
.takes_value(true) .takes_value(true)
.default_value(default_max_snapshot_to_retain) .default_value(default_full_snapshot_archive_interval_slots)
.help("The maximum number of snapshots to hold on to when purging older snapshots.") .help("Number of slots between generating full snapshots")
)
.arg(
Arg::with_name("maximum_full_snapshots_to_retain")
.long("maximum-full-snapshots-to-retain")
.alias("maximum-snapshots-to-retain")
.value_name("NUMBER")
.takes_value(true)
.default_value(default_maximum_full_snapshot_archives_to_retain)
.help("The maximum number of full snapshot archives to hold on to when purging older snapshots.")
)
.arg(
Arg::with_name("maximum_incremental_snapshots_to_retain")
.long("maximum-incremental-snapshots-to-retain")
.value_name("NUMBER")
.takes_value(true)
.default_value(default_maximum_incremental_snapshot_archives_to_retain)
.help("The maximum number of incremental snapshot archives to hold on to when purging older snapshots.")
) )
.arg( .arg(
Arg::with_name("minimal_snapshot_download_speed") Arg::with_name("minimal_snapshot_download_speed")
@ -1464,12 +1500,20 @@ pub fn main() {
.help("Skip the check for PoH speed."), .help("Skip the check for PoH speed."),
) )
.arg( .arg(
Arg::with_name("accounts_hash_interval_slots") Arg::with_name("accounts-hash-interval-slots")
.long("accounts-hash-slots") .long("accounts-hash-interval-slots")
.value_name("ACCOUNTS_HASH_INTERVAL_SLOTS") .value_name("NUMBER")
.takes_value(true) .takes_value(true)
.default_value("100") .default_value("100")
.help("Number of slots between generating accounts hash."), .help("Number of slots between generating accounts hash.")
.validator(|val| {
if val.eq("0") {
Err(String::from("Accounts hash interval cannot be zero"))
}
else {
Ok(())
}
}),
) )
.arg( .arg(
Arg::with_name("snapshot_version") Arg::with_name("snapshot_version")
@ -2637,10 +2681,11 @@ pub fn main() {
.collect() .collect()
}); });
let snapshot_interval_slots = value_t_or_exit!(matches, "snapshot_interval_slots", u64);
let maximum_local_snapshot_age = value_t_or_exit!(matches, "maximum_local_snapshot_age", u64); let maximum_local_snapshot_age = value_t_or_exit!(matches, "maximum_local_snapshot_age", u64);
let maximum_full_snapshot_archives_to_retain = let maximum_full_snapshot_archives_to_retain =
value_t_or_exit!(matches, "maximum_snapshots_to_retain", usize); value_t_or_exit!(matches, "maximum_full_snapshots_to_retain", usize);
let maximum_incremental_snapshot_archives_to_retain =
value_t_or_exit!(matches, "maximum_incremental_snapshots_to_retain", usize);
let minimal_snapshot_download_speed = let minimal_snapshot_download_speed =
value_t_or_exit!(matches, "minimal_snapshot_download_speed", f32); value_t_or_exit!(matches, "minimal_snapshot_download_speed", f32);
let maximum_snapshot_download_abort = let maximum_snapshot_download_abort =
@ -2680,36 +2725,53 @@ pub fn main() {
exit(1) exit(1)
}) })
}); });
validator_config.snapshot_config = Some(SnapshotConfig {
full_snapshot_archive_interval_slots: if snapshot_interval_slots > 0 { let incremental_snapshot_interval_slots =
snapshot_interval_slots value_t_or_exit!(matches, "incremental_snapshot_interval_slots", u64);
let (full_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots) =
if incremental_snapshot_interval_slots > 0 {
if matches.is_present("incremental_snapshots") {
(
value_t_or_exit!(matches, "full_snapshot_interval_slots", u64),
incremental_snapshot_interval_slots,
)
} else {
(incremental_snapshot_interval_slots, Slot::MAX)
}
} else { } else {
std::u64::MAX (Slot::MAX, Slot::MAX)
}, };
incremental_snapshot_archive_interval_slots: Slot::MAX,
validator_config.snapshot_config = Some(SnapshotConfig {
full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots,
bank_snapshots_dir, bank_snapshots_dir,
snapshot_archives_dir: snapshot_archives_dir.clone(), snapshot_archives_dir: snapshot_archives_dir.clone(),
archive_format, archive_format,
snapshot_version, snapshot_version,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain: maximum_incremental_snapshot_archives_to_retain,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
}); });
validator_config.accounts_hash_interval_slots = validator_config.accounts_hash_interval_slots =
value_t_or_exit!(matches, "accounts_hash_interval_slots", u64); value_t_or_exit!(matches, "accounts-hash-interval-slots", u64);
if validator_config.accounts_hash_interval_slots == 0 {
eprintln!("Accounts hash interval should not be 0.");
exit(1);
}
if !is_snapshot_config_valid( if !is_snapshot_config_valid(
snapshot_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots,
validator_config.accounts_hash_interval_slots, validator_config.accounts_hash_interval_slots,
) { ) {
eprintln!("Invalid snapshot interval provided ({}), must be a multiple of accounts_hash_interval_slots ({})", eprintln!("Invalid snapshot configuration provided: snapshot intervals are incompatible. \
snapshot_interval_slots, \n\t- full snapshot interval MUST be a multiple of accounts hash interval (if enabled) \
validator_config.accounts_hash_interval_slots, \n\t- incremental snapshot interval MUST be a multiple of accounts hash interval (if enabled) \
); \n\t- full snapshot interval MUST be larger than incremental snapshot interval (if enabled) \
\nSnapshot configuration values: \
\n\tfull snapshot interval: {} \
\n\tincremental snapshot interval: {} \
\n\taccounts hash interval: {}",
if full_snapshot_archive_interval_slots == Slot::MAX { "disabled".to_string() } else { full_snapshot_archive_interval_slots.to_string() },
if incremental_snapshot_archive_interval_slots == Slot::MAX { "disabled".to_string() } else { incremental_snapshot_archive_interval_slots.to_string() },
validator_config.accounts_hash_interval_slots);
exit(1); exit(1);
} }
@ -2856,7 +2918,7 @@ pub fn main() {
solana_metrics::set_panic_hook("validator"); solana_metrics::set_panic_hook("validator");
solana_entry::entry::init_poh(); solana_entry::entry::init_poh();
solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir); snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir);
let identity_keypair = Arc::new(identity_keypair); let identity_keypair = Arc::new(identity_keypair);