gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make background services aware of incremental snapshots (#19401)

Browse Source 
AccountsBackgroundService now knows about incremental snapshots.  It is
now also in charge of deciding if an AccountsPackage is destined to be a
SnapshotPackage or not (or just used by AccountsHashVerifier).

!!! New behavior changes !!!

Taking snapshots (both bank and archive) **MUST** succeed.

This is required because of how the last full snapshot slot is
calculated, which is used by AccountsBackgroundService when calling
`clean_accounts()`.

File system calls are now unwrapped and will result in a crash. As Trent told me:

>Well I think if a snapshot fails due to some IO error, it's very likely that the operator is going to have to intervene before it works.  We should exit error in this case, otherwise the validator might happily spin for several more hours, never successfully writing a complete snapshot, before something else brings it down.  This would leave the validator's last local snapshot many more slots behind than it would be had we exited outright and potentially force the operator to abandon ledger continuity in favor of a quick catchup

Other errors will set the `exit` flag to `true`, and the node will gracefully shutdown.

Fixes #19167 
Fixes #19168
This commit is contained in:
Brooks Prumo
2021-08-31 18:33:27 -05:00
committed by GitHub
parent 718ab7c12e
commit fe9ee9134a
9 changed files with 408 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
core/tests/snapshots.rs
View File

@ -66,7 +66,9 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_archive_info::FullSnapshotArchiveInfo,
snapshot_config::{LastFullSnapshotSlot, SnapshotConfig},
snapshot_package::{AccountsPackage, PendingSnapshotPackage},
snapshot_package::{
AccountsPackage, PendingSnapshotPackage, SnapshotPackage, SnapshotType,
},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
@ -255,18 +257,19 @@ mod tests {
snapshot_request_receiver,
accounts_package_sender,
};
for slot in 0..last_slot {
let mut bank = Bank::new_from_parent(&bank_forks[slot], &Pubkey::default(), slot + 1);
for slot in 1..=last_slot {
let mut bank = Bank::new_from_parent(&bank_forks[slot - 1], &Pubkey::default(), slot);
f(&mut bank, mint_keypair);
let bank = bank_forks.insert(bank);
// Set root to make sure we don't end up with too many account storage entries
// and to allow snapshotting of bank and the purging logic on status_cache to
// kick in
if slot % set_root_interval == 0 || slot == last_slot - 1 {
if slot % set_root_interval == 0 || slot == last_slot {
// set_root should send a snapshot request
bank_forks.set_root(bank.slot(), &request_sender, None);
bank.update_accounts_hash();
snapshot_request_handler.handle_snapshot_requests(false, false, false, 0);
snapshot_request_handler
.handle_snapshot_requests(false, false, false, 0, &mut None);
}
}
@ -277,7 +280,7 @@ mod tests {
let last_bank_snapshot_info =
snapshot_utils::get_highest_bank_snapshot_info(bank_snapshots_dir)
.expect("no bank snapshots found in path");
let accounts_package = AccountsPackage::new_for_full_snapshot(
let accounts_package = AccountsPackage::new(
last_bank,
&last_bank_snapshot_info,
bank_snapshots_dir,
@ -287,13 +290,10 @@ mod tests {
ArchiveFormat::TarBzip2,
snapshot_version,
None,
Some(SnapshotType::FullSnapshot),
)
.unwrap();
let snapshot_package = snapshot_utils::process_accounts_package(
accounts_package,
Some(last_bank.get_thread_pool()),
None,
);
let snapshot_package = SnapshotPackage::from(accounts_package);
snapshot_utils::archive_snapshot_package(
&snapshot_package,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -415,8 +415,9 @@ mod tests {
bank_snapshots_dir,
snapshot_archives_dir,
snapshot_config.snapshot_version,
&snapshot_config.archive_format,
snapshot_config.archive_format,
None,
Some(SnapshotType::FullSnapshot),
)
.unwrap();
@ -504,8 +505,6 @@ mod tests {
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
);
let thread_pool = accounts_db::make_min_priority_thread_pool();
let _package_receiver = std::thread::Builder::new()
.name("package-receiver".to_string())
.spawn(move || {
@ -515,11 +514,7 @@ mod tests {
accounts_package = new_accounts_package;
}
let snapshot_package = solana_runtime::snapshot_utils::process_accounts_package(
accounts_package,
Some(&thread_pool),
None,
);
let snapshot_package = SnapshotPackage::from(accounts_package);
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
}
@ -625,7 +620,7 @@ mod tests {
run_bank_forks_snapshot_n(
snapshot_version,
cluster_type,
(MAX_CACHE_ENTRIES * 2 + 1) as u64,
(MAX_CACHE_ENTRIES * 2) as u64,
|bank, mint_keypair| {
let tx = system_transaction::transfer(
mint_keypair,
@ -712,14 +707,19 @@ mod tests {
// set_root sends a snapshot request
bank_forks.set_root(bank.slot(), &request_sender, None);
bank.update_accounts_hash();
snapshot_request_handler.handle_snapshot_requests(false, false, false, 0);
snapshot_request_handler.handle_snapshot_requests(
false,
false,
false,
0,
&mut last_full_snapshot_slot,
);
}
// Since AccountsBackgroundService isn't running, manually make a full snapshot archive
// at the right interval
if slot % FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS == 0 {
make_full_snapshot_archive(&bank, &snapshot_test_config.snapshot_config).unwrap();
last_full_snapshot_slot = Some(slot);
}
// Similarly, make an incremental snapshot archive at the right interval, but only if
// there's been at least one full snapshot first, and a full snapshot wasn't already
@ -764,7 +764,7 @@ mod tests {
"did not find bank snapshot with this path",
)
})?;
snapshot_utils::package_process_and_archive_full_snapshot(
snapshot_utils::package_and_archive_full_snapshot(
bank,
&bank_snapshot_info,
&snapshot_config.bank_snapshots_dir,
@ -772,7 +772,6 @@ mod tests {
bank.get_snapshot_storages(None),
snapshot_config.archive_format,
snapshot_config.snapshot_version,
None,
snapshot_config.maximum_snapshots_to_retain,
)?;
@ -800,7 +799,7 @@ mod tests {
)
})?;
let storages = bank.get_snapshot_storages(Some(incremental_snapshot_base_slot));
snapshot_utils::package_process_and_archive_incremental_snapshot(
snapshot_utils::package_and_archive_incremental_snapshot(
bank,
incremental_snapshot_base_slot,
&bank_snapshot_info,
@ -809,7 +808,6 @@ mod tests {
storages,
snapshot_config.archive_format,
snapshot_config.snapshot_version,
None,
snapshot_config.maximum_snapshots_to_retain,
)?;
@ -857,9 +855,8 @@ mod tests {
const INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot = BANK_SNAPSHOT_INTERVAL_SLOTS * 3;
const FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS: Slot =
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS * 5;
const LAST_SLOT: Slot = FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS * 3 - 1;
const EXPECTED_SLOT_FOR_LAST_SNAPSHOT_ARCHIVE: Slot =
LAST_SLOT + 1 - FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS;
const LAST_SLOT: Slot = FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS * 3
+ INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS * 2;
info!("Running snapshots with background services test...");
trace!(
@ -949,6 +946,7 @@ mod tests {
false,
false,
true,
None,
);
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
@ -1019,9 +1017,14 @@ mod tests {
)
.unwrap();
assert_eq!(deserialized_bank.slot(), LAST_SLOT,);
assert_eq!(
deserialized_bank.slot(),
EXPECTED_SLOT_FOR_LAST_SNAPSHOT_ARCHIVE
deserialized_bank,
**bank_forks
.read()
.unwrap()
.get(deserialized_bank.slot())
.unwrap()
);
// Stop the background services