gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add incremental snapshot utils (#18504)

Browse Source 
This commit adds high-level functions for creating and loading-from
incremental snapshots, plus all low-level functions required to perform
those tasks.  This commit **does not** add taking incremental snapshots
as part of a running validator, nor starting up a node with an
incremental snapshot; just laying ground work.

Additionally, `snapshot_utils` and `serde_snapshot` have been
refactored to use a common code paths for the different snapshots.

Also of note, some renaming has happened:
  1. Snapshots are now either `full_` or `incremental_` throughout the
     codebase.  If not specified, the code applies to both.
  2. Bank snapshots now are called "bank snapshots"
     (before they were called "slot snapshots", "bank snapshots", or
      just "snapshots").  The one exception is within `Bank`, where they
     are still just "snapshots", because they are already "bank
     snapshots".
  3. Snapshot archives now have `_archive` in the code.  This
     should clear up an ambiguity between bank snapshots and snapshot
     archives.
This commit is contained in:
Brooks Prumo
2021-07-22 14:40:37 -05:00
committed by GitHub
parent 7fc4cfebc8
commit d1debcd971
19 changed files with 1773 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
validator/src/main.rs
View File

@ -47,7 +47,9 @@ use {
},
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
snapshot_config::SnapshotConfig,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
},
solana_sdk::{
clock::{Slot, DEFAULT_S_PER_SLOT},
@ -475,8 +477,10 @@ fn get_rpc_node(
blacklist_timeout = Instant::now();
let mut highest_snapshot_hash: Option<(Slot, Hash)> =
snapshot_utils::get_highest_snapshot_archive_info(snapshot_output_dir).map(
|snapshot_archive_info| (snapshot_archive_info.slot, snapshot_archive_info.hash),
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map(
|snapshot_archive_info| {
(*snapshot_archive_info.slot(), *snapshot_archive_info.hash())
},
);
let eligible_rpc_peers = if snapshot_not_required {
rpc_peers
@ -858,7 +862,7 @@ fn rpc_bootstrap(
let mut use_local_snapshot = false;
if let Some(highest_local_snapshot_slot) =
snapshot_utils::get_highest_snapshot_archive_slot(snapshot_output_dir)
snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_output_dir)
{
if highest_local_snapshot_slot
> snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)
@ -900,7 +904,7 @@ fn rpc_bootstrap(
{
snapshot_config.maximum_snapshots_to_retain
} else {
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN
};
let ret = download_snapshot(
&rpc_contact_info.rpc,
@ -1041,7 +1045,7 @@ pub fn main() {
.send_transaction_leader_forward_count
.to_string();
let default_rpc_threads = num_cpus::get().to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string();
let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string();
let default_accounts_shrink_optimize_total_space =