Add incremental snapshot utils (#18504)

This commit adds high-level functions for creating and loading-from
incremental snapshots, plus all low-level functions required to perform
those tasks.  This commit **does not** add taking incremental snapshots
as part of a running validator, nor starting up a node with an
incremental snapshot; just laying ground work.

Additionally, `snapshot_utils` and `serde_snapshot` have been
refactored to use a common code paths for the different snapshots.

Also of note, some renaming has happened:
  1. Snapshots are now either `full_` or `incremental_` throughout the
     codebase.  If not specified, the code applies to both.
  2. Bank snapshots now are called "bank snapshots"
     (before they were called "slot snapshots", "bank snapshots", or
      just "snapshots").  The one exception is within `Bank`, where they
     are still just "snapshots", because they are already "bank
     snapshots".
  3. Snapshot archives now have `_archive` in the code.  This
     should clear up an ambiguity between bank snapshots and snapshot
     archives.

ledger/src/bank_forks_utils.rs

@@ -11,7 +11,7 @@ use solana_entry::entry::VerifyRecyclers;
 use solana_runtime::{
     bank_forks::BankForks,
     snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, SnapshotArchiveInfo},
+    snapshot_utils::{self, FullSnapshotArchiveInfo},
 };
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
 use std::{fs, path::PathBuf, process, result};
@@ -53,9 +53,11 @@ pub fn load(
         fs::create_dir_all(&snapshot_config.snapshot_path)
             .expect("Couldn't create snapshot directory");
 
-        if let Some(snapshot_archive_info) = snapshot_utils::get_highest_snapshot_archive_info(
-            &snapshot_config.snapshot_package_output_path,
-        ) {
+        if let Some(full_snapshot_archive_info) =
+            snapshot_utils::get_highest_full_snapshot_archive_info(
+                &snapshot_config.snapshot_package_output_path,
+            )
+        {
             return load_from_snapshot(
                 genesis_config,
                 blockstore,
@@ -65,7 +67,7 @@ pub fn load(
                 process_options,
                 transaction_status_sender,
                 cache_block_meta_sender,
-                &snapshot_archive_info,
+                &full_snapshot_archive_info,
             );
         } else {
             info!("No snapshot package available; will load from genesis");
@@ -113,11 +115,11 @@ fn load_from_snapshot(
     process_options: ProcessOptions,
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
-    snapshot_archive_info: &SnapshotArchiveInfo,
+    full_snapshot_archive_info: &FullSnapshotArchiveInfo,
 ) -> LoadResult {
     info!(
         "Loading snapshot package: {:?}",
-        &snapshot_archive_info.path
+        full_snapshot_archive_info.path()
     );
 
     // Fail hard here if snapshot fails to load, don't silently continue
@@ -126,12 +128,13 @@ fn load_from_snapshot(
         process::exit(1);
     }
 
-    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archive(
+    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
         &account_paths,
         &process_options.frozen_accounts,
         &snapshot_config.snapshot_path,
-        &snapshot_archive_info.path,
-        snapshot_archive_info.archive_format,
+        full_snapshot_archive_info.path(),
+        None,
+        *full_snapshot_archive_info.archive_format(),
         genesis_config,
         process_options.debug_keys.clone(),
         Some(&crate::builtins::get(process_options.bpf_jit)),
@@ -152,10 +155,18 @@ fn load_from_snapshot(
         deserialized_bank.get_accounts_hash(),
     );
 
-    if deserialized_bank_slot_and_hash != (snapshot_archive_info.slot, snapshot_archive_info.hash) {
+    if deserialized_bank_slot_and_hash
+        != (
+            *full_snapshot_archive_info.slot(),
+            *full_snapshot_archive_info.hash(),
+        )
+    {
         error!(
             "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
-            (snapshot_archive_info.slot, snapshot_archive_info.hash),
+            (
+                full_snapshot_archive_info.slot(),
+                full_snapshot_archive_info.hash()
+            ),
             deserialized_bank_slot_and_hash
         );
         process::exit(1);

ledger/src/blockstore_processor.rs

@@ -567,7 +567,16 @@ fn do_process_blockstore_from_root(
         ("slot", bank_forks.root(), i64),
         ("forks", initial_forks.len(), i64),
         ("calculate_capitalization_us", time_cap.as_us(), i64),
-        ("untar_us", timings.untar_us, i64),
+        (
+            "full_snapshot_untar_us",
+            timings.full_snapshot_untar_us,
+            i64
+        ),
+        (
+            "incremental_snapshot_untar_us",
+            timings.incremental_snapshot_untar_us,
+            i64
+        ),
         (
             "rebuild_bank_from_snapshots_us",
             timings.rebuild_bank_from_snapshots_us,