Refactor genesis download/load/check functions (#17276)

* Refactor genesis ingest functions

* Consolidate genesis.bin/genesis.tar.bz2 references

Cargo.lock

@@ -4549,6 +4549,15 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "solana-genesis-utils"
+version = "1.7.0"
+dependencies = [
+ "solana-download-utils",
+ "solana-runtime",
+ "solana-sdk",
+]
+
 [[package]]
 name = "solana-gossip"
 version = "1.7.0"
@@ -5442,6 +5451,7 @@ dependencies = [
  "solana-core",
  "solana-download-utils",
  "solana-faucet",
+ "solana-genesis-utils",
  "solana-ledger",
  "solana-logger 1.7.0",
  "solana-metrics",

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "perf",
     "validator",
     "genesis",
+    "genesis-utils",
     "gossip",
     "install",
     "keygen",

core/src/rpc_service.rs

@@ -26,7 +26,10 @@ use solana_runtime::{
     commitment::BlockCommitmentCache,
     snapshot_utils,
 };
-use solana_sdk::{hash::Hash, native_token::lamports_to_sol, pubkey::Pubkey};
+use solana_sdk::{
+    genesis_config::DEFAULT_GENESIS_DOWNLOAD_PATH, hash::Hash, native_token::lamports_to_sol,
+    pubkey::Pubkey,
+};
 use std::{
     collections::HashSet,
     net::SocketAddr,
@@ -101,7 +104,7 @@ impl RpcRequestMiddleware {
 
     fn is_file_get_path(&self, path: &str) -> bool {
         match path {
-            "/genesis.tar.bz2" => true,
+            DEFAULT_GENESIS_DOWNLOAD_PATH => true,
             _ => {
                 if self.snapshot_config.is_some() {
                     self.snapshot_archive_path_regex.is_match(path)
@@ -136,7 +139,7 @@ impl RpcRequestMiddleware {
         let stem = path.split_at(1).1; // Drop leading '/' from path
         let filename = {
             match path {
-                "/genesis.tar.bz2" => {
+                DEFAULT_GENESIS_DOWNLOAD_PATH => {
                     inc_new_counter_info!("rpc-get_genesis", 1);
                     self.ledger_path.join(stem)
                 }
@@ -488,7 +491,10 @@ mod tests {
         bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion,
         snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
     };
-    use solana_sdk::{genesis_config::ClusterType, signature::Signer};
+    use solana_sdk::{
+        genesis_config::{ClusterType, DEFAULT_GENESIS_ARCHIVE},
+        signature::Signer,
+    };
     use std::io::Write;
     use std::net::{IpAddr, Ipv4Addr};
 
@@ -592,8 +598,8 @@ mod tests {
             RpcHealth::stub(),
         );
 
-        assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
+        assert!(rrm.is_file_get_path(DEFAULT_GENESIS_DOWNLOAD_PATH));
-        assert!(!rrm.is_file_get_path("genesis.tar.bz2"));
+        assert!(!rrm.is_file_get_path(DEFAULT_GENESIS_ARCHIVE));
 
         assert!(!rrm.is_file_get_path("/snapshot.tar.bz2")); // This is a redirect
 
@@ -629,7 +635,7 @@ mod tests {
         let ledger_path = get_tmp_ledger_path!();
         std::fs::create_dir(&ledger_path).unwrap();
 
-        let genesis_path = ledger_path.join("genesis.tar.bz2");
+        let genesis_path = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
         let rrm = RpcRequestMiddleware::new(
             ledger_path.clone(),
             None,
@@ -638,7 +644,7 @@ mod tests {
         );
 
         // File does not exist => request should fail.
-        let action = rrm.process_file_get("/genesis.tar.bz2");
+        let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
         if let RequestMiddlewareAction::Respond { response, .. } = action {
             let response = runtime.block_on(response);
             let response = response.unwrap();
@@ -653,7 +659,7 @@ mod tests {
         }
 
         // Normal file exist => request should succeed.
-        let action = rrm.process_file_get("/genesis.tar.bz2");
+        let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
         if let RequestMiddlewareAction::Respond { response, .. } = action {
             let response = runtime.block_on(response);
             let response = response.unwrap();
@@ -672,7 +678,7 @@ mod tests {
             symlink::symlink_file("wrong", &genesis_path).unwrap();
 
             // File is a symbolic link => request should fail.
-            let action = rrm.process_file_get("/genesis.tar.bz2");
+            let action = rrm.process_file_get(DEFAULT_GENESIS_DOWNLOAD_PATH);
             if let RequestMiddlewareAction::Respond { response, .. } = action {
                 let response = runtime.block_on(response);
                 let response = response.unwrap();

download-utils/src/lib.rs

@@ -3,8 +3,7 @@ use console::Emoji;
 use indicatif::{ProgressBar, ProgressStyle};
 use log::*;
 use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils};
-use solana_sdk::clock::Slot;
+use solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash};
-use solana_sdk::hash::Hash;
 use std::fs::{self, File};
 use std::io;
 use std::io::Read;
@@ -158,11 +157,11 @@ pub fn download_genesis_if_missing(
 ) -> Result<PathBuf, String> {
     if !genesis_package.exists() {
         let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
-        let tmp_genesis_package = tmp_genesis_path.join("genesis.tar.bz2");
+        let tmp_genesis_package = tmp_genesis_path.join(DEFAULT_GENESIS_ARCHIVE);
 
         let _ignored = fs::remove_dir_all(&tmp_genesis_path);
         download_file(
-            &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
+            &format!("http://{}/{}", rpc_addr, DEFAULT_GENESIS_ARCHIVE),
             &tmp_genesis_package,
             use_progress_bar,
         )?;

genesis-utils/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "solana-genesis-utils"
+version = "1.7.0"
+description = "Solana Genesis Utils"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-download-utils"
+edition = "2018"
+
+[dependencies]
+solana-sdk = { path = "../sdk", version = "=1.7.0" }
+solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
+solana-runtime = { path = "../runtime", version = "=1.7.0" }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_genesis_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

genesis-utils/src/lib.rs

@@ -0,0 +1,75 @@
+use solana_download_utils::download_genesis_if_missing;
+use solana_runtime::hardened_unpack::unpack_genesis_archive;
+use solana_sdk::{
+    genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE},
+    hash::Hash,
+};
+use std::net::SocketAddr;
+
+fn check_genesis_hash(
+    genesis_config: &GenesisConfig,
+    expected_genesis_hash: Option<Hash>,
+) -> Result<(), String> {
+    let genesis_hash = genesis_config.hash();
+
+    if let Some(expected_genesis_hash) = expected_genesis_hash {
+        if expected_genesis_hash != genesis_hash {
+            return Err(format!(
+                "Genesis hash mismatch: expected {} but downloaded genesis hash is {}",
+                expected_genesis_hash, genesis_hash,
+            ));
+        }
+    }
+
+    Ok(())
+}
+
+fn load_local_genesis(
+    ledger_path: &std::path::Path,
+    expected_genesis_hash: Option<Hash>,
+) -> Result<GenesisConfig, String> {
+    let existing_genesis = GenesisConfig::load(&ledger_path)
+        .map_err(|err| format!("Failed to load genesis config: {}", err))?;
+    check_genesis_hash(&existing_genesis, expected_genesis_hash)?;
+
+    Ok(existing_genesis)
+}
+
+pub fn download_then_check_genesis_hash(
+    rpc_addr: &SocketAddr,
+    ledger_path: &std::path::Path,
+    expected_genesis_hash: Option<Hash>,
+    max_genesis_archive_unpacked_size: u64,
+    no_genesis_fetch: bool,
+    use_progress_bar: bool,
+) -> Result<GenesisConfig, String> {
+    if no_genesis_fetch {
+        let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
+        return Ok(genesis_config);
+    }
+
+    let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
+    let genesis_config = if let Ok(tmp_genesis_package) =
+        download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
+    {
+        unpack_genesis_archive(
+            &tmp_genesis_package,
+            &ledger_path,
+            max_genesis_archive_unpacked_size,
+        )
+        .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
+
+        let downloaded_genesis = GenesisConfig::load(&ledger_path)
+            .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
+
+        check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
+        std::fs::rename(tmp_genesis_package, genesis_package)
+            .map_err(|err| format!("Unable to rename: {:?}", err))?;
+
+        downloaded_genesis
+    } else {
+        load_local_genesis(ledger_path, expected_genesis_hash)?
+    };
+
+    Ok(genesis_config)
+}

ledger/src/blockstore.rs

@@ -28,7 +28,7 @@ use solana_rayon_threadlimit::get_thread_count;
 use solana_runtime::hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};
 use solana_sdk::{
     clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, MS_PER_TICK},
-    genesis_config::GenesisConfig,
+    genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE},
     hash::Hash,
     pubkey::Pubkey,
     sanitize::Sanitize,
@@ -3434,13 +3434,13 @@ pub fn create_new_ledger(
     // Explicitly close the blockstore before we create the archived genesis file
     drop(blockstore);
 
-    let archive_path = ledger_path.join("genesis.tar.bz2");
+    let archive_path = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
     let args = vec![
         "jcfhS",
         archive_path.to_str().unwrap(),
         "-C",
         ledger_path.to_str().unwrap(),
-        "genesis.bin",
+        DEFAULT_GENESIS_FILE,
         "rocksdb",
     ];
     let output = std::process::Command::new("tar")
@@ -3478,18 +3478,24 @@ pub fn create_new_ledger(
             let mut error_messages = String::new();
 
             fs::rename(
-                &ledger_path.join("genesis.tar.bz2"),
+                &ledger_path.join(DEFAULT_GENESIS_ARCHIVE),
-                ledger_path.join("genesis.tar.bz2.failed"),
+                ledger_path.join(format!("{}.failed", DEFAULT_GENESIS_ARCHIVE)),
             )
             .unwrap_or_else(|e| {
-                error_messages += &format!("/failed to stash problematic genesis.tar.bz2: {}", e)
+                error_messages += &format!(
+                    "/failed to stash problematic {}: {}",
+                    DEFAULT_GENESIS_ARCHIVE, e
+                )
             });
             fs::rename(
-                &ledger_path.join("genesis.bin"),
+                &ledger_path.join(DEFAULT_GENESIS_FILE),
-                ledger_path.join("genesis.bin.failed"),
+                ledger_path.join(format!("{}.failed", DEFAULT_GENESIS_FILE)),
             )
             .unwrap_or_else(|e| {
-                error_messages += &format!("/failed to stash problematic genesis.bin: {}", e)
+                error_messages += &format!(
+                    "/failed to stash problematic {}: {}",
+                    DEFAULT_GENESIS_FILE, e
+                )
             });
             fs::rename(
                 &ledger_path.join("rocksdb"),

runtime/src/hardened_unpack.rs

@@ -1,3 +1,4 @@
+use solana_sdk::genesis_config::{DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE};
 use {
     bzip2::bufread::BzDecoder,
     log::*,
@@ -280,7 +281,7 @@ pub fn open_genesis_config(
     max_genesis_archive_unpacked_size: u64,
 ) -> GenesisConfig {
     GenesisConfig::load(&ledger_path).unwrap_or_else(|load_err| {
-        let genesis_package = ledger_path.join("genesis.tar.bz2");
+        let genesis_package = ledger_path.join(DEFAULT_GENESIS_ARCHIVE);
         unpack_genesis_archive(
             &genesis_package,
             ledger_path,
@@ -348,8 +349,8 @@ fn is_valid_genesis_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
     trace!("validating: {:?} {:?}", parts, kind);
     #[allow(clippy::match_like_matches_macro)]
     match (parts, kind) {
-        (["genesis.bin"], GNUSparse) => true,
+        ([DEFAULT_GENESIS_FILE], GNUSparse) => true,
-        (["genesis.bin"], Regular) => true,
+        ([DEFAULT_GENESIS_FILE], Regular) => true,
         (["rocksdb"], Directory) => true,
         (["rocksdb", _], GNUSparse) => true,
         (["rocksdb", _], Regular) => true,

sdk/src/genesis_config.rs

@@ -32,6 +32,10 @@ use std::{
     time::{SystemTime, UNIX_EPOCH},
 };
 
+pub const DEFAULT_GENESIS_FILE: &str = "genesis.bin";
+pub const DEFAULT_GENESIS_ARCHIVE: &str = "genesis.tar.bz2";
+pub const DEFAULT_GENESIS_DOWNLOAD_PATH: &str = "/genesis.tar.bz2";
+
 // deprecated default that is no longer used
 pub const UNUSED_DEFAULT: u64 = 1024;
 
@@ -152,7 +156,7 @@ impl GenesisConfig {
     }
 
     fn genesis_filename(ledger_path: &Path) -> PathBuf {
-        Path::new(ledger_path).join("genesis.bin")
+        Path::new(ledger_path).join(DEFAULT_GENESIS_FILE)
     }
 
     pub fn load(ledger_path: &Path) -> Result<Self, std::io::Error> {

validator/Cargo.toml

@@ -34,6 +34,7 @@ solana-client = { path = "../client", version = "=1.7.0" }
 solana-core = { path = "../core", version = "=1.7.0" }
 solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
 solana-faucet = { path = "../faucet", version = "=1.7.0" }
+solana-genesis-utils = { path = "../genesis-utils", version = "=1.7.0" }
 solana-ledger = { path = "../ledger", version = "=1.7.0" }
 solana-logger = { path = "../logger", version = "=1.7.0" }
 solana-metrics = { path = "../metrics", version = "=1.7.0" }

validator/src/main.rs

@@ -34,7 +34,8 @@ use {
             is_snapshot_config_invalid, Validator, ValidatorConfig, ValidatorStartProgress,
         },
     },
-    solana_download_utils::{download_genesis_if_missing, download_snapshot},
+    solana_download_utils::download_snapshot,
+    solana_genesis_utils::download_then_check_genesis_hash,
     solana_ledger::blockstore_db::BlockstoreRecoveryMode,
     solana_perf::recycler::enable_recycler_warming,
     solana_rpc::rpc_pubsub_service::PubSubConfig,
@@ -43,13 +44,12 @@ use {
             AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
         },
         bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
-        hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
+        hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         snapshot_utils::{get_highest_snapshot_archive_path, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
     },
     solana_sdk::{
         clock::{Slot, DEFAULT_S_PER_SLOT},
         commitment_config::CommitmentConfig,
-        genesis_config::GenesisConfig,
         hash::Hash,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
@@ -648,74 +648,6 @@ fn validators_set(
     }
 }
 
-fn check_genesis_hash(
-    genesis_config: &GenesisConfig,
-    expected_genesis_hash: Option<Hash>,
-) -> Result<(), String> {
-    let genesis_hash = genesis_config.hash();
-
-    if let Some(expected_genesis_hash) = expected_genesis_hash {
-        if expected_genesis_hash != genesis_hash {
-            return Err(format!(
-                "Genesis hash mismatch: expected {} but downloaded genesis hash is {}",
-                expected_genesis_hash, genesis_hash,
-            ));
-        }
-    }
-
-    Ok(())
-}
-
-fn load_local_genesis(
-    ledger_path: &std::path::Path,
-    expected_genesis_hash: Option<Hash>,
-) -> Result<GenesisConfig, String> {
-    let existing_genesis = GenesisConfig::load(&ledger_path)
-        .map_err(|err| format!("Failed to load genesis config: {}", err))?;
-    check_genesis_hash(&existing_genesis, expected_genesis_hash)?;
-
-    Ok(existing_genesis)
-}
-
-fn download_then_check_genesis_hash(
-    rpc_addr: &SocketAddr,
-    ledger_path: &std::path::Path,
-    expected_genesis_hash: Option<Hash>,
-    max_genesis_archive_unpacked_size: u64,
-    no_genesis_fetch: bool,
-    use_progress_bar: bool,
-) -> Result<Hash, String> {
-    if no_genesis_fetch {
-        let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
-        return Ok(genesis_config.hash());
-    }
-
-    let genesis_package = ledger_path.join("genesis.tar.bz2");
-    let genesis_config = if let Ok(tmp_genesis_package) =
-        download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
-    {
-        unpack_genesis_archive(
-            &tmp_genesis_package,
-            &ledger_path,
-            max_genesis_archive_unpacked_size,
-        )
-        .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
-
-        let downloaded_genesis = GenesisConfig::load(&ledger_path)
-            .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
-
-        check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
-        std::fs::rename(tmp_genesis_package, genesis_package)
-            .map_err(|err| format!("Unable to rename: {:?}", err))?;
-
-        downloaded_genesis
-    } else {
-        load_local_genesis(ledger_path, expected_genesis_hash)?
-    };
-
-    Ok(genesis_config.hash())
-}
-
 fn verify_reachable_ports(
     node: &Node,
     cluster_entrypoint: &ContactInfo,
@@ -872,7 +804,7 @@ fn rpc_bootstrap(
             Err(err) => Err(format!("Failed to get RPC node version: {}", err)),
         }
         .and_then(|_| {
-            let genesis_hash = download_then_check_genesis_hash(
+            let genesis_config = download_then_check_genesis_hash(
                 &rpc_contact_info.rpc,
                 &ledger_path,
                 validator_config.expected_genesis_hash,
@@ -881,7 +813,8 @@ fn rpc_bootstrap(
                 use_progress_bar,
             );
 
-            if let Ok(genesis_hash) = genesis_hash {
+            if let Ok(genesis_config) = genesis_config {
+                let genesis_hash = genesis_config.hash();
                 if validator_config.expected_genesis_hash.is_none() {
                     info!("Expected genesis hash set to {}", genesis_hash);
                     validator_config.expected_genesis_hash = Some(genesis_hash);