gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Various snapshot-related code clean up (bp #14487) (#14513)

Browse Source 
* Create account paths once

(cherry picked from commit fe0ba4a429)

* Replace incorrect symlink_dir usage with symlink_file

(cherry picked from commit f2a7f561a0)

* Reduce TempDir exposure

(cherry picked from commit 9f70f7dc3e)

* Rename AccountsPackage::root to AccountsPackage::slot

(cherry picked from commit 141e6706e6)

* Rename CompressionType to ArchiveFormat

(cherry picked from commit 7be6770808)

Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
mergify[bot]
2021-01-09 18:29:09 +00:00
committed by GitHub
parent ae73cc8d05
commit 484bd48b35
14 changed files with 129 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
core/src/accounts_hash_verifier.rs
View File

@@ -83,17 +83,17 @@ impl AccountsHashVerifier {
snapshot_interval_slots: u64, snapshot_interval_slots: u64,
) { ) {
if fault_injection_rate_slots != 0 if fault_injection_rate_slots != 0
&& accounts_package.root % fault_injection_rate_slots == 0 && accounts_package.slot % fault_injection_rate_slots == 0
{ {
// For testing, publish an invalid hash to gossip. // For testing, publish an invalid hash to gossip.
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use solana_sdk::hash::extend_and_hash; use solana_sdk::hash::extend_and_hash;
warn!("inserting fault at slot: {}", accounts_package.root); warn!("inserting fault at slot: {}", accounts_package.slot);
let rand = thread_rng().gen_range(0, 10); let rand = thread_rng().gen_range(0, 10);
let hash = extend_and_hash(&accounts_package.hash, &[rand]); let hash = extend_and_hash(&accounts_package.hash, &[rand]);
hashes.push((accounts_package.root, hash)); hashes.push((accounts_package.slot, hash));
} else { } else {
hashes.push((accounts_package.root, accounts_package.hash)); hashes.push((accounts_package.slot, accounts_package.hash));
} }
while hashes.len() > MAX_SNAPSHOT_HASHES { while hashes.len() > MAX_SNAPSHOT_HASHES {
@@ -175,7 +175,7 @@ mod tests {
use super::*; use super::*;
use crate::cluster_info::make_accounts_hashes_message; use crate::cluster_info::make_accounts_hashes_message;
use crate::contact_info::ContactInfo; use crate::contact_info::ContactInfo;
use solana_runtime::bank_forks::CompressionType; use solana_runtime::bank_forks::ArchiveFormat;
use solana_runtime::snapshot_utils::SnapshotVersion; use solana_runtime::snapshot_utils::SnapshotVersion;
use solana_sdk::{ use solana_sdk::{
hash::hash, hash::hash,
@@ -234,12 +234,12 @@ mod tests {
let accounts_package = AccountsPackage { let accounts_package = AccountsPackage {
hash: hash(&[i as u8]), hash: hash(&[i as u8]),
block_height: 100 + i as u64, block_height: 100 + i as u64,
root: 100 + i as u64, slot: 100 + i as u64,
slot_deltas: vec![], slot_deltas: vec![],
snapshot_links, snapshot_links,
tar_output_file: PathBuf::from("."), tar_output_file: PathBuf::from("."),
storages: vec![], storages: vec![],
compression: CompressionType::Bzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
}; };

6
core/src/rpc_service.rs
View File

@@ -439,9 +439,7 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo}, genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path, get_tmp_ledger_path,
}; };
use solana_runtime::{ use solana_runtime::{bank::Bank, bank_forks::ArchiveFormat, snapshot_utils::SnapshotVersion};
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
};
use solana_sdk::{genesis_config::ClusterType, signature::Signer}; use solana_sdk::{genesis_config::ClusterType, signature::Signer};
use std::net::{IpAddr, Ipv4Addr}; use std::net::{IpAddr, Ipv4Addr};
@@ -534,7 +532,7 @@ mod tests {
snapshot_interval_slots: 0, snapshot_interval_slots: 0,
snapshot_package_output_path: PathBuf::from("/"), snapshot_package_output_path: PathBuf::from("/"),
snapshot_path: PathBuf::from("/"), snapshot_path: PathBuf::from("/"),
compression: CompressionType::Bzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
}), }),
bank_forks, bank_forks,

10
core/src/snapshot_packager_service.rs
View File

@@ -51,7 +51,7 @@ impl SnapshotPackagerService {
{ {
warn!("Failed to create snapshot archive: {}", err); warn!("Failed to create snapshot archive: {}", err);
} else { } else {
hashes.push((snapshot_package.root, snapshot_package.hash)); hashes.push((snapshot_package.slot, snapshot_package.hash));
while hashes.len() > MAX_SNAPSHOT_HASHES { while hashes.len() > MAX_SNAPSHOT_HASHES {
hashes.remove(0); hashes.remove(0);
} }
@@ -81,7 +81,7 @@ mod tests {
use solana_runtime::{ use solana_runtime::{
accounts_db::AccountStorageEntry, accounts_db::AccountStorageEntry,
bank::BankSlotDelta, bank::BankSlotDelta,
bank_forks::CompressionType, bank_forks::ArchiveFormat,
snapshot_package::AccountsPackage, snapshot_package::AccountsPackage,
snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
}; };
@@ -163,7 +163,7 @@ mod tests {
let output_tar_path = snapshot_utils::get_snapshot_archive_path( let output_tar_path = snapshot_utils::get_snapshot_archive_path(
&snapshot_package_output_path, &snapshot_package_output_path,
&(42, Hash::default()), &(42, Hash::default()),
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
); );
let snapshot_package = AccountsPackage::new( let snapshot_package = AccountsPackage::new(
5, 5,
@@ -173,7 +173,7 @@ mod tests {
vec![storage_entries], vec![storage_entries],
output_tar_path.clone(), output_tar_path.clone(),
Hash::default(), Hash::default(),
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
SnapshotVersion::default(), SnapshotVersion::default(),
); );
@@ -198,7 +198,7 @@ mod tests {
output_tar_path, output_tar_path,
snapshots_dir, snapshots_dir,
accounts_dir, accounts_dir,
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
); );
} }
} }

4
core/src/test_validator.rs
View File

@@ -8,7 +8,7 @@ use {
solana_client::rpc_client::RpcClient, solana_client::rpc_client::RpcClient,
solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger}, solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger},
solana_runtime::{ solana_runtime::{
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
genesis_utils::create_genesis_config_with_leader_ex, genesis_utils::create_genesis_config_with_leader_ex,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
}, },
@@ -354,7 +354,7 @@ impl TestValidator {
snapshot_interval_slots: 100, snapshot_interval_slots: 100,
snapshot_path: ledger_path.join("snapshot"), snapshot_path: ledger_path.join("snapshot"),
snapshot_package_output_path: ledger_path.to_path_buf(), snapshot_package_output_path: ledger_path.to_path_buf(),
compression: CompressionType::NoCompression, archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
}), }),
enforce_ulimit_nofile: false, enforce_ulimit_nofile: false,

16
core/tests/snapshots.rs
View File

@@ -45,7 +45,7 @@ mod tests {
use solana_runtime::{ use solana_runtime::{
accounts_background_service::{ABSRequestSender, SnapshotRequestHandler}, accounts_background_service::{ABSRequestSender, SnapshotRequestHandler},
bank::{Bank, BankSlotDelta}, bank::{Bank, BankSlotDelta},
bank_forks::{BankForks, CompressionType, SnapshotConfig}, bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo}, genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils, snapshot_utils,
snapshot_utils::SnapshotVersion, snapshot_utils::SnapshotVersion,
@@ -106,7 +106,7 @@ mod tests {
snapshot_interval_slots, snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()), snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version, snapshot_version,
}; };
bank_forks.set_snapshot_config(Some(snapshot_config.clone())); bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
@@ -146,9 +146,9 @@ mod tests {
snapshot_utils::get_snapshot_archive_path( snapshot_utils::get_snapshot_archive_path(
snapshot_package_output_path, snapshot_package_output_path,
&(old_last_bank.slot(), old_last_bank.get_accounts_hash()), &(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
), ),
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
old_genesis_config, old_genesis_config,
None, None,
None, None,
@@ -226,7 +226,7 @@ mod tests {
last_bank.src.slot_deltas(&last_bank.src.roots()), last_bank.src.slot_deltas(&last_bank.src.roots()),
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_package_output_path,
last_bank.get_snapshot_storages(), last_bank.get_snapshot_storages(),
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
snapshot_version, snapshot_version,
) )
.unwrap(); .unwrap();
@@ -347,7 +347,7 @@ mod tests {
&snapshot_path, &snapshot_path,
&snapshot_package_output_path, &snapshot_package_output_path,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
&snapshot_config.compression, &snapshot_config.archive_format,
) )
.unwrap(); .unwrap();
@@ -375,7 +375,7 @@ mod tests {
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
snapshot_package_output_path, snapshot_package_output_path,
&(slot, accounts_hash), &(slot, accounts_hash),
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
)); ));
} }
} }
@@ -432,7 +432,7 @@ mod tests {
saved_accounts_dir saved_accounts_dir
.path() .path()
.join(accounts_dir.path().file_name().unwrap()), .join(accounts_dir.path().file_name().unwrap()),
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
); );
} }

8
download-utils/src/lib.rs
View File

@@ -1,7 +1,7 @@
use console::Emoji; use console::Emoji;
use indicatif::{ProgressBar, ProgressStyle}; use indicatif::{ProgressBar, ProgressStyle};
use log::*; use log::*;
use solana_runtime::{bank_forks::CompressionType, snapshot_utils}; use solana_runtime::{bank_forks::ArchiveFormat, snapshot_utils};
use solana_sdk::clock::Slot; use solana_sdk::clock::Slot;
use solana_sdk::hash::Hash; use solana_sdk::hash::Hash;
use std::fs::{self, File}; use std::fs::{self, File};
@@ -175,9 +175,9 @@ pub fn download_snapshot(
snapshot_utils::purge_old_snapshot_archives(ledger_path); snapshot_utils::purge_old_snapshot_archives(ledger_path);
for compression in &[ for compression in &[
CompressionType::Zstd, ArchiveFormat::TarZstd,
CompressionType::Gzip, ArchiveFormat::TarGzip,
CompressionType::Bzip2, ArchiveFormat::TarBzip2,
] { ] {
let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path( let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
ledger_path, ledger_path,

6
ledger-tool/src/main.rs
View File

@@ -25,7 +25,7 @@ use solana_ledger::{
}; };
use solana_runtime::{ use solana_runtime::{
bank::{Bank, RewardCalculationEvent}, bank::{Bank, RewardCalculationEvent},
bank_forks::{BankForks, CompressionType, SnapshotConfig}, bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
snapshot_utils, snapshot_utils,
snapshot_utils::SnapshotVersion, snapshot_utils::SnapshotVersion,
@@ -667,7 +667,7 @@ fn load_bank_forks(
snapshot_interval_slots: 0, // Value doesn't matter snapshot_interval_slots: 0, // Value doesn't matter
snapshot_package_output_path, snapshot_package_output_path,
snapshot_path, snapshot_path,
compression: CompressionType::Bzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
}) })
}; };
@@ -1947,7 +1947,7 @@ fn main() {
bank.src.slot_deltas(&bank.src.roots()), bank.src.slot_deltas(&bank.src.roots()),
output_directory, output_directory,
storages, storages,
CompressionType::Zstd, ArchiveFormat::TarZstd,
snapshot_version, snapshot_version,
) )
}) })

10
local-cluster/tests/local_cluster.rs
View File

@@ -32,7 +32,7 @@ use solana_local_cluster::{
local_cluster::{ClusterConfig, LocalCluster}, local_cluster::{ClusterConfig, LocalCluster},
}; };
use solana_runtime::{ use solana_runtime::{
bank_forks::{CompressionType, SnapshotConfig}, bank_forks::{ArchiveFormat, SnapshotConfig},
snapshot_utils, snapshot_utils,
}; };
use solana_sdk::{ use solana_sdk::{
@@ -1054,7 +1054,7 @@ fn test_snapshot_download() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path( let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path, &validator_snapshot_test_config.snapshot_output_path,
&archive_snapshot_hash, &archive_snapshot_hash,
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
); );
// Download the snapshot, then boot a validator from it. // Download the snapshot, then boot a validator from it.
@@ -1124,7 +1124,7 @@ fn test_snapshot_restart_tower() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path( let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path, &validator_snapshot_test_config.snapshot_output_path,
&archive_snapshot_hash, &archive_snapshot_hash,
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
); );
fs::hard_link(archive_filename, &validator_archive_path).unwrap(); fs::hard_link(archive_filename, &validator_archive_path).unwrap();
@@ -1189,7 +1189,7 @@ fn test_snapshots_blockstore_floor() {
let validator_archive_path = snapshot_utils::get_snapshot_archive_path( let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
&validator_snapshot_test_config.snapshot_output_path, &validator_snapshot_test_config.snapshot_output_path,
&(archive_slot, archive_hash), &(archive_slot, archive_hash),
&CompressionType::Bzip2, &ArchiveFormat::TarBzip2,
); );
fs::hard_link(archive_filename, &validator_archive_path).unwrap(); fs::hard_link(archive_filename, &validator_archive_path).unwrap();
let slot_floor = archive_slot; let slot_floor = archive_slot;
@@ -2446,7 +2446,7 @@ fn setup_snapshot_validator_config(
snapshot_interval_slots, snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()), snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(), snapshot_version: snapshot_utils::SnapshotVersion::default(),
}; };

2
runtime/src/accounts_background_service.rs
View File

@@ -112,7 +112,7 @@ impl SnapshotRequestHandler {
&self.snapshot_config.snapshot_path, &self.snapshot_config.snapshot_path,
&self.snapshot_config.snapshot_package_output_path, &self.snapshot_config.snapshot_package_output_path,
self.snapshot_config.snapshot_version, self.snapshot_config.snapshot_version,
&self.snapshot_config.compression, &self.snapshot_config.archive_format,
); );
if r.is_err() { if r.is_err() {
warn!( warn!(

12
runtime/src/bank_forks.rs
View File

@@ -18,11 +18,11 @@ use std::{
pub use crate::snapshot_utils::SnapshotVersion; pub use crate::snapshot_utils::SnapshotVersion;
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub enum CompressionType { pub enum ArchiveFormat {
Bzip2, TarBzip2,
Gzip, TarGzip,
Zstd, TarZstd,
NoCompression, Tar,
} }
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
@@ -36,7 +36,7 @@ pub struct SnapshotConfig {
// Where to place the snapshots for recent slots // Where to place the snapshots for recent slots
pub snapshot_path: PathBuf, pub snapshot_path: PathBuf,
pub compression: CompressionType, pub archive_format: ArchiveFormat,
// Snapshot version to generate // Snapshot version to generate
pub snapshot_version: SnapshotVersion, pub snapshot_version: SnapshotVersion,

8
runtime/src/serde_snapshot.rs
View File

@@ -288,6 +288,12 @@ where
map map
}; };
// Ensure all account paths exist
for path in &accounts_db.paths {
std::fs::create_dir_all(path)
.unwrap_or_else(|err| panic!("Failed to create directory {}: {}", path.display(), err));
}
let mut last_log_update = Instant::now(); let mut last_log_update = Instant::now();
let mut remaining_slots_to_process = storage.len(); let mut remaining_slots_to_process = storage.len();
@@ -307,8 +313,6 @@ where
let path_index = thread_rng().gen_range(0, accounts_db.paths.len()); let path_index = thread_rng().gen_range(0, accounts_db.paths.len());
let local_dir = &accounts_db.paths[path_index]; let local_dir = &accounts_db.paths[path_index];
std::fs::create_dir_all(local_dir).expect("Create directory failed");
// Move the corresponding AppendVec from the snapshot into the directory pointed // Move the corresponding AppendVec from the snapshot into the directory pointed
// at by `local_dir` // at by `local_dir`
let append_vec_relative_path = let append_vec_relative_path =

14
runtime/src/snapshot_package.rs
View File

@@ -1,4 +1,4 @@
use crate::bank_forks::CompressionType; use crate::bank_forks::ArchiveFormat;
use crate::snapshot_utils::SnapshotVersion; use crate::snapshot_utils::SnapshotVersion;
use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
use solana_sdk::clock::Slot; use solana_sdk::clock::Slot;
@@ -15,38 +15,38 @@ pub type AccountsPackageSendError = SendError<AccountsPackage>;
#[derive(Debug)] #[derive(Debug)]
pub struct AccountsPackage { pub struct AccountsPackage {
pub root: Slot, pub slot: Slot,
pub block_height: Slot, pub block_height: Slot,
pub slot_deltas: Vec<BankSlotDelta>, pub slot_deltas: Vec<BankSlotDelta>,
pub snapshot_links: TempDir, pub snapshot_links: TempDir,
pub storages: SnapshotStorages, pub storages: SnapshotStorages,
pub tar_output_file: PathBuf, pub tar_output_file: PathBuf,
pub hash: Hash, pub hash: Hash,
pub compression: CompressionType, pub archive_format: ArchiveFormat,
pub snapshot_version: SnapshotVersion, pub snapshot_version: SnapshotVersion,
} }
impl AccountsPackage { impl AccountsPackage {
pub fn new( pub fn new(
root: Slot, slot: Slot,
block_height: u64, block_height: u64,
slot_deltas: Vec<BankSlotDelta>, slot_deltas: Vec<BankSlotDelta>,
snapshot_links: TempDir, snapshot_links: TempDir,
storages: SnapshotStorages, storages: SnapshotStorages,
tar_output_file: PathBuf, tar_output_file: PathBuf,
hash: Hash, hash: Hash,
compression: CompressionType, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
) -> Self { ) -> Self {
Self { Self {
root, slot,
block_height, block_height,
slot_deltas, slot_deltas,
snapshot_links, snapshot_links,
storages, storages,
tar_output_file, tar_output_file,
hash, hash,
compression, archive_format,
snapshot_version, snapshot_version,
} }
} }

116
runtime/src/snapshot_utils.rs
View File

@@ -1,7 +1,7 @@
use crate::{ use crate::{
accounts_index::AccountIndex, accounts_index::AccountIndex,
bank::{Bank, BankSlotDelta, Builtins}, bank::{Bank, BankSlotDelta, Builtins},
bank_forks::CompressionType, bank_forks::ArchiveFormat,
hardened_unpack::{unpack_snapshot, UnpackError}, hardened_unpack::{unpack_snapshot, UnpackError},
serde_snapshot::{ serde_snapshot::{
bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
@@ -27,7 +27,6 @@ use std::{
str::FromStr, str::FromStr,
}; };
use tar::Archive; use tar::Archive;
use tempfile::TempDir;
use thiserror::Error; use thiserror::Error;
pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
@@ -159,7 +158,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
snapshot_package_output_path: P, snapshot_package_output_path: P,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
compression: CompressionType, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
) -> Result<AccountsPackage> { ) -> Result<AccountsPackage> {
// Hard link all the snapshots we need for this package // Hard link all the snapshots we need for this package
@@ -181,7 +180,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
let snapshot_package_output_file = get_snapshot_archive_path( let snapshot_package_output_file = get_snapshot_archive_path(
&snapshot_package_output_path, &snapshot_package_output_path,
&(bank.slot(), bank.get_accounts_hash()), &(bank.slot(), bank.get_accounts_hash()),
&compression, &archive_format,
); );
let package = AccountsPackage::new( let package = AccountsPackage::new(
@@ -192,19 +191,19 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_storages, snapshot_storages,
snapshot_package_output_file, snapshot_package_output_file,
bank.get_accounts_hash(), bank.get_accounts_hash(),
compression, archive_format,
snapshot_version, snapshot_version,
); );
Ok(package) Ok(package)
} }
fn get_compression_ext(compression: &CompressionType) -> &'static str { fn get_archive_ext(archive_format: &ArchiveFormat) -> &'static str {
match compression { match archive_format {
CompressionType::Bzip2 => ".tar.bz2", ArchiveFormat::TarBzip2 => ".tar.bz2",
CompressionType::Gzip => ".tar.gz", ArchiveFormat::TarGzip => ".tar.gz",
CompressionType::Zstd => ".tar.zst", ArchiveFormat::TarZstd => ".tar.zst",
CompressionType::NoCompression => ".tar", ArchiveFormat::Tar => ".tar",
} }
} }
@@ -230,13 +229,16 @@ pub fn remove_tmp_snapshot_archives(snapshot_path: &Path) {
pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()> { pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()> {
info!( info!(
"Generating snapshot archive for slot {}", "Generating snapshot archive for slot {}",
snapshot_package.root snapshot_package.slot
); );
serialize_status_cache( serialize_status_cache(
snapshot_package.root, snapshot_package.slot,
&snapshot_package.slot_deltas, &snapshot_package.slot_deltas,
&snapshot_package.snapshot_links, &snapshot_package
.snapshot_links
.path()
.join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
)?; )?;
let mut timer = Measure::start("snapshot_package-package_snapshots"); let mut timer = Measure::start("snapshot_package-package_snapshots");
@@ -274,10 +276,10 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
)); ));
// `storage_path` - The file path where the AppendVec itself is located // `storage_path` - The file path where the AppendVec itself is located
// `output_path` - The directory where the AppendVec will be placed in the staging directory. // `output_path` - The file path where the AppendVec will be placed in the staging directory.
let storage_path = let storage_path =
fs::canonicalize(storage_path).expect("Could not get absolute path for accounts"); fs::canonicalize(storage_path).expect("Could not get absolute path for accounts");
symlink::symlink_dir(storage_path, &output_path)?; symlink::symlink_file(storage_path, &output_path)?;
if !output_path.is_file() { if !output_path.is_file() {
return Err(SnapshotError::StoragePathSymlinkInvalid); return Err(SnapshotError::StoragePathSymlinkInvalid);
} }
@@ -289,7 +291,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?; f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
} }
let file_ext = get_compression_ext(&snapshot_package.compression); let file_ext = get_archive_ext(&snapshot_package.archive_format);
// Tar the staging directory into the archive at `archive_path` // Tar the staging directory into the archive at `archive_path`
// //
@@ -320,23 +322,23 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
Some(tar_output) => { Some(tar_output) => {
let mut archive_file = fs::File::create(&archive_path)?; let mut archive_file = fs::File::create(&archive_path)?;
match snapshot_package.compression { match snapshot_package.archive_format {
CompressionType::Bzip2 => { ArchiveFormat::TarBzip2 => {
let mut encoder = let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best); bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best);
io::copy(tar_output, &mut encoder)?; io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?; let _ = encoder.finish()?;
} }
CompressionType::Gzip => { ArchiveFormat::TarGzip => {
let mut encoder = let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default()); flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
io::copy(tar_output, &mut encoder)?; io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?; let _ = encoder.finish()?;
} }
CompressionType::NoCompression => { ArchiveFormat::Tar => {
io::copy(tar_output, &mut archive_file)?; io::copy(tar_output, &mut archive_file)?;
} }
CompressionType::Zstd => { ArchiveFormat::TarZstd => {
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?; let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
io::copy(tar_output, &mut encoder)?; io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?; let _ = encoder.finish()?;
@@ -361,13 +363,13 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
info!( info!(
"Successfully created {:?}. slot: {}, elapsed ms: {}, size={}", "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}",
snapshot_package.tar_output_file, snapshot_package.tar_output_file,
snapshot_package.root, snapshot_package.slot,
timer.as_ms(), timer.as_ms(),
metadata.len() metadata.len()
); );
datapoint_info!( datapoint_info!(
"snapshot-package", "snapshot-package",
("slot", snapshot_package.root, i64), ("slot", snapshot_package.slot, i64),
("duration_ms", timer.as_ms(), i64), ("duration_ms", timer.as_ms(), i64),
("size", metadata.len(), i64) ("size", metadata.len(), i64)
); );
@@ -546,14 +548,10 @@ pub fn add_snapshot<P: AsRef<Path>>(
fn serialize_status_cache( fn serialize_status_cache(
slot: Slot, slot: Slot,
slot_deltas: &[BankSlotDelta], slot_deltas: &[BankSlotDelta],
snapshot_links: &TempDir, status_cache_path: &Path,
) -> Result<()> { ) -> Result<()> {
// the status cache is stored as snapshot_path/status_cache
let snapshot_status_cache_file_path =
snapshot_links.path().join(SNAPSHOT_STATUS_CACHE_FILE_NAME);
let mut status_cache_serialize = Measure::start("status_cache_serialize-ms"); let mut status_cache_serialize = Measure::start("status_cache_serialize-ms");
let consumed_size = serialize_snapshot_data_file(&snapshot_status_cache_file_path, |stream| { let consumed_size = serialize_snapshot_data_file(status_cache_path, |stream| {
serialize_into(stream, slot_deltas)?; serialize_into(stream, slot_deltas)?;
Ok(()) Ok(())
})?; })?;
@@ -585,7 +583,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
frozen_account_pubkeys: &[Pubkey], frozen_account_pubkeys: &[Pubkey],
snapshot_path: &PathBuf, snapshot_path: &PathBuf,
snapshot_tar: P, snapshot_tar: P,
compression: CompressionType, archive_format: ArchiveFormat,
genesis_config: &GenesisConfig, genesis_config: &GenesisConfig,
debug_keys: Option<Arc<HashSet<Pubkey>>>, debug_keys: Option<Arc<HashSet<Pubkey>>>,
additional_builtins: Option<&Builtins>, additional_builtins: Option<&Builtins>,
@@ -595,7 +593,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
let unpack_dir = tempfile::Builder::new() let unpack_dir = tempfile::Builder::new()
.prefix(TMP_SNAPSHOT_DIR_PREFIX) .prefix(TMP_SNAPSHOT_DIR_PREFIX)
.tempdir_in(snapshot_path)?; .tempdir_in(snapshot_path)?;
untar_snapshot_in(&snapshot_tar, &unpack_dir, compression)?; untar_snapshot_in(&snapshot_tar, &unpack_dir, archive_format)?;
let mut measure = Measure::start("bank rebuild from snapshot"); let mut measure = Measure::start("bank rebuild from snapshot");
let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR); let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
@@ -629,27 +627,27 @@ pub fn bank_from_archive<P: AsRef<Path>>(
pub fn get_snapshot_archive_path<P: AsRef<Path>>( pub fn get_snapshot_archive_path<P: AsRef<Path>>(
snapshot_output_dir: P, snapshot_output_dir: P,
snapshot_hash: &(Slot, Hash), snapshot_hash: &(Slot, Hash),
compression: &CompressionType, archive_format: &ArchiveFormat,
) -> PathBuf { ) -> PathBuf {
snapshot_output_dir.as_ref().join(format!( snapshot_output_dir.as_ref().join(format!(
"snapshot-{}-{}{}", "snapshot-{}-{}{}",
snapshot_hash.0, snapshot_hash.0,
snapshot_hash.1, snapshot_hash.1,
get_compression_ext(compression), get_archive_ext(archive_format),
)) ))
} }
fn compression_type_from_str(compress: &str) -> Option<CompressionType> { fn archive_format_from_str(archive_format: &str) -> Option<ArchiveFormat> {
match compress { match archive_format {
"tar.bz2" => Some(CompressionType::Bzip2), "tar.bz2" => Some(ArchiveFormat::TarBzip2),
"tar.gz" => Some(CompressionType::Gzip), "tar.gz" => Some(ArchiveFormat::TarGzip),
"tar.zst" => Some(CompressionType::Zstd), "tar.zst" => Some(ArchiveFormat::TarZstd),
"tar" => Some(CompressionType::NoCompression), "tar" => Some(ArchiveFormat::Tar),
_ => None, _ => None,
} }
} }
fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionType)> { fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, ArchiveFormat)> {
let snapshot_filename_regex = let snapshot_filename_regex =
Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$").unwrap(); Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.(tar|tar\.bz2|tar\.zst|tar\.gz)$").unwrap();
@@ -658,12 +656,12 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy
let hash_str = captures.get(2).unwrap().as_str(); let hash_str = captures.get(2).unwrap().as_str();
let ext = captures.get(3).unwrap().as_str(); let ext = captures.get(3).unwrap().as_str();
if let (Ok(slot), Ok(hash), Some(compression)) = ( if let (Ok(slot), Ok(hash), Some(archive_format)) = (
slot_str.parse::<Slot>(), slot_str.parse::<Slot>(),
hash_str.parse::<Hash>(), hash_str.parse::<Hash>(),
compression_type_from_str(ext), archive_format_from_str(ext),
) { ) {
return Some((slot, hash, compression)); return Some((slot, hash, archive_format));
} }
} }
None None
@@ -671,7 +669,7 @@ fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash, CompressionTy
pub fn get_snapshot_archives<P: AsRef<Path>>( pub fn get_snapshot_archives<P: AsRef<Path>>(
snapshot_output_dir: P, snapshot_output_dir: P,
) -> Vec<(PathBuf, (Slot, Hash, CompressionType))> { ) -> Vec<(PathBuf, (Slot, Hash, ArchiveFormat))> {
match fs::read_dir(&snapshot_output_dir) { match fs::read_dir(&snapshot_output_dir) {
Err(err) => { Err(err) => {
info!("Unable to read snapshot directory: {}", err); info!("Unable to read snapshot directory: {}", err);
@@ -702,7 +700,7 @@ pub fn get_snapshot_archives<P: AsRef<Path>>(
pub fn get_highest_snapshot_archive_path<P: AsRef<Path>>( pub fn get_highest_snapshot_archive_path<P: AsRef<Path>>(
snapshot_output_dir: P, snapshot_output_dir: P,
) -> Option<(PathBuf, (Slot, Hash, CompressionType))> { ) -> Option<(PathBuf, (Slot, Hash, ArchiveFormat))> {
let archives = get_snapshot_archives(snapshot_output_dir); let archives = get_snapshot_archives(snapshot_output_dir);
archives.into_iter().next() archives.into_iter().next()
} }
@@ -720,27 +718,27 @@ pub fn purge_old_snapshot_archives<P: AsRef<Path>>(snapshot_output_dir: P) {
pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>( pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_tar: P, snapshot_tar: P,
unpack_dir: Q, unpack_dir: Q,
compression: CompressionType, archive_format: ArchiveFormat,
) -> Result<()> { ) -> Result<()> {
let mut measure = Measure::start("snapshot untar"); let mut measure = Measure::start("snapshot untar");
let tar_name = File::open(&snapshot_tar)?; let tar_name = File::open(&snapshot_tar)?;
match compression { match archive_format {
CompressionType::Bzip2 => { ArchiveFormat::TarBzip2 => {
let tar = BzDecoder::new(BufReader::new(tar_name)); let tar = BzDecoder::new(BufReader::new(tar_name));
let mut archive = Archive::new(tar); let mut archive = Archive::new(tar);
unpack_snapshot(&mut archive, unpack_dir)?; unpack_snapshot(&mut archive, unpack_dir)?;
} }
CompressionType::Gzip => { ArchiveFormat::TarGzip => {
let tar = GzDecoder::new(BufReader::new(tar_name)); let tar = GzDecoder::new(BufReader::new(tar_name));
let mut archive = Archive::new(tar); let mut archive = Archive::new(tar);
unpack_snapshot(&mut archive, unpack_dir)?; unpack_snapshot(&mut archive, unpack_dir)?;
} }
CompressionType::Zstd => { ArchiveFormat::TarZstd => {
let tar = zstd::stream::read::Decoder::new(BufReader::new(tar_name))?; let tar = zstd::stream::read::Decoder::new(BufReader::new(tar_name))?;
let mut archive = Archive::new(tar); let mut archive = Archive::new(tar);
unpack_snapshot(&mut archive, unpack_dir)?; unpack_snapshot(&mut archive, unpack_dir)?;
} }
CompressionType::NoCompression => { ArchiveFormat::Tar => {
let tar = BufReader::new(tar_name); let tar = BufReader::new(tar_name);
let mut archive = Archive::new(tar); let mut archive = Archive::new(tar);
unpack_snapshot(&mut archive, unpack_dir)?; unpack_snapshot(&mut archive, unpack_dir)?;
@@ -839,7 +837,7 @@ pub fn verify_snapshot_archive<P, Q, R>(
snapshot_archive: P, snapshot_archive: P,
snapshots_to_verify: Q, snapshots_to_verify: Q,
storages_to_verify: R, storages_to_verify: R,
compression: CompressionType, archive_format: ArchiveFormat,
) where ) where
P: AsRef<Path>, P: AsRef<Path>,
Q: AsRef<Path>, Q: AsRef<Path>,
@@ -847,7 +845,7 @@ pub fn verify_snapshot_archive<P, Q, R>(
{ {
let temp_dir = tempfile::TempDir::new().unwrap(); let temp_dir = tempfile::TempDir::new().unwrap();
let unpack_dir = temp_dir.path(); let unpack_dir = temp_dir.path();
untar_snapshot_in(snapshot_archive, &unpack_dir, compression).unwrap(); untar_snapshot_in(snapshot_archive, &unpack_dir, archive_format).unwrap();
// Check snapshots are the same // Check snapshots are the same
let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR); let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR);
@@ -878,7 +876,7 @@ pub fn snapshot_bank(
snapshot_path: &Path, snapshot_path: &Path,
snapshot_package_output_path: &Path, snapshot_package_output_path: &Path,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
compression: &CompressionType, archive_format: &ArchiveFormat,
) -> Result<()> { ) -> Result<()> {
let storages: Vec<_> = root_bank.get_snapshot_storages(); let storages: Vec<_> = root_bank.get_snapshot_storages();
let mut add_snapshot_time = Measure::start("add-snapshot-ms"); let mut add_snapshot_time = Measure::start("add-snapshot-ms");
@@ -899,7 +897,7 @@ pub fn snapshot_bank(
status_cache_slot_deltas, status_cache_slot_deltas,
snapshot_package_output_path, snapshot_package_output_path,
storages, storages,
compression.clone(), archive_format.clone(),
snapshot_version, snapshot_version,
)?; )?;
@@ -1024,15 +1022,15 @@ mod tests {
fn test_snapshot_hash_of() { fn test_snapshot_hash_of() {
assert_eq!( assert_eq!(
snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())), snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())),
Some((42, Hash::default(), CompressionType::Bzip2)) Some((42, Hash::default(), ArchiveFormat::TarBzip2))
); );
assert_eq!( assert_eq!(
snapshot_hash_of(&format!("snapshot-43-{}.tar.zst", Hash::default())), snapshot_hash_of(&format!("snapshot-43-{}.tar.zst", Hash::default())),
Some((43, Hash::default(), CompressionType::Zstd)) Some((43, Hash::default(), ArchiveFormat::TarZstd))
); );
assert_eq!( assert_eq!(
snapshot_hash_of(&format!("snapshot-42-{}.tar", Hash::default())), snapshot_hash_of(&format!("snapshot-42-{}.tar", Hash::default())),
Some((42, Hash::default(), CompressionType::NoCompression)) Some((42, Hash::default(), ArchiveFormat::Tar))
); );
assert!(snapshot_hash_of("invalid").is_none()); assert!(snapshot_hash_of("invalid").is_none());

31
validator/src/main.rs
View File

@@ -29,7 +29,7 @@ use solana_ledger::blockstore_db::BlockstoreRecoveryMode;
use solana_perf::recycler::enable_recycler_warming; use solana_perf::recycler::enable_recycler_warming;
use solana_runtime::{ use solana_runtime::{
accounts_index::AccountIndex, accounts_index::AccountIndex,
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
snapshot_utils::get_highest_snapshot_archive_path, snapshot_utils::get_highest_snapshot_archive_path,
}; };
@@ -1359,13 +1359,14 @@ pub fn main() {
other than increasing the account balance"), other than increasing the account balance"),
) )
.arg( .arg(
Arg::with_name("snapshot_compression") Arg::with_name("snapshot_archive_format")
.long("snapshot-compression") .long("snapshot-archive-format")
.possible_values(&["bz2", "gzip", "zstd", "none"]) .alias("snapshot-compression") // Legacy name used by Solana v1.5.x and older
.possible_values(&["bz2", "gzip", "zstd", "tar", "none"])
.default_value("zstd") .default_value("zstd")
.value_name("COMPRESSION_TYPE") .value_name("ARCHIVE_TYPE")
.takes_value(true) .takes_value(true)
.help("Type of snapshot compression to use."), .help("Snapshot archive format to use."),
) )
.arg( .arg(
Arg::with_name("max_genesis_archive_unpacked_size") Arg::with_name("max_genesis_archive_unpacked_size")
@@ -1670,14 +1671,14 @@ pub fn main() {
exit(1); exit(1);
}); });
let snapshot_compression = { let archive_format = {
let compression_str = value_t_or_exit!(matches, "snapshot_compression", String); let archive_format_str = value_t_or_exit!(matches, "snapshot_archive_format", String);
match compression_str.as_str() { match archive_format_str.as_str() {
"bz2" => CompressionType::Bzip2, "bz2" => ArchiveFormat::TarBzip2,
"gzip" => CompressionType::Gzip, "gzip" => ArchiveFormat::TarGzip,
"zstd" => CompressionType::Zstd, "zstd" => ArchiveFormat::TarZstd,
"none" => CompressionType::NoCompression, "tar" | "none" => ArchiveFormat::Tar,
_ => panic!("Compression type not recognized: {}", compression_str), _ => panic!("Archive format not recognized: {}", archive_format_str),
} }
}; };
@@ -1698,7 +1699,7 @@ pub fn main() {
}, },
snapshot_path, snapshot_path,
snapshot_package_output_path: ledger_path.clone(), snapshot_package_output_path: ledger_path.clone(),
compression: snapshot_compression, archive_format,
snapshot_version, snapshot_version,
}); });