gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move download code to download-utils crate (#8704)

Browse Source
This commit is contained in:
sakridge
2020-03-07 07:08:01 -08:00
committed by GitHub
parent a7d1346d51
commit 97986a5241
9 changed files with 319 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
validator/Cargo.toml
View File

@ -9,18 +9,16 @@ license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
bzip2 = "0.3.3"
clap = "2.33.0"
chrono = { version = "0.4.10", features = ["serde"] }
console = "0.9.2"
log = "0.4.8"
indicatif = "0.14.0"
rand = "0.6.5"
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls", "json"] }
serde_json = "1.0.46"
solana-clap-utils = { path = "../clap-utils", version = "1.1.0" }
solana-client = { path = "../client", version = "1.1.0" }
solana-core = { path = "../core", version = "1.1.0" }
solana-download-utils = { path = "../download-utils", version = "1.1.0" }
solana-faucet = { path = "../faucet", version = "1.1.0" }
solana-ledger = { path = "../ledger", version = "1.1.0" }
solana-logger = { path = "../logger", version = "1.1.0" }
@ -31,7 +29,6 @@ solana-runtime = { path = "../runtime", version = "1.1.0" }
solana-sdk = { path = "../sdk", version = "1.1.0" }
solana-vote-program = { path = "../programs/vote", version = "1.1.0" }
solana-vote-signer = { path = "../vote-signer", version = "1.1.0" }
tar = "0.4.26"
[target."cfg(unix)".dependencies]
gag = "0.1.10"

206
validator/src/main.rs
View File

@ -1,9 +1,6 @@
use bzip2::bufread::BzDecoder;
use clap::{
crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg, ArgMatches,
};
use console::Emoji;
use indicatif::{ProgressBar, ProgressStyle};
use log::*;
use rand::{thread_rng, Rng};
use solana_clap_utils::{
@ -20,11 +17,11 @@ use solana_core::{
rpc::JsonRpcConfig,
validator::{Validator, ValidatorConfig},
};
use solana_download_utils::{download_genesis, download_snapshot};
use solana_ledger::bank_forks::SnapshotConfig;
use solana_perf::recycler::enable_recycler_warming;
use solana_sdk::{
clock::Slot,
genesis_config::GenesisConfig,
hash::Hash,
pubkey::Pubkey,
signature::{Keypair, Signer},
@ -32,9 +29,8 @@ use solana_sdk::{
use std::{
collections::HashSet,
fs::{self, File},
io::{self, Read},
net::{SocketAddr, TcpListener, UdpSocket},
path::{Path, PathBuf},
path::PathBuf,
process::exit,
str::FromStr,
sync::{
@ -65,122 +61,6 @@ fn hash_validator(hash: String) -> Result<(), String> {
.map_err(|e| format!("{:?}", e))
}
static TRUCK: Emoji = Emoji("🚚 ", "");
static SPARKLE: Emoji = Emoji("✨ ", "");
/// Creates a new process bar for processing that will take an unknown amount of time
fn new_spinner_progress_bar() -> ProgressBar {
let progress_bar = ProgressBar::new(42);
progress_bar
.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
progress_bar.enable_steady_tick(100);
progress_bar
}
fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
if destination_file.is_file() {
return Err(format!("{:?} already exists", destination_file));
}
let download_start = Instant::now();
fs::create_dir_all(destination_file.parent().unwrap()).map_err(|err| err.to_string())?;
let temp_destination_file = destination_file.with_extension(".tmp");
let progress_bar = new_spinner_progress_bar();
progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
let response = reqwest::blocking::Client::new()
.get(url)
.send()
.and_then(|response| response.error_for_status())
.map_err(|err| {
progress_bar.finish_and_clear();
err.to_string()
})?;
let download_size = {
response
.headers()
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|content_length| content_length.to_str().ok())
.and_then(|content_length| content_length.parse().ok())
.unwrap_or(0)
};
progress_bar.set_length(download_size);
progress_bar.set_style(
ProgressStyle::default_bar()
.template(&format!(
"{}{}Downloading {} {}",
"{spinner:.green} ",
TRUCK,
url,
"[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
))
.progress_chars("=> "),
);
struct DownloadProgress<R> {
progress_bar: ProgressBar,
response: R,
}
impl<R: Read> Read for DownloadProgress<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.response.read(buf).map(|n| {
self.progress_bar.inc(n as u64);
n
})
}
}
let mut source = DownloadProgress {
progress_bar,
response,
};
File::create(&temp_destination_file)
.and_then(|mut file| std::io::copy(&mut source, &mut file))
.map_err(|err| format!("Unable to write {:?}: {:?}", temp_destination_file, err))?;
source.progress_bar.finish_and_clear();
info!(
" {}{}",
SPARKLE,
format!(
"Downloaded {} ({} bytes) in {:?}",
url,
download_size,
Instant::now().duration_since(download_start),
)
);
std::fs::rename(temp_destination_file, destination_file)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
Ok(())
}
fn extract_archive(archive_filename: &Path, destination_dir: &Path) -> Result<(), String> {
info!("Extracting {:?}...", archive_filename);
let extract_start = Instant::now();
fs::create_dir_all(destination_dir).map_err(|err| err.to_string())?;
let tar_bz2 = File::open(&archive_filename)
.map_err(|err| format!("Unable to open {:?}: {:?}", archive_filename, err))?;
let tar = BzDecoder::new(std::io::BufReader::new(tar_bz2));
let mut archive = tar::Archive::new(tar);
archive
.unpack(destination_dir)
.map_err(|err| format!("Unable to unpack {:?}: {:?}", archive_filename, err))?;
info!(
"Extracted {:?} in {:?}",
archive_filename,
Instant::now().duration_since(extract_start)
);
Ok(())
}
fn get_shred_rpc_peers(
cluster_info: &Arc<RwLock<ClusterInfo>>,
expected_shred_version: Option<u16>,
@ -458,74 +338,6 @@ fn check_vote_account(
Ok(())
}
fn download_genesis(
rpc_addr: &SocketAddr,
ledger_path: &Path,
validator_config: &mut ValidatorConfig,
) -> Result<(), String> {
let genesis_package = ledger_path.join("genesis.tar.bz2");
let genesis_config = if !genesis_package.exists() {
let tmp_genesis_path = ledger_path.join("tmp-genesis");
let tmp_genesis_package = tmp_genesis_path.join("genesis.tar.bz2");
let _ignored = fs::remove_dir_all(&tmp_genesis_path);
download_file(
&format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
&tmp_genesis_package,
)?;
extract_archive(&tmp_genesis_package, &ledger_path)?;
let tmp_genesis_config = GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
if let Some(expected_genesis_hash) = validator_config.expected_genesis_hash {
if expected_genesis_hash != tmp_genesis_config.hash() {
return Err(format!(
"Genesis hash mismatch: expected {} but downloaded genesis hash is {}",
expected_genesis_hash,
tmp_genesis_config.hash(),
));
}
}
std::fs::rename(tmp_genesis_package, genesis_package)
.map_err(|err| format!("Unable to rename: {:?}", err))?;
tmp_genesis_config
} else {
GenesisConfig::load(&ledger_path)
.map_err(|err| format!("Failed to load genesis config: {}", err))?
};
if validator_config.expected_genesis_hash.is_none() {
info!("Expected genesis hash set to {}", genesis_config.hash());
// If no particular genesis hash is expected use the one that's here
validator_config.expected_genesis_hash = Some(genesis_config.hash());
}
Ok(())
}
fn download_snapshot(
rpc_addr: &SocketAddr,
ledger_path: &Path,
snapshot_hash: (Slot, Hash),
) -> Result<(), String> {
let snapshot_package =
solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path, &snapshot_hash);
if snapshot_package.exists() {
Ok(())
} else {
download_file(
&format!(
"http://{}/{}",
rpc_addr,
snapshot_package.file_name().unwrap().to_str().unwrap()
),
&snapshot_package,
)
}
}
// This function is duplicated in ledger-tool/src/main.rs...
fn hardforks_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Slot>> {
if matches.is_present(name) {
@ -1157,11 +969,19 @@ pub fn main() {
Err(err) => Err(format!("Failed to get RPC node version: {}", err)),
}
.and_then(|_| {
download_genesis(
let genesis_hash = download_genesis(
&rpc_contact_info.rpc,
&ledger_path,
&mut validator_config,
)
validator_config.expected_genesis_hash,
);
if let Ok(genesis_hash) = genesis_hash {
if validator_config.expected_genesis_hash.is_none() {
info!("Expected genesis hash set to {}", genesis_hash);
validator_config.expected_genesis_hash = Some(genesis_hash);
}
}
genesis_hash
})
.and_then(|_| {
if let Some(expected_genesis_hash) = validator_config.expected_genesis_hash {