From fcabc6f7995660e6405b61cd455ab316239b940e Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 24 Feb 2020 13:37:14 -0700 Subject: [PATCH] Rename snapshot.tar.bz2 to snapshot--.tar.bz2 --- Cargo.lock | 51 ++++++++++-------- core/Cargo.toml | 1 + core/src/rpc_service.rs | 57 +++++++++++++++++---- core/src/snapshot_packager_service.rs | 6 ++- ledger-tool/src/main.rs | 5 +- ledger/Cargo.toml | 1 + ledger/src/bank_forks.rs | 5 +- ledger/src/bank_forks_utils.rs | 66 +++++++++++++----------- ledger/src/snapshot_utils.rs | 74 +++++++++++++++++++++++++-- validator/src/main.rs | 51 ++++++++---------- 10 files changed, 220 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c78d29740f..199c272667 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,7 +197,7 @@ dependencies = [ "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "which 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -584,7 +584,7 @@ dependencies = [ "encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -900,7 +900,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -998,7 +998,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1010,7 +1010,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1022,7 +1022,7 @@ dependencies = [ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1316,7 +1316,7 @@ dependencies = [ "bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1624,7 +1624,7 @@ dependencies = [ "console 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1840,8 +1840,8 @@ dependencies = [ "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2656,7 +2656,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2976,13 +2976,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.1" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3000,7 +3000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex-syntax" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -3831,6 +3831,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3882,7 +3883,7 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4053,6 +4054,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4716,7 +4718,7 @@ dependencies = [ "lalrpop 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "solana_libra_failure_ext 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)", "solana_libra_types 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -5229,6 +5231,14 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "threshold_crypto" version = "0.3.2" @@ -5286,7 +5296,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -6376,10 +6386,10 @@ dependencies = [ "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" -"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" +"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" "checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" -"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" +"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum rental 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "01916ebd9fc2e81978a5dc9542a2fa47f5bb2ca3402e14c7cc42d6e3c5123e1f" "checksum rental-impl 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "82260d54cf2cbe9608df161f7e7c98e81fae702aa13af9e4d5d39dc2ffb25ab6" @@ -6493,6 +6503,7 @@ dependencies = [ "checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99" "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum threshold_crypto 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95be1032c63011f20b01c5edb64930e2b51512782b43b458b1e3449613d70f87" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tiny-bip39 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92" diff --git a/core/Cargo.toml b/core/Cargo.toml index 38f4d76042..f2e65e459b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -37,6 +37,7 @@ num-traits = "0.2" rand = "0.6.5" rand_chacha = "0.1.1" rayon = "1.2.0" +regex = "1.3.4" serde = "1.0.104" serde_derive = "1.0.103" serde_json = "1.0.46" diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index 5ddecebb34..d0747ecb6c 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -9,6 +9,7 @@ use jsonrpc_http_server::{ hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware, RequestMiddlewareAction, ServerBuilder, }; +use regex::Regex; use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore}; use solana_sdk::hash::Hash; use std::{ @@ -28,13 +29,17 @@ pub struct JsonRpcService { close_handle: Option, } -#[derive(Default)] struct RpcRequestMiddleware { ledger_path: PathBuf, + snapshot_archive_path_regex: Regex, } impl RpcRequestMiddleware { pub fn new(ledger_path: PathBuf) -> Self { - Self { ledger_path } + Self { + ledger_path, + snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$") + .unwrap(), + } } fn not_found() -> hyper::Response { @@ -51,9 +56,19 @@ impl RpcRequestMiddleware { .unwrap() } - fn get(&self, filename: &str) -> RequestMiddlewareAction { - info!("get {}", filename); - let filename = self.ledger_path.join(filename); + fn is_get_path(&self, path: &str) -> bool { + match path { + "/genesis.tar.bz2" => true, + _ => self.snapshot_archive_path_regex.is_match(path), + } + } + + fn get(&self, path: &str) -> RequestMiddlewareAction { + let filename = self.ledger_path.join( + path.split_at(1).1, // Drop leading '/' from path + ); + info!("get {} -> {:?}", path, filename); + RequestMiddlewareAction::Respond { should_validate_hosts: true, response: Box::new( @@ -73,13 +88,14 @@ impl RpcRequestMiddleware { impl RequestMiddleware for RpcRequestMiddleware { fn on_request(&self, request: hyper::Request) -> RequestMiddlewareAction { trace!("request uri: {}", request.uri()); - match request.uri().path() { - "/snapshot.tar.bz2" => self.get("snapshot.tar.bz2"), - "/genesis.tar.bz2" => self.get("genesis.tar.bz2"), - _ => RequestMiddlewareAction::Proceed { + + if self.is_get_path(request.uri().path()) { + self.get(request.uri().path()) + } else { + RequestMiddlewareAction::Proceed { should_continue_on_invalid_cors: false, request, - }, + } } } } @@ -234,4 +250,25 @@ mod tests { rpc_service.exit(); rpc_service.join().unwrap(); } + + #[test] + fn test_is_get_path() { + let rrm = RpcRequestMiddleware::new(PathBuf::from("/")); + + assert!(rrm.is_get_path("/genesis.tar.bz2")); + assert!(!rrm.is_get_path("genesis.tar.bz2")); + + assert!(!rrm.is_get_path("/snapshot.tar.bz2")); + + assert!( + rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2") + ); + assert!(!rrm.is_get_path( + "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2" + )); + + assert!(!rrm.is_get_path("/")); + assert!(!rrm.is_get_path("..")); + assert!(!rrm.is_get_path("🎣")); + } } diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 2b619cefaa..12de848a03 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -154,8 +154,10 @@ mod tests { } // Create a packageable snapshot - let output_tar_path = - snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path); + let output_tar_path = snapshot_utils::get_snapshot_archive_path( + &snapshot_package_output_path, + &(42, Hash::default()), + ); let snapshot_package = SnapshotPackage::new( 5, vec![], diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 00c80aab84..5c5b4ef764 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -915,7 +915,10 @@ fn main() { snapshot_utils::package_snapshot( &bank, &slot_snapshot_paths, - snapshot_utils::get_snapshot_archive_path(output_directory), + snapshot_utils::get_snapshot_archive_path( + output_directory, + &(bank.slot(), bank.hash()), + ), &temp_dir, &bank.src.roots(), storages, diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 829d3d96c7..5b6ffaec0f 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -24,6 +24,7 @@ rand = "0.6.5" rand_chacha = "0.1.1" rayon = "1.2.0" reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] } +regex = "1.3.4" serde = "1.0.104" serde_bytes = "0.11.3" solana-client = { path = "../client", version = "1.1.0" } diff --git a/ledger/src/bank_forks.rs b/ledger/src/bank_forks.rs index b60dd3eae5..ae309e5599 100644 --- a/ledger/src/bank_forks.rs +++ b/ledger/src/bank_forks.rs @@ -192,7 +192,10 @@ impl BankForks { root, &root_bank.src.roots(), snapshot_package_sender.as_ref().unwrap(), - snapshot_utils::get_snapshot_archive_path(&config.snapshot_package_output_path), + snapshot_utils::get_snapshot_archive_path( + &config.snapshot_package_output_path, + &(root_bank.slot(), root_bank.hash()), + ), ); if r.is_err() { warn!("Error generating snapshot for bank: {}, err: {:?}", root, r); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 1f280e1389..c58449eb12 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -52,40 +52,44 @@ pub fn load( fs::create_dir_all(&snapshot_config.snapshot_path) .expect("Couldn't create snapshot directory"); - let tar = snapshot_utils::get_snapshot_archive_path( + match snapshot_utils::get_highest_snapshot_archive_path( &snapshot_config.snapshot_package_output_path, - ); - if tar.exists() { - info!("Loading snapshot package: {:?}", tar); - // Fail hard here if snapshot fails to load, don't silently continue + ) { + Some(tar) => { + if tar.exists() { + info!("Loading snapshot package: {:?}", tar); + // Fail hard here if snapshot fails to load, don't silently continue - if account_paths.is_empty() { - panic!("Account paths not present when booting from snapshot") + if account_paths.is_empty() { + panic!("Account paths not present when booting from snapshot") + } + + let deserialized_bank = snapshot_utils::bank_from_archive( + &account_paths, + &snapshot_config.snapshot_path, + &tar, + ) + .expect("Load from snapshot failed"); + + let snapshot_hash = ( + deserialized_bank.slot(), + deserialized_bank.get_accounts_hash(), + ); + return to_loadresult( + blockstore_processor::process_blockstore_from_root( + genesis_config, + blockstore, + Arc::new(deserialized_bank), + &process_options, + &VerifyRecyclers::default(), + ), + Some(snapshot_hash), + ); + } else { + info!("Snapshot package does not exist: {:?}", tar); + } } - - let deserialized_bank = snapshot_utils::bank_from_archive( - &account_paths, - &snapshot_config.snapshot_path, - &tar, - ) - .expect("Load from snapshot failed"); - - let snapshot_hash = ( - deserialized_bank.slot(), - deserialized_bank.get_accounts_hash(), - ); - return to_loadresult( - blockstore_processor::process_blockstore_from_root( - genesis_config, - blockstore, - Arc::new(deserialized_bank), - &process_options, - &VerifyRecyclers::default(), - ), - Some(snapshot_hash), - ); - } else { - info!("Snapshot package does not exist: {:?}", tar); + None => info!("No snapshot package available"), } } else { info!("Snapshots disabled"); diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index b7f27c9830..004785b3b7 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -3,6 +3,7 @@ use bincode::serialize_into; use bzip2::bufread::BzDecoder; use fs_extra::dir::CopyOptions; use log::*; +use regex::Regex; use solana_measure::measure::Measure; use solana_runtime::{ accounts_db::{SnapshotStorage, SnapshotStorages}, @@ -11,7 +12,7 @@ use solana_runtime::{ MAX_SNAPSHOT_DATA_FILE_SIZE, }, }; -use solana_sdk::clock::Slot; +use solana_sdk::{clock::Slot, hash::Hash}; use std::{ cmp::Ordering, env, @@ -205,6 +206,13 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<() let metadata = fs::metadata(&archive_path)?; fs::rename(&archive_path, &snapshot_package.tar_output_file)?; + // Keep around at most two snapshot archives + let archives = get_snapshot_archives(snapshot_package.tar_output_file.parent().unwrap()); + for old_archive in archives.into_iter().skip(2) { + fs::remove_file(old_archive.0) + .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err)); + } + timer.stop(); info!( "Successfully created tarball. slot: {}, elapsed ms: {}, size={}", @@ -497,8 +505,59 @@ fn is_snapshot_compression_disabled() -> bool { } } -pub fn get_snapshot_archive_path>(snapshot_output_dir: P) -> PathBuf { - snapshot_output_dir.as_ref().join("snapshot.tar.bz2") +pub fn get_snapshot_archive_path>( + snapshot_output_dir: P, + snapshot_hash: &(Slot, Hash), +) -> PathBuf { + snapshot_output_dir.as_ref().join(format!( + "snapshot-{}-{}.tar.bz2", + snapshot_hash.0, snapshot_hash.1 + )) +} + +fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash)> { + let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.tar\.bz2$").unwrap(); + + if let Some(captures) = snapshot_filename_regex.captures(archive_filename) { + let slot_str = captures.get(1).unwrap().as_str(); + let hash_str = captures.get(2).unwrap().as_str(); + + if let (Ok(slot), Ok(hash)) = (slot_str.parse::(), hash_str.parse::()) { + return Some((slot, hash)); + } + } + None +} + +fn get_snapshot_archives>(snapshot_output_dir: P) -> Vec<(PathBuf, (Slot, Hash))> { + let files = fs::read_dir(&snapshot_output_dir) + .unwrap_or_else(|err| panic!("Unable to read snapshot directory: {}", err)); + + let mut archives: Vec<_> = files + .filter_map(|entry| { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_file() { + if let Some(snapshot_hash) = + snapshot_hash_of(path.file_name().unwrap().to_str().unwrap()) + { + return Some((path, snapshot_hash)); + } + } + } + None + }) + .collect(); + + archives.sort_by(|a, b| (b.1).0.cmp(&(a.1).0)); // reverse sort by slot + archives +} + +pub fn get_highest_snapshot_archive_path>( + snapshot_output_dir: P, +) -> Option { + let archives = get_snapshot_archives(snapshot_output_dir); + archives.into_iter().next().map(|archive| archive.0) } pub fn untar_snapshot_in, Q: AsRef>( @@ -733,4 +792,13 @@ mod tests { ); assert_matches!(result, Err(SnapshotError::IO(ref message)) if message.to_string().starts_with("invalid snapshot data file")); } + + #[test] + fn test_snapshot_hash_of() { + assert_eq!( + snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())), + Some((42, Hash::default())) + ); + assert!(snapshot_hash_of("invalid").is_none()); + } } diff --git a/validator/src/main.rs b/validator/src/main.rs index 1da96609b8..0f410462ff 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -80,7 +80,7 @@ fn new_spinner_progress_bar() -> ProgressBar { progress_bar } -fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Result<(), String> { +fn download_file(url: &str, destination_file: &Path) -> Result<(), String> { if destination_file.is_file() { return Err(format!("{:?} already exists", destination_file)); } @@ -96,15 +96,9 @@ fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Resu let client = reqwest::blocking::Client::new(); let response = client.get(url).send().map_err(|err| err.to_string())?; - if response.status() == reqwest::StatusCode::NOT_FOUND && not_found_ok { - progress_bar.finish_and_clear(); - info!("Archive not found at {}", url); - return Ok(()); - } - let response = response .error_for_status() - .map_err(|err| format!("Unable to get: {:?}", err))?; + .map_err(|err| format!("Unable to download {}: {}", url, err))?; let download_size = { response .headers() @@ -460,7 +454,6 @@ fn download_genesis( download_file( &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"), &tmp_genesis_package, - false, )?; extract_archive(&tmp_genesis_package, &ledger_path)?; @@ -496,28 +489,22 @@ fn download_genesis( fn download_snapshot( rpc_addr: &SocketAddr, ledger_path: &Path, - snapshot_hash: Option<(Slot, Hash)>, + snapshot_hash: (Slot, Hash), ) -> Result<(), String> { - if snapshot_hash.is_none() { - return Ok(()); - } - - let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path); + let snapshot_package = + solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path, &snapshot_hash); if snapshot_package.exists() { - fs::remove_file(&snapshot_package) - .map_err(|err| format!("error removing {:?}: {}", snapshot_package, err))?; + Ok(()) + } else { + download_file( + &format!( + "http://{}/{}", + rpc_addr, + snapshot_package.file_name().unwrap().to_str().unwrap() + ), + &snapshot_package, + ) } - download_file( - &format!( - "http://{}/{}", - rpc_addr, - snapshot_package.file_name().unwrap().to_str().unwrap() - ), - &snapshot_package, - true, - )?; - - Ok(()) } // This function is duplicated in ledger-tool/src/main.rs... @@ -1144,7 +1131,13 @@ pub fn main() { } Ok(()) }) - .and_then(|_| download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash)) + .and_then(|_| { + if let Some(snapshot_hash) = snapshot_hash { + download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash) + } else { + Ok(()) + } + }) .and_then(|_| { if !validator_config.voting_disabled && !no_check_vote_account { check_vote_account(