Rename snapshot.tar.bz2 to snapshot-<slot>-<hash>.tar.bz2 (bp #8482) (#8500)

automerge

Cargo.lock

@@ -193,7 +193,7 @@ dependencies = [
  "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
  "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "which 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -586,7 +586,7 @@ dependencies = [
  "encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -925,7 +925,7 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -1023,7 +1023,7 @@ dependencies = [
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1035,7 +1035,7 @@ dependencies = [
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1047,7 +1047,7 @@ dependencies = [
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1339,7 +1339,7 @@ dependencies = [
  "bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1640,7 +1640,7 @@ dependencies = [
  "console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1823,8 +1823,8 @@ dependencies = [
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2586,7 +2586,7 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2657,7 +2657,7 @@ dependencies = [
  "error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2918,13 +2918,13 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.3.1"
+version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2942,7 +2942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.12"
+version = "0.6.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
@@ -3803,6 +3803,7 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3857,7 +3858,7 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4032,6 +4033,7 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4714,7 +4716,7 @@ dependencies = [
  "lalrpop 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lalrpop-util 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana_libra_failure_ext 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana_libra_types 0.0.1-sol4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -5300,6 +5302,14 @@ dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "threshold_crypto"
 version = "0.3.2"
@@ -5357,7 +5367,7 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -6374,10 +6384,10 @@ dependencies = [
 "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828"
 "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
-"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd"
+"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8"
 "checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9"
 "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
-"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
+"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06"
 "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
 "checksum rental 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "01916ebd9fc2e81978a5dc9542a2fa47f5bb2ca3402e14c7cc42d6e3c5123e1f"
 "checksum rental-impl 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "82260d54cf2cbe9608df161f7e7c98e81fae702aa13af9e4d5d39dc2ffb25ab6"
@@ -6493,6 +6503,7 @@ dependencies = [
 "checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99"
 "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
+"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
 "checksum threshold_crypto 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95be1032c63011f20b01c5edb64930e2b51512782b43b458b1e3449613d70f87"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 "checksum tiny-bip39 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92"

core/Cargo.toml

@@ -38,6 +38,7 @@ num-traits = "0.2"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
 rayon = "1.2.0"
+regex = "1.3.4"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.44"

core/src/rpc_service.rs

@@ -9,6 +9,7 @@ use jsonrpc_http_server::{
     hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware,
     RequestMiddlewareAction, ServerBuilder,
 };
+use regex::Regex;
 use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
 use solana_sdk::hash::Hash;
 use std::{
@@ -28,13 +29,17 @@ pub struct JsonRpcService {
     close_handle: Option<CloseHandle>,
 }
 
-#[derive(Default)]
 struct RpcRequestMiddleware {
     ledger_path: PathBuf,
+    snapshot_archive_path_regex: Regex,
 }
 impl RpcRequestMiddleware {
     pub fn new(ledger_path: PathBuf) -> Self {
-        Self { ledger_path }
+        Self {
+            ledger_path,
+            snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$")
+                .unwrap(),
+        }
     }
 
     fn not_found() -> hyper::Response<hyper::Body> {
@@ -51,9 +56,19 @@ impl RpcRequestMiddleware {
             .unwrap()
     }
 
-    fn get(&self, filename: &str) -> RequestMiddlewareAction {
+    fn is_get_path(&self, path: &str) -> bool {
-        info!("get {}", filename);
+        match path {
-        let filename = self.ledger_path.join(filename);
+            "/genesis.tar.bz2" => true,
+            _ => self.snapshot_archive_path_regex.is_match(path),
+        }
+    }
+
+    fn get(&self, path: &str) -> RequestMiddlewareAction {
+        let filename = self.ledger_path.join(
+            path.split_at(1).1, // Drop leading '/' from path
+        );
+        info!("get {} -> {:?}", path, filename);
+
         RequestMiddlewareAction::Respond {
             should_validate_hosts: true,
             response: Box::new(
@@ -73,13 +88,14 @@ impl RpcRequestMiddleware {
 impl RequestMiddleware for RpcRequestMiddleware {
     fn on_request(&self, request: hyper::Request<hyper::Body>) -> RequestMiddlewareAction {
         trace!("request uri: {}", request.uri());
-        match request.uri().path() {
+
-            "/snapshot.tar.bz2" => self.get("snapshot.tar.bz2"),
+        if self.is_get_path(request.uri().path()) {
-            "/genesis.tar.bz2" => self.get("genesis.tar.bz2"),
+            self.get(request.uri().path())
-            _ => RequestMiddlewareAction::Proceed {
+        } else {
+            RequestMiddlewareAction::Proceed {
                 should_continue_on_invalid_cors: false,
                 request,
-            },
+            }
         }
     }
 }
@@ -234,4 +250,25 @@ mod tests {
         rpc_service.exit();
         rpc_service.join().unwrap();
     }
+
+    #[test]
+    fn test_is_get_path() {
+        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"));
+
+        assert!(rrm.is_get_path("/genesis.tar.bz2"));
+        assert!(!rrm.is_get_path("genesis.tar.bz2"));
+
+        assert!(!rrm.is_get_path("/snapshot.tar.bz2"));
+
+        assert!(
+            rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2")
+        );
+        assert!(!rrm.is_get_path(
+            "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
+        ));
+
+        assert!(!rrm.is_get_path("/"));
+        assert!(!rrm.is_get_path(".."));
+        assert!(!rrm.is_get_path("🎣"));
+    }
 }

core/src/snapshot_packager_service.rs

@@ -154,8 +154,10 @@ mod tests {
         }
 
         // Create a packageable snapshot
-        let output_tar_path =
+        let output_tar_path = snapshot_utils::get_snapshot_archive_path(
-            snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
+            &snapshot_package_output_path,
+            &(42, Hash::default()),
+        );
         let snapshot_package = SnapshotPackage::new(
             5,
             vec![],

core/tests/bank_forks.rs

@@ -67,13 +67,19 @@ mod tests {
         }
     }
 
-    fn restore_from_snapshot(old_bank_forks: &BankForks, account_paths: Vec<PathBuf>) {
+    fn restore_from_snapshot(
+        old_bank_forks: &BankForks,
+        old_last_slot: Slot,
+        account_paths: Vec<PathBuf>,
+    ) {
         let (snapshot_path, snapshot_package_output_path) = old_bank_forks
             .snapshot_config
             .as_ref()
             .map(|c| (&c.snapshot_path, &c.snapshot_package_output_path))
             .unwrap();
 
+        let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
+
         let deserialized_bank = snapshot_utils::bank_from_archive(
             &account_paths,
             &old_bank_forks
@@ -81,7 +87,10 @@ mod tests {
                 .as_ref()
                 .unwrap()
                 .snapshot_path,
-            snapshot_utils::get_snapshot_archive_path(snapshot_package_output_path),
+            snapshot_utils::get_snapshot_archive_path(
+                snapshot_package_output_path,
+                &(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
+            ),
         )
         .unwrap();
 
@@ -139,18 +148,20 @@ mod tests {
             slot_snapshot_paths
                 .last()
                 .expect("no snapshots found in path"),
-            snapshot_utils::get_snapshot_archive_path(
-                &snapshot_config.snapshot_package_output_path,
-            ),
             &snapshot_config.snapshot_path,
             &last_bank.src.roots(),
+            &snapshot_config.snapshot_package_output_path,
             storages,
         )
         .unwrap();
 
         snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
 
-        restore_from_snapshot(bank_forks, vec![accounts_dir.path().to_path_buf()]);
+        restore_from_snapshot(
+            bank_forks,
+            last_slot,
+            vec![accounts_dir.path().to_path_buf()],
+        );
     }
 
     #[test]
@@ -221,9 +232,8 @@ mod tests {
         let saved_snapshots_dir = TempDir::new().unwrap();
         let saved_accounts_dir = TempDir::new().unwrap();
         let saved_slot = 4;
-        let saved_tar = snapshot_config
+        let mut saved_archive_path = None;
-            .snapshot_package_output_path
+
-            .join(saved_slot.to_string());
         for forks in 0..MAX_CACHE_ENTRIES + 2 {
             let bank = Bank::new_from_parent(
                 &bank_forks[forks as u64],
@@ -235,6 +245,7 @@ mod tests {
             let tx = system_transaction::transfer(&mint_keypair, &key1, 1, genesis_config.hash());
             assert_eq!(bank.process_transaction(&tx), Ok(()));
             bank.squash();
+            let accounts_hash = bank.update_accounts_hash();
             bank_forks.insert(bank);
 
             let package_sender = {
@@ -249,14 +260,7 @@ mod tests {
             };
 
             bank_forks
-                .generate_snapshot(
+                .generate_snapshot(slot, &vec![], &package_sender)
-                    slot,
-                    &vec![],
-                    &package_sender,
-                    snapshot_config
-                        .snapshot_package_output_path
-                        .join(slot.to_string()),
-                )
                 .unwrap();
 
             if slot == saved_slot as u64 {
@@ -282,6 +286,11 @@ mod tests {
                     &options,
                 )
                 .unwrap();
+
+                saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
+                    &snapshot_config.snapshot_package_output_path,
+                    &(slot, accounts_hash),
+                ));
             }
         }
 
@@ -319,7 +328,7 @@ mod tests {
             .join()
             .expect("SnapshotPackagerService exited with error");
 
-        // Check the tar we cached the state for earlier was generated correctly
+        // Check the archive we cached the state for earlier was generated correctly
 
         // before we compare, stick an empty status_cache in this dir so that the package comparision works
         // This is needed since the status_cache is added by the packager and is not collected from
@@ -338,7 +347,7 @@ mod tests {
         .unwrap();
 
         snapshot_utils::verify_snapshot_archive(
-            saved_tar,
+            saved_archive_path.unwrap(),
             saved_snapshots_dir.path(),
             saved_accounts_dir
                 .path()

ledger-tool/src/main.rs

@@ -909,9 +909,9 @@ fn main() {
                             snapshot_utils::package_snapshot(
                                 &bank,
                                 &slot_snapshot_paths,
-                                snapshot_utils::get_snapshot_archive_path(output_directory),
                                 &temp_dir,
                                 &bank.src.roots(),
+                                output_directory,
                                 storages,
                             )
                         })

ledger/Cargo.toml

@@ -26,6 +26,7 @@ rand = "0.6.5"
 rand_chacha = "0.1.1"
 rayon = "1.2.0"
 reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
+regex = "1.3.4"
 serde = "1.0.104"
 serde_bytes = "0.11.3"
 serde_derive = "1.0.103"

ledger/src/bank_forks.rs

@@ -10,7 +10,7 @@ use solana_sdk::{clock::Slot, timing};
 use std::{
     collections::{HashMap, HashSet},
     ops::Index,
-    path::{Path, PathBuf},
+    path::PathBuf,
     sync::Arc,
     time::Instant,
 };
@@ -192,7 +192,6 @@ impl BankForks {
                     root,
                     &root_bank.src.roots(),
                     snapshot_package_sender.as_ref().unwrap(),
-                    snapshot_utils::get_snapshot_archive_path(&config.snapshot_package_output_path),
                 );
                 if r.is_err() {
                     warn!("Error generating snapshot for bank: {}, err: {:?}", root, r);
@@ -236,12 +235,11 @@ impl BankForks {
         }
     }
 
-    pub fn generate_snapshot<P: AsRef<Path>>(
+    pub fn generate_snapshot(
         &self,
         root: Slot,
         slots_to_snapshot: &[Slot],
         snapshot_package_sender: &SnapshotPackageSender,
-        tar_output_file: P,
     ) -> Result<()> {
         let config = self.snapshot_config.as_ref().unwrap();
 
@@ -267,9 +265,9 @@ impl BankForks {
         let package = snapshot_utils::package_snapshot(
             &bank,
             latest_slot_snapshot_paths,
-            tar_output_file,
             &config.snapshot_path,
             slots_to_snapshot,
+            &config.snapshot_package_output_path,
             storages,
         )?;
 

ledger/src/bank_forks_utils.rs

@@ -10,7 +10,7 @@ use crate::{
 };
 use log::*;
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
-use std::{fs, path::PathBuf, result, sync::Arc};
+use std::{fs, path::PathBuf, process, result, sync::Arc};
 
 pub type LoadResult = result::Result<
     (
@@ -52,25 +52,38 @@ pub fn load(
         fs::create_dir_all(&snapshot_config.snapshot_path)
             .expect("Couldn't create snapshot directory");
 
-        let tar = snapshot_utils::get_snapshot_archive_path(
+        match snapshot_utils::get_highest_snapshot_archive_path(
             &snapshot_config.snapshot_package_output_path,
-        );
+        ) {
-        if tar.exists() {
+            Some((archive_filename, archive_snapshot_hash)) => {
-            info!("Loading snapshot package: {:?}", tar);
+                info!("Loading snapshot package: {:?}", archive_filename);
                 // Fail hard here if snapshot fails to load, don't silently continue
 
                 if account_paths.is_empty() {
-                panic!("Account paths not present when booting from snapshot")
+                    error!("Account paths not present when booting from snapshot");
+                    process::exit(1);
                 }
 
                 let deserialized_bank = snapshot_utils::bank_from_archive(
                     &account_paths,
                     &snapshot_config.snapshot_path,
-                &tar,
+                    &archive_filename,
                 )
                 .expect("Load from snapshot failed");
 
-            let snapshot_hash = (deserialized_bank.slot(), deserialized_bank.hash());
+                let deserialized_snapshot_hash = (
+                    deserialized_bank.slot(),
+                    deserialized_bank.get_accounts_hash(),
+                );
+
+                if deserialized_snapshot_hash != archive_snapshot_hash {
+                    error!(
+                        "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
+                        archive_snapshot_hash, deserialized_snapshot_hash
+                    );
+                    process::exit(1);
+                }
+
                 return to_loadresult(
                     blockstore_processor::process_blockstore_from_root(
                         genesis_config,
@@ -79,10 +92,10 @@ pub fn load(
                         &process_options,
                         &VerifyRecyclers::default(),
                     ),
-                Some(snapshot_hash),
+                    Some(deserialized_snapshot_hash),
                 );
-        } else {
+            }
-            info!("Snapshot package does not exist: {:?}", tar);
+            None => info!("No snapshot package available"),
         }
     } else {
         info!("Snapshots disabled");

ledger/src/snapshot_utils.rs

@@ -3,6 +3,7 @@ use bincode::serialize_into;
 use bzip2::bufread::BzDecoder;
 use fs_extra::dir::CopyOptions;
 use log::*;
+use regex::Regex;
 use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts_db::{SnapshotStorage, SnapshotStorages},
@@ -11,7 +12,7 @@ use solana_runtime::{
         MAX_SNAPSHOT_DATA_FILE_SIZE,
     },
 };
-use solana_sdk::clock::Slot;
+use solana_sdk::{clock::Slot, hash::Hash};
 use std::{
     cmp::Ordering,
     env,
@@ -81,9 +82,9 @@ impl SlotSnapshotPaths {
 pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     bank: &Bank,
     snapshot_files: &SlotSnapshotPaths,
-    snapshot_package_output_file: P,
     snapshot_path: Q,
     slots_to_snapshot: &[Slot],
+    snapshot_package_output_path: P,
     snapshot_storages: SnapshotStorages,
 ) -> Result<SnapshotPackage> {
     // Hard link all the snapshots we need for this package
@@ -100,13 +101,18 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     // any temporary state created for the SnapshotPackage (like the snapshot_hard_links_dir)
     snapshot_files.copy_snapshot_directory(snapshot_hard_links_dir.path())?;
 
+    let snapshot_package_output_file = get_snapshot_archive_path(
+        &snapshot_package_output_path,
+        &(bank.slot(), bank.get_accounts_hash()),
+    );
+
     let package = SnapshotPackage::new(
         bank.slot(),
         bank.src.slot_deltas(slots_to_snapshot),
         snapshot_hard_links_dir,
         snapshot_storages,
-        snapshot_package_output_file.as_ref().to_path_buf(),
+        snapshot_package_output_file,
-        bank.hash(),
+        bank.get_accounts_hash(),
     );
 
     Ok(package)
@@ -205,9 +211,17 @@ pub fn archive_snapshot_package(snapshot_package: &SnapshotPackage) -> Result<()
     let metadata = fs::metadata(&archive_path)?;
     fs::rename(&archive_path, &snapshot_package.tar_output_file)?;
 
+    // Keep around at most two snapshot archives
+    let archives = get_snapshot_archives(snapshot_package.tar_output_file.parent().unwrap());
+    for old_archive in archives.into_iter().skip(2) {
+        fs::remove_file(old_archive.0)
+            .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err));
+    }
+
     timer.stop();
     info!(
-        "Successfully created tarball. slot: {}, elapsed ms: {}, size={}",
+        "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}",
+        snapshot_package.tar_output_file,
         snapshot_package.root,
         timer.as_ms(),
         metadata.len()
@@ -415,32 +429,6 @@ pub fn remove_snapshot<P: AsRef<Path>>(slot: Slot, snapshot_path: P) -> Result<(
     Ok(())
 }
 
-pub fn bank_slot_from_archive<P: AsRef<Path>>(snapshot_tar: P) -> Result<Slot> {
-    let tempdir = tempfile::TempDir::new()?;
-    untar_snapshot_in(&snapshot_tar, &tempdir)?;
-    let unpacked_snapshots_dir = tempdir.path().join(TAR_SNAPSHOTS_DIR);
-    let local_account_paths = vec![tempdir.path().join("account_dummy")];
-    let unpacked_accounts_dir = tempdir.path().join(TAR_ACCOUNTS_DIR);
-    let snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
-    let last_root_paths = snapshot_paths
-        .last()
-        .ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
-    let bank = deserialize_snapshot_data_file(
-        &last_root_paths.snapshot_file_path,
-        MAX_SNAPSHOT_DATA_FILE_SIZE,
-        |stream| {
-            let bank: Bank = deserialize_from_snapshot(stream.by_ref())?;
-            bank.rc.accounts_from_stream(
-                stream.by_ref(),
-                &local_account_paths,
-                &unpacked_accounts_dir,
-            )?;
-            Ok(bank)
-        },
-    )?;
-    Ok(bank.slot())
-}
-
 pub fn bank_from_archive<P: AsRef<Path>>(
     account_paths: &[PathBuf],
     snapshot_path: &PathBuf,
@@ -503,8 +491,59 @@ fn is_snapshot_compression_disabled() -> bool {
     }
 }
 
-pub fn get_snapshot_archive_path<P: AsRef<Path>>(snapshot_output_dir: P) -> PathBuf {
+pub fn get_snapshot_archive_path<P: AsRef<Path>>(
-    snapshot_output_dir.as_ref().join("snapshot.tar.bz2")
+    snapshot_output_dir: P,
+    snapshot_hash: &(Slot, Hash),
+) -> PathBuf {
+    snapshot_output_dir.as_ref().join(format!(
+        "snapshot-{}-{}.tar.bz2",
+        snapshot_hash.0, snapshot_hash.1
+    ))
+}
+
+fn snapshot_hash_of(archive_filename: &str) -> Option<(Slot, Hash)> {
+    let snapshot_filename_regex = Regex::new(r"snapshot-(\d+)-([[:alnum:]]+)\.tar\.bz2$").unwrap();
+
+    if let Some(captures) = snapshot_filename_regex.captures(archive_filename) {
+        let slot_str = captures.get(1).unwrap().as_str();
+        let hash_str = captures.get(2).unwrap().as_str();
+
+        if let (Ok(slot), Ok(hash)) = (slot_str.parse::<Slot>(), hash_str.parse::<Hash>()) {
+            return Some((slot, hash));
+        }
+    }
+    None
+}
+
+fn get_snapshot_archives<P: AsRef<Path>>(snapshot_output_dir: P) -> Vec<(PathBuf, (Slot, Hash))> {
+    let files = fs::read_dir(&snapshot_output_dir)
+        .unwrap_or_else(|err| panic!("Unable to read snapshot directory: {}", err));
+
+    let mut archives: Vec<_> = files
+        .filter_map(|entry| {
+            if let Ok(entry) = entry {
+                let path = entry.path();
+                if path.is_file() {
+                    if let Some(snapshot_hash) =
+                        snapshot_hash_of(path.file_name().unwrap().to_str().unwrap())
+                    {
+                        return Some((path, snapshot_hash));
+                    }
+                }
+            }
+            None
+        })
+        .collect();
+
+    archives.sort_by(|a, b| (b.1).0.cmp(&(a.1).0)); // reverse sort by slot
+    archives
+}
+
+pub fn get_highest_snapshot_archive_path<P: AsRef<Path>>(
+    snapshot_output_dir: P,
+) -> Option<(PathBuf, (Slot, Hash))> {
+    let archives = get_snapshot_archives(snapshot_output_dir);
+    archives.into_iter().next()
 }
 
 pub fn untar_snapshot_in<P: AsRef<Path>, Q: AsRef<Path>>(
@@ -607,7 +646,7 @@ fn get_io_error(error: &str) -> SnapshotError {
 }
 
 pub fn verify_snapshot_archive<P, Q, R>(
-    snapshot_tar: P,
+    snapshot_archive: P,
     snapshots_to_verify: Q,
     storages_to_verify: R,
 ) where
@@ -617,7 +656,7 @@ pub fn verify_snapshot_archive<P, Q, R>(
 {
     let temp_dir = tempfile::TempDir::new().unwrap();
     let unpack_dir = temp_dir.path();
-    untar_snapshot_in(snapshot_tar, &unpack_dir).unwrap();
+    untar_snapshot_in(snapshot_archive, &unpack_dir).unwrap();
 
     // Check snapshots are the same
     let unpacked_snapshots = unpack_dir.join(&TAR_SNAPSHOTS_DIR);
@@ -739,4 +778,13 @@ mod tests {
         );
         assert_matches!(result, Err(SnapshotError::IO(ref message)) if message.to_string().starts_with("invalid snapshot data file"));
     }
+
+    #[test]
+    fn test_snapshot_hash_of() {
+        assert_eq!(
+            snapshot_hash_of(&format!("snapshot-42-{}.tar.bz2", Hash::default())),
+            Some((42, Hash::default()))
+        );
+        assert!(snapshot_hash_of("invalid").is_none());
+    }
 }

local-cluster/tests/local_cluster.rs

@@ -18,10 +18,11 @@ use solana_local_cluster::{
 };
 use solana_sdk::{
     client::SyncClient,
-    clock,
+    clock::{self, Slot},
     commitment_config::CommitmentConfig,
     epoch_schedule::{EpochSchedule, MINIMUM_SLOTS_PER_EPOCH},
     genesis_config::OperatingMode,
+    hash::Hash,
     poh_config::PohConfig,
     signature::{Keypair, Signer},
 };
@@ -648,14 +649,16 @@ fn test_snapshot_restart_tower() {
         .as_ref()
         .unwrap()
         .snapshot_package_output_path;
-    let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
-    wait_for_next_snapshot(&cluster, &tar);
 
-    // Copy tar to validator's snapshot output directory
+    let (archive_filename, archive_snapshot_hash) =
-    let validator_tar_path = snapshot_utils::get_snapshot_archive_path(
+        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);
+
+    // Copy archive to validator's snapshot output directory
+    let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
         &validator_snapshot_test_config.snapshot_output_path,
+        &archive_snapshot_hash,
     );
-    fs::hard_link(tar, &validator_tar_path).unwrap();
+    fs::hard_link(archive_filename, &validator_archive_path).unwrap();
 
     // Restart validator from snapshot, the validator's tower state in this snapshot
     // will contain slots < the root bank of the snapshot. Validator should not panic.
@@ -704,21 +707,23 @@ fn test_snapshots_blockstore_floor() {
 
     trace!("Waiting for snapshot tar to be generated with slot",);
 
-    let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
+    let (archive_filename, (archive_slot, archive_hash)) = loop {
-    loop {
+        let archive =
-        if tar.exists() {
+            snapshot_utils::get_highest_snapshot_archive_path(&snapshot_package_output_path);
-            trace!("snapshot tar exists");
+        if archive.is_some() {
-            break;
+            trace!("snapshot exists");
+            break archive.unwrap();
         }
         sleep(Duration::from_millis(5000));
-    }
+    };
 
-    // Copy tar to validator's snapshot output directory
+    // Copy archive to validator's snapshot output directory
-    let validator_tar_path = snapshot_utils::get_snapshot_archive_path(
+    let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
         &validator_snapshot_test_config.snapshot_output_path,
+        &(archive_slot, archive_hash),
     );
-    fs::hard_link(tar, &validator_tar_path).unwrap();
+    fs::hard_link(archive_filename, &validator_archive_path).unwrap();
-    let slot_floor = snapshot_utils::bank_slot_from_archive(&validator_tar_path).unwrap();
+    let slot_floor = archive_slot;
 
     // Start up a new node from a snapshot
     let validator_stake = 5;
@@ -814,8 +819,7 @@ fn test_snapshots_restart_validity() {
 
         expected_balances.extend(new_balances);
 
-        let tar = snapshot_utils::get_snapshot_archive_path(&snapshot_package_output_path);
+        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);
-        wait_for_next_snapshot(&cluster, &tar);
 
         // Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
         // which may delete the old accounts on exit at any point
@@ -952,7 +956,10 @@ fn test_no_voting() {
     }
 }
 
-fn wait_for_next_snapshot<P: AsRef<Path>>(cluster: &LocalCluster, tar: P) {
+fn wait_for_next_snapshot(
+    cluster: &LocalCluster,
+    snapshot_package_output_path: &Path,
+) -> (PathBuf, (Slot, Hash)) {
     // Get slot after which this was generated
     let client = cluster
         .get_validator_client(&cluster.entry_point_info.id)
@@ -964,17 +971,18 @@ fn wait_for_next_snapshot<P: AsRef<Path>>(cluster: &LocalCluster, tar: P) {
     // Wait for a snapshot for a bank >= last_slot to be made so we know that the snapshot
     // must include the transactions just pushed
     trace!(
-        "Waiting for snapshot tar to be generated with slot > {}",
+        "Waiting for snapshot archive to be generated with slot > {}",
         last_slot
     );
     loop {
-        if tar.as_ref().exists() {
+        if let Some((filename, (slot, hash))) =
-            trace!("snapshot tar exists");
+            snapshot_utils::get_highest_snapshot_archive_path(snapshot_package_output_path)
-            let slot = snapshot_utils::bank_slot_from_archive(&tar).unwrap();
+        {
+            trace!("snapshot for slot {} exists", slot);
             if slot >= last_slot {
-                break;
+                return (filename, (slot, hash));
             }
-            trace!("snapshot tar slot {} < last_slot {}", slot, last_slot);
+            trace!("snapshot slot {} < last_slot {}", slot, last_slot);
         }
         sleep(Duration::from_millis(5000));
     }

runtime/src/bank.rs

@@ -496,6 +496,14 @@ impl Bank {
         *self.hash.read().unwrap()
     }
 
+    pub fn get_accounts_hash(&self) -> Hash {
+        self.hash()
+    }
+
+    pub fn update_accounts_hash(&self) -> Hash {
+        self.hash()
+    }
+
     pub fn is_frozen(&self) -> bool {
         *self.hash.read().unwrap() != Hash::default()
     }

validator/src/main.rs

@@ -80,7 +80,7 @@ fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
-fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Result<(), String> {
+fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
     if destination_file.is_file() {
         return Err(format!("{:?} already exists", destination_file));
     }
@@ -96,15 +96,9 @@ fn download_file(url: &str, destination_file: &Path, not_found_ok: bool) -> Resu
     let client = reqwest::blocking::Client::new();
     let response = client.get(url).send().map_err(|err| err.to_string())?;
 
-    if response.status() == reqwest::StatusCode::NOT_FOUND && not_found_ok {
-        progress_bar.finish_and_clear();
-        info!("Archive not found at {}", url);
-        return Ok(());
-    }
-
     let response = response
         .error_for_status()
-        .map_err(|err| format!("Unable to get: {:?}", err))?;
+        .map_err(|err| format!("Unable to download {}: {}", url, err))?;
     let download_size = {
         response
             .headers()
@@ -438,7 +432,6 @@ fn download_genesis(
         download_file(
             &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
             &tmp_genesis_package,
-            false,
         )?;
         extract_archive(&tmp_genesis_package, &ledger_path)?;
 
@@ -474,17 +467,13 @@ fn download_genesis(
 fn download_snapshot(
     rpc_addr: &SocketAddr,
     ledger_path: &Path,
-    snapshot_hash: Option<(Slot, Hash)>,
+    snapshot_hash: (Slot, Hash),
 ) -> Result<(), String> {
-    if snapshot_hash.is_none() {
+    let snapshot_package =
-        return Ok(());
+        solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path, &snapshot_hash);
-    }
-
-    let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_archive_path(ledger_path);
     if snapshot_package.exists() {
-        fs::remove_file(&snapshot_package)
+        Ok(())
-            .map_err(|err| format!("error removing {:?}: {}", snapshot_package, err))?;
+    } else {
-    }
         download_file(
             &format!(
                 "http://{}/{}",
@@ -492,10 +481,8 @@ fn download_snapshot(
                 snapshot_package.file_name().unwrap().to_str().unwrap()
             ),
             &snapshot_package,
-        true,
+        )
-    )?;
+    }
-
-    Ok(())
 }
 
 // This function is duplicated in ledger-tool/src/main.rs...
@@ -1122,7 +1109,13 @@ pub fn main() {
                     }
                     Ok(())
                 })
-                .and_then(|_| download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash))
+                .and_then(|_| {
+                    if let Some(snapshot_hash) = snapshot_hash {
+                        download_snapshot(&rpc_contact_info.rpc, &ledger_path, snapshot_hash)
+                    } else {
+                        Ok(())
+                    }
+                })
                 .and_then(|_| {
                     if !validator_config.voting_disabled && !no_check_vote_account {
                         check_vote_account(