Snapshot pipefitting through the validator cli (#5617)

* Handle 404 errors better

* Snapshot pipefitting through the validator cli

* Add download progress bar

* Log the current entrypoint slot

core/src/snapshot_utils.rs

@@ -14,7 +14,6 @@ use std::fs::File;
 use std::io::{BufReader, BufWriter, Error as IOError, ErrorKind};
 use std::path::{Path, PathBuf};
 use tar::Archive;
-use tempfile::TempDir;
 
 const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache";
 
@@ -92,29 +91,43 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     Ok(package)
 }
 
-pub fn get_snapshot_paths<P: AsRef<Path>>(snapshot_path: P) -> Vec<SlotSnapshotPaths> {
-    let paths = fs::read_dir(&snapshot_path).expect("Invalid snapshot path");
-    let mut names = paths
-        .filter_map(|entry| {
-            entry.ok().and_then(|e| {
-                e.path()
-                    .file_name()
-                    .and_then(|n| n.to_str().map(|s| s.parse::<u64>().ok()))
-                    .unwrap_or(None)
-            })
-        })
-        .map(|slot| {
-            let snapshot_path = snapshot_path.as_ref().join(slot.to_string());
-            SlotSnapshotPaths {
-                slot,
-                snapshot_file_path: snapshot_path.join(get_snapshot_file_name(slot)),
-                snapshot_status_cache_path: snapshot_path.join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
-            }
-        })
-        .collect::<Vec<SlotSnapshotPaths>>();
+pub fn get_snapshot_paths<P: AsRef<Path>>(snapshot_path: P) -> Vec<SlotSnapshotPaths>
+where
+    P: std::fmt::Debug,
+{
+    match fs::read_dir(&snapshot_path) {
+        Ok(paths) => {
+            let mut names = paths
+                .filter_map(|entry| {
+                    entry.ok().and_then(|e| {
+                        e.path()
+                            .file_name()
+                            .and_then(|n| n.to_str().map(|s| s.parse::<u64>().ok()))
+                            .unwrap_or(None)
+                    })
+                })
+                .map(|slot| {
+                    let snapshot_path = snapshot_path.as_ref().join(slot.to_string());
+                    SlotSnapshotPaths {
+                        slot,
+                        snapshot_file_path: snapshot_path.join(get_snapshot_file_name(slot)),
+                        snapshot_status_cache_path: snapshot_path
+                            .join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
+                    }
+                })
+                .collect::<Vec<SlotSnapshotPaths>>();
 
-    names.sort();
-    names
+            names.sort();
+            names
+        }
+        Err(err) => {
+            info!(
+                "Unable to read snapshot directory {:?}: {}",
+                snapshot_path, err
+            );
+            vec![]
+        }
+    }
 }
 
 pub fn add_snapshot<P: AsRef<Path>>(
@@ -172,7 +185,7 @@ pub fn remove_snapshot<P: AsRef<Path>>(slot: u64, snapshot_path: P) -> Result<()
 }
 
 pub fn bank_slot_from_archive<P: AsRef<Path>>(snapshot_tar: P) -> Result<u64> {
-    let tempdir = TempDir::new()?;
+    let tempdir = tempfile::TempDir::new()?;
     untar_snapshot_in(&snapshot_tar, &tempdir)?;
     let unpacked_snapshots_dir = tempdir.path().join(TAR_SNAPSHOTS_DIR);
     let snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
@@ -191,7 +204,7 @@ pub fn bank_from_archive<P: AsRef<Path>>(
     snapshot_tar: P,
 ) -> Result<Bank> {
     // Untar the snapshot into a temp directory under `snapshot_config.snapshot_path()`
-    let unpack_dir = tempfile::tempdir_in(snapshot_config.snapshot_path())?;
+    let unpack_dir = tempfile::tempdir_in(&snapshot_config.snapshot_path)?;
     untar_snapshot_in(&snapshot_tar, &unpack_dir)?;
 
     let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR);
@@ -199,14 +212,19 @@ pub fn bank_from_archive<P: AsRef<Path>>(
     let snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir);
     let bank = rebuild_bank_from_snapshots(account_paths, &snapshot_paths, unpacked_accounts_dir)?;
 
-    // Move the unpacked snapshots into `snapshot_config.snapshot_path()`
-    let dir_files = fs::read_dir(unpacked_snapshots_dir).expect("Invalid snapshot path");
+    // Move the unpacked snapshots into `snapshot_config.snapshot_path`
+    let dir_files = fs::read_dir(&unpacked_snapshots_dir).unwrap_or_else(|err| {
+        panic!(
+            "Invalid snapshot path {:?}: {}",
+            unpacked_snapshots_dir, err
+        )
+    });
     let paths: Vec<PathBuf> = dir_files
         .filter_map(|entry| entry.ok().map(|e| e.path()))
         .collect();
     let mut copy_options = CopyOptions::new();
     copy_options.overwrite = true;
-    fs_extra::move_items(&paths, snapshot_config.snapshot_path(), &copy_options)?;
+    fs_extra::move_items(&paths, &snapshot_config.snapshot_path, &copy_options)?;
 
     Ok(bank)
 }
@@ -238,7 +256,7 @@ where
     let last_root_paths = snapshot_paths
         .last()
         .ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?;
-    info!("Load from {:?}", &last_root_paths.snapshot_file_path);
+    info!("Loading from {:?}", &last_root_paths.snapshot_file_path);
     let file = File::open(&last_root_paths.snapshot_file_path)?;
     let mut stream = BufReader::new(file);
     let bank: Bank = deserialize_from(&mut stream).map_err(|e| get_io_error(&e.to_string()))?;