Add non-progress bar download status (#13370) (#13942)

download-utils/src/lib.rs

@@ -23,7 +23,11 @@ fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
-pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
+pub fn download_file(
+    url: &str,
+    destination_file: &Path,
+    use_progress_bar: bool,
+) -> Result<(), String> {
     if destination_file.is_file() {
         return Err(format!("{:?} already exists", destination_file));
     }
@@ -34,7 +38,9 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
     let temp_destination_file = destination_file.with_extension(".tmp");
 
     let progress_bar = new_spinner_progress_bar();
-    progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
+    if use_progress_bar {
+        progress_bar.set_message(&format!("{}Downloading {}...", TRUCK, url));
+    }
 
     let response = reqwest::blocking::Client::new()
         .get(url)
@@ -53,28 +59,51 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
             .and_then(|content_length| content_length.parse().ok())
             .unwrap_or(0)
     };
-    progress_bar.set_length(download_size);
+
-    progress_bar.set_style(
+    if use_progress_bar {
-        ProgressStyle::default_bar()
+        progress_bar.set_length(download_size);
-            .template(&format!(
+        progress_bar.set_style(
-                "{}{}Downloading {} {}",
+            ProgressStyle::default_bar()
-                "{spinner:.green} ",
+                .template(&format!(
-                TRUCK,
+                    "{}{}Downloading {} {}",
-                url,
+                    "{spinner:.green} ",
-                "[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
+                    TRUCK,
-            ))
+                    url,
-            .progress_chars("=> "),
+                    "[{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})"
-    );
+                ))
+                .progress_chars("=> "),
+        );
+    } else {
+        info!("Downloading {} bytes from {}", download_size, url);
+    }
 
     struct DownloadProgress<R> {
         progress_bar: ProgressBar,
         response: R,
+        last_print: Instant,
+        current_bytes: usize,
+        download_size: f32,
+        use_progress_bar: bool,
     }
 
     impl<R: Read> Read for DownloadProgress<R> {
         fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
             self.response.read(buf).map(|n| {
-                self.progress_bar.inc(n as u64);
+                if self.use_progress_bar {
+                    self.progress_bar.inc(n as u64);
+                } else {
+                    self.current_bytes += n;
+                    if self.last_print.elapsed().as_secs() > 5 {
+                        let bytes_f32 = self.current_bytes as f32;
+                        info!(
+                            "downloaded {} bytes {:.1}% {:.1} bytes/s",
+                            self.current_bytes,
+                            100f32 * (bytes_f32 / self.download_size),
+                            bytes_f32 / self.last_print.elapsed().as_secs_f32(),
+                        );
+                        self.last_print = Instant::now();
+                    }
+                }
                 n
             })
         }
@@ -83,6 +112,10 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
     let mut source = DownloadProgress {
         progress_bar,
         response,
+        last_print: Instant::now(),
+        current_bytes: 0,
+        download_size: (download_size as f32).max(1f32),
+        use_progress_bar,
     };
 
     File::create(&temp_destination_file)
@@ -110,6 +143,7 @@ pub fn download_file(url: &str, destination_file: &Path) -> Result<(), String> {
 pub fn download_genesis_if_missing(
     rpc_addr: &SocketAddr,
     genesis_package: &Path,
+    use_progress_bar: bool,
 ) -> Result<PathBuf, String> {
     if !genesis_package.exists() {
         let tmp_genesis_path = genesis_package.parent().unwrap().join("tmp-genesis");
@@ -119,6 +153,7 @@ pub fn download_genesis_if_missing(
         download_file(
             &format!("http://{}/{}", rpc_addr, "genesis.tar.bz2"),
             &tmp_genesis_package,
+            use_progress_bar,
         )?;
 
         Ok(tmp_genesis_package)
@@ -131,6 +166,7 @@ pub fn download_snapshot(
     rpc_addr: &SocketAddr,
     ledger_path: &Path,
     desired_snapshot_hash: (Slot, Hash),
+    use_progress_bar: bool,
 ) -> Result<(), String> {
     // Remove all snapshot not matching the desired hash
     let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path);
@@ -171,6 +207,7 @@ pub fn download_snapshot(
                         .unwrap()
                 ),
                 &desired_snapshot_package,
+                use_progress_bar,
             )
             .is_ok()
             {

local-cluster/tests/local_cluster.rs

@@ -1052,6 +1052,7 @@ fn test_snapshot_download() {
         &cluster.entry_point_info.rpc,
         &validator_archive_path,
         archive_snapshot_hash,
+        false,
     )
     .unwrap();
 

validator/src/main.rs

@@ -447,6 +447,7 @@ fn download_then_check_genesis_hash(
     expected_genesis_hash: Option<Hash>,
     max_genesis_archive_unpacked_size: u64,
     no_genesis_fetch: bool,
+    use_progress_bar: bool,
 ) -> Result<Hash, String> {
     if no_genesis_fetch {
         let genesis_config = load_local_genesis(ledger_path, expected_genesis_hash)?;
@@ -454,26 +455,27 @@ fn download_then_check_genesis_hash(
     }
 
     let genesis_package = ledger_path.join("genesis.tar.bz2");
-    let genesis_config =
+    let genesis_config = if let Ok(tmp_genesis_package) =
-        if let Ok(tmp_genesis_package) = download_genesis_if_missing(rpc_addr, &genesis_package) {
+        download_genesis_if_missing(rpc_addr, &genesis_package, use_progress_bar)
-            unpack_genesis_archive(
+    {
-                &tmp_genesis_package,
+        unpack_genesis_archive(
-                &ledger_path,
+            &tmp_genesis_package,
-                max_genesis_archive_unpacked_size,
+            &ledger_path,
-            )
+            max_genesis_archive_unpacked_size,
-            .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
+        )
+        .map_err(|err| format!("Failed to unpack downloaded genesis config: {}", err))?;
 
-            let downloaded_genesis = GenesisConfig::load(&ledger_path)
+        let downloaded_genesis = GenesisConfig::load(&ledger_path)
-                .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
+            .map_err(|err| format!("Failed to load downloaded genesis config: {}", err))?;
 
-            check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
+        check_genesis_hash(&downloaded_genesis, expected_genesis_hash)?;
-            std::fs::rename(tmp_genesis_package, genesis_package)
+        std::fs::rename(tmp_genesis_package, genesis_package)
-                .map_err(|err| format!("Unable to rename: {:?}", err))?;
+            .map_err(|err| format!("Unable to rename: {:?}", err))?;
 
-            downloaded_genesis
+        downloaded_genesis
-        } else {
+    } else {
-            load_local_genesis(ledger_path, expected_genesis_hash)?
+        load_local_genesis(ledger_path, expected_genesis_hash)?
-        };
+    };
 
     Ok(genesis_config.hash())
 }
@@ -636,6 +638,7 @@ fn rpc_bootstrap(
     validator_config: &mut ValidatorConfig,
     bootstrap_config: RpcBootstrapConfig,
     no_port_check: bool,
+    use_progress_bar: bool,
     maximum_local_snapshot_age: Slot,
 ) {
     if !no_port_check {
@@ -694,6 +697,7 @@ fn rpc_bootstrap(
                 validator_config.expected_genesis_hash,
                 bootstrap_config.max_genesis_archive_unpacked_size,
                 bootstrap_config.no_genesis_fetch,
+                use_progress_bar,
             );
 
             if let Ok(genesis_hash) = genesis_hash {
@@ -747,6 +751,7 @@ fn rpc_bootstrap(
                                 &rpc_contact_info.rpc,
                                 &ledger_path,
                                 snapshot_hash,
+                                use_progress_bar,
                             );
                             gossip_service.join().unwrap();
                             ret
@@ -814,6 +819,7 @@ fn create_validator(
     mut validator_config: ValidatorConfig,
     rpc_bootstrap_config: RpcBootstrapConfig,
     no_port_check: bool,
+    use_progress_bar: bool,
     maximum_local_snapshot_age: Slot,
 ) -> Validator {
     if validator_config.cuda {
@@ -834,6 +840,7 @@ fn create_validator(
             &mut validator_config,
             rpc_bootstrap_config,
             no_port_check,
+            use_progress_bar,
             maximum_local_snapshot_age,
         );
     }
@@ -1670,6 +1677,7 @@ pub fn main() {
             Some(logfile)
         }
     };
+    let use_progress_bar = logfile.is_none();
     let _logger_thread = start_logger(logfile);
 
     // Default to RUST_BACKTRACE=1 for more informative validator logs
@@ -1763,6 +1771,7 @@ pub fn main() {
         validator_config,
         rpc_bootstrap_config,
         no_port_check,
+        use_progress_bar,
         maximum_local_snapshot_age,
     );