gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add test_incremental_snapshot_download() to local-cluster (#19746)

Browse Source
This commit is contained in:
Brooks Prumo
2021-09-13 21:44:48 -05:00
committed by GitHub
parent b57e86abf2
commit 79ade5ec68
6 changed files with 385 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
download-utils/src/lib.rs
View File

@@ -2,7 +2,10 @@
use console::Emoji;
use indicatif::{ProgressBar, ProgressStyle};
use log::*;
use solana_runtime::{snapshot_utils, snapshot_utils::ArchiveFormat};
use solana_runtime::{
snapshot_package::SnapshotType,
snapshot_utils::{self, ArchiveFormat},
};
use solana_sdk::{clock::Slot, genesis_config::DEFAULT_GENESIS_ARCHIVE, hash::Hash};
use std::fs::{self, File};
use std::io;
@@ -244,13 +247,16 @@ pub fn download_genesis_if_missing(
}
}
pub fn download_snapshot<'a, 'b>(
/// Download a snapshot archive from `rpc_addr`. Use `snapshot_type` to specify downloading either
/// a full snapshot or an incremental snapshot.
pub fn download_snapshot_archive<'a, 'b>(
rpc_addr: &SocketAddr,
snapshot_archives_dir: &Path,
desired_snapshot_hash: (Slot, Hash),
use_progress_bar: bool,
snapshot_type: SnapshotType,
maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize,
use_progress_bar: bool,
progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
) -> Result<(), String> {
snapshot_utils::purge_old_snapshot_archives(
@@ -259,20 +265,31 @@ pub fn download_snapshot<'a, 'b>(
maximum_incremental_snapshot_archives_to_retain,
);
for compression in &[
for archive_format in [
ArchiveFormat::TarZstd,
ArchiveFormat::TarGzip,
ArchiveFormat::TarBzip2,
ArchiveFormat::Tar, // `solana-test-validator` creates uncompressed snapshots
] {
let desired_snapshot_package = snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir.to_path_buf(),
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
*compression,
);
let destination_path = match snapshot_type {
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir.to_path_buf(),
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
archive_format,
),
SnapshotType::IncrementalSnapshot(base_slot) => {
snapshot_utils::build_incremental_snapshot_archive_path(
snapshot_archives_dir.to_path_buf(),
base_slot,
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
archive_format,
)
}
};
if desired_snapshot_package.is_file() {
if destination_path.is_file() {
return Ok(());
}
@@ -280,13 +297,9 @@ pub fn download_snapshot<'a, 'b>(
&format!(
"http://{}/{}",
rpc_addr,
desired_snapshot_package
.file_name()
.unwrap()
.to_str()
.unwrap()
destination_path.file_name().unwrap().to_str().unwrap()
),
&desired_snapshot_package,
&destination_path,
use_progress_bar,
progress_notify_callback,
) {
@@ -295,7 +308,7 @@ pub fn download_snapshot<'a, 'b>(
}
}
Err(format!(
"Failed to download a snapshot for slot {} from {}",
"Failed to download a snapshot archive for slot {} from {}",
desired_snapshot_hash.0, rpc_addr
))
}