gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Separate remotely downloaded snapshot archives (#23510)

Browse Source 
* seperate remotely downloaded snapshot archives

* add str const for snapshot download dir

* only walk remote sub directory

* move directory creation outside of loop

* move is_remote to traits

* clippy simplify

* clippy

* clippy

* add unittest

* fix local cluster tests

* look for remote snapshot archive in remote foler

* create remote dir in tests

* use snapshot download dir constant

* extract build_remote_dir fn

* fix build

* code review - walking snapshot archived dirs explicitly

* fix build

* fix build

* fix comments

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* clippy

* borrow to avoid copy

Co-authored-by: Brooks Prumo <brooks@prumo.org>
This commit is contained in:
HaoranYi
2022-03-14 14:03:59 -05:00
committed by GitHub
parent 5ea6a1e500
commit 0c684721d8
5 changed files with 146 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
download-utils/src/lib.rs
View File

@@ -268,6 +268,10 @@ pub fn download_snapshot_archive<'a, 'b>(
maximum_incremental_snapshot_archives_to_retain,
);
let snapshot_archives_remote_dir =
snapshot_utils::build_snapshot_archives_remote_dir(snapshot_archives_dir);
fs::create_dir_all(&snapshot_archives_remote_dir).unwrap();
for archive_format in [
ArchiveFormat::TarZstd,
ArchiveFormat::TarGzip,
@@ -276,14 +280,14 @@ pub fn download_snapshot_archive<'a, 'b>(
] {
let destination_path = match snapshot_type {
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir,
&snapshot_archives_remote_dir,
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
archive_format,
),
SnapshotType::IncrementalSnapshot(base_slot) => {
snapshot_utils::build_incremental_snapshot_archive_path(
snapshot_archives_dir,
&snapshot_archives_remote_dir,
base_slot,
desired_snapshot_hash.0,
&desired_snapshot_hash.1,