gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Separate remotely downloaded snapshot archives (#23510)

Browse Source 
* seperate remotely downloaded snapshot archives

* add str const for snapshot download dir

* only walk remote sub directory

* move directory creation outside of loop

* move is_remote to traits

* clippy simplify

* clippy

* clippy

* add unittest

* fix local cluster tests

* look for remote snapshot archive in remote foler

* create remote dir in tests

* use snapshot download dir constant

* extract build_remote_dir fn

* fix build

* code review - walking snapshot archived dirs explicitly

* fix build

* fix build

* fix comments

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* clippy

* borrow to avoid copy

Co-authored-by: Brooks Prumo <brooks@prumo.org>
This commit is contained in:
HaoranYi
2022-03-14 14:03:59 -05:00
committed by GitHub
parent 5ea6a1e500
commit 0c684721d8
5 changed files with 146 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
local-cluster/tests/local_cluster.rs
View File

@ -1083,6 +1083,22 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
}
};
let copy_files_with_remote = |from: &Path, to: &Path| {
copy_files(from, to);
let remote_from = snapshot_utils::build_snapshot_archives_remote_dir(from);
let remote_to = snapshot_utils::build_snapshot_archives_remote_dir(to);
let _ = fs::create_dir_all(&remote_from);
let _ = fs::create_dir_all(&remote_to);
copy_files(&remote_from, &remote_to);
};
let delete_files_with_remote = |from: &Path| {
delete_files(from);
let remote_dir = snapshot_utils::build_snapshot_archives_remote_dir(from);
let _ = fs::create_dir_all(&remote_dir);
delete_files(&remote_dir);
};
// After downloading the snapshots, copy them over to a backup directory. Later we'll need to
// restart the node and guarantee that the only snapshots present are these initial ones. So,
// the easiest way to do that is create a backup now, delete the ones on the node before
@ -1092,7 +1108,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
"Backing up validator snapshots to dir: {}...",
backup_validator_snapshot_archives_dir.path().display()
);
copy_files(
copy_files_with_remote(
validator_snapshot_test_config.snapshot_archives_dir.path(),
backup_validator_snapshot_archives_dir.path(),
);
@ -1170,8 +1186,8 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
trace!(
"Delete all the snapshots on the validator and restore the originals from the backup..."
);
delete_files(validator_snapshot_test_config.snapshot_archives_dir.path());
copy_files(
delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path());
copy_files_with_remote(
backup_validator_snapshot_archives_dir.path(),
validator_snapshot_test_config.snapshot_archives_dir.path(),
);