gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Do not shell out for tar (#19043)

Browse Source 
When making a snapshot archive, we used to shell out and call `tar -S`
for sparse file support.  The tar crate supports sparse files, so no
need to do this anymore.

Fixes #10860

(cherry picked from commit 68cc71409e)

# Conflicts:
#	runtime/src/snapshot_utils.rs
This commit is contained in:
Brooks Prumo
2021-08-04 17:07:55 -05:00
committed by mergify[bot]
parent 200c5c9fd6
commit df929bda38
1 changed files with 36 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
runtime/src/snapshot_utils.rs
View File

@ -28,15 +28,13 @@ use {
collections::HashSet, collections::HashSet,
fmt, fmt,
fs::{self, File}, fs::{self, File},
io::{ io::{BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write},
self, BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write,
},
path::{Path, PathBuf}, path::{Path, PathBuf},
process::{self, ExitStatus}, process::ExitStatus,
str::FromStr, str::FromStr,
sync::Arc, sync::Arc,
}, },
tar::Archive, tar::{self, Archive},
thiserror::Error, thiserror::Error,
}; };
@ -313,69 +311,46 @@ pub fn archive_snapshot_package(
let file_ext = get_archive_ext(snapshot_package.archive_format); let file_ext = get_archive_ext(snapshot_package.archive_format);
// Tar the staging directory into the archive at `archive_path` // Tar the staging directory into the archive at `archive_path`
//
// system `tar` program is used for -S (sparse file support)
let archive_path = tar_dir.join(format!( let archive_path = tar_dir.join(format!(
"{}{}.{}", "{}{}.{}",
TMP_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext TMP_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
)); ));
let mut tar = process::Command::new("tar") {
.args(&[ let mut archive_file = fs::File::create(&archive_path)?;
"chS",
"-C",
staging_dir.path().to_str().unwrap(),
"accounts",
"snapshots",
"version",
])
.stdin(process::Stdio::null())
.stdout(process::Stdio::piped())
.stderr(process::Stdio::inherit())
.spawn()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process spawn"))?;
match &mut tar.stdout { let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
None => { let mut archive = tar::Builder::new(encoder);
return Err(SnapshotError::Io(IoError::new( for dir in &["accounts", "snapshots"] {
ErrorKind::Other, archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
"tar stdout unavailable".to_string(), }
))); archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
} archive.into_inner()?;
Some(tar_output) => { Ok(())
let mut archive_file = fs::File::create(&archive_path)?; };
match snapshot_package.archive_format { match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => { ArchiveFormat::TarBzip2 => {
let mut encoder = let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best); bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best);
io::copy(tar_output, &mut encoder)?; do_archive_files(&mut encoder)?;
let _ = encoder.finish()?; encoder.finish()?;
} }
ArchiveFormat::TarGzip => { ArchiveFormat::TarGzip => {
let mut encoder = let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default()); flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
io::copy(tar_output, &mut encoder)?; do_archive_files(&mut encoder)?;
let _ = encoder.finish()?; encoder.finish()?;
} }
ArchiveFormat::Tar => { ArchiveFormat::TarZstd => {
io::copy(tar_output, &mut archive_file)?; let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
} do_archive_files(&mut encoder)?;
ArchiveFormat::TarZstd => { encoder.finish()?;
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?; }
io::copy(tar_output, &mut encoder)?; ArchiveFormat::Tar => {
let _ = encoder.finish()?; do_archive_files(&mut archive_file)?;
} }
}; };
}
}
let tar_exit_status = tar
.wait()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process wait"))?;
if !tar_exit_status.success() {
warn!("tar command failed with exit code: {}", tar_exit_status);
return Err(SnapshotError::ArchiveGenerationFailure(tar_exit_status));
} }
// Atomically move the archive into position for other validators to find // Atomically move the archive into position for other validators to find