gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Do not shell out for tar (#19043)

Browse Source 
When making a snapshot archive, we used to shell out and call `tar -S`
for sparse file support.  The tar crate supports sparse files, so no
need to do this anymore.

Fixes #10860

(cherry picked from commit 68cc71409e)

# Conflicts:
#	runtime/src/snapshot_utils.rs
This commit is contained in:
Brooks Prumo
2021-08-04 17:07:55 -05:00
committed by mergify[bot]
parent 200c5c9fd6
commit df929bda38
1 changed files with 36 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
runtime/src/snapshot_utils.rs
View File

@ -28,15 +28,13 @@ use {
collections::HashSet,
fmt,
fs::{self, File},
io::{
self, BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write,
},
io::{BufReader, BufWriter, Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write},
path::{Path, PathBuf},
process::{self, ExitStatus},
process::ExitStatus,
str::FromStr,
sync::Arc,
},
tar::Archive,
tar::{self, Archive},
thiserror::Error,
};
@ -313,69 +311,46 @@ pub fn archive_snapshot_package(
let file_ext = get_archive_ext(snapshot_package.archive_format);
// Tar the staging directory into the archive at `archive_path`
//
// system `tar` program is used for -S (sparse file support)
let archive_path = tar_dir.join(format!(
"{}{}.{}",
TMP_SNAPSHOT_PREFIX, snapshot_package.slot, file_ext
));
let mut tar = process::Command::new("tar")
.args(&[
"chS",
"-C",
staging_dir.path().to_str().unwrap(),
"accounts",
"snapshots",
"version",
])
.stdin(process::Stdio::null())
.stdout(process::Stdio::piped())
.stderr(process::Stdio::inherit())
.spawn()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process spawn"))?;
{
let mut archive_file = fs::File::create(&archive_path)?;
match &mut tar.stdout {
None => {
return Err(SnapshotError::Io(IoError::new(
ErrorKind::Other,
"tar stdout unavailable".to_string(),
)));
}
Some(tar_output) => {
let mut archive_file = fs::File::create(&archive_path)?;
let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
let mut archive = tar::Builder::new(encoder);
for dir in &["accounts", "snapshots"] {
archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
}
archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
archive.into_inner()?;
Ok(())
};
match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => {
let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best);
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
ArchiveFormat::TarGzip => {
let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
ArchiveFormat::Tar => {
io::copy(tar_output, &mut archive_file)?;
}
ArchiveFormat::TarZstd => {
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
io::copy(tar_output, &mut encoder)?;
let _ = encoder.finish()?;
}
};
}
}
let tar_exit_status = tar
.wait()
.map_err(|e| SnapshotError::IoWithSource(e, "tar process wait"))?;
if !tar_exit_status.success() {
warn!("tar command failed with exit code: {}", tar_exit_status);
return Err(SnapshotError::ArchiveGenerationFailure(tar_exit_status));
match snapshot_package.archive_format {
ArchiveFormat::TarBzip2 => {
let mut encoder =
bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::Best);
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::TarGzip => {
let mut encoder =
flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::TarZstd => {
let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
do_archive_files(&mut encoder)?;
encoder.finish()?;
}
ArchiveFormat::Tar => {
do_archive_files(&mut archive_file)?;
}
};
}
// Atomically move the archive into position for other validators to find