gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Pass around --max-genesis-archive-unpacked-size (#9161)

Browse Source 
automerge
This commit is contained in:
Ryo Onodera
2020-04-30 10:53:34 +09:00
committed by GitHub
parent a0514eb2ae
commit a91236012d
12 changed files with 208 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
ledger/src/blockstore.rs
View File

@ -10,6 +10,7 @@ use crate::{
blockstore_meta::*,
entry::{create_ticks, Entry},
erasure::ErasureConfig,
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
leader_schedule_cache::LeaderScheduleCache,
next_slots_iterator::NextSlotsIterator,
shred::{Result as ShredResult, Shred, Shredder},
@ -45,6 +46,7 @@ use std::{
cmp,
collections::HashMap,
fs,
io::{Error as IOError, ErrorKind},
path::{Path, PathBuf},
rc::Rc,
sync::{
@ -2622,7 +2624,11 @@ fn calculate_stake_weighted_timestamp(
// Creates a new ledger with slot 0 full of ticks (and only ticks).
//
// Returns the blockhash that can be used to append entries with.
pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) -> Result<Hash> {
pub fn create_new_ledger(
ledger_path: &Path,
genesis_config: &GenesisConfig,
max_genesis_archive_unpacked_size: u64,
) -> Result<Hash> {
Blockstore::destroy(ledger_path)?;
genesis_config.write(&ledger_path)?;
@ -2658,7 +2664,6 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) ->
.output()
.unwrap();
if !output.status.success() {
use std::io::{Error as IOError, ErrorKind};
use std::str::from_utf8;
error!("tar stdout: {}", from_utf8(&output.stdout).unwrap_or("?"));
error!("tar stderr: {}", from_utf8(&output.stderr).unwrap_or("?"));
@ -2672,6 +2677,54 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_config: &GenesisConfig) ->
)));
}
// ensure the genesis archive can be unpacked and it is under
// max_genesis_archive_unpacked_size, immedately after creating it above.
{
let temp_dir = tempfile::TempDir::new().unwrap();
// unpack into a temp dir, while completely discarding the unpacked files
let unpack_check = unpack_genesis_archive(
&archive_path,
&temp_dir.into_path(),
max_genesis_archive_unpacked_size,
);
if let Err(unpack_err) = unpack_check {
// stash problematic original archived genesis related files to
// examine them later and to prevent validator and ledger-tool from
// naively consuming them
let mut error_messages = String::new();
fs::rename(
&ledger_path.join("genesis.tar.bz2"),
ledger_path.join("genesis.tar.bz2.failed"),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic genesis.tar.bz2: {}", e)
});
fs::rename(
&ledger_path.join("genesis.bin"),
ledger_path.join("genesis.bin.failed"),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic genesis.bin: {}", e)
});
fs::rename(
&ledger_path.join("rocksdb"),
ledger_path.join("rocksdb.failed"),
)
.unwrap_or_else(|e| {
error_messages += &format!("/failed to stash problematic rocksdb: {}", e)
});
return Err(BlockstoreError::IO(IOError::new(
ErrorKind::Other,
format!(
"Error checking to unpack genesis archive: {}{}",
unpack_err, error_messages
),
)));
}
}
Ok(last_hash)
}
@ -2739,7 +2792,12 @@ pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> boo
// ticks)
pub fn create_new_ledger_from_name(name: &str, genesis_config: &GenesisConfig) -> (PathBuf, Hash) {
let ledger_path = get_ledger_path_from_name(name);
let blockhash = create_new_ledger(&ledger_path, genesis_config).unwrap();
let blockhash = create_new_ledger(
&ledger_path,
genesis_config,
MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
)
.unwrap();
(ledger_path, blockhash)
}

3
ledger/src/blockstore_db.rs
View File

@ -1,4 +1,4 @@
use crate::blockstore_meta;
use crate::{blockstore_meta, hardened_unpack::UnpackError};
use bincode::{deserialize, serialize};
use byteorder::{BigEndian, ByteOrder};
use log::*;
@ -55,6 +55,7 @@ pub enum BlockstoreError {
Serialize(#[from] Box<bincode::ErrorKind>),
FsExtraError(#[from] fs_extra::error::Error),
SlotCleanedUp,
UnpackError(#[from] UnpackError),
}
pub type Result<T> = std::result::Result<T, BlockstoreError>;

53
ledger/src/hardened_unpack.rs
View File

@ -19,9 +19,9 @@ use thiserror::Error;
#[derive(Error, Debug)]
pub enum UnpackError {
#[error("IO error")]
#[error("IO error: {0}")]
IO(#[from] std::io::Error),
#[error("Archive error")]
#[error("Archive error: {0}")]
Archive(String),
}
@ -29,15 +29,15 @@ pub type Result<T> = std::result::Result<T, UnpackError>;
const MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE: u64 = 500 * 1024 * 1024 * 1024; // 500 GiB
const MAX_SNAPSHOT_ARCHIVE_UNPACKED_COUNT: u64 = 500_000;
const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 1024 * 1024 * 1024; // 1024 MiB
pub const MAX_GENESIS_ARCHIVE_UNPACKED_SIZE: u64 = 10 * 1024 * 1024; // 10 MiB
const MAX_GENESIS_ARCHIVE_UNPACKED_COUNT: u64 = 100;
fn checked_total_size_sum(total_size: u64, entry_size: u64, limit_size: u64) -> Result<u64> {
let total_size = total_size.saturating_add(entry_size);
if total_size > limit_size {
return Err(UnpackError::Archive(format!(
"too large snapshot: {:?}",
total_size
"too large archive: {} than limit: {}",
total_size, limit_size,
)));
}
Ok(total_size)
@ -151,10 +151,18 @@ fn is_valid_snapshot_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool
}
}
pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig {
pub fn open_genesis_config(
ledger_path: &Path,
max_genesis_archive_unpacked_size: u64,
) -> GenesisConfig {
GenesisConfig::load(&ledger_path).unwrap_or_else(|load_err| {
let genesis_package = ledger_path.join("genesis.tar.bz2");
unpack_genesis_archive(&genesis_package, ledger_path).unwrap_or_else(|unpack_err| {
unpack_genesis_archive(
&genesis_package,
ledger_path,
max_genesis_archive_unpacked_size,
)
.unwrap_or_else(|unpack_err| {
warn!(
"Failed to open ledger genesis_config at {:?}: {}, {}",
ledger_path, load_err, unpack_err,
@ -170,17 +178,20 @@ pub fn open_genesis_config(ledger_path: &Path) -> GenesisConfig {
pub fn unpack_genesis_archive(
archive_filename: &Path,
destination_dir: &Path,
) -> std::result::Result<(), String> {
max_genesis_archive_unpacked_size: u64,
) -> std::result::Result<(), UnpackError> {
info!("Extracting {:?}...", archive_filename);
let extract_start = Instant::now();
fs::create_dir_all(destination_dir).map_err(|err| err.to_string())?;
let tar_bz2 = File::open(&archive_filename)
.map_err(|err| format!("Unable to open {:?}: {:?}", archive_filename, err))?;
fs::create_dir_all(destination_dir)?;
let tar_bz2 = File::open(&archive_filename)?;
let tar = BzDecoder::new(BufReader::new(tar_bz2));
let mut archive = Archive::new(tar);
unpack_genesis(&mut archive, destination_dir)
.map_err(|err| format!("Unable to unpack {:?}: {:?}", archive_filename, err))?;
unpack_genesis(
&mut archive,
destination_dir,
max_genesis_archive_unpacked_size,
)?;
info!(
"Extracted {:?} in {:?}",
archive_filename,
@ -189,11 +200,15 @@ pub fn unpack_genesis_archive(
Ok(())
}
fn unpack_genesis<A: Read, P: AsRef<Path>>(archive: &mut Archive<A>, unpack_dir: P) -> Result<()> {
fn unpack_genesis<A: Read, P: AsRef<Path>>(
archive: &mut Archive<A>,
unpack_dir: P,
max_genesis_archive_unpacked_size: u64,
) -> Result<()> {
unpack_archive(
archive,
unpack_dir,
MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
max_genesis_archive_unpacked_size,
MAX_GENESIS_ARCHIVE_UNPACKED_COUNT,
is_valid_genesis_archive_entry,
)
@ -311,7 +326,9 @@ mod tests {
}
fn finalize_and_unpack_genesis(archive: tar::Builder<Vec<u8>>) -> Result<()> {
with_finalize_and_unpack(archive, |a, b| unpack_genesis(a, b))
with_finalize_and_unpack(archive, |a, b| {
unpack_genesis(a, b, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)
})
}
#[test]
@ -440,7 +457,7 @@ mod tests {
let mut archive = Builder::new(Vec::new());
archive.append(&header, data).unwrap();
let result = finalize_and_unpack_snapshot(archive);
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 1125899906842624");
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 1125899906842624 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE));
}
#[test]
@ -456,7 +473,7 @@ mod tests {
let result =
checked_total_size_sum(u64::max_value() - 2, 2, MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE);
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == *"too large snapshot: 18446744073709551615");
assert_matches!(result, Err(UnpackError::Archive(ref message)) if message.to_string() == format!("too large archive: 18446744073709551615 than limit: {}", MAX_SNAPSHOT_ARCHIVE_UNPACKED_SIZE));
}
#[test]