Cap file size for snapshot data files (#7182)

* save limit deserialize

* save

* Save

* Clean up

* rustfmt

* rustfmt

* Just comment out to please CI

* Fix ci...

* Move code

* Rustfmt

* Crean up control flow

* Add another comment

* Introduce predetermined constant limit on snapshot data files (deserialize side)

* Introduce predetermined constant limit on snapshot data files (serialize side)

* rustfmt

* Tweak message

* Revert dynamic memory limit

* Limit size of snapshot data file (de)serialization

* Fix test breakage

* Clean up

* Fix uses formatting

* Rename: deserialize_{for,from}_snapshot

* Simplify comment

* Use Slot

* Provide slot for status cache

* Align variable name with snapshot_status_cache_file_path

* Define serialize_snapshot_data_file_with_metrics

* Fix build.......

* De-marco serialize_snapshot_data_file_with_metrics

* Revert u64 => Slot
This commit is contained in:
Ryo Onodera
2020-01-10 09:49:36 +09:00
committed by GitHub
parent 73c93cc345
commit 865c42465a
6 changed files with 412 additions and 91 deletions

View File

@@ -20,6 +20,7 @@
use crate::accounts_index::AccountsIndex;
use crate::append_vec::{AppendVec, StoredAccount, StoredMeta};
use crate::bank::deserialize_from_snapshot;
use bincode::{deserialize_from, serialize_into};
use byteorder::{ByteOrder, LittleEndian};
use fs_extra::dir::CopyOptions;
@@ -459,8 +460,8 @@ impl AccountsDB {
) -> Result<(), IOError> {
let _len: usize =
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
let storage: AccountStorage =
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
let storage: AccountStorage = deserialize_from_snapshot(&mut stream)
.map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
// Remap the deserialized AppendVec paths to point to correct local paths
let new_storage_map: Result<HashMap<Slot, SlotStores>, IOError> = storage

View File

@@ -59,6 +59,7 @@ use std::{
};
pub const SECONDS_PER_YEAR: f64 = (365.25 * 24.0 * 60.0 * 60.0);
pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5;
@@ -1876,6 +1877,19 @@ pub fn goto_end_of_slot(bank: &mut Bank) {
}
}
// This guards against possible memory exhaustions in bincode when restoring
// the full state from snapshot data files by imposing a fixed hard limit with
// ample of headrooms for such a usecase.
pub fn deserialize_from_snapshot<R, T>(reader: R) -> bincode::Result<T>
where
R: Read,
T: serde::de::DeserializeOwned,
{
bincode::config()
.limit(MAX_SNAPSHOT_DATA_FILE_SIZE)
.deserialize_from(reader)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1887,7 +1901,7 @@ mod tests {
},
status_cache::MAX_CACHE_ENTRIES,
};
use bincode::{deserialize_from, serialize_into, serialized_size};
use bincode::{serialize_into, serialized_size};
use solana_sdk::instruction::AccountMeta;
use solana_sdk::system_program::solana_system_program;
use solana_sdk::{
@@ -4318,7 +4332,7 @@ mod tests {
serialize_into(&mut writer, &bank2.rc).unwrap();
let mut rdr = Cursor::new(&buf[..]);
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
let mut dbank: Bank = deserialize_from_snapshot(&mut rdr).unwrap();
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
// Create a new set of directories for this bank's accounts