Cap file size for snapshot data files (#7182)
* save limit deserialize * save * Save * Clean up * rustfmt * rustfmt * Just comment out to please CI * Fix ci... * Move code * Rustfmt * Crean up control flow * Add another comment * Introduce predetermined constant limit on snapshot data files (deserialize side) * Introduce predetermined constant limit on snapshot data files (serialize side) * rustfmt * Tweak message * Revert dynamic memory limit * Limit size of snapshot data file (de)serialization * Fix test breakage * Clean up * Fix uses formatting * Rename: deserialize_{for,from}_snapshot * Simplify comment * Use Slot * Provide slot for status cache * Align variable name with snapshot_status_cache_file_path * Define serialize_snapshot_data_file_with_metrics * Fix build....... * De-marco serialize_snapshot_data_file_with_metrics * Revert u64 => Slot
This commit is contained in:
@@ -20,6 +20,7 @@
|
||||
|
||||
use crate::accounts_index::AccountsIndex;
|
||||
use crate::append_vec::{AppendVec, StoredAccount, StoredMeta};
|
||||
use crate::bank::deserialize_from_snapshot;
|
||||
use bincode::{deserialize_from, serialize_into};
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use fs_extra::dir::CopyOptions;
|
||||
@@ -459,8 +460,8 @@ impl AccountsDB {
|
||||
) -> Result<(), IOError> {
|
||||
let _len: usize =
|
||||
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
|
||||
let storage: AccountStorage =
|
||||
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
|
||||
let storage: AccountStorage = deserialize_from_snapshot(&mut stream)
|
||||
.map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
|
||||
|
||||
// Remap the deserialized AppendVec paths to point to correct local paths
|
||||
let new_storage_map: Result<HashMap<Slot, SlotStores>, IOError> = storage
|
||||
|
@@ -59,6 +59,7 @@ use std::{
|
||||
};
|
||||
|
||||
pub const SECONDS_PER_YEAR: f64 = (365.25 * 24.0 * 60.0 * 60.0);
|
||||
pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
|
||||
|
||||
pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5;
|
||||
|
||||
@@ -1876,6 +1877,19 @@ pub fn goto_end_of_slot(bank: &mut Bank) {
|
||||
}
|
||||
}
|
||||
|
||||
// This guards against possible memory exhaustions in bincode when restoring
|
||||
// the full state from snapshot data files by imposing a fixed hard limit with
|
||||
// ample of headrooms for such a usecase.
|
||||
pub fn deserialize_from_snapshot<R, T>(reader: R) -> bincode::Result<T>
|
||||
where
|
||||
R: Read,
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
bincode::config()
|
||||
.limit(MAX_SNAPSHOT_DATA_FILE_SIZE)
|
||||
.deserialize_from(reader)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1887,7 +1901,7 @@ mod tests {
|
||||
},
|
||||
status_cache::MAX_CACHE_ENTRIES,
|
||||
};
|
||||
use bincode::{deserialize_from, serialize_into, serialized_size};
|
||||
use bincode::{serialize_into, serialized_size};
|
||||
use solana_sdk::instruction::AccountMeta;
|
||||
use solana_sdk::system_program::solana_system_program;
|
||||
use solana_sdk::{
|
||||
@@ -4318,7 +4332,7 @@ mod tests {
|
||||
serialize_into(&mut writer, &bank2.rc).unwrap();
|
||||
|
||||
let mut rdr = Cursor::new(&buf[..]);
|
||||
let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
|
||||
let mut dbank: Bank = deserialize_from_snapshot(&mut rdr).unwrap();
|
||||
let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
|
||||
|
||||
// Create a new set of directories for this bank's accounts
|
||||
|
Reference in New Issue
Block a user