diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 5b49723470..475e0c5fa8 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -349,6 +349,7 @@ mod tests { snapshots_dir, accounts_dir, archive_format, + snapshot_utils::VerifyBank::Deterministic, ); } } diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 3a7e70a7d3..a578fe2802 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -558,6 +558,7 @@ mod tests { saved_snapshots_dir.path(), saved_accounts_dir.path(), ArchiveFormat::TarBzip2, + snapshot_utils::VerifyBank::NonDeterministic(saved_slot), ); } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 2c467c1978..28e36b0e4a 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -894,7 +894,7 @@ impl NonceInfo for NonceFull { // Bank's common fields shared by all supported snapshot versions for deserialization. // Sync fields with BankFieldsToSerialize! This is paired with it. // All members are made public to remain Bank's members private and to make versioned deserializer workable on this -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq)] pub(crate) struct BankFieldsToDeserialize { pub(crate) blockhash_queue: BlockhashQueue, pub(crate) ancestors: AncestorsForSerialization, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 956af7a836..33a368da3f 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -63,7 +63,7 @@ pub(crate) enum SerdeStyle { const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; -#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample, PartialEq)] struct AccountsDbFields( HashMap>, StoredMetaWriteVersion, @@ -137,7 +137,7 @@ impl SnapshotAccountsDbFields { } } -trait TypeContext<'a> { +trait TypeContext<'a>: PartialEq { type SerializableAccountStorageEntry: Serialize + DeserializeOwned + From<&'a AccountStorageEntry> @@ -189,6 +189,23 @@ where .deserialize_from::(reader) } +/// used by tests to compare contents of serialized bank fields +/// serialized format is not deterministic - likely due to randomness in structs like hashmaps +pub(crate) fn compare_two_serialized_banks( + path1: impl AsRef, + path2: impl AsRef, +) -> std::result::Result { + use std::fs::File; + let file1 = File::open(path1)?; + let mut stream1 = BufReader::new(file1); + let file2 = File::open(path2)?; + let mut stream2 = BufReader::new(file2); + + let fields1 = newer::Context::deserialize_bank_fields(&mut stream1)?; + let fields2 = newer::Context::deserialize_bank_fields(&mut stream2)?; + Ok(fields1 == fields2) +} + #[allow(clippy::too_many_arguments)] pub(crate) fn bank_from_streams( serde_style: SerdeStyle, diff --git a/runtime/src/serde_snapshot/newer.rs b/runtime/src/serde_snapshot/newer.rs index 4d486f128a..f54b11bede 100644 --- a/runtime/src/serde_snapshot/newer.rs +++ b/runtime/src/serde_snapshot/newer.rs @@ -176,6 +176,7 @@ impl<'a> From> for SerializableVersionedB #[cfg(RUSTC_WITH_SPECIALIZATION)] impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {} +#[derive(PartialEq)] pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 4a701a0a3d..4369b099bb 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1649,11 +1649,22 @@ fn get_io_error(error: &str) -> SnapshotError { SnapshotError::Io(IoError::new(ErrorKind::Other, error)) } +#[derive(Debug, Copy, Clone)] +/// allow tests to specify what happened to the serialized format +pub enum VerifyBank { + /// the bank's serialized format is expected to be identical to what we are comparing against + Deterministic, + /// the serialized bank was 'reserialized' into a non-deterministic format at the specified slot + /// so, deserialize both files and compare deserialized results + NonDeterministic(Slot), +} + pub fn verify_snapshot_archive( snapshot_archive: P, snapshots_to_verify: Q, storages_to_verify: R, archive_format: ArchiveFormat, + verify_bank: VerifyBank, ) where P: AsRef, Q: AsRef, @@ -1672,6 +1683,17 @@ pub fn verify_snapshot_archive( // Check snapshots are the same let unpacked_snapshots = unpack_dir.join("snapshots"); + if let VerifyBank::NonDeterministic(slot) = verify_bank { + // file contents may be different, but deserialized structs should be equal + let slot = slot.to_string(); + let p1 = snapshots_to_verify.as_ref().join(&slot).join(&slot); + let p2 = unpacked_snapshots.join(&slot).join(&slot); + + assert!(crate::serde_snapshot::compare_two_serialized_banks(&p1, &p2).unwrap()); + std::fs::remove_file(p1).unwrap(); + std::fs::remove_file(p2).unwrap(); + } + assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap()); // Check the account entries are the same