Snapshot Packaging Service (#5262)

* Snapshot serialization and packaging

runtime/src/accounts_db.rs

@@ -36,6 +36,7 @@ use std::fmt;
 use std::fs::remove_dir_all;
 use std::io::{BufReader, Cursor, Error, ErrorKind, Read};
 use std::path::Path;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
 use sys_info;
@@ -81,8 +82,8 @@ pub type InstructionLoaders = Vec<Vec<(Pubkey, Account)>>;
 // Each fork has a set of storage entries.
 type ForkStores = HashMap<usize, Arc<AccountStorageEntry>>;
 
-#[derive(Default, Debug)]
-pub struct AccountStorage(HashMap<Fork, ForkStores>);
+#[derive(Clone, Default, Debug)]
+pub struct AccountStorage(pub HashMap<Fork, ForkStores>);
 
 struct AccountStorageVisitor;
 
@@ -122,11 +123,20 @@ impl Serialize for AccountStorage {
             len += storage.len();
         }
         let mut map = serializer.serialize_map(Some(len))?;
+        let mut count = 0;
+        let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
         for fork_storage in self.0.values() {
             for (storage_id, account_storage_entry) in fork_storage {
                 map.serialize_entry(storage_id, &**account_storage_entry)?;
+                count += 1;
             }
         }
+        serialize_account_storage_timer.stop();
+        datapoint_info!(
+            "serialize_account_storage_ms",
+            ("duration", serialize_account_storage_timer.as_ms(), i64),
+            ("num_entries", count, i64),
+        );
         map.end()
     }
 }
@@ -166,7 +176,7 @@ pub struct AccountStorageEntry {
 }
 
 impl AccountStorageEntry {
-    pub fn new(path: &str, fork_id: Fork, id: usize, file_size: u64) -> Self {
+    pub fn new(path: &Path, fork_id: Fork, id: usize, file_size: u64) -> Self {
         let tail = format!("{}.{}", fork_id, id);
         let path = Path::new(path).join(&tail);
         let accounts = AppendVec::new(&path, true, file_size as usize);
@@ -208,6 +218,14 @@ impl AccountStorageEntry {
         self.count_and_status.read().unwrap().0
     }
 
+    pub fn fork_id(&self) -> Fork {
+        self.fork_id
+    }
+
+    pub fn append_vec_id(&self) -> AppendVecId {
+        self.id
+    }
+
     fn add_account(&self) {
         let mut count_and_status = self.count_and_status.write().unwrap();
         *count_and_status = (count_and_status.0 + 1, count_and_status.1);
@@ -252,6 +270,10 @@ impl AccountStorageEntry {
         }
         count_and_status.0
     }
+
+    pub fn get_path(&self) -> PathBuf {
+        self.accounts.get_path()
+    }
 }
 
 pub fn get_paths_vec(paths: &str) -> Vec<String> {
@@ -409,7 +431,7 @@ impl AccountsDB {
         Ok(())
     }
 
-    fn new_storage_entry(&self, fork_id: Fork, path: &str, size: u64) -> AccountStorageEntry {
+    fn new_storage_entry(&self, fork_id: Fork, path: &Path, size: u64) -> AccountStorageEntry {
         AccountStorageEntry::new(
             path,
             fork_id,
@@ -568,7 +590,7 @@ impl AccountsDB {
     ) -> Arc<AccountStorageEntry> {
         let paths = self.paths.read().unwrap();
         let path_index = thread_rng().gen_range(0, paths.len());
-        let store = Arc::new(self.new_storage_entry(fork_id, &paths[path_index], size));
+        let store = Arc::new(self.new_storage_entry(fork_id, &Path::new(&paths[path_index]), size));
         fork_storage.insert(store.id, store.clone());
         store
     }

runtime/src/append_vec.rs

@@ -235,6 +235,10 @@ impl AppendVec {
         Some((meta, stored.0.clone_account()))
     }
 
+    pub fn get_path(&self) -> PathBuf {
+        self.path.clone()
+    }
+
     pub fn accounts<'a>(&'a self, mut start: usize) -> Vec<StoredAccount<'a>> {
         let mut accounts = vec![];
         while let Some((account, next)) = self.get_account(start) {

runtime/src/bank.rs

@@ -3,6 +3,7 @@
 //! on behalf of the caller, and a low-level API for when they have
 //! already been signed and verified.
 use crate::accounts::Accounts;
+use crate::accounts_db::AccountStorageEntry;
 use crate::accounts_db::{
     AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
 };
@@ -83,6 +84,15 @@ impl BankRc {
         self.accounts.update_from_stream(stream)
     }
 
+    pub fn get_storage_entries(&self) -> Vec<Arc<AccountStorageEntry>> {
+        let r_storage = self.accounts.accounts_db.storage.read().unwrap();
+        r_storage
+            .0
+            .values()
+            .flat_map(|fork_store| fork_store.values().cloned())
+            .collect()
+    }
+
     fn get_io_error(error: &str) -> std::io::Error {
         warn!("BankRc error: {:?}", error);
         std::io::Error::new(std::io::ErrorKind::Other, error)
@@ -167,6 +177,10 @@ impl StatusCacheRc {
         let sc = status_cache_rc.status_cache.write().unwrap();
         self.status_cache.write().unwrap().append(&sc);
     }
+
+    pub fn purge_roots(&self) {
+        self.status_cache.write().unwrap().purge_roots();
+    }
 }
 
 /// Manager for the state of all accounts and programs after processing its entries.

runtime/src/status_cache.rs

@@ -6,7 +6,7 @@ use solana_sdk::hash::Hash;
 use solana_sdk::signature::Signature;
 use std::collections::{HashMap, HashSet};
 
-const MAX_CACHE_ENTRIES: usize = solana_sdk::timing::MAX_HASH_AGE_IN_SECONDS;
+pub const MAX_CACHE_ENTRIES: usize = solana_sdk::timing::MAX_HASH_AGE_IN_SECONDS;
 const CACHED_SIGNATURE_SIZE: usize = 20;
 
 // Store forks in a single chunk of memory to avoid another lookup.
@@ -104,14 +104,7 @@ impl<T: Serialize + Clone> StatusCache<T> {
     /// After MAX_CACHE_ENTRIES, roots are removed, and any old signatures are cleared.
     pub fn add_root(&mut self, fork: ForkId) {
         self.roots.insert(fork);
-        if self.roots.len() > MAX_CACHE_ENTRIES {
-            if let Some(min) = self.roots.iter().min().cloned() {
-                self.roots.remove(&min);
-                for cache in self.cache.iter_mut() {
-                    cache.retain(|_, (fork, _, _)| *fork > min);
-                }
-            }
-        }
+        self.purge_roots();
     }
 
     /// Insert a new signature for a specific fork.
@@ -136,6 +129,17 @@ impl<T: Serialize + Clone> StatusCache<T> {
         sig_forks.push((fork, res));
     }
 
+    pub fn purge_roots(&mut self) {
+        if self.roots.len() > MAX_CACHE_ENTRIES {
+            if let Some(min) = self.roots.iter().min().cloned() {
+                self.roots.remove(&min);
+                for cache in self.cache.iter_mut() {
+                    cache.retain(|_, (fork, _, _)| *fork > min);
+                }
+            }
+        }
+    }
+
     fn insert_entry(
         &mut self,
         transaction_blockhash: &Hash,