Introduce automatic ABI maintenance mechanism (2/2; rollout) (#8012)

* Introduce automatic ABI maintenance mechanism (2/2; rollout)

* Fix stable clippy

* Change to symlink

* Freeze abi of Tower

* fmt...

* Improve dev-experience!

* Update BankSlotDelta

$ diff -u /tmp/abi8/*7dg6BreYxTuxiVz6aLvk3p2Z7GQk2cJqfGvC9h4FAoSj* /tmp/abi8/*9chBcbXVJ4fK7uGgydQzam5aHipaAKFw6V4LDFpjbE4w*
--- /tmp/abi8/bank__BankSlotDelta_frozen_abi__test_abi_digest_7dg6BreYxTuxiVz6aLvk3p2Z7GQk2cJqfGvC9h4FAoSj      2020-06-18 18:01:22.831228087 +0900
+++ /tmp/abi8/bank__BankSlotDelta_frozen_abi__test_abi_digest_9chBcbXVJ4fK7uGgydQzam5aHipaAKFw6V4LDFpjbE4w      2020-07-03 15:59:58.430695244 +0900
@@ -140,7 +140,7 @@
                                                         field u8
                                                             primitive u8
                                                         field solana_sdk::instruction::InstructionError
-                                                            enum InstructionError (variants = 34)
+                                                            enum InstructionError (variants = 35)
                                                                 variant(0) GenericError (unit)
                                                                 variant(1) InvalidArgument (unit)
                                                                 variant(2) InvalidInstructionData (unit)
@@ -176,6 +176,7 @@
                                                                 variant(31) CallDepth (unit)
                                                                 variant(32) MissingAccount (unit)
                                                                 variant(33) ReentrancyNotAllowed (unit)
+                                                                variant(34) MaxSeedLengthExceeded (unit)
                                                     variant(9) CallChainTooDeep (unit)
                                                     variant(10) MissingSignatureForFee (unit)
                                                     variant(11) InvalidAccountIndex (unit)

* Fix some merge conflicts...

runtime/Cargo.toml

@@ -37,6 +37,7 @@ solana-measure = { path = "../measure", version = "1.3.0" }
 solana-metrics = { path = "../metrics", version = "1.3.0" }
 solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.0" }
 solana-sdk = { path = "../sdk", version = "1.3.0" }
+solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.3.0" }
 solana-stake-program = { path = "../programs/stake", version = "1.3.0" }
 solana-vote-program = { path = "../programs/vote", version = "1.3.0" }
 symlink = "0.1.0"
@@ -55,3 +56,6 @@ solana-noop-program = { path = "../programs/noop", version = "1.3.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
+
+[build-dependencies]
+rustc_version = "0.2"

runtime/build.rs

@@ -0,0 +1 @@
+../sdk/build.rs

runtime/src/accounts.rs

@@ -31,13 +31,13 @@ use std::{
     sync::{Arc, Mutex, RwLock},
 };
 
-#[derive(Default, Debug)]
+#[derive(Default, Debug, AbiExample)]
 pub(crate) struct ReadonlyLock {
     lock_count: Mutex<u64>,
 }
 
 /// This structure handles synchronization for db
-#[derive(Default, Debug)]
+#[derive(Default, Debug, AbiExample)]
 pub struct Accounts {
     /// my slot
     pub slot: Slot,

runtime/src/accounts_db.rs

@@ -132,7 +132,7 @@ impl AccountStorage {
     }
 }
 
-#[derive(Debug, Eq, PartialEq, Copy, Clone, Deserialize, Serialize)]
+#[derive(Debug, Eq, PartialEq, Copy, Clone, Deserialize, Serialize, AbiExample, AbiEnumVisitor)]
 pub enum AccountStorageStatus {
     Available = 0,
     Full = 1,
@@ -324,7 +324,7 @@ pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, Vec<PathBu
     Ok((temp_dirs, paths))
 }
 
-#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, AbiExample)]
 pub struct BankHashStats {
     pub num_updated_accounts: u64,
     pub num_removed_accounts: u64,
@@ -358,7 +358,7 @@ impl BankHashStats {
     }
 }
 
-#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, AbiExample)]
 pub struct BankHashInfo {
     pub hash: Hash,
     pub snapshot_hash: Hash,
@@ -431,6 +431,21 @@ fn make_min_priority_thread_pool() -> ThreadPool {
         .unwrap()
 }
 
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+impl solana_sdk::abi_example::AbiExample for AccountsDB {
+    fn example() -> Self {
+        let accounts_db = AccountsDB::new_single();
+        let key = Pubkey::default();
+        let some_data_len = 5;
+        let some_slot: Slot = 0;
+        let account = Account::new(1, some_data_len, &key);
+        accounts_db.store(some_slot, &[(&key, &account)]);
+        accounts_db.add_root(0);
+
+        accounts_db
+    }
+}
+
 impl Default for AccountsDB {
     fn default() -> Self {
         let num_threads = get_thread_count();

runtime/src/append_vec.rs

@@ -100,7 +100,7 @@ impl<'a> StoredAccount<'a> {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, AbiExample)]
 #[allow(clippy::mutex_atomic)]
 pub struct AppendVec {
     path: PathBuf,

runtime/src/bank.rs

@@ -74,6 +74,7 @@ pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0;
 pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5;
 
 type BankStatusCache = StatusCache<Result<()>>;
+#[frozen_abi(digest = "9chBcbXVJ4fK7uGgydQzam5aHipaAKFw6V4LDFpjbE4w")]
 pub type BankSlotDelta = SlotDelta<Result<()>>;
 type TransactionAccountRefCells = Vec<Rc<RefCell<Account>>>;
 type TransactionLoaderRefCells = Vec<Vec<(Pubkey, RefCell<Account>)>>;
@@ -107,6 +108,20 @@ pub struct BankRc {
     pub(crate) slot: Slot,
 }
 
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+use solana_sdk::abi_example::AbiExample;
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+impl AbiExample for BankRc {
+    fn example() -> Self {
+        BankRc {
+            // Set parent to None to cut the recursion into another Bank
+            parent: RwLock::new(None),
+            accounts: AbiExample::example(),
+            slot: AbiExample::example(),
+        }
+    }
+}
+
 impl BankRc {
     pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self {
         Self {
@@ -121,7 +136,7 @@ impl BankRc {
     }
 }
 
-#[derive(Default)]
+#[derive(Default, AbiExample)]
 pub struct StatusCacheRc {
     /// where all the Accounts are stored
     /// A cache of signature statuses
@@ -188,7 +203,7 @@ impl HashAgeKind {
     }
 }
 
-#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize)]
+#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
 struct UnusedAccounts {
     unused1: HashSet<Pubkey>,
     unused2: HashSet<Pubkey>,
@@ -196,7 +211,8 @@ struct UnusedAccounts {
 }
 
 /// Manager for the state of all accounts and programs after processing its entries.
-#[derive(Default, Deserialize, Serialize)]
+#[frozen_abi(digest = "GsvisJfTaHxmTX4s6gQs2bZtxVggB7F325XDdXTA573p")]
+#[derive(Default, Deserialize, Serialize, AbiExample)]
 pub struct Bank {
     /// References to accounts, parent and signature status
     #[serde(skip)]

runtime/src/blockhash_queue.rs

@@ -4,7 +4,7 @@ use solana_sdk::{
 };
 use std::collections::HashMap;
 
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, AbiExample)]
 struct HashAge {
     fee_calculator: FeeCalculator,
     hash_height: u64,
@@ -12,7 +12,8 @@ struct HashAge {
 }
 
 /// Low memory overhead, so can be cloned for every checkpoint
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[frozen_abi(digest = "EwaoLA34VN18E95GvjmkeStUpWqTeBd7FGpd7mppLmEw")]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, AbiExample)]
 pub struct BlockhashQueue {
     /// updated whenever an hash is registered
     hash_height: u64,

runtime/src/bloom.rs

@@ -11,7 +11,7 @@ pub trait BloomHashIndex {
     fn hash_at_index(&self, hash_index: u64) -> u64;
 }
 
-#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq, AbiExample)]
 pub struct Bloom<T: BloomHashIndex> {
     pub keys: Vec<u64>,
     pub bits: BitVec<u64>,

runtime/src/epoch_stakes.rs

@@ -7,13 +7,13 @@ use std::{collections::HashMap, sync::Arc};
 pub type NodeIdToVoteAccounts = HashMap<Pubkey, NodeVoteAccounts>;
 pub type EpochAuthorizedVoters = HashMap<Pubkey, Pubkey>;
 
-#[derive(Clone, Serialize, Debug, Deserialize, Default, PartialEq)]
+#[derive(Clone, Serialize, Debug, Deserialize, Default, PartialEq, AbiExample)]
 pub struct NodeVoteAccounts {
     pub vote_accounts: Vec<Pubkey>,
     pub total_stake: u64,
 }
 
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize, AbiExample)]
 pub struct EpochStakes {
     stakes: Arc<Stakes>,
     total_stake: u64,

runtime/src/lib.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
 pub mod accounts;
 pub mod accounts_db;
 pub mod accounts_index;
@@ -38,5 +39,8 @@ extern crate solana_metrics;
 #[macro_use]
 extern crate serde_derive;
 
+#[macro_use]
+extern crate solana_sdk_macro_frozen_abi;
+
 extern crate fs_extra;
 extern crate tempfile;

runtime/src/message_processor.rs

@@ -262,6 +262,16 @@ impl Clone for MessageProcessor {
         }
     }
 }
+
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+impl ::solana_sdk::abi_example::AbiExample for MessageProcessor {
+    fn example() -> Self {
+        // MessageProcessor's fields are #[serde(skip)]-ed and not Serialize
+        // so, just rely on Default anyway.
+        MessageProcessor::default()
+    }
+}
+
 impl MessageProcessor {
     /// Add a static entrypoint to intercept instructions before the dynamic loader.
     pub fn add_program(&mut self, program_id: Pubkey, process_instruction: ProcessInstruction) {

runtime/src/rent_collector.rs

@@ -4,7 +4,7 @@ use solana_sdk::{
     rent::Rent, sysvar,
 };
 
-#[derive(Default, Serialize, Deserialize, Clone)]
+#[derive(Default, Serialize, Deserialize, Clone, AbiExample)]
 pub struct RentCollector {
     pub epoch: Epoch,
     pub epoch_schedule: EpochSchedule,

runtime/src/serde_snapshot/future.rs

@@ -7,6 +7,9 @@ pub(super) struct SerializableAccountStorageEntry {
     accounts_current_len: usize,
 }
 
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+impl solana_sdk::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {}
+
 impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
     fn from(rhs: &AccountStorageEntry) -> Self {
         Self {

runtime/src/serde_snapshot/legacy.rs

@@ -1,3 +1,5 @@
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+use solana_sdk::abi_example::IgnoreAsHelper;
 use {super::*, solana_measure::measure::Measure, std::cell::RefCell};
 
 // Serializable version of AccountStorageEntry for snapshot format
@@ -8,6 +10,9 @@ pub(super) struct SerializableAccountStorageEntry {
     count_and_status: (usize, AccountStorageStatus),
 }
 
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+impl IgnoreAsHelper for SerializableAccountStorageEntry {}
+
 impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
     fn from(rhs: &AccountStorageEntry) -> Self {
         Self {
@@ -30,6 +35,9 @@ struct SerializableAppendVec {
     current_len: usize,
 }
 
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+impl IgnoreAsHelper for SerializableAppendVec {}
+
 impl From<&AppendVec> for SerializableAppendVec {
     fn from(rhs: &AppendVec) -> SerializableAppendVec {
         SerializableAppendVec {

runtime/src/serde_snapshot/tests.rs

@@ -278,3 +278,50 @@ fn test_bank_serialize_newer() {
 fn test_bank_serialize_older() {
     test_bank_serialize_style(SerdeStyle::OLDER)
 }
+
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+mod test_bank_rc_serialize {
+    use super::*;
+
+    // These some what long test harness is required to freeze the ABI of
+    // BankRc's serialization due to versioned nature
+    #[frozen_abi(digest = "HfCP74JKqPdeAccNJEj7KEoNxtsmX3zRqc2rpTy1NC7H")]
+    #[derive(Serialize, AbiExample)]
+    pub struct BandRcAbiTestWrapperFuture {
+        #[serde(serialize_with = "wrapper_future")]
+        bank_rc: BankRc,
+    }
+
+    pub fn wrapper_future<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
+        (SerializableBankRc::<future::Context> {
+            bank_rc,
+            snapshot_storages: &snapshot_storages,
+            phantom: std::marker::PhantomData::default(),
+        })
+        .serialize(s)
+    }
+
+    #[frozen_abi(digest = "43niyekyWwreLALcdEeFFpd7h8U6pgSXGqfKBRw8H7Vy")]
+    #[derive(Serialize, AbiExample)]
+    pub struct BandRcAbiTestWrapperLegacy {
+        #[serde(serialize_with = "wrapper_legacy")]
+        bank_rc: BankRc,
+    }
+
+    pub fn wrapper_legacy<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
+        (SerializableBankRc::<legacy::Context> {
+            bank_rc,
+            snapshot_storages: &snapshot_storages,
+            phantom: std::marker::PhantomData::default(),
+        })
+        .serialize(s)
+    }
+}

runtime/src/serde_snapshot/utils.rs

@@ -2,6 +2,8 @@ use serde::{
     ser::{SerializeSeq, SerializeTuple},
     Serialize, Serializer,
 };
+#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+use solana_sdk::abi_example::IgnoreAsHelper;
 
 // consumes an iterator and returns an object that will serialize as a serde seq
 #[allow(dead_code)]
@@ -15,6 +17,9 @@ where
         iter: std::cell::RefCell<Option<I>>,
     }
 
+    #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+    impl<I> IgnoreAsHelper for SerializableSequencedIterator<I> {}
+
     impl<I> Serialize for SerializableSequencedIterator<I>
     where
         I: IntoIterator,
@@ -51,6 +56,9 @@ where
         iter: std::cell::RefCell<Option<I>>,
     }
 
+    #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+    impl<I> IgnoreAsHelper for SerializableSequencedIterator<I> {}
+
     impl<I> Serialize for SerializableSequencedIterator<I>
     where
         I: IntoIterator,
@@ -87,6 +95,9 @@ where
         iter: std::cell::RefCell<Option<I>>,
     }
 
+    #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
+    impl<I> IgnoreAsHelper for SerializableMappedIterator<I> {}
+
     impl<K, V, I> Serialize for SerializableMappedIterator<I>
     where
         K: Serialize,

runtime/src/stakes.rs

@@ -7,7 +7,7 @@ use solana_stake_program::stake_state::{new_stake_history_entry, Delegation, Sta
 use solana_vote_program::vote_state::VoteState;
 use std::collections::HashMap;
 
-#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize)]
+#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
 pub struct Stakes {
     /// vote accounts
     vote_accounts: HashMap<Pubkey, (u64, Account)>,

runtime/src/status_cache.rs

@@ -41,7 +41,7 @@ pub struct SignatureConfirmationStatus<T> {
     pub status: T,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, AbiExample)]
 pub struct StatusCache<T: Serialize + Clone> {
     cache: StatusMap<T>,
     roots: HashSet<Slot>,