Create snapshots sparsely (#4815) (#4816)

(cherry picked from commit c5e6ebb496)

.appveyor.yml

@@ -4,7 +4,7 @@ version: '{build}'
 branches:
   only:
   - master
-  - /v[0-9.]+/
+  - /^v[0-9.]+/
 
 cache:
   - '%USERPROFILE%\.cargo'

.travis.yml

@@ -17,7 +17,7 @@ script:
 branches:
   only:
     - master
-    - /v.*/
+    - /^v\d+\.\d+(\.\d+)?(-\S*)?$/
 
 notifications:
   slack:

Cargo.lock

@@ -2727,6 +2727,7 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-derive 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-logger 0.16.0",

core/src/bank_forks.rs

@@ -329,8 +329,9 @@ impl BankForks {
         names.sort();
         let mut bank_maps = vec![];
         let status_cache_rc = StatusCacheRc::default();
+        let id = (names[names.len() - 1] + 1) as usize;
         let mut bank0 =
-            Bank::create_with_genesis(&genesis_block, account_paths.clone(), &status_cache_rc);
+            Bank::create_with_genesis(&genesis_block, account_paths.clone(), &status_cache_rc, id);
         bank0.freeze();
         let bank_root = BankForks::load_snapshots(
             &names,

core/src/replicator.rs

@@ -23,7 +23,6 @@ use solana_sdk::account_utils::State;
 use solana_sdk::client::{AsyncClient, SyncClient};
 use solana_sdk::hash::{Hash, Hasher};
 use solana_sdk::message::Message;
-use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
 use solana_sdk::timing::timestamp;
 use solana_sdk::transaction::Transaction;
@@ -303,7 +302,7 @@ impl Replicator {
         })
     }
 
-    pub fn run(&mut self, mining_pool_pubkey: Pubkey) {
+    pub fn run(&mut self) {
         info!("waiting for ledger download");
         self.thread_handles.pop().unwrap().join().unwrap();
         self.encrypt_ledger()
@@ -330,11 +329,11 @@ impl Replicator {
                     }
                 };
             self.blockhash = storage_blockhash;
-            self.redeem_rewards(&mining_pool_pubkey);
+            self.redeem_rewards();
         }
     }
 
-    fn redeem_rewards(&self, mining_pool_pubkey: &Pubkey) {
+    fn redeem_rewards(&self) {
         let nodes = self.cluster_info.read().unwrap().tvu_peers();
         let client = crate::gossip_service::get_client(&nodes);
 
@@ -347,7 +346,6 @@ impl Replicator {
                     let ix = storage_instruction::claim_reward(
                         &self.keypair.pubkey(),
                         &self.storage_keypair.pubkey(),
-                        mining_pool_pubkey,
                     );
                     let message = Message::new_with_payer(vec![ix], Some(&self.keypair.pubkey()));
                     if let Err(e) = client.send_message(&[&self.keypair], message) {

core/tests/local_cluster.rs

@@ -118,6 +118,7 @@ fn test_leader_failure_4() {
     );
 }
 #[test]
+#[ignore]
 fn test_two_unbalanced_stakes() {
     solana_logger::setup();
     let mut validator_config = ValidatorConfig::default();

genesis/src/main.rs

@@ -147,14 +147,6 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 .required(true)
                 .help("Path to file containing the bootstrap leader's storage keypair"),
         )
-        .arg(
-            Arg::with_name("storage_mining_pool_lamports")
-                .long("storage-mining-pool-lamports")
-                .value_name("LAMPORTS")
-                .takes_value(true)
-                .required(true)
-                .help("Number of lamports to assign to the storage mining pool"),
-        )
         .arg(
             Arg::with_name("bootstrap_leader_lamports")
                 .long("bootstrap-leader-lamports")
@@ -261,7 +253,6 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     let bootstrap_leader_lamports = value_t_or_exit!(matches, "bootstrap_leader_lamports", u64);
     let bootstrap_leader_stake_lamports =
         value_t_or_exit!(matches, "bootstrap_leader_stake_lamports", u64);
-    let storage_pool_lamports = value_t_or_exit!(matches, "storage_mining_pool_lamports", u64);
 
     let bootstrap_leader_keypair = read_keypair(bootstrap_leader_keypair_file)?;
     let bootstrap_vote_keypair = read_keypair(bootstrap_vote_keypair_file)?;
@@ -306,12 +297,6 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     1,
                 ),
             ),
-            (
-                "StorageMiningPoo111111111111111111111111111"
-                    .parse()
-                    .unwrap(),
-                storage_contract::create_mining_pool_account(storage_pool_lamports),
-            ),
         ])
         .native_instruction_processors(&[
             solana_bpf_loader_program!(),
@@ -524,6 +509,8 @@ mod tests {
         )
         .expect("builder");
 
+        builder = solana_storage_api::rewards_pools::genesis(builder);
+
         remove_file(path).unwrap();
 
         let genesis_block = builder.clone().build();

multinode-demo/fullnode.sh

@@ -490,7 +490,7 @@ while true; do
           rm -rf "$new_state_dir" "$new_state_archive"
           cp -a "$state_dir" "$new_state_dir"
           cd "$new_state_dir"
-          tar zcf "$new_state_archive" ./*
+          tar zcfS "$new_state_archive" ./*
         )
         ln -f "$new_state_archive" "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz
         rm -rf "$new_state_dir" "$new_state_archive"

multinode-demo/setup.sh

@@ -23,7 +23,6 @@ default_arg --ledger "$SOLANA_RSYNC_CONFIG_DIR"/ledger
 default_arg --mint "$SOLANA_CONFIG_DIR"/mint-keypair.json
 default_arg --lamports 100000000000000
 default_arg --bootstrap-leader-lamports 424242
-default_arg --storage-mining-pool-lamports 100000000
 default_arg --target-lamports-per-signature 42
 default_arg --target-signatures-per-slot 42
 default_arg --hashes-per-tick auto

net/remote/remote-node.sh

@@ -224,7 +224,7 @@ local|tar)
         ./multinode-demo/drone.sh > drone.log 2>&1 &
       fi
       export BLOCKEXPLORER_GEOIP_WHITELIST=$PWD/net/config/geoip.yml
-      npm install @solana/blockexplorer@1
+      npm install @solana/blockexplorer@1.10.3
       npx solana-blockexplorer > blockexplorer.log 2>&1 &
 
       # Confirm the blockexplorer is accessible

net/scripts/add-solana-user-authorized_keys.sh

@@ -14,6 +14,7 @@ set -ex
 # 2. Inline ~/.ssh/id-solana-testnet.pub below
 cat > /solana-authorized_keys <<EOF
 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFBNwLw0i+rI312gWshojFlNw9NV7WfaKeeUsYADqOvM2o4yrO2pPw+sgW8W+/rPpVyH7zU9WVRgTME8NgFV1Vc=
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGqZAwAZeBl0buOMz4FpUYrtpwk1L5aGKlbd7lI8dpbSx5WVRPWCVKhWzsGMtDUIfmozdzJouk1LPyihghTDgsE=
 EOF
 
 sudo -u solana bash -c "

programs/stake_api/src/stake_instruction.rs

@@ -31,6 +31,24 @@ pub enum StakeInstruction {
     ///    2 - RewardsPool Stake Account from which to redeem credits
     ///    3 - Rewards syscall Account that carries points values
     RedeemVoteCredits,
+
+    /// Withdraw unstaked lamports from the stake account
+    ///
+    /// Expects 3 Accounts:
+    ///    0 - Delegate StakeAccount
+    ///    1 - System account to which the lamports will be transferred,
+    ///    2 - Syscall Account that carries epoch
+    ///
+    /// The u64 is the portion of the Stake account balance to be withdrawn,
+    ///    must be <= StakeAccount.lamports - staked lamports
+    Withdraw(u64),
+
+    /// Deactivates the stake in the account
+    ///
+    /// Expects 2 Accounts:
+    ///    0 - Delegate StakeAccount
+    ///    1 - Syscall Account that carries epoch
+    Deactivate,
 }
 
 pub fn create_stake_account(
@@ -77,6 +95,23 @@ pub fn delegate_stake(stake_pubkey: &Pubkey, vote_pubkey: &Pubkey, stake: u64) -
     Instruction::new(id(), &StakeInstruction::DelegateStake(stake), account_metas)
 }
 
+pub fn withdraw(stake_pubkey: &Pubkey, to_pubkey: &Pubkey, lamports: u64) -> Instruction {
+    let account_metas = vec![
+        AccountMeta::new(*stake_pubkey, true),
+        AccountMeta::new(*to_pubkey, false),
+        AccountMeta::new(syscall::current::id(), false),
+    ];
+    Instruction::new(id(), &StakeInstruction::Withdraw(lamports), account_metas)
+}
+
+pub fn deactivate_stake(stake_pubkey: &Pubkey) -> Instruction {
+    let account_metas = vec![
+        AccountMeta::new(*stake_pubkey, true),
+        AccountMeta::new(syscall::current::id(), false),
+    ];
+    Instruction::new(id(), &StakeInstruction::Deactivate, account_metas)
+}
+
 pub fn process_instruction(
     _program_id: &Pubkey,
     keyed_accounts: &mut [KeyedAccount],
@@ -123,6 +158,27 @@ pub fn process_instruction(
                 &syscall::rewards::from_keyed_account(&rest[0])?,
             )
         }
+        StakeInstruction::Withdraw(lamports) => {
+            if rest.len() != 2 {
+                Err(InstructionError::InvalidInstructionData)?;
+            }
+            let (to, syscall) = &mut rest.split_at_mut(1);
+            let mut to = &mut to[0];
+
+            me.withdraw(
+                lamports,
+                &mut to,
+                &syscall::current::from_keyed_account(&syscall[0])?,
+            )
+        }
+        StakeInstruction::Deactivate => {
+            if rest.len() != 1 {
+                Err(InstructionError::InvalidInstructionData)?;
+            }
+            let syscall = &rest[0];
+
+            me.deactivate_stake(&syscall::current::from_keyed_account(&syscall)?)
+        }
     }
 }
 
@@ -168,6 +224,14 @@ mod tests {
             process_instruction(&delegate_stake(&Pubkey::default(), &Pubkey::default(), 0)),
             Err(InstructionError::InvalidAccountData),
         );
+        assert_eq!(
+            process_instruction(&withdraw(&Pubkey::default(), &Pubkey::new_rand(), 100)),
+            Err(InstructionError::InvalidAccountData),
+        );
+        assert_eq!(
+            process_instruction(&deactivate_stake(&Pubkey::default())),
+            Err(InstructionError::InvalidAccountData),
+        );
     }
 
     #[test]
@@ -250,6 +314,76 @@ mod tests {
             ),
             Err(InstructionError::InvalidAccountData),
         );
+
+        // Tests 3rd keyed account is of correct type (Current instead of rewards) in withdraw
+        assert_eq!(
+            super::process_instruction(
+                &Pubkey::default(),
+                &mut [
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(
+                        &syscall::rewards::id(),
+                        false,
+                        &mut syscall::rewards::create_account(1, 0.0, 0.0)
+                    ),
+                ],
+                &serialize(&StakeInstruction::Withdraw(42)).unwrap(),
+            ),
+            Err(InstructionError::InvalidArgument),
+        );
+
+        // Tests correct number of accounts are provided in withdraw
+        assert_eq!(
+            super::process_instruction(
+                &Pubkey::default(),
+                &mut [
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(
+                        &syscall::current::id(),
+                        false,
+                        &mut syscall::rewards::create_account(1, 0.0, 0.0)
+                    ),
+                ],
+                &serialize(&StakeInstruction::Withdraw(42)).unwrap(),
+            ),
+            Err(InstructionError::InvalidInstructionData),
+        );
+
+        // Tests 2nd keyed account is of correct type (Current instead of rewards) in deactivate
+        assert_eq!(
+            super::process_instruction(
+                &Pubkey::default(),
+                &mut [
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(
+                        &syscall::rewards::id(),
+                        false,
+                        &mut syscall::rewards::create_account(1, 0.0, 0.0)
+                    ),
+                ],
+                &serialize(&StakeInstruction::Deactivate).unwrap(),
+            ),
+            Err(InstructionError::InvalidArgument),
+        );
+
+        // Tests correct number of accounts are provided in deactivate
+        assert_eq!(
+            super::process_instruction(
+                &Pubkey::default(),
+                &mut [
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(&Pubkey::default(), false, &mut Account::default()),
+                    KeyedAccount::new(
+                        &syscall::current::id(),
+                        false,
+                        &mut syscall::rewards::create_account(1, 0.0, 0.0)
+                    ),
+                ],
+                &serialize(&StakeInstruction::Deactivate).unwrap(),
+            ),
+            Err(InstructionError::InvalidInstructionData),
+        );
     }
 
 }

programs/stake_api/src/stake_state.rs

@@ -12,6 +12,7 @@ use solana_sdk::pubkey::Pubkey;
 use solana_sdk::syscall;
 use solana_sdk::timing::Epoch;
 use solana_vote_api::vote_state::VoteState;
+use std::cmp;
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
 pub enum StakeState {
@@ -196,6 +197,12 @@ pub trait StakeAccount {
         rewards_account: &mut KeyedAccount,
         rewards: &syscall::rewards::Rewards,
     ) -> Result<(), InstructionError>;
+    fn withdraw(
+        &mut self,
+        lamports: u64,
+        to: &mut KeyedAccount,
+        current: &syscall::current::Current,
+    ) -> Result<(), InstructionError>;
 }
 
 impl<'a> StakeAccount for KeyedAccount<'a> {
@@ -281,6 +288,44 @@ impl<'a> StakeAccount for KeyedAccount<'a> {
             Err(InstructionError::InvalidAccountData)
         }
     }
+    fn withdraw(
+        &mut self,
+        lamports: u64,
+        to: &mut KeyedAccount,
+        current: &syscall::current::Current,
+    ) -> Result<(), InstructionError> {
+        if self.signer_key().is_none() {
+            return Err(InstructionError::MissingRequiredSignature);
+        }
+
+        match self.state()? {
+            StakeState::Stake(mut stake) => {
+                let staked = if stake.stake(current.epoch) == 0 {
+                    0
+                } else {
+                    // Assume full stake if the stake is under warmup/cooldown
+                    stake.stake
+                };
+                if lamports > self.account.lamports.saturating_sub(staked) {
+                    return Err(InstructionError::InsufficientFunds);
+                }
+                self.account.lamports -= lamports;
+                // Adjust the stake (in case balance dropped below stake)
+                stake.stake = cmp::min(stake.stake, self.account.lamports);
+                to.account.lamports += lamports;
+                Ok(())
+            }
+            StakeState::Uninitialized => {
+                if lamports > self.account.lamports {
+                    return Err(InstructionError::InsufficientFunds);
+                }
+                self.account.lamports -= lamports;
+                to.account.lamports += lamports;
+                Ok(())
+            }
+            _ => Err(InstructionError::InvalidAccountData),
+        }
+    }
 }
 
 // utility function, used by Bank, tests, genesis
@@ -316,6 +361,7 @@ mod tests {
     use solana_sdk::account::Account;
     use solana_sdk::pubkey::Pubkey;
     use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_program;
     use solana_vote_api::vote_state;
 
     #[test]
@@ -418,6 +464,181 @@ mod tests {
         assert_eq!(stake.stake(STAKE_WARMUP_EPOCHS * 42), 0);
     }
 
+    #[test]
+    fn test_deactivate_stake() {
+        let stake_pubkey = Pubkey::new_rand();
+        let stake_lamports = 42;
+        let mut stake_account =
+            Account::new(stake_lamports, std::mem::size_of::<StakeState>(), &id());
+
+        let current = syscall::current::Current::default();
+
+        // unsigned keyed account
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, false, &mut stake_account);
+        assert_eq!(
+            stake_keyed_account.deactivate_stake(&current),
+            Err(InstructionError::MissingRequiredSignature)
+        );
+
+        // signed keyed account but not staked yet
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, true, &mut stake_account);
+        assert_eq!(
+            stake_keyed_account.deactivate_stake(&current),
+            Err(InstructionError::InvalidAccountData)
+        );
+
+        // Staking
+        let vote_pubkey = Pubkey::new_rand();
+        let mut vote_account =
+            vote_state::create_account(&vote_pubkey, &Pubkey::new_rand(), 0, 100);
+        let mut vote_keyed_account = KeyedAccount::new(&vote_pubkey, false, &mut vote_account);
+        vote_keyed_account.set_state(&VoteState::default()).unwrap();
+        assert_eq!(
+            stake_keyed_account.delegate_stake(&vote_keyed_account, stake_lamports, &current),
+            Ok(())
+        );
+
+        // Deactivate after staking
+        assert_eq!(stake_keyed_account.deactivate_stake(&current), Ok(()));
+    }
+
+    #[test]
+    fn test_withdraw_stake() {
+        let stake_pubkey = Pubkey::new_rand();
+        let mut total_lamports = 100;
+        let stake_lamports = 42;
+        let mut stake_account =
+            Account::new(total_lamports, std::mem::size_of::<StakeState>(), &id());
+
+        let current = syscall::current::Current::default();
+
+        let to = Pubkey::new_rand();
+        let mut to_account = Account::new(1, 0, &system_program::id());
+        let mut to_keyed_account = KeyedAccount::new(&to, false, &mut to_account);
+
+        // unsigned keyed account
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, false, &mut stake_account);
+        assert_eq!(
+            stake_keyed_account.withdraw(total_lamports, &mut to_keyed_account, &current),
+            Err(InstructionError::MissingRequiredSignature)
+        );
+
+        // signed keyed account but uninitialized
+        // try withdrawing more than balance
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, true, &mut stake_account);
+        assert_eq!(
+            stake_keyed_account.withdraw(total_lamports + 1, &mut to_keyed_account, &current),
+            Err(InstructionError::InsufficientFunds)
+        );
+
+        // try withdrawing some (enough for rest of the test to carry forward)
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, true, &mut stake_account);
+        assert_eq!(
+            stake_keyed_account.withdraw(5, &mut to_keyed_account, &current),
+            Ok(())
+        );
+        total_lamports -= 5;
+
+        // Stake some lamports (available lampoorts for withdrawls will reduce)
+        let vote_pubkey = Pubkey::new_rand();
+        let mut vote_account =
+            vote_state::create_account(&vote_pubkey, &Pubkey::new_rand(), 0, 100);
+        let mut vote_keyed_account = KeyedAccount::new(&vote_pubkey, false, &mut vote_account);
+        vote_keyed_account.set_state(&VoteState::default()).unwrap();
+        assert_eq!(
+            stake_keyed_account.delegate_stake(&vote_keyed_account, stake_lamports, &current),
+            Ok(())
+        );
+
+        // Try to withdraw more than what's available
+        assert_eq!(
+            stake_keyed_account.withdraw(
+                total_lamports - stake_lamports + 1,
+                &mut to_keyed_account,
+                &current
+            ),
+            Err(InstructionError::InsufficientFunds)
+        );
+
+        // Try to withdraw all unstaked lamports
+        assert_eq!(
+            stake_keyed_account.withdraw(
+                total_lamports - stake_lamports,
+                &mut to_keyed_account,
+                &current
+            ),
+            Ok(())
+        );
+    }
+
+    #[test]
+    fn test_withdraw_stake_before_warmup() {
+        let stake_pubkey = Pubkey::new_rand();
+        let total_lamports = 100;
+        let stake_lamports = 42;
+        let mut stake_account =
+            Account::new(total_lamports, std::mem::size_of::<StakeState>(), &id());
+
+        let current = syscall::current::Current::default();
+        let mut future = syscall::current::Current::default();
+        future.epoch += 16;
+
+        let to = Pubkey::new_rand();
+        let mut to_account = Account::new(1, 0, &system_program::id());
+        let mut to_keyed_account = KeyedAccount::new(&to, false, &mut to_account);
+
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, true, &mut stake_account);
+
+        // Stake some lamports (available lampoorts for withdrawls will reduce)
+        let vote_pubkey = Pubkey::new_rand();
+        let mut vote_account =
+            vote_state::create_account(&vote_pubkey, &Pubkey::new_rand(), 0, 100);
+        let mut vote_keyed_account = KeyedAccount::new(&vote_pubkey, false, &mut vote_account);
+        vote_keyed_account.set_state(&VoteState::default()).unwrap();
+        assert_eq!(
+            stake_keyed_account.delegate_stake(&vote_keyed_account, stake_lamports, &future),
+            Ok(())
+        );
+
+        // Try to withdraw including staked
+        assert_eq!(
+            stake_keyed_account.withdraw(
+                total_lamports - stake_lamports + 1,
+                &mut to_keyed_account,
+                &current
+            ),
+            Ok(())
+        );
+    }
+
+    #[test]
+    fn test_withdraw_stake_invalid_state() {
+        let stake_pubkey = Pubkey::new_rand();
+        let total_lamports = 100;
+        let mut stake_account =
+            Account::new(total_lamports, std::mem::size_of::<StakeState>(), &id());
+
+        let current = syscall::current::Current::default();
+        let mut future = syscall::current::Current::default();
+        future.epoch += 16;
+
+        let to = Pubkey::new_rand();
+        let mut to_account = Account::new(1, 0, &system_program::id());
+        let mut to_keyed_account = KeyedAccount::new(&to, false, &mut to_account);
+
+        let mut stake_keyed_account = KeyedAccount::new(&stake_pubkey, true, &mut stake_account);
+        let stake_state = StakeState::MiningPool {
+            epoch: 0,
+            point_value: 0.0,
+        };
+        stake_keyed_account.set_state(&stake_state).unwrap();
+
+        assert_eq!(
+            stake_keyed_account.withdraw(total_lamports, &mut to_keyed_account, &current),
+            Err(InstructionError::InvalidAccountData)
+        );
+    }
+
     #[test]
     fn test_stake_state_calculate_rewards() {
         let mut vote_state = VoteState::default();

programs/storage_api/Cargo.toml

@@ -12,6 +12,7 @@ edition = "2018"
 assert_matches = "1.3.0"
 bincode = "1.1.4"
 log = "0.4.2"
+rand = "0.6.5"
 num-derive = "0.2"
 num-traits = "0.2"
 serde = "1.0.92"

programs/storage_api/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod rewards_pools;
 pub mod storage_contract;
 pub mod storage_instruction;
 pub mod storage_processor;

programs/storage_api/src/rewards_pools.rs

@@ -0,0 +1,63 @@
+//! rewards_pools
+//! * initialize genesis with rewards pools
+//! * keep track of rewards
+//! * own mining pools
+
+use crate::storage_contract::create_rewards_pool;
+use rand::{thread_rng, Rng};
+use solana_sdk::genesis_block::Builder;
+use solana_sdk::hash::{hash, Hash};
+use solana_sdk::pubkey::Pubkey;
+
+// base rewards pool ID
+const ID: [u8; 32] = [
+    6, 162, 25, 123, 127, 71, 141, 232, 129, 171, 58, 183, 79, 88, 181, 17, 163, 11, 51, 111, 22,
+    123, 67, 115, 5, 131, 109, 161, 16, 0, 0, 0,
+];
+
+solana_sdk::solana_name_id!(ID, "StorageMiningPoo111111111111111111111111111");
+
+// to cut down on collisions for redemptions, we make multiple accounts
+pub const NUM_REWARDS_POOLS: usize = 32;
+
+pub fn genesis(mut builder: Builder) -> Builder {
+    let mut pubkey = id();
+
+    for _i in 0..NUM_REWARDS_POOLS {
+        builder = builder.rewards_pool(pubkey, create_rewards_pool());
+        pubkey = Pubkey::new(hash(pubkey.as_ref()).as_ref());
+    }
+    builder
+}
+
+pub fn random_id() -> Pubkey {
+    let mut id = Hash::new(&ID);
+
+    for _i in 0..thread_rng().gen_range(0, NUM_REWARDS_POOLS) {
+        id = hash(id.as_ref());
+    }
+
+    Pubkey::new(id.as_ref())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::genesis_block::Builder;
+
+    #[test]
+    fn test() {
+        let builder = Builder::new();
+
+        let genesis_block = genesis(builder).build();
+
+        for _i in 0..NUM_REWARDS_POOLS {
+            let id = random_id();
+            assert!(genesis_block
+                .rewards_pools
+                .iter()
+                .position(|x| x.0 == id)
+                .is_some());
+        }
+    }
+}

programs/storage_api/src/storage_contract.rs

@@ -79,7 +79,7 @@ pub enum StorageContract {
         reward_validations: BTreeMap<usize, BTreeMap<Pubkey, Vec<ProofStatus>>>,
     },
 
-    MiningPool,
+    RewardsPool,
 }
 
 // utility function, used by Bank, tests, genesis
@@ -99,17 +99,6 @@ pub fn create_validator_storage_account(owner: Pubkey, lamports: u64) -> Account
     storage_account
 }
 
-// utility function, used by genesis
-pub fn create_mining_pool_account(lamports: u64) -> Account {
-    let mut storage_account = Account::new(lamports, STORAGE_ACCOUNT_SPACE as usize, &crate::id());
-
-    storage_account
-        .set_state(&StorageContract::MiningPool)
-        .expect("set_state");
-
-    storage_account
-}
-
 pub struct StorageAccount<'a> {
     pub(crate) id: Pubkey,
     account: &'a mut Account,
@@ -120,16 +109,6 @@ impl<'a> StorageAccount<'a> {
         Self { id, account }
     }
 
-    pub fn initialize_mining_pool(&mut self) -> Result<(), InstructionError> {
-        let storage_contract = &mut self.account.state()?;
-        if let StorageContract::Uninitialized = storage_contract {
-            *storage_contract = StorageContract::MiningPool;
-            self.account.set_state(storage_contract)
-        } else {
-            Err(InstructionError::AccountAlreadyInitialized)?
-        }
-    }
-
     pub fn initialize_replicator_storage(&mut self, owner: Pubkey) -> Result<(), InstructionError> {
         let storage_contract = &mut self.account.state()?;
         if let StorageContract::Uninitialized = storage_contract {
@@ -379,7 +358,7 @@ impl<'a> StorageAccount<'a> {
 
     pub fn claim_storage_reward(
         &mut self,
-        mining_pool: &mut KeyedAccount,
+        rewards_pool: &mut KeyedAccount,
         owner: &mut StorageAccount,
     ) -> Result<(), InstructionError> {
         let mut storage_contract = &mut self.account.state()?;
@@ -397,12 +376,13 @@ impl<'a> StorageAccount<'a> {
             }
 
             let pending = *pending_lamports;
-            if mining_pool.account.lamports < pending {
+            if rewards_pool.account.lamports < pending {
+                println!("reward pool has {}", rewards_pool.account.lamports);
                 Err(InstructionError::CustomError(
                     StorageError::RewardPoolDepleted as u32,
                 ))?
             }
-            mining_pool.account.lamports -= pending;
+            rewards_pool.account.lamports -= pending;
             owner.account.lamports += pending;
             //clear pending_lamports
             *pending_lamports = 0;
@@ -432,7 +412,7 @@ impl<'a> StorageAccount<'a> {
             let total_proofs = checked_proofs.len() as u64;
             let num_validations = count_valid_proofs(&checked_proofs);
             let reward = num_validations * REPLICATOR_REWARD * (num_validations / total_proofs);
-            mining_pool.account.lamports -= reward;
+            rewards_pool.account.lamports -= reward;
             owner.account.lamports += reward;
             self.account.set_state(storage_contract)
         } else {
@@ -441,6 +421,10 @@ impl<'a> StorageAccount<'a> {
     }
 }
 
+pub fn create_rewards_pool() -> Account {
+    Account::new_data(std::u64::MAX, &StorageContract::RewardsPool, &crate::id()).unwrap()
+}
+
 /// Store the result of a proof validation into the replicator account
 fn store_validation_result(
     me: &Pubkey,

programs/storage_api/src/storage_instruction.rs

@@ -1,5 +1,5 @@
-use crate::id;
 use crate::storage_contract::{ProofStatus, STORAGE_ACCOUNT_SPACE};
+use crate::{id, rewards_pools};
 use serde_derive::{Deserialize, Serialize};
 use solana_sdk::hash::Hash;
 use solana_sdk::instruction::{AccountMeta, Instruction};
@@ -10,11 +10,10 @@ use solana_sdk::system_instruction;
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum StorageInstruction {
-    /// Initialize the account as a mining pool, validator or replicator
+    /// Initialize the account as a validator or replicator
     ///
     /// Expects 1 Account:
     ///    0 - Account to be initialized
-    InitializeMiningPool,
     InitializeValidatorStorage {
         owner: Pubkey,
     },
@@ -128,27 +127,6 @@ pub fn create_replicator_storage_account(
     ]
 }
 
-pub fn create_mining_pool_account(
-    from_pubkey: &Pubkey,
-    storage_pubkey: &Pubkey,
-    lamports: u64,
-) -> Vec<Instruction> {
-    vec![
-        system_instruction::create_account(
-            from_pubkey,
-            storage_pubkey,
-            lamports,
-            STORAGE_ACCOUNT_SPACE,
-            &id(),
-        ),
-        Instruction::new(
-            id(),
-            &StorageInstruction::InitializeMiningPool,
-            vec![AccountMeta::new(*storage_pubkey, false)],
-        ),
-    ]
-}
-
 pub fn mining_proof(
     storage_pubkey: &Pubkey,
     sha_state: Hash,
@@ -200,15 +178,11 @@ pub fn proof_validation(
     Instruction::new(id(), &storage_instruction, account_metas)
 }
 
-pub fn claim_reward(
-    owner_pubkey: &Pubkey,
-    storage_pubkey: &Pubkey,
-    mining_pool_pubkey: &Pubkey,
-) -> Instruction {
+pub fn claim_reward(owner_pubkey: &Pubkey, storage_pubkey: &Pubkey) -> Instruction {
     let storage_instruction = StorageInstruction::ClaimStorageReward;
     let account_metas = vec![
         AccountMeta::new(*storage_pubkey, false),
-        AccountMeta::new(*mining_pool_pubkey, false),
+        AccountMeta::new(rewards_pools::random_id(), false),
         AccountMeta::new(*owner_pubkey, false),
     ];
     Instruction::new(id(), &storage_instruction, account_metas)

programs/storage_api/src/storage_processor.rs

@@ -20,12 +20,6 @@ pub fn process_instruction(
     let mut storage_account = StorageAccount::new(*me[0].unsigned_key(), &mut me[0].account);
 
     match bincode::deserialize(data).map_err(|_| InstructionError::InvalidInstructionData)? {
-        StorageInstruction::InitializeMiningPool => {
-            if !rest.is_empty() {
-                Err(InstructionError::InvalidArgument)?;
-            }
-            storage_account.initialize_mining_pool()
-        }
         StorageInstruction::InitializeReplicatorStorage { owner } => {
             if !rest.is_empty() {
                 Err(InstructionError::InvalidArgument)?;

programs/storage_program/tests/storage_processor.rs

@@ -3,10 +3,10 @@ use bincode::deserialize;
 use log::*;
 use solana_runtime::bank::Bank;
 use solana_runtime::bank_client::BankClient;
+use solana_runtime::genesis_utils::{create_genesis_block, GenesisBlockInfo};
 use solana_sdk::account::{create_keyed_accounts, Account, KeyedAccount};
 use solana_sdk::account_utils::State;
 use solana_sdk::client::SyncClient;
-use solana_sdk::genesis_block::create_genesis_block;
 use solana_sdk::hash::{hash, Hash};
 use solana_sdk::instruction::{Instruction, InstructionError};
 use solana_sdk::message::Message;
@@ -53,7 +53,11 @@ fn test_account_owner() {
     let validator_storage_pubkey = Pubkey::new_rand();
     let replicator_storage_pubkey = Pubkey::new_rand();
 
-    let (genesis_block, mint_keypair) = create_genesis_block(1000);
+    let GenesisBlockInfo {
+        genesis_block,
+        mint_keypair,
+        ..
+    } = create_genesis_block(1000);
     let mut bank = Bank::new(&genesis_block);
     let mint_pubkey = mint_keypair.pubkey();
     bank.add_instruction_processor(id(), process_instruction);
@@ -259,7 +263,11 @@ fn test_submit_mining_ok() {
 #[test]
 fn test_validate_mining() {
     solana_logger::setup();
-    let (mut genesis_block, mint_keypair) = create_genesis_block(100_000);
+    let GenesisBlockInfo {
+        mut genesis_block,
+        mint_keypair,
+        ..
+    } = create_genesis_block(100_000);
     genesis_block
         .native_instruction_processors
         .push(solana_storage_program::solana_storage_program!());
@@ -276,9 +284,6 @@ fn test_validate_mining() {
     let validator_storage_keypair = Keypair::new();
     let validator_storage_id = validator_storage_keypair.pubkey();
 
-    let mining_pool_keypair = Keypair::new();
-    let mining_pool_pubkey = mining_pool_keypair.pubkey();
-
     let bank = Bank::new(&genesis_block);
     let bank = Arc::new(bank);
     let bank_client = BankClient::new_shared(&bank);
@@ -291,12 +296,6 @@ fn test_validate_mining() {
         &[&replicator_1_storage_id, &replicator_2_storage_id],
         10,
     );
-    let message = Message::new(storage_instruction::create_mining_pool_account(
-        &mint_pubkey,
-        &mining_pool_pubkey,
-        10_000,
-    ));
-    bank_client.send_message(&[&mint_keypair], message).unwrap();
 
     // create a new bank in segment 2
     let bank = Arc::new(Bank::new_from_parent(
@@ -407,7 +406,6 @@ fn test_validate_mining() {
         vec![storage_instruction::claim_reward(
             &owner_pubkey,
             &validator_storage_id,
-            &mining_pool_pubkey,
         )],
         Some(&mint_pubkey),
     );
@@ -431,7 +429,6 @@ fn test_validate_mining() {
         vec![storage_instruction::claim_reward(
             &owner_pubkey,
             &replicator_1_storage_id,
-            &mining_pool_pubkey,
         )],
         Some(&mint_pubkey),
     );
@@ -447,7 +444,6 @@ fn test_validate_mining() {
         vec![storage_instruction::claim_reward(
             &owner_pubkey,
             &replicator_2_storage_id,
-            &mining_pool_pubkey,
         )],
         Some(&mint_pubkey),
     );
@@ -561,7 +557,11 @@ fn get_storage_blockhash<C: SyncClient>(client: &C, account: &Pubkey) -> Hash {
 
 #[test]
 fn test_bank_storage() {
-    let (mut genesis_block, mint_keypair) = create_genesis_block(1000);
+    let GenesisBlockInfo {
+        mut genesis_block,
+        mint_keypair,
+        ..
+    } = create_genesis_block(1000);
     genesis_block
         .native_instruction_processors
         .push(solana_storage_program::solana_storage_program!());

replicator/src/main.rs

@@ -2,7 +2,6 @@ use clap::{crate_description, crate_name, crate_version, App, Arg};
 use solana::cluster_info::{Node, FULLNODE_PORT_RANGE};
 use solana::contact_info::ContactInfo;
 use solana::replicator::Replicator;
-use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
 use std::net::SocketAddr;
 use std::process::exit;
@@ -70,10 +69,6 @@ fn main() {
         Keypair::new()
     };
 
-    let storage_mining_pool_pubkey = "StorageMiningPoo111111111111111111111111111"
-        .parse::<Pubkey>()
-        .unwrap();
-
     let entrypoint_addr = matches
         .value_of("entrypoint")
         .map(|entrypoint| {
@@ -106,6 +101,6 @@ fn main() {
     )
     .unwrap();
 
-    replicator.run(storage_mining_pool_pubkey);
+    replicator.run();
     replicator.close();
 }

run.sh

@@ -61,7 +61,6 @@ else
 fi
 solana-keygen new -f -o "$dataDir"/drone-keypair.json
 solana-keygen new -f -o "$dataDir"/leader-storage-account-keypair.json
-solana-keygen new -f -o "$dataDir"/storage-mining-pool-keypair.json
 
 leaderVoteAccountPubkey=$(\
   solana-wallet \
@@ -72,8 +71,6 @@ leaderVoteAccountPubkey=$(\
 solana-genesis \
   --lamports 1000000000 \
   --bootstrap-leader-lamports 10000000 \
-  --storage-mining-pool-lamports 100000000 \
-  --storage-mining-pool-keypair "$dataDir"/storage-mining-pool-keypair.json \
   --target-lamports-per-signature 42 \
   --target-signatures-per-slot 42 \
   --hashes-per-tick sleep \

runtime/src/accounts_db.rs

@@ -245,6 +245,8 @@ impl AccountStorageEntry {
 
         if count > 0 {
             *count_and_status = (count - 1, status);
+        } else {
+            warn!("count value 0 for fork {}", self.fork_id);
         }
         count_and_status.0
     }
@@ -260,7 +262,7 @@ pub struct AccountsDB {
     pub storage: RwLock<AccountStorage>,
 
     /// distribute the accounts across storage lists
-    next_id: AtomicUsize,
+    pub next_id: AtomicUsize,
 
     /// write version
     write_version: AtomicUsize,
@@ -325,9 +327,7 @@ impl AccountsDB {
 
         let _len: usize = deserialize_from(&mut stream)
             .map_err(|_| AccountsDB::get_io_error("len deserialize error"))?;
-        let accounts_index: AccountsIndex<AccountInfo> = deserialize_from(&mut stream)
-            .map_err(|_| AccountsDB::get_io_error("accounts index deserialize error"))?;
-        let storage: AccountStorage = deserialize_from(&mut stream)
+        let mut storage: AccountStorage = deserialize_from(&mut stream)
             .map_err(|_| AccountsDB::get_io_error("storage deserialize error"))?;
         let version: u64 = deserialize_from(&mut stream)
             .map_err(|_| AccountsDB::get_io_error("write version deserialize error"))?;
@@ -341,11 +341,13 @@ impl AccountsDB {
         ids.sort();
 
         {
-            let mut index = self.accounts_index.write().unwrap();
-            let union = index.roots.union(&accounts_index.roots);
-            index.roots = union.cloned().collect();
-            index.last_root = accounts_index.last_root;
             let mut stores = self.storage.write().unwrap();
+            if let Some((_, store0)) = storage.0.remove_entry(&0) {
+                let fork_storage0 = stores.0.entry(0).or_insert_with(HashMap::new);
+                for (id, store) in store0.iter() {
+                    fork_storage0.insert(*id, store.clone());
+                }
+            }
             stores.0.extend(storage.0);
         }
         self.next_id
@@ -627,8 +629,8 @@ impl AccountsDB {
     }
 
     fn generate_index(&self) {
-        let mut forks: Vec<Fork> = self.storage.read().unwrap().0.keys().cloned().collect();
-
+        let storage = self.storage.read().unwrap();
+        let mut forks: Vec<Fork> = storage.0.keys().cloned().collect();
         forks.sort();
         let mut accounts_index = self.accounts_index.write().unwrap();
         accounts_index.roots.insert(0);
@@ -655,8 +657,12 @@ impl AccountsDB {
             while let Some(maps) = accumulator.pop() {
                 AccountsDB::merge(&mut account_maps, &maps);
             }
-            for (pubkey, (_, account_info)) in account_maps.iter() {
-                accounts_index.add_index(*fork_id, pubkey, account_info.clone());
+            if !account_maps.is_empty() {
+                accounts_index.roots.insert(*fork_id);
+                let mut _reclaims: Vec<(u64, AccountInfo)> = vec![];
+                for (pubkey, (_, account_info)) in account_maps.iter() {
+                    accounts_index.insert(*fork_id, pubkey, account_info.clone(), &mut _reclaims);
+                }
             }
         }
     }
@@ -668,15 +674,11 @@ impl Serialize for AccountsDB {
         S: serde::ser::Serializer,
     {
         use serde::ser::Error;
-        let accounts_index = self.accounts_index.read().unwrap();
         let storage = self.storage.read().unwrap();
-        let len = serialized_size(&*accounts_index).unwrap()
-            + serialized_size(&*storage).unwrap()
-            + std::mem::size_of::<u64>() as u64;
+        let len = serialized_size(&*storage).unwrap() + std::mem::size_of::<u64>() as u64;
         let mut buf = vec![0u8; len as usize];
         let mut wr = Cursor::new(&mut buf[..]);
         let version: u64 = self.write_version.load(Ordering::Relaxed) as u64;
-        serialize_into(&mut wr, &*accounts_index).map_err(Error::custom)?;
         serialize_into(&mut wr, &*storage).map_err(Error::custom)?;
         serialize_into(&mut wr, &version).map_err(Error::custom)?;
         let len = wr.position() as usize;
@@ -880,7 +882,7 @@ mod tests {
             ACCOUNT_DATA_FILE_SIZE as usize / 3,
             0,
         );
-        assert!(check_storage(&db, 2));
+        assert!(check_storage(&db, 0, 2));
 
         let pubkey = Pubkey::new_rand();
         let account = Account::new(1, ACCOUNT_DATA_FILE_SIZE as usize / 3, &pubkey);
@@ -991,12 +993,17 @@ mod tests {
         }
     }
 
-    fn check_storage(accounts: &AccountsDB, count: usize) -> bool {
-        let stores = accounts.storage.read().unwrap();
-        assert_eq!(stores.0[&0].len(), 1);
-        assert_eq!(stores.0[&0][&0].status(), AccountStorageStatus::Available);
-        assert_eq!(stores.0[&0][&0].count(), count);
-        stores.0[&0][&0].count() == count
+    fn check_storage(accounts: &AccountsDB, fork: Fork, count: usize) -> bool {
+        let storage = accounts.storage.read().unwrap();
+        assert_eq!(storage.0[&fork].len(), 1);
+        let fork_storage = storage.0.get(&fork).unwrap();
+        let mut total_count: usize = 0;
+        for store in fork_storage.values() {
+            assert_eq!(store.status(), AccountStorageStatus::Available);
+            total_count += store.count();
+        }
+        assert_eq!(total_count, count);
+        total_count == count
     }
 
     fn check_accounts(
@@ -1057,7 +1064,7 @@ mod tests {
         let mut pubkeys: Vec<Pubkey> = vec![];
         create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
         update_accounts(&accounts, &pubkeys, 0, 99);
-        assert_eq!(check_storage(&accounts, 100), true);
+        assert_eq!(check_storage(&accounts, 0, 100), true);
     }
 
     #[test]
@@ -1232,7 +1239,7 @@ mod tests {
         let accounts = AccountsDB::new(&paths.paths);
         let mut pubkeys: Vec<Pubkey> = vec![];
         create_account(&accounts, &mut pubkeys, 0, 100, 0, 0);
-        assert_eq!(check_storage(&accounts, 100), true);
+        assert_eq!(check_storage(&accounts, 0, 100), true);
         check_accounts(&accounts, &pubkeys, 0, 100, 1);
         modify_accounts(&accounts, &pubkeys, 0, 100, 2);
         check_accounts(&accounts, &pubkeys, 0, 100, 2);
@@ -1244,6 +1251,8 @@ mod tests {
         let mut buf = vec![0u8; serialized_size(&accounts).unwrap() as usize];
         let mut writer = Cursor::new(&mut buf[..]);
         serialize_into(&mut writer, &accounts).unwrap();
+        assert!(check_storage(&accounts, 0, 100));
+        assert!(check_storage(&accounts, 1, 10));
 
         let mut reader = BufReader::new(&buf[..]);
         let daccounts = AccountsDB::new(&paths.paths);
@@ -1254,6 +1263,8 @@ mod tests {
         );
         check_accounts(&daccounts, &pubkeys, 0, 100, 2);
         check_accounts(&daccounts, &pubkeys1, 1, 10, 1);
+        assert!(check_storage(&daccounts, 0, 100));
+        assert!(check_storage(&daccounts, 1, 10));
     }
 
     #[test]

runtime/src/accounts_index.rs

@@ -1,21 +1,18 @@
 use hashbrown::HashMap;
 use log::*;
-use serde::{Deserialize, Serialize};
 use solana_sdk::pubkey::Pubkey;
 use std::collections;
 use std::collections::HashSet;
 
 pub type Fork = u64;
 
-#[derive(Debug, Default, Deserialize, Serialize)]
+#[derive(Debug, Default)]
 pub struct AccountsIndex<T> {
-    #[serde(skip)]
     pub account_maps: HashMap<Pubkey, Vec<(Fork, T)>>,
 
     pub roots: HashSet<Fork>,
 
     //This value that needs to be stored to recover the index from AppendVec
-    #[serde(skip)]
     pub last_root: Fork,
 }
 

runtime/src/append_vec.rs

@@ -325,6 +325,7 @@ impl Serialize for AppendVec {
             + std::mem::size_of::<usize>() as u64;
         let mut buf = vec![0u8; len as usize];
         let mut wr = Cursor::new(&mut buf[..]);
+        self.map.flush().map_err(Error::custom)?;
         serialize_into(&mut wr, &self.path).map_err(Error::custom)?;
         serialize_into(&mut wr, &(self.current_len.load(Ordering::Relaxed) as u64))
             .map_err(Error::custom)?;

runtime/src/bank.rs

@@ -4,7 +4,7 @@
 //! already been signed and verified.
 use crate::accounts::Accounts;
 use crate::accounts_db::{
-    ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
+    AppendVecId, ErrorCounters, InstructionAccounts, InstructionCredits, InstructionLoaders,
 };
 use crate::accounts_index::Fork;
 use crate::blockhash_queue::BlockhashQueue;
@@ -62,8 +62,12 @@ pub struct BankRc {
 }
 
 impl BankRc {
-    pub fn new(account_paths: Option<String>) -> Self {
+    pub fn new(account_paths: Option<String>, id: AppendVecId) -> Self {
         let accounts = Accounts::new(account_paths);
+        accounts
+            .accounts_db
+            .next_id
+            .store(id as usize, Ordering::Relaxed);
         BankRc {
             accounts: Arc::new(accounts),
             parent: RwLock::new(None),
@@ -374,9 +378,10 @@ impl Bank {
         genesis_block: &GenesisBlock,
         account_paths: Option<String>,
         status_cache_rc: &StatusCacheRc,
+        id: AppendVecId,
     ) -> Self {
         let mut bank = Self::default();
-        bank.set_bank_rc(&BankRc::new(account_paths), &status_cache_rc);
+        bank.set_bank_rc(&BankRc::new(account_paths, id), &status_cache_rc);
         bank.process_genesis_block(genesis_block);
         bank.ancestors.insert(0, 0);
         bank
@@ -2583,7 +2588,7 @@ mod tests {
         let mut dbank: Bank = deserialize_from(&mut rdr).unwrap();
         let mut reader = BufReader::new(&buf[rdr.position() as usize..]);
         dbank.set_bank_rc(
-            &BankRc::new(Some(bank0.accounts().paths.clone())),
+            &BankRc::new(Some(bank0.accounts().paths.clone()), 0),
             &StatusCacheRc::default(),
         );
         assert!(dbank.rc.update_from_stream(&mut reader).is_ok());

runtime/src/genesis_utils.rs

@@ -70,6 +70,7 @@ pub fn create_genesis_block_with_leader(
         ]);
 
     builder = solana_stake_api::rewards_pools::genesis(builder);
+    builder = solana_storage_api::rewards_pools::genesis(builder);
 
     GenesisBlockInfo {
         genesis_block: builder.build(),

runtime/src/message_processor.rs

@@ -186,7 +186,10 @@ impl MessageProcessor {
         let program_id = instruction.program_id(&message.account_keys);
         // TODO: the runtime should be checking read/write access to memory
         // we are trusting the hard-coded programs not to clobber or allocate
-        let pre_total: u64 = program_accounts.iter().map(|a| a.lamports).sum();
+        let pre_total: u128 = program_accounts
+            .iter()
+            .map(|a| u128::from(a.lamports))
+            .sum();
         let pre_data: Vec<_> = program_accounts
             .iter_mut()
             .map(|a| (a.owner, a.lamports, a.data.clone()))
@@ -215,7 +218,10 @@ impl MessageProcessor {
             }
         }
         // The total sum of all the lamports in all the accounts cannot change.
-        let post_total: u64 = program_accounts.iter().map(|a| a.lamports).sum();
+        let post_total: u128 = program_accounts
+            .iter()
+            .map(|a| u128::from(a.lamports))
+            .sum();
         if pre_total != post_total {
             return Err(InstructionError::UnbalancedInstruction);
         }

wallet/src/wallet.rs

@@ -52,12 +52,13 @@ pub enum WalletCommand {
     ShowVoteAccount(Pubkey),
     CreateStakeAccount(Pubkey, u64),
     DelegateStake(Keypair, Pubkey, u64),
+    WithdrawStake(Keypair, Pubkey, u64),
+    DeactivateStake(Keypair),
     RedeemVoteCredits(Pubkey, Pubkey),
     ShowStakeAccount(Pubkey),
-    CreateStorageMiningPoolAccount(Pubkey, u64),
     CreateReplicatorStorageAccount(Pubkey, Pubkey),
     CreateValidatorStorageAccount(Pubkey, Pubkey),
-    ClaimStorageReward(Pubkey, Pubkey, Pubkey),
+    ClaimStorageReward(Pubkey, Pubkey),
     ShowStorageAccount(Pubkey),
     Deploy(String),
     GetTransactionCount,
@@ -248,6 +249,23 @@ pub fn parse_command(
                 stake,
             ))
         }
+        ("withdraw-stake", Some(matches)) => {
+            let staking_account_keypair =
+                keypair_of(matches, "staking_account_keypair_file").unwrap();
+            let destination_account_pubkey =
+                value_of(matches, "destination_account_pubkey").unwrap();
+            let lamports = matches.value_of("lamports").unwrap().parse()?;
+            Ok(WalletCommand::WithdrawStake(
+                staking_account_keypair,
+                destination_account_pubkey,
+                lamports,
+            ))
+        }
+        ("deactivate-stake", Some(matches)) => {
+            let staking_account_keypair =
+                keypair_of(matches, "staking_account_keypair_file").unwrap();
+            Ok(WalletCommand::DeactivateStake(staking_account_keypair))
+        }
         ("redeem-vote-credits", Some(matches)) => {
             let staking_account_pubkey = value_of(matches, "staking_account_pubkey").unwrap();
             let voting_account_pubkey = value_of(matches, "voting_account_pubkey").unwrap();
@@ -278,12 +296,9 @@ pub fn parse_command(
         }
         ("claim-storage-reward", Some(matches)) => {
             let node_account_pubkey = value_of(matches, "node_account_pubkey").unwrap();
-            let storage_mining_pool_account_pubkey =
-                value_of(matches, "storage_mining_pool_account_pubkey").unwrap();
             let storage_account_pubkey = value_of(matches, "storage_account_pubkey").unwrap();
             Ok(WalletCommand::ClaimStorageReward(
                 node_account_pubkey,
-                storage_mining_pool_account_pubkey,
                 storage_account_pubkey,
             ))
         }
@@ -558,6 +573,24 @@ fn process_create_stake_account(
     Ok(signature_str.to_string())
 }
 
+fn process_deactivate_stake_account(
+    rpc_client: &RpcClient,
+    config: &WalletConfig,
+    staking_account_keypair: &Keypair,
+) -> ProcessResult {
+    let (recent_blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let ixs = stake_instruction::deactivate_stake(&staking_account_keypair.pubkey());
+    let mut tx = Transaction::new_signed_with_payer(
+        vec![ixs],
+        Some(&config.keypair.pubkey()),
+        &[&config.keypair, &staking_account_keypair],
+        recent_blockhash,
+    );
+    let signature_str = rpc_client
+        .send_and_confirm_transaction(&mut tx, &[&config.keypair, &staking_account_keypair])?;
+    Ok(signature_str.to_string())
+}
+
 fn process_delegate_stake(
     rpc_client: &RpcClient,
     config: &WalletConfig,
@@ -584,6 +617,32 @@ fn process_delegate_stake(
     Ok(signature_str.to_string())
 }
 
+fn process_withdraw_stake(
+    rpc_client: &RpcClient,
+    config: &WalletConfig,
+    staking_account_keypair: &Keypair,
+    destination_account_pubkey: &Pubkey,
+    lamports: u64,
+) -> ProcessResult {
+    let (recent_blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let ixs = vec![stake_instruction::withdraw(
+        &staking_account_keypair.pubkey(),
+        destination_account_pubkey,
+        lamports,
+    )];
+
+    let mut tx = Transaction::new_signed_with_payer(
+        ixs,
+        Some(&config.keypair.pubkey()),
+        &[&config.keypair, &staking_account_keypair],
+        recent_blockhash,
+    );
+
+    let signature_str = rpc_client
+        .send_and_confirm_transaction(&mut tx, &[&config.keypair, &staking_account_keypair])?;
+    Ok(signature_str.to_string())
+}
+
 fn process_redeem_vote_credits(
     rpc_client: &RpcClient,
     config: &WalletConfig,
@@ -626,23 +685,6 @@ fn process_show_stake_account(
     }
 }
 
-fn process_create_storage_mining_pool_account(
-    rpc_client: &RpcClient,
-    config: &WalletConfig,
-    storage_account_pubkey: &Pubkey,
-    lamports: u64,
-) -> ProcessResult {
-    let (recent_blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let ixs = storage_instruction::create_mining_pool_account(
-        &config.keypair.pubkey(),
-        storage_account_pubkey,
-        lamports,
-    );
-    let mut tx = Transaction::new_signed_instructions(&[&config.keypair], ixs, recent_blockhash);
-    let signature_str = rpc_client.send_and_confirm_transaction(&mut tx, &[&config.keypair])?;
-    Ok(signature_str.to_string())
-}
-
 fn process_create_replicator_storage_account(
     rpc_client: &RpcClient,
     config: &WalletConfig,
@@ -683,16 +725,12 @@ fn process_claim_storage_reward(
     rpc_client: &RpcClient,
     config: &WalletConfig,
     node_account_pubkey: &Pubkey,
-    storage_mining_pool_account_pubkey: &Pubkey,
     storage_account_pubkey: &Pubkey,
 ) -> ProcessResult {
     let (recent_blockhash, _fee_calculator) = rpc_client.get_recent_blockhash()?;
 
-    let instruction = storage_instruction::claim_reward(
-        node_account_pubkey,
-        storage_account_pubkey,
-        storage_mining_pool_account_pubkey,
-    );
+    let instruction =
+        storage_instruction::claim_reward(node_account_pubkey, storage_account_pubkey);
     let signers = [&config.keypair];
     let message = Message::new_with_payer(vec![instruction], Some(&signers[0].pubkey()));
 
@@ -1022,6 +1060,23 @@ pub fn process_command(config: &WalletConfig) -> ProcessResult {
             )
         }
 
+        WalletCommand::WithdrawStake(
+            staking_account_keypair,
+            destination_account_pubkey,
+            lamports,
+        ) => process_withdraw_stake(
+            &rpc_client,
+            config,
+            &staking_account_keypair,
+            &destination_account_pubkey,
+            *lamports,
+        ),
+
+        // Deactivate stake account
+        WalletCommand::DeactivateStake(staking_account_keypair) => {
+            process_deactivate_stake_account(&rpc_client, config, &staking_account_keypair)
+        }
+
         WalletCommand::RedeemVoteCredits(staking_account_pubkey, voting_account_pubkey) => {
             process_redeem_vote_credits(
                 &rpc_client,
@@ -1035,15 +1090,6 @@ pub fn process_command(config: &WalletConfig) -> ProcessResult {
             process_show_stake_account(&rpc_client, config, &staking_account_pubkey)
         }
 
-        WalletCommand::CreateStorageMiningPoolAccount(storage_account_pubkey, lamports) => {
-            process_create_storage_mining_pool_account(
-                &rpc_client,
-                config,
-                &storage_account_pubkey,
-                *lamports,
-            )
-        }
-
         WalletCommand::CreateReplicatorStorageAccount(
             storage_account_owner,
             storage_account_pubkey,
@@ -1063,17 +1109,14 @@ pub fn process_command(config: &WalletConfig) -> ProcessResult {
             )
         }
 
-        WalletCommand::ClaimStorageReward(
-            node_account_pubkey,
-            storage_mining_pool_account_pubkey,
-            storage_account_pubkey,
-        ) => process_claim_storage_reward(
-            &rpc_client,
-            config,
-            node_account_pubkey,
-            &storage_mining_pool_account_pubkey,
-            &storage_account_pubkey,
-        ),
+        WalletCommand::ClaimStorageReward(node_account_pubkey, storage_account_pubkey) => {
+            process_claim_storage_reward(
+                &rpc_client,
+                config,
+                node_account_pubkey,
+                &storage_account_pubkey,
+            )
+        }
 
         WalletCommand::ShowStorageAccount(storage_account_pubkey) => {
             process_show_storage_account(&rpc_client, config, &storage_account_pubkey)
@@ -1401,6 +1444,47 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .help("The number of lamports to stake, must be less than the stake account's balance."),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("deactivate-stake")
+                .about("Deactivate the delegated stake from the staking account")
+                .arg(
+                    Arg::with_name("staking_account_keypair_file")
+                        .index(1)
+                        .value_name("KEYPAIR_FILE")
+                        .takes_value(true)
+                        .required(true)
+                        .help("Keypair file for the staking account, for signing the delegate transaction."),
+                )
+        )
+        .subcommand(
+            SubCommand::with_name("withdraw-stake")
+                .about("Withdraw the unstaked lamports from the stake account")
+                .arg(
+                    Arg::with_name("staking_account_keypair_file")
+                        .index(1)
+                        .value_name("KEYPAIR_FILE")
+                        .takes_value(true)
+                        .required(true)
+                        .help("Keypair file for the staking account, for signing the withdraw transaction."),
+                )
+                .arg(
+                    Arg::with_name("destination_account_pubkey")
+                        .index(2)
+                        .value_name("PUBKEY")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_pubkey)
+                        .help("The account where the lamports should be transfered"),
+                )
+                .arg(
+                    Arg::with_name("lamports")
+                        .index(3)
+                        .value_name("NUM")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The number of lamports to to withdraw from the stake account."),
+                ),
+        )
         .subcommand(
             SubCommand::with_name("redeem-vote-credits")
                 .about("Redeem credits in the staking account")
@@ -1518,15 +1602,6 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .validator(is_pubkey)
                         .help("The node account to credit the rewards to"),
                 )
-                .arg(
-                    Arg::with_name("storage_mining_pool_account_pubkey")
-                        .index(2)
-                        .value_name("MINING POOL PUBKEY")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_pubkey)
-                        .help("Mining pool account to redeem credits from"),
-                )
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(3)
@@ -1837,6 +1912,35 @@ mod tests {
             WalletCommand::DelegateStake(keypair, pubkey, 42)
         );
 
+        let keypair_file = make_tmp_path("keypair_file");
+        gen_keypair_file(&keypair_file).unwrap();
+        let keypair = read_keypair(&keypair_file).unwrap();
+        // Test Withdraw from Stake Account
+        let test_withdraw_stake = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-stake",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+        ]);
+        assert_eq!(
+            parse_command(&pubkey, &test_withdraw_stake).unwrap(),
+            WalletCommand::WithdrawStake(keypair, pubkey, 42)
+        );
+
+        // Test Deactivate Stake Subcommand
+        let keypair_file = make_tmp_path("keypair_file");
+        gen_keypair_file(&keypair_file).unwrap();
+        let keypair = read_keypair(&keypair_file).unwrap();
+        let test_deactivate_stake =
+            test_commands
+                .clone()
+                .get_matches_from(vec!["test", "deactivate-stake", &keypair_file]);
+        assert_eq!(
+            parse_command(&pubkey, &test_deactivate_stake).unwrap(),
+            WalletCommand::DeactivateStake(keypair)
+        );
+
         // Test Deploy Subcommand
         let test_deploy =
             test_commands
@@ -2006,6 +2110,17 @@ mod tests {
         let signature = process_command(&config);
         assert_eq!(signature.unwrap(), SIGNATURE.to_string());
 
+        let bob_keypair = Keypair::new();
+        let to_pubkey = Pubkey::new_rand();
+        config.command = WalletCommand::WithdrawStake(bob_keypair.into(), to_pubkey, 100);
+        let signature = process_command(&config);
+        assert_eq!(signature.unwrap(), SIGNATURE.to_string());
+
+        let bob_keypair = Keypair::new();
+        config.command = WalletCommand::DeactivateStake(bob_keypair.into());
+        let signature = process_command(&config);
+        assert_eq!(signature.unwrap(), SIGNATURE.to_string());
+
         config.command = WalletCommand::GetTransactionCount;
         assert_eq!(process_command(&config).unwrap(), "1234");