Move src/ into core/src. Top-level crate is now called solana-workspace

core/src/bank_forks.rs

@@ -0,0 +1,124 @@
+//! The `bank_forks` module implments BankForks a DAG of checkpointed Banks
+
+use solana_runtime::bank::Bank;
+use std::collections::HashMap;
+use std::ops::Index;
+use std::sync::Arc;
+
+pub struct BankForks {
+    banks: HashMap<u64, Arc<Bank>>,
+    working_bank: Arc<Bank>,
+}
+
+impl Index<u64> for BankForks {
+    type Output = Arc<Bank>;
+    fn index(&self, bank_id: u64) -> &Arc<Bank> {
+        &self.banks[&bank_id]
+    }
+}
+
+impl BankForks {
+    pub fn new(bank_id: u64, bank: Bank) -> Self {
+        let mut banks = HashMap::new();
+        let working_bank = Arc::new(bank);
+        banks.insert(bank_id, working_bank.clone());
+        Self {
+            banks,
+            working_bank,
+        }
+    }
+    pub fn frozen_banks(&self) -> HashMap<u64, Arc<Bank>> {
+        let mut frozen_banks: Vec<Arc<Bank>> = vec![];
+        frozen_banks.extend(self.banks.values().filter(|v| v.is_frozen()).cloned());
+        frozen_banks.extend(
+            self.banks
+                .iter()
+                .flat_map(|(_, v)| v.parents())
+                .filter(|v| v.is_frozen()),
+        );
+        frozen_banks.into_iter().map(|b| (b.slot(), b)).collect()
+    }
+    pub fn active_banks(&self) -> Vec<u64> {
+        self.banks.iter().map(|(k, _v)| *k).collect()
+    }
+    pub fn get(&self, bank_id: u64) -> Option<&Arc<Bank>> {
+        self.banks.get(&bank_id)
+    }
+
+    pub fn new_from_banks(initial_banks: &[Arc<Bank>]) -> Self {
+        let mut banks = HashMap::new();
+        let working_bank = initial_banks[0].clone();
+        for bank in initial_banks {
+            banks.insert(bank.slot(), bank.clone());
+        }
+        Self {
+            banks,
+            working_bank,
+        }
+    }
+
+    // TODO: use the bank's own ID instead of receiving a parameter?
+    pub fn insert(&mut self, bank_id: u64, bank: Bank) {
+        let mut bank = Arc::new(bank);
+        self.banks.insert(bank_id, bank.clone());
+
+        if bank_id > self.working_bank.slot() {
+            self.working_bank = bank.clone()
+        }
+
+        // TODO: this really only needs to look at the first
+        //  parent if we're always calling insert()
+        //  when we construct a child bank
+        while let Some(parent) = bank.parent() {
+            self.banks.remove(&parent.slot());
+            bank = parent;
+        }
+    }
+
+    // TODO: really want to kill this...
+    pub fn working_bank(&self) -> Arc<Bank> {
+        self.working_bank.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_bank_forks() {
+        let (genesis_block, _) = GenesisBlock::new(10_000);
+        let bank = Bank::new(&genesis_block);
+        let mut bank_forks = BankForks::new(0, bank);
+        let child_bank = Bank::new_from_parent(&bank_forks[0u64], Pubkey::default(), 1);
+        child_bank.register_tick(&Hash::default());
+        bank_forks.insert(1, child_bank);
+        assert_eq!(bank_forks[1u64].tick_height(), 1);
+        assert_eq!(bank_forks.working_bank().tick_height(), 1);
+    }
+
+    #[test]
+    fn test_bank_forks_frozen_banks() {
+        let (genesis_block, _) = GenesisBlock::new(10_000);
+        let bank = Bank::new(&genesis_block);
+        let mut bank_forks = BankForks::new(0, bank);
+        let child_bank = Bank::new_from_parent(&bank_forks[0u64], Pubkey::default(), 1);
+        bank_forks.insert(1, child_bank);
+        assert!(bank_forks.frozen_banks().get(&0).is_some());
+        assert!(bank_forks.frozen_banks().get(&1).is_none());
+    }
+
+    #[test]
+    fn test_bank_forks_active_banks() {
+        let (genesis_block, _) = GenesisBlock::new(10_000);
+        let bank = Bank::new(&genesis_block);
+        let mut bank_forks = BankForks::new(0, bank);
+        let child_bank = Bank::new_from_parent(&bank_forks[0u64], Pubkey::default(), 1);
+        bank_forks.insert(1, child_bank);
+        assert_eq!(bank_forks.active_banks(), vec![1]);
+    }
+
+}

core/src/banking_stage.rs

@@ -0,0 +1,733 @@
+//! The `banking_stage` processes Transaction messages. It is intended to be used
+//! to contruct a software pipeline. The stage uses all available CPU cores and
+//! can do its processing in parallel with signature verification on the GPU.
+
+use crate::entry::Entry;
+use crate::leader_confirmation_service::LeaderConfirmationService;
+use crate::packet::Packets;
+use crate::packet::SharedPackets;
+use crate::poh_recorder::{PohRecorder, PohRecorderError, WorkingBank};
+use crate::result::{Error, Result};
+use crate::service::Service;
+use crate::sigverify_stage::VerifiedPackets;
+use bincode::deserialize;
+use solana_metrics::counter::Counter;
+use solana_runtime::bank::{self, Bank, BankError};
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing::{self, duration_as_us, MAX_RECENT_TICK_HASHES};
+use solana_sdk::transaction::Transaction;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError};
+use std::sync::{Arc, Mutex};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+use std::time::Instant;
+use sys_info;
+
+pub type UnprocessedPackets = Vec<(SharedPackets, usize)>; // `usize` is the index of the first unprocessed packet in `SharedPackets`
+
+// number of threads is 1 until mt bank is ready
+pub const NUM_THREADS: u32 = 10;
+
+/// Stores the stage's thread handle and output receiver.
+pub struct BankingStage {
+    bank_thread_hdls: Vec<JoinHandle<UnprocessedPackets>>,
+    exit: Arc<AtomicBool>,
+    leader_confirmation_service: LeaderConfirmationService,
+}
+
+impl BankingStage {
+    /// Create the stage using `bank`. Exit when `verified_receiver` is dropped.
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        bank: &Arc<Bank>,
+        poh_recorder: &Arc<Mutex<PohRecorder>>,
+        verified_receiver: Receiver<VerifiedPackets>,
+        max_tick_height: u64,
+        leader_id: Pubkey,
+    ) -> (Self, Receiver<Vec<(Entry, u64)>>) {
+        let (entry_sender, entry_receiver) = channel();
+        let working_bank = WorkingBank {
+            bank: bank.clone(),
+            sender: entry_sender,
+            min_tick_height: bank.tick_height(),
+            max_tick_height,
+        };
+
+        info!(
+            "new working bank {} {} {}",
+            working_bank.min_tick_height,
+            working_bank.max_tick_height,
+            poh_recorder.lock().unwrap().poh.tick_height
+        );
+        poh_recorder.lock().unwrap().set_working_bank(working_bank);
+
+        let shared_verified_receiver = Arc::new(Mutex::new(verified_receiver));
+
+        // Single thread to generate entries from many banks.
+        // This thread talks to poh_service and broadcasts the entries once they have been recorded.
+        // Once an entry has been recorded, its last_id is registered with the bank.
+        let exit = Arc::new(AtomicBool::new(false));
+
+        // Single thread to compute confirmation
+        let leader_confirmation_service =
+            LeaderConfirmationService::new(&bank, leader_id, exit.clone());
+
+        // Many banks that process transactions in parallel.
+        let bank_thread_hdls: Vec<JoinHandle<UnprocessedPackets>> = (0..Self::num_threads())
+            .map(|_| {
+                let thread_verified_receiver = shared_verified_receiver.clone();
+                let thread_poh_recorder = poh_recorder.clone();
+                let thread_bank = bank.clone();
+                Builder::new()
+                    .name("solana-banking-stage-tx".to_string())
+                    .spawn(move || {
+                        let mut unprocessed_packets: UnprocessedPackets = vec![];
+                        loop {
+                            match Self::process_packets(
+                                &thread_bank,
+                                &thread_verified_receiver,
+                                &thread_poh_recorder,
+                            ) {
+                                Err(Error::RecvTimeoutError(RecvTimeoutError::Timeout)) => (),
+                                Ok(more_unprocessed_packets) => {
+                                    unprocessed_packets.extend(more_unprocessed_packets);
+                                }
+                                Err(err) => {
+                                    debug!("solana-banking-stage-tx: exit due to {:?}", err);
+                                    break;
+                                }
+                            }
+                        }
+                        unprocessed_packets
+                    })
+                    .unwrap()
+            })
+            .collect();
+        (
+            Self {
+                bank_thread_hdls,
+                exit,
+                leader_confirmation_service,
+            },
+            entry_receiver,
+        )
+    }
+
+    pub fn num_threads() -> u32 {
+        sys_info::cpu_num().unwrap_or(NUM_THREADS)
+    }
+
+    /// Convert the transactions from a blob of binary data to a vector of transactions
+    fn deserialize_transactions(p: &Packets) -> Vec<Option<Transaction>> {
+        p.packets
+            .iter()
+            .map(|x| deserialize(&x.data[0..x.meta.size]).ok())
+            .collect()
+    }
+
+    fn record_transactions(
+        txs: &[Transaction],
+        results: &[bank::Result<()>],
+        poh: &Arc<Mutex<PohRecorder>>,
+    ) -> Result<()> {
+        let processed_transactions: Vec<_> = results
+            .iter()
+            .zip(txs.iter())
+            .filter_map(|(r, x)| match r {
+                Ok(_) => Some(x.clone()),
+                Err(BankError::ProgramError(index, err)) => {
+                    info!("program error {:?}, {:?}", index, err);
+                    Some(x.clone())
+                }
+                Err(ref e) => {
+                    debug!("process transaction failed {:?}", e);
+                    None
+                }
+            })
+            .collect();
+        debug!("processed: {} ", processed_transactions.len());
+        // unlock all the accounts with errors which are filtered by the above `filter_map`
+        if !processed_transactions.is_empty() {
+            let hash = Transaction::hash(&processed_transactions);
+            // record and unlock will unlock all the successfull transactions
+            poh.lock().unwrap().record(hash, processed_transactions)?;
+        }
+        Ok(())
+    }
+
+    pub fn process_and_record_transactions(
+        bank: &Bank,
+        txs: &[Transaction],
+        poh: &Arc<Mutex<PohRecorder>>,
+    ) -> Result<()> {
+        let now = Instant::now();
+        // Once accounts are locked, other threads cannot encode transactions that will modify the
+        // same account state
+        let lock_results = bank.lock_accounts(txs);
+        let lock_time = now.elapsed();
+
+        let now = Instant::now();
+        // Use a shorter maximum age when adding transactions into the pipeline.  This will reduce
+        // the likelihood of any single thread getting starved and processing old ids.
+        // TODO: Banking stage threads should be prioritized to complete faster then this queue
+        // expires.
+        let (loaded_accounts, results) =
+            bank.load_and_execute_transactions(txs, lock_results, MAX_RECENT_TICK_HASHES / 2);
+        let load_execute_time = now.elapsed();
+
+        let record_time = {
+            let now = Instant::now();
+            Self::record_transactions(txs, &results, poh)?;
+            now.elapsed()
+        };
+
+        let commit_time = {
+            let now = Instant::now();
+            bank.commit_transactions(txs, &loaded_accounts, &results);
+            now.elapsed()
+        };
+
+        let now = Instant::now();
+        // Once the accounts are new transactions can enter the pipeline to process them
+        bank.unlock_accounts(&txs, &results);
+        let unlock_time = now.elapsed();
+        debug!(
+            "lock: {}us load_execute: {}us record: {}us commit: {}us unlock: {}us txs_len: {}",
+            duration_as_us(&lock_time),
+            duration_as_us(&load_execute_time),
+            duration_as_us(&record_time),
+            duration_as_us(&commit_time),
+            duration_as_us(&unlock_time),
+            txs.len(),
+        );
+        Ok(())
+    }
+
+    /// Sends transactions to the bank.
+    ///
+    /// Returns the number of transactions successfully processed by the bank, which may be less
+    /// than the total number if max PoH height was reached and the bank halted
+    fn process_transactions(
+        bank: &Bank,
+        transactions: &[Transaction],
+        poh: &Arc<Mutex<PohRecorder>>,
+    ) -> Result<(usize)> {
+        let mut chunk_start = 0;
+        while chunk_start != transactions.len() {
+            let chunk_end = chunk_start + Entry::num_will_fit(&transactions[chunk_start..]);
+
+            let result = Self::process_and_record_transactions(
+                bank,
+                &transactions[chunk_start..chunk_end],
+                poh,
+            );
+            trace!("process_transcations: {:?}", result);
+            if let Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached)) = result {
+                info!("process transactions: max height reached");
+                break;
+            }
+            result?;
+            chunk_start = chunk_end;
+        }
+        Ok(chunk_start)
+    }
+
+    /// Process the incoming packets
+    pub fn process_packets(
+        bank: &Bank,
+        verified_receiver: &Arc<Mutex<Receiver<VerifiedPackets>>>,
+        poh: &Arc<Mutex<PohRecorder>>,
+    ) -> Result<UnprocessedPackets> {
+        let recv_start = Instant::now();
+        let mms = verified_receiver
+            .lock()
+            .unwrap()
+            .recv_timeout(Duration::from_millis(100))?;
+        let mut reqs_len = 0;
+        let mms_len = mms.len();
+        info!(
+            "@{:?} process start stalled for: {:?}ms batches: {}",
+            timing::timestamp(),
+            timing::duration_as_ms(&recv_start.elapsed()),
+            mms.len(),
+        );
+        inc_new_counter_info!("banking_stage-entries_received", mms_len);
+        let count = mms.iter().map(|x| x.1.len()).sum();
+        let proc_start = Instant::now();
+        let mut new_tx_count = 0;
+
+        let mut unprocessed_packets = vec![];
+        let mut bank_shutdown = false;
+        for (msgs, vers) in mms {
+            if bank_shutdown {
+                unprocessed_packets.push((msgs, 0));
+                continue;
+            }
+
+            let transactions = Self::deserialize_transactions(&msgs.read().unwrap());
+            reqs_len += transactions.len();
+
+            debug!("transactions received {}", transactions.len());
+            let (verified_transactions, verified_transaction_index): (Vec<_>, Vec<_>) =
+                transactions
+                    .into_iter()
+                    .zip(vers)
+                    .zip(0..)
+                    .filter_map(|((tx, ver), index)| match tx {
+                        None => None,
+                        Some(tx) => {
+                            if tx.verify_refs() && ver != 0 {
+                                Some((tx, index))
+                            } else {
+                                None
+                            }
+                        }
+                    })
+                    .unzip();
+
+            debug!("verified transactions {}", verified_transactions.len());
+
+            let processed = Self::process_transactions(bank, &verified_transactions, poh)?;
+            if processed < verified_transactions.len() {
+                bank_shutdown = true;
+                // Collect any unprocessed transactions in this batch for forwarding
+                unprocessed_packets.push((msgs, verified_transaction_index[processed]));
+            }
+            new_tx_count += processed;
+        }
+
+        inc_new_counter_info!(
+            "banking_stage-time_ms",
+            timing::duration_as_ms(&proc_start.elapsed()) as usize
+        );
+        let total_time_s = timing::duration_as_s(&proc_start.elapsed());
+        let total_time_ms = timing::duration_as_ms(&proc_start.elapsed());
+        info!(
+            "@{:?} done processing transaction batches: {} time: {:?}ms reqs: {} reqs/s: {}",
+            timing::timestamp(),
+            mms_len,
+            total_time_ms,
+            reqs_len,
+            (reqs_len as f32) / (total_time_s)
+        );
+        inc_new_counter_info!("banking_stage-process_packets", count);
+        inc_new_counter_info!("banking_stage-process_transactions", new_tx_count);
+
+        Ok(unprocessed_packets)
+    }
+
+    pub fn join_and_collect_unprocessed_packets(&mut self) -> UnprocessedPackets {
+        let mut unprocessed_packets: UnprocessedPackets = vec![];
+        for bank_thread_hdl in self.bank_thread_hdls.drain(..) {
+            match bank_thread_hdl.join() {
+                Ok(more_unprocessed_packets) => {
+                    unprocessed_packets.extend(more_unprocessed_packets)
+                }
+                err => warn!("bank_thread_hdl join failed: {:?}", err),
+            }
+        }
+        unprocessed_packets
+    }
+}
+
+impl Service for BankingStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for bank_thread_hdl in self.bank_thread_hdls {
+            bank_thread_hdl.join()?;
+        }
+        self.exit.store(true, Ordering::Relaxed);
+        self.leader_confirmation_service.join()?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::entry::EntrySlice;
+    use crate::packet::to_packets;
+    use crate::poh_service::{PohService, PohServiceConfig};
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::native_program::ProgramError;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
+    use std::thread::sleep;
+
+    fn create_test_recorder(bank: &Arc<Bank>) -> (Arc<Mutex<PohRecorder>>, PohService) {
+        let exit = Arc::new(AtomicBool::new(false));
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(
+            bank.tick_height(),
+            bank.last_id(),
+        )));
+        let poh_service = PohService::new(
+            poh_recorder.clone(),
+            &PohServiceConfig::default(),
+            exit.clone(),
+        );
+        (poh_recorder, poh_service)
+    }
+
+    #[test]
+    fn test_banking_stage_shutdown1() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let (verified_sender, verified_receiver) = channel();
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (banking_stage, _entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            DEFAULT_TICKS_PER_SLOT,
+            genesis_block.bootstrap_leader_id,
+        );
+        drop(verified_sender);
+        banking_stage.join().unwrap();
+        poh_service.close().unwrap();
+    }
+
+    #[test]
+    fn test_banking_stage_tick() {
+        let (mut genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        genesis_block.ticks_per_slot = 4;
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let start_hash = bank.last_id();
+        let (verified_sender, verified_receiver) = channel();
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (banking_stage, entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            genesis_block.ticks_per_slot - 1,
+            genesis_block.bootstrap_leader_id,
+        );
+        sleep(Duration::from_millis(600));
+        drop(verified_sender);
+
+        let entries: Vec<_> = entry_receiver
+            .iter()
+            .flat_map(|x| x.into_iter().map(|e| e.0))
+            .collect();
+        assert_eq!(entries.len(), genesis_block.ticks_per_slot as usize - 1);
+        assert!(entries.verify(&start_hash));
+        assert_eq!(entries[entries.len() - 1].hash, bank.last_id());
+        banking_stage.join().unwrap();
+        poh_service.close().unwrap();
+    }
+
+    #[test]
+    fn test_banking_stage_entries_only() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let start_hash = bank.last_id();
+        let (verified_sender, verified_receiver) = channel();
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (banking_stage, entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            DEFAULT_TICKS_PER_SLOT,
+            genesis_block.bootstrap_leader_id,
+        );
+
+        // good tx
+        let keypair = mint_keypair;
+        let tx = SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, start_hash, 0);
+
+        // good tx, but no verify
+        let tx_no_ver =
+            SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, start_hash, 0);
+
+        // bad tx, AccountNotFound
+        let keypair = Keypair::new();
+        let tx_anf = SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, start_hash, 0);
+
+        // send 'em over
+        let packets = to_packets(&[tx, tx_no_ver, tx_anf]);
+
+        // glad they all fit
+        assert_eq!(packets.len(), 1);
+        verified_sender // tx, no_ver, anf
+            .send(vec![(packets[0].clone(), vec![1u8, 0u8, 1u8])])
+            .unwrap();
+
+        drop(verified_sender);
+
+        //receive entries + ticks
+        let entries: Vec<Vec<Entry>> = entry_receiver
+            .iter()
+            .map(|x| x.into_iter().map(|e| e.0).collect())
+            .collect();
+
+        assert!(entries.len() >= 1);
+
+        let mut last_id = start_hash;
+        entries.iter().for_each(|entries| {
+            assert_eq!(entries.len(), 1);
+            assert!(entries.verify(&last_id));
+            last_id = entries.last().unwrap().hash;
+        });
+        drop(entry_receiver);
+        banking_stage.join().unwrap();
+        poh_service.close().unwrap();
+    }
+    #[test]
+    #[ignore] //flaky
+    fn test_banking_stage_entryfication() {
+        // In this attack we'll demonstrate that a verifier can interpret the ledger
+        // differently if either the server doesn't signal the ledger to add an
+        // Entry OR if the verifier tries to parallelize across multiple Entries.
+        let (genesis_block, mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let (verified_sender, verified_receiver) = channel();
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (banking_stage, entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            DEFAULT_TICKS_PER_SLOT,
+            genesis_block.bootstrap_leader_id,
+        );
+
+        // Process a batch that includes a transaction that receives two tokens.
+        let alice = Keypair::new();
+        let tx = SystemTransaction::new_account(
+            &mint_keypair,
+            alice.pubkey(),
+            2,
+            genesis_block.hash(),
+            0,
+        );
+
+        let packets = to_packets(&[tx]);
+        verified_sender
+            .send(vec![(packets[0].clone(), vec![1u8])])
+            .unwrap();
+
+        // Process a second batch that spends one of those tokens.
+        let tx = SystemTransaction::new_account(
+            &alice,
+            mint_keypair.pubkey(),
+            1,
+            genesis_block.hash(),
+            0,
+        );
+        let packets = to_packets(&[tx]);
+        verified_sender
+            .send(vec![(packets[0].clone(), vec![1u8])])
+            .unwrap();
+        drop(verified_sender);
+        banking_stage.join().unwrap();
+
+        // Collect the ledger and feed it to a new bank.
+        let entries: Vec<_> = entry_receiver
+            .iter()
+            .flat_map(|x| x.into_iter().map(|e| e.0))
+            .collect();
+        // same assertion as running through the bank, really...
+        assert!(entries.len() >= 2);
+
+        // Assert the user holds one token, not two. If the stage only outputs one
+        // entry, then the second transaction will be rejected, because it drives
+        // the account balance below zero before the credit is added.
+        let bank = Bank::new(&genesis_block);
+        for entry in entries {
+            bank.process_transactions(&entry.transactions)
+                .iter()
+                .for_each(|x| assert_eq!(*x, Ok(())));
+        }
+        assert_eq!(bank.get_balance(&alice.pubkey()), 1);
+        poh_service.close().unwrap();
+    }
+
+    // Test that when the max_tick_height is reached, the banking stage exits
+    #[test]
+    fn test_max_tick_height_shutdown() {
+        solana_logger::setup();
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let (verified_sender, verified_receiver) = channel();
+        let max_tick_height = 10;
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (banking_stage, _entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            max_tick_height,
+            genesis_block.bootstrap_leader_id,
+        );
+
+        loop {
+            let bank_tick_height = bank.tick_height();
+            if bank_tick_height >= max_tick_height {
+                break;
+            }
+            sleep(Duration::from_millis(10));
+        }
+
+        drop(verified_sender);
+        banking_stage.join().unwrap();
+        poh_service.close().unwrap();
+    }
+
+    #[test]
+    fn test_returns_unprocessed_packet() {
+        solana_logger::setup();
+        let (genesis_block, mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let ticks_per_slot = 1;
+        let (verified_sender, verified_receiver) = channel();
+        let (poh_recorder, poh_service) = create_test_recorder(&bank);
+        let (mut banking_stage, _entry_receiver) = BankingStage::new(
+            &bank,
+            &poh_recorder,
+            verified_receiver,
+            ticks_per_slot,
+            genesis_block.bootstrap_leader_id,
+        );
+
+        // Wait for Poh recorder to hit max height
+        loop {
+            let bank_tick_height = bank.tick_height();
+            if bank_tick_height >= ticks_per_slot {
+                break;
+            }
+            sleep(Duration::from_millis(10));
+        }
+
+        // Now send a transaction to the banking stage
+        let transaction = SystemTransaction::new_account(
+            &mint_keypair,
+            Keypair::new().pubkey(),
+            2,
+            genesis_block.hash(),
+            0,
+        );
+
+        let packets = to_packets(&[transaction]);
+        verified_sender
+            .send(vec![(packets[0].clone(), vec![1u8])])
+            .unwrap();
+
+        // Shut down the banking stage, it should give back the transaction
+        drop(verified_sender);
+        let unprocessed_packets = banking_stage.join_and_collect_unprocessed_packets();
+        assert_eq!(unprocessed_packets.len(), 1);
+        let (packets, start_index) = &unprocessed_packets[0];
+        assert_eq!(packets.read().unwrap().packets.len(), 1); // TODO: maybe compare actual packet contents too
+        assert_eq!(*start_index, 0);
+        poh_service.close().unwrap();
+    }
+
+    #[test]
+    fn test_bank_record_transactions() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(10_000);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let (entry_sender, entry_receiver) = channel();
+        let working_bank = WorkingBank {
+            bank: bank.clone(),
+            sender: entry_sender,
+            min_tick_height: bank.tick_height(),
+            max_tick_height: std::u64::MAX,
+        };
+
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(
+            bank.tick_height(),
+            bank.last_id(),
+        )));
+        poh_recorder.lock().unwrap().set_working_bank(working_bank);
+        let pubkey = Keypair::new().pubkey();
+
+        let transactions = vec![
+            SystemTransaction::new_move(&mint_keypair, pubkey, 1, genesis_block.hash(), 0),
+            SystemTransaction::new_move(&mint_keypair, pubkey, 1, genesis_block.hash(), 0),
+        ];
+
+        let mut results = vec![Ok(()), Ok(())];
+        BankingStage::record_transactions(&transactions, &results, &poh_recorder).unwrap();
+        let entries = entry_receiver.recv().unwrap();
+        assert_eq!(entries[0].0.transactions.len(), transactions.len());
+
+        // ProgramErrors should still be recorded
+        results[0] = Err(BankError::ProgramError(
+            1,
+            ProgramError::ResultWithNegativeTokens,
+        ));
+        BankingStage::record_transactions(&transactions, &results, &poh_recorder).unwrap();
+        let entries = entry_receiver.recv().unwrap();
+        assert_eq!(entries[0].0.transactions.len(), transactions.len());
+
+        // Other BankErrors should not be recorded
+        results[0] = Err(BankError::AccountNotFound);
+        BankingStage::record_transactions(&transactions, &results, &poh_recorder).unwrap();
+        let entries = entry_receiver.recv().unwrap();
+        assert_eq!(entries[0].0.transactions.len(), transactions.len() - 1);
+    }
+
+    #[test]
+    fn test_bank_process_and_record_transactions() {
+        solana_logger::setup();
+        let (genesis_block, mint_keypair) = GenesisBlock::new(10_000);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let pubkey = Keypair::new().pubkey();
+
+        let transactions = vec![SystemTransaction::new_move(
+            &mint_keypair,
+            pubkey,
+            1,
+            genesis_block.hash(),
+            0,
+        )];
+
+        let (entry_sender, entry_receiver) = channel();
+        let working_bank = WorkingBank {
+            bank: bank.clone(),
+            sender: entry_sender,
+            min_tick_height: bank.tick_height(),
+            max_tick_height: bank.tick_height() + 1,
+        };
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(
+            bank.tick_height(),
+            bank.last_id(),
+        )));
+        poh_recorder.lock().unwrap().set_working_bank(working_bank);
+
+        BankingStage::process_and_record_transactions(&bank, &transactions, &poh_recorder).unwrap();
+        poh_recorder.lock().unwrap().tick();
+
+        let mut need_tick = true;
+        // read entries until I find mine, might be ticks...
+        while let Ok(entries) = entry_receiver.recv() {
+            for (entry, _) in entries {
+                if !entry.is_tick() {
+                    trace!("got entry");
+                    assert_eq!(entry.transactions.len(), transactions.len());
+                    assert_eq!(bank.get_balance(&pubkey), 1);
+                    need_tick = false;
+                } else {
+                    break;
+                }
+            }
+        }
+
+        assert_eq!(need_tick, false);
+
+        let transactions = vec![SystemTransaction::new_move(
+            &mint_keypair,
+            pubkey,
+            2,
+            genesis_block.hash(),
+            0,
+        )];
+
+        assert_matches!(
+            BankingStage::process_and_record_transactions(&bank, &transactions, &poh_recorder,),
+            Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached))
+        );
+
+        assert_eq!(bank.get_balance(&pubkey), 1);
+    }
+}

core/src/blob_fetch_stage.rs

@@ -0,0 +1,46 @@
+//! The `blob_fetch_stage` pulls blobs from UDP sockets and sends it to a channel.
+
+use crate::service::Service;
+use crate::streamer::{self, BlobSender};
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::{self, JoinHandle};
+
+pub struct BlobFetchStage {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl BlobFetchStage {
+    pub fn new(socket: Arc<UdpSocket>, sender: &BlobSender, exit: Arc<AtomicBool>) -> Self {
+        Self::new_multi_socket(vec![socket], sender, exit)
+    }
+    pub fn new_multi_socket(
+        sockets: Vec<Arc<UdpSocket>>,
+        sender: &BlobSender,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let thread_hdls: Vec<_> = sockets
+            .into_iter()
+            .map(|socket| streamer::blob_receiver(socket, exit.clone(), sender.clone()))
+            .collect();
+
+        Self { exit, thread_hdls }
+    }
+
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for BlobFetchStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

core/src/blockstream.rs

@@ -0,0 +1,237 @@
+//! The `blockstream` module provides a method for streaming entries out via a
+//! local unix socket, to provide client services such as a block explorer with
+//! real-time access to entries.
+
+use crate::entry::Entry;
+use crate::result::Result;
+use chrono::{SecondsFormat, Utc};
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use std::cell::RefCell;
+use std::io::prelude::*;
+use std::net::Shutdown;
+use std::os::unix::net::UnixStream;
+use std::path::Path;
+
+pub trait EntryWriter: std::fmt::Debug {
+    fn write(&self, payload: String) -> Result<()>;
+}
+
+#[derive(Debug, Default)]
+pub struct EntryVec {
+    values: RefCell<Vec<String>>,
+}
+
+impl EntryWriter for EntryVec {
+    fn write(&self, payload: String) -> Result<()> {
+        self.values.borrow_mut().push(payload);
+        Ok(())
+    }
+}
+
+impl EntryVec {
+    pub fn new() -> Self {
+        EntryVec {
+            values: RefCell::new(Vec::new()),
+        }
+    }
+
+    pub fn entries(&self) -> Vec<String> {
+        self.values.borrow().clone()
+    }
+}
+
+#[derive(Debug)]
+pub struct EntrySocket {
+    socket: String,
+}
+
+const MESSAGE_TERMINATOR: &str = "\n";
+
+impl EntryWriter for EntrySocket {
+    fn write(&self, payload: String) -> Result<()> {
+        let mut socket = UnixStream::connect(Path::new(&self.socket))?;
+        socket.write_all(payload.as_bytes())?;
+        socket.write_all(MESSAGE_TERMINATOR.as_bytes())?;
+        socket.shutdown(Shutdown::Write)?;
+        Ok(())
+    }
+}
+
+pub trait BlockstreamEvents {
+    fn emit_entry_event(
+        &self,
+        slot: u64,
+        tick_height: u64,
+        leader_id: Pubkey,
+        entries: &Entry,
+    ) -> Result<()>;
+    fn emit_block_event(
+        &self,
+        slot: u64,
+        tick_height: u64,
+        leader_id: Pubkey,
+        last_id: Hash,
+    ) -> Result<()>;
+}
+
+#[derive(Debug)]
+pub struct Blockstream<T: EntryWriter> {
+    pub output: T,
+}
+
+impl<T> BlockstreamEvents for Blockstream<T>
+where
+    T: EntryWriter,
+{
+    fn emit_entry_event(
+        &self,
+        slot: u64,
+        tick_height: u64,
+        leader_id: Pubkey,
+        entry: &Entry,
+    ) -> Result<()> {
+        let json_entry = serde_json::to_string(&entry)?;
+        let payload = format!(
+            r#"{{"dt":"{}","t":"entry","s":{},"h":{},"l":"{:?}","entry":{}}}"#,
+            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
+            slot,
+            tick_height,
+            leader_id,
+            json_entry,
+        );
+        self.output.write(payload)?;
+        Ok(())
+    }
+
+    fn emit_block_event(
+        &self,
+        slot: u64,
+        tick_height: u64,
+        leader_id: Pubkey,
+        last_id: Hash,
+    ) -> Result<()> {
+        let payload = format!(
+            r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":"{:?}","id":"{:?}"}}"#,
+            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
+            slot,
+            tick_height,
+            leader_id,
+            last_id,
+        );
+        self.output.write(payload)?;
+        Ok(())
+    }
+}
+
+pub type SocketBlockstream = Blockstream<EntrySocket>;
+
+impl SocketBlockstream {
+    pub fn new(socket: String) -> Self {
+        Blockstream {
+            output: EntrySocket { socket },
+        }
+    }
+}
+
+pub type MockBlockstream = Blockstream<EntryVec>;
+
+impl MockBlockstream {
+    pub fn new(_: String) -> Self {
+        Blockstream {
+            output: EntryVec::new(),
+        }
+    }
+
+    pub fn entries(&self) -> Vec<String> {
+        self.output.entries()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::entry::Entry;
+    use chrono::{DateTime, FixedOffset};
+    use serde_json::Value;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_blockstream() -> () {
+        let blockstream = MockBlockstream::new("test_stream".to_string());
+        let ticks_per_slot = 5;
+
+        let mut last_id = Hash::default();
+        let mut entries = Vec::new();
+        let mut expected_entries = Vec::new();
+
+        let tick_height_initial = 0;
+        let tick_height_final = tick_height_initial + ticks_per_slot + 2;
+        let mut curr_slot = 0;
+        let leader_id = Keypair::new().pubkey();
+
+        for tick_height in tick_height_initial..=tick_height_final {
+            if tick_height == 5 {
+                blockstream
+                    .emit_block_event(curr_slot, tick_height - 1, leader_id, last_id)
+                    .unwrap();
+                curr_slot += 1;
+            }
+            let entry = Entry::new(&mut last_id, 1, vec![]); // just ticks
+            last_id = entry.hash;
+            blockstream
+                .emit_entry_event(curr_slot, tick_height, leader_id, &entry)
+                .unwrap();
+            expected_entries.push(entry.clone());
+            entries.push(entry);
+        }
+
+        assert_eq!(
+            blockstream.entries().len() as u64,
+            // one entry per tick (0..=N+2) is +3, plus one block
+            ticks_per_slot + 3 + 1
+        );
+
+        let mut j = 0;
+        let mut matched_entries = 0;
+        let mut matched_slots = HashSet::new();
+        let mut matched_blocks = HashSet::new();
+
+        for item in blockstream.entries() {
+            let json: Value = serde_json::from_str(&item).unwrap();
+            let dt_str = json["dt"].as_str().unwrap();
+
+            // Ensure `ts` field parses as valid DateTime
+            let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
+
+            let item_type = json["t"].as_str().unwrap();
+            match item_type {
+                "block" => {
+                    let id = json["id"].to_string();
+                    matched_blocks.insert(id);
+                }
+
+                "entry" => {
+                    let slot = json["s"].as_u64().unwrap();
+                    matched_slots.insert(slot);
+                    let entry_obj = json["entry"].clone();
+                    let entry: Entry = serde_json::from_value(entry_obj).unwrap();
+
+                    assert_eq!(entry, expected_entries[j]);
+                    matched_entries += 1;
+                    j += 1;
+                }
+
+                _ => {
+                    assert!(false, "unknown item type {}", item);
+                }
+            }
+        }
+
+        assert_eq!(matched_entries, expected_entries.len());
+        assert_eq!(matched_slots.len(), 2);
+        assert_eq!(matched_blocks.len(), 1);
+    }
+}

core/src/blockstream_service.rs

@@ -0,0 +1,197 @@
+//! The `blockstream_service` implements optional streaming of entries and block metadata
+//! using the `blockstream` module, providing client services such as a block explorer with
+//! real-time access to entries.
+
+use crate::blockstream::BlockstreamEvents;
+#[cfg(test)]
+use crate::blockstream::MockBlockstream as Blockstream;
+#[cfg(not(test))]
+use crate::blockstream::SocketBlockstream as Blockstream;
+use crate::blocktree::Blocktree;
+use crate::result::{Error, Result};
+use crate::service::Service;
+use solana_sdk::pubkey::Pubkey;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError};
+use std::sync::Arc;
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct BlockstreamService {
+    t_blockstream: JoinHandle<()>,
+}
+
+impl BlockstreamService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        slot_full_receiver: Receiver<(u64, Pubkey)>,
+        blocktree: Arc<Blocktree>,
+        blockstream_socket: String,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let mut blockstream = Blockstream::new(blockstream_socket);
+        let t_blockstream = Builder::new()
+            .name("solana-blockstream".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                if let Err(e) =
+                    Self::process_entries(&slot_full_receiver, &blocktree, &mut blockstream)
+                {
+                    match e {
+                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                        _ => info!("Error from process_entries: {:?}", e),
+                    }
+                }
+            })
+            .unwrap();
+        Self { t_blockstream }
+    }
+    fn process_entries(
+        slot_full_receiver: &Receiver<(u64, Pubkey)>,
+        blocktree: &Arc<Blocktree>,
+        blockstream: &mut Blockstream,
+    ) -> Result<()> {
+        let timeout = Duration::new(1, 0);
+        let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
+
+        let entries = blocktree.get_slot_entries(slot, 0, None).unwrap();
+        let blocktree_meta = blocktree.meta(slot).unwrap().unwrap();
+        let _parent_slot = if slot == 0 {
+            None
+        } else {
+            Some(blocktree_meta.parent_slot)
+        };
+        let ticks_per_slot = entries
+            .iter()
+            .filter(|entry| entry.is_tick())
+            .fold(0, |acc, _| acc + 1);
+        let mut tick_height = if slot > 0 {
+            ticks_per_slot * slot - 1
+        } else {
+            0
+        };
+
+        for (i, entry) in entries.iter().enumerate() {
+            if entry.is_tick() {
+                tick_height += 1;
+            }
+            blockstream
+                .emit_entry_event(slot, tick_height, slot_leader, &entry)
+                .unwrap_or_else(|e| {
+                    debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
+                });
+            if i == entries.len() - 1 {
+                blockstream
+                    .emit_block_event(slot, tick_height, slot_leader, entry.hash)
+                    .unwrap_or_else(|e| {
+                        debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
+                    });
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Service for BlockstreamService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.t_blockstream.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::blocktree::create_new_tmp_ledger;
+    use crate::entry::{create_ticks, Entry};
+    use chrono::{DateTime, FixedOffset};
+    use serde_json::Value;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use std::sync::mpsc::channel;
+
+    #[test]
+    fn test_blockstream_service_process_entries() {
+        let ticks_per_slot = 5;
+        let leader_id = Keypair::new().pubkey();
+
+        // Set up genesis block and blocktree
+        let (mut genesis_block, _mint_keypair) = GenesisBlock::new(1000);
+        genesis_block.ticks_per_slot = ticks_per_slot;
+
+        let (ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap();
+
+        // Set up blockstream
+        let mut blockstream = Blockstream::new("test_stream".to_string());
+
+        // Set up dummy channel to receive a full-slot notification
+        let (slot_full_sender, slot_full_receiver) = channel();
+
+        // Create entries - 4 ticks + 1 populated entry + 1 tick
+        let mut entries = create_ticks(4, Hash::default());
+
+        let keypair = Keypair::new();
+        let mut last_id = entries[3].hash;
+        let tx = SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, Hash::default(), 0);
+        let entry = Entry::new(&mut last_id, 1, vec![tx]);
+        last_id = entry.hash;
+        entries.push(entry);
+        let final_tick = create_ticks(1, last_id);
+        entries.extend_from_slice(&final_tick);
+
+        let expected_entries = entries.clone();
+        let expected_tick_heights = [5, 6, 7, 8, 8, 9];
+
+        blocktree.write_entries(1, 0, 0, &entries).unwrap();
+
+        slot_full_sender.send((1, leader_id)).unwrap();
+        BlockstreamService::process_entries(
+            &slot_full_receiver,
+            &Arc::new(blocktree),
+            &mut blockstream,
+        )
+        .unwrap();
+        assert_eq!(blockstream.entries().len(), 7);
+
+        let (entry_events, block_events): (Vec<Value>, Vec<Value>) = blockstream
+            .entries()
+            .iter()
+            .map(|item| {
+                let json: Value = serde_json::from_str(&item).unwrap();
+                let dt_str = json["dt"].as_str().unwrap();
+                // Ensure `ts` field parses as valid DateTime
+                let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
+                json
+            })
+            .partition(|json| {
+                let item_type = json["t"].as_str().unwrap();
+                item_type == "entry"
+            });
+        for (i, json) in entry_events.iter().enumerate() {
+            let height = json["h"].as_u64().unwrap();
+            assert_eq!(height, expected_tick_heights[i]);
+            let entry_obj = json["entry"].clone();
+            let tx = entry_obj["transactions"].as_array().unwrap();
+            if tx.len() == 0 {
+                // TODO: There is a bug in Transaction deserialize methods such that
+                // `serde_json::from_str` does not work for populated Entries.
+                // Remove this `if` when fixed.
+                let entry: Entry = serde_json::from_value(entry_obj).unwrap();
+                assert_eq!(entry, expected_entries[i]);
+            }
+        }
+        for json in block_events {
+            let slot = json["s"].as_u64().unwrap();
+            assert_eq!(1, slot);
+            let height = json["h"].as_u64().unwrap();
+            assert_eq!(2 * ticks_per_slot - 1, height);
+        }
+    }
+}

core/src/blocktree_processor.rs

@@ -0,0 +1,690 @@
+use crate::bank_forks::BankForks;
+use crate::blocktree::Blocktree;
+use crate::entry::{Entry, EntrySlice};
+use crate::leader_schedule_utils;
+use rayon::prelude::*;
+use solana_metrics::counter::Counter;
+use solana_runtime::bank::{Bank, BankError, Result};
+use solana_sdk::genesis_block::GenesisBlock;
+use solana_sdk::timing::duration_as_ms;
+use solana_sdk::timing::MAX_RECENT_TICK_HASHES;
+use std::sync::Arc;
+use std::time::Instant;
+
+pub fn process_entry(bank: &Bank, entry: &Entry) -> Result<()> {
+    if !entry.is_tick() {
+        first_err(&bank.process_transactions(&entry.transactions))?;
+    } else {
+        bank.register_tick(&entry.hash);
+    }
+    Ok(())
+}
+
+fn first_err(results: &[Result<()>]) -> Result<()> {
+    for r in results {
+        r.clone()?;
+    }
+    Ok(())
+}
+
+fn par_execute_entries(bank: &Bank, entries: &[(&Entry, Vec<Result<()>>)]) -> Result<()> {
+    inc_new_counter_info!("bank-par_execute_entries-count", entries.len());
+    let results: Vec<Result<()>> = entries
+        .into_par_iter()
+        .map(|(e, lock_results)| {
+            let results = bank.load_execute_and_commit_transactions(
+                &e.transactions,
+                lock_results.to_vec(),
+                MAX_RECENT_TICK_HASHES,
+            );
+            bank.unlock_accounts(&e.transactions, &results);
+            first_err(&results)
+        })
+        .collect();
+
+    first_err(&results)
+}
+
+/// process entries in parallel
+/// 1. In order lock accounts for each entry while the lock succeeds, up to a Tick entry
+/// 2. Process the locked group in parallel
+/// 3. Register the `Tick` if it's available
+/// 4. Update the leader scheduler, goto 1
+fn par_process_entries(bank: &Bank, entries: &[Entry]) -> Result<()> {
+    // accumulator for entries that can be processed in parallel
+    let mut mt_group = vec![];
+    for entry in entries {
+        if entry.is_tick() {
+            // if its a tick, execute the group and register the tick
+            par_execute_entries(bank, &mt_group)?;
+            bank.register_tick(&entry.hash);
+            mt_group = vec![];
+            continue;
+        }
+        // try to lock the accounts
+        let lock_results = bank.lock_accounts(&entry.transactions);
+        // if any of the locks error out
+        // execute the current group
+        if first_err(&lock_results).is_err() {
+            par_execute_entries(bank, &mt_group)?;
+            mt_group = vec![];
+            //reset the lock and push the entry
+            bank.unlock_accounts(&entry.transactions, &lock_results);
+            let lock_results = bank.lock_accounts(&entry.transactions);
+            mt_group.push((entry, lock_results));
+        } else {
+            // push the entry to the mt_group
+            mt_group.push((entry, lock_results));
+        }
+    }
+    par_execute_entries(bank, &mt_group)?;
+    Ok(())
+}
+
+/// Process an ordered list of entries.
+pub fn process_entries(bank: &Bank, entries: &[Entry]) -> Result<()> {
+    par_process_entries(bank, entries)
+}
+
+/// Process an ordered list of entries, populating a circular buffer "tail"
+/// as we go.
+fn process_block(bank: &Bank, entries: &[Entry]) -> Result<()> {
+    for entry in entries {
+        process_entry(bank, entry)?;
+    }
+
+    Ok(())
+}
+
+#[derive(Debug, PartialEq)]
+pub struct BankForksInfo {
+    pub bank_id: u64,
+    pub entry_height: u64,
+}
+
+pub fn process_blocktree(
+    genesis_block: &GenesisBlock,
+    blocktree: &Blocktree,
+    account_paths: Option<String>,
+) -> Result<(BankForks, Vec<BankForksInfo>)> {
+    let now = Instant::now();
+    info!("processing ledger...");
+
+    // Setup bank for slot 0
+    let mut pending_slots = {
+        let slot = 0;
+        let bank = Arc::new(Bank::new_with_paths(&genesis_block, account_paths));
+        let entry_height = 0;
+        let last_entry_hash = bank.last_id();
+        vec![(slot, bank, entry_height, last_entry_hash)]
+    };
+
+    let mut fork_info = vec![];
+    while !pending_slots.is_empty() {
+        let (slot, starting_bank, starting_entry_height, mut last_entry_hash) =
+            pending_slots.pop().unwrap();
+
+        let bank = Arc::new(Bank::new_from_parent(
+            &starting_bank,
+            leader_schedule_utils::slot_leader_at(slot, &starting_bank),
+            starting_bank.slot(),
+        ));
+        let mut entry_height = starting_entry_height;
+
+        // Load the metadata for this slot
+        let meta = blocktree
+            .meta(slot)
+            .map_err(|err| {
+                warn!("Failed to load meta for slot {}: {:?}", slot, err);
+                BankError::LedgerVerificationFailed
+            })?
+            .unwrap();
+
+        // Fetch all entries for this slot
+        let mut entries = blocktree.get_slot_entries(slot, 0, None).map_err(|err| {
+            warn!("Failed to load entries for slot {}: {:?}", slot, err);
+            BankError::LedgerVerificationFailed
+        })?;
+
+        if slot == 0 {
+            // The first entry in the ledger is a pseudo-tick used only to ensure the number of ticks
+            // in slot 0 is the same as the number of ticks in all subsequent slots.  It is not
+            // processed by the bank, skip over it.
+            if entries.is_empty() {
+                warn!("entry0 not present");
+                return Err(BankError::LedgerVerificationFailed);
+            }
+            let entry0 = &entries[0];
+            if !(entry0.is_tick() && entry0.verify(&last_entry_hash)) {
+                warn!("Ledger proof of history failed at entry0");
+                return Err(BankError::LedgerVerificationFailed);
+            }
+            last_entry_hash = entry0.hash;
+            entry_height += 1;
+            entries = entries.drain(1..).collect();
+        }
+
+        if !entries.is_empty() {
+            if !entries.verify(&last_entry_hash) {
+                warn!("Ledger proof of history failed at entry: {}", entry_height);
+                return Err(BankError::LedgerVerificationFailed);
+            }
+
+            process_block(&bank, &entries).map_err(|err| {
+                warn!("Failed to process entries for slot {}: {:?}", slot, err);
+                BankError::LedgerVerificationFailed
+            })?;
+
+            last_entry_hash = entries.last().unwrap().hash;
+            entry_height += entries.len() as u64;
+        }
+
+        let slot_complete =
+            leader_schedule_utils::num_ticks_left_in_slot(&bank, bank.tick_height()) == 0;
+        if !slot_complete {
+            // Slot was not complete, clear out any partial entries
+            // TODO: Walk |meta.next_slots| and clear all child slots too?
+            blocktree.reset_slot_consumed(slot).map_err(|err| {
+                warn!("Failed to reset partial slot {}: {:?}", slot, err);
+                BankError::LedgerVerificationFailed
+            })?;
+
+            let bfi = BankForksInfo {
+                bank_id: slot,
+                entry_height: starting_entry_height,
+            };
+            fork_info.push((starting_bank, bfi));
+            continue;
+        }
+
+        // TODO merge with locktower, voting
+        bank.squash();
+
+        if meta.next_slots.is_empty() {
+            // Reached the end of this fork.  Record the final entry height and last entry.hash
+
+            let bfi = BankForksInfo {
+                bank_id: slot,
+                entry_height,
+            };
+            fork_info.push((bank, bfi));
+            continue;
+        }
+
+        // This is a fork point, create a new child bank for each fork
+        pending_slots.extend(meta.next_slots.iter().map(|next_slot| {
+            let child_bank = Arc::new(Bank::new_from_parent(
+                &bank,
+                leader_schedule_utils::slot_leader_at(*next_slot, &bank),
+                *next_slot,
+            ));
+            trace!("Add child bank for slot={}", next_slot);
+            // bank_forks.insert(*next_slot, child_bank);
+            (*next_slot, child_bank, entry_height, last_entry_hash)
+        }));
+
+        // reverse sort by slot, so the next slot to be processed can be pop()ed
+        // TODO: remove me once leader_scheduler can hang with out-of-order slots?
+        pending_slots.sort_by(|a, b| b.0.cmp(&a.0));
+    }
+
+    let (banks, bank_forks_info): (Vec<_>, Vec<_>) = fork_info.into_iter().unzip();
+    let bank_forks = BankForks::new_from_banks(&banks);
+    info!(
+        "processed ledger in {}ms, forks={}...",
+        duration_as_ms(&now.elapsed()),
+        bank_forks_info.len(),
+    );
+
+    Ok((bank_forks, bank_forks_info))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::blocktree::create_new_tmp_ledger;
+    use crate::blocktree::tests::entries_to_blobs;
+    use crate::entry::{create_ticks, next_entry, Entry};
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+
+    fn fill_blocktree_slot_with_ticks(
+        blocktree: &Blocktree,
+        ticks_per_slot: u64,
+        slot: u64,
+        parent_slot: u64,
+        last_entry_hash: Hash,
+    ) -> Hash {
+        let entries = create_ticks(ticks_per_slot, last_entry_hash);
+        let last_entry_hash = entries.last().unwrap().hash;
+
+        let blobs = entries_to_blobs(&entries, slot, parent_slot);
+        blocktree.insert_data_blobs(blobs.iter()).unwrap();
+
+        last_entry_hash
+    }
+
+    #[test]
+    fn test_process_blocktree_with_incomplete_slot() {
+        solana_logger::setup();
+
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(10_000);
+        let ticks_per_slot = genesis_block.ticks_per_slot;
+
+        /*
+          Build a blocktree in the ledger with the following fork structure:
+
+               slot 0
+                 |
+               slot 1
+                 |
+               slot 2
+
+           where slot 1 is incomplete (missing 1 tick at the end)
+        */
+
+        // Create a new ledger with slot 0 full of ticks
+        let (ledger_path, mut last_id) = create_new_tmp_ledger!(&genesis_block);
+        debug!("ledger_path: {:?}", ledger_path);
+
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot)
+            .expect("Expected to successfully open database ledger");
+
+        // Write slot 1
+        // slot 1, points at slot 0.  Missing one tick
+        {
+            let parent_slot = 0;
+            let slot = 1;
+            let mut entries = create_ticks(ticks_per_slot, last_id);
+            last_id = entries.last().unwrap().hash;
+
+            entries.pop();
+
+            let blobs = entries_to_blobs(&entries, slot, parent_slot);
+            blocktree.insert_data_blobs(blobs.iter()).unwrap();
+        }
+
+        // slot 2, points at slot 1
+        fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 1, last_id);
+
+        let (mut _bank_forks, bank_forks_info) =
+            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+
+        assert_eq!(bank_forks_info.len(), 1);
+        assert_eq!(
+            bank_forks_info[0],
+            BankForksInfo {
+                bank_id: 1, // never finished first slot
+                entry_height: ticks_per_slot,
+            }
+        );
+    }
+
+    #[test]
+    fn test_process_blocktree_with_two_forks() {
+        solana_logger::setup();
+
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(10_000);
+        let ticks_per_slot = genesis_block.ticks_per_slot;
+
+        // Create a new ledger with slot 0 full of ticks
+        let (ledger_path, last_id) = create_new_tmp_ledger!(&genesis_block);
+        debug!("ledger_path: {:?}", ledger_path);
+        let mut last_entry_hash = last_id;
+
+        /*
+            Build a blocktree in the ledger with the following fork structure:
+
+                 slot 0
+                   |
+                 slot 1
+                 /   \
+            slot 2   |
+               /     |
+            slot 3   |
+                     |
+                   slot 4
+
+        */
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot)
+            .expect("Expected to successfully open database ledger");
+
+        // Fork 1, ending at slot 3
+        let last_slot1_entry_hash =
+            fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 1, 0, last_entry_hash);
+        last_entry_hash =
+            fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 2, 1, last_slot1_entry_hash);
+        let last_fork1_entry_hash =
+            fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 3, 2, last_entry_hash);
+
+        // Fork 2, ending at slot 4
+        let last_fork2_entry_hash =
+            fill_blocktree_slot_with_ticks(&blocktree, ticks_per_slot, 4, 1, last_slot1_entry_hash);
+
+        info!("last_fork1_entry.hash: {:?}", last_fork1_entry_hash);
+        info!("last_fork2_entry.hash: {:?}", last_fork2_entry_hash);
+
+        let (bank_forks, bank_forks_info) =
+            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+
+        assert_eq!(bank_forks_info.len(), 2); // There are two forks
+        assert_eq!(
+            bank_forks_info[0],
+            BankForksInfo {
+                bank_id: 3, // Fork 1's head is slot 3
+                entry_height: ticks_per_slot * 4,
+            }
+        );
+        assert_eq!(
+            bank_forks_info[1],
+            BankForksInfo {
+                bank_id: 4, // Fork 2's head is slot 4
+                entry_height: ticks_per_slot * 3,
+            }
+        );
+
+        // Ensure bank_forks holds the right banks
+        for info in bank_forks_info {
+            assert_eq!(bank_forks[info.bank_id].slot(), info.bank_id);
+        }
+    }
+
+    #[test]
+    fn test_first_err() {
+        assert_eq!(first_err(&[Ok(())]), Ok(()));
+        assert_eq!(
+            first_err(&[Ok(()), Err(BankError::DuplicateSignature)]),
+            Err(BankError::DuplicateSignature)
+        );
+        assert_eq!(
+            first_err(&[
+                Ok(()),
+                Err(BankError::DuplicateSignature),
+                Err(BankError::AccountInUse)
+            ]),
+            Err(BankError::DuplicateSignature)
+        );
+        assert_eq!(
+            first_err(&[
+                Ok(()),
+                Err(BankError::AccountInUse),
+                Err(BankError::DuplicateSignature)
+            ]),
+            Err(BankError::AccountInUse)
+        );
+        assert_eq!(
+            first_err(&[
+                Err(BankError::AccountInUse),
+                Ok(()),
+                Err(BankError::DuplicateSignature)
+            ]),
+            Err(BankError::AccountInUse)
+        );
+    }
+
+    #[test]
+    fn test_process_empty_entry_is_registered() {
+        solana_logger::setup();
+
+        let (genesis_block, mint_keypair) = GenesisBlock::new(2);
+        let bank = Bank::new(&genesis_block);
+        let keypair = Keypair::new();
+        let slot_entries = create_ticks(genesis_block.ticks_per_slot - 1, genesis_block.hash());
+        let tx = SystemTransaction::new_account(
+            &mint_keypair,
+            keypair.pubkey(),
+            1,
+            slot_entries.last().unwrap().hash,
+            0,
+        );
+
+        // First, ensure the TX is rejected because of the unregistered last ID
+        assert_eq!(
+            bank.process_transaction(&tx),
+            Err(BankError::LastIdNotFound)
+        );
+
+        // Now ensure the TX is accepted despite pointing to the ID of an empty entry.
+        par_process_entries(&bank, &slot_entries).unwrap();
+        assert_eq!(bank.process_transaction(&tx), Ok(()));
+    }
+
+    #[test]
+    fn test_process_ledger_simple() {
+        solana_logger::setup();
+        let leader_pubkey = Keypair::new().pubkey();
+        let (genesis_block, mint_keypair) = GenesisBlock::new_with_leader(100, leader_pubkey, 50);
+        let (ledger_path, mut last_entry_hash) = create_new_tmp_ledger!(&genesis_block);
+        debug!("ledger_path: {:?}", ledger_path);
+
+        let mut entries = vec![];
+        let last_id = genesis_block.hash();
+        for _ in 0..3 {
+            // Transfer one token from the mint to a random account
+            let keypair = Keypair::new();
+            let tx = SystemTransaction::new_account(&mint_keypair, keypair.pubkey(), 1, last_id, 0);
+            let entry = Entry::new(&last_entry_hash, 1, vec![tx]);
+            last_entry_hash = entry.hash;
+            entries.push(entry);
+
+            // Add a second Transaction that will produce a
+            // ProgramError<0, ResultWithNegativeTokens> error when processed
+            let keypair2 = Keypair::new();
+            let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0);
+            let entry = Entry::new(&last_entry_hash, 1, vec![tx]);
+            last_entry_hash = entry.hash;
+            entries.push(entry);
+        }
+
+        // Fill up the rest of slot 1 with ticks
+        entries.extend(create_ticks(genesis_block.ticks_per_slot, last_entry_hash));
+
+        let blocktree =
+            Blocktree::open(&ledger_path).expect("Expected to successfully open database ledger");
+        blocktree.write_entries(1, 0, 0, &entries).unwrap();
+        let entry_height = genesis_block.ticks_per_slot + entries.len() as u64;
+        let (bank_forks, bank_forks_info) =
+            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+
+        assert_eq!(bank_forks_info.len(), 1);
+        assert_eq!(
+            bank_forks_info[0],
+            BankForksInfo {
+                bank_id: 1,
+                entry_height,
+            }
+        );
+
+        let bank = bank_forks[1].clone();
+        assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 50 - 3);
+        assert_eq!(bank.tick_height(), 2 * genesis_block.ticks_per_slot - 1);
+        assert_eq!(bank.last_id(), entries.last().unwrap().hash);
+    }
+
+    #[test]
+    fn test_process_ledger_with_one_tick_per_slot() {
+        let (mut genesis_block, _mint_keypair) = GenesisBlock::new(123);
+        genesis_block.ticks_per_slot = 1;
+        let (ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        let blocktree = Blocktree::open(&ledger_path).unwrap();
+        let (bank_forks, bank_forks_info) =
+            process_blocktree(&genesis_block, &blocktree, None).unwrap();
+
+        assert_eq!(bank_forks_info.len(), 1);
+        assert_eq!(
+            bank_forks_info[0],
+            BankForksInfo {
+                bank_id: 0,
+                entry_height: 1,
+            }
+        );
+        let bank = bank_forks[0].clone();
+        assert_eq!(bank.tick_height(), 0);
+    }
+
+    #[test]
+    fn test_par_process_entries_tick() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(1000);
+        let bank = Bank::new(&genesis_block);
+
+        // ensure bank can process a tick
+        assert_eq!(bank.tick_height(), 0);
+        let tick = next_entry(&genesis_block.hash(), 1, vec![]);
+        assert_eq!(par_process_entries(&bank, &[tick.clone()]), Ok(()));
+        assert_eq!(bank.tick_height(), 1);
+    }
+
+    #[test]
+    fn test_par_process_entries_2_entries_collision() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
+        let bank = Bank::new(&genesis_block);
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+
+        let last_id = bank.last_id();
+
+        // ensure bank can process 2 entries that have a common account and no tick is registered
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair1.pubkey(), 2, bank.last_id(), 0);
+        let entry_1 = next_entry(&last_id, 1, vec![tx]);
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair2.pubkey(), 2, bank.last_id(), 0);
+        let entry_2 = next_entry(&entry_1.hash, 1, vec![tx]);
+        assert_eq!(par_process_entries(&bank, &[entry_1, entry_2]), Ok(()));
+        assert_eq!(bank.get_balance(&keypair1.pubkey()), 2);
+        assert_eq!(bank.get_balance(&keypair2.pubkey()), 2);
+        assert_eq!(bank.last_id(), last_id);
+    }
+
+    #[test]
+    fn test_par_process_entries_2_txes_collision() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
+        let bank = Bank::new(&genesis_block);
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        let keypair3 = Keypair::new();
+
+        // fund: put 4 in each of 1 and 2
+        assert_matches!(
+            bank.transfer(4, &mint_keypair, keypair1.pubkey(), bank.last_id()),
+            Ok(_)
+        );
+        assert_matches!(
+            bank.transfer(4, &mint_keypair, keypair2.pubkey(), bank.last_id()),
+            Ok(_)
+        );
+
+        // construct an Entry whose 2nd transaction would cause a lock conflict with previous entry
+        let entry_1_to_mint = next_entry(
+            &bank.last_id(),
+            1,
+            vec![SystemTransaction::new_account(
+                &keypair1,
+                mint_keypair.pubkey(),
+                1,
+                bank.last_id(),
+                0,
+            )],
+        );
+
+        let entry_2_to_3_mint_to_1 = next_entry(
+            &entry_1_to_mint.hash,
+            1,
+            vec![
+                SystemTransaction::new_account(&keypair2, keypair3.pubkey(), 2, bank.last_id(), 0), // should be fine
+                SystemTransaction::new_account(
+                    &keypair1,
+                    mint_keypair.pubkey(),
+                    2,
+                    bank.last_id(),
+                    0,
+                ), // will collide
+            ],
+        );
+
+        assert_eq!(
+            par_process_entries(&bank, &[entry_1_to_mint, entry_2_to_3_mint_to_1]),
+            Ok(())
+        );
+
+        assert_eq!(bank.get_balance(&keypair1.pubkey()), 1);
+        assert_eq!(bank.get_balance(&keypair2.pubkey()), 2);
+        assert_eq!(bank.get_balance(&keypair3.pubkey()), 2);
+    }
+
+    #[test]
+    fn test_par_process_entries_2_entries_par() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
+        let bank = Bank::new(&genesis_block);
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        let keypair3 = Keypair::new();
+        let keypair4 = Keypair::new();
+
+        //load accounts
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair1.pubkey(), 1, bank.last_id(), 0);
+        assert_eq!(bank.process_transaction(&tx), Ok(()));
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair2.pubkey(), 1, bank.last_id(), 0);
+        assert_eq!(bank.process_transaction(&tx), Ok(()));
+
+        // ensure bank can process 2 entries that do not have a common account and no tick is registered
+        let last_id = bank.last_id();
+        let tx = SystemTransaction::new_account(&keypair1, keypair3.pubkey(), 1, bank.last_id(), 0);
+        let entry_1 = next_entry(&last_id, 1, vec![tx]);
+        let tx = SystemTransaction::new_account(&keypair2, keypair4.pubkey(), 1, bank.last_id(), 0);
+        let entry_2 = next_entry(&entry_1.hash, 1, vec![tx]);
+        assert_eq!(par_process_entries(&bank, &[entry_1, entry_2]), Ok(()));
+        assert_eq!(bank.get_balance(&keypair3.pubkey()), 1);
+        assert_eq!(bank.get_balance(&keypair4.pubkey()), 1);
+        assert_eq!(bank.last_id(), last_id);
+    }
+
+    #[test]
+    fn test_par_process_entries_2_entries_tick() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
+        let bank = Bank::new(&genesis_block);
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        let keypair3 = Keypair::new();
+        let keypair4 = Keypair::new();
+
+        //load accounts
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair1.pubkey(), 1, bank.last_id(), 0);
+        assert_eq!(bank.process_transaction(&tx), Ok(()));
+        let tx =
+            SystemTransaction::new_account(&mint_keypair, keypair2.pubkey(), 1, bank.last_id(), 0);
+        assert_eq!(bank.process_transaction(&tx), Ok(()));
+
+        let last_id = bank.last_id();
+        while last_id == bank.last_id() {
+            bank.register_tick(&Hash::default());
+        }
+
+        // ensure bank can process 2 entries that do not have a common account and tick is registered
+        let tx = SystemTransaction::new_account(&keypair2, keypair3.pubkey(), 1, last_id, 0);
+        let entry_1 = next_entry(&last_id, 1, vec![tx]);
+        let tick = next_entry(&entry_1.hash, 1, vec![]);
+        let tx = SystemTransaction::new_account(&keypair1, keypair4.pubkey(), 1, bank.last_id(), 0);
+        let entry_2 = next_entry(&tick.hash, 1, vec![tx]);
+        assert_eq!(
+            par_process_entries(&bank, &[entry_1.clone(), tick.clone(), entry_2.clone()]),
+            Ok(())
+        );
+        assert_eq!(bank.get_balance(&keypair3.pubkey()), 1);
+        assert_eq!(bank.get_balance(&keypair4.pubkey()), 1);
+
+        // ensure that an error is returned for an empty account (keypair2)
+        let tx = SystemTransaction::new_account(&keypair2, keypair3.pubkey(), 1, bank.last_id(), 0);
+        let entry_3 = next_entry(&entry_2.hash, 1, vec![tx]);
+        assert_eq!(
+            par_process_entries(&bank, &[entry_3]),
+            Err(BankError::AccountNotFound)
+        );
+    }
+}

core/src/broadcast_stage.rs

@@ -0,0 +1,390 @@
+//! A stage to broadcast data from a leader node to validators
+//!
+use crate::blocktree::Blocktree;
+use crate::cluster_info::{ClusterInfo, ClusterInfoError, NodeInfo, DATA_PLANE_FANOUT};
+use crate::entry::Entry;
+use crate::entry::EntrySlice;
+#[cfg(feature = "erasure")]
+use crate::erasure::CodingGenerator;
+use crate::packet::index_blobs;
+use crate::result::{Error, Result};
+use crate::service::Service;
+use crate::staking_utils;
+use rayon::prelude::*;
+use solana_metrics::counter::Counter;
+use solana_metrics::{influxdb, submit};
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing::duration_as_ms;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError};
+use std::sync::{Arc, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::{Duration, Instant};
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum BroadcastStageReturnType {
+    LeaderRotation,
+    ChannelDisconnected,
+    ExitSignal,
+}
+
+struct Broadcast {
+    id: Pubkey,
+    blob_index: u64,
+
+    #[cfg(feature = "erasure")]
+    coding_generator: CodingGenerator,
+}
+
+impl Broadcast {
+    fn run(
+        &mut self,
+        slot_height: u64,
+        max_tick_height: u64,
+        broadcast_table: &[NodeInfo],
+        receiver: &Receiver<Vec<(Entry, u64)>>,
+        sock: &UdpSocket,
+        blocktree: &Arc<Blocktree>,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let entries = receiver.recv_timeout(timer)?;
+        let now = Instant::now();
+        let mut num_entries = entries.len();
+        let mut ventries = Vec::new();
+        let mut last_tick = entries.last().map(|v| v.1).unwrap_or(0);
+        ventries.push(entries);
+
+        while let Ok(entries) = receiver.try_recv() {
+            num_entries += entries.len();
+            last_tick = entries.last().map(|v| v.1).unwrap_or(0);
+            ventries.push(entries);
+        }
+
+        inc_new_counter_info!("broadcast_service-entries_received", num_entries);
+
+        let to_blobs_start = Instant::now();
+
+        let blobs: Vec<_> = ventries
+            .into_par_iter()
+            .flat_map(|p| {
+                let entries: Vec<_> = p.into_iter().map(|e| e.0).collect();
+                entries.to_shared_blobs()
+            })
+            .collect();
+
+        // TODO: blob_index should be slot-relative...
+        index_blobs(&blobs, &self.id, &mut self.blob_index, slot_height);
+        let parent = {
+            if slot_height == 0 {
+                0
+            } else {
+                slot_height - 1
+            }
+        };
+        for b in blobs.iter() {
+            b.write().unwrap().set_parent(parent);
+        }
+
+        let to_blobs_elapsed = duration_as_ms(&to_blobs_start.elapsed());
+
+        let broadcast_start = Instant::now();
+
+        inc_new_counter_info!("streamer-broadcast-sent", blobs.len());
+
+        assert!(last_tick <= max_tick_height);
+        let contains_last_tick = last_tick == max_tick_height;
+
+        if contains_last_tick {
+            blobs.last().unwrap().write().unwrap().set_is_last_in_slot();
+        }
+
+        blocktree.write_shared_blobs(&blobs)?;
+
+        // Send out data
+        ClusterInfo::broadcast(&self.id, contains_last_tick, &broadcast_table, sock, &blobs)?;
+
+        // Fill in the coding blob data from the window data blobs
+        #[cfg(feature = "erasure")]
+        {
+            let coding = self.coding_generator.next(&blobs)?;
+
+            // send out erasures
+            ClusterInfo::broadcast(&self.id, false, &broadcast_table, sock, &coding)?;
+        }
+
+        let broadcast_elapsed = duration_as_ms(&broadcast_start.elapsed());
+
+        inc_new_counter_info!(
+            "broadcast_service-time_ms",
+            duration_as_ms(&now.elapsed()) as usize
+        );
+        info!(
+            "broadcast: {} entries, blob time {} broadcast time {}",
+            num_entries, to_blobs_elapsed, broadcast_elapsed
+        );
+
+        submit(
+            influxdb::Point::new("broadcast-service")
+                .add_field(
+                    "transmit-index",
+                    influxdb::Value::Integer(self.blob_index as i64),
+                )
+                .to_owned(),
+        );
+
+        Ok(())
+    }
+}
+
+// Implement a destructor for the BroadcastStage thread to signal it exited
+// even on panics
+struct Finalizer {
+    exit_sender: Arc<AtomicBool>,
+}
+
+impl Finalizer {
+    fn new(exit_sender: Arc<AtomicBool>) -> Self {
+        Finalizer { exit_sender }
+    }
+}
+// Implement a destructor for Finalizer.
+impl Drop for Finalizer {
+    fn drop(&mut self) {
+        self.exit_sender.clone().store(true, Ordering::Relaxed);
+    }
+}
+
+pub struct BroadcastStage {
+    thread_hdl: JoinHandle<BroadcastStageReturnType>,
+}
+
+impl BroadcastStage {
+    #[allow(clippy::too_many_arguments)]
+    fn run(
+        slot_height: u64,
+        bank: &Arc<Bank>,
+        sock: &UdpSocket,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        blob_index: u64,
+        receiver: &Receiver<Vec<(Entry, u64)>>,
+        exit_signal: &Arc<AtomicBool>,
+        blocktree: &Arc<Blocktree>,
+    ) -> BroadcastStageReturnType {
+        let me = cluster_info.read().unwrap().my_data().clone();
+
+        let mut broadcast = Broadcast {
+            id: me.id,
+            blob_index,
+            #[cfg(feature = "erasure")]
+            coding_generator: CodingGenerator::new(),
+        };
+
+        let max_tick_height = (slot_height + 1) * bank.ticks_per_slot() - 1;
+
+        loop {
+            if exit_signal.load(Ordering::Relaxed) {
+                return BroadcastStageReturnType::ExitSignal;
+            }
+            let mut broadcast_table = cluster_info
+                .read()
+                .unwrap()
+                .sorted_tvu_peers(&staking_utils::node_stakes(&bank));
+            // Layer 1, leader nodes are limited to the fanout size.
+            broadcast_table.truncate(DATA_PLANE_FANOUT);
+            inc_new_counter_info!("broadcast_service-num_peers", broadcast_table.len() + 1);
+            if let Err(e) = broadcast.run(
+                slot_height,
+                max_tick_height,
+                &broadcast_table,
+                receiver,
+                sock,
+                blocktree,
+            ) {
+                match e {
+                    Error::RecvTimeoutError(RecvTimeoutError::Disconnected) | Error::SendError => {
+                        return BroadcastStageReturnType::ChannelDisconnected;
+                    }
+                    Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                    Error::ClusterInfoError(ClusterInfoError::NoPeers) => (), // TODO: Why are the unit-tests throwing hundreds of these?
+                    _ => {
+                        inc_new_counter_info!("streamer-broadcaster-error", 1, 1);
+                        error!("broadcaster error: {:?}", e);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Service to broadcast messages from the leader to layer 1 nodes.
+    /// See `cluster_info` for network layer definitions.
+    /// # Arguments
+    /// * `sock` - Socket to send from.
+    /// * `exit` - Boolean to signal system exit.
+    /// * `cluster_info` - ClusterInfo structure
+    /// * `window` - Cache of blobs that we have broadcast
+    /// * `receiver` - Receive channel for blobs to be retransmitted to all the layer 1 nodes.
+    /// * `exit_sender` - Set to true when this service exits, allows rest of Tpu to exit cleanly.
+    /// Otherwise, when a Tpu closes, it only closes the stages that come after it. The stages
+    /// that come before could be blocked on a receive, and never notice that they need to
+    /// exit. Now, if any stage of the Tpu closes, it will lead to closing the WriteStage (b/c
+    /// WriteStage is the last stage in the pipeline), which will then close Broadcast service,
+    /// which will then close FetchStage in the Tpu, and then the rest of the Tpu,
+    /// completing the cycle.
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        slot_height: u64,
+        bank: &Arc<Bank>,
+        sock: UdpSocket,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        blob_index: u64,
+        receiver: Receiver<Vec<(Entry, u64)>>,
+        exit_sender: Arc<AtomicBool>,
+        blocktree: &Arc<Blocktree>,
+    ) -> Self {
+        let exit_signal = Arc::new(AtomicBool::new(false));
+        let blocktree = blocktree.clone();
+        let bank = bank.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-broadcaster".to_string())
+            .spawn(move || {
+                let _exit = Finalizer::new(exit_sender);
+                Self::run(
+                    slot_height,
+                    &bank,
+                    &sock,
+                    &cluster_info,
+                    blob_index,
+                    &receiver,
+                    &exit_signal,
+                    &blocktree,
+                )
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+}
+
+impl Service for BroadcastStage {
+    type JoinReturnType = BroadcastStageReturnType;
+
+    fn join(self) -> thread::Result<BroadcastStageReturnType> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::blocktree::{get_tmp_ledger_path, Blocktree};
+    use crate::cluster_info::{ClusterInfo, Node};
+    use crate::entry::create_ticks;
+    use crate::service::Service;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
+    use std::sync::atomic::AtomicBool;
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    struct MockBroadcastStage {
+        blocktree: Arc<Blocktree>,
+        broadcast_service: BroadcastStage,
+    }
+
+    fn setup_dummy_broadcast_service(
+        slot_height: u64,
+        leader_pubkey: Pubkey,
+        ledger_path: &str,
+        entry_receiver: Receiver<Vec<(Entry, u64)>>,
+        blob_index: u64,
+    ) -> MockBroadcastStage {
+        // Make the database ledger
+        let blocktree = Arc::new(Blocktree::open(ledger_path).unwrap());
+
+        // Make the leader node and scheduler
+        let leader_info = Node::new_localhost_with_pubkey(leader_pubkey);
+
+        // Make a node to broadcast to
+        let buddy_keypair = Keypair::new();
+        let broadcast_buddy = Node::new_localhost_with_pubkey(buddy_keypair.pubkey());
+
+        // Fill the cluster_info with the buddy's info
+        let mut cluster_info = ClusterInfo::new(leader_info.info.clone());
+        cluster_info.insert_info(broadcast_buddy.info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let exit_sender = Arc::new(AtomicBool::new(false));
+        let bank = Arc::new(Bank::default());
+
+        // Start up the broadcast stage
+        let broadcast_service = BroadcastStage::new(
+            slot_height,
+            &bank,
+            leader_info.sockets.broadcast,
+            cluster_info,
+            blob_index,
+            entry_receiver,
+            exit_sender,
+            &blocktree,
+        );
+
+        MockBroadcastStage {
+            blocktree,
+            broadcast_service,
+        }
+    }
+
+    #[test]
+    #[ignore]
+    //TODO this test won't work since broadcast stage no longer edits the ledger
+    fn test_broadcast_ledger() {
+        let ledger_path = get_tmp_ledger_path("test_broadcast_ledger");
+        {
+            // Create the leader scheduler
+            let leader_keypair = Keypair::new();
+            let start_tick_height = 0;
+            let max_tick_height = start_tick_height + DEFAULT_TICKS_PER_SLOT;
+
+            let (entry_sender, entry_receiver) = channel();
+            let broadcast_service = setup_dummy_broadcast_service(
+                0,
+                leader_keypair.pubkey(),
+                &ledger_path,
+                entry_receiver,
+                0,
+            );
+
+            let ticks = create_ticks(max_tick_height - start_tick_height, Hash::default());
+            for (i, tick) in ticks.into_iter().enumerate() {
+                entry_sender
+                    .send(vec![(tick, i as u64 + 1)])
+                    .expect("Expect successful send to broadcast service");
+            }
+
+            sleep(Duration::from_millis(2000));
+            let blocktree = broadcast_service.blocktree;
+            let mut blob_index = 0;
+            for i in 0..max_tick_height - start_tick_height {
+                let slot = (start_tick_height + i + 1) / DEFAULT_TICKS_PER_SLOT;
+
+                let result = blocktree.get_data_blob(slot, blob_index).unwrap();
+
+                blob_index += 1;
+                assert!(result.is_some());
+            }
+
+            drop(entry_sender);
+            broadcast_service
+                .broadcast_service
+                .join()
+                .expect("Expect successful join of broadcast service");
+        }
+
+        Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
+    }
+}

core/src/chacha.rs

@@ -0,0 +1,171 @@
+use crate::blocktree::Blocktree;
+use std::fs::File;
+use std::io;
+use std::io::{BufWriter, Write};
+use std::path::Path;
+use std::sync::Arc;
+
+use crate::storage_stage::ENTRIES_PER_SEGMENT;
+
+pub const CHACHA_BLOCK_SIZE: usize = 64;
+pub const CHACHA_KEY_SIZE: usize = 32;
+
+#[link(name = "cpu-crypt")]
+extern "C" {
+    fn chacha20_cbc_encrypt(
+        input: *const u8,
+        output: *mut u8,
+        in_len: usize,
+        key: *const u8,
+        ivec: *mut u8,
+    );
+}
+
+pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mut [u8]) {
+    unsafe {
+        chacha20_cbc_encrypt(
+            input.as_ptr(),
+            output.as_mut_ptr(),
+            input.len(),
+            key.as_ptr(),
+            ivec.as_mut_ptr(),
+        );
+    }
+}
+
+pub fn chacha_cbc_encrypt_ledger(
+    blocktree: &Arc<Blocktree>,
+    slice: u64,
+    out_path: &Path,
+    ivec: &mut [u8; CHACHA_BLOCK_SIZE],
+) -> io::Result<usize> {
+    let mut out_file =
+        BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
+    const BUFFER_SIZE: usize = 8 * 1024;
+    let mut buffer = [0; BUFFER_SIZE];
+    let mut encrypted_buffer = [0; BUFFER_SIZE];
+    let key = [0; CHACHA_KEY_SIZE];
+    let mut total_entries = 0;
+    let mut total_size = 0;
+    let mut entry = slice;
+
+    loop {
+        match blocktree.read_blobs_bytes(entry, ENTRIES_PER_SEGMENT - total_entries, &mut buffer, 0)
+        {
+            Ok((num_entries, entry_len)) => {
+                debug!(
+                    "chacha: encrypting slice: {} num_entries: {} entry_len: {}",
+                    slice, num_entries, entry_len
+                );
+                debug!("read {} bytes", entry_len);
+                let mut size = entry_len as usize;
+                if size == 0 {
+                    break;
+                }
+
+                if size < BUFFER_SIZE {
+                    // We are on the last block, round to the nearest key_size
+                    // boundary
+                    size = (size + CHACHA_KEY_SIZE - 1) & !(CHACHA_KEY_SIZE - 1);
+                }
+                total_size += size;
+
+                chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
+                if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
+                    warn!("Error writing file! {:?}", res);
+                    return Err(res);
+                }
+
+                total_entries += num_entries;
+                entry += num_entries;
+            }
+            Err(e) => {
+                info!("Error encrypting file: {:?}", e);
+                break;
+            }
+        }
+    }
+    Ok(total_size)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::blocktree::get_tmp_ledger_path;
+    use crate::blocktree::Blocktree;
+    use crate::chacha::chacha_cbc_encrypt_ledger;
+    use crate::entry::Entry;
+    use ring::signature::Ed25519KeyPair;
+    use solana_sdk::budget_transaction::BudgetTransaction;
+    use solana_sdk::hash::{hash, Hash, Hasher};
+    use solana_sdk::signature::KeypairUtil;
+    use std::fs::remove_file;
+    use std::fs::File;
+    use std::io::Read;
+    use std::path::Path;
+    use std::sync::Arc;
+    use untrusted::Input;
+
+    fn make_tiny_deterministic_test_entries(num: usize) -> Vec<Entry> {
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        let pkcs = [
+            48, 83, 2, 1, 1, 48, 5, 6, 3, 43, 101, 112, 4, 34, 4, 32, 109, 148, 235, 20, 97, 127,
+            43, 194, 109, 43, 121, 76, 54, 38, 234, 14, 108, 68, 209, 227, 137, 191, 167, 144, 177,
+            174, 57, 182, 79, 198, 196, 93, 161, 35, 3, 33, 0, 116, 121, 255, 78, 31, 95, 179, 172,
+            30, 125, 206, 87, 88, 78, 46, 145, 25, 154, 161, 252, 3, 58, 235, 116, 39, 148, 193,
+            150, 111, 61, 20, 226,
+        ];
+        let keypair = Ed25519KeyPair::from_pkcs8(Input::from(&pkcs)).unwrap();
+
+        let mut id = one;
+        let mut num_hashes = 0;
+        (0..num)
+            .map(|_| {
+                Entry::new_mut(
+                    &mut id,
+                    &mut num_hashes,
+                    vec![BudgetTransaction::new_signature(
+                        &keypair,
+                        keypair.pubkey(),
+                        keypair.pubkey(),
+                        one,
+                    )],
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn test_encrypt_ledger() {
+        solana_logger::setup();
+        let ledger_dir = "chacha_test_encrypt_file";
+        let ledger_path = get_tmp_ledger_path(ledger_dir);
+        let ticks_per_slot = 16;
+        let blocktree = Arc::new(Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap());
+        let out_path = Path::new("test_chacha_encrypt_file_output.txt.enc");
+
+        let entries = make_tiny_deterministic_test_entries(32);
+        blocktree.write_entries(0, 0, 0, &entries).unwrap();
+
+        let mut key = hex!(
+            "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
+                            abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
+        );
+        chacha_cbc_encrypt_ledger(&blocktree, 0, out_path, &mut key).unwrap();
+        let mut out_file = File::open(out_path).unwrap();
+        let mut buf = vec![];
+        let size = out_file.read_to_end(&mut buf).unwrap();
+        let mut hasher = Hasher::default();
+        hasher.hash(&buf[..size]);
+
+        use bs58;
+        //  golden needs to be updated if blob stuff changes....
+        let golden = Hash::new(
+            &bs58::decode("BCNVsE19CCpsvGseZTCEEM1qSiX1ridku2w155VveqEu")
+                .into_vec()
+                .unwrap(),
+        );
+        assert_eq!(hasher.result(), golden);
+        remove_file(out_path).unwrap();
+    }
+}

core/src/chacha_cuda.rs

@@ -0,0 +1,205 @@
+// Module used by validators to approve storage mining proofs
+// // in parallel using the GPU
+
+use crate::blocktree::Blocktree;
+use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
+use crate::sigverify::{
+    chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state,
+};
+use solana_sdk::hash::Hash;
+use std::io;
+use std::mem::size_of;
+use std::sync::Arc;
+
+use crate::storage_stage::ENTRIES_PER_SEGMENT;
+
+// Encrypt a file with multiple starting IV states, determined by ivecs.len()
+//
+// Then sample each block at the offsets provided by samples argument with sha256
+// and return the vec of sha states
+pub fn chacha_cbc_encrypt_file_many_keys(
+    blocktree: &Arc<Blocktree>,
+    segment: u64,
+    ivecs: &mut [u8],
+    samples: &[u64],
+) -> io::Result<Vec<Hash>> {
+    if ivecs.len() % CHACHA_BLOCK_SIZE != 0 {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!(
+                "bad IV length({}) not divisible by {} ",
+                ivecs.len(),
+                CHACHA_BLOCK_SIZE,
+            ),
+        ));
+    }
+
+    let mut buffer = [0; 8 * 1024];
+    let num_keys = ivecs.len() / CHACHA_BLOCK_SIZE;
+    let mut sha_states = vec![0; num_keys * size_of::<Hash>()];
+    let mut int_sha_states = vec![0; num_keys * 112];
+    let keys: Vec<u8> = vec![0; num_keys * CHACHA_KEY_SIZE]; // keys not used ATM, uniqueness comes from IV
+    let mut entry = segment;
+    let mut total_entries = 0;
+    let mut total_entry_len = 0;
+    let mut time: f32 = 0.0;
+    unsafe {
+        chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32);
+    }
+    loop {
+        match blocktree.read_blobs_bytes(entry, ENTRIES_PER_SEGMENT - total_entries, &mut buffer, 0)
+        {
+            Ok((num_entries, entry_len)) => {
+                debug!(
+                    "chacha_cuda: encrypting segment: {} num_entries: {} entry_len: {}",
+                    segment, num_entries, entry_len
+                );
+                let entry_len_usz = entry_len as usize;
+                unsafe {
+                    chacha_cbc_encrypt_many_sample(
+                        buffer[..entry_len_usz].as_ptr(),
+                        int_sha_states.as_mut_ptr(),
+                        entry_len_usz,
+                        keys.as_ptr(),
+                        ivecs.as_mut_ptr(),
+                        num_keys as u32,
+                        samples.as_ptr(),
+                        samples.len() as u32,
+                        total_entry_len,
+                        &mut time,
+                    );
+                }
+
+                total_entry_len += entry_len;
+                total_entries += num_entries;
+                entry += num_entries;
+                debug!(
+                    "total entries: {} entry: {} segment: {} entries_per_segment: {}",
+                    total_entries, entry, segment, ENTRIES_PER_SEGMENT
+                );
+                if (entry - segment) >= ENTRIES_PER_SEGMENT {
+                    break;
+                }
+            }
+            Err(e) => {
+                info!("Error encrypting file: {:?}", e);
+                break;
+            }
+        }
+    }
+    unsafe {
+        chacha_end_sha_state(
+            int_sha_states.as_ptr(),
+            sha_states.as_mut_ptr(),
+            num_keys as u32,
+        );
+    }
+    let mut res = Vec::new();
+    for x in 0..num_keys {
+        let start = x * size_of::<Hash>();
+        let end = start + size_of::<Hash>();
+        res.push(Hash::new(&sha_states[start..end]));
+    }
+    Ok(res)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::blocktree::get_tmp_ledger_path;
+    use crate::blocktree::Blocktree;
+    use crate::chacha::chacha_cbc_encrypt_ledger;
+    use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
+    use crate::entry::make_tiny_test_entries;
+    use crate::replicator::sample_file;
+    use solana_sdk::hash::Hash;
+    use std::fs::{remove_dir_all, remove_file};
+    use std::path::Path;
+    use std::sync::Arc;
+
+    #[test]
+    fn test_encrypt_file_many_keys_single() {
+        solana_logger::setup();
+
+        let entries = make_tiny_test_entries(32);
+        let ledger_dir = "test_encrypt_file_many_keys_single";
+        let ledger_path = get_tmp_ledger_path(ledger_dir);
+        let ticks_per_slot = 16;
+        let blocktree = Arc::new(Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap());
+
+        blocktree.write_entries(0, 0, 0, &entries).unwrap();
+
+        let out_path = Path::new("test_chacha_encrypt_file_many_keys_single_output.txt.enc");
+
+        let samples = [0];
+        let mut ivecs = hex!(
+            "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
+                              abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
+        );
+
+        let mut cpu_iv = ivecs.clone();
+        chacha_cbc_encrypt_ledger(&blocktree, 0, out_path, &mut cpu_iv).unwrap();
+
+        let ref_hash = sample_file(&out_path, &samples).unwrap();
+
+        let hashes =
+            chacha_cbc_encrypt_file_many_keys(&blocktree, 0, &mut ivecs, &samples).unwrap();
+
+        assert_eq!(hashes[0], ref_hash);
+
+        let _ignored = remove_dir_all(&ledger_path);
+        let _ignored = remove_file(out_path);
+    }
+
+    #[test]
+    fn test_encrypt_file_many_keys_multiple_keys() {
+        solana_logger::setup();
+
+        let entries = make_tiny_test_entries(32);
+        let ledger_dir = "test_encrypt_file_many_keys_multiple";
+        let ledger_path = get_tmp_ledger_path(ledger_dir);
+        let ticks_per_slot = 16;
+        let blocktree = Arc::new(Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap());
+        blocktree.write_entries(0, 0, 0, &entries).unwrap();
+
+        let out_path = Path::new("test_chacha_encrypt_file_many_keys_multiple_output.txt.enc");
+
+        let samples = [0, 1, 3, 4, 5, 150];
+        let mut ivecs = Vec::new();
+        let mut ref_hashes: Vec<Hash> = vec![];
+        for i in 0..2 {
+            let mut ivec = hex!(
+                "abc123abc123abc123abc123abc123abc123abababababababababababababab
+                                 abc123abc123abc123abc123abc123abc123abababababababababababababab"
+            );
+            ivec[0] = i;
+            ivecs.extend(ivec.clone().iter());
+            chacha_cbc_encrypt_ledger(&blocktree.clone(), 0, out_path, &mut ivec).unwrap();
+
+            ref_hashes.push(sample_file(&out_path, &samples).unwrap());
+            info!(
+                "ivec: {:?} hash: {:?} ivecs: {:?}",
+                ivec.to_vec(),
+                ref_hashes.last(),
+                ivecs
+            );
+        }
+
+        let hashes =
+            chacha_cbc_encrypt_file_many_keys(&blocktree, 0, &mut ivecs, &samples).unwrap();
+
+        assert_eq!(hashes, ref_hashes);
+
+        let _ignored = remove_dir_all(&ledger_path);
+        let _ignored = remove_file(out_path);
+    }
+
+    #[test]
+    fn test_encrypt_file_many_keys_bad_key_length() {
+        let mut keys = hex!("abc123");
+        let ledger_dir = "test_encrypt_file_many_keys_bad_key_length";
+        let ledger_path = get_tmp_ledger_path(ledger_dir);
+        let samples = [0];
+        let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
+        assert!(chacha_cbc_encrypt_file_many_keys(&blocktree, 0, &mut keys, &samples,).is_err());
+    }
+}

core/src/client.rs

@@ -0,0 +1,13 @@
+use crate::cluster_info::{NodeInfo, FULLNODE_PORT_RANGE};
+use crate::thin_client::ThinClient;
+use std::time::Duration;
+
+pub fn mk_client(r: &NodeInfo) -> ThinClient {
+    let (_, transactions_socket) = solana_netutil::bind_in_range(FULLNODE_PORT_RANGE).unwrap();
+    ThinClient::new(r.rpc, r.tpu, transactions_socket)
+}
+
+pub fn mk_client_with_timeout(r: &NodeInfo, timeout: Duration) -> ThinClient {
+    let (_, transactions_socket) = solana_netutil::bind_in_range(FULLNODE_PORT_RANGE).unwrap();
+    ThinClient::new_with_timeout(r.rpc, r.tpu, transactions_socket, timeout)
+}

core/src/cluster_info_vote_listener.rs

@@ -0,0 +1,69 @@
+use crate::cluster_info::{ClusterInfo, GOSSIP_SLEEP_MILLIS};
+use crate::packet;
+use crate::result::Result;
+use crate::service::Service;
+use crate::streamer::PacketSender;
+use solana_metrics::counter::Counter;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct ClusterInfoVoteListener {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl ClusterInfoVoteListener {
+    pub fn new(
+        exit: Arc<AtomicBool>,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        sender: PacketSender,
+    ) -> Self {
+        let exit1 = exit.clone();
+        let thread = Builder::new()
+            .name("solana-cluster_info_vote_listener".to_string())
+            .spawn(move || {
+                let _ = Self::recv_loop(&exit1, &cluster_info, &sender);
+            })
+            .unwrap();
+        Self {
+            exit,
+            thread_hdls: vec![thread],
+        }
+    }
+    fn recv_loop(
+        exit: &Arc<AtomicBool>,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        sender: &PacketSender,
+    ) -> Result<()> {
+        let mut last_ts = 0;
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                return Ok(());
+            }
+            let (votes, new_ts) = cluster_info.read().unwrap().get_votes(last_ts);
+            last_ts = new_ts;
+            inc_new_counter_info!("cluster_info_vote_listener-recv_count", votes.len());
+            let msgs = packet::to_packets(&votes);
+            for m in msgs {
+                sender.send(m)?;
+            }
+            sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS));
+        }
+    }
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for ClusterInfoVoteListener {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

core/src/cluster_tests.rs

@@ -0,0 +1,41 @@
+/// Cluster independant integration tests
+///
+/// All tests must start from an entry point and a funding keypair and
+/// discover the rest of the network.
+use crate::client::mk_client;
+use crate::contact_info::ContactInfo;
+use crate::gossip_service::discover;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::system_transaction::SystemTransaction;
+
+/// Spend and verify from every node in the network
+pub fn spend_and_verify_all_nodes(
+    entry_point_info: &ContactInfo,
+    funding_keypair: &Keypair,
+    nodes: usize,
+) {
+    let cluster_nodes = discover(&entry_point_info, nodes);
+    assert!(cluster_nodes.len() >= nodes);
+    for ingress_node in &cluster_nodes {
+        let random_keypair = Keypair::new();
+        let mut client = mk_client(&ingress_node);
+        let bal = client
+            .poll_get_balance(&funding_keypair.pubkey())
+            .expect("balance in source");
+        assert!(bal > 0);
+        let mut transaction = SystemTransaction::new_move(
+            &funding_keypair,
+            random_keypair.pubkey(),
+            1,
+            client.get_last_id(),
+            0,
+        );
+        let sig = client
+            .retry_transfer(&funding_keypair, &mut transaction, 5)
+            .unwrap();
+        for validator in &cluster_nodes {
+            let mut client = mk_client(&validator);
+            client.poll_for_signature(&sig).unwrap();
+        }
+    }
+}

core/src/contact_info.rs

@@ -0,0 +1,304 @@
+use crate::rpc_service::RPC_PORT;
+use bincode::serialize;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil, Signable, Signature};
+use solana_sdk::timing::timestamp;
+use std::cmp::{Ord, Ordering, PartialEq, PartialOrd};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+/// Structure representing a node on the network
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ContactInfo {
+    pub id: Pubkey,
+    /// signature of this ContactInfo
+    pub signature: Signature,
+    /// gossip address
+    pub gossip: SocketAddr,
+    /// address to connect to for replication
+    pub tvu: SocketAddr,
+    /// transactions address
+    pub tpu: SocketAddr,
+    /// storage data address
+    pub storage_addr: SocketAddr,
+    /// address to which to send JSON-RPC requests
+    pub rpc: SocketAddr,
+    /// websocket for JSON-RPC push notifications
+    pub rpc_pubsub: SocketAddr,
+    /// latest wallclock picked
+    pub wallclock: u64,
+}
+
+impl Ord for ContactInfo {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.id.cmp(&other.id)
+    }
+}
+
+impl PartialOrd for ContactInfo {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for ContactInfo {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+impl Eq for ContactInfo {}
+
+#[macro_export]
+macro_rules! socketaddr {
+    ($ip:expr, $port:expr) => {
+        SocketAddr::from((Ipv4Addr::from($ip), $port))
+    };
+    ($str:expr) => {{
+        let a: SocketAddr = $str.parse().unwrap();
+        a
+    }};
+}
+#[macro_export]
+macro_rules! socketaddr_any {
+    () => {
+        socketaddr!(0, 0)
+    };
+}
+
+impl Default for ContactInfo {
+    fn default() -> Self {
+        ContactInfo {
+            id: Pubkey::default(),
+            gossip: socketaddr_any!(),
+            tvu: socketaddr_any!(),
+            tpu: socketaddr_any!(),
+            storage_addr: socketaddr_any!(),
+            rpc: socketaddr_any!(),
+            rpc_pubsub: socketaddr_any!(),
+            wallclock: 0,
+            signature: Signature::default(),
+        }
+    }
+}
+
+impl ContactInfo {
+    pub fn new(
+        id: Pubkey,
+        gossip: SocketAddr,
+        tvu: SocketAddr,
+        tpu: SocketAddr,
+        storage_addr: SocketAddr,
+        rpc: SocketAddr,
+        rpc_pubsub: SocketAddr,
+        now: u64,
+    ) -> Self {
+        ContactInfo {
+            id,
+            signature: Signature::default(),
+            gossip,
+            tvu,
+            tpu,
+            storage_addr,
+            rpc,
+            rpc_pubsub,
+            wallclock: now,
+        }
+    }
+
+    pub fn new_localhost(id: Pubkey, now: u64) -> Self {
+        Self::new(
+            id,
+            socketaddr!("127.0.0.1:1234"),
+            socketaddr!("127.0.0.1:1235"),
+            socketaddr!("127.0.0.1:1236"),
+            socketaddr!("127.0.0.1:1237"),
+            socketaddr!("127.0.0.1:1238"),
+            socketaddr!("127.0.0.1:1239"),
+            now,
+        )
+    }
+
+    #[cfg(test)]
+    /// ContactInfo with multicast addresses for adversarial testing.
+    pub fn new_multicast() -> Self {
+        let addr = socketaddr!("224.0.1.255:1000");
+        assert!(addr.ip().is_multicast());
+        Self::new(
+            Keypair::new().pubkey(),
+            addr,
+            addr,
+            addr,
+            addr,
+            addr,
+            addr,
+            0,
+        )
+    }
+    fn next_port(addr: &SocketAddr, nxt: u16) -> SocketAddr {
+        let mut nxt_addr = *addr;
+        nxt_addr.set_port(addr.port() + nxt);
+        nxt_addr
+    }
+    pub fn new_with_pubkey_socketaddr(pubkey: Pubkey, bind_addr: &SocketAddr) -> Self {
+        let tpu_addr = *bind_addr;
+        let gossip_addr = Self::next_port(&bind_addr, 1);
+        let tvu_addr = Self::next_port(&bind_addr, 2);
+        let rpc_addr = SocketAddr::new(bind_addr.ip(), RPC_PORT);
+        let rpc_pubsub_addr = SocketAddr::new(bind_addr.ip(), RPC_PORT + 1);
+        ContactInfo::new(
+            pubkey,
+            gossip_addr,
+            tvu_addr,
+            tpu_addr,
+            "0.0.0.0:0".parse().unwrap(),
+            rpc_addr,
+            rpc_pubsub_addr,
+            timestamp(),
+        )
+    }
+    pub fn new_with_socketaddr(bind_addr: &SocketAddr) -> Self {
+        let keypair = Keypair::new();
+        Self::new_with_pubkey_socketaddr(keypair.pubkey(), bind_addr)
+    }
+    //
+    pub fn new_entry_point(gossip_addr: &SocketAddr) -> Self {
+        let daddr: SocketAddr = socketaddr!("0.0.0.0:0");
+        ContactInfo::new(
+            Pubkey::default(),
+            *gossip_addr,
+            daddr,
+            daddr,
+            daddr,
+            daddr,
+            daddr,
+            timestamp(),
+        )
+    }
+    fn is_valid_ip(addr: IpAddr) -> bool {
+        !(addr.is_unspecified() || addr.is_multicast())
+        // || (addr.is_loopback() && !cfg_test))
+        // TODO: boot loopback in production networks
+    }
+    /// port must not be 0
+    /// ip must be specified and not mulitcast
+    /// loopback ip is only allowed in tests
+    pub fn is_valid_address(addr: &SocketAddr) -> bool {
+        (addr.port() != 0) && Self::is_valid_ip(addr.ip())
+    }
+}
+
+impl Signable for ContactInfo {
+    fn pubkey(&self) -> Pubkey {
+        self.id
+    }
+
+    fn signable_data(&self) -> Vec<u8> {
+        #[derive(Serialize)]
+        struct SignData {
+            id: Pubkey,
+            gossip: SocketAddr,
+            tvu: SocketAddr,
+            tpu: SocketAddr,
+            storage_addr: SocketAddr,
+            rpc: SocketAddr,
+            rpc_pubsub: SocketAddr,
+            wallclock: u64,
+        }
+
+        let me = self;
+        let data = SignData {
+            id: me.id,
+            gossip: me.gossip,
+            tvu: me.tvu,
+            tpu: me.tpu,
+            storage_addr: me.storage_addr,
+            rpc: me.rpc,
+            rpc_pubsub: me.rpc_pubsub,
+            wallclock: me.wallclock,
+        };
+        serialize(&data).expect("failed to serialize ContactInfo")
+    }
+
+    fn get_signature(&self) -> Signature {
+        self.signature
+    }
+
+    fn set_signature(&mut self, signature: Signature) {
+        self.signature = signature
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_valid_address() {
+        assert!(cfg!(test));
+        let bad_address_port = socketaddr!("127.0.0.1:0");
+        assert!(!ContactInfo::is_valid_address(&bad_address_port));
+        let bad_address_unspecified = socketaddr!(0, 1234);
+        assert!(!ContactInfo::is_valid_address(&bad_address_unspecified));
+        let bad_address_multicast = socketaddr!([224, 254, 0, 0], 1234);
+        assert!(!ContactInfo::is_valid_address(&bad_address_multicast));
+        let loopback = socketaddr!("127.0.0.1:1234");
+        assert!(ContactInfo::is_valid_address(&loopback));
+        //        assert!(!ContactInfo::is_valid_ip_internal(loopback.ip(), false));
+    }
+    #[test]
+    fn test_default() {
+        let ci = ContactInfo::default();
+        assert!(ci.gossip.ip().is_unspecified());
+        assert!(ci.tvu.ip().is_unspecified());
+        assert!(ci.rpc.ip().is_unspecified());
+        assert!(ci.rpc_pubsub.ip().is_unspecified());
+        assert!(ci.tpu.ip().is_unspecified());
+        assert!(ci.storage_addr.ip().is_unspecified());
+    }
+    #[test]
+    fn test_multicast() {
+        let ci = ContactInfo::new_multicast();
+        assert!(ci.gossip.ip().is_multicast());
+        assert!(ci.tvu.ip().is_multicast());
+        assert!(ci.rpc.ip().is_multicast());
+        assert!(ci.rpc_pubsub.ip().is_multicast());
+        assert!(ci.tpu.ip().is_multicast());
+        assert!(ci.storage_addr.ip().is_multicast());
+    }
+    #[test]
+    fn test_entry_point() {
+        let addr = socketaddr!("127.0.0.1:10");
+        let ci = ContactInfo::new_entry_point(&addr);
+        assert_eq!(ci.gossip, addr);
+        assert!(ci.tvu.ip().is_unspecified());
+        assert!(ci.rpc.ip().is_unspecified());
+        assert!(ci.rpc_pubsub.ip().is_unspecified());
+        assert!(ci.tpu.ip().is_unspecified());
+        assert!(ci.storage_addr.ip().is_unspecified());
+    }
+    #[test]
+    fn test_socketaddr() {
+        let addr = socketaddr!("127.0.0.1:10");
+        let ci = ContactInfo::new_with_socketaddr(&addr);
+        assert_eq!(ci.tpu, addr);
+        assert_eq!(ci.gossip.port(), 11);
+        assert_eq!(ci.tvu.port(), 12);
+        assert_eq!(ci.rpc.port(), 8899);
+        assert_eq!(ci.rpc_pubsub.port(), 8900);
+        assert!(ci.storage_addr.ip().is_unspecified());
+    }
+    #[test]
+    fn replayed_data_new_with_socketaddr_with_pubkey() {
+        let keypair = Keypair::new();
+        let d1 = ContactInfo::new_with_pubkey_socketaddr(
+            keypair.pubkey().clone(),
+            &socketaddr!("127.0.0.1:1234"),
+        );
+        assert_eq!(d1.id, keypair.pubkey());
+        assert_eq!(d1.gossip, socketaddr!("127.0.0.1:1235"));
+        assert_eq!(d1.tvu, socketaddr!("127.0.0.1:1236"));
+        assert_eq!(d1.tpu, socketaddr!("127.0.0.1:1234"));
+        assert_eq!(d1.rpc, socketaddr!("127.0.0.1:8899"));
+        assert_eq!(d1.rpc_pubsub, socketaddr!("127.0.0.1:8900"));
+    }
+}

core/src/crds.rs

@@ -0,0 +1,317 @@
+//! This module implements Cluster Replicated Data Store for
+//! asynchronous updates in a distributed network.
+//!
+//! Data is stored in the CrdsValue type, each type has a specific
+//! CrdsValueLabel.  Labels are semantically grouped into a single record
+//! that is identified by a Pubkey.
+//! * 1 Pubkey maps many CrdsValueLabels
+//! * 1 CrdsValueLabel maps to 1 CrdsValue
+//! The Label, the record Pubkey, and all the record labels can be derived
+//! from a single CrdsValue.
+//!
+//! The actual data is stored in a single map of
+//! `CrdsValueLabel(Pubkey) -> CrdsValue` This allows for partial record
+//! updates to be propagated through the network.
+//!
+//! This means that full `Record` updates are not atomic.
+//!
+//! Additional labels can be added by appending them to the CrdsValueLabel,
+//! CrdsValue enums.
+//!
+//! Merge strategy is implemented in:
+//!     impl PartialOrd for VersionedCrdsValue
+//!
+//! A value is updated to a new version if the labels match, and the value
+//! wallclock is later, or the value hash is greater.
+
+use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use bincode::serialize;
+use indexmap::map::IndexMap;
+use solana_sdk::hash::{hash, Hash};
+use solana_sdk::pubkey::Pubkey;
+use std::cmp;
+
+#[derive(Clone)]
+pub struct Crds {
+    /// Stores the map of labels and values
+    pub table: IndexMap<CrdsValueLabel, VersionedCrdsValue>,
+}
+
+#[derive(PartialEq, Debug)]
+pub enum CrdsError {
+    InsertFailed,
+}
+
+/// This structure stores some local metadata associated with the CrdsValue
+/// The implementation of PartialOrd ensures that the "highest" version is always picked to be
+/// stored in the Crds
+#[derive(PartialEq, Debug, Clone)]
+pub struct VersionedCrdsValue {
+    pub value: CrdsValue,
+    /// local time when inserted
+    pub insert_timestamp: u64,
+    /// local time when updated
+    pub local_timestamp: u64,
+    /// value hash
+    pub value_hash: Hash,
+}
+
+impl PartialOrd for VersionedCrdsValue {
+    fn partial_cmp(&self, other: &VersionedCrdsValue) -> Option<cmp::Ordering> {
+        if self.value.label() != other.value.label() {
+            None
+        } else if self.value.wallclock() == other.value.wallclock() {
+            Some(self.value_hash.cmp(&other.value_hash))
+        } else {
+            Some(self.value.wallclock().cmp(&other.value.wallclock()))
+        }
+    }
+}
+impl VersionedCrdsValue {
+    pub fn new(local_timestamp: u64, value: CrdsValue) -> Self {
+        let value_hash = hash(&serialize(&value).unwrap());
+        VersionedCrdsValue {
+            value,
+            insert_timestamp: local_timestamp,
+            local_timestamp,
+            value_hash,
+        }
+    }
+}
+
+impl Default for Crds {
+    fn default() -> Self {
+        Crds {
+            table: IndexMap::new(),
+        }
+    }
+}
+
+impl Crds {
+    /// must be called atomically with `insert_versioned`
+    pub fn new_versioned(&self, local_timestamp: u64, value: CrdsValue) -> VersionedCrdsValue {
+        VersionedCrdsValue::new(local_timestamp, value)
+    }
+    /// insert the new value, returns the old value if insert succeeds
+    pub fn insert_versioned(
+        &mut self,
+        new_value: VersionedCrdsValue,
+    ) -> Result<Option<VersionedCrdsValue>, CrdsError> {
+        let label = new_value.value.label();
+        let wallclock = new_value.value.wallclock();
+        let do_insert = self
+            .table
+            .get(&label)
+            .map(|current| new_value > *current)
+            .unwrap_or(true);
+        if do_insert {
+            let old = self.table.insert(label, new_value);
+            Ok(old)
+        } else {
+            trace!("INSERT FAILED data: {} new.wallclock: {}", label, wallclock,);
+            Err(CrdsError::InsertFailed)
+        }
+    }
+    pub fn insert(
+        &mut self,
+        value: CrdsValue,
+        local_timestamp: u64,
+    ) -> Result<Option<VersionedCrdsValue>, CrdsError> {
+        let new_value = self.new_versioned(local_timestamp, value);
+        self.insert_versioned(new_value)
+    }
+    pub fn lookup(&self, label: &CrdsValueLabel) -> Option<&CrdsValue> {
+        self.table.get(label).map(|x| &x.value)
+    }
+
+    pub fn lookup_versioned(&self, label: &CrdsValueLabel) -> Option<&VersionedCrdsValue> {
+        self.table.get(label)
+    }
+
+    fn update_label_timestamp(&mut self, id: &CrdsValueLabel, now: u64) {
+        if let Some(e) = self.table.get_mut(id) {
+            e.local_timestamp = cmp::max(e.local_timestamp, now);
+        }
+    }
+
+    /// Update the timestamp's of all the labels that are assosciated with Pubkey
+    pub fn update_record_timestamp(&mut self, pubkey: Pubkey, now: u64) {
+        for label in &CrdsValue::record_labels(pubkey) {
+            self.update_label_timestamp(label, now);
+        }
+    }
+
+    /// find all the keys that are older or equal to min_ts
+    pub fn find_old_labels(&self, min_ts: u64) -> Vec<CrdsValueLabel> {
+        self.table
+            .iter()
+            .filter_map(|(k, v)| {
+                if v.local_timestamp <= min_ts {
+                    Some(k)
+                } else {
+                    None
+                }
+            })
+            .cloned()
+            .collect()
+    }
+
+    pub fn remove(&mut self, key: &CrdsValueLabel) {
+        self.table.remove(key);
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::contact_info::ContactInfo;
+    use crate::crds_value::LeaderId;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_insert() {
+        let mut crds = Crds::default();
+        let val = CrdsValue::LeaderId(LeaderId::default());
+        assert_eq!(crds.insert(val.clone(), 0).ok(), Some(None));
+        assert_eq!(crds.table.len(), 1);
+        assert!(crds.table.contains_key(&val.label()));
+        assert_eq!(crds.table[&val.label()].local_timestamp, 0);
+    }
+    #[test]
+    fn test_update_old() {
+        let mut crds = Crds::default();
+        let val = CrdsValue::LeaderId(LeaderId::default());
+        assert_eq!(crds.insert(val.clone(), 0), Ok(None));
+        assert_eq!(crds.insert(val.clone(), 1), Err(CrdsError::InsertFailed));
+        assert_eq!(crds.table[&val.label()].local_timestamp, 0);
+    }
+    #[test]
+    fn test_update_new() {
+        let mut crds = Crds::default();
+        let original = CrdsValue::LeaderId(LeaderId::default());
+        assert_matches!(crds.insert(original.clone(), 0), Ok(_));
+        let val = CrdsValue::LeaderId(LeaderId::new(Pubkey::default(), Pubkey::default(), 1));
+        assert_eq!(
+            crds.insert(val.clone(), 1).unwrap().unwrap().value,
+            original
+        );
+        assert_eq!(crds.table[&val.label()].local_timestamp, 1);
+    }
+    #[test]
+    fn test_update_timestsamp() {
+        let mut crds = Crds::default();
+        let val = CrdsValue::LeaderId(LeaderId::default());
+        assert_eq!(crds.insert(val.clone(), 0), Ok(None));
+
+        crds.update_label_timestamp(&val.label(), 1);
+        assert_eq!(crds.table[&val.label()].local_timestamp, 1);
+        assert_eq!(crds.table[&val.label()].insert_timestamp, 0);
+
+        let val2 = CrdsValue::ContactInfo(ContactInfo::default());
+        assert_eq!(val2.label().pubkey(), val.label().pubkey());
+        assert_matches!(crds.insert(val2.clone(), 0), Ok(None));
+
+        crds.update_record_timestamp(val.label().pubkey(), 2);
+        assert_eq!(crds.table[&val.label()].local_timestamp, 2);
+        assert_eq!(crds.table[&val.label()].insert_timestamp, 0);
+        assert_eq!(crds.table[&val2.label()].local_timestamp, 2);
+        assert_eq!(crds.table[&val2.label()].insert_timestamp, 0);
+
+        crds.update_record_timestamp(val.label().pubkey(), 1);
+        assert_eq!(crds.table[&val.label()].local_timestamp, 2);
+        assert_eq!(crds.table[&val.label()].insert_timestamp, 0);
+
+        let mut ci = ContactInfo::default();
+        ci.wallclock += 1;
+        let val3 = CrdsValue::ContactInfo(ci);
+        assert_matches!(crds.insert(val3.clone(), 3), Ok(Some(_)));
+        assert_eq!(crds.table[&val2.label()].local_timestamp, 3);
+        assert_eq!(crds.table[&val2.label()].insert_timestamp, 3);
+    }
+    #[test]
+    fn test_find_old_records() {
+        let mut crds = Crds::default();
+        let val = CrdsValue::LeaderId(LeaderId::default());
+        assert_eq!(crds.insert(val.clone(), 1), Ok(None));
+
+        assert!(crds.find_old_labels(0).is_empty());
+        assert_eq!(crds.find_old_labels(1), vec![val.label()]);
+        assert_eq!(crds.find_old_labels(2), vec![val.label()]);
+    }
+    #[test]
+    fn test_remove() {
+        let mut crds = Crds::default();
+        let val = CrdsValue::LeaderId(LeaderId::default());
+        assert_matches!(crds.insert(val.clone(), 1), Ok(_));
+
+        assert_eq!(crds.find_old_labels(1), vec![val.label()]);
+        crds.remove(&val.label());
+        assert!(crds.find_old_labels(1).is_empty());
+    }
+    #[test]
+    fn test_equal() {
+        let key = Keypair::new();
+        let v1 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), Pubkey::default(), 0)),
+        );
+        let v2 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), Pubkey::default(), 0)),
+        );
+        assert!(!(v1 != v2));
+        assert!(v1 == v2);
+    }
+    #[test]
+    fn test_hash_order() {
+        let key = Keypair::new();
+        let v1 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), Pubkey::default(), 0)),
+        );
+        let v2 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), key.pubkey(), 0)),
+        );
+        assert!(v1 != v2);
+        assert!(!(v1 == v2));
+        if v1 > v2 {
+            assert!(v2 < v1)
+        } else {
+            assert!(v2 > v1)
+        }
+    }
+    #[test]
+    fn test_wallclock_order() {
+        let key = Keypair::new();
+        let v1 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), Pubkey::default(), 1)),
+        );
+        let v2 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(key.pubkey(), Pubkey::default(), 0)),
+        );
+        assert!(v1 > v2);
+        assert!(!(v1 < v2));
+        assert!(v1 != v2);
+        assert!(!(v1 == v2));
+    }
+    #[test]
+    fn test_label_order() {
+        let v1 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(Keypair::new().pubkey(), Pubkey::default(), 0)),
+        );
+        let v2 = VersionedCrdsValue::new(
+            1,
+            CrdsValue::LeaderId(LeaderId::new(Keypair::new().pubkey(), Pubkey::default(), 0)),
+        );
+        assert!(v1 != v2);
+        assert!(!(v1 == v2));
+        assert!(!(v1 < v2));
+        assert!(!(v1 > v2));
+        assert!(!(v2 < v1));
+        assert!(!(v2 > v1));
+    }
+}

core/src/crds_gossip.rs

@@ -0,0 +1,216 @@
+//! Crds Gossip
+//! This module ties together Crds and the push and pull gossip overlays.  The interface is
+//! designed to run with a simulator or over a UDP network connection with messages up to a
+//! packet::BLOB_DATA_SIZE size.
+
+use crate::crds::Crds;
+use crate::crds_gossip_error::CrdsGossipError;
+use crate::crds_gossip_pull::CrdsGossipPull;
+use crate::crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE};
+use crate::crds_value::CrdsValue;
+use hashbrown::HashMap;
+use solana_runtime::bloom::Bloom;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+
+///The min size for bloom filters
+pub const CRDS_GOSSIP_BLOOM_SIZE: usize = 1000;
+
+#[derive(Clone)]
+pub struct CrdsGossip {
+    pub crds: Crds,
+    pub id: Pubkey,
+    pub push: CrdsGossipPush,
+    pull: CrdsGossipPull,
+}
+
+impl Default for CrdsGossip {
+    fn default() -> Self {
+        CrdsGossip {
+            crds: Crds::default(),
+            id: Pubkey::default(),
+            push: CrdsGossipPush::default(),
+            pull: CrdsGossipPull::default(),
+        }
+    }
+}
+
+impl CrdsGossip {
+    pub fn set_self(&mut self, id: Pubkey) {
+        self.id = id;
+    }
+    /// process a push message to the network
+    pub fn process_push_message(&mut self, values: &[CrdsValue], now: u64) -> Vec<Pubkey> {
+        let results: Vec<_> = values
+            .iter()
+            .map(|val| {
+                self.push
+                    .process_push_message(&mut self.crds, val.clone(), now)
+            })
+            .collect();
+        results
+            .into_iter()
+            .zip(values)
+            .filter_map(|(r, d)| {
+                if r == Err(CrdsGossipError::PushMessagePrune) {
+                    Some(d.label().pubkey())
+                } else if let Ok(Some(val)) = r {
+                    self.pull
+                        .record_old_hash(val.value_hash, val.local_timestamp);
+                    None
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn new_push_messages(&mut self, now: u64) -> (Pubkey, Vec<Pubkey>, Vec<CrdsValue>) {
+        let (peers, values) = self.push.new_push_messages(&self.crds, now);
+        (self.id, peers, values)
+    }
+
+    /// add the `from` to the peer's filter of nodes
+    pub fn process_prune_msg(
+        &mut self,
+        peer: Pubkey,
+        destination: Pubkey,
+        origin: &[Pubkey],
+        wallclock: u64,
+        now: u64,
+    ) -> Result<(), CrdsGossipError> {
+        let expired = now > wallclock + self.push.prune_timeout;
+        if expired {
+            return Err(CrdsGossipError::PruneMessageTimeout);
+        }
+        if self.id == destination {
+            self.push.process_prune_msg(peer, origin);
+            Ok(())
+        } else {
+            Err(CrdsGossipError::BadPruneDestination)
+        }
+    }
+
+    /// refresh the push active set
+    /// * ratio - number of actives to rotate
+    pub fn refresh_push_active_set(&mut self, stakes: &HashMap<Pubkey, u64>) {
+        self.push.refresh_push_active_set(
+            &self.crds,
+            stakes,
+            self.id,
+            self.pull.pull_request_time.len(),
+            CRDS_GOSSIP_NUM_ACTIVE,
+        )
+    }
+
+    /// generate a random request
+    pub fn new_pull_request(
+        &self,
+        now: u64,
+        stakes: &HashMap<Pubkey, u64>,
+    ) -> Result<(Pubkey, Bloom<Hash>, CrdsValue), CrdsGossipError> {
+        self.pull.new_pull_request(&self.crds, self.id, now, stakes)
+    }
+
+    /// time when a request to `from` was initiated
+    /// This is used for weighted random selection durring `new_pull_request`
+    /// It's important to use the local nodes request creation time as the weight
+    /// instaad of the response received time otherwise failed nodes will increase their weight.
+    pub fn mark_pull_request_creation_time(&mut self, from: Pubkey, now: u64) {
+        self.pull.mark_pull_request_creation_time(from, now)
+    }
+    /// process a pull request and create a response
+    pub fn process_pull_request(
+        &mut self,
+        caller: CrdsValue,
+        filter: Bloom<Hash>,
+        now: u64,
+    ) -> Vec<CrdsValue> {
+        self.pull
+            .process_pull_request(&mut self.crds, caller, filter, now)
+    }
+    /// process a pull response
+    pub fn process_pull_response(
+        &mut self,
+        from: Pubkey,
+        response: Vec<CrdsValue>,
+        now: u64,
+    ) -> usize {
+        self.pull
+            .process_pull_response(&mut self.crds, from, response, now)
+    }
+    pub fn purge(&mut self, now: u64) {
+        if now > self.push.msg_timeout {
+            let min = now - self.push.msg_timeout;
+            self.push.purge_old_pending_push_messages(&self.crds, min);
+        }
+        if now > 5 * self.push.msg_timeout {
+            let min = now - 5 * self.push.msg_timeout;
+            self.push.purge_old_pushed_once_messages(min);
+        }
+        if now > self.pull.crds_timeout {
+            let min = now - self.pull.crds_timeout;
+            self.pull.purge_active(&mut self.crds, self.id, min);
+        }
+        if now > 5 * self.pull.crds_timeout {
+            let min = now - 5 * self.pull.crds_timeout;
+            self.pull.purge_purged(min);
+        }
+    }
+}
+
+/// Computes a normalized(log of actual stake) stake
+pub fn get_stake<S: std::hash::BuildHasher>(id: &Pubkey, stakes: &HashMap<Pubkey, u64, S>) -> f32 {
+    // cap the max balance to u32 max (it should be plenty)
+    let bal = f64::from(u32::max_value()).min(*stakes.get(id).unwrap_or(&0) as f64);
+    1_f32.max((bal as f32).ln())
+}
+
+/// Computes bounded weight given some max, a time since last selected, and a stake value
+/// The minimum stake is 1 and not 0 to allow 'time since last' picked to factor in.
+pub fn get_weight(max_weight: f32, time_since_last_selected: u32, stake: f32) -> f32 {
+    let mut weight = time_since_last_selected as f32 * stake;
+    if weight.is_infinite() {
+        weight = max_weight;
+    }
+    1.0_f32.max(weight.min(max_weight))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::cluster_info::NodeInfo;
+    use solana_sdk::hash::hash;
+    use solana_sdk::timing::timestamp;
+
+    #[test]
+    fn test_prune_errors() {
+        let mut crds_gossip = CrdsGossip::default();
+        crds_gossip.id = Pubkey::new(&[0; 32]);
+        let id = crds_gossip.id;
+        let ci = NodeInfo::new_localhost(Pubkey::new(&[1; 32]), 0);
+        let prune_pubkey = Pubkey::new(&[2; 32]);
+        crds_gossip
+            .crds
+            .insert(CrdsValue::ContactInfo(ci.clone()), 0)
+            .unwrap();
+        crds_gossip.refresh_push_active_set(&HashMap::new());
+        let now = timestamp();
+        //incorrect dest
+        let mut res = crds_gossip.process_prune_msg(
+            ci.id,
+            Pubkey::new(hash(&[1; 32]).as_ref()),
+            &[prune_pubkey],
+            now,
+            now,
+        );
+        assert_eq!(res.err(), Some(CrdsGossipError::BadPruneDestination));
+        //correct dest
+        res = crds_gossip.process_prune_msg(ci.id, id, &[prune_pubkey], now, now);
+        res.unwrap();
+        //test timeout
+        let timeout = now + crds_gossip.push.prune_timeout * 2;
+        res = crds_gossip.process_prune_msg(ci.id, id, &[prune_pubkey], now, timeout);
+        assert_eq!(res.err(), Some(CrdsGossipError::PruneMessageTimeout));
+    }
+}

core/src/crds_gossip_error.rs

@@ -0,0 +1,9 @@
+#[derive(PartialEq, Debug)]
+pub enum CrdsGossipError {
+    NoPeers,
+    PushMessageTimeout,
+    PushMessagePrune,
+    PushMessageOldVersion,
+    BadPruneDestination,
+    PruneMessageTimeout,
+}

core/src/crds_gossip_pull.rs

@@ -0,0 +1,417 @@
+//! Crds Gossip Pull overlay
+//! This module implements the anti-entropy protocol for the network.
+//!
+//! The basic strategy is as follows:
+//! 1. Construct a bloom filter of the local data set
+//! 2. Randomly ask a node on the network for data that is not contained in the bloom filter.
+//!
+//! Bloom filters have a false positive rate.  Each requests uses a different bloom filter
+//! with random hash functions.  So each subsequent request will have a different distribution
+//! of false positives.
+
+use crate::contact_info::ContactInfo;
+use crate::crds::Crds;
+use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_BLOOM_SIZE};
+use crate::crds_gossip_error::CrdsGossipError;
+use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use crate::packet::BLOB_DATA_SIZE;
+use bincode::serialized_size;
+use hashbrown::HashMap;
+use rand;
+use rand::distributions::{Distribution, WeightedIndex};
+use solana_runtime::bloom::Bloom;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use std::cmp;
+use std::collections::VecDeque;
+
+pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;
+
+#[derive(Clone)]
+pub struct CrdsGossipPull {
+    /// timestamp of last request
+    pub pull_request_time: HashMap<Pubkey, u64>,
+    /// hash and insert time
+    purged_values: VecDeque<(Hash, u64)>,
+    /// max bytes per message
+    pub max_bytes: usize,
+    pub crds_timeout: u64,
+}
+
+impl Default for CrdsGossipPull {
+    fn default() -> Self {
+        Self {
+            purged_values: VecDeque::new(),
+            pull_request_time: HashMap::new(),
+            max_bytes: BLOB_DATA_SIZE,
+            crds_timeout: CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
+        }
+    }
+}
+impl CrdsGossipPull {
+    /// generate a random request
+    pub fn new_pull_request(
+        &self,
+        crds: &Crds,
+        self_id: Pubkey,
+        now: u64,
+        stakes: &HashMap<Pubkey, u64>,
+    ) -> Result<(Pubkey, Bloom<Hash>, CrdsValue), CrdsGossipError> {
+        let options = self.pull_options(crds, &self_id, now, stakes);
+        if options.is_empty() {
+            return Err(CrdsGossipError::NoPeers);
+        }
+        let filter = self.build_crds_filter(crds);
+        let index = WeightedIndex::new(options.iter().map(|weighted| weighted.0)).unwrap();
+        let random = index.sample(&mut rand::thread_rng());
+        let self_info = crds
+            .lookup(&CrdsValueLabel::ContactInfo(self_id))
+            .unwrap_or_else(|| panic!("self_id invalid {}", self_id));
+        Ok((options[random].1.id, filter, self_info.clone()))
+    }
+
+    fn pull_options<'a>(
+        &self,
+        crds: &'a Crds,
+        self_id: &Pubkey,
+        now: u64,
+        stakes: &HashMap<Pubkey, u64>,
+    ) -> Vec<(f32, &'a ContactInfo)> {
+        crds.table
+            .values()
+            .filter_map(|v| v.value.contact_info())
+            .filter(|v| v.id != *self_id && ContactInfo::is_valid_address(&v.gossip))
+            .map(|item| {
+                let max_weight = f32::from(u16::max_value()) - 1.0;
+                let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
+                let since = ((now - req_time) / 1024) as u32;
+                let stake = get_stake(&item.id, stakes);
+                let weight = get_weight(max_weight, since, stake);
+                (weight, item)
+            })
+            .collect()
+    }
+
+    /// time when a request to `from` was initiated
+    /// This is used for weighted random selection during `new_pull_request`
+    /// It's important to use the local nodes request creation time as the weight
+    /// instead of the response received time otherwise failed nodes will increase their weight.
+    pub fn mark_pull_request_creation_time(&mut self, from: Pubkey, now: u64) {
+        self.pull_request_time.insert(from, now);
+    }
+
+    /// Store an old hash in the purged values set
+    pub fn record_old_hash(&mut self, hash: Hash, timestamp: u64) {
+        self.purged_values.push_back((hash, timestamp))
+    }
+
+    /// process a pull request and create a response
+    pub fn process_pull_request(
+        &mut self,
+        crds: &mut Crds,
+        caller: CrdsValue,
+        mut filter: Bloom<Hash>,
+        now: u64,
+    ) -> Vec<CrdsValue> {
+        let rv = self.filter_crds_values(crds, &mut filter);
+        let key = caller.label().pubkey();
+        let old = crds.insert(caller, now);
+        if let Some(val) = old.ok().and_then(|opt| opt) {
+            self.purged_values
+                .push_back((val.value_hash, val.local_timestamp))
+        }
+        crds.update_record_timestamp(key, now);
+        rv
+    }
+    /// process a pull response
+    pub fn process_pull_response(
+        &mut self,
+        crds: &mut Crds,
+        from: Pubkey,
+        response: Vec<CrdsValue>,
+        now: u64,
+    ) -> usize {
+        let mut failed = 0;
+        for r in response {
+            let owner = r.label().pubkey();
+            let old = crds.insert(r, now);
+            failed += old.is_err() as usize;
+            old.ok().map(|opt| {
+                crds.update_record_timestamp(owner, now);
+                opt.map(|val| {
+                    self.purged_values
+                        .push_back((val.value_hash, val.local_timestamp))
+                })
+            });
+        }
+        crds.update_record_timestamp(from, now);
+        failed
+    }
+    /// build a filter of the current crds table
+    fn build_crds_filter(&self, crds: &Crds) -> Bloom<Hash> {
+        let num = cmp::max(
+            CRDS_GOSSIP_BLOOM_SIZE,
+            crds.table.values().count() + self.purged_values.len(),
+        );
+        let mut bloom = Bloom::random(num, 0.1, 4 * 1024 * 8 - 1);
+        for v in crds.table.values() {
+            bloom.add(&v.value_hash);
+        }
+        for (value_hash, _insert_timestamp) in &self.purged_values {
+            bloom.add(value_hash);
+        }
+        bloom
+    }
+    /// filter values that fail the bloom filter up to max_bytes
+    fn filter_crds_values(&self, crds: &Crds, filter: &mut Bloom<Hash>) -> Vec<CrdsValue> {
+        let mut max_bytes = self.max_bytes as isize;
+        let mut ret = vec![];
+        for v in crds.table.values() {
+            if filter.contains(&v.value_hash) {
+                continue;
+            }
+            max_bytes -= serialized_size(&v.value).unwrap() as isize;
+            if max_bytes < 0 {
+                break;
+            }
+            ret.push(v.value.clone());
+        }
+        ret
+    }
+    /// Purge values from the crds that are older then `active_timeout`
+    /// The value_hash of an active item is put into self.purged_values queue
+    pub fn purge_active(&mut self, crds: &mut Crds, self_id: Pubkey, min_ts: u64) {
+        let old = crds.find_old_labels(min_ts);
+        let mut purged: VecDeque<_> = old
+            .iter()
+            .filter(|label| label.pubkey() != self_id)
+            .filter_map(|label| {
+                let rv = crds
+                    .lookup_versioned(label)
+                    .map(|val| (val.value_hash, val.local_timestamp));
+                crds.remove(label);
+                rv
+            })
+            .collect();
+        self.purged_values.append(&mut purged);
+    }
+    /// Purge values from the `self.purged_values` queue that are older then purge_timeout
+    pub fn purge_purged(&mut self, min_ts: u64) {
+        let cnt = self
+            .purged_values
+            .iter()
+            .take_while(|v| v.1 < min_ts)
+            .count();
+        self.purged_values.drain(..cnt);
+    }
+}
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::contact_info::ContactInfo;
+    use crate::crds_value::LeaderId;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_new_pull_with_stakes() {
+        let mut crds = Crds::default();
+        let mut stakes = HashMap::new();
+        let node = CrdsGossipPull::default();
+        let me = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        crds.insert(me.clone(), 0).unwrap();
+        for i in 1..=30 {
+            let entry =
+                CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+            let id = entry.label().pubkey();
+            crds.insert(entry.clone(), 0).unwrap();
+            stakes.insert(id, i * 100);
+        }
+        let now = 1024;
+        let mut options = node.pull_options(&crds, &me.label().pubkey(), now, &stakes);
+        assert!(!options.is_empty());
+        options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
+        // check that the highest stake holder is also the heaviest weighted.
+        assert_eq!(
+            *stakes.get(&options.get(0).unwrap().1.id).unwrap(),
+            3000_u64
+        );
+    }
+
+    #[test]
+    fn test_new_pull_request() {
+        let mut crds = Crds::default();
+        let entry = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let id = entry.label().pubkey();
+        let node = CrdsGossipPull::default();
+        assert_eq!(
+            node.new_pull_request(&crds, id, 0, &HashMap::new()),
+            Err(CrdsGossipError::NoPeers)
+        );
+
+        crds.insert(entry.clone(), 0).unwrap();
+        assert_eq!(
+            node.new_pull_request(&crds, id, 0, &HashMap::new()),
+            Err(CrdsGossipError::NoPeers)
+        );
+
+        let new = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        crds.insert(new.clone(), 0).unwrap();
+        let req = node.new_pull_request(&crds, id, 0, &HashMap::new());
+        let (to, _, self_info) = req.unwrap();
+        assert_eq!(to, new.label().pubkey());
+        assert_eq!(self_info, entry);
+    }
+
+    #[test]
+    fn test_new_mark_creation_time() {
+        let mut crds = Crds::default();
+        let entry = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let node_id = entry.label().pubkey();
+        let mut node = CrdsGossipPull::default();
+        crds.insert(entry.clone(), 0).unwrap();
+        let old = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        crds.insert(old.clone(), 0).unwrap();
+        let new = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        crds.insert(new.clone(), 0).unwrap();
+
+        // set request creation time to max_value
+        node.mark_pull_request_creation_time(new.label().pubkey(), u64::max_value());
+
+        // odds of getting the other request should be 1 in u64::max_value()
+        for _ in 0..10 {
+            let req = node.new_pull_request(&crds, node_id, u64::max_value(), &HashMap::new());
+            let (to, _, self_info) = req.unwrap();
+            assert_eq!(to, old.label().pubkey());
+            assert_eq!(self_info, entry);
+        }
+    }
+
+    #[test]
+    fn test_process_pull_request() {
+        let mut node_crds = Crds::default();
+        let entry = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let node_id = entry.label().pubkey();
+        let node = CrdsGossipPull::default();
+        node_crds.insert(entry.clone(), 0).unwrap();
+        let new = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        node_crds.insert(new.clone(), 0).unwrap();
+        let req = node.new_pull_request(&node_crds, node_id, 0, &HashMap::new());
+
+        let mut dest_crds = Crds::default();
+        let mut dest = CrdsGossipPull::default();
+        let (_, filter, caller) = req.unwrap();
+        let rsp = dest.process_pull_request(&mut dest_crds, caller.clone(), filter, 1);
+        assert!(rsp.is_empty());
+        assert!(dest_crds.lookup(&caller.label()).is_some());
+        assert_eq!(
+            dest_crds
+                .lookup_versioned(&caller.label())
+                .unwrap()
+                .insert_timestamp,
+            1
+        );
+        assert_eq!(
+            dest_crds
+                .lookup_versioned(&caller.label())
+                .unwrap()
+                .local_timestamp,
+            1
+        );
+    }
+    #[test]
+    fn test_process_pull_request_response() {
+        let mut node_crds = Crds::default();
+        let entry = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let node_id = entry.label().pubkey();
+        let mut node = CrdsGossipPull::default();
+        node_crds.insert(entry.clone(), 0).unwrap();
+        let new = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        node_crds.insert(new.clone(), 0).unwrap();
+
+        let mut dest = CrdsGossipPull::default();
+        let mut dest_crds = Crds::default();
+        let new = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        dest_crds.insert(new.clone(), 0).unwrap();
+
+        // node contains a key from the dest node, but at an older local timestamp
+        let dest_id = new.label().pubkey();
+        let same_key = CrdsValue::LeaderId(LeaderId::new(dest_id, dest_id, 1));
+        node_crds.insert(same_key.clone(), 0).unwrap();
+        assert_eq!(
+            node_crds
+                .lookup_versioned(&same_key.label())
+                .unwrap()
+                .local_timestamp,
+            0
+        );
+        let mut done = false;
+        for _ in 0..30 {
+            // there is a chance of a false positive with bloom filters
+            let req = node.new_pull_request(&node_crds, node_id, 0, &HashMap::new());
+            let (_, filter, caller) = req.unwrap();
+            let rsp = dest.process_pull_request(&mut dest_crds, caller, filter, 0);
+            // if there is a false positive this is empty
+            // prob should be around 0.1 per iteration
+            if rsp.is_empty() {
+                continue;
+            }
+
+            assert_eq!(rsp.len(), 1);
+            let failed = node.process_pull_response(&mut node_crds, node_id, rsp, 1);
+            assert_eq!(failed, 0);
+            assert_eq!(
+                node_crds
+                    .lookup_versioned(&new.label())
+                    .unwrap()
+                    .local_timestamp,
+                1
+            );
+            // verify that the whole record was updated for dest since this is a response from dest
+            assert_eq!(
+                node_crds
+                    .lookup_versioned(&same_key.label())
+                    .unwrap()
+                    .local_timestamp,
+                1
+            );
+            done = true;
+            break;
+        }
+        assert!(done);
+    }
+    #[test]
+    fn test_gossip_purge() {
+        let mut node_crds = Crds::default();
+        let entry = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let node_label = entry.label();
+        let node_id = node_label.pubkey();
+        let mut node = CrdsGossipPull::default();
+        node_crds.insert(entry.clone(), 0).unwrap();
+        let old = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        node_crds.insert(old.clone(), 0).unwrap();
+        let value_hash = node_crds.lookup_versioned(&old.label()).unwrap().value_hash;
+
+        //verify self is valid
+        assert_eq!(node_crds.lookup(&node_label).unwrap().label(), node_label);
+
+        // purge
+        node.purge_active(&mut node_crds, node_id, 1);
+
+        //verify self is still valid after purge
+        assert_eq!(node_crds.lookup(&node_label).unwrap().label(), node_label);
+
+        assert_eq!(node_crds.lookup_versioned(&old.label()), None);
+        assert_eq!(node.purged_values.len(), 1);
+        for _ in 0..30 {
+            // there is a chance of a false positive with bloom filters
+            // assert that purged value is still in the set
+            // chance of 30 consecutive false positives is 0.1^30
+            let filter = node.build_crds_filter(&node_crds);
+            assert!(filter.contains(&value_hash));
+        }
+
+        // purge the value
+        node.purge_purged(1);
+        assert_eq!(node.purged_values.len(), 0);
+    }
+}

core/src/crds_gossip_push.rs

@@ -0,0 +1,508 @@
+//! Crds Gossip Push overlay
+//! This module is used to propagate recently created CrdsValues across the network
+//! Eager push strategy is based on Plumtree
+//! http://asc.di.fct.unl.pt/~jleitao/pdf/srds07-leitao.pdf
+//!
+//! Main differences are:
+//! 1. There is no `max hop`.  Messages are signed with a local wallclock.  If they are outside of
+//!    the local nodes wallclock window they are drooped silently.
+//! 2. The prune set is stored in a Bloom filter.
+
+use crate::contact_info::ContactInfo;
+use crate::crds::{Crds, VersionedCrdsValue};
+use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_BLOOM_SIZE};
+use crate::crds_gossip_error::CrdsGossipError;
+use crate::crds_value::{CrdsValue, CrdsValueLabel};
+use crate::packet::BLOB_DATA_SIZE;
+use bincode::serialized_size;
+use hashbrown::HashMap;
+use indexmap::map::IndexMap;
+use rand;
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::seq::SliceRandom;
+use solana_runtime::bloom::Bloom;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing::timestamp;
+use std::cmp;
+
+pub const CRDS_GOSSIP_NUM_ACTIVE: usize = 30;
+pub const CRDS_GOSSIP_PUSH_FANOUT: usize = 6;
+pub const CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS: u64 = 5000;
+pub const CRDS_GOSSIP_PRUNE_MSG_TIMEOUT_MS: u64 = 500;
+
+#[derive(Clone)]
+pub struct CrdsGossipPush {
+    /// max bytes per message
+    pub max_bytes: usize,
+    /// active set of validators for push
+    active_set: IndexMap<Pubkey, Bloom<Pubkey>>,
+    /// push message queue
+    push_messages: HashMap<CrdsValueLabel, Hash>,
+    pushed_once: HashMap<Hash, u64>,
+    pub num_active: usize,
+    pub push_fanout: usize,
+    pub msg_timeout: u64,
+    pub prune_timeout: u64,
+}
+
+impl Default for CrdsGossipPush {
+    fn default() -> Self {
+        Self {
+            max_bytes: BLOB_DATA_SIZE,
+            active_set: IndexMap::new(),
+            push_messages: HashMap::new(),
+            pushed_once: HashMap::new(),
+            num_active: CRDS_GOSSIP_NUM_ACTIVE,
+            push_fanout: CRDS_GOSSIP_PUSH_FANOUT,
+            msg_timeout: CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS,
+            prune_timeout: CRDS_GOSSIP_PRUNE_MSG_TIMEOUT_MS,
+        }
+    }
+}
+impl CrdsGossipPush {
+    pub fn num_pending(&self) -> usize {
+        self.push_messages.len()
+    }
+    /// process a push message to the network
+    pub fn process_push_message(
+        &mut self,
+        crds: &mut Crds,
+        value: CrdsValue,
+        now: u64,
+    ) -> Result<Option<VersionedCrdsValue>, CrdsGossipError> {
+        if now > value.wallclock() + self.msg_timeout {
+            return Err(CrdsGossipError::PushMessageTimeout);
+        }
+        if now + self.msg_timeout < value.wallclock() {
+            return Err(CrdsGossipError::PushMessageTimeout);
+        }
+        let label = value.label();
+
+        let new_value = crds.new_versioned(now, value);
+        let value_hash = new_value.value_hash;
+        if self.pushed_once.get(&value_hash).is_some() {
+            return Err(CrdsGossipError::PushMessagePrune);
+        }
+        let old = crds.insert_versioned(new_value);
+        if old.is_err() {
+            return Err(CrdsGossipError::PushMessageOldVersion);
+        }
+        self.push_messages.insert(label, value_hash);
+        self.pushed_once.insert(value_hash, now);
+        Ok(old.ok().and_then(|opt| opt))
+    }
+
+    /// New push message to broadcast to peers.
+    /// Returns a list of Pubkeys for the selected peers and a list of values to send to all the
+    /// peers.
+    /// The list of push messages is created such that all the randomly selected peers have not
+    /// pruned the source addresses.
+    pub fn new_push_messages(&mut self, crds: &Crds, now: u64) -> (Vec<Pubkey>, Vec<CrdsValue>) {
+        let max = self.active_set.len();
+        let mut nodes: Vec<_> = (0..max).collect();
+        nodes.shuffle(&mut rand::thread_rng());
+        let peers: Vec<Pubkey> = nodes
+            .into_iter()
+            .filter_map(|n| self.active_set.get_index(n))
+            .take(self.push_fanout)
+            .map(|n| *n.0)
+            .collect();
+        let mut total_bytes: usize = 0;
+        let mut values = vec![];
+        for (label, hash) in &self.push_messages {
+            let mut failed = false;
+            for p in &peers {
+                let filter = self.active_set.get_mut(p);
+                failed |= filter.is_none() || filter.unwrap().contains(&label.pubkey());
+            }
+            if failed {
+                continue;
+            }
+            let res = crds.lookup_versioned(label);
+            if res.is_none() {
+                continue;
+            }
+            let version = res.unwrap();
+            if version.value_hash != *hash {
+                continue;
+            }
+            let value = &version.value;
+            if value.wallclock() > now || value.wallclock() + self.msg_timeout < now {
+                continue;
+            }
+            total_bytes += serialized_size(value).unwrap() as usize;
+            if total_bytes > self.max_bytes {
+                break;
+            }
+            values.push(value.clone());
+        }
+        for v in &values {
+            self.push_messages.remove(&v.label());
+        }
+        (peers, values)
+    }
+
+    /// add the `from` to the peer's filter of nodes
+    pub fn process_prune_msg(&mut self, peer: Pubkey, origins: &[Pubkey]) {
+        for origin in origins {
+            if let Some(p) = self.active_set.get_mut(&peer) {
+                p.add(origin)
+            }
+        }
+    }
+
+    fn compute_need(num_active: usize, active_set_len: usize, ratio: usize) -> usize {
+        let num = active_set_len / ratio;
+        cmp::min(num_active, (num_active - active_set_len) + num)
+    }
+
+    /// refresh the push active set
+    /// * ratio - active_set.len()/ratio is the number of actives to rotate
+    pub fn refresh_push_active_set(
+        &mut self,
+        crds: &Crds,
+        stakes: &HashMap<Pubkey, u64>,
+        self_id: Pubkey,
+        network_size: usize,
+        ratio: usize,
+    ) {
+        let need = Self::compute_need(self.num_active, self.active_set.len(), ratio);
+        let mut new_items = HashMap::new();
+
+        let mut options: Vec<_> = self.push_options(crds, &self_id, stakes);
+        if options.is_empty() {
+            return;
+        }
+        while new_items.len() < need {
+            let index = WeightedIndex::new(options.iter().map(|weighted| weighted.0));
+            if index.is_err() {
+                break;
+            }
+            let index = index.unwrap();
+            let index = index.sample(&mut rand::thread_rng());
+            let item = options[index].1;
+            options.remove(index);
+            if self.active_set.get(&item.id).is_some() {
+                continue;
+            }
+            if new_items.get(&item.id).is_some() {
+                continue;
+            }
+            let size = cmp::max(CRDS_GOSSIP_BLOOM_SIZE, network_size);
+            let bloom = Bloom::random(size, 0.1, 1024 * 8 * 4);
+            new_items.insert(item.id, bloom);
+        }
+        let mut keys: Vec<Pubkey> = self.active_set.keys().cloned().collect();
+        keys.shuffle(&mut rand::thread_rng());
+        let num = keys.len() / ratio;
+        for k in &keys[..num] {
+            self.active_set.remove(k);
+        }
+        for (k, v) in new_items {
+            self.active_set.insert(k, v);
+        }
+    }
+
+    fn push_options<'a>(
+        &self,
+        crds: &'a Crds,
+        self_id: &Pubkey,
+        stakes: &HashMap<Pubkey, u64>,
+    ) -> Vec<(f32, &'a ContactInfo)> {
+        crds.table
+            .values()
+            .filter(|v| v.value.contact_info().is_some())
+            .map(|v| (v.value.contact_info().unwrap(), v))
+            .filter(|(info, _)| info.id != *self_id && ContactInfo::is_valid_address(&info.gossip))
+            .map(|(info, value)| {
+                let max_weight = f32::from(u16::max_value()) - 1.0;
+                let last_updated: u64 = value.local_timestamp;
+                let since = ((timestamp() - last_updated) / 1024) as u32;
+                let stake = get_stake(&info.id, stakes);
+                let weight = get_weight(max_weight, since, stake);
+                (weight, info)
+            })
+            .collect()
+    }
+
+    /// purge old pending push messages
+    pub fn purge_old_pending_push_messages(&mut self, crds: &Crds, min_time: u64) {
+        let old_msgs: Vec<CrdsValueLabel> = self
+            .push_messages
+            .iter()
+            .filter_map(|(k, hash)| {
+                if let Some(versioned) = crds.lookup_versioned(k) {
+                    if versioned.value.wallclock() < min_time || versioned.value_hash != *hash {
+                        Some(k)
+                    } else {
+                        None
+                    }
+                } else {
+                    Some(k)
+                }
+            })
+            .cloned()
+            .collect();
+        for k in old_msgs {
+            self.push_messages.remove(&k);
+        }
+    }
+    /// purge old pushed_once messages
+    pub fn purge_old_pushed_once_messages(&mut self, min_time: u64) {
+        let old_msgs: Vec<Hash> = self
+            .pushed_once
+            .iter()
+            .filter_map(|(k, v)| if *v < min_time { Some(k) } else { None })
+            .cloned()
+            .collect();
+        for k in old_msgs {
+            self.pushed_once.remove(&k);
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::contact_info::ContactInfo;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_process_push() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let value = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        let label = value.label();
+        // push a new message
+        assert_eq!(
+            push.process_push_message(&mut crds, value.clone(), 0),
+            Ok(None)
+        );
+        assert_eq!(crds.lookup(&label), Some(&value));
+
+        // push it again
+        assert_eq!(
+            push.process_push_message(&mut crds, value.clone(), 0),
+            Err(CrdsGossipError::PushMessagePrune)
+        );
+    }
+    #[test]
+    fn test_process_push_old_version() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let mut ci = ContactInfo::new_localhost(Keypair::new().pubkey(), 0);
+        ci.wallclock = 1;
+        let value = CrdsValue::ContactInfo(ci.clone());
+
+        // push a new message
+        assert_eq!(push.process_push_message(&mut crds, value, 0), Ok(None));
+
+        // push an old version
+        ci.wallclock = 0;
+        let value = CrdsValue::ContactInfo(ci.clone());
+        assert_eq!(
+            push.process_push_message(&mut crds, value, 0),
+            Err(CrdsGossipError::PushMessageOldVersion)
+        );
+    }
+    #[test]
+    fn test_process_push_timeout() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let timeout = push.msg_timeout;
+        let mut ci = ContactInfo::new_localhost(Keypair::new().pubkey(), 0);
+
+        // push a version to far in the future
+        ci.wallclock = timeout + 1;
+        let value = CrdsValue::ContactInfo(ci.clone());
+        assert_eq!(
+            push.process_push_message(&mut crds, value, 0),
+            Err(CrdsGossipError::PushMessageTimeout)
+        );
+
+        // push a version to far in the past
+        ci.wallclock = 0;
+        let value = CrdsValue::ContactInfo(ci.clone());
+        assert_eq!(
+            push.process_push_message(&mut crds, value, timeout + 1),
+            Err(CrdsGossipError::PushMessageTimeout)
+        );
+    }
+    #[test]
+    fn test_process_push_update() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let mut ci = ContactInfo::new_localhost(Keypair::new().pubkey(), 0);
+        ci.wallclock = 0;
+        let value_old = CrdsValue::ContactInfo(ci.clone());
+
+        // push a new message
+        assert_eq!(
+            push.process_push_message(&mut crds, value_old.clone(), 0),
+            Ok(None)
+        );
+
+        // push an old version
+        ci.wallclock = 1;
+        let value = CrdsValue::ContactInfo(ci.clone());
+        assert_eq!(
+            push.process_push_message(&mut crds, value, 0)
+                .unwrap()
+                .unwrap()
+                .value,
+            value_old
+        );
+    }
+    #[test]
+    fn test_compute_need() {
+        assert_eq!(CrdsGossipPush::compute_need(30, 0, 10), 30);
+        assert_eq!(CrdsGossipPush::compute_need(30, 1, 10), 29);
+        assert_eq!(CrdsGossipPush::compute_need(30, 30, 10), 3);
+        assert_eq!(CrdsGossipPush::compute_need(30, 29, 10), 3);
+    }
+    #[test]
+    fn test_refresh_active_set() {
+        solana_logger::setup();
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let value1 = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+
+        assert_eq!(crds.insert(value1.clone(), 0), Ok(None));
+        push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+
+        assert!(push.active_set.get(&value1.label().pubkey()).is_some());
+        let value2 = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert!(push.active_set.get(&value2.label().pubkey()).is_none());
+        assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
+        for _ in 0..30 {
+            push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+            if push.active_set.get(&value2.label().pubkey()).is_some() {
+                break;
+            }
+        }
+        assert!(push.active_set.get(&value2.label().pubkey()).is_some());
+
+        for _ in 0..push.num_active {
+            let value2 =
+                CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+            assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
+        }
+        push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+        assert_eq!(push.active_set.len(), push.num_active);
+    }
+    #[test]
+    fn test_active_set_refresh_with_bank() {
+        let time = timestamp() - 1024; //make sure there's at least a 1 second delay
+        let mut crds = Crds::default();
+        let push = CrdsGossipPush::default();
+        let mut stakes = HashMap::new();
+        for i in 1..=100 {
+            let peer =
+                CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), time));
+            let id = peer.label().pubkey();
+            crds.insert(peer.clone(), time).unwrap();
+            stakes.insert(id, i * 100);
+        }
+        let mut options = push.push_options(&crds, &Pubkey::default(), &stakes);
+        assert!(!options.is_empty());
+        options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
+        // check that the highest stake holder is also the heaviest weighted.
+        assert_eq!(
+            *stakes.get(&options.get(0).unwrap().1.id).unwrap(),
+            10_000_u64
+        );
+    }
+    #[test]
+    fn test_new_push_messages() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let peer = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
+        push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+
+        let new_msg =
+            CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert_eq!(
+            push.process_push_message(&mut crds, new_msg.clone(), 0),
+            Ok(None)
+        );
+        assert_eq!(push.active_set.len(), 1);
+        assert_eq!(
+            push.new_push_messages(&crds, 0),
+            (vec![peer.label().pubkey()], vec![new_msg])
+        );
+    }
+    #[test]
+    fn test_process_prune() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let peer = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
+        push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+
+        let new_msg =
+            CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert_eq!(
+            push.process_push_message(&mut crds, new_msg.clone(), 0),
+            Ok(None)
+        );
+        push.process_prune_msg(peer.label().pubkey(), &[new_msg.label().pubkey()]);
+        assert_eq!(
+            push.new_push_messages(&crds, 0),
+            (vec![peer.label().pubkey()], vec![])
+        );
+    }
+    #[test]
+    fn test_purge_old_pending_push_messages() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let peer = CrdsValue::ContactInfo(ContactInfo::new_localhost(Keypair::new().pubkey(), 0));
+        assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
+        push.refresh_push_active_set(&crds, &HashMap::new(), Pubkey::default(), 1, 1);
+
+        let mut ci = ContactInfo::new_localhost(Keypair::new().pubkey(), 0);
+        ci.wallclock = 1;
+        let new_msg = CrdsValue::ContactInfo(ci.clone());
+        assert_eq!(
+            push.process_push_message(&mut crds, new_msg.clone(), 1),
+            Ok(None)
+        );
+        push.purge_old_pending_push_messages(&crds, 0);
+        assert_eq!(
+            push.new_push_messages(&crds, 0),
+            (vec![peer.label().pubkey()], vec![])
+        );
+    }
+
+    #[test]
+    fn test_purge_old_pushed_once_messages() {
+        let mut crds = Crds::default();
+        let mut push = CrdsGossipPush::default();
+        let mut ci = ContactInfo::new_localhost(Keypair::new().pubkey(), 0);
+        ci.wallclock = 0;
+        let value = CrdsValue::ContactInfo(ci.clone());
+        let label = value.label();
+        // push a new message
+        assert_eq!(
+            push.process_push_message(&mut crds, value.clone(), 0),
+            Ok(None)
+        );
+        assert_eq!(crds.lookup(&label), Some(&value));
+
+        // push it again
+        assert_eq!(
+            push.process_push_message(&mut crds, value.clone(), 0),
+            Err(CrdsGossipError::PushMessagePrune)
+        );
+
+        // purge the old pushed
+        push.purge_old_pushed_once_messages(1);
+
+        // push it again
+        assert_eq!(
+            push.process_push_message(&mut crds, value.clone(), 0),
+            Err(CrdsGossipError::PushMessageOldVersion)
+        );
+    }
+}

core/src/crds_value.rs

@@ -0,0 +1,276 @@
+use crate::contact_info::ContactInfo;
+use bincode::serialize;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, Signable, Signature};
+use solana_sdk::transaction::Transaction;
+use std::fmt;
+
+/// CrdsValue that is replicated across the cluster
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub enum CrdsValue {
+    /// * Merge Strategy - Latest wallclock is picked
+    ContactInfo(ContactInfo),
+    /// * Merge Strategy - Latest wallclock is picked
+    Vote(Vote),
+    /// * Merge Strategy - Latest wallclock is picked
+    LeaderId(LeaderId),
+}
+
+#[derive(Serialize, Deserialize, Default, Clone, Debug, PartialEq)]
+pub struct LeaderId {
+    pub id: Pubkey,
+    pub signature: Signature,
+    pub leader_id: Pubkey,
+    pub wallclock: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct Vote {
+    pub transaction: Transaction,
+    pub signature: Signature,
+    pub wallclock: u64,
+}
+
+impl Signable for LeaderId {
+    fn pubkey(&self) -> Pubkey {
+        self.id
+    }
+
+    fn signable_data(&self) -> Vec<u8> {
+        #[derive(Serialize)]
+        struct SignData {
+            id: Pubkey,
+            leader_id: Pubkey,
+            wallclock: u64,
+        }
+        let data = SignData {
+            id: self.id,
+            leader_id: self.leader_id,
+            wallclock: self.wallclock,
+        };
+        serialize(&data).expect("unable to serialize LeaderId")
+    }
+
+    fn get_signature(&self) -> Signature {
+        self.signature
+    }
+
+    fn set_signature(&mut self, signature: Signature) {
+        self.signature = signature
+    }
+}
+
+impl Signable for Vote {
+    fn pubkey(&self) -> Pubkey {
+        self.transaction.account_keys[0]
+    }
+
+    fn signable_data(&self) -> Vec<u8> {
+        #[derive(Serialize)]
+        struct SignData {
+            transaction: Transaction,
+            wallclock: u64,
+        }
+        let data = SignData {
+            transaction: self.transaction.clone(),
+            wallclock: self.wallclock,
+        };
+        serialize(&data).expect("unable to serialize Vote")
+    }
+
+    fn get_signature(&self) -> Signature {
+        self.signature
+    }
+
+    fn set_signature(&mut self, signature: Signature) {
+        self.signature = signature
+    }
+}
+
+/// Type of the replicated value
+/// These are labels for values in a record that is associated with `Pubkey`
+#[derive(PartialEq, Hash, Eq, Clone, Debug)]
+pub enum CrdsValueLabel {
+    ContactInfo(Pubkey),
+    Vote(Pubkey),
+    LeaderId(Pubkey),
+}
+
+impl fmt::Display for CrdsValueLabel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
+            CrdsValueLabel::Vote(_) => write!(f, "Vote({})", self.pubkey()),
+            CrdsValueLabel::LeaderId(_) => write!(f, "LeaderId({})", self.pubkey()),
+        }
+    }
+}
+
+impl CrdsValueLabel {
+    pub fn pubkey(&self) -> Pubkey {
+        match self {
+            CrdsValueLabel::ContactInfo(p) => *p,
+            CrdsValueLabel::Vote(p) => *p,
+            CrdsValueLabel::LeaderId(p) => *p,
+        }
+    }
+}
+
+impl LeaderId {
+    pub fn new(id: Pubkey, leader_id: Pubkey, wallclock: u64) -> Self {
+        LeaderId {
+            id,
+            signature: Signature::default(),
+            leader_id,
+            wallclock,
+        }
+    }
+}
+
+impl Vote {
+    // TODO: it might make sense for the transaction to encode the wallclock in the userdata
+    pub fn new(transaction: Transaction, wallclock: u64) -> Self {
+        Vote {
+            transaction,
+            signature: Signature::default(),
+            wallclock,
+        }
+    }
+}
+
+impl CrdsValue {
+    /// Totally unsecure unverfiable wallclock of the node that generated this message
+    /// Latest wallclock is always picked.
+    /// This is used to time out push messages.
+    pub fn wallclock(&self) -> u64 {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => contact_info.wallclock,
+            CrdsValue::Vote(vote) => vote.wallclock,
+            CrdsValue::LeaderId(leader_id) => leader_id.wallclock,
+        }
+    }
+    pub fn label(&self) -> CrdsValueLabel {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => {
+                CrdsValueLabel::ContactInfo(contact_info.pubkey())
+            }
+            CrdsValue::Vote(vote) => CrdsValueLabel::Vote(vote.pubkey()),
+            CrdsValue::LeaderId(leader_id) => CrdsValueLabel::LeaderId(leader_id.pubkey()),
+        }
+    }
+    pub fn contact_info(&self) -> Option<&ContactInfo> {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => Some(contact_info),
+            _ => None,
+        }
+    }
+    pub fn leader_id(&self) -> Option<&LeaderId> {
+        match self {
+            CrdsValue::LeaderId(leader_id) => Some(leader_id),
+            _ => None,
+        }
+    }
+    pub fn vote(&self) -> Option<&Vote> {
+        match self {
+            CrdsValue::Vote(vote) => Some(vote),
+            _ => None,
+        }
+    }
+    /// Return all the possible labels for a record identified by Pubkey.
+    pub fn record_labels(key: Pubkey) -> [CrdsValueLabel; 3] {
+        [
+            CrdsValueLabel::ContactInfo(key),
+            CrdsValueLabel::Vote(key),
+            CrdsValueLabel::LeaderId(key),
+        ]
+    }
+}
+
+impl Signable for CrdsValue {
+    fn sign(&mut self, keypair: &Keypair) {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => contact_info.sign(keypair),
+            CrdsValue::Vote(vote) => vote.sign(keypair),
+            CrdsValue::LeaderId(leader_id) => leader_id.sign(keypair),
+        };
+    }
+    fn verify(&self) -> bool {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => contact_info.verify(),
+            CrdsValue::Vote(vote) => vote.verify(),
+            CrdsValue::LeaderId(leader_id) => leader_id.verify(),
+        }
+    }
+
+    fn pubkey(&self) -> Pubkey {
+        match self {
+            CrdsValue::ContactInfo(contact_info) => contact_info.pubkey(),
+            CrdsValue::Vote(vote) => vote.pubkey(),
+            CrdsValue::LeaderId(leader_id) => leader_id.pubkey(),
+        }
+    }
+
+    fn signable_data(&self) -> Vec<u8> {
+        unimplemented!()
+    }
+
+    fn get_signature(&self) -> Signature {
+        unimplemented!()
+    }
+
+    fn set_signature(&mut self, _: Signature) {
+        unimplemented!()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::contact_info::ContactInfo;
+    use crate::test_tx::test_tx;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::timing::timestamp;
+
+    #[test]
+    fn test_labels() {
+        let mut hits = [false; 3];
+        // this method should cover all the possible labels
+        for v in &CrdsValue::record_labels(Pubkey::default()) {
+            match v {
+                CrdsValueLabel::ContactInfo(_) => hits[0] = true,
+                CrdsValueLabel::Vote(_) => hits[1] = true,
+                CrdsValueLabel::LeaderId(_) => hits[2] = true,
+            }
+        }
+        assert!(hits.iter().all(|x| *x));
+    }
+    #[test]
+    fn test_keys_and_values() {
+        let v = CrdsValue::LeaderId(LeaderId::default());
+        let key = v.clone().leader_id().unwrap().id;
+        assert_eq!(v.wallclock(), 0);
+        assert_eq!(v.label(), CrdsValueLabel::LeaderId(key));
+
+        let v = CrdsValue::ContactInfo(ContactInfo::default());
+        assert_eq!(v.wallclock(), 0);
+        let key = v.clone().contact_info().unwrap().id;
+        assert_eq!(v.label(), CrdsValueLabel::ContactInfo(key));
+
+        let v = CrdsValue::Vote(Vote::new(test_tx(), 0));
+        assert_eq!(v.wallclock(), 0);
+        let key = v.clone().vote().unwrap().transaction.account_keys[0];
+        assert_eq!(v.label(), CrdsValueLabel::Vote(key));
+    }
+    #[test]
+    fn test_signature() {
+        let keypair = Keypair::new();
+        let fake_keypair = Keypair::new();
+        let leader = LeaderId::new(keypair.pubkey(), Pubkey::default(), timestamp());
+        let mut v = CrdsValue::LeaderId(leader);
+        v.sign(&keypair);
+        assert!(v.verify());
+        v.sign(&fake_keypair);
+        assert!(!v.verify());
+    }
+
+}

core/src/db_window.rs

@@ -0,0 +1,527 @@
+//! Set of functions for emulating windowing functions from a database ledger implementation
+use crate::blocktree::*;
+#[cfg(feature = "erasure")]
+use crate::erasure;
+use crate::packet::{SharedBlob, BLOB_HEADER_SIZE};
+use crate::result::Result;
+use crate::streamer::BlobSender;
+use solana_metrics::counter::Counter;
+use solana_sdk::pubkey::Pubkey;
+use std::borrow::Borrow;
+use std::sync::Arc;
+
+pub const MAX_REPAIR_LENGTH: usize = 128;
+
+pub fn retransmit_blobs(dq: &[SharedBlob], retransmit: &BlobSender, id: &Pubkey) -> Result<()> {
+    let mut retransmit_queue: Vec<SharedBlob> = Vec::new();
+    for b in dq {
+        // Don't add blobs generated by this node to the retransmit queue
+        if b.read().unwrap().id() != *id {
+            retransmit_queue.push(b.clone());
+        }
+    }
+
+    if !retransmit_queue.is_empty() {
+        inc_new_counter_info!("streamer-recv_window-retransmit", retransmit_queue.len());
+        retransmit.send(retransmit_queue)?;
+    }
+    Ok(())
+}
+
+/// Process a blob: Add blob to the ledger window.
+pub fn process_blob(blocktree: &Arc<Blocktree>, blob: &SharedBlob) -> Result<()> {
+    let is_coding = blob.read().unwrap().is_coding();
+
+    // Check if the blob is in the range of our known leaders. If not, we return.
+    let (slot, pix) = {
+        let r_blob = blob.read().unwrap();
+        (r_blob.slot(), r_blob.index())
+    };
+
+    // TODO: Once the original leader signature is added to the blob, make sure that
+    // the blob was originally generated by the expected leader for this slot
+
+    // Insert the new blob into block tree
+    if is_coding {
+        let blob = &blob.read().unwrap();
+        blocktree.put_coding_blob_bytes(slot, pix, &blob.data[..BLOB_HEADER_SIZE + blob.size()])?;
+    } else {
+        blocktree.insert_data_blobs(vec![(*blob.read().unwrap()).borrow()])?;
+    }
+
+    #[cfg(feature = "erasure")]
+    {
+        // TODO: Support per-slot erasure. Issue: https://github.com/solana-labs/solana/issues/2441
+        if let Err(e) = try_erasure(blocktree, 0) {
+            trace!(
+                "erasure::recover failed to write recovered coding blobs. Err: {:?}",
+                e
+            );
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "erasure")]
+fn try_erasure(blocktree: &Arc<Blocktree>, slot_index: u64) -> Result<()> {
+    let meta = blocktree.meta(slot_index)?;
+
+    if let Some(meta) = meta {
+        let (data, coding) = erasure::recover(blocktree, slot_index, meta.consumed)?;
+        for c in coding {
+            let c = c.read().unwrap();
+            blocktree.put_coding_blob_bytes(
+                0,
+                c.index(),
+                &c.data[..BLOB_HEADER_SIZE + c.size()],
+            )?;
+        }
+
+        blocktree.write_shared_blobs(data)
+    } else {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::blocktree::get_tmp_ledger_path;
+    #[cfg(all(feature = "erasure", test))]
+    use crate::entry::reconstruct_entries_from_blobs;
+    use crate::entry::{make_tiny_test_entries, EntrySlice};
+    #[cfg(all(feature = "erasure", test))]
+    use crate::erasure::test::{generate_blocktree_from_window, setup_window_ledger};
+    #[cfg(all(feature = "erasure", test))]
+    use crate::erasure::{NUM_CODING, NUM_DATA};
+    use crate::packet::{index_blobs, Blob, Packet, Packets, SharedBlob, PACKET_DATA_SIZE};
+    use crate::streamer::{receiver, responder, PacketReceiver};
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    fn get_msgs(r: PacketReceiver, num: &mut usize) {
+        for _t in 0..5 {
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(m) => *num += m.read().unwrap().packets.len(),
+                e => info!("error {:?}", e),
+            }
+            if *num == 10 {
+                break;
+            }
+        }
+    }
+    #[test]
+    pub fn streamer_debug() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+
+    #[test]
+    pub fn streamer_send_test() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
+
+        let addr = read.local_addr().unwrap();
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let (s_reader, r_reader) = channel();
+        let t_receiver = receiver(
+            Arc::new(read),
+            exit.clone(),
+            s_reader,
+            "window-streamer-test",
+        );
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let t_responder = responder("streamer_send_test", Arc::new(send), r_responder);
+            let mut msgs = Vec::new();
+            for i in 0..10 {
+                let b = SharedBlob::default();
+                {
+                    let mut w = b.write().unwrap();
+                    w.data[0] = i as u8;
+                    w.meta.size = PACKET_DATA_SIZE;
+                    w.meta.set_addr(&addr);
+                }
+                msgs.push(b);
+            }
+            s_responder.send(msgs).expect("send");
+            t_responder
+        };
+
+        let mut num = 0;
+        get_msgs(r_reader, &mut num);
+        assert_eq!(num, 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+    /*
+        #[test]
+        pub fn test_send_to_retransmit_stage() {
+            let leader = Keypair::new().pubkey();
+            let nonleader = Keypair::new().pubkey();
+            let mut leader_scheduler = LeaderScheduler::default();
+            leader_scheduler.set_leader_schedule(vec![leader]);
+            let leader_scheduler = Arc::new(RwLock::new(leader_scheduler));
+            let blob = SharedBlob::default();
+
+            let (blob_sender, blob_receiver) = channel();
+
+            // Expect all blobs to be sent to retransmit_stage
+            blob.write().unwrap().forward(false);
+            retransmit_blobs(
+                &vec![blob.clone()],
+                &leader_scheduler,
+                &blob_sender,
+                &nonleader,
+            )
+            .expect("Expect successful retransmit");
+            let _ = blob_receiver
+                .try_recv()
+                .expect("Expect input blob to be retransmitted");
+
+            blob.write().unwrap().forward(true);
+            retransmit_blobs(
+                &vec![blob.clone()],
+                &leader_scheduler,
+                &blob_sender,
+                &nonleader,
+            )
+            .expect("Expect successful retransmit");
+            let output_blob = blob_receiver
+                .try_recv()
+                .expect("Expect input blob to be retransmitted");
+
+            // retransmit_blobs shouldn't be modifying the blob. That is retransmit stage's job now
+            assert_eq!(*output_blob[0].read().unwrap(), *blob.read().unwrap());
+
+            // Expect blob from leader while currently leader to not be retransmitted
+            // Even when forward is set
+            blob.write().unwrap().forward(true);
+            retransmit_blobs(&vec![blob], &leader_scheduler, &blob_sender, &leader)
+                .expect("Expect successful retransmit");
+            assert!(blob_receiver.try_recv().is_err());
+        }
+    */
+    #[test]
+    pub fn test_find_missing_data_indexes_sanity() {
+        let slot = 0;
+
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+        // Early exit conditions
+        let empty: Vec<u64> = vec![];
+        assert_eq!(blocktree.find_missing_data_indexes(slot, 0, 0, 1), empty);
+        assert_eq!(blocktree.find_missing_data_indexes(slot, 5, 5, 1), empty);
+        assert_eq!(blocktree.find_missing_data_indexes(slot, 4, 3, 1), empty);
+        assert_eq!(blocktree.find_missing_data_indexes(slot, 1, 2, 0), empty);
+
+        let mut blobs = make_tiny_test_entries(2).to_blobs();
+
+        const ONE: u64 = 1;
+        const OTHER: u64 = 4;
+
+        blobs[0].set_index(ONE);
+        blobs[1].set_index(OTHER);
+
+        // Insert one blob at index = first_index
+        blocktree.write_blobs(&blobs).unwrap();
+
+        const STARTS: u64 = OTHER * 2;
+        const END: u64 = OTHER * 3;
+        const MAX: usize = 10;
+        // The first blob has index = first_index. Thus, for i < first_index,
+        // given the input range of [i, first_index], the missing indexes should be
+        // [i, first_index - 1]
+        for start in 0..STARTS {
+            let result = blocktree.find_missing_data_indexes(
+                slot, start, // start
+                END,   //end
+                MAX,   //max
+            );
+            let expected: Vec<u64> = (start..END).filter(|i| *i != ONE && *i != OTHER).collect();
+            assert_eq!(result, expected);
+        }
+
+        drop(blocktree);
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_find_missing_data_indexes() {
+        let slot = 0;
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+        // Write entries
+        let gap = 10;
+        assert!(gap > 3);
+        let num_entries = 10;
+        let mut blobs = make_tiny_test_entries(num_entries).to_blobs();
+        for (i, b) in blobs.iter_mut().enumerate() {
+            b.set_index(i as u64 * gap);
+            b.set_slot(slot);
+        }
+        blocktree.write_blobs(&blobs).unwrap();
+
+        // Index of the first blob is 0
+        // Index of the second blob is "gap"
+        // Thus, the missing indexes should then be [1, gap - 1] for the input index
+        // range of [0, gap)
+        let expected: Vec<u64> = (1..gap).collect();
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 0, gap, gap as usize),
+            expected
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 1, gap, (gap - 1) as usize),
+            expected,
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 0, gap - 1, (gap - 1) as usize),
+            &expected[..expected.len() - 1],
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, gap - 2, gap, gap as usize),
+            vec![gap - 2, gap - 1],
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, gap - 2, gap, 1),
+            vec![gap - 2],
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 0, gap, 1),
+            vec![1],
+        );
+
+        // Test with end indexes that are greater than the last item in the ledger
+        let mut expected: Vec<u64> = (1..gap).collect();
+        expected.push(gap + 1);
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap + 2) as usize),
+            expected,
+        );
+        assert_eq!(
+            blocktree.find_missing_data_indexes(slot, 0, gap + 2, (gap - 1) as usize),
+            &expected[..expected.len() - 1],
+        );
+
+        for i in 0..num_entries as u64 {
+            for j in 0..i {
+                let expected: Vec<u64> = (j..i)
+                    .flat_map(|k| {
+                        let begin = k * gap + 1;
+                        let end = (k + 1) * gap;
+                        (begin..end)
+                    })
+                    .collect();
+                assert_eq!(
+                    blocktree.find_missing_data_indexes(
+                        slot,
+                        j * gap,
+                        i * gap,
+                        ((i - j) * gap) as usize
+                    ),
+                    expected,
+                );
+            }
+        }
+
+        drop(blocktree);
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_find_missing_data_indexes_slots() {
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+        let num_entries_per_slot = 10;
+        let num_slots = 2;
+        let mut blobs = make_tiny_test_entries(num_slots * num_entries_per_slot).to_blobs();
+
+        // Insert every nth entry for each slot
+        let nth = 3;
+        for (i, b) in blobs.iter_mut().enumerate() {
+            b.set_index(((i % num_entries_per_slot) * nth) as u64);
+            b.set_slot((i / num_entries_per_slot) as u64);
+        }
+
+        blocktree.write_blobs(&blobs).unwrap();
+
+        let mut expected: Vec<u64> = (0..num_entries_per_slot)
+            .flat_map(|x| ((nth * x + 1) as u64..(nth * x + nth) as u64))
+            .collect();
+
+        // For each slot, find all missing indexes in the range [0, num_entries_per_slot * nth]
+        for slot_height in 0..num_slots {
+            assert_eq!(
+                blocktree.find_missing_data_indexes(
+                    slot_height as u64,
+                    0,
+                    (num_entries_per_slot * nth) as u64,
+                    num_entries_per_slot * nth as usize
+                ),
+                expected,
+            );
+        }
+
+        // Test with a limit on the number of returned entries
+        for slot_height in 0..num_slots {
+            assert_eq!(
+                blocktree.find_missing_data_indexes(
+                    slot_height as u64,
+                    0,
+                    (num_entries_per_slot * nth) as u64,
+                    num_entries_per_slot * (nth - 1)
+                )[..],
+                expected[..num_entries_per_slot * (nth - 1)],
+            );
+        }
+
+        // Try to find entries in the range [num_entries_per_slot * nth..num_entries_per_slot * (nth + 1)
+        // that don't exist in the ledger.
+        let extra_entries =
+            (num_entries_per_slot * nth) as u64..(num_entries_per_slot * (nth + 1)) as u64;
+        expected.extend(extra_entries);
+
+        // For each slot, find all missing indexes in the range [0, num_entries_per_slot * nth]
+        for slot_height in 0..num_slots {
+            assert_eq!(
+                blocktree.find_missing_data_indexes(
+                    slot_height as u64,
+                    0,
+                    (num_entries_per_slot * (nth + 1)) as u64,
+                    num_entries_per_slot * (nth + 1),
+                ),
+                expected,
+            );
+        }
+    }
+
+    #[test]
+    pub fn test_no_missing_blob_indexes() {
+        let slot = 0;
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+        // Write entries
+        let num_entries = 10;
+        let shared_blobs = make_tiny_test_entries(num_entries).to_shared_blobs();
+
+        index_blobs(&shared_blobs, &Keypair::new().pubkey(), &mut 0, slot);
+
+        let blob_locks: Vec<_> = shared_blobs.iter().map(|b| b.read().unwrap()).collect();
+        let blobs: Vec<&Blob> = blob_locks.iter().map(|b| &**b).collect();
+        blocktree.write_blobs(blobs).unwrap();
+
+        let empty: Vec<u64> = vec![];
+        for i in 0..num_entries as u64 {
+            for j in 0..i {
+                assert_eq!(
+                    blocktree.find_missing_data_indexes(slot, j, i, (i - j) as usize),
+                    empty
+                );
+            }
+        }
+
+        drop(blocktree);
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[cfg(all(feature = "erasure", test))]
+    #[test]
+    pub fn test_try_erasure() {
+        // Setup the window
+        let offset = 0;
+        let num_blobs = NUM_DATA + 2;
+        let slot_height = 0;
+        let mut window = setup_window_ledger(offset, num_blobs, false, slot_height);
+        let end_index = (offset + num_blobs) % window.len();
+
+        // Test erasing a data block and an erasure block
+        let coding_start = offset - (offset % NUM_DATA) + (NUM_DATA - NUM_CODING);
+
+        let erased_index = coding_start % window.len();
+
+        // Create a hole in the window
+        let erased_data = window[erased_index].data.clone();
+        let erased_coding = window[erased_index].coding.clone().unwrap();
+        window[erased_index].data = None;
+        window[erased_index].coding = None;
+
+        // Generate the blocktree from the window
+        let ledger_path = get_tmp_ledger_path!();
+        let blocktree = Arc::new(generate_blocktree_from_window(&ledger_path, &window, false));
+
+        try_erasure(&blocktree, 0).expect("Expected successful erasure attempt");
+        window[erased_index].data = erased_data;
+
+        {
+            let data_blobs: Vec<_> = window[erased_index..end_index]
+                .iter()
+                .map(|slot| slot.data.clone().unwrap())
+                .collect();
+
+            let locks: Vec<_> = data_blobs.iter().map(|blob| blob.read().unwrap()).collect();
+
+            let locked_data: Vec<&Blob> = locks.iter().map(|lock| &**lock).collect();
+
+            let (expected, _) = reconstruct_entries_from_blobs(locked_data).unwrap();
+
+            assert_eq!(
+                blocktree
+                    .get_slot_entries(
+                        0,
+                        erased_index as u64,
+                        Some((end_index - erased_index) as u64)
+                    )
+                    .unwrap(),
+                expected
+            );
+        }
+
+        let erased_coding_l = erased_coding.read().unwrap();
+        assert_eq!(
+            &blocktree
+                .get_coding_blob_bytes(slot_height, erased_index as u64)
+                .unwrap()
+                .unwrap()[BLOB_HEADER_SIZE..],
+            &erased_coding_l.data()[..erased_coding_l.size() as usize],
+        );
+    }
+
+    #[test]
+    fn test_process_blob() {
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Arc::new(Blocktree::open(&blocktree_path).unwrap());
+        let num_entries = 10;
+        let original_entries = make_tiny_test_entries(num_entries);
+        let shared_blobs = original_entries.clone().to_shared_blobs();
+
+        index_blobs(&shared_blobs, &Keypair::new().pubkey(), &mut 0, 0);
+
+        for blob in shared_blobs.iter().rev() {
+            process_blob(&blocktree, blob).expect("Expect successful processing of blob");
+        }
+
+        assert_eq!(
+            blocktree.get_slot_entries(0, 0, None).unwrap(),
+            original_entries
+        );
+
+        drop(blocktree);
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+}

core/src/entry.rs

@@ -0,0 +1,687 @@
+//! The `entry` module is a fundamental building block of Proof of History. It contains a
+//! unique ID that is the hash of the Entry before it, plus the hash of the
+//! transactions within it. Entries cannot be reordered, and its field `num_hashes`
+//! represents an approximate amount of time since the last Entry was created.
+use crate::packet::{Blob, SharedBlob, BLOB_DATA_SIZE};
+use crate::poh::Poh;
+use crate::result::Result;
+use bincode::{deserialize, serialize_into, serialized_size};
+use chrono::prelude::Utc;
+use rayon::prelude::*;
+use solana_sdk::budget_transaction::BudgetTransaction;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::transaction::Transaction;
+use solana_sdk::vote_program::Vote;
+use solana_sdk::vote_transaction::VoteTransaction;
+use std::borrow::Borrow;
+use std::io::Cursor;
+use std::mem::size_of;
+use std::sync::mpsc::{Receiver, Sender};
+use std::sync::{Arc, RwLock};
+
+pub type EntrySender = Sender<Vec<Entry>>;
+pub type EntryReceiver = Receiver<Vec<Entry>>;
+
+/// Each Entry contains three pieces of data. The `num_hashes` field is the number
+/// of hashes performed since the previous entry.  The `hash` field is the result
+/// of hashing `hash` from the previous entry `num_hashes` times.  The `transactions`
+/// field points to Transactions that took place shortly before `hash` was generated.
+///
+/// If you divide `num_hashes` by the amount of time it takes to generate a new hash, you
+/// get a duration estimate since the last Entry. Since processing power increases
+/// over time, one should expect the duration `num_hashes` represents to decrease proportionally.
+/// An upper bound on Duration can be estimated by assuming each hash was generated by the
+/// world's fastest processor at the time the entry was recorded. Or said another way, it
+/// is physically not possible for a shorter duration to have occurred if one assumes the
+/// hash was computed by the world's fastest processor at that time. The hash chain is both
+/// a Verifiable Delay Function (VDF) and a Proof of Work (not to be confused with Proof of
+/// Work consensus!)
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Entry {
+    /// The number of hashes since the previous Entry ID.
+    pub num_hashes: u64,
+
+    /// The SHA-256 hash `num_hashes` after the previous Entry ID.
+    pub hash: Hash,
+
+    /// An unordered list of transactions that were observed before the Entry ID was
+    /// generated. They may have been observed before a previous Entry ID but were
+    /// pushed back into this list to ensure deterministic interpretation of the ledger.
+    pub transactions: Vec<Transaction>,
+}
+
+impl Entry {
+    /// Creates the next Entry `num_hashes` after `start_hash`.
+    pub fn new(prev_hash: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Self {
+        let entry = {
+            if num_hashes == 0 && transactions.is_empty() {
+                Entry {
+                    num_hashes: 0,
+                    hash: *prev_hash,
+                    transactions,
+                }
+            } else if num_hashes == 0 {
+                // If you passed in transactions, but passed in num_hashes == 0, then
+                // next_hash will generate the next hash and set num_hashes == 1
+                let hash = next_hash(prev_hash, 1, &transactions);
+                Entry {
+                    num_hashes: 1,
+                    hash,
+                    transactions,
+                }
+            } else {
+                // Otherwise, the next Entry `num_hashes` after `start_hash`.
+                // If you wanted a tick for instance, then pass in num_hashes = 1
+                // and transactions = empty
+                let hash = next_hash(prev_hash, num_hashes, &transactions);
+                Entry {
+                    num_hashes,
+                    hash,
+                    transactions,
+                }
+            }
+        };
+
+        let size = Entry::serialized_size(&entry.transactions[..]);
+        if size > BLOB_DATA_SIZE as u64 {
+            panic!(
+                "Serialized entry size too large: {} ({} transactions):",
+                size,
+                entry.transactions.len()
+            );
+        }
+        entry
+    }
+
+    pub fn to_shared_blob(&self) -> SharedBlob {
+        let blob = self.to_blob();
+        Arc::new(RwLock::new(blob))
+    }
+
+    pub fn to_blob(&self) -> Blob {
+        let mut blob = Blob::default();
+        let pos = {
+            let mut out = Cursor::new(blob.data_mut());
+            serialize_into(&mut out, &self).expect("failed to serialize output");
+            out.position() as usize
+        };
+        blob.set_size(pos);
+        blob
+    }
+
+    /// Estimate serialized_size of Entry without creating an Entry.
+    pub fn serialized_size(transactions: &[Transaction]) -> u64 {
+        let txs_size: u64 = transactions
+            .iter()
+            .map(|tx| tx.serialized_size().unwrap())
+            .sum();
+        // num_hashes   +    hash  +              txs
+        (2 * size_of::<u64>() + size_of::<Hash>()) as u64 + txs_size
+    }
+
+    pub fn num_will_fit(transactions: &[Transaction]) -> usize {
+        if transactions.is_empty() {
+            return 0;
+        }
+        let mut num = transactions.len();
+        let mut upper = transactions.len();
+        let mut lower = 1; // if one won't fit, we have a lot of TODOs
+        let mut next = transactions.len(); // optimistic
+        loop {
+            debug!(
+                "num {}, upper {} lower {} next {} transactions.len() {}",
+                num,
+                upper,
+                lower,
+                next,
+                transactions.len()
+            );
+            if Self::serialized_size(&transactions[..num]) <= BLOB_DATA_SIZE as u64 {
+                next = (upper + num) / 2;
+                lower = num;
+                debug!("num {} fits, maybe too well? trying {}", num, next);
+            } else {
+                next = (lower + num) / 2;
+                upper = num;
+                debug!("num {} doesn't fit! trying {}", num, next);
+            }
+            // same as last time
+            if next == num {
+                debug!("converged on num {}", num);
+                break;
+            }
+            num = next;
+        }
+        num
+    }
+
+    /// Creates the next Tick Entry `num_hashes` after `start_hash`.
+    pub fn new_mut(
+        start_hash: &mut Hash,
+        num_hashes: &mut u64,
+        transactions: Vec<Transaction>,
+    ) -> Self {
+        let entry = Self::new(start_hash, *num_hashes, transactions);
+        *start_hash = entry.hash;
+        *num_hashes = 0;
+        assert!(serialized_size(&entry).unwrap() <= BLOB_DATA_SIZE as u64);
+        entry
+    }
+
+    /// Creates a Entry from the number of hashes `num_hashes`
+    /// since the previous transaction and that resulting `hash`.
+
+    #[cfg(test)]
+    pub fn new_tick(num_hashes: u64, hash: &Hash) -> Self {
+        Entry {
+            num_hashes,
+            hash: *hash,
+            transactions: vec![],
+        }
+    }
+
+    /// Verifies self.hash is the result of hashing a `start_hash` `self.num_hashes` times.
+    /// If the transaction is not a Tick, then hash that as well.
+    pub fn verify(&self, start_hash: &Hash) -> bool {
+        let ref_hash = next_hash(start_hash, self.num_hashes, &self.transactions);
+        if self.hash != ref_hash {
+            warn!(
+                "next_hash is invalid expected: {:?} actual: {:?}",
+                self.hash, ref_hash
+            );
+            return false;
+        }
+        true
+    }
+
+    pub fn is_tick(&self) -> bool {
+        self.transactions.is_empty()
+    }
+}
+
+/// Creates the hash `num_hashes` after `start_hash`. If the transaction contains
+/// a signature, the final hash will be a hash of both the previous ID and
+/// the signature.  If num_hashes is zero and there's no transaction data,
+///  start_hash is returned.
+fn next_hash(start_hash: &Hash, num_hashes: u64, transactions: &[Transaction]) -> Hash {
+    if num_hashes == 0 && transactions.is_empty() {
+        return *start_hash;
+    }
+
+    let mut poh = Poh::new(*start_hash, 0);
+
+    for _ in 1..num_hashes {
+        poh.hash();
+    }
+
+    if transactions.is_empty() {
+        poh.tick().hash
+    } else {
+        poh.record(Transaction::hash(transactions)).hash
+    }
+}
+
+pub fn reconstruct_entries_from_blobs<I>(blobs: I) -> Result<(Vec<Entry>, u64)>
+where
+    I: IntoIterator,
+    I::Item: Borrow<Blob>,
+{
+    let mut entries: Vec<Entry> = vec![];
+    let mut num_ticks = 0;
+
+    for blob in blobs.into_iter() {
+        let entry: Entry = {
+            let msg_size = blob.borrow().size();
+            deserialize(&blob.borrow().data()[..msg_size])?
+        };
+
+        if entry.is_tick() {
+            num_ticks += 1
+        }
+        entries.push(entry)
+    }
+    Ok((entries, num_ticks))
+}
+
+// an EntrySlice is a slice of Entries
+pub trait EntrySlice {
+    /// Verifies the hashes and counts of a slice of transactions are all consistent.
+    fn verify(&self, start_hash: &Hash) -> bool;
+    fn to_shared_blobs(&self) -> Vec<SharedBlob>;
+    fn to_blobs(&self) -> Vec<Blob>;
+    fn votes(&self) -> Vec<(Pubkey, Vote, Hash)>;
+}
+
+impl EntrySlice for [Entry] {
+    fn verify(&self, start_hash: &Hash) -> bool {
+        let genesis = [Entry {
+            num_hashes: 0,
+            hash: *start_hash,
+            transactions: vec![],
+        }];
+        let entry_pairs = genesis.par_iter().chain(self).zip(self);
+        entry_pairs.all(|(x0, x1)| {
+            let r = x1.verify(&x0.hash);
+            if !r {
+                warn!(
+                    "entry invalid!: x0: {:?}, x1: {:?} num txs: {}",
+                    x0.hash,
+                    x1.hash,
+                    x1.transactions.len()
+                );
+            }
+            r
+        })
+    }
+
+    fn to_blobs(&self) -> Vec<Blob> {
+        self.iter().map(|entry| entry.to_blob()).collect()
+    }
+
+    fn to_shared_blobs(&self) -> Vec<SharedBlob> {
+        self.iter().map(|entry| entry.to_shared_blob()).collect()
+    }
+
+    fn votes(&self) -> Vec<(Pubkey, Vote, Hash)> {
+        self.iter()
+            .flat_map(|entry| {
+                entry
+                    .transactions
+                    .iter()
+                    .flat_map(VoteTransaction::get_votes)
+            })
+            .collect()
+    }
+}
+
+pub fn next_entry_mut(start: &mut Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
+    let entry = Entry::new(&start, num_hashes, transactions);
+    *start = entry.hash;
+    entry
+}
+
+/// Creates the next entries for given transactions, outputs
+/// updates start_hash to hash of last Entry, sets num_hashes to 0
+pub fn next_entries_mut(
+    start_hash: &mut Hash,
+    num_hashes: &mut u64,
+    transactions: Vec<Transaction>,
+) -> Vec<Entry> {
+    // TODO: ?? find a number that works better than |?
+    //                                               V
+    if transactions.is_empty() || transactions.len() == 1 {
+        vec![Entry::new_mut(start_hash, num_hashes, transactions)]
+    } else {
+        let mut chunk_start = 0;
+        let mut entries = Vec::new();
+
+        while chunk_start < transactions.len() {
+            let mut chunk_end = transactions.len();
+            let mut upper = chunk_end;
+            let mut lower = chunk_start;
+            let mut next = chunk_end; // be optimistic that all will fit
+
+            // binary search for how many transactions will fit in an Entry (i.e. a BLOB)
+            loop {
+                debug!(
+                    "chunk_end {}, upper {} lower {} next {} transactions.len() {}",
+                    chunk_end,
+                    upper,
+                    lower,
+                    next,
+                    transactions.len()
+                );
+                if Entry::serialized_size(&transactions[chunk_start..chunk_end])
+                    <= BLOB_DATA_SIZE as u64
+                {
+                    next = (upper + chunk_end) / 2;
+                    lower = chunk_end;
+                    debug!(
+                        "chunk_end {} fits, maybe too well? trying {}",
+                        chunk_end, next
+                    );
+                } else {
+                    next = (lower + chunk_end) / 2;
+                    upper = chunk_end;
+                    debug!("chunk_end {} doesn't fit! trying {}", chunk_end, next);
+                }
+                // same as last time
+                if next == chunk_end {
+                    debug!("converged on chunk_end {}", chunk_end);
+                    break;
+                }
+                chunk_end = next;
+            }
+            entries.push(Entry::new_mut(
+                start_hash,
+                num_hashes,
+                transactions[chunk_start..chunk_end].to_vec(),
+            ));
+            chunk_start = chunk_end;
+        }
+
+        entries
+    }
+}
+
+/// Creates the next Entries for given transactions
+pub fn next_entries(
+    start_hash: &Hash,
+    num_hashes: u64,
+    transactions: Vec<Transaction>,
+) -> Vec<Entry> {
+    let mut hash = *start_hash;
+    let mut num_hashes = num_hashes;
+    next_entries_mut(&mut hash, &mut num_hashes, transactions)
+}
+
+pub fn create_ticks(num_ticks: u64, mut hash: Hash) -> Vec<Entry> {
+    let mut ticks = Vec::with_capacity(num_ticks as usize);
+    for _ in 0..num_ticks {
+        let new_tick = next_entry_mut(&mut hash, 1, vec![]);
+        ticks.push(new_tick);
+    }
+
+    ticks
+}
+
+pub fn make_tiny_test_entries_from_hash(start: &Hash, num: usize) -> Vec<Entry> {
+    let keypair = Keypair::new();
+
+    let mut hash = *start;
+    let mut num_hashes = 0;
+    (0..num)
+        .map(|_| {
+            Entry::new_mut(
+                &mut hash,
+                &mut num_hashes,
+                vec![BudgetTransaction::new_timestamp(
+                    &keypair,
+                    keypair.pubkey(),
+                    keypair.pubkey(),
+                    Utc::now(),
+                    *start,
+                )],
+            )
+        })
+        .collect()
+}
+
+pub fn make_tiny_test_entries(num: usize) -> Vec<Entry> {
+    let zero = Hash::default();
+    let one = solana_sdk::hash::hash(&zero.as_ref());
+    make_tiny_test_entries_from_hash(&one, num)
+}
+
+pub fn make_large_test_entries(num_entries: usize) -> Vec<Entry> {
+    let zero = Hash::default();
+    let one = solana_sdk::hash::hash(&zero.as_ref());
+    let keypair = Keypair::new();
+
+    let tx = BudgetTransaction::new_timestamp(
+        &keypair,
+        keypair.pubkey(),
+        keypair.pubkey(),
+        Utc::now(),
+        one,
+    );
+
+    let serialized_size = tx.serialized_size().unwrap();
+    let num_txs = BLOB_DATA_SIZE / serialized_size as usize;
+    let txs = vec![tx; num_txs];
+    let entry = next_entries(&one, 1, txs)[0].clone();
+    vec![entry; num_entries]
+}
+
+#[cfg(test)]
+pub fn make_consecutive_blobs(
+    id: &Pubkey,
+    num_blobs_to_make: u64,
+    start_height: u64,
+    start_hash: Hash,
+    addr: &std::net::SocketAddr,
+) -> Vec<SharedBlob> {
+    let entries = create_ticks(num_blobs_to_make, start_hash);
+
+    let blobs = entries.to_shared_blobs();
+    let mut index = start_height;
+    for blob in &blobs {
+        let mut blob = blob.write().unwrap();
+        blob.set_index(index);
+        blob.set_id(id);
+        blob.forward(true);
+        blob.meta.set_addr(addr);
+        index += 1;
+    }
+    blobs
+}
+
+#[cfg(test)]
+/// Creates the next Tick or Transaction Entry `num_hashes` after `start_hash`.
+pub fn next_entry(prev_hash: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
+    assert!(num_hashes > 0 || transactions.is_empty());
+    Entry {
+        num_hashes,
+        hash: next_hash(prev_hash, num_hashes, &transactions),
+        transactions,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::entry::Entry;
+    use crate::packet::{to_blobs, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
+    use solana_sdk::hash::hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+    #[test]
+    fn test_entry_verify() {
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        assert!(Entry::new_tick(0, &zero).verify(&zero)); // base case, never used
+        assert!(!Entry::new_tick(0, &zero).verify(&one)); // base case, bad
+        assert!(next_entry(&zero, 1, vec![]).verify(&zero)); // inductive step
+        assert!(!next_entry(&zero, 1, vec![]).verify(&one)); // inductive step, bad
+    }
+
+    #[test]
+    fn test_transaction_reorder_attack() {
+        let zero = Hash::default();
+
+        // First, verify entries
+        let keypair = Keypair::new();
+        let tx0 = SystemTransaction::new_account(&keypair, keypair.pubkey(), 0, zero, 0);
+        let tx1 = SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, zero, 0);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_witness_reorder_attack() {
+        let zero = Hash::default();
+
+        // First, verify entries
+        let keypair = Keypair::new();
+        let tx0 = BudgetTransaction::new_timestamp(
+            &keypair,
+            keypair.pubkey(),
+            keypair.pubkey(),
+            Utc::now(),
+            zero,
+        );
+        let tx1 =
+            BudgetTransaction::new_signature(&keypair, keypair.pubkey(), keypair.pubkey(), zero);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two witness transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_next_entry() {
+        let zero = Hash::default();
+        let tick = next_entry(&zero, 1, vec![]);
+        assert_eq!(tick.num_hashes, 1);
+        assert_ne!(tick.hash, zero);
+
+        let tick = next_entry(&zero, 0, vec![]);
+        assert_eq!(tick.num_hashes, 0);
+        assert_eq!(tick.hash, zero);
+
+        let keypair = Keypair::new();
+        let tx0 = BudgetTransaction::new_timestamp(
+            &keypair,
+            keypair.pubkey(),
+            keypair.pubkey(),
+            Utc::now(),
+            zero,
+        );
+        let entry0 = next_entry(&zero, 1, vec![tx0.clone()]);
+        assert_eq!(entry0.num_hashes, 1);
+        assert_eq!(entry0.hash, next_hash(&zero, 1, &vec![tx0]));
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_next_entry_panic() {
+        let zero = Hash::default();
+        let keypair = Keypair::new();
+        let tx = SystemTransaction::new_account(&keypair, keypair.pubkey(), 0, zero, 0);
+        next_entry(&zero, 0, vec![tx]);
+    }
+
+    #[test]
+    fn test_serialized_size() {
+        let zero = Hash::default();
+        let keypair = Keypair::new();
+        let tx = SystemTransaction::new_account(&keypair, keypair.pubkey(), 0, zero, 0);
+        let entry = next_entry(&zero, 1, vec![tx.clone()]);
+        assert_eq!(
+            Entry::serialized_size(&[tx]),
+            serialized_size(&entry).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_verify_slice() {
+        solana_logger::setup();
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        assert!(vec![][..].verify(&zero)); // base case
+        assert!(vec![Entry::new_tick(0, &zero)][..].verify(&zero)); // singleton case 1
+        assert!(!vec![Entry::new_tick(0, &zero)][..].verify(&one)); // singleton case 2, bad
+        assert!(vec![next_entry(&zero, 0, vec![]); 2][..].verify(&zero)); // inductive step
+
+        let mut bad_ticks = vec![next_entry(&zero, 0, vec![]); 2];
+        bad_ticks[1].hash = one;
+        assert!(!bad_ticks.verify(&zero)); // inductive step, bad
+    }
+
+    fn make_test_entries() -> Vec<Entry> {
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        let keypair = Keypair::new();
+        let vote_account = Keypair::new();
+        let tx0 = VoteTransaction::new_vote(&vote_account, 1, one, 1);
+        let tx1 = BudgetTransaction::new_timestamp(
+            &keypair,
+            keypair.pubkey(),
+            keypair.pubkey(),
+            Utc::now(),
+            one,
+        );
+        //
+        // TODO: this magic number and the mix of transaction types
+        //       is designed to fill up a Blob more or less exactly,
+        //       to get near enough the threshold that
+        //       deserialization falls over if it uses the wrong size()
+        //       parameter to index into blob.data()
+        //
+        // magic numbers -----------------+
+        //                                |
+        //                                V
+        let mut transactions = vec![tx0; 362];
+        transactions.extend(vec![tx1; 100]);
+        next_entries(&zero, 0, transactions)
+    }
+
+    #[test]
+    fn test_entries_to_shared_blobs() {
+        solana_logger::setup();
+        let entries = make_test_entries();
+
+        let blob_q = entries.to_blobs();
+
+        assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap().0, entries);
+    }
+
+    #[test]
+    fn test_bad_blobs_attack() {
+        solana_logger::setup();
+        let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        let blobs_q = to_blobs(vec![(0, addr)]).unwrap(); // <-- attack!
+        assert!(reconstruct_entries_from_blobs(blobs_q).is_err());
+    }
+
+    #[test]
+    fn test_next_entries() {
+        solana_logger::setup();
+        let hash = Hash::default();
+        let next_hash = solana_sdk::hash::hash(&hash.as_ref());
+        let keypair = Keypair::new();
+        let vote_account = Keypair::new();
+        let tx_small = VoteTransaction::new_vote(&vote_account, 1, next_hash, 2);
+        let tx_large = BudgetTransaction::new(&keypair, keypair.pubkey(), 1, next_hash);
+
+        let tx_small_size = tx_small.serialized_size().unwrap() as usize;
+        let tx_large_size = tx_large.serialized_size().unwrap() as usize;
+        let entry_size = serialized_size(&Entry {
+            num_hashes: 0,
+            hash: Hash::default(),
+            transactions: vec![],
+        })
+        .unwrap() as usize;
+        assert!(tx_small_size < tx_large_size);
+        assert!(tx_large_size < PACKET_DATA_SIZE);
+
+        let threshold = (BLOB_DATA_SIZE - entry_size) / tx_small_size;
+
+        // verify no split
+        let transactions = vec![tx_small.clone(); threshold];
+        let entries0 = next_entries(&hash, 0, transactions.clone());
+        assert_eq!(entries0.len(), 1);
+        assert!(entries0.verify(&hash));
+
+        // verify the split with uniform transactions
+        let transactions = vec![tx_small.clone(); threshold * 2];
+        let entries0 = next_entries(&hash, 0, transactions.clone());
+        assert_eq!(entries0.len(), 2);
+        assert!(entries0.verify(&hash));
+
+        // verify the split with small transactions followed by large
+        // transactions
+        let mut transactions = vec![tx_small.clone(); BLOB_DATA_SIZE / tx_small_size];
+        let large_transactions = vec![tx_large.clone(); BLOB_DATA_SIZE / tx_large_size];
+
+        transactions.extend(large_transactions);
+
+        let entries0 = next_entries(&hash, 0, transactions.clone());
+        assert!(entries0.len() >= 2);
+        assert!(entries0.verify(&hash));
+    }
+
+}

core/src/fetch_stage.rs

@@ -0,0 +1,57 @@
+//! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
+
+use crate::service::Service;
+use crate::streamer::{self, PacketReceiver, PacketSender};
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::{self, JoinHandle};
+
+pub struct FetchStage {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl FetchStage {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(sockets: Vec<UdpSocket>, exit: Arc<AtomicBool>) -> (Self, PacketReceiver) {
+        let (sender, receiver) = channel();
+        (Self::new_with_sender(sockets, exit, &sender), receiver)
+    }
+    pub fn new_with_sender(
+        sockets: Vec<UdpSocket>,
+        exit: Arc<AtomicBool>,
+        sender: &PacketSender,
+    ) -> Self {
+        let tx_sockets = sockets.into_iter().map(Arc::new).collect();
+        Self::new_multi_socket(tx_sockets, exit, &sender)
+    }
+    fn new_multi_socket(
+        sockets: Vec<Arc<UdpSocket>>,
+        exit: Arc<AtomicBool>,
+        sender: &PacketSender,
+    ) -> Self {
+        let thread_hdls: Vec<_> = sockets
+            .into_iter()
+            .map(|socket| streamer::receiver(socket, exit.clone(), sender.clone(), "fetch-stage"))
+            .collect();
+
+        Self { exit, thread_hdls }
+    }
+
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for FetchStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

core/src/fullnode.rs

@@ -0,0 +1,776 @@
+//! The `fullnode` module hosts all the fullnode microservices.
+
+use crate::bank_forks::BankForks;
+use crate::blocktree::Blocktree;
+use crate::blocktree_processor::{self, BankForksInfo};
+use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
+use crate::entry::create_ticks;
+use crate::entry::next_entry_mut;
+use crate::entry::Entry;
+use crate::gossip_service::GossipService;
+use crate::poh_recorder::PohRecorder;
+use crate::poh_service::{PohService, PohServiceConfig};
+use crate::rpc_pubsub_service::PubSubService;
+use crate::rpc_service::JsonRpcService;
+use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::service::Service;
+use crate::storage_stage::StorageState;
+use crate::tpu::Tpu;
+use crate::tvu::{Sockets, Tvu, TvuRotationInfo, TvuRotationReceiver};
+use crate::voting_keypair::VotingKeypair;
+use solana_metrics::counter::Counter;
+use solana_sdk::genesis_block::GenesisBlock;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::system_transaction::SystemTransaction;
+use solana_sdk::timing::timestamp;
+use solana_sdk::vote_transaction::VoteTransaction;
+use std::net::UdpSocket;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::JoinHandle;
+use std::thread::{spawn, Result};
+use std::time::Duration;
+
+struct NodeServices {
+    tpu: Tpu,
+    tvu: Tvu,
+}
+
+impl NodeServices {
+    fn new(tpu: Tpu, tvu: Tvu) -> Self {
+        NodeServices { tpu, tvu }
+    }
+
+    fn join(self) -> Result<()> {
+        self.tpu.join()?;
+        //tvu will never stop unless exit is signaled
+        self.tvu.join()?;
+        Ok(())
+    }
+
+    fn exit(&self) {
+        self.tpu.exit();
+        self.tvu.exit();
+    }
+}
+
+pub struct FullnodeConfig {
+    pub sigverify_disabled: bool,
+    pub voting_disabled: bool,
+    pub blockstream: Option<String>,
+    pub storage_rotate_count: u64,
+    pub tick_config: PohServiceConfig,
+    pub account_paths: Option<String>,
+}
+impl Default for FullnodeConfig {
+    fn default() -> Self {
+        // TODO: remove this, temporary parameter to configure
+        // storage amount differently for test configurations
+        // so tests don't take forever to run.
+        const NUM_HASHES_FOR_STORAGE_ROTATE: u64 = 1024;
+        Self {
+            sigverify_disabled: false,
+            voting_disabled: false,
+            blockstream: None,
+            storage_rotate_count: NUM_HASHES_FOR_STORAGE_ROTATE,
+            tick_config: PohServiceConfig::default(),
+            account_paths: None,
+        }
+    }
+}
+
+pub struct Fullnode {
+    id: Pubkey,
+    exit: Arc<AtomicBool>,
+    rpc_service: Option<JsonRpcService>,
+    rpc_pubsub_service: Option<PubSubService>,
+    gossip_service: GossipService,
+    sigverify_disabled: bool,
+    tpu_sockets: Vec<UdpSocket>,
+    broadcast_socket: UdpSocket,
+    node_services: NodeServices,
+    rotation_receiver: TvuRotationReceiver,
+    blocktree: Arc<Blocktree>,
+    poh_service: PohService,
+    poh_recorder: Arc<Mutex<PohRecorder>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+}
+
+impl Fullnode {
+    pub fn new<T>(
+        mut node: Node,
+        keypair: &Arc<Keypair>,
+        ledger_path: &str,
+        voting_keypair: T,
+        entrypoint_info_option: Option<&NodeInfo>,
+        config: &FullnodeConfig,
+    ) -> Self
+    where
+        T: 'static + KeypairUtil + Sync + Send,
+    {
+        info!("creating bank...");
+
+        let id = keypair.pubkey();
+        assert_eq!(id, node.info.id);
+
+        let (bank_forks, bank_forks_info, blocktree, ledger_signal_receiver) =
+            new_banks_from_blocktree(ledger_path, config.account_paths.clone());
+
+        let exit = Arc::new(AtomicBool::new(false));
+        let bank_info = &bank_forks_info[0];
+        let bank = bank_forks[bank_info.bank_id].clone();
+
+        info!("starting PoH... {} {}", bank.tick_height(), bank.last_id(),);
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(
+            bank.tick_height(),
+            bank.last_id(),
+        )));
+        let poh_service = PohService::new(poh_recorder.clone(), &config.tick_config, exit.clone());
+
+        info!("node info: {:?}", node.info);
+        info!("node entrypoint_info: {:?}", entrypoint_info_option);
+        info!(
+            "node local gossip address: {}",
+            node.sockets.gossip.local_addr().unwrap()
+        );
+
+        let blocktree = Arc::new(blocktree);
+        let bank_forks = Arc::new(RwLock::new(bank_forks));
+
+        node.info.wallclock = timestamp();
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_keypair(
+            node.info.clone(),
+            keypair.clone(),
+        )));
+
+        // TODO: The RPC service assumes that there is a drone running on the cluster
+        //       entrypoint, which is a bad assumption.
+        //       See https://github.com/solana-labs/solana/issues/1830 for the removal of drone
+        //       from the RPC API
+        let drone_addr = {
+            let mut entrypoint_drone_addr = match entrypoint_info_option {
+                Some(entrypoint_info_info) => entrypoint_info_info.rpc,
+                None => node.info.rpc,
+            };
+            entrypoint_drone_addr.set_port(solana_drone::drone::DRONE_PORT);
+            entrypoint_drone_addr
+        };
+
+        let storage_state = StorageState::new();
+
+        let rpc_service = JsonRpcService::new(
+            &cluster_info,
+            SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()),
+            drone_addr,
+            storage_state.clone(),
+        );
+
+        let subscriptions = Arc::new(RpcSubscriptions::default());
+        let rpc_pubsub_service = PubSubService::new(
+            &subscriptions,
+            SocketAddr::new(
+                IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+                node.info.rpc_pubsub.port(),
+            ),
+        );
+
+        let gossip_service = GossipService::new(
+            &cluster_info,
+            Some(blocktree.clone()),
+            Some(bank_forks.clone()),
+            node.sockets.gossip,
+            exit.clone(),
+        );
+
+        // Insert the entrypoint info, should only be None if this node
+        // is the bootstrap leader
+        if let Some(entrypoint_info) = entrypoint_info_option {
+            cluster_info
+                .write()
+                .unwrap()
+                .insert_info(entrypoint_info.clone());
+        }
+
+        let sockets = Sockets {
+            repair: node
+                .sockets
+                .repair
+                .try_clone()
+                .expect("Failed to clone repair socket"),
+            retransmit: node
+                .sockets
+                .retransmit
+                .try_clone()
+                .expect("Failed to clone retransmit socket"),
+            fetch: node
+                .sockets
+                .tvu
+                .iter()
+                .map(|s| s.try_clone().expect("Failed to clone TVU Sockets"))
+                .collect(),
+        };
+
+        let voting_keypair_option = if config.voting_disabled {
+            None
+        } else {
+            Some(Arc::new(voting_keypair))
+        };
+
+        // Setup channel for rotation indications
+        let (rotation_sender, rotation_receiver) = channel();
+
+        let tvu = Tvu::new(
+            voting_keypair_option,
+            &bank_forks,
+            &bank_forks_info,
+            &cluster_info,
+            sockets,
+            blocktree.clone(),
+            config.storage_rotate_count,
+            &rotation_sender,
+            &storage_state,
+            config.blockstream.as_ref(),
+            ledger_signal_receiver,
+            &subscriptions,
+        );
+        let tpu = Tpu::new(id, &cluster_info);
+
+        inc_new_counter_info!("fullnode-new", 1);
+        Self {
+            id,
+            sigverify_disabled: config.sigverify_disabled,
+            gossip_service,
+            rpc_service: Some(rpc_service),
+            rpc_pubsub_service: Some(rpc_pubsub_service),
+            node_services: NodeServices::new(tpu, tvu),
+            exit,
+            tpu_sockets: node.sockets.tpu,
+            broadcast_socket: node.sockets.broadcast,
+            rotation_receiver,
+            blocktree,
+            poh_service,
+            poh_recorder,
+            bank_forks,
+        }
+    }
+
+    fn rotate(&mut self, rotation_info: TvuRotationInfo) {
+        trace!(
+            "{:?}: rotate for slot={} to leader={:?}",
+            self.id,
+            rotation_info.slot,
+            rotation_info.leader_id,
+        );
+
+        if let Some(ref mut rpc_service) = self.rpc_service {
+            // TODO: This is not the correct bank.  Instead TVU should pass along the
+            // frozen Bank for each completed block for RPC to use from it's notion of the "best"
+            // available fork (until we want to surface multiple forks to RPC)
+            rpc_service.set_bank(&self.bank_forks.read().unwrap().working_bank());
+        }
+
+        if rotation_info.leader_id == self.id {
+            debug!("{:?} rotating to leader role", self.id);
+            let tpu_bank = self
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(rotation_info.slot)
+                .unwrap()
+                .clone();
+            self.node_services.tpu.switch_to_leader(
+                &tpu_bank,
+                &self.poh_recorder,
+                self.tpu_sockets
+                    .iter()
+                    .map(|s| s.try_clone().expect("Failed to clone TPU sockets"))
+                    .collect(),
+                self.broadcast_socket
+                    .try_clone()
+                    .expect("Failed to clone broadcast socket"),
+                self.sigverify_disabled,
+                rotation_info.slot,
+                &self.blocktree,
+            );
+        } else {
+            self.node_services.tpu.switch_to_forwarder(
+                rotation_info.leader_id,
+                self.tpu_sockets
+                    .iter()
+                    .map(|s| s.try_clone().expect("Failed to clone TPU sockets"))
+                    .collect(),
+            );
+        }
+    }
+
+    // Runs a thread to manage node role transitions.  The returned closure can be used to signal the
+    // node to exit.
+    pub fn start(
+        mut self,
+        rotation_notifier: Option<Sender<u64>>,
+    ) -> (JoinHandle<()>, Arc<AtomicBool>, Receiver<bool>) {
+        let (sender, receiver) = channel();
+        let exit = self.exit.clone();
+        let timeout = Duration::from_secs(1);
+        let handle = spawn(move || loop {
+            if self.exit.load(Ordering::Relaxed) {
+                debug!("node shutdown requested");
+                self.close().expect("Unable to close node");
+                let _ = sender.send(true);
+                break;
+            }
+
+            match self.rotation_receiver.recv_timeout(timeout) {
+                Ok(rotation_info) => {
+                    trace!("{:?}: rotate at slot={}", self.id, rotation_info.slot);
+                    //TODO: this will be called by the TVU every time it votes
+                    //instead of here
+                    info!(
+                        "reset PoH... {} {}",
+                        rotation_info.tick_height, rotation_info.last_id
+                    );
+                    self.poh_recorder
+                        .lock()
+                        .unwrap()
+                        .reset(rotation_info.tick_height, rotation_info.last_id);
+                    let slot = rotation_info.slot;
+                    self.rotate(rotation_info);
+                    debug!("role transition complete");
+                    if let Some(ref rotation_notifier) = rotation_notifier {
+                        rotation_notifier.send(slot).unwrap();
+                    }
+                }
+                Err(RecvTimeoutError::Timeout) => continue,
+                _ => (),
+            }
+        });
+        (handle, exit, receiver)
+    }
+
+    pub fn run(self, rotation_notifier: Option<Sender<u64>>) -> impl FnOnce() {
+        let (_, exit, receiver) = self.start(rotation_notifier);
+        move || {
+            exit.store(true, Ordering::Relaxed);
+            receiver.recv().unwrap();
+            debug!("node shutdown complete");
+        }
+    }
+
+    // Used for notifying many nodes in parallel to exit
+    fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+        // Need to force the poh_recorder to drop the WorkingBank,
+        // which contains the channel to BroadcastStage. This should be
+        // sufficient as long as no other rotations are happening that
+        // can cause the Tpu to restart a BankingStage and reset a
+        // WorkingBank in poh_recorder. It follows no other rotations can be
+        // in motion because exit()/close() are only called by the run() loop
+        // which is the sole initiator of rotations.
+        self.poh_recorder.lock().unwrap().clear_bank();
+        if let Some(ref rpc_service) = self.rpc_service {
+            rpc_service.exit();
+        }
+        if let Some(ref rpc_pubsub_service) = self.rpc_pubsub_service {
+            rpc_pubsub_service.exit();
+        }
+        self.node_services.exit();
+        self.poh_service.exit()
+    }
+
+    fn close(self) -> Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+pub fn new_banks_from_blocktree(
+    blocktree_path: &str,
+    account_paths: Option<String>,
+) -> (BankForks, Vec<BankForksInfo>, Blocktree, Receiver<bool>) {
+    let genesis_block =
+        GenesisBlock::load(blocktree_path).expect("Expected to successfully open genesis block");
+
+    let (blocktree, ledger_signal_receiver) =
+        Blocktree::open_with_config_signal(blocktree_path, genesis_block.ticks_per_slot)
+            .expect("Expected to successfully open database ledger");
+
+    let (bank_forks, bank_forks_info) =
+        blocktree_processor::process_blocktree(&genesis_block, &blocktree, account_paths)
+            .expect("process_blocktree failed");
+
+    (
+        bank_forks,
+        bank_forks_info,
+        blocktree,
+        ledger_signal_receiver,
+    )
+}
+
+impl Service for Fullnode {
+    type JoinReturnType = ();
+
+    fn join(self) -> Result<()> {
+        if let Some(rpc_service) = self.rpc_service {
+            rpc_service.join()?;
+        }
+        if let Some(rpc_pubsub_service) = self.rpc_pubsub_service {
+            rpc_pubsub_service.join()?;
+        }
+
+        self.gossip_service.join()?;
+        self.node_services.join()?;
+        trace!("exit node_services!");
+        self.poh_service.join()?;
+        trace!("exit poh!");
+        Ok(())
+    }
+}
+
+// Create entries such the node identified by active_keypair will be added to the active set for
+// leader selection, and append `num_ending_ticks` empty tick entries.
+pub fn make_active_set_entries(
+    active_keypair: &Arc<Keypair>,
+    token_source: &Keypair,
+    stake: u64,
+    slot_height_to_vote_on: u64,
+    last_id: &Hash,
+    num_ending_ticks: u64,
+) -> (Vec<Entry>, VotingKeypair) {
+    // 1) Assume the active_keypair node has no tokens staked
+    let transfer_tx =
+        SystemTransaction::new_account(&token_source, active_keypair.pubkey(), stake, *last_id, 0);
+    let mut last_entry_hash = *last_id;
+    let transfer_entry = next_entry_mut(&mut last_entry_hash, 1, vec![transfer_tx]);
+
+    // 2) Create and register a vote account for active_keypair
+    let voting_keypair = VotingKeypair::new_local(active_keypair);
+    let vote_account_id = voting_keypair.pubkey();
+
+    let new_vote_account_tx =
+        VoteTransaction::fund_staking_account(active_keypair, vote_account_id, *last_id, 1, 1);
+    let new_vote_account_entry = next_entry_mut(&mut last_entry_hash, 1, vec![new_vote_account_tx]);
+
+    // 3) Create vote entry
+    let vote_tx = VoteTransaction::new_vote(&voting_keypair, slot_height_to_vote_on, *last_id, 0);
+    let vote_entry = next_entry_mut(&mut last_entry_hash, 1, vec![vote_tx]);
+
+    // 4) Create `num_ending_ticks` empty ticks
+    let mut entries = vec![transfer_entry, new_vote_account_entry, vote_entry];
+    let empty_ticks = create_ticks(num_ending_ticks, last_entry_hash);
+    entries.extend(empty_ticks);
+
+    (entries, voting_keypair)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::blocktree::{create_new_tmp_ledger, tmp_copy_blocktree};
+    use crate::entry::make_consecutive_blobs;
+    use crate::streamer::responder;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::timing::DEFAULT_SLOTS_PER_EPOCH;
+    use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
+    use std::fs::remove_dir_all;
+
+    #[test]
+    fn validator_exit() {
+        let leader_keypair = Keypair::new();
+        let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
+
+        let validator_keypair = Keypair::new();
+        let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
+        let (genesis_block, _mint_keypair) =
+            GenesisBlock::new_with_leader(10_000, leader_keypair.pubkey(), 1000);
+        let (validator_ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        let validator = Fullnode::new(
+            validator_node,
+            &Arc::new(validator_keypair),
+            &validator_ledger_path,
+            Keypair::new(),
+            Some(&leader_node.info),
+            &FullnodeConfig::default(),
+        );
+        validator.close().unwrap();
+        remove_dir_all(validator_ledger_path).unwrap();
+    }
+
+    #[test]
+    fn validator_parallel_exit() {
+        let leader_keypair = Keypair::new();
+        let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
+
+        let mut ledger_paths = vec![];
+        let validators: Vec<Fullnode> = (0..2)
+            .map(|_| {
+                let validator_keypair = Keypair::new();
+                let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
+                let (genesis_block, _mint_keypair) =
+                    GenesisBlock::new_with_leader(10_000, leader_keypair.pubkey(), 1000);
+                let (validator_ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+                ledger_paths.push(validator_ledger_path.clone());
+                Fullnode::new(
+                    validator_node,
+                    &Arc::new(validator_keypair),
+                    &validator_ledger_path,
+                    Keypair::new(),
+                    Some(&leader_node.info),
+                    &FullnodeConfig::default(),
+                )
+            })
+            .collect();
+
+        // Each validator can exit in parallel to speed many sequential calls to `join`
+        validators.iter().for_each(|v| v.exit());
+        // While join is called sequentially, the above exit call notified all the
+        // validators to exit from all their threads
+        validators.into_iter().for_each(|validator| {
+            validator.join().unwrap();
+        });
+
+        for path in ledger_paths {
+            remove_dir_all(path).unwrap();
+        }
+    }
+
+    #[test]
+    fn test_leader_to_leader_transition() {
+        solana_logger::setup();
+
+        let bootstrap_leader_keypair = Keypair::new();
+        let bootstrap_leader_node =
+            Node::new_localhost_with_pubkey(bootstrap_leader_keypair.pubkey());
+
+        // Once the bootstrap leader hits the second epoch, because there are no other choices in
+        // the active set, this leader will remain the leader in the second epoch. In the second
+        // epoch, check that the same leader knows to shut down and restart as a leader again.
+        let ticks_per_slot = 5;
+        let slots_per_epoch = 2;
+
+        let voting_keypair = Keypair::new();
+        let fullnode_config = FullnodeConfig::default();
+
+        let (mut genesis_block, _mint_keypair) =
+            GenesisBlock::new_with_leader(10_000, bootstrap_leader_keypair.pubkey(), 500);
+        genesis_block.ticks_per_slot = ticks_per_slot;
+        genesis_block.slots_per_epoch = slots_per_epoch;
+
+        let (bootstrap_leader_ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        // Start the bootstrap leader
+        let bootstrap_leader = Fullnode::new(
+            bootstrap_leader_node,
+            &Arc::new(bootstrap_leader_keypair),
+            &bootstrap_leader_ledger_path,
+            voting_keypair,
+            None,
+            &fullnode_config,
+        );
+
+        let (rotation_sender, rotation_receiver) = channel();
+        let bootstrap_leader_exit = bootstrap_leader.run(Some(rotation_sender));
+
+        // Wait for the bootstrap leader to transition.  Since there are no other nodes in the
+        // cluster it will continue to be the leader
+        assert_eq!(rotation_receiver.recv().unwrap(), 1);
+        bootstrap_leader_exit();
+    }
+
+    #[test]
+    #[ignore]
+    fn test_ledger_role_transition() {
+        solana_logger::setup();
+
+        let fullnode_config = FullnodeConfig::default();
+        let ticks_per_slot = DEFAULT_TICKS_PER_SLOT;
+
+        // Create the leader and validator nodes
+        let bootstrap_leader_keypair = Arc::new(Keypair::new());
+        let validator_keypair = Arc::new(Keypair::new());
+        let (bootstrap_leader_node, validator_node, bootstrap_leader_ledger_path, _, _) =
+            setup_leader_validator(
+                &bootstrap_leader_keypair,
+                &validator_keypair,
+                ticks_per_slot,
+                0,
+            );
+        let bootstrap_leader_info = bootstrap_leader_node.info.clone();
+
+        let validator_ledger_path = tmp_copy_blocktree!(&bootstrap_leader_ledger_path);
+
+        let ledger_paths = vec![
+            bootstrap_leader_ledger_path.clone(),
+            validator_ledger_path.clone(),
+        ];
+
+        {
+            // Test that a node knows to transition to a validator based on parsing the ledger
+            let bootstrap_leader = Fullnode::new(
+                bootstrap_leader_node,
+                &bootstrap_leader_keypair,
+                &bootstrap_leader_ledger_path,
+                Keypair::new(),
+                Some(&bootstrap_leader_info),
+                &fullnode_config,
+            );
+            let (rotation_sender, rotation_receiver) = channel();
+            let bootstrap_leader_exit = bootstrap_leader.run(Some(rotation_sender));
+            assert_eq!(rotation_receiver.recv().unwrap(), (DEFAULT_SLOTS_PER_EPOCH));
+
+            // Test that a node knows to transition to a leader based on parsing the ledger
+            let validator = Fullnode::new(
+                validator_node,
+                &validator_keypair,
+                &validator_ledger_path,
+                Keypair::new(),
+                Some(&bootstrap_leader_info),
+                &fullnode_config,
+            );
+
+            let (rotation_sender, rotation_receiver) = channel();
+            let validator_exit = validator.run(Some(rotation_sender));
+            assert_eq!(rotation_receiver.recv().unwrap(), (DEFAULT_SLOTS_PER_EPOCH));
+
+            validator_exit();
+            bootstrap_leader_exit();
+        }
+        for path in ledger_paths {
+            Blocktree::destroy(&path).expect("Expected successful database destruction");
+            let _ignored = remove_dir_all(&path);
+        }
+    }
+
+    // TODO: Rework this test or TVU (make_consecutive_blobs sends blobs that can't be handled by
+    //       the replay_stage)
+    #[test]
+    #[ignore]
+    fn test_validator_to_leader_transition() {
+        solana_logger::setup();
+        // Make leader and validator node
+        let ticks_per_slot = 10;
+        let slots_per_epoch = 4;
+        let leader_keypair = Arc::new(Keypair::new());
+        let validator_keypair = Arc::new(Keypair::new());
+        let fullnode_config = FullnodeConfig::default();
+        let (leader_node, validator_node, validator_ledger_path, ledger_initial_len, last_id) =
+            setup_leader_validator(&leader_keypair, &validator_keypair, ticks_per_slot, 0);
+
+        let leader_id = leader_keypair.pubkey();
+        let validator_info = validator_node.info.clone();
+
+        info!("leader: {:?}", leader_id);
+        info!("validator: {:?}", validator_info.id);
+
+        let voting_keypair = Keypair::new();
+
+        // Start the validator
+        let validator = Fullnode::new(
+            validator_node,
+            &validator_keypair,
+            &validator_ledger_path,
+            voting_keypair,
+            Some(&leader_node.info),
+            &fullnode_config,
+        );
+
+        let blobs_to_send = slots_per_epoch * ticks_per_slot + ticks_per_slot;
+
+        // Send blobs to the validator from our mock leader
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let blob_sockets: Vec<Arc<UdpSocket>> =
+                leader_node.sockets.tvu.into_iter().map(Arc::new).collect();
+            let t_responder = responder(
+                "test_validator_to_leader_transition",
+                blob_sockets[0].clone(),
+                r_responder,
+            );
+
+            let tvu_address = &validator_info.tvu;
+
+            let msgs = make_consecutive_blobs(
+                &leader_id,
+                blobs_to_send,
+                ledger_initial_len,
+                last_id,
+                &tvu_address,
+            )
+            .into_iter()
+            .rev()
+            .collect();
+            s_responder.send(msgs).expect("send");
+            t_responder
+        };
+
+        info!("waiting for validator to rotate into the leader role");
+        let (rotation_sender, rotation_receiver) = channel();
+        let validator_exit = validator.run(Some(rotation_sender));
+        let rotation = rotation_receiver.recv().unwrap();
+        assert_eq!(rotation, blobs_to_send);
+
+        // Close the validator so that rocksdb has locks available
+        validator_exit();
+        let (bank_forks, bank_forks_info, _, _) =
+            new_banks_from_blocktree(&validator_ledger_path, None);
+        let bank = bank_forks.working_bank();
+        let entry_height = bank_forks_info[0].entry_height;
+
+        assert!(bank.tick_height() >= bank.ticks_per_slot() * bank.slots_per_epoch());
+
+        assert!(entry_height >= ledger_initial_len);
+
+        // Shut down
+        t_responder.join().expect("responder thread join");
+        Blocktree::destroy(&validator_ledger_path)
+            .expect("Expected successful database destruction");
+        let _ignored = remove_dir_all(&validator_ledger_path).unwrap();
+    }
+
+    fn setup_leader_validator(
+        leader_keypair: &Arc<Keypair>,
+        validator_keypair: &Arc<Keypair>,
+        ticks_per_slot: u64,
+        num_ending_slots: u64,
+    ) -> (Node, Node, String, u64, Hash) {
+        info!("validator: {}", validator_keypair.pubkey());
+        info!("leader: {}", leader_keypair.pubkey());
+
+        let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
+        let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
+
+        let (mut genesis_block, mint_keypair) =
+            GenesisBlock::new_with_leader(10_000, leader_node.info.id, 500);
+        genesis_block.ticks_per_slot = ticks_per_slot;
+
+        let (ledger_path, last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        // Add entries so that the validator is in the active set, then finish up the slot with
+        // ticks (and maybe add extra slots full of empty ticks)
+        let (entries, _) = make_active_set_entries(
+            validator_keypair,
+            &mint_keypair,
+            10,
+            0,
+            &last_id,
+            ticks_per_slot * (num_ending_slots + 1),
+        );
+
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap();
+        let last_id = entries.last().unwrap().hash;
+        let entry_height = ticks_per_slot + entries.len() as u64;
+        blocktree.write_entries(1, 0, 0, entries).unwrap();
+
+        (
+            leader_node,
+            validator_node,
+            ledger_path,
+            entry_height,
+            last_id,
+        )
+    }
+}

core/src/gen_keys.rs

@@ -0,0 +1,68 @@
+//! The `signature` module provides functionality for public, and private keys.
+
+use rand::{Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use rayon::prelude::*;
+use solana_sdk::signature::Keypair;
+use untrusted::Input;
+
+pub struct GenKeys {
+    generator: ChaChaRng,
+}
+
+impl GenKeys {
+    pub fn new(seed: [u8; 32]) -> GenKeys {
+        let generator = ChaChaRng::from_seed(seed);
+        GenKeys { generator }
+    }
+
+    fn gen_seed(&mut self) -> [u8; 32] {
+        let mut seed = [0u8; 32];
+        self.generator.fill(&mut seed);
+        seed
+    }
+
+    fn gen_n_seeds(&mut self, n: u64) -> Vec<[u8; 32]> {
+        (0..n).map(|_| self.gen_seed()).collect()
+    }
+
+    pub fn gen_n_keypairs(&mut self, n: u64) -> Vec<Keypair> {
+        self.gen_n_seeds(n)
+            .into_par_iter()
+            .map(|seed| Keypair::from_seed_unchecked(Input::from(&seed)).unwrap())
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    pub use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::KeypairUtil;
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_new_key_is_deterministic() {
+        let seed = [0u8; 32];
+        let mut gen0 = GenKeys::new(seed);
+        let mut gen1 = GenKeys::new(seed);
+
+        for _ in 0..100 {
+            assert_eq!(gen0.gen_seed().to_vec(), gen1.gen_seed().to_vec());
+        }
+    }
+
+    fn gen_n_pubkeys(seed: [u8; 32], n: u64) -> HashSet<Pubkey> {
+        GenKeys::new(seed)
+            .gen_n_keypairs(n)
+            .into_iter()
+            .map(|x| x.pubkey())
+            .collect()
+    }
+
+    #[test]
+    fn test_gen_n_pubkeys_deterministic() {
+        let seed = [0u8; 32];
+        assert_eq!(gen_n_pubkeys(seed, 50), gen_n_pubkeys(seed, 50));
+    }
+}

core/src/gossip_service.rs

@@ -0,0 +1,184 @@
+//! The `gossip_service` module implements the network control plane.
+
+use crate::bank_forks::BankForks;
+use crate::blocktree::Blocktree;
+use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
+use crate::service::Service;
+use crate::streamer;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::thread::{self, JoinHandle};
+use std::time::Duration;
+
+pub struct GossipService {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl GossipService {
+    pub fn new(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        blocktree: Option<Arc<Blocktree>>,
+        bank_forks: Option<Arc<RwLock<BankForks>>>,
+        gossip_socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let (request_sender, request_receiver) = channel();
+        let gossip_socket = Arc::new(gossip_socket);
+        trace!(
+            "GossipService: id: {}, listening on: {:?}",
+            &cluster_info.read().unwrap().my_data().id,
+            gossip_socket.local_addr().unwrap()
+        );
+        let t_receiver =
+            streamer::blob_receiver(gossip_socket.clone(), exit.clone(), request_sender);
+        let (response_sender, response_receiver) = channel();
+        let t_responder = streamer::responder("gossip", gossip_socket, response_receiver);
+        let t_listen = ClusterInfo::listen(
+            cluster_info.clone(),
+            blocktree,
+            request_receiver,
+            response_sender.clone(),
+            exit.clone(),
+        );
+        let t_gossip = ClusterInfo::gossip(
+            cluster_info.clone(),
+            bank_forks,
+            response_sender,
+            exit.clone(),
+        );
+        let thread_hdls = vec![t_receiver, t_responder, t_listen, t_gossip];
+        Self { exit, thread_hdls }
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit.store(true, Ordering::Relaxed);
+        self.join()
+    }
+}
+
+pub fn make_listening_node(
+    leader: &NodeInfo,
+) -> (GossipService, Arc<RwLock<ClusterInfo>>, Node, Pubkey) {
+    let keypair = Keypair::new();
+    let exit = Arc::new(AtomicBool::new(false));
+    let new_node = Node::new_localhost_with_pubkey(keypair.pubkey());
+    let new_node_info = new_node.info.clone();
+    let id = new_node.info.id;
+    let mut new_node_cluster_info = ClusterInfo::new_with_keypair(new_node_info, Arc::new(keypair));
+    new_node_cluster_info.insert_info(leader.clone());
+    new_node_cluster_info.set_leader(leader.id);
+    let new_node_cluster_info_ref = Arc::new(RwLock::new(new_node_cluster_info));
+    let gossip_service = GossipService::new(
+        &new_node_cluster_info_ref,
+        None,
+        None,
+        new_node
+            .sockets
+            .gossip
+            .try_clone()
+            .expect("Failed to clone gossip"),
+        exit.clone(),
+    );
+
+    (gossip_service, new_node_cluster_info_ref, new_node, id)
+}
+
+pub fn discover(entry_point_info: &NodeInfo, num_nodes: usize) -> Vec<NodeInfo> {
+    converge(entry_point_info, num_nodes)
+}
+
+//TODO: deprecate this in favor of discover
+pub fn converge(node: &NodeInfo, num_nodes: usize) -> Vec<NodeInfo> {
+    info!("Wait for convergence with {} nodes", num_nodes);
+    // Let's spy on the network
+    let (gossip_service, spy_ref, id) = make_spy_node(node);
+    trace!(
+        "converge spy_node {} looking for at least {} nodes",
+        id,
+        num_nodes
+    );
+
+    // Wait for the cluster to converge
+    for _ in 0..15 {
+        let rpc_peers = spy_ref.read().unwrap().rpc_peers();
+        if rpc_peers.len() >= num_nodes {
+            debug!(
+                "converge found {}/{} nodes: {:?}",
+                rpc_peers.len(),
+                num_nodes,
+                rpc_peers
+            );
+            gossip_service.close().unwrap();
+            return rpc_peers;
+        }
+        debug!(
+            "spy_node: {} converge found {}/{} nodes, need {} more",
+            id,
+            rpc_peers.len(),
+            num_nodes,
+            num_nodes - rpc_peers.len()
+        );
+        sleep(Duration::new(1, 0));
+    }
+    panic!("Failed to converge");
+}
+
+pub fn make_spy_node(leader: &NodeInfo) -> (GossipService, Arc<RwLock<ClusterInfo>>, Pubkey) {
+    let keypair = Keypair::new();
+    let exit = Arc::new(AtomicBool::new(false));
+    let mut spy = Node::new_localhost_with_pubkey(keypair.pubkey());
+    let id = spy.info.id;
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    spy.info.tvu = daddr;
+    spy.info.rpc = daddr;
+    let mut spy_cluster_info = ClusterInfo::new_with_keypair(spy.info, Arc::new(keypair));
+    spy_cluster_info.insert_info(leader.clone());
+    spy_cluster_info.set_leader(leader.id);
+    let spy_cluster_info_ref = Arc::new(RwLock::new(spy_cluster_info));
+    let gossip_service = GossipService::new(
+        &spy_cluster_info_ref,
+        None,
+        None,
+        spy.sockets.gossip,
+        exit.clone(),
+    );
+
+    (gossip_service, spy_cluster_info_ref, id)
+}
+
+impl Service for GossipService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster_info::{ClusterInfo, Node};
+    use std::sync::atomic::AtomicBool;
+    use std::sync::{Arc, RwLock};
+
+    #[test]
+    #[ignore]
+    // test that stage will exit when flag is set
+    fn test_exit() {
+        let exit = Arc::new(AtomicBool::new(false));
+        let tn = Node::new_localhost();
+        let cluster_info = ClusterInfo::new(tn.info.clone());
+        let c = Arc::new(RwLock::new(cluster_info));
+        let d = GossipService::new(&c, None, None, tn.sockets.gossip, exit.clone());
+        d.close().expect("thread join");
+    }
+}

core/src/leader_confirmation_service.rs

@@ -0,0 +1,201 @@
+//! The `leader_confirmation_service` module implements the tools necessary
+//! to generate a thread which regularly calculates the last confirmation times
+//! observed by the leader
+
+use crate::service::Service;
+use solana_metrics::{influxdb, submit};
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::sleep;
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ConfirmationError {
+    NoValidSupermajority,
+}
+
+pub const COMPUTE_CONFIRMATION_MS: u64 = 100;
+
+pub struct LeaderConfirmationService {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl LeaderConfirmationService {
+    fn get_last_supermajority_timestamp(
+        bank: &Arc<Bank>,
+        leader_id: Pubkey,
+        last_valid_validator_timestamp: u64,
+    ) -> result::Result<u64, ConfirmationError> {
+        let mut total_stake = 0;
+
+        // Hold an accounts_db read lock as briefly as possible, just long enough to collect all
+        // the vote states
+        let vote_states = bank.vote_states(|_, vote_state| leader_id != vote_state.delegate_id);
+
+        let slots_and_stakes: Vec<(u64, u64)> = vote_states
+            .iter()
+            .filter_map(|(_, vote_state)| {
+                let validator_stake = bank.get_balance(&vote_state.delegate_id);
+                total_stake += validator_stake;
+                vote_state
+                    .votes
+                    .back()
+                    .map(|vote| (vote.slot_height, validator_stake))
+            })
+            .collect();
+
+        let super_majority_stake = (2 * total_stake) / 3;
+
+        if let Some(last_valid_validator_timestamp) =
+            bank.get_confirmation_timestamp(slots_and_stakes, super_majority_stake)
+        {
+            return Ok(last_valid_validator_timestamp);
+        }
+
+        if last_valid_validator_timestamp != 0 {
+            let now = timing::timestamp();
+            submit(
+                influxdb::Point::new(&"leader-confirmation")
+                    .add_field(
+                        "duration_ms",
+                        influxdb::Value::Integer((now - last_valid_validator_timestamp) as i64),
+                    )
+                    .to_owned(),
+            );
+        }
+
+        Err(ConfirmationError::NoValidSupermajority)
+    }
+
+    pub fn compute_confirmation(
+        bank: &Arc<Bank>,
+        leader_id: Pubkey,
+        last_valid_validator_timestamp: &mut u64,
+    ) {
+        if let Ok(super_majority_timestamp) =
+            Self::get_last_supermajority_timestamp(bank, leader_id, *last_valid_validator_timestamp)
+        {
+            let now = timing::timestamp();
+            let confirmation_ms = now - super_majority_timestamp;
+
+            *last_valid_validator_timestamp = super_majority_timestamp;
+
+            submit(
+                influxdb::Point::new(&"leader-confirmation")
+                    .add_field(
+                        "duration_ms",
+                        influxdb::Value::Integer(confirmation_ms as i64),
+                    )
+                    .to_owned(),
+            );
+        }
+    }
+
+    /// Create a new LeaderConfirmationService for computing confirmation.
+    pub fn new(bank: &Arc<Bank>, leader_id: Pubkey, exit: Arc<AtomicBool>) -> Self {
+        let bank = bank.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-leader-confirmation-service".to_string())
+            .spawn(move || {
+                let mut last_valid_validator_timestamp = 0;
+                loop {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+                    Self::compute_confirmation(
+                        &bank,
+                        leader_id,
+                        &mut last_valid_validator_timestamp,
+                    );
+                    sleep(Duration::from_millis(COMPUTE_CONFIRMATION_MS));
+                }
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+}
+
+impl Service for LeaderConfirmationService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::voting_keypair::tests::{new_vote_account, push_vote};
+    use crate::voting_keypair::VotingKeypair;
+    use bincode::serialize;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::vote_transaction::VoteTransaction;
+    use std::sync::Arc;
+
+    #[test]
+    fn test_compute_confirmation() {
+        solana_logger::setup();
+
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1234);
+        let bank = Arc::new(Bank::new(&genesis_block));
+
+        // Move the bank up 10 slots
+        let mut tick_hash = genesis_block.hash();
+        while bank.slot_height() < 10 {
+            tick_hash = hash(&serialize(&tick_hash).unwrap());
+            bank.register_tick(&tick_hash);
+        }
+        let last_id = bank.last_id();
+
+        // Create a total of 10 vote accounts, each will have a balance of 1 (after giving 1 to
+        // their vote account), for a total staking pool of 10 tokens.
+        let vote_accounts: Vec<_> = (0..10)
+            .map(|i| {
+                // Create new validator to vote
+                let validator_keypair = Arc::new(Keypair::new());
+                let voting_keypair = VotingKeypair::new_local(&validator_keypair);
+                let voting_pubkey = voting_keypair.pubkey();
+
+                // Give the validator some tokens
+                bank.transfer(2, &mint_keypair, validator_keypair.pubkey(), last_id)
+                    .unwrap();
+                new_vote_account(&validator_keypair, &voting_pubkey, &bank, 1);
+
+                if i < 6 {
+                    push_vote(&voting_keypair, &bank, (i + 1) as u64);
+                }
+                (voting_keypair, validator_keypair)
+            })
+            .collect();
+
+        // There isn't 2/3 consensus, so the bank's confirmation value should be the default
+        let mut last_confirmation_time = 0;
+        LeaderConfirmationService::compute_confirmation(
+            &bank,
+            genesis_block.bootstrap_leader_id,
+            &mut last_confirmation_time,
+        );
+        assert_eq!(last_confirmation_time, 0);
+
+        // Get another validator to vote, so we now have 2/3 consensus
+        let voting_keypair = &vote_accounts[7].0;
+        let vote_tx = VoteTransaction::new_vote(voting_keypair, 7, last_id, 0);
+        bank.process_transaction(&vote_tx).unwrap();
+
+        LeaderConfirmationService::compute_confirmation(
+            &bank,
+            genesis_block.bootstrap_leader_id,
+            &mut last_confirmation_time,
+        );
+        assert!(last_confirmation_time > 0);
+    }
+}

core/src/leader_schedule.rs

@@ -0,0 +1,65 @@
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::SeedableRng;
+use rand_chacha::ChaChaRng;
+use solana_sdk::pubkey::Pubkey;
+use std::ops::Index;
+
+/// Stake-weighted leader schedule for one epoch.
+#[derive(Debug, PartialEq)]
+pub struct LeaderSchedule {
+    slot_leaders: Vec<Pubkey>,
+}
+
+impl LeaderSchedule {
+    // Note: passing in zero stakers will cause a panic.
+    pub fn new(ids_and_stakes: &[(Pubkey, u64)], seed: [u8; 32], len: u64) -> Self {
+        let (ids, stakes): (Vec<_>, Vec<_>) = ids_and_stakes.iter().cloned().unzip();
+        let rng = &mut ChaChaRng::from_seed(seed);
+        let weighted_index = WeightedIndex::new(stakes).unwrap();
+        let slot_leaders = (0..len).map(|_| ids[weighted_index.sample(rng)]).collect();
+        Self { slot_leaders }
+    }
+}
+
+impl Index<usize> for LeaderSchedule {
+    type Output = Pubkey;
+    fn index(&self, index: usize) -> &Pubkey {
+        &self.slot_leaders[index % self.slot_leaders.len()]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_leader_schedule_index() {
+        let pubkey0 = Keypair::new().pubkey();
+        let pubkey1 = Keypair::new().pubkey();
+        let leader_schedule = LeaderSchedule {
+            slot_leaders: vec![pubkey0, pubkey1],
+        };
+        assert_eq!(leader_schedule[0], pubkey0);
+        assert_eq!(leader_schedule[1], pubkey1);
+        assert_eq!(leader_schedule[2], pubkey0);
+    }
+
+    #[test]
+    fn test_leader_schedule_basic() {
+        let num_keys = 10;
+        let stakes: Vec<_> = (0..num_keys)
+            .map(|i| (Keypair::new().pubkey(), i))
+            .collect();
+
+        let seed = Keypair::new().pubkey();
+        let mut seed_bytes = [0u8; 32];
+        seed_bytes.copy_from_slice(seed.as_ref());
+        let len = num_keys * 10;
+        let leader_schedule = LeaderSchedule::new(&stakes, seed_bytes, len);
+        let leader_schedule2 = LeaderSchedule::new(&stakes, seed_bytes, len);
+        assert_eq!(leader_schedule.slot_leaders.len() as u64, len);
+        // Check that the same schedule is reproducibly generated
+        assert_eq!(leader_schedule, leader_schedule2);
+    }
+}

core/src/leader_schedule_utils.rs

@@ -0,0 +1,169 @@
+use crate::leader_schedule::LeaderSchedule;
+use crate::staking_utils;
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+
+/// Return the leader schedule for the given epoch.
+fn leader_schedule(epoch_height: u64, bank: &Bank) -> LeaderSchedule {
+    let stakes = staking_utils::node_stakes_at_epoch(bank, epoch_height);
+    let mut seed = [0u8; 32];
+    seed[0..8].copy_from_slice(&epoch_height.to_le_bytes());
+    let mut stakes: Vec<_> = stakes.into_iter().collect();
+    sort_stakes(&mut stakes);
+    LeaderSchedule::new(&stakes, seed, bank.slots_per_epoch())
+}
+
+fn sort_stakes(stakes: &mut Vec<(Pubkey, u64)>) {
+    // Sort first by stake. If stakes are the same, sort by pubkey to ensure a
+    // deterministic result.
+    // Note: Use unstable sort, because we dedup right after to remove the equal elements.
+    stakes.sort_unstable_by(|(l_id, l_stake), (r_id, r_stake)| {
+        if r_stake == l_stake {
+            r_id.cmp(&l_id)
+        } else {
+            r_stake.cmp(&l_stake)
+        }
+    });
+
+    // Now that it's sorted, we can do an O(n) dedup.
+    stakes.dedup();
+}
+
+/// Return the leader for the slot at the slot_index and epoch_height returned
+/// by the given function.
+fn slot_leader_by<F>(bank: &Bank, get_slot_index: F) -> Pubkey
+where
+    F: Fn(u64, u64, u64) -> (u64, u64),
+{
+    let (slot_index, epoch_height) = get_slot_index(
+        bank.slot_index(),
+        bank.epoch_height(),
+        bank.slots_per_epoch(),
+    );
+    let leader_schedule = leader_schedule(epoch_height, bank);
+    leader_schedule[slot_index as usize]
+}
+
+/// Return the leader for the current slot.
+pub fn slot_leader(bank: &Bank) -> Pubkey {
+    slot_leader_by(bank, |slot_index, epoch_height, _| {
+        (slot_index, epoch_height)
+    })
+}
+
+/// Return the leader for the given slot.
+pub fn slot_leader_at(slot: u64, bank: &Bank) -> Pubkey {
+    slot_leader_by(bank, |_, _, _| {
+        (slot % bank.slots_per_epoch(), slot / bank.slots_per_epoch())
+    })
+}
+
+/// Return the epoch height and slot index of the slot before the current slot.
+fn prev_slot_leader_index(slot_index: u64, epoch_height: u64, slots_per_epoch: u64) -> (u64, u64) {
+    if epoch_height == 0 && slot_index == 0 {
+        return (0, 0);
+    }
+
+    if slot_index == 0 {
+        (slots_per_epoch - 1, epoch_height - 1)
+    } else {
+        (slot_index - 1, epoch_height)
+    }
+}
+
+/// Return the slot_index and epoch height of the slot following the current slot.
+fn next_slot_leader_index(slot_index: u64, epoch_height: u64, slots_per_epoch: u64) -> (u64, u64) {
+    if slot_index + 1 == slots_per_epoch {
+        (0, epoch_height + 1)
+    } else {
+        (slot_index + 1, epoch_height)
+    }
+}
+
+/// Return the leader for the slot before the current slot.
+pub fn prev_slot_leader(bank: &Bank) -> Pubkey {
+    slot_leader_by(bank, prev_slot_leader_index)
+}
+
+/// Return the leader for the slot following the current slot.
+pub fn next_slot_leader(bank: &Bank) -> Pubkey {
+    slot_leader_by(bank, next_slot_leader_index)
+}
+
+// Returns the number of ticks remaining from the specified tick_height to the end of the
+// slot implied by the tick_height
+pub fn num_ticks_left_in_slot(bank: &Bank, tick_height: u64) -> u64 {
+    bank.ticks_per_slot() - tick_height % bank.ticks_per_slot() - 1
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::staking_utils;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_leader_schedule_via_bank() {
+        let pubkey = Keypair::new().pubkey();
+        let (genesis_block, _mint_keypair) = GenesisBlock::new_with_leader(2, pubkey, 2);
+        let bank = Bank::new(&genesis_block);
+
+        let ids_and_stakes: Vec<_> = staking_utils::node_stakes(&bank).into_iter().collect();
+        let seed = [0u8; 32];
+        let leader_schedule =
+            LeaderSchedule::new(&ids_and_stakes, seed, genesis_block.slots_per_epoch);
+
+        assert_eq!(leader_schedule[0], pubkey);
+        assert_eq!(leader_schedule[1], pubkey);
+        assert_eq!(leader_schedule[2], pubkey);
+    }
+
+    #[test]
+    fn test_leader_scheduler1_basic() {
+        let pubkey = Keypair::new().pubkey();
+        let genesis_block = GenesisBlock::new_with_leader(2, pubkey, 2).0;
+        let bank = Bank::new(&genesis_block);
+        assert_eq!(slot_leader(&bank), pubkey);
+    }
+
+    #[test]
+    fn test_leader_scheduler1_prev_slot_leader_index() {
+        assert_eq!(prev_slot_leader_index(0, 0, 2), (0, 0));
+        assert_eq!(prev_slot_leader_index(1, 0, 2), (0, 0));
+        assert_eq!(prev_slot_leader_index(0, 1, 2), (1, 0));
+    }
+
+    #[test]
+    fn test_leader_scheduler1_next_slot_leader_index() {
+        assert_eq!(next_slot_leader_index(0, 0, 2), (1, 0));
+        assert_eq!(next_slot_leader_index(1, 0, 2), (0, 1));
+    }
+
+    #[test]
+    fn test_sort_stakes_basic() {
+        let pubkey0 = Keypair::new().pubkey();
+        let pubkey1 = Keypair::new().pubkey();
+        let mut stakes = vec![(pubkey0, 1), (pubkey1, 2)];
+        sort_stakes(&mut stakes);
+        assert_eq!(stakes, vec![(pubkey1, 2), (pubkey0, 1)]);
+    }
+
+    #[test]
+    fn test_sort_stakes_with_dup() {
+        let pubkey0 = Keypair::new().pubkey();
+        let pubkey1 = Keypair::new().pubkey();
+        let mut stakes = vec![(pubkey0, 1), (pubkey1, 2), (pubkey0, 1)];
+        sort_stakes(&mut stakes);
+        assert_eq!(stakes, vec![(pubkey1, 2), (pubkey0, 1)]);
+    }
+
+    #[test]
+    fn test_sort_stakes_with_equal_stakes() {
+        let pubkey0 = Pubkey::default();
+        let pubkey1 = Keypair::new().pubkey();
+        let mut stakes = vec![(pubkey0, 1), (pubkey1, 1)];
+        sort_stakes(&mut stakes);
+        assert_eq!(stakes, vec![(pubkey1, 1), (pubkey0, 1)]);
+    }
+}

core/src/lib.rs

@@ -0,0 +1,100 @@
+//! The `solana` library implements the Solana high-performance blockchain architecture.
+//! It includes a full Rust implementation of the architecture (see
+//! [Fullnode](server/struct.Fullnode.html)) as well as hooks to GPU implementations of its most
+//! paralellizable components (i.e. [SigVerify](sigverify/index.html)).  It also includes
+//! command-line tools to spin up fullnodes and a Rust library
+//! (see [ThinClient](thin_client/struct.ThinClient.html)) to interact with them.
+//!
+
+#![cfg_attr(feature = "unstable", feature(test))]
+pub mod bank_forks;
+pub mod banking_stage;
+pub mod blob_fetch_stage;
+pub mod broadcast_stage;
+#[cfg(feature = "chacha")]
+pub mod chacha;
+#[cfg(all(feature = "chacha", feature = "cuda"))]
+pub mod chacha_cuda;
+pub mod client;
+pub mod cluster_info_vote_listener;
+pub mod crds;
+pub mod crds_gossip;
+pub mod crds_gossip_error;
+pub mod crds_gossip_pull;
+pub mod crds_gossip_push;
+pub mod crds_value;
+#[macro_use]
+pub mod contact_info;
+#[macro_use]
+pub mod blocktree;
+pub mod blockstream;
+pub mod blockstream_service;
+pub mod blocktree_processor;
+pub mod cluster_info;
+pub mod cluster_tests;
+pub mod db_window;
+pub mod entry;
+#[cfg(feature = "erasure")]
+pub mod erasure;
+pub mod fetch_stage;
+pub mod fullnode;
+pub mod gen_keys;
+pub mod gossip_service;
+pub mod leader_confirmation_service;
+pub mod leader_schedule;
+pub mod leader_schedule_utils;
+pub mod local_cluster;
+pub mod local_vote_signer_service;
+pub mod packet;
+pub mod poh;
+pub mod poh_recorder;
+pub mod poh_service;
+pub mod recvmmsg;
+pub mod repair_service;
+pub mod replay_stage;
+pub mod replicator;
+pub mod result;
+pub mod retransmit_stage;
+pub mod rpc;
+pub mod rpc_mock;
+pub mod rpc_pubsub;
+pub mod rpc_pubsub_service;
+pub mod rpc_request;
+pub mod rpc_service;
+pub mod rpc_status;
+pub mod rpc_subscriptions;
+pub mod service;
+pub mod sigverify;
+pub mod sigverify_stage;
+pub mod staking_utils;
+pub mod storage_stage;
+pub mod streamer;
+pub mod test_tx;
+pub mod thin_client;
+pub mod tpu;
+pub mod tpu_forwarder;
+pub mod tvu;
+pub mod voting_keypair;
+#[cfg(test)]
+pub mod window;
+pub mod window_service;
+
+#[cfg(test)]
+#[cfg(any(feature = "chacha", feature = "cuda"))]
+#[macro_use]
+extern crate hex_literal;
+
+#[macro_use]
+extern crate log;
+
+#[macro_use]
+extern crate serde_derive;
+#[macro_use]
+extern crate serde_json;
+
+#[macro_use]
+extern crate solana_metrics;
+
+#[cfg(test)]
+#[macro_use]
+extern crate matches;

core/src/local_cluster.rs

@@ -0,0 +1,195 @@
+use crate::blocktree::{create_new_tmp_ledger, tmp_copy_blocktree};
+use crate::client::mk_client;
+use crate::cluster_info::{Node, NodeInfo};
+use crate::fullnode::{Fullnode, FullnodeConfig};
+use crate::gossip_service::discover;
+use crate::thin_client::retry_get_balance;
+use crate::thin_client::ThinClient;
+use crate::voting_keypair::VotingKeypair;
+use solana_sdk::genesis_block::GenesisBlock;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::system_transaction::SystemTransaction;
+use solana_sdk::vote_program::VoteState;
+use solana_sdk::vote_transaction::VoteTransaction;
+use std::fs::remove_dir_all;
+use std::io::{Error, ErrorKind, Result};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::JoinHandle;
+
+pub struct LocalCluster {
+    /// Keypair with funding to particpiate in the network
+    pub funding_keypair: Keypair,
+    /// Entry point from which the rest of the network can be discovered
+    pub entry_point_info: NodeInfo,
+    fullnode_hdls: Vec<(JoinHandle<()>, Arc<AtomicBool>)>,
+    ledger_paths: Vec<String>,
+}
+
+impl LocalCluster {
+    pub fn new(num_nodes: usize, cluster_lamports: u64, lamports_per_node: u64) -> Self {
+        let leader_keypair = Arc::new(Keypair::new());
+        let leader_pubkey = leader_keypair.pubkey();
+        let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
+        let (genesis_block, mint_keypair) =
+            GenesisBlock::new_with_leader(cluster_lamports, leader_pubkey, lamports_per_node);
+        let (genesis_ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+        let leader_ledger_path = tmp_copy_blocktree!(&genesis_ledger_path);
+        let mut ledger_paths = vec![];
+        ledger_paths.push(genesis_ledger_path.clone());
+        ledger_paths.push(leader_ledger_path.clone());
+        let voting_keypair = VotingKeypair::new_local(&leader_keypair);
+        let fullnode_config = FullnodeConfig::default();
+        let leader_node_info = leader_node.info.clone();
+        let leader_server = Fullnode::new(
+            leader_node,
+            &leader_keypair,
+            &leader_ledger_path,
+            voting_keypair,
+            None,
+            &fullnode_config,
+        );
+        let (thread, exit, _) = leader_server.start(None);
+        let mut fullnode_hdls = vec![(thread, exit)];
+        let mut client = mk_client(&leader_node_info);
+        for _ in 0..(num_nodes - 1) {
+            let validator_keypair = Arc::new(Keypair::new());
+            let voting_keypair = VotingKeypair::new_local(&validator_keypair);
+            let validator_pubkey = validator_keypair.pubkey();
+            let validator_node = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
+            let ledger_path = tmp_copy_blocktree!(&genesis_ledger_path);
+            ledger_paths.push(ledger_path.clone());
+
+            // Send each validator some tokens to vote
+            let validator_balance = Self::transfer(
+                &mut client,
+                &mint_keypair,
+                &validator_pubkey,
+                lamports_per_node,
+            );
+            info!(
+                "validator {} balance {}",
+                validator_pubkey, validator_balance
+            );
+
+            Self::create_and_fund_vote_account(
+                &mut client,
+                voting_keypair.pubkey(),
+                &validator_keypair,
+                1,
+            )
+            .unwrap();
+            let validator_server = Fullnode::new(
+                validator_node,
+                &validator_keypair,
+                &ledger_path,
+                voting_keypair,
+                Some(&leader_node_info),
+                &fullnode_config,
+            );
+            let (thread, exit, _) = validator_server.start(None);
+            fullnode_hdls.push((thread, exit));
+        }
+        discover(&leader_node_info, num_nodes);
+        Self {
+            funding_keypair: mint_keypair,
+            entry_point_info: leader_node_info,
+            fullnode_hdls,
+            ledger_paths,
+        }
+    }
+
+    pub fn exit(&self) {
+        for node in &self.fullnode_hdls {
+            node.1.store(true, Ordering::Relaxed);
+        }
+    }
+    pub fn close(&mut self) {
+        self.exit();
+        while let Some(node) = self.fullnode_hdls.pop() {
+            node.0.join().expect("join");
+        }
+        for path in &self.ledger_paths {
+            remove_dir_all(path).unwrap();
+        }
+    }
+
+    fn transfer(
+        client: &mut ThinClient,
+        source_keypair: &Keypair,
+        dest_pubkey: &Pubkey,
+        lamports: u64,
+    ) -> u64 {
+        trace!("getting leader last_id");
+        let last_id = client.get_last_id();
+        let mut tx =
+            SystemTransaction::new_account(&source_keypair, *dest_pubkey, lamports, last_id, 0);
+        info!(
+            "executing transfer of {} from {} to {}",
+            lamports,
+            source_keypair.pubkey(),
+            *dest_pubkey
+        );
+        client
+            .retry_transfer(&source_keypair, &mut tx, 5)
+            .expect("client transfer");
+        retry_get_balance(client, dest_pubkey, Some(lamports)).expect("get balance")
+    }
+
+    fn create_and_fund_vote_account(
+        client: &mut ThinClient,
+        vote_account: Pubkey,
+        from_account: &Arc<Keypair>,
+        amount: u64,
+    ) -> Result<()> {
+        // Create the vote account if necessary
+        if client.poll_get_balance(&vote_account).unwrap_or(0) == 0 {
+            let mut transaction = VoteTransaction::fund_staking_account(
+                from_account,
+                vote_account,
+                client.get_last_id(),
+                amount,
+                1,
+            );
+
+            client
+                .retry_transfer(&from_account, &mut transaction, 5)
+                .expect("client transfer");
+            retry_get_balance(client, &vote_account, Some(amount)).expect("get balance");
+        }
+
+        info!("Checking for vote account registration");
+        let vote_account_user_data = client.get_account_userdata(&vote_account);
+        if let Ok(Some(vote_account_user_data)) = vote_account_user_data {
+            if let Ok(vote_state) = VoteState::deserialize(&vote_account_user_data) {
+                if vote_state.delegate_id == from_account.pubkey() {
+                    return Ok(());
+                }
+            }
+        }
+
+        Err(Error::new(
+            ErrorKind::Other,
+            "expected successful vote account registration",
+        ))
+    }
+}
+
+impl Drop for LocalCluster {
+    fn drop(&mut self) {
+        self.close()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_local_cluster_start_and_exit() {
+        solana_logger::setup();
+        let network = LocalCluster::new(1, 100, 2);
+        drop(network)
+    }
+}

core/src/local_vote_signer_service.rs

@@ -0,0 +1,44 @@
+//! The `local_vote_signer_service` can be started locally to sign fullnode votes
+
+use crate::cluster_info::FULLNODE_PORT_RANGE;
+use crate::service::Service;
+use solana_vote_signer::rpc::VoteSignerRpcService;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::{self, Builder, JoinHandle};
+
+pub struct LocalVoteSignerService {
+    thread: JoinHandle<()>,
+    exit: Arc<AtomicBool>,
+}
+
+impl Service for LocalVoteSignerService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.exit.store(true, Ordering::Relaxed);
+        self.thread.join()
+    }
+}
+
+impl LocalVoteSignerService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> (Self, SocketAddr) {
+        let addr = match solana_netutil::find_available_port_in_range(FULLNODE_PORT_RANGE) {
+            Ok(port) => SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port),
+            Err(_e) => panic!("Failed to find an available port for local vote signer service"),
+        };
+        let exit = Arc::new(AtomicBool::new(false));
+        let thread_exit = exit.clone();
+        let thread = Builder::new()
+            .name("solana-vote-signer".to_string())
+            .spawn(move || {
+                let service = VoteSignerRpcService::new(addr, thread_exit);
+                service.join().unwrap();
+            })
+            .unwrap();
+
+        (Self { thread, exit }, addr)
+    }
+}

core/src/packet.rs

@@ -0,0 +1,600 @@
+//! The `packet` module defines data structures and methods to pull data from the network.
+use crate::recvmmsg::{recv_mmsg, NUM_RCVMMSGS};
+use crate::result::{Error, Result};
+use bincode::{serialize, serialize_into};
+use byteorder::{ByteOrder, LittleEndian};
+use serde::Serialize;
+use solana_metrics::counter::Counter;
+pub use solana_sdk::packet::PACKET_DATA_SIZE;
+use solana_sdk::pubkey::Pubkey;
+use std::cmp;
+use std::fmt;
+use std::io;
+use std::mem::size_of;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket};
+use std::sync::{Arc, RwLock};
+
+pub type SharedPackets = Arc<RwLock<Packets>>;
+pub type SharedBlob = Arc<RwLock<Blob>>;
+pub type SharedBlobs = Vec<SharedBlob>;
+
+pub const NUM_PACKETS: usize = 1024 * 8;
+pub const BLOB_SIZE: usize = (64 * 1024 - 128); // wikipedia says there should be 20b for ipv4 headers
+pub const BLOB_DATA_SIZE: usize = BLOB_SIZE - (BLOB_HEADER_SIZE * 2);
+pub const NUM_BLOBS: usize = (NUM_PACKETS * PACKET_DATA_SIZE) / BLOB_SIZE;
+
+#[derive(Clone, Default, Debug, PartialEq)]
+#[repr(C)]
+pub struct Meta {
+    pub size: usize,
+    pub num_retransmits: u64,
+    pub addr: [u16; 8],
+    pub port: u16,
+    pub v6: bool,
+}
+
+#[derive(Clone)]
+#[repr(C)]
+pub struct Packet {
+    pub data: [u8; PACKET_DATA_SIZE],
+    pub meta: Meta,
+}
+
+impl fmt::Debug for Packet {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Packet {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+impl Default for Packet {
+    fn default() -> Packet {
+        Packet {
+            data: [0u8; PACKET_DATA_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+impl Meta {
+    pub fn addr(&self) -> SocketAddr {
+        if !self.v6 {
+            let addr = [
+                self.addr[0] as u8,
+                self.addr[1] as u8,
+                self.addr[2] as u8,
+                self.addr[3] as u8,
+            ];
+            let ipv4: Ipv4Addr = From::<[u8; 4]>::from(addr);
+            SocketAddr::new(IpAddr::V4(ipv4), self.port)
+        } else {
+            let ipv6: Ipv6Addr = From::<[u16; 8]>::from(self.addr);
+            SocketAddr::new(IpAddr::V6(ipv6), self.port)
+        }
+    }
+
+    pub fn set_addr(&mut self, a: &SocketAddr) {
+        match *a {
+            SocketAddr::V4(v4) => {
+                let ip = v4.ip().octets();
+                self.addr[0] = u16::from(ip[0]);
+                self.addr[1] = u16::from(ip[1]);
+                self.addr[2] = u16::from(ip[2]);
+                self.addr[3] = u16::from(ip[3]);
+                self.addr[4] = 0;
+                self.addr[5] = 0;
+                self.addr[6] = 0;
+                self.addr[7] = 0;
+                self.v6 = false;
+            }
+            SocketAddr::V6(v6) => {
+                self.addr = v6.ip().segments();
+                self.v6 = true;
+            }
+        }
+        self.port = a.port();
+    }
+}
+
+#[derive(Debug)]
+pub struct Packets {
+    pub packets: Vec<Packet>,
+}
+
+//auto derive doesn't support large arrays
+impl Default for Packets {
+    fn default() -> Packets {
+        Packets {
+            packets: vec![Packet::default(); NUM_PACKETS],
+        }
+    }
+}
+
+impl Packets {
+    pub fn set_addr(&mut self, addr: &SocketAddr) {
+        for m in self.packets.iter_mut() {
+            m.meta.set_addr(&addr);
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct Blob {
+    pub data: [u8; BLOB_SIZE],
+    pub meta: Meta,
+}
+
+impl PartialEq for Blob {
+    fn eq(&self, other: &Blob) -> bool {
+        self.data.iter().zip(other.data.iter()).all(|(a, b)| a == b) && self.meta == other.meta
+    }
+}
+
+impl fmt::Debug for Blob {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Blob {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+//auto derive doesn't support large arrays
+impl Default for Blob {
+    fn default() -> Blob {
+        Blob {
+            data: [0u8; BLOB_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum BlobError {
+    /// the Blob's meta and data are not self-consistent
+    BadState,
+    /// Blob verification failed
+    VerificationFailed,
+}
+
+impl Packets {
+    fn run_read_from(&mut self, socket: &UdpSocket) -> Result<usize> {
+        self.packets.resize(NUM_PACKETS, Packet::default());
+        let mut i = 0;
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until it's readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        trace!("receiving on {}", socket.local_addr().unwrap());
+        loop {
+            match recv_mmsg(socket, &mut self.packets[i..]) {
+                Err(_) if i > 0 => {
+                    inc_new_counter_info!("packets-recv_count", i);
+                    debug!("got {:?} messages on {}", i, socket.local_addr().unwrap());
+                    return Ok(i);
+                }
+                Err(e) => {
+                    trace!("recv_from err {:?}", e);
+                    return Err(Error::IO(e));
+                }
+                Ok(npkts) => {
+                    if i == 0 {
+                        socket.set_nonblocking(true)?;
+                    }
+                    trace!("got {} packets", npkts);
+                    i += npkts;
+                    if npkts != NUM_RCVMMSGS || i >= 1024 {
+                        inc_new_counter_info!("packets-recv_count", i);
+                        return Ok(i);
+                    }
+                }
+            }
+        }
+    }
+    pub fn recv_from(&mut self, socket: &UdpSocket) -> Result<()> {
+        let sz = self.run_read_from(socket)?;
+        self.packets.resize(sz, Packet::default());
+        debug!("recv_from: {}", sz);
+        Ok(())
+    }
+    pub fn send_to(&self, socket: &UdpSocket) -> Result<()> {
+        for p in &self.packets {
+            let a = p.meta.addr();
+            socket.send_to(&p.data[..p.meta.size], &a)?;
+        }
+        Ok(())
+    }
+}
+
+pub fn to_packets_chunked<T: Serialize>(xs: &[T], chunks: usize) -> Vec<SharedPackets> {
+    let mut out = vec![];
+    for x in xs.chunks(chunks) {
+        let p = SharedPackets::default();
+        p.write()
+            .unwrap()
+            .packets
+            .resize(x.len(), Packet::default());
+        for (i, o) in x.iter().zip(p.write().unwrap().packets.iter_mut()) {
+            let mut wr = io::Cursor::new(&mut o.data[..]);
+            serialize_into(&mut wr, &i).expect("serialize request");
+            let len = wr.position() as usize;
+            o.meta.size = len;
+        }
+        out.push(p);
+    }
+    out
+}
+
+pub fn to_packets<T: Serialize>(xs: &[T]) -> Vec<SharedPackets> {
+    to_packets_chunked(xs, NUM_PACKETS)
+}
+
+pub fn to_blob<T: Serialize>(resp: T, rsp_addr: SocketAddr) -> Result<Blob> {
+    let mut b = Blob::default();
+    let v = serialize(&resp)?;
+    let len = v.len();
+    assert!(len <= BLOB_SIZE);
+    b.data[..len].copy_from_slice(&v);
+    b.meta.size = len;
+    b.meta.set_addr(&rsp_addr);
+    Ok(b)
+}
+
+pub fn to_blobs<T: Serialize>(rsps: Vec<(T, SocketAddr)>) -> Result<Vec<Blob>> {
+    let mut blobs = Vec::new();
+    for (resp, rsp_addr) in rsps {
+        blobs.push(to_blob(resp, rsp_addr)?);
+    }
+    Ok(blobs)
+}
+
+pub fn to_shared_blob<T: Serialize>(resp: T, rsp_addr: SocketAddr) -> Result<SharedBlob> {
+    let blob = Arc::new(RwLock::new(to_blob(resp, rsp_addr)?));
+    Ok(blob)
+}
+
+pub fn to_shared_blobs<T: Serialize>(rsps: Vec<(T, SocketAddr)>) -> Result<SharedBlobs> {
+    let mut blobs = Vec::new();
+    for (resp, rsp_addr) in rsps {
+        blobs.push(to_shared_blob(resp, rsp_addr)?);
+    }
+    Ok(blobs)
+}
+
+macro_rules! range {
+    ($prev:expr, $type:ident) => {
+        $prev..$prev + size_of::<$type>()
+    };
+}
+
+const PARENT_RANGE: std::ops::Range<usize> = range!(0, u64);
+const SLOT_RANGE: std::ops::Range<usize> = range!(PARENT_RANGE.end, u64);
+const INDEX_RANGE: std::ops::Range<usize> = range!(SLOT_RANGE.end, u64);
+const ID_RANGE: std::ops::Range<usize> = range!(INDEX_RANGE.end, Pubkey);
+const FORWARD_RANGE: std::ops::Range<usize> = range!(ID_RANGE.end, bool);
+const FLAGS_RANGE: std::ops::Range<usize> = range!(FORWARD_RANGE.end, u32);
+const SIZE_RANGE: std::ops::Range<usize> = range!(FLAGS_RANGE.end, u64);
+
+macro_rules! align {
+    ($x:expr, $align:expr) => {
+        $x + ($align * 8 - 1) & !($align * 8 - 1)
+    };
+}
+
+pub const BLOB_HEADER_SIZE: usize = align!(SIZE_RANGE.end, 8);
+
+pub const BLOB_FLAG_IS_LAST_IN_SLOT: u32 = 0x2;
+
+pub const BLOB_FLAG_IS_CODING: u32 = 0x1;
+
+impl Blob {
+    pub fn new(data: &[u8]) -> Self {
+        let mut blob = Self::default();
+        let data_len = cmp::min(data.len(), blob.data.len());
+        let bytes = &data[..data_len];
+        blob.data[..data_len].copy_from_slice(bytes);
+        blob.meta.size = blob.data_size() as usize;
+        blob
+    }
+
+    pub fn parent(&self) -> u64 {
+        LittleEndian::read_u64(&self.data[PARENT_RANGE])
+    }
+    pub fn set_parent(&mut self, ix: u64) {
+        LittleEndian::write_u64(&mut self.data[PARENT_RANGE], ix);
+    }
+    pub fn slot(&self) -> u64 {
+        LittleEndian::read_u64(&self.data[SLOT_RANGE])
+    }
+    pub fn set_slot(&mut self, ix: u64) {
+        LittleEndian::write_u64(&mut self.data[SLOT_RANGE], ix);
+    }
+    pub fn index(&self) -> u64 {
+        LittleEndian::read_u64(&self.data[INDEX_RANGE])
+    }
+    pub fn set_index(&mut self, ix: u64) {
+        LittleEndian::write_u64(&mut self.data[INDEX_RANGE], ix);
+    }
+
+    /// sender id, we use this for identifying if its a blob from the leader that we should
+    /// retransmit.  eventually blobs should have a signature that we can use for spam filtering
+    pub fn id(&self) -> Pubkey {
+        Pubkey::new(&self.data[ID_RANGE])
+    }
+
+    pub fn set_id(&mut self, id: &Pubkey) {
+        self.data[ID_RANGE].copy_from_slice(id.as_ref())
+    }
+
+    /// Used to determine whether or not this blob should be forwarded in retransmit
+    /// A bool is used here instead of a flag because this item is not intended to be signed when
+    /// blob signatures are introduced
+    pub fn should_forward(&self) -> bool {
+        self.data[FORWARD_RANGE][0] & 0x1 == 1
+    }
+
+    pub fn forward(&mut self, forward: bool) {
+        self.data[FORWARD_RANGE][0] = u8::from(forward)
+    }
+
+    pub fn flags(&self) -> u32 {
+        LittleEndian::read_u32(&self.data[FLAGS_RANGE])
+    }
+    pub fn set_flags(&mut self, ix: u32) {
+        LittleEndian::write_u32(&mut self.data[FLAGS_RANGE], ix);
+    }
+
+    pub fn is_coding(&self) -> bool {
+        (self.flags() & BLOB_FLAG_IS_CODING) != 0
+    }
+
+    pub fn set_coding(&mut self) {
+        let flags = self.flags();
+        self.set_flags(flags | BLOB_FLAG_IS_CODING);
+    }
+
+    pub fn set_is_last_in_slot(&mut self) {
+        let flags = self.flags();
+        self.set_flags(flags | BLOB_FLAG_IS_LAST_IN_SLOT);
+    }
+
+    pub fn is_last_in_slot(&self) -> bool {
+        (self.flags() & BLOB_FLAG_IS_LAST_IN_SLOT) != 0
+    }
+
+    pub fn data_size(&self) -> u64 {
+        LittleEndian::read_u64(&self.data[SIZE_RANGE])
+    }
+
+    pub fn set_data_size(&mut self, ix: u64) {
+        LittleEndian::write_u64(&mut self.data[SIZE_RANGE], ix);
+    }
+
+    pub fn data(&self) -> &[u8] {
+        &self.data[BLOB_HEADER_SIZE..]
+    }
+    pub fn data_mut(&mut self) -> &mut [u8] {
+        &mut self.data[BLOB_HEADER_SIZE..]
+    }
+    pub fn size(&self) -> usize {
+        let size = self.data_size() as usize;
+
+        if size > BLOB_HEADER_SIZE && size == self.meta.size {
+            size - BLOB_HEADER_SIZE
+        } else {
+            0
+        }
+    }
+
+    pub fn set_size(&mut self, size: usize) {
+        let new_size = size + BLOB_HEADER_SIZE;
+        self.meta.size = new_size;
+        self.set_data_size(new_size as u64);
+    }
+
+    pub fn recv_blob(socket: &UdpSocket, r: &SharedBlob) -> io::Result<()> {
+        let mut p = r.write().unwrap();
+        trace!("receiving on {}", socket.local_addr().unwrap());
+
+        let (nrecv, from) = socket.recv_from(&mut p.data)?;
+        p.meta.size = nrecv;
+        p.meta.set_addr(&from);
+        trace!("got {} bytes from {}", nrecv, from);
+        Ok(())
+    }
+
+    pub fn recv_from(socket: &UdpSocket) -> Result<SharedBlobs> {
+        let mut v = Vec::new();
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until it's readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        for i in 0..NUM_BLOBS {
+            let r = SharedBlob::default();
+
+            match Blob::recv_blob(socket, &r) {
+                Err(_) if i > 0 => {
+                    trace!("got {:?} messages on {}", i, socket.local_addr().unwrap());
+                    break;
+                }
+                Err(e) => {
+                    if e.kind() != io::ErrorKind::WouldBlock {
+                        info!("recv_from err {:?}", e);
+                    }
+                    return Err(Error::IO(e));
+                }
+                Ok(()) => {
+                    if i == 0 {
+                        socket.set_nonblocking(true)?;
+                    }
+                }
+            }
+            v.push(r);
+        }
+        Ok(v)
+    }
+    pub fn send_to(socket: &UdpSocket, v: SharedBlobs) -> Result<()> {
+        for r in v {
+            {
+                let p = r.read().unwrap();
+                let a = p.meta.addr();
+                if let Err(e) = socket.send_to(&p.data[..p.meta.size], &a) {
+                    warn!(
+                        "error sending {} byte packet to {:?}: {:?}",
+                        p.meta.size, a, e
+                    );
+                    Err(e)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+pub fn index_blobs(blobs: &[SharedBlob], id: &Pubkey, blob_index: &mut u64, slot: u64) {
+    // enumerate all the blobs, those are the indices
+    for blob in blobs.iter() {
+        let mut blob = blob.write().unwrap();
+
+        blob.set_index(*blob_index);
+        blob.set_slot(slot);
+        blob.set_id(id);
+        blob.forward(true);
+        *blob_index += 1;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::packet::{
+        to_packets, Blob, Meta, Packet, Packets, SharedBlob, SharedPackets, NUM_PACKETS,
+        PACKET_DATA_SIZE,
+    };
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use std::io;
+    use std::io::Write;
+    use std::net::{Ipv4Addr, SocketAddr, UdpSocket};
+
+    #[test]
+    fn test_packets_set_addr() {
+        // test that the address is actually being updated
+        let send_addr = socketaddr!([127, 0, 0, 1], 123);
+        let packets = vec![Packet::default()];
+        let mut msgs = Packets { packets };
+        msgs.set_addr(&send_addr);
+        assert_eq!(SocketAddr::from(msgs.packets[0].meta.addr()), send_addr);
+    }
+
+    #[test]
+    pub fn packet_send_recv() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let p = SharedPackets::default();
+        p.write().unwrap().packets.resize(10, Packet::default());
+        for m in p.write().unwrap().packets.iter_mut() {
+            m.meta.set_addr(&addr);
+            m.meta.size = PACKET_DATA_SIZE;
+        }
+        p.read().unwrap().send_to(&sender).unwrap();
+        p.write().unwrap().recv_from(&reader).unwrap();
+        for m in p.write().unwrap().packets.iter_mut() {
+            assert_eq!(m.meta.size, PACKET_DATA_SIZE);
+            assert_eq!(m.meta.addr(), saddr);
+        }
+    }
+
+    #[test]
+    fn test_to_packets() {
+        let keypair = Keypair::new();
+        let hash = Hash::new(&[1; 32]);
+        let tx = SystemTransaction::new_account(&keypair, keypair.pubkey(), 1, hash, 0);
+        let rv = to_packets(&vec![tx.clone(); 1]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), 1);
+
+        let rv = to_packets(&vec![tx.clone(); NUM_PACKETS]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+
+        let rv = to_packets(&vec![tx.clone(); NUM_PACKETS + 1]);
+        assert_eq!(rv.len(), 2);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+        assert_eq!(rv[1].read().unwrap().packets.len(), 1);
+    }
+
+    #[test]
+    pub fn blob_send_recv() {
+        trace!("start");
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let p = SharedBlob::default();
+        p.write().unwrap().meta.set_addr(&addr);
+        p.write().unwrap().meta.size = 1024;
+        let v = vec![p];
+        Blob::send_to(&sender, v).unwrap();
+        trace!("send_to");
+        let rv = Blob::recv_from(&reader).unwrap();
+        trace!("recv_from");
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().meta.size, 1024);
+    }
+
+    #[cfg(all(feature = "ipv6", test))]
+    #[test]
+    pub fn blob_ipv6_send_recv() {
+        let reader = UdpSocket::bind("[::1]:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("[::1]:0").expect("bind");
+        let p = SharedBlob::default();
+        p.as_mut().unwrap().meta.set_addr(&addr);
+        p.as_mut().unwrap().meta.size = 1024;
+        let mut v = VecDeque::default();
+        v.push_back(p);
+        Blob::send_to(&r, &sender, &mut v).unwrap();
+        let mut rv = Blob::recv_from(&reader).unwrap();
+        let rp = rv.pop_front().unwrap();
+        assert_eq!(rp.as_mut().meta.size, 1024);
+    }
+
+    #[test]
+    pub fn debug_trait() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn blob_test() {
+        let mut b = Blob::default();
+        b.set_index(<u64>::max_value());
+        assert_eq!(b.index(), <u64>::max_value());
+        b.data_mut()[0] = 1;
+        assert_eq!(b.data()[0], 1);
+        assert_eq!(b.index(), <u64>::max_value());
+        assert_eq!(b.meta, Meta::default());
+    }
+    #[test]
+    fn test_blob_forward() {
+        let mut b = Blob::default();
+        assert!(!b.should_forward());
+        b.forward(true);
+        assert!(b.should_forward());
+    }
+
+}

core/src/poh.rs

@@ -0,0 +1,194 @@
+//! The `Poh` module provhashes an object for generating a Proof of History.
+//! It records Hashes items on behalf of its users.
+use solana_sdk::hash::{hash, hashv, Hash};
+
+pub struct Poh {
+    pub hash: Hash,
+    num_hashes: u64,
+    pub tick_height: u64,
+}
+
+#[derive(Debug)]
+pub struct PohEntry {
+    pub tick_height: u64,
+    pub num_hashes: u64,
+    pub hash: Hash,
+    pub mixin: Option<Hash>,
+}
+
+impl Poh {
+    pub fn new(hash: Hash, tick_height: u64) -> Self {
+        Poh {
+            num_hashes: 0,
+            hash,
+            tick_height,
+        }
+    }
+
+    pub fn hash(&mut self) {
+        self.hash = hash(&self.hash.as_ref());
+        self.num_hashes += 1;
+    }
+
+    pub fn record(&mut self, mixin: Hash) -> PohEntry {
+        self.hash = hashv(&[&self.hash.as_ref(), &mixin.as_ref()]);
+
+        let num_hashes = self.num_hashes + 1;
+        self.num_hashes = 0;
+
+        PohEntry {
+            tick_height: self.tick_height,
+            num_hashes,
+            hash: self.hash,
+            mixin: Some(mixin),
+        }
+    }
+
+    // emissions of Ticks (i.e. PohEntries without a mixin) allows
+    //  valhashators to parallelize the work of catching up
+    pub fn tick(&mut self) -> PohEntry {
+        self.hash();
+
+        let num_hashes = self.num_hashes;
+        self.num_hashes = 0;
+        self.tick_height += 1;
+
+        PohEntry {
+            tick_height: self.tick_height,
+            num_hashes,
+            hash: self.hash,
+            mixin: None,
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn verify(initial_hash: Hash, entries: &[PohEntry]) -> bool {
+    let mut current_hash = initial_hash;
+
+    for entry in entries {
+        assert!(entry.num_hashes != 0);
+
+        for _ in 1..entry.num_hashes {
+            current_hash = hash(&current_hash.as_ref());
+        }
+        current_hash = match entry.mixin {
+            Some(mixin) => hashv(&[&current_hash.as_ref(), &mixin.as_ref()]),
+            None => hash(&current_hash.as_ref()),
+        };
+        if current_hash != entry.hash {
+            return false;
+        }
+    }
+
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::poh::{verify, Poh, PohEntry};
+    use solana_sdk::hash::{hash, hashv, Hash};
+
+    #[test]
+    fn test_poh_verify() {
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        let two = hash(&one.as_ref());
+        let one_with_zero = hashv(&[&zero.as_ref(), &zero.as_ref()]);
+
+        let mut poh = Poh::new(zero, 0);
+        assert_eq!(
+            verify(
+                zero,
+                &[poh.tick(), poh.record(zero), poh.record(zero), poh.tick(),],
+            ),
+            true
+        );
+
+        assert_eq!(
+            verify(
+                zero,
+                &[PohEntry {
+                    tick_height: 0,
+                    num_hashes: 1,
+                    hash: one,
+                    mixin: None,
+                }],
+            ),
+            true
+        );
+        assert_eq!(
+            verify(
+                zero,
+                &[PohEntry {
+                    tick_height: 0,
+                    num_hashes: 2,
+                    hash: two,
+                    mixin: None,
+                }]
+            ),
+            true
+        );
+
+        assert_eq!(
+            verify(
+                zero,
+                &[PohEntry {
+                    tick_height: 0,
+                    num_hashes: 1,
+                    hash: one_with_zero,
+                    mixin: Some(zero),
+                }]
+            ),
+            true
+        );
+        assert_eq!(
+            verify(
+                zero,
+                &[PohEntry {
+                    tick_height: 0,
+                    num_hashes: 1,
+                    hash: zero,
+                    mixin: None
+                }]
+            ),
+            false
+        );
+
+        assert_eq!(
+            verify(
+                zero,
+                &[
+                    PohEntry {
+                        tick_height: 0,
+                        num_hashes: 1,
+                        hash: one_with_zero,
+                        mixin: Some(zero),
+                    },
+                    PohEntry {
+                        tick_height: 0,
+                        num_hashes: 1,
+                        hash: hash(&one_with_zero.as_ref()),
+                        mixin: None
+                    },
+                ]
+            ),
+            true
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_poh_verify_assert() {
+        verify(
+            Hash::default(),
+            &[PohEntry {
+                tick_height: 0,
+                num_hashes: 0,
+                hash: Hash::default(),
+                mixin: None,
+            }],
+        );
+    }
+
+}

core/src/poh_recorder.rs

@@ -0,0 +1,415 @@
+//! The `poh_recorder` module provides an object for synchronizing with Proof of History.
+//! It synchronizes PoH, bank's register_tick and the ledger
+//!
+//! PohRecorder will send ticks or entries to a WorkingBank, if the current range of ticks is
+//! within the specified WorkingBank range.
+//!
+//! For Ticks:
+//! * tick must be > WorkingBank::min_tick_height && tick must be <= WorkingBank::man_tick_height
+//!
+//! For Entries:
+//! * recorded entry must be >= WorkingBank::min_tick_height && entry must be < WorkingBank::man_tick_height
+//!
+use crate::entry::Entry;
+use crate::poh::Poh;
+use crate::result::{Error, Result};
+use solana_runtime::bank::Bank;
+use solana_sdk::hash::Hash;
+use solana_sdk::transaction::Transaction;
+use std::sync::mpsc::Sender;
+use std::sync::Arc;
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum PohRecorderError {
+    InvalidCallingObject,
+    MaxHeightReached,
+    MinHeightNotReached,
+}
+
+#[derive(Clone)]
+pub struct WorkingBank {
+    pub bank: Arc<Bank>,
+    pub sender: Sender<Vec<(Entry, u64)>>,
+    pub min_tick_height: u64,
+    pub max_tick_height: u64,
+}
+
+pub struct PohRecorder {
+    pub poh: Poh,
+    tick_cache: Vec<(Entry, u64)>,
+    working_bank: Option<WorkingBank>,
+}
+
+impl PohRecorder {
+    pub fn clear_bank(&mut self) {
+        self.working_bank = None;
+    }
+
+    pub fn hash(&mut self) {
+        // TODO: amortize the cost of this lock by doing the loop in here for
+        // some min amount of hashes
+        self.poh.hash();
+    }
+
+    // synchronize PoH with a bank
+    pub fn reset(&mut self, tick_height: u64, last_id: Hash) {
+        let mut cache = vec![];
+        info!(
+            "reset poh from: {},{} to: {},{}",
+            self.poh.hash, self.poh.tick_height, last_id, tick_height,
+        );
+        std::mem::swap(&mut cache, &mut self.tick_cache);
+        self.poh = Poh::new(last_id, tick_height);
+    }
+
+    pub fn set_working_bank(&mut self, working_bank: WorkingBank) {
+        trace!("new working bank");
+        self.working_bank = Some(working_bank);
+    }
+
+    // Flush cache will delay flushing the cache for a bank until it past the WorkingBank::min_tick_height
+    // On a record flush will flush the cache at the WorkingBank::min_tick_height, since a record
+    // occurs after the min_tick_height was generated
+    fn flush_cache(&mut self, tick: bool) -> Result<()> {
+        // check_tick_height is called before flush cache, so it cannot overrun the bank
+        // so a bank that is so late that it's slot fully generated before it starts recording
+        // will fail instead of broadcasting any ticks
+        let working_bank = self
+            .working_bank
+            .as_ref()
+            .ok_or(Error::PohRecorderError(PohRecorderError::MaxHeightReached))?;
+        if self.poh.tick_height < working_bank.min_tick_height {
+            return Err(Error::PohRecorderError(
+                PohRecorderError::MinHeightNotReached,
+            ));
+        }
+        if tick && self.poh.tick_height == working_bank.min_tick_height {
+            return Err(Error::PohRecorderError(
+                PohRecorderError::MinHeightNotReached,
+            ));
+        }
+
+        let cnt = self
+            .tick_cache
+            .iter()
+            .take_while(|x| x.1 <= working_bank.max_tick_height)
+            .count();
+        let e = if cnt > 0 {
+            debug!(
+                "flush_cache: bank_id: {} tick_height: {} max: {} sending: {}",
+                working_bank.bank.slot(),
+                working_bank.bank.tick_height(),
+                working_bank.max_tick_height,
+                cnt,
+            );
+            let cache = &self.tick_cache[..cnt];
+            for t in cache {
+                working_bank.bank.register_tick(&t.0.hash);
+            }
+            working_bank.sender.send(cache.to_vec())
+        } else {
+            Ok(())
+        };
+        if self.poh.tick_height >= working_bank.max_tick_height {
+            info!("poh_record: max_tick_height reached, setting working bank to None");
+            self.working_bank = None;
+        }
+        if e.is_err() {
+            info!("WorkingBank::sender disconnected {:?}", e);
+            //revert the cache, but clear the working bank
+            self.working_bank = None;
+        } else {
+            //commit the flush
+            let _ = self.tick_cache.drain(..cnt);
+        }
+
+        Ok(())
+    }
+
+    pub fn tick(&mut self) {
+        // Register and send the entry out while holding the lock if the max PoH height
+        // hasn't been reached.
+        // This guarantees PoH order and Entry production and banks LastId queue is the same
+        let tick = self.generate_tick();
+        trace!("tick {}", tick.1);
+        self.tick_cache.push(tick);
+        let _ = self.flush_cache(true);
+    }
+
+    pub fn record(&mut self, mixin: Hash, txs: Vec<Transaction>) -> Result<()> {
+        // Register and send the entry out while holding the lock.
+        // This guarantees PoH order and Entry production and banks LastId queue is the same.
+        self.flush_cache(false)?;
+        self.record_and_send_txs(mixin, txs)
+    }
+
+    /// A recorder to synchronize PoH with the following data structures
+    /// * bank - the LastId's queue is updated on `tick` and `record` events
+    /// * sender - the Entry channel that outputs to the ledger
+    pub fn new(tick_height: u64, last_entry_hash: Hash) -> Self {
+        let poh = Poh::new(last_entry_hash, tick_height);
+        PohRecorder {
+            poh,
+            tick_cache: vec![],
+            working_bank: None,
+        }
+    }
+
+    fn record_and_send_txs(&mut self, mixin: Hash, txs: Vec<Transaction>) -> Result<()> {
+        let working_bank = self
+            .working_bank
+            .as_ref()
+            .ok_or(Error::PohRecorderError(PohRecorderError::MaxHeightReached))?;
+        let poh_entry = self.poh.record(mixin);
+        assert!(!txs.is_empty(), "Entries without transactions are used to track real-time passing in the ledger and can only be generated with PohRecorder::tick function");
+        let recorded_entry = Entry {
+            num_hashes: poh_entry.num_hashes,
+            hash: poh_entry.hash,
+            transactions: txs,
+        };
+        trace!("sending entry {}", recorded_entry.is_tick());
+        working_bank
+            .sender
+            .send(vec![(recorded_entry, poh_entry.tick_height)])?;
+        Ok(())
+    }
+
+    fn generate_tick(&mut self) -> (Entry, u64) {
+        let tick = self.poh.tick();
+        assert_ne!(tick.tick_height, 0);
+        (
+            Entry {
+                num_hashes: tick.num_hashes,
+                hash: tick.hash,
+                transactions: vec![],
+            },
+            tick.tick_height,
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_tx::test_tx;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::hash;
+    use std::sync::mpsc::channel;
+    use std::sync::Arc;
+
+    #[test]
+    fn test_poh_recorder_no_zero_tick() {
+        let prev_hash = Hash::default();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.tick_cache.len(), 1);
+        assert_eq!(poh_recorder.tick_cache[0].1, 1);
+        assert_eq!(poh_recorder.poh.tick_height, 1);
+    }
+
+    #[test]
+    fn test_poh_recorder_tick_height_is_last_tick() {
+        let prev_hash = Hash::default();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+        poh_recorder.tick();
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.tick_cache.len(), 2);
+        assert_eq!(poh_recorder.tick_cache[1].1, 2);
+        assert_eq!(poh_recorder.poh.tick_height, 2);
+    }
+
+    #[test]
+    fn test_poh_recorder_reset_clears_cache() {
+        let mut poh_recorder = PohRecorder::new(0, Hash::default());
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.tick_cache.len(), 1);
+        poh_recorder.reset(0, Hash::default());
+        assert_eq!(poh_recorder.tick_cache.len(), 0);
+    }
+
+    #[test]
+    fn test_poh_recorder_clear() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, _) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 2,
+            max_tick_height: 3,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        assert!(poh_recorder.working_bank.is_some());
+        poh_recorder.clear_bank();
+        assert!(poh_recorder.working_bank.is_none());
+    }
+
+    #[test]
+    fn test_poh_recorder_tick_sent_after_min() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 2,
+            max_tick_height: 3,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+        poh_recorder.tick();
+        //tick height equal to min_tick_height
+        //no tick has been sent
+        assert_eq!(poh_recorder.tick_cache.last().unwrap().1, 2);
+        assert!(entry_receiver.try_recv().is_err());
+
+        // all ticks are sent after height > min
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.poh.tick_height, 3);
+        assert_eq!(poh_recorder.tick_cache.len(), 0);
+        let e = entry_receiver.recv().expect("recv 1");
+        assert_eq!(e.len(), 3);
+        assert!(poh_recorder.working_bank.is_none());
+    }
+
+    #[test]
+    fn test_poh_recorder_tick_sent_upto_and_including_max() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        poh_recorder.tick();
+        poh_recorder.tick();
+        poh_recorder.tick();
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.tick_cache.last().unwrap().1, 4);
+        assert_eq!(poh_recorder.poh.tick_height, 4);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 2,
+            max_tick_height: 3,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+
+        assert_eq!(poh_recorder.poh.tick_height, 5);
+        assert!(poh_recorder.working_bank.is_none());
+        let e = entry_receiver.recv().expect("recv 1");
+        assert_eq!(e.len(), 3);
+    }
+
+    #[test]
+    fn test_poh_recorder_record_to_early() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 2,
+            max_tick_height: 3,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+        let tx = test_tx();
+        let h1 = hash(b"hello world!");
+        assert!(poh_recorder.record(h1, vec![tx.clone()]).is_err());
+        assert!(entry_receiver.try_recv().is_err());
+    }
+
+    #[test]
+    fn test_poh_recorder_record_at_min_passes() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 1,
+            max_tick_height: 2,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.tick_cache.len(), 1);
+        assert_eq!(poh_recorder.poh.tick_height, 1);
+        let tx = test_tx();
+        let h1 = hash(b"hello world!");
+        assert!(poh_recorder.record(h1, vec![tx.clone()]).is_ok());
+        assert_eq!(poh_recorder.tick_cache.len(), 0);
+
+        //tick in the cache + entry
+        let e = entry_receiver.recv().expect("recv 1");
+        assert_eq!(e.len(), 1);
+        assert!(e[0].0.is_tick());
+        let e = entry_receiver.recv().expect("recv 2");
+        assert!(!e[0].0.is_tick());
+    }
+
+    #[test]
+    fn test_poh_recorder_record_at_max_fails() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 1,
+            max_tick_height: 2,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.poh.tick_height, 2);
+        let tx = test_tx();
+        let h1 = hash(b"hello world!");
+        assert!(poh_recorder.record(h1, vec![tx.clone()]).is_err());
+
+        let e = entry_receiver.recv().expect("recv 1");
+        assert_eq!(e.len(), 2);
+        assert!(e[0].0.is_tick());
+        assert!(e[1].0.is_tick());
+    }
+
+    #[test]
+    fn test_poh_cache_on_disconnect() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let mut poh_recorder = PohRecorder::new(0, prev_hash);
+
+        let working_bank = WorkingBank {
+            bank,
+            sender: entry_sender,
+            min_tick_height: 2,
+            max_tick_height: 3,
+        };
+        poh_recorder.set_working_bank(working_bank);
+        poh_recorder.tick();
+        poh_recorder.tick();
+        assert_eq!(poh_recorder.poh.tick_height, 2);
+        drop(entry_receiver);
+        poh_recorder.tick();
+        assert!(poh_recorder.working_bank.is_none());
+        assert_eq!(poh_recorder.tick_cache.len(), 3);
+    }
+}

core/src/poh_service.rs

@@ -0,0 +1,241 @@
+//! The `poh_service` module implements a service that records the passing of
+//! "ticks", a measure of time in the PoH stream
+
+use crate::poh_recorder::PohRecorder;
+use crate::service::Service;
+use solana_sdk::timing::NUM_TICKS_PER_SECOND;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::SyncSender;
+use std::sync::{Arc, Mutex};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+#[derive(Clone)]
+pub enum PohServiceConfig {
+    /// * `Tick` - Run full PoH thread.  Tick is a rough estimate of how many hashes to roll before
+    ///            transmitting a new entry.
+    Tick(usize),
+    /// * `Sleep`- Low power mode.  Sleep is a rough estimate of how long to sleep before rolling 1
+    ///            PoH once and producing 1 tick.
+    Sleep(Duration),
+    /// each node in simulation will be blocked until the receiver reads their step
+    Step(SyncSender<()>),
+}
+
+impl Default for PohServiceConfig {
+    fn default() -> PohServiceConfig {
+        // TODO: Change this to Tick to enable PoH
+        PohServiceConfig::Sleep(Duration::from_millis(1000 / NUM_TICKS_PER_SECOND))
+    }
+}
+
+pub struct PohService {
+    tick_producer: JoinHandle<()>,
+    poh_exit: Arc<AtomicBool>,
+}
+
+impl PohService {
+    pub fn exit(&self) {
+        self.poh_exit.store(true, Ordering::Relaxed);
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+
+    pub fn new(
+        poh_recorder: Arc<Mutex<PohRecorder>>,
+        config: &PohServiceConfig,
+        poh_exit: Arc<AtomicBool>,
+    ) -> Self {
+        // PohService is a headless producer, so when it exits it should notify the banking stage.
+        // Since channel are not used to talk between these threads an AtomicBool is used as a
+        // signal.
+        let poh_exit_ = poh_exit.clone();
+        // Single thread to generate ticks
+        let config = config.clone();
+        let tick_producer = Builder::new()
+            .name("solana-poh-service-tick_producer".to_string())
+            .spawn(move || {
+                let poh_recorder = poh_recorder;
+                Self::tick_producer(&poh_recorder, &config, &poh_exit_);
+                poh_exit_.store(true, Ordering::Relaxed);
+            })
+            .unwrap();
+
+        Self {
+            tick_producer,
+            poh_exit,
+        }
+    }
+
+    fn tick_producer(
+        poh: &Arc<Mutex<PohRecorder>>,
+        config: &PohServiceConfig,
+        poh_exit: &AtomicBool,
+    ) {
+        loop {
+            match config {
+                PohServiceConfig::Tick(num) => {
+                    for _ in 1..*num {
+                        poh.lock().unwrap().hash();
+                    }
+                }
+                PohServiceConfig::Sleep(duration) => {
+                    sleep(*duration);
+                }
+                PohServiceConfig::Step(sender) => {
+                    let r = sender.send(());
+                    if r.is_err() {
+                        break;
+                    }
+                }
+            }
+            poh.lock().unwrap().tick();
+            if poh_exit.load(Ordering::Relaxed) {
+                return;
+            }
+        }
+    }
+}
+
+impl Service for PohService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.tick_producer.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::poh_recorder::WorkingBank;
+    use crate::result::Result;
+    use crate::test_tx::test_tx;
+    use solana_runtime::bank::Bank;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::hash;
+    use std::sync::mpsc::channel;
+    use std::sync::mpsc::RecvError;
+
+    #[test]
+    fn test_poh_service() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(bank.tick_height(), prev_hash)));
+        let exit = Arc::new(AtomicBool::new(false));
+        let working_bank = WorkingBank {
+            bank: bank.clone(),
+            sender: entry_sender,
+            min_tick_height: bank.tick_height(),
+            max_tick_height: std::u64::MAX,
+        };
+
+        let entry_producer: JoinHandle<Result<()>> = {
+            let poh_recorder = poh_recorder.clone();
+            let exit = exit.clone();
+
+            Builder::new()
+                .name("solana-poh-service-entry_producer".to_string())
+                .spawn(move || {
+                    loop {
+                        // send some data
+                        let h1 = hash(b"hello world!");
+                        let tx = test_tx();
+                        poh_recorder.lock().unwrap().record(h1, vec![tx]).unwrap();
+
+                        if exit.load(Ordering::Relaxed) {
+                            break Ok(());
+                        }
+                    }
+                })
+                .unwrap()
+        };
+
+        const HASHES_PER_TICK: u64 = 2;
+        let poh_service = PohService::new(
+            poh_recorder.clone(),
+            &PohServiceConfig::Tick(HASHES_PER_TICK as usize),
+            Arc::new(AtomicBool::new(false)),
+        );
+        poh_recorder.lock().unwrap().set_working_bank(working_bank);
+
+        // get some events
+        let mut hashes = 0;
+        let mut need_tick = true;
+        let mut need_entry = true;
+        let mut need_partial = true;
+
+        while need_tick || need_entry || need_partial {
+            for entry in entry_receiver.recv().unwrap() {
+                let entry = &entry.0;
+                if entry.is_tick() {
+                    assert!(entry.num_hashes <= HASHES_PER_TICK);
+
+                    if entry.num_hashes == HASHES_PER_TICK {
+                        need_tick = false;
+                    } else {
+                        need_partial = false;
+                    }
+
+                    hashes += entry.num_hashes;
+
+                    assert_eq!(hashes, HASHES_PER_TICK);
+
+                    hashes = 0;
+                } else {
+                    assert!(entry.num_hashes >= 1);
+                    need_entry = false;
+                    hashes += entry.num_hashes - 1;
+                }
+            }
+        }
+        exit.store(true, Ordering::Relaxed);
+        poh_service.exit();
+        let _ = poh_service.join().unwrap();
+        let _ = entry_producer.join().unwrap();
+    }
+
+    #[test]
+    fn test_poh_service_drops_working_bank() {
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let prev_hash = bank.last_id();
+        let (entry_sender, entry_receiver) = channel();
+        let poh_recorder = Arc::new(Mutex::new(PohRecorder::new(bank.tick_height(), prev_hash)));
+        let exit = Arc::new(AtomicBool::new(false));
+        let working_bank = WorkingBank {
+            bank: bank.clone(),
+            sender: entry_sender,
+            min_tick_height: bank.tick_height() + 3,
+            max_tick_height: bank.tick_height() + 5,
+        };
+
+        let poh_service = PohService::new(
+            poh_recorder.clone(),
+            &PohServiceConfig::default(),
+            Arc::new(AtomicBool::new(false)),
+        );
+
+        poh_recorder.lock().unwrap().set_working_bank(working_bank);
+
+        // all 5 ticks are expected, there is no tick 0
+        // First 4 ticks must be sent all at once, since bank shouldn't see them until
+        // the after bank's min_tick_height(3) is reached.
+        let entries = entry_receiver.recv().expect("recv 1");
+        assert_eq!(entries.len(), 4);
+        let entries = entry_receiver.recv().expect("recv 2");
+        assert_eq!(entries.len(), 1);
+
+        //WorkingBank should be dropped by the PohService thread as well
+        assert_eq!(entry_receiver.recv(), Err(RecvError));
+
+        exit.store(true, Ordering::Relaxed);
+        poh_service.exit();
+        let _ = poh_service.join().unwrap();
+    }
+}

core/src/recvmmsg.rs

@@ -0,0 +1,212 @@
+//! The `recvmmsg` module provides recvmmsg() API implementation
+
+use crate::packet::Packet;
+use std::cmp;
+use std::io;
+use std::net::UdpSocket;
+
+pub const NUM_RCVMMSGS: usize = 16;
+
+#[cfg(not(target_os = "linux"))]
+pub fn recv_mmsg(socket: &UdpSocket, packets: &mut [Packet]) -> io::Result<usize> {
+    let mut i = 0;
+    let count = cmp::min(NUM_RCVMMSGS, packets.len());
+    for p in packets.iter_mut().take(count) {
+        p.meta.size = 0;
+        match socket.recv_from(&mut p.data) {
+            Err(_) if i > 0 => {
+                break;
+            }
+            Err(e) => {
+                return Err(e);
+            }
+            Ok((nrecv, from)) => {
+                p.meta.size = nrecv;
+                p.meta.set_addr(&from);
+                if i == 0 {
+                    socket.set_nonblocking(true)?;
+                }
+            }
+        }
+        i += 1;
+    }
+    Ok(i)
+}
+
+#[cfg(target_os = "linux")]
+pub fn recv_mmsg(sock: &UdpSocket, packets: &mut [Packet]) -> io::Result<usize> {
+    use libc::{
+        c_void, iovec, mmsghdr, recvmmsg, sockaddr_in, socklen_t, time_t, timespec, MSG_WAITFORONE,
+    };
+    use nix::sys::socket::InetAddr;
+    use std::mem;
+    use std::os::unix::io::AsRawFd;
+
+    let mut hdrs: [mmsghdr; NUM_RCVMMSGS] = unsafe { mem::zeroed() };
+    let mut iovs: [iovec; NUM_RCVMMSGS] = unsafe { mem::zeroed() };
+    let mut addr: [sockaddr_in; NUM_RCVMMSGS] = unsafe { mem::zeroed() };
+    let addrlen = mem::size_of_val(&addr) as socklen_t;
+
+    let sock_fd = sock.as_raw_fd();
+
+    let count = cmp::min(iovs.len(), packets.len());
+
+    for i in 0..count {
+        iovs[i].iov_base = packets[i].data.as_mut_ptr() as *mut c_void;
+        iovs[i].iov_len = packets[i].data.len();
+
+        hdrs[i].msg_hdr.msg_name = &mut addr[i] as *mut _ as *mut _;
+        hdrs[i].msg_hdr.msg_namelen = addrlen;
+        hdrs[i].msg_hdr.msg_iov = &mut iovs[i];
+        hdrs[i].msg_hdr.msg_iovlen = 1;
+    }
+    let mut ts = timespec {
+        tv_sec: 1 as time_t,
+        tv_nsec: 0,
+    };
+
+    let npkts =
+        match unsafe { recvmmsg(sock_fd, &mut hdrs[0], count as u32, MSG_WAITFORONE, &mut ts) } {
+            -1 => return Err(io::Error::last_os_error()),
+            n => {
+                for i in 0..n as usize {
+                    let mut p = &mut packets[i];
+                    p.meta.size = hdrs[i].msg_len as usize;
+                    let inet_addr = InetAddr::V4(addr[i]);
+                    p.meta.set_addr(&inet_addr.to_std());
+                }
+                n as usize
+            }
+        };
+
+    Ok(npkts)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::packet::PACKET_DATA_SIZE;
+    use crate::recvmmsg::*;
+    use std::time::{Duration, Instant};
+
+    #[test]
+    pub fn test_recv_mmsg_one_iter() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let sent = NUM_RCVMMSGS - 1;
+        for _ in 0..sent {
+            let data = [0; PACKET_DATA_SIZE];
+            sender.send_to(&data[..], &addr).unwrap();
+        }
+
+        let mut packets = vec![Packet::default(); NUM_RCVMMSGS];
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(sent, recv);
+        for i in 0..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr);
+        }
+    }
+
+    #[test]
+    pub fn test_recv_mmsg_multi_iter() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let sent = NUM_RCVMMSGS + 10;
+        for _ in 0..sent {
+            let data = [0; PACKET_DATA_SIZE];
+            sender.send_to(&data[..], &addr).unwrap();
+        }
+
+        let mut packets = vec![Packet::default(); NUM_RCVMMSGS * 2];
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(NUM_RCVMMSGS, recv);
+        for i in 0..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr);
+        }
+
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(sent - NUM_RCVMMSGS, recv);
+        for i in 0..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr);
+        }
+    }
+
+    #[test]
+    pub fn test_recv_mmsg_multi_iter_timeout() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        reader.set_read_timeout(Some(Duration::new(5, 0))).unwrap();
+        reader.set_nonblocking(false).unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let sent = NUM_RCVMMSGS;
+        for _ in 0..sent {
+            let data = [0; PACKET_DATA_SIZE];
+            sender.send_to(&data[..], &addr).unwrap();
+        }
+
+        let start = Instant::now();
+        let mut packets = vec![Packet::default(); NUM_RCVMMSGS * 2];
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(NUM_RCVMMSGS, recv);
+        for i in 0..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr);
+        }
+        reader.set_nonblocking(true).unwrap();
+
+        let _recv = recv_mmsg(&reader, &mut packets[..]);
+        assert!(start.elapsed().as_secs() < 5);
+    }
+
+    #[test]
+    pub fn test_recv_mmsg_multi_addrs() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+
+        let sender1 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr1 = sender1.local_addr().unwrap();
+        let sent1 = NUM_RCVMMSGS - 1;
+
+        let sender2 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr2 = sender2.local_addr().unwrap();
+        let sent2 = NUM_RCVMMSGS + 1;
+
+        for _ in 0..sent1 {
+            let data = [0; PACKET_DATA_SIZE];
+            sender1.send_to(&data[..], &addr).unwrap();
+        }
+
+        for _ in 0..sent2 {
+            let data = [0; PACKET_DATA_SIZE];
+            sender2.send_to(&data[..], &addr).unwrap();
+        }
+
+        let mut packets = vec![Packet::default(); NUM_RCVMMSGS * 2];
+
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(NUM_RCVMMSGS, recv);
+        for i in 0..sent1 {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr1);
+        }
+
+        for i in sent1..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr2);
+        }
+
+        let recv = recv_mmsg(&reader, &mut packets[..]).unwrap();
+        assert_eq!(sent1 + sent2 - NUM_RCVMMSGS, recv);
+        for i in 0..recv {
+            assert_eq!(packets[i].meta.size, PACKET_DATA_SIZE);
+            assert_eq!(packets[i].meta.addr(), saddr2);
+        }
+    }
+}

core/src/repair_service.rs

@@ -0,0 +1,362 @@
+//! The `repair_service` module implements the tools necessary to generate a thread which
+//! regularly finds missing blobs in the ledger and sends repair requests for those blobs
+
+use crate::blocktree::{Blocktree, SlotMeta};
+use crate::cluster_info::ClusterInfo;
+use crate::result::Result;
+use crate::service::Service;
+use solana_metrics::{influxdb, submit};
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+
+pub const MAX_REPAIR_LENGTH: usize = 16;
+pub const REPAIR_MS: u64 = 100;
+pub const MAX_REPAIR_TRIES: u64 = 128;
+
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RepairType {
+    HighestBlob(u64, u64),
+    Blob(u64, u64),
+}
+
+#[derive(Default)]
+struct RepairInfo {
+    max_slot: u64,
+    repair_tries: u64,
+}
+
+impl RepairInfo {
+    fn new() -> Self {
+        RepairInfo {
+            max_slot: 0,
+            repair_tries: 0,
+        }
+    }
+}
+
+pub struct RepairService {
+    t_repair: JoinHandle<()>,
+}
+
+impl RepairService {
+    fn run(
+        blocktree: &Arc<Blocktree>,
+        exit: &Arc<AtomicBool>,
+        repair_socket: &Arc<UdpSocket>,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+    ) {
+        let mut repair_info = RepairInfo::new();
+        let id = cluster_info.read().unwrap().id();
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            let repairs = Self::generate_repairs(blocktree, MAX_REPAIR_LENGTH, &mut repair_info);
+
+            if let Ok(repairs) = repairs {
+                let reqs: Vec<_> = repairs
+                    .into_iter()
+                    .filter_map(|repair_request| {
+                        let (slot_height, blob_index, is_highest_request) = {
+                            match repair_request {
+                                RepairType::Blob(s, i) => (s, i, false),
+                                RepairType::HighestBlob(s, i) => (s, i, true),
+                            }
+                        };
+                        cluster_info
+                            .read()
+                            .unwrap()
+                            .window_index_request(slot_height, blob_index, is_highest_request)
+                            .map(|result| (result, slot_height, blob_index))
+                            .ok()
+                    })
+                    .collect();
+
+                for ((to, req), slot_height, blob_index) in reqs {
+                    if let Ok(local_addr) = repair_socket.local_addr() {
+                        submit(
+                            influxdb::Point::new("repair_service")
+                                .add_field(
+                                    "repair_slot",
+                                    influxdb::Value::Integer(slot_height as i64),
+                                )
+                                .to_owned()
+                                .add_field(
+                                    "repair_blob",
+                                    influxdb::Value::Integer(blob_index as i64),
+                                )
+                                .to_owned()
+                                .add_field("to", influxdb::Value::String(to.to_string()))
+                                .to_owned()
+                                .add_field("from", influxdb::Value::String(local_addr.to_string()))
+                                .to_owned()
+                                .add_field("id", influxdb::Value::String(id.to_string()))
+                                .to_owned(),
+                        );
+                    }
+
+                    repair_socket.send_to(&req, to).unwrap_or_else(|e| {
+                        info!("{} repair req send_to({}) error {:?}", id, to, e);
+                        0
+                    });
+                }
+            }
+            sleep(Duration::from_millis(REPAIR_MS));
+        }
+    }
+
+    pub fn new(
+        blocktree: Arc<Blocktree>,
+        exit: Arc<AtomicBool>,
+        repair_socket: Arc<UdpSocket>,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+    ) -> Self {
+        let t_repair = Builder::new()
+            .name("solana-repair-service".to_string())
+            .spawn(move || Self::run(&blocktree, &exit, &repair_socket, &cluster_info))
+            .unwrap();
+
+        RepairService { t_repair }
+    }
+
+    fn process_slot(
+        blocktree: &Blocktree,
+        slot_height: u64,
+        slot: &SlotMeta,
+        max_repairs: usize,
+    ) -> Vec<RepairType> {
+        if slot.is_full() {
+            vec![]
+        } else if slot.consumed == slot.received {
+            vec![RepairType::HighestBlob(slot_height, slot.received)]
+        } else {
+            let reqs = blocktree.find_missing_data_indexes(
+                slot_height,
+                slot.consumed,
+                slot.received,
+                max_repairs,
+            );
+
+            reqs.into_iter()
+                .map(|i| RepairType::Blob(slot_height, i))
+                .collect()
+        }
+    }
+
+    fn generate_repairs(
+        blocktree: &Blocktree,
+        max_repairs: usize,
+        repair_info: &mut RepairInfo,
+    ) -> Result<(Vec<RepairType>)> {
+        // Slot height and blob indexes for blobs we want to repair
+        let mut repairs: Vec<RepairType> = vec![];
+        let mut current_slot_height = Some(0);
+        while repairs.len() < max_repairs && current_slot_height.is_some() {
+            if current_slot_height.unwrap() > repair_info.max_slot {
+                repair_info.repair_tries = 0;
+                repair_info.max_slot = current_slot_height.unwrap();
+            }
+
+            let slot = blocktree.meta(current_slot_height.unwrap())?;
+            if slot.is_some() {
+                let slot = slot.unwrap();
+                let new_repairs = Self::process_slot(
+                    blocktree,
+                    current_slot_height.unwrap(),
+                    &slot,
+                    max_repairs - repairs.len(),
+                );
+                repairs.extend(new_repairs);
+            }
+            current_slot_height = blocktree.get_next_slot(current_slot_height.unwrap())?;
+        }
+
+        // Only increment repair_tries if the ledger contains every blob for every slot
+        if repairs.is_empty() {
+            repair_info.repair_tries += 1;
+        }
+
+        // Optimistically try the next slot if we haven't gotten any repairs
+        // for a while
+        if repair_info.repair_tries >= MAX_REPAIR_TRIES {
+            repairs.push(RepairType::HighestBlob(repair_info.max_slot + 1, 0))
+        }
+
+        Ok(repairs)
+    }
+}
+
+impl Service for RepairService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.t_repair.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::blocktree::tests::{make_many_slot_entries, make_slot_entries};
+    use crate::blocktree::{get_tmp_ledger_path, Blocktree};
+    use crate::entry::create_ticks;
+    use crate::entry::{make_tiny_test_entries, EntrySlice};
+    use solana_sdk::hash::Hash;
+
+    #[test]
+    pub fn test_repair_missed_future_slot() {
+        let blocktree_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+            let mut blobs = create_ticks(1, Hash::default()).to_blobs();
+            blobs[0].set_index(0);
+            blobs[0].set_slot(0);
+            blobs[0].set_is_last_in_slot();
+
+            blocktree.write_blobs(&blobs).unwrap();
+
+            let mut repair_info = RepairInfo::new();
+            // We have all the blobs for all the slots in the ledger, wait for optimistic
+            // future repair after MAX_REPAIR_TRIES
+            for i in 0..MAX_REPAIR_TRIES {
+                // Check that repair tries to patch the empty slot
+                assert_eq!(repair_info.repair_tries, i);
+                assert_eq!(repair_info.max_slot, 0);
+                let expected = if i == MAX_REPAIR_TRIES - 1 {
+                    vec![RepairType::HighestBlob(1, 0)]
+                } else {
+                    vec![]
+                };
+                assert_eq!(
+                    RepairService::generate_repairs(&blocktree, 2, &mut repair_info).unwrap(),
+                    expected
+                );
+            }
+
+            // Insert a bigger blob, see that we the MAX_REPAIR_TRIES gets reset
+            let mut blobs = create_ticks(1, Hash::default()).to_blobs();
+            blobs[0].set_index(0);
+            blobs[0].set_slot(1);
+            blobs[0].set_is_last_in_slot();
+
+            blocktree.write_blobs(&blobs).unwrap();
+            assert_eq!(
+                RepairService::generate_repairs(&blocktree, 2, &mut repair_info).unwrap(),
+                vec![]
+            );
+            assert_eq!(repair_info.repair_tries, 1);
+            assert_eq!(repair_info.max_slot, 1);
+        }
+
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_repair_empty_slot() {
+        let blocktree_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+            let mut blobs = make_tiny_test_entries(1).to_blobs();
+            blobs[0].set_index(1);
+            blobs[0].set_slot(2);
+
+            let mut repair_info = RepairInfo::new();
+
+            // Write this blob to slot 2, should chain to slot 1, which we haven't received
+            // any blobs for
+            blocktree.write_blobs(&blobs).unwrap();
+            // Check that repair tries to patch the empty slot
+            assert_eq!(
+                RepairService::generate_repairs(&blocktree, 2, &mut repair_info).unwrap(),
+                vec![RepairType::HighestBlob(0, 0), RepairType::Blob(2, 0)]
+            );
+        }
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_generate_repairs() {
+        let blocktree_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+            let nth = 3;
+            let num_entries_per_slot = 5 * nth;
+            let num_slots = 2;
+
+            let mut repair_info = RepairInfo::new();
+
+            // Create some blobs
+            let (blobs, _) =
+                make_many_slot_entries(0, num_slots as u64, num_entries_per_slot as u64);
+
+            // write every nth blob
+            let blobs_to_write: Vec<_> = blobs.iter().step_by(nth as usize).collect();
+
+            blocktree.write_blobs(blobs_to_write).unwrap();
+
+            let missing_indexes_per_slot: Vec<u64> = (0..num_entries_per_slot / nth - 1)
+                .flat_map(|x| ((nth * x + 1) as u64..(nth * x + nth) as u64))
+                .collect();
+
+            let expected: Vec<RepairType> = (0..num_slots)
+                .flat_map(|slot_height| {
+                    missing_indexes_per_slot
+                        .iter()
+                        .map(move |blob_index| RepairType::Blob(slot_height as u64, *blob_index))
+                })
+                .collect();
+
+            // Across all slots, find all missing indexes in the range [0, num_entries_per_slot]
+            assert_eq!(
+                RepairService::generate_repairs(&blocktree, std::usize::MAX, &mut repair_info)
+                    .unwrap(),
+                expected
+            );
+
+            assert_eq!(
+                RepairService::generate_repairs(&blocktree, expected.len() - 2, &mut repair_info)
+                    .unwrap()[..],
+                expected[0..expected.len() - 2]
+            );
+        }
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_generate_highest_repair() {
+        let blocktree_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Blocktree::open(&blocktree_path).unwrap();
+
+            let num_entries_per_slot = 10;
+
+            let mut repair_info = RepairInfo::new();
+
+            // Create some blobs
+            let (mut blobs, _) = make_slot_entries(0, 0, num_entries_per_slot as u64);
+
+            // Remove is_last flag on last blob
+            blobs.last_mut().unwrap().set_flags(0);
+
+            blocktree.write_blobs(&blobs).unwrap();
+
+            // We didn't get the last blob for this slot, so ask for the highest blob for that slot
+            let expected: Vec<RepairType> = vec![RepairType::HighestBlob(0, num_entries_per_slot)];
+
+            assert_eq!(
+                RepairService::generate_repairs(&blocktree, std::usize::MAX, &mut repair_info)
+                    .unwrap(),
+                expected
+            );
+        }
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+    }
+}

core/src/replay_stage.rs

@@ -0,0 +1,440 @@
+//! The `replay_stage` replays transactions broadcast by the leader.
+
+use crate::bank_forks::BankForks;
+use crate::blocktree::Blocktree;
+use crate::blocktree_processor;
+use crate::blocktree_processor::BankForksInfo;
+use crate::cluster_info::ClusterInfo;
+use crate::entry::{Entry, EntryReceiver, EntrySender, EntrySlice};
+use crate::leader_schedule_utils;
+use crate::packet::BlobError;
+use crate::result;
+use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::service::Service;
+use crate::tvu::{TvuRotationInfo, TvuRotationSender};
+use solana_metrics::counter::Counter;
+use solana_runtime::bank::Bank;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::KeypairUtil;
+use solana_sdk::timing::duration_as_ms;
+use solana_sdk::vote_transaction::VoteTransaction;
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError};
+use std::sync::{Arc, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+use std::time::Instant;
+
+pub const MAX_ENTRY_RECV_PER_ITER: usize = 512;
+
+// Implement a destructor for the ReplayStage thread to signal it exited
+// even on panics
+struct Finalizer {
+    exit_sender: Arc<AtomicBool>,
+}
+
+impl Finalizer {
+    fn new(exit_sender: Arc<AtomicBool>) -> Self {
+        Finalizer { exit_sender }
+    }
+}
+// Implement a destructor for Finalizer.
+impl Drop for Finalizer {
+    fn drop(&mut self) {
+        self.exit_sender.clone().store(true, Ordering::Relaxed);
+    }
+}
+
+pub struct ReplayStage {
+    t_replay: JoinHandle<result::Result<()>>,
+    exit: Arc<AtomicBool>,
+}
+
+impl ReplayStage {
+    #[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
+    pub fn new<T>(
+        my_id: Pubkey,
+        voting_keypair: Option<Arc<T>>,
+        blocktree: Arc<Blocktree>,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        _bank_forks_info: &[BankForksInfo],
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        exit: Arc<AtomicBool>,
+        to_leader_sender: &TvuRotationSender,
+        ledger_signal_receiver: Receiver<bool>,
+        subscriptions: &Arc<RpcSubscriptions>,
+    ) -> (Self, Receiver<(u64, Pubkey)>, EntryReceiver)
+    where
+        T: 'static + KeypairUtil + Send + Sync,
+    {
+        let (forward_entry_sender, forward_entry_receiver) = channel();
+        let (slot_full_sender, slot_full_receiver) = channel();
+        trace!("replay stage");
+        let exit_ = exit.clone();
+        let to_leader_sender = to_leader_sender.clone();
+        let subscriptions = subscriptions.clone();
+        let bank_forks = bank_forks.clone();
+
+        let mut progress = HashMap::new();
+
+        // Start the replay stage loop
+        let t_replay = Builder::new()
+            .name("solana-replay-stage".to_string())
+            .spawn(move || {
+                let _exit = Finalizer::new(exit_.clone());
+                loop {
+                    let now = Instant::now();
+                    // Stop getting entries if we get exit signal
+                    if exit_.load(Ordering::Relaxed) {
+                        break;
+                    }
+                    Self::generate_new_bank_forks(&blocktree, &mut bank_forks.write().unwrap());
+                    let live_bank_ids = bank_forks.read().unwrap().active_banks();
+                    trace!("live banks {:?}", live_bank_ids);
+                    let mut votable: Vec<u64> = vec![];
+                    for bank_id in live_bank_ids {
+                        let bank = bank_forks.read().unwrap().get(bank_id).unwrap().clone();
+                        if !Self::is_tpu(&bank, my_id) {
+                            Self::replay_blocktree_into_bank(
+                                &bank,
+                                &blocktree,
+                                &mut progress,
+                                &forward_entry_sender,
+                            )?;
+                        }
+                        let max_tick_height = (bank_id + 1) * bank.ticks_per_slot() - 1;
+                        if bank.tick_height() == max_tick_height {
+                            bank.freeze();
+                            votable.push(bank_id);
+                            progress.remove(&bank_id);
+                            let id = leader_schedule_utils::slot_leader_at(bank.slot(), &bank);
+                            if let Err(e) = slot_full_sender.send((bank.slot(), id)) {
+                                info!("{} slot_full alert failed: {:?}", my_id, e);
+                            }
+                        }
+                    }
+                    // TODO: fork selection
+                    // vote on the latest one for now
+                    votable.sort();
+
+                    if let Some(latest_slot_vote) = votable.last() {
+                        let parent = bank_forks
+                            .read()
+                            .unwrap()
+                            .get(*latest_slot_vote)
+                            .unwrap()
+                            .clone();
+                        let next_slot = *latest_slot_vote + 1;
+                        let next_leader = leader_schedule_utils::slot_leader_at(next_slot, &parent);
+                        cluster_info.write().unwrap().set_leader(next_leader);
+
+                        subscriptions.notify_subscribers(&parent);
+
+                        if let Some(ref voting_keypair) = voting_keypair {
+                            let keypair = voting_keypair.as_ref();
+                            let vote = VoteTransaction::new_vote(
+                                keypair,
+                                *latest_slot_vote,
+                                parent.last_id(),
+                                0,
+                            );
+                            cluster_info.write().unwrap().push_vote(vote);
+                        }
+                        if next_leader == my_id {
+                            let tpu_bank = Bank::new_from_parent(&parent, my_id, next_slot);
+                            bank_forks.write().unwrap().insert(next_slot, tpu_bank);
+                        }
+                        debug!(
+                            "to_leader_sender: me: {} next_slot: {} next_leader: {}",
+                            my_id, next_slot, next_leader
+                        );
+                        to_leader_sender.send(TvuRotationInfo {
+                            tick_height: parent.tick_height(),
+                            last_id: parent.last_id(),
+                            slot: next_slot,
+                            leader_id: next_leader,
+                        })?;
+                    }
+                    inc_new_counter_info!(
+                        "replicate_stage-duration",
+                        duration_as_ms(&now.elapsed()) as usize
+                    );
+                    let timer = Duration::from_millis(100);
+                    let result = ledger_signal_receiver.recv_timeout(timer);
+                    match result {
+                        Err(RecvTimeoutError::Timeout) => continue,
+                        Err(_) => break,
+                        Ok(_) => debug!("blocktree signal"),
+                    };
+                }
+                Ok(())
+            })
+            .unwrap();
+        (
+            Self { t_replay, exit },
+            slot_full_receiver,
+            forward_entry_receiver,
+        )
+    }
+
+    pub fn replay_blocktree_into_bank(
+        bank: &Bank,
+        blocktree: &Blocktree,
+        progress: &mut HashMap<u64, (Hash, usize)>,
+        forward_entry_sender: &EntrySender,
+    ) -> result::Result<()> {
+        let (entries, num) = Self::load_blocktree_entries(bank, blocktree, progress)?;
+        let len = entries.len();
+        let result =
+            Self::replay_entries_into_bank(bank, entries, progress, forward_entry_sender, num);
+        if result.is_ok() {
+            trace!("verified entries {}", len);
+            inc_new_counter_info!("replicate-stage_process_entries", len);
+        } else {
+            info!("debug to verify entries {}", len);
+            //TODO: mark this fork as failed
+            inc_new_counter_info!("replicate-stage_failed_process_entries", len);
+        }
+        Ok(())
+    }
+
+    pub fn load_blocktree_entries(
+        bank: &Bank,
+        blocktree: &Blocktree,
+        progress: &mut HashMap<u64, (Hash, usize)>,
+    ) -> result::Result<(Vec<Entry>, usize)> {
+        let bank_id = bank.slot();
+        let bank_progress = &mut progress.entry(bank_id).or_insert((bank.last_id(), 0));
+        blocktree.get_slot_entries_with_blob_count(bank_id, bank_progress.1 as u64, None)
+    }
+
+    pub fn replay_entries_into_bank(
+        bank: &Bank,
+        entries: Vec<Entry>,
+        progress: &mut HashMap<u64, (Hash, usize)>,
+        forward_entry_sender: &EntrySender,
+        num: usize,
+    ) -> result::Result<()> {
+        let bank_progress = &mut progress.entry(bank.slot()).or_insert((bank.last_id(), 0));
+        let result = Self::verify_and_process_entries(&bank, &entries, &bank_progress.0);
+        bank_progress.1 += num;
+        if let Some(last_entry) = entries.last() {
+            bank_progress.0 = last_entry.hash;
+        }
+        if result.is_ok() {
+            forward_entry_sender.send(entries)?;
+        }
+        result
+    }
+
+    pub fn is_tpu(bank: &Bank, my_id: Pubkey) -> bool {
+        my_id == leader_schedule_utils::slot_leader(&bank)
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+
+    pub fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+
+    pub fn verify_and_process_entries(
+        bank: &Bank,
+        entries: &[Entry],
+        last_entry: &Hash,
+    ) -> result::Result<()> {
+        if !entries.verify(last_entry) {
+            trace!(
+                "entry verification failed {} {} {} {}",
+                entries.len(),
+                bank.tick_height(),
+                last_entry,
+                bank.last_id()
+            );
+            return Err(result::Error::BlobError(BlobError::VerificationFailed));
+        }
+        blocktree_processor::process_entries(bank, entries)?;
+
+        Ok(())
+    }
+
+    fn generate_new_bank_forks(blocktree: &Blocktree, forks: &mut BankForks) {
+        // Find the next slot that chains to the old slot
+        let frozen_banks = forks.frozen_banks();
+        let frozen_bank_ids: Vec<u64> = frozen_banks.keys().cloned().collect();
+        trace!("generate new forks {:?}", frozen_bank_ids);
+        let next_slots = blocktree
+            .get_slots_since(&frozen_bank_ids)
+            .expect("Db error");
+        for (parent_id, children) in next_slots {
+            let parent_bank = frozen_banks
+                .get(&parent_id)
+                .expect("missing parent in bank forks")
+                .clone();
+            for child_id in children {
+                let new_fork = forks.get(child_id).is_none();
+                if new_fork {
+                    let leader = leader_schedule_utils::slot_leader_at(child_id, &parent_bank);
+                    trace!("new fork:{} parent:{}", child_id, parent_id);
+                    forks.insert(
+                        child_id,
+                        Bank::new_from_parent(&parent_bank, leader, child_id),
+                    );
+                }
+            }
+        }
+    }
+}
+
+impl Service for ReplayStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.t_replay.join().map(|_| ())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::blocktree::create_new_tmp_ledger;
+    use crate::cluster_info::{ClusterInfo, Node};
+    use crate::entry::create_ticks;
+    use crate::entry::{next_entry_mut, Entry};
+    use crate::fullnode::new_banks_from_blocktree;
+    use crate::replay_stage::ReplayStage;
+    use crate::result::Error;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use std::fs::remove_dir_all;
+    use std::sync::atomic::AtomicBool;
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+
+    #[test]
+    fn test_vote_error_replay_stage_correctness() {
+        solana_logger::setup();
+        // Set up dummy node to host a ReplayStage
+        let my_keypair = Keypair::new();
+        let my_id = my_keypair.pubkey();
+        let my_node = Node::new_localhost_with_pubkey(my_id);
+
+        // Create keypair for the leader
+        let leader_id = Keypair::new().pubkey();
+
+        let (genesis_block, _mint_keypair) = GenesisBlock::new_with_leader(10_000, leader_id, 500);
+
+        let (my_ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        // Set up the cluster info
+        let cluster_info_me = Arc::new(RwLock::new(ClusterInfo::new(my_node.info.clone())));
+
+        // Set up the replay stage
+        let exit = Arc::new(AtomicBool::new(false));
+        let voting_keypair = Arc::new(Keypair::new());
+        let (to_leader_sender, _to_leader_receiver) = channel();
+        {
+            let (bank_forks, bank_forks_info, blocktree, l_receiver) =
+                new_banks_from_blocktree(&my_ledger_path, None);
+            let bank = bank_forks.working_bank();
+
+            let blocktree = Arc::new(blocktree);
+            let (replay_stage, _slot_full_receiver, ledger_writer_recv) = ReplayStage::new(
+                my_keypair.pubkey(),
+                Some(voting_keypair.clone()),
+                blocktree.clone(),
+                &Arc::new(RwLock::new(bank_forks)),
+                &bank_forks_info,
+                cluster_info_me.clone(),
+                exit.clone(),
+                &to_leader_sender,
+                l_receiver,
+                &Arc::new(RpcSubscriptions::default()),
+            );
+
+            let keypair = voting_keypair.as_ref();
+            let vote = VoteTransaction::new_vote(keypair, 0, bank.last_id(), 0);
+            cluster_info_me.write().unwrap().push_vote(vote);
+
+            info!("Send ReplayStage an entry, should see it on the ledger writer receiver");
+            let next_tick = create_ticks(1, bank.last_id());
+            blocktree.write_entries(1, 0, 0, next_tick.clone()).unwrap();
+
+            let received_tick = ledger_writer_recv
+                .recv()
+                .expect("Expected to receive an entry on the ledger writer receiver");
+
+            assert_eq!(next_tick[0], received_tick[0]);
+
+            replay_stage
+                .close()
+                .expect("Expect successful ReplayStage exit");
+        }
+        let _ignored = remove_dir_all(&my_ledger_path);
+    }
+
+    #[test]
+    fn test_replay_stage_poh_ok_entry_receiver() {
+        let (forward_entry_sender, forward_entry_receiver) = channel();
+        let genesis_block = GenesisBlock::new(10_000).0;
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let mut last_id = bank.last_id();
+        let mut entries = Vec::new();
+        for _ in 0..5 {
+            let entry = next_entry_mut(&mut last_id, 1, vec![]); //just ticks
+            entries.push(entry);
+        }
+
+        let mut progress = HashMap::new();
+        let res = ReplayStage::replay_entries_into_bank(
+            &bank,
+            entries.clone(),
+            &mut progress,
+            &forward_entry_sender,
+            0,
+        );
+        assert!(res.is_ok(), "replay failed {:?}", res);
+        let res = forward_entry_receiver.try_recv();
+        match res {
+            Ok(_) => (),
+            Err(e) => assert!(false, "Entries were not sent correctly {:?}", e),
+        }
+    }
+
+    #[test]
+    fn test_replay_stage_poh_error_entry_receiver() {
+        let (forward_entry_sender, forward_entry_receiver) = channel();
+        let mut entries = Vec::new();
+        for _ in 0..5 {
+            let entry = Entry::new(&mut Hash::default(), 1, vec![]); //just broken entries
+            entries.push(entry);
+        }
+
+        let genesis_block = GenesisBlock::new(10_000).0;
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let mut progress = HashMap::new();
+        let res = ReplayStage::replay_entries_into_bank(
+            &bank,
+            entries.clone(),
+            &mut progress,
+            &forward_entry_sender,
+            0,
+        );
+
+        match res {
+            Ok(_) => assert!(false, "Should have failed because entries are broken"),
+            Err(Error::BlobError(BlobError::VerificationFailed)) => (),
+            Err(e) => assert!(
+                false,
+                "Should have failed because with blob error, instead, got {:?}",
+                e
+            ),
+        }
+        assert!(forward_entry_receiver.try_recv().is_err());
+    }
+}

core/src/replicator.rs

@@ -0,0 +1,465 @@
+use crate::blob_fetch_stage::BlobFetchStage;
+use crate::blocktree::Blocktree;
+use crate::blocktree_processor;
+#[cfg(feature = "chacha")]
+use crate::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
+use crate::client::mk_client;
+use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
+use crate::gossip_service::GossipService;
+use crate::result::{self, Result};
+use crate::rpc_request::{RpcClient, RpcRequest, RpcRequestHandler};
+use crate::service::Service;
+use crate::storage_stage::{get_segment_from_entry, ENTRIES_PER_SEGMENT};
+use crate::streamer::BlobReceiver;
+use crate::thin_client::{retry_get_balance, ThinClient};
+use crate::window_service::WindowService;
+use rand::thread_rng;
+use rand::Rng;
+use solana_drone::drone::{request_airdrop_transaction, DRONE_PORT};
+use solana_sdk::genesis_block::GenesisBlock;
+use solana_sdk::hash::{Hash, Hasher};
+use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
+use solana_sdk::storage_program::StorageTransaction;
+use std::fs::File;
+use std::io;
+use std::io::BufReader;
+use std::io::Read;
+use std::io::Seek;
+use std::io::SeekFrom;
+use std::io::{Error, ErrorKind};
+use std::mem::size_of;
+use std::net::UdpSocket;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::time::{Duration, Instant};
+
+pub struct Replicator {
+    gossip_service: GossipService,
+    fetch_stage: BlobFetchStage,
+    window_service: WindowService,
+    pub retransmit_receiver: BlobReceiver,
+    exit: Arc<AtomicBool>,
+    entry_height: u64,
+}
+
+pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
+    let in_file = File::open(in_path)?;
+    let metadata = in_file.metadata()?;
+    let mut buffer_file = BufReader::new(in_file);
+
+    let mut hasher = Hasher::default();
+    let sample_size = size_of::<Hash>();
+    let sample_size64 = sample_size as u64;
+    let mut buf = vec![0; sample_size];
+
+    let file_len = metadata.len();
+    if file_len < sample_size64 {
+        return Err(Error::new(ErrorKind::Other, "file too short!"));
+    }
+    for offset in sample_offsets {
+        if *offset > (file_len - sample_size64) / sample_size64 {
+            return Err(Error::new(ErrorKind::Other, "offset too large"));
+        }
+        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
+        trace!("sampling @ {} ", *offset);
+        match buffer_file.read(&mut buf) {
+            Ok(size) => {
+                assert_eq!(size, buf.len());
+                hasher.hash(&buf);
+            }
+            Err(e) => {
+                warn!("Error sampling file");
+                return Err(e);
+            }
+        }
+    }
+
+    Ok(hasher.result())
+}
+
+fn get_entry_heights_from_last_id(
+    signature: &ring::signature::Signature,
+    storage_entry_height: u64,
+) -> u64 {
+    let signature_vec = signature.as_ref();
+    let mut segment_index = u64::from(signature_vec[0])
+        | (u64::from(signature_vec[1]) << 8)
+        | (u64::from(signature_vec[1]) << 16)
+        | (u64::from(signature_vec[2]) << 24);
+    let max_segment_index = get_segment_from_entry(storage_entry_height);
+    segment_index %= max_segment_index as u64;
+    segment_index * ENTRIES_PER_SEGMENT
+}
+
+impl Replicator {
+    /// Returns a Result that contains a replicator on success
+    ///
+    /// # Arguments
+    /// * `ledger_path` - path to where the ledger will be stored.
+    /// Causes panic if none
+    /// * `node` - The replicator node
+    /// * `leader_info` - NodeInfo representing the leader
+    /// * `keypair` - Keypair for this replicator
+    /// * `timeout` - (optional) timeout for polling for leader/downloading the ledger. Defaults to
+    /// 30 seconds
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        ledger_path: &str,
+        node: Node,
+        leader_info: &NodeInfo,
+        keypair: &Keypair,
+        timeout: Option<Duration>,
+    ) -> Result<Self> {
+        let exit = Arc::new(AtomicBool::new(false));
+        let timeout = timeout.unwrap_or_else(|| Duration::new(30, 0));
+
+        info!("Replicator: id: {}", keypair.pubkey());
+        info!("Creating cluster info....");
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(node.info.clone())));
+
+        let leader_pubkey = leader_info.id;
+        {
+            let mut cluster_info_w = cluster_info.write().unwrap();
+            cluster_info_w.insert_info(leader_info.clone());
+            cluster_info_w.set_leader(leader_pubkey);
+        }
+
+        // Create Blocktree, eventually will simply repurpose the input
+        // ledger path as the Blocktree path once we replace the ledger with
+        // Blocktree. Note for now, this ledger will not contain any of the existing entries
+        // in the ledger located at ledger_path, and will only append on newly received
+        // entries after being passed to window_service
+        let blocktree =
+            Blocktree::open(ledger_path).expect("Expected to be able to open database ledger");
+
+        let genesis_block =
+            GenesisBlock::load(ledger_path).expect("Expected to successfully open genesis block");
+
+        let (_bank_forks, _bank_forks_info) =
+            blocktree_processor::process_blocktree(&genesis_block, &blocktree, None)
+                .expect("process_blocktree failed");
+
+        let blocktree = Arc::new(blocktree);
+
+        //TODO(sagar) Does replicator need a bank also ?
+        let gossip_service = GossipService::new(
+            &cluster_info,
+            Some(blocktree.clone()),
+            None,
+            node.sockets.gossip,
+            exit.clone(),
+        );
+
+        info!("polling for leader");
+        let leader = Self::poll_for_leader(&cluster_info, timeout)?;
+
+        info!("Got leader: {:?}", leader);
+
+        let (storage_last_id, storage_entry_height) =
+            Self::poll_for_last_id_and_entry_height(&cluster_info)?;
+
+        let signature = keypair.sign(storage_last_id.as_ref());
+        let entry_height = get_entry_heights_from_last_id(&signature, storage_entry_height);
+
+        info!("replicating entry_height: {}", entry_height);
+
+        let repair_socket = Arc::new(node.sockets.repair);
+        let mut blob_sockets: Vec<Arc<UdpSocket>> =
+            node.sockets.tvu.into_iter().map(Arc::new).collect();
+        blob_sockets.push(repair_socket.clone());
+        let (blob_fetch_sender, blob_fetch_receiver) = channel();
+        let fetch_stage =
+            BlobFetchStage::new_multi_socket(blob_sockets, &blob_fetch_sender, exit.clone());
+
+        // todo: pull blobs off the retransmit_receiver and recycle them?
+        let (retransmit_sender, retransmit_receiver) = channel();
+
+        let window_service = WindowService::new(
+            blocktree.clone(),
+            cluster_info.clone(),
+            blob_fetch_receiver,
+            retransmit_sender,
+            repair_socket,
+            exit.clone(),
+        );
+
+        info!("window created, waiting for ledger download done");
+        let _start = Instant::now();
+        let mut _received_so_far = 0;
+
+        /*while !done.load(Ordering::Relaxed) {
+            sleep(Duration::from_millis(100));
+
+            let elapsed = start.elapsed();
+            received_so_far += entry_receiver.try_recv().map(|v| v.len()).unwrap_or(0);
+
+            if received_so_far == 0 && elapsed > timeout {
+                return Err(result::Error::IO(io::Error::new(
+                    ErrorKind::TimedOut,
+                    "Timed out waiting to receive any blocks",
+                )));
+            }
+        }*/
+
+        info!("Done receiving entries from window_service");
+
+        let mut node_info = node.info.clone();
+        node_info.tvu = "0.0.0.0:0".parse().unwrap();
+        {
+            let mut cluster_info_w = cluster_info.write().unwrap();
+            cluster_info_w.insert_info(node_info);
+        }
+
+        let mut client = mk_client(&leader);
+
+        Self::get_airdrop_tokens(&mut client, keypair, &leader_info);
+        info!("Done downloading ledger at {}", ledger_path);
+
+        let ledger_path = Path::new(ledger_path);
+        let ledger_data_file_encrypted = ledger_path.join("ledger.enc");
+        let mut sampling_offsets = Vec::new();
+
+        #[cfg(not(feature = "chacha"))]
+        sampling_offsets.push(0);
+
+        #[cfg(feature = "chacha")]
+        {
+            use crate::storage_stage::NUM_STORAGE_SAMPLES;
+            use rand::{Rng, SeedableRng};
+            use rand_chacha::ChaChaRng;
+
+            let mut ivec = [0u8; 64];
+            ivec.copy_from_slice(signature.as_ref());
+
+            let num_encrypted_bytes = chacha_cbc_encrypt_ledger(
+                &blocktree,
+                entry_height,
+                &ledger_data_file_encrypted,
+                &mut ivec,
+            )?;
+
+            let num_chacha_blocks = num_encrypted_bytes / CHACHA_BLOCK_SIZE;
+            let mut rng_seed = [0u8; 32];
+            rng_seed.copy_from_slice(&signature.as_ref()[0..32]);
+            let mut rng = ChaChaRng::from_seed(rng_seed);
+            for _ in 0..NUM_STORAGE_SAMPLES {
+                sampling_offsets.push(rng.gen_range(0, num_chacha_blocks) as u64);
+            }
+        }
+
+        info!("Done encrypting the ledger");
+
+        match sample_file(&ledger_data_file_encrypted, &sampling_offsets) {
+            Ok(hash) => {
+                let last_id = client.get_last_id();
+                info!("sampled hash: {}", hash);
+                let mut tx = StorageTransaction::new_mining_proof(
+                    &keypair,
+                    hash,
+                    last_id,
+                    entry_height,
+                    Signature::new(signature.as_ref()),
+                );
+                client
+                    .retry_transfer(&keypair, &mut tx, 10)
+                    .expect("transfer didn't work!");
+            }
+            Err(e) => info!("Error occurred while sampling: {:?}", e),
+        }
+
+        Ok(Self {
+            gossip_service,
+            fetch_stage,
+            window_service,
+            retransmit_receiver,
+            exit,
+            entry_height,
+        })
+    }
+
+    pub fn close(self) {
+        self.exit.store(true, Ordering::Relaxed);
+        self.join()
+    }
+
+    pub fn join(self) {
+        self.gossip_service.join().unwrap();
+        self.fetch_stage.join().unwrap();
+        self.window_service.join().unwrap();
+
+        // Drain the queue here to prevent self.retransmit_receiver from being dropped
+        // before the window_service thread is joined
+        let mut retransmit_queue_count = 0;
+        while let Ok(_blob) = self.retransmit_receiver.recv_timeout(Duration::new(1, 0)) {
+            retransmit_queue_count += 1;
+        }
+        debug!("retransmit channel count: {}", retransmit_queue_count);
+    }
+
+    pub fn entry_height(&self) -> u64 {
+        self.entry_height
+    }
+
+    fn poll_for_leader(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        timeout: Duration,
+    ) -> Result<NodeInfo> {
+        let start = Instant::now();
+        loop {
+            if let Some(l) = cluster_info.read().unwrap().get_gossip_top_leader() {
+                return Ok(l.clone());
+            }
+
+            let elapsed = start.elapsed();
+            if elapsed > timeout {
+                return Err(result::Error::IO(io::Error::new(
+                    ErrorKind::TimedOut,
+                    "Timed out waiting to receive any blocks",
+                )));
+            }
+
+            sleep(Duration::from_millis(900));
+            info!("{}", cluster_info.read().unwrap().node_info_trace());
+        }
+    }
+
+    fn poll_for_last_id_and_entry_height(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+    ) -> Result<(String, u64)> {
+        for _ in 0..10 {
+            let rpc_client = {
+                let cluster_info = cluster_info.read().unwrap();
+                let rpc_peers = cluster_info.rpc_peers();
+                debug!("rpc peers: {:?}", rpc_peers);
+                let node_idx = thread_rng().gen_range(0, rpc_peers.len());
+                RpcClient::new_from_socket(rpc_peers[node_idx].rpc)
+            };
+
+            let storage_last_id = rpc_client
+                .make_rpc_request(2, RpcRequest::GetStorageMiningLastId, None)
+                .expect("rpc request")
+                .to_string();
+            let storage_entry_height = rpc_client
+                .make_rpc_request(2, RpcRequest::GetStorageMiningEntryHeight, None)
+                .expect("rpc request")
+                .as_u64()
+                .unwrap();
+            if get_segment_from_entry(storage_entry_height) != 0 {
+                return Ok((storage_last_id, storage_entry_height));
+            }
+            info!("max entry_height: {}", storage_entry_height);
+            sleep(Duration::from_secs(3));
+        }
+        Err(Error::new(
+            ErrorKind::Other,
+            "Couldn't get last_id or entry_height",
+        ))?
+    }
+
+    fn get_airdrop_tokens(client: &mut ThinClient, keypair: &Keypair, leader_info: &NodeInfo) {
+        if retry_get_balance(client, &keypair.pubkey(), None).is_none() {
+            let mut drone_addr = leader_info.tpu;
+            drone_addr.set_port(DRONE_PORT);
+
+            let airdrop_amount = 1;
+
+            let last_id = client.get_last_id();
+            match request_airdrop_transaction(
+                &drone_addr,
+                &keypair.pubkey(),
+                airdrop_amount,
+                last_id,
+            ) {
+                Ok(transaction) => {
+                    let signature = client.transfer_signed(&transaction).unwrap();
+                    client.poll_for_signature(&signature).unwrap();
+                }
+                Err(err) => {
+                    panic!(
+                        "Error requesting airdrop: {:?} to addr: {:?} amount: {}",
+                        err, drone_addr, airdrop_amount
+                    );
+                }
+            };
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::replicator::sample_file;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use std::fs::File;
+    use std::fs::{create_dir_all, remove_file};
+    use std::io::Write;
+    use std::mem::size_of;
+    use std::path::PathBuf;
+
+    fn tmp_file_path(name: &str) -> PathBuf {
+        use std::env;
+        let out_dir = env::var("OUT_DIR").unwrap_or_else(|_| "target".to_string());
+        let keypair = Keypair::new();
+
+        let mut path = PathBuf::new();
+        path.push(out_dir);
+        path.push("tmp");
+        create_dir_all(&path).unwrap();
+
+        path.push(format!("{}-{}", name, keypair.pubkey()));
+        path
+    }
+
+    #[test]
+    fn test_sample_file() {
+        solana_logger::setup();
+        let in_path = tmp_file_path("test_sample_file_input.txt");
+        let num_strings = 4096;
+        let string = "12foobar";
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..num_strings {
+                in_file.write(string.as_bytes()).unwrap();
+            }
+        }
+        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
+        let samples: Vec<_> = (0..num_samples).collect();
+        let res = sample_file(&in_path, samples.as_slice());
+        let ref_hash: Hash = Hash::new(&[
+            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
+            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
+        ]);
+        let res = res.unwrap();
+        assert_eq!(res, ref_hash);
+
+        // Sample just past the end
+        assert!(sample_file(&in_path, &[num_samples]).is_err());
+        remove_file(&in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_invalid_offset() {
+        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..4096 {
+                in_file.write("123456foobar".as_bytes()).unwrap();
+            }
+        }
+        let samples = [0, 200000];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+        remove_file(in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_missing_file() {
+        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
+        let samples = [0, 5];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+    }
+}

core/src/result.rs

@@ -0,0 +1,180 @@
+//! The `result` module exposes a Result type that propagates one of many different Error types.
+
+use crate::blocktree;
+use crate::cluster_info;
+#[cfg(feature = "erasure")]
+use crate::erasure;
+use crate::packet;
+use crate::poh_recorder;
+use bincode;
+use serde_json;
+use solana_runtime::bank;
+use std;
+use std::any::Any;
+
+#[derive(Debug)]
+pub enum Error {
+    IO(std::io::Error),
+    JSON(serde_json::Error),
+    AddrParse(std::net::AddrParseError),
+    JoinError(Box<dyn Any + Send + 'static>),
+    RecvError(std::sync::mpsc::RecvError),
+    RecvTimeoutError(std::sync::mpsc::RecvTimeoutError),
+    TryRecvError(std::sync::mpsc::TryRecvError),
+    Serialize(std::boxed::Box<bincode::ErrorKind>),
+    BankError(bank::BankError),
+    ClusterInfoError(cluster_info::ClusterInfoError),
+    BlobError(packet::BlobError),
+    #[cfg(feature = "erasure")]
+    ErasureError(erasure::ErasureError),
+    SendError,
+    PohRecorderError(poh_recorder::PohRecorderError),
+    BlocktreeError(blocktree::BlocktreeError),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "solana error")
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl std::convert::From<std::sync::mpsc::RecvError> for Error {
+    fn from(e: std::sync::mpsc::RecvError) -> Error {
+        Error::RecvError(e)
+    }
+}
+impl std::convert::From<std::sync::mpsc::TryRecvError> for Error {
+    fn from(e: std::sync::mpsc::TryRecvError) -> Error {
+        Error::TryRecvError(e)
+    }
+}
+impl std::convert::From<std::sync::mpsc::RecvTimeoutError> for Error {
+    fn from(e: std::sync::mpsc::RecvTimeoutError) -> Error {
+        Error::RecvTimeoutError(e)
+    }
+}
+impl std::convert::From<bank::BankError> for Error {
+    fn from(e: bank::BankError) -> Error {
+        Error::BankError(e)
+    }
+}
+impl std::convert::From<cluster_info::ClusterInfoError> for Error {
+    fn from(e: cluster_info::ClusterInfoError) -> Error {
+        Error::ClusterInfoError(e)
+    }
+}
+#[cfg(feature = "erasure")]
+impl std::convert::From<erasure::ErasureError> for Error {
+    fn from(e: erasure::ErasureError) -> Error {
+        Error::ErasureError(e)
+    }
+}
+impl<T> std::convert::From<std::sync::mpsc::SendError<T>> for Error {
+    fn from(_e: std::sync::mpsc::SendError<T>) -> Error {
+        Error::SendError
+    }
+}
+impl std::convert::From<Box<dyn Any + Send + 'static>> for Error {
+    fn from(e: Box<dyn Any + Send + 'static>) -> Error {
+        Error::JoinError(e)
+    }
+}
+impl std::convert::From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Error {
+        Error::IO(e)
+    }
+}
+impl std::convert::From<serde_json::Error> for Error {
+    fn from(e: serde_json::Error) -> Error {
+        Error::JSON(e)
+    }
+}
+impl std::convert::From<std::net::AddrParseError> for Error {
+    fn from(e: std::net::AddrParseError) -> Error {
+        Error::AddrParse(e)
+    }
+}
+impl std::convert::From<std::boxed::Box<bincode::ErrorKind>> for Error {
+    fn from(e: std::boxed::Box<bincode::ErrorKind>) -> Error {
+        Error::Serialize(e)
+    }
+}
+impl std::convert::From<poh_recorder::PohRecorderError> for Error {
+    fn from(e: poh_recorder::PohRecorderError) -> Error {
+        Error::PohRecorderError(e)
+    }
+}
+impl std::convert::From<blocktree::BlocktreeError> for Error {
+    fn from(e: blocktree::BlocktreeError) -> Error {
+        Error::BlocktreeError(e)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::result::Error;
+    use crate::result::Result;
+    use serde_json;
+    use std::io;
+    use std::io::Write;
+    use std::net::SocketAddr;
+    use std::panic;
+    use std::sync::mpsc::channel;
+    use std::sync::mpsc::RecvError;
+    use std::sync::mpsc::RecvTimeoutError;
+    use std::thread;
+
+    fn addr_parse_error() -> Result<SocketAddr> {
+        let r = "12fdfasfsafsadfs".parse()?;
+        Ok(r)
+    }
+
+    fn join_error() -> Result<()> {
+        panic::set_hook(Box::new(|_info| {}));
+        let r = thread::spawn(|| panic!("hi")).join()?;
+        Ok(r)
+    }
+    fn json_error() -> Result<()> {
+        let r = serde_json::from_slice("=342{;;;;:}".as_bytes())?;
+        Ok(r)
+    }
+    fn send_error() -> Result<()> {
+        let (s, r) = channel();
+        drop(r);
+        s.send(())?;
+        Ok(())
+    }
+
+    #[test]
+    fn from_test() {
+        assert_matches!(addr_parse_error(), Err(Error::AddrParse(_)));
+        assert_matches!(Error::from(RecvError {}), Error::RecvError(_));
+        assert_matches!(
+            Error::from(RecvTimeoutError::Timeout),
+            Error::RecvTimeoutError(_)
+        );
+        assert_matches!(send_error(), Err(Error::SendError));
+        assert_matches!(join_error(), Err(Error::JoinError(_)));
+        let ioe = io::Error::new(io::ErrorKind::NotFound, "hi");
+        assert_matches!(Error::from(ioe), Error::IO(_));
+    }
+    #[test]
+    fn fmt_test() {
+        write!(io::sink(), "{:?}", addr_parse_error()).unwrap();
+        write!(io::sink(), "{:?}", Error::from(RecvError {})).unwrap();
+        write!(io::sink(), "{:?}", Error::from(RecvTimeoutError::Timeout)).unwrap();
+        write!(io::sink(), "{:?}", send_error()).unwrap();
+        write!(io::sink(), "{:?}", join_error()).unwrap();
+        write!(io::sink(), "{:?}", json_error()).unwrap();
+        write!(
+            io::sink(),
+            "{:?}",
+            Error::from(io::Error::new(io::ErrorKind::NotFound, "hi"))
+        )
+        .unwrap();
+    }
+}

core/src/retransmit_stage.rs

@@ -0,0 +1,167 @@
+//! The `retransmit_stage` retransmits blobs between validators
+
+use crate::bank_forks::BankForks;
+use crate::blocktree::Blocktree;
+use crate::cluster_info::{
+    compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT, GROW_LAYER_CAPACITY,
+    NEIGHBORHOOD_SIZE,
+};
+use crate::packet::SharedBlob;
+use crate::result::{Error, Result};
+use crate::service::Service;
+use crate::staking_utils;
+use crate::streamer::BlobReceiver;
+use crate::window_service::WindowService;
+use solana_metrics::counter::Counter;
+use solana_metrics::{influxdb, submit};
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::channel;
+use std::sync::mpsc::RecvTimeoutError;
+use std::sync::{Arc, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+
+fn retransmit(
+    bank_forks: &Arc<RwLock<BankForks>>,
+    cluster_info: &Arc<RwLock<ClusterInfo>>,
+    r: &BlobReceiver,
+    sock: &UdpSocket,
+) -> Result<()> {
+    let timer = Duration::new(1, 0);
+    let mut dq = r.recv_timeout(timer)?;
+    while let Ok(mut nq) = r.try_recv() {
+        dq.append(&mut nq);
+    }
+
+    submit(
+        influxdb::Point::new("retransmit-stage")
+            .add_field("count", influxdb::Value::Integer(dq.len() as i64))
+            .to_owned(),
+    );
+    let (neighbors, children) = compute_retransmit_peers(
+        &staking_utils::node_stakes(&bank_forks.read().unwrap().working_bank()),
+        cluster_info,
+        DATA_PLANE_FANOUT,
+        NEIGHBORHOOD_SIZE,
+        GROW_LAYER_CAPACITY,
+    );
+    for b in &dq {
+        if b.read().unwrap().should_forward() {
+            ClusterInfo::retransmit_to(&cluster_info, &neighbors, &copy_for_neighbors(b), sock)?;
+        }
+        // Always send blobs to children
+        ClusterInfo::retransmit_to(&cluster_info, &children, b, sock)?;
+    }
+    Ok(())
+}
+
+/// Modifies a blob for neighbors nodes
+#[inline]
+fn copy_for_neighbors(b: &SharedBlob) -> SharedBlob {
+    let mut blob = b.read().unwrap().clone();
+    // Disable blob forwarding for neighbors
+    blob.forward(false);
+    Arc::new(RwLock::new(blob))
+}
+
+/// Service to retransmit messages from the leader or layer 1 to relevant peer nodes.
+/// See `cluster_info` for network layer definitions.
+/// # Arguments
+/// * `sock` - Socket to read from.  Read timeout is set to 1.
+/// * `exit` - Boolean to signal system exit.
+/// * `cluster_info` - This structure needs to be updated and populated by the bank and via gossip.
+/// * `recycler` - Blob recycler.
+/// * `r` - Receive channel for blobs to be retransmitted to all the layer 1 nodes.
+fn retransmitter(
+    sock: Arc<UdpSocket>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    cluster_info: Arc<RwLock<ClusterInfo>>,
+    r: BlobReceiver,
+) -> JoinHandle<()> {
+    Builder::new()
+        .name("solana-retransmitter".to_string())
+        .spawn(move || {
+            trace!("retransmitter started");
+            loop {
+                if let Err(e) = retransmit(&bank_forks, &cluster_info, &r, &sock) {
+                    match e {
+                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                        _ => {
+                            inc_new_counter_info!("streamer-retransmit-error", 1, 1);
+                        }
+                    }
+                }
+            }
+            trace!("exiting retransmitter");
+        })
+        .unwrap()
+}
+
+pub struct RetransmitStage {
+    thread_hdls: Vec<JoinHandle<()>>,
+    window_service: WindowService,
+}
+
+impl RetransmitStage {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        bank_forks: &Arc<RwLock<BankForks>>,
+        blocktree: Arc<Blocktree>,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        retransmit_socket: Arc<UdpSocket>,
+        repair_socket: Arc<UdpSocket>,
+        fetch_stage_receiver: BlobReceiver,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let (retransmit_sender, retransmit_receiver) = channel();
+
+        let t_retransmit = retransmitter(
+            retransmit_socket,
+            bank_forks.clone(),
+            cluster_info.clone(),
+            retransmit_receiver,
+        );
+        let window_service = WindowService::new(
+            blocktree,
+            cluster_info.clone(),
+            fetch_stage_receiver,
+            retransmit_sender,
+            repair_socket,
+            exit,
+        );
+
+        let thread_hdls = vec![t_retransmit];
+        Self {
+            thread_hdls,
+            window_service,
+        }
+    }
+}
+
+impl Service for RetransmitStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        self.window_service.join()?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Test that blobs always come out with forward unset for neighbors
+    #[test]
+    fn test_blob_for_neighbors() {
+        let blob = SharedBlob::default();
+        blob.write().unwrap().forward(true);
+        let for_hoodies = copy_for_neighbors(&blob);
+        assert!(!for_hoodies.read().unwrap().should_forward());
+    }
+}

core/src/rpc.rs

@@ -0,0 +1,646 @@
+//! The `rpc` module implements the Solana RPC interface.
+
+use crate::cluster_info::ClusterInfo;
+use crate::packet::PACKET_DATA_SIZE;
+use crate::rpc_status::RpcSignatureStatus;
+use crate::storage_stage::StorageState;
+use bincode::{deserialize, serialize};
+use bs58;
+use jsonrpc_core::{Error, ErrorCode, Metadata, Result};
+use jsonrpc_derive::rpc;
+use solana_drone::drone::request_airdrop_transaction;
+use solana_runtime::bank::{self, Bank, BankError};
+use solana_sdk::account::Account;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Signature;
+use solana_sdk::transaction::Transaction;
+use std::mem;
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::time::{Duration, Instant};
+
+#[derive(Clone)]
+pub struct JsonRpcRequestProcessor {
+    bank: Option<Arc<Bank>>,
+    storage_state: StorageState,
+}
+
+impl JsonRpcRequestProcessor {
+    fn bank(&self) -> Result<&Arc<Bank>> {
+        self.bank.as_ref().ok_or(Error {
+            code: ErrorCode::InternalError,
+            message: "No bank available".into(),
+            data: None,
+        })
+    }
+
+    pub fn set_bank(&mut self, bank: &Arc<Bank>) {
+        self.bank = Some(bank.clone());
+    }
+
+    pub fn new(storage_state: StorageState) -> Self {
+        JsonRpcRequestProcessor {
+            bank: None,
+            storage_state,
+        }
+    }
+
+    pub fn get_account_info(&self, pubkey: Pubkey) -> Result<Account> {
+        self.bank()?
+            .get_account(&pubkey)
+            .ok_or_else(Error::invalid_request)
+    }
+
+    pub fn get_balance(&self, pubkey: Pubkey) -> Result<u64> {
+        let val = self.bank()?.get_balance(&pubkey);
+        Ok(val)
+    }
+
+    fn get_last_id(&self) -> Result<String> {
+        let id = self.bank()?.last_id();
+        Ok(bs58::encode(id).into_string())
+    }
+
+    pub fn get_signature_status(&self, signature: Signature) -> Option<bank::Result<()>> {
+        self.bank()
+            .ok()
+            .and_then(|bank| bank.get_signature_status(&signature))
+    }
+
+    fn get_transaction_count(&self) -> Result<u64> {
+        Ok(self.bank()?.transaction_count() as u64)
+    }
+
+    fn get_storage_mining_last_id(&self) -> Result<String> {
+        let id = self.storage_state.get_last_id();
+        Ok(bs58::encode(id).into_string())
+    }
+
+    fn get_storage_mining_entry_height(&self) -> Result<u64> {
+        let entry_height = self.storage_state.get_entry_height();
+        Ok(entry_height)
+    }
+
+    fn get_storage_pubkeys_for_entry_height(&self, entry_height: u64) -> Result<Vec<Pubkey>> {
+        Ok(self
+            .storage_state
+            .get_pubkeys_for_entry_height(entry_height))
+    }
+}
+
+fn get_leader_addr(cluster_info: &Arc<RwLock<ClusterInfo>>) -> Result<SocketAddr> {
+    if let Some(leader_data) = cluster_info.read().unwrap().leader_data() {
+        Ok(leader_data.tpu)
+    } else {
+        Err(Error {
+            code: ErrorCode::InternalError,
+            message: "No leader detected".into(),
+            data: None,
+        })
+    }
+}
+
+fn verify_pubkey(input: String) -> Result<Pubkey> {
+    let pubkey_vec = bs58::decode(input).into_vec().map_err(|err| {
+        info!("verify_pubkey: invalid input: {:?}", err);
+        Error::invalid_request()
+    })?;
+    if pubkey_vec.len() != mem::size_of::<Pubkey>() {
+        info!(
+            "verify_pubkey: invalid pubkey_vec length: {}",
+            pubkey_vec.len()
+        );
+        Err(Error::invalid_request())
+    } else {
+        Ok(Pubkey::new(&pubkey_vec))
+    }
+}
+
+fn verify_signature(input: &str) -> Result<Signature> {
+    let signature_vec = bs58::decode(input).into_vec().map_err(|err| {
+        info!("verify_signature: invalid input: {}: {:?}", input, err);
+        Error::invalid_request()
+    })?;
+    if signature_vec.len() != mem::size_of::<Signature>() {
+        info!(
+            "verify_signature: invalid signature_vec length: {}",
+            signature_vec.len()
+        );
+        Err(Error::invalid_request())
+    } else {
+        Ok(Signature::new(&signature_vec))
+    }
+}
+
+#[derive(Clone)]
+pub struct Meta {
+    pub request_processor: Arc<RwLock<JsonRpcRequestProcessor>>,
+    pub cluster_info: Arc<RwLock<ClusterInfo>>,
+    pub rpc_addr: SocketAddr,
+    pub drone_addr: SocketAddr,
+}
+impl Metadata for Meta {}
+
+#[rpc]
+pub trait RpcSol {
+    type Metadata;
+
+    #[rpc(meta, name = "confirmTransaction")]
+    fn confirm_transaction(&self, _: Self::Metadata, _: String) -> Result<bool>;
+
+    #[rpc(meta, name = "getAccountInfo")]
+    fn get_account_info(&self, _: Self::Metadata, _: String) -> Result<Account>;
+
+    #[rpc(meta, name = "getBalance")]
+    fn get_balance(&self, _: Self::Metadata, _: String) -> Result<u64>;
+
+    #[rpc(meta, name = "getLastId")]
+    fn get_last_id(&self, _: Self::Metadata) -> Result<String>;
+
+    #[rpc(meta, name = "getSignatureStatus")]
+    fn get_signature_status(&self, _: Self::Metadata, _: String) -> Result<RpcSignatureStatus>;
+
+    #[rpc(meta, name = "getTransactionCount")]
+    fn get_transaction_count(&self, _: Self::Metadata) -> Result<u64>;
+
+    #[rpc(meta, name = "requestAirdrop")]
+    fn request_airdrop(&self, _: Self::Metadata, _: String, _: u64) -> Result<String>;
+
+    #[rpc(meta, name = "sendTransaction")]
+    fn send_transaction(&self, _: Self::Metadata, _: Vec<u8>) -> Result<String>;
+
+    #[rpc(meta, name = "getStorageMiningLastId")]
+    fn get_storage_mining_last_id(&self, _: Self::Metadata) -> Result<String>;
+
+    #[rpc(meta, name = "getStorageMiningEntryHeight")]
+    fn get_storage_mining_entry_height(&self, _: Self::Metadata) -> Result<u64>;
+
+    #[rpc(meta, name = "getStoragePubkeysForEntryHeight")]
+    fn get_storage_pubkeys_for_entry_height(
+        &self,
+        _: Self::Metadata,
+        _: u64,
+    ) -> Result<Vec<Pubkey>>;
+}
+
+pub struct RpcSolImpl;
+impl RpcSol for RpcSolImpl {
+    type Metadata = Meta;
+
+    fn confirm_transaction(&self, meta: Self::Metadata, id: String) -> Result<bool> {
+        info!("confirm_transaction rpc request received: {:?}", id);
+        self.get_signature_status(meta, id)
+            .map(|status| status == RpcSignatureStatus::Confirmed)
+    }
+
+    fn get_account_info(&self, meta: Self::Metadata, id: String) -> Result<Account> {
+        info!("get_account_info rpc request received: {:?}", id);
+        let pubkey = verify_pubkey(id)?;
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_account_info(pubkey)
+    }
+
+    fn get_balance(&self, meta: Self::Metadata, id: String) -> Result<u64> {
+        info!("get_balance rpc request received: {:?}", id);
+        let pubkey = verify_pubkey(id)?;
+        meta.request_processor.read().unwrap().get_balance(pubkey)
+    }
+
+    fn get_last_id(&self, meta: Self::Metadata) -> Result<String> {
+        info!("get_last_id rpc request received");
+        meta.request_processor.read().unwrap().get_last_id()
+    }
+
+    fn get_signature_status(&self, meta: Self::Metadata, id: String) -> Result<RpcSignatureStatus> {
+        info!("get_signature_status rpc request received: {:?}", id);
+        let signature = verify_signature(&id)?;
+        let res = meta
+            .request_processor
+            .read()
+            .unwrap()
+            .get_signature_status(signature);
+
+        let status = {
+            if res.is_none() {
+                RpcSignatureStatus::SignatureNotFound
+            } else {
+                match res.unwrap() {
+                    Ok(_) => RpcSignatureStatus::Confirmed,
+                    Err(BankError::AccountInUse) => RpcSignatureStatus::AccountInUse,
+                    Err(BankError::AccountLoadedTwice) => RpcSignatureStatus::AccountLoadedTwice,
+                    Err(BankError::ProgramError(_, _)) => RpcSignatureStatus::ProgramRuntimeError,
+                    Err(err) => {
+                        trace!("mapping {:?} to GenericFailure", err);
+                        RpcSignatureStatus::GenericFailure
+                    }
+                }
+            }
+        };
+        info!("get_signature_status rpc request status: {:?}", status);
+        Ok(status)
+    }
+
+    fn get_transaction_count(&self, meta: Self::Metadata) -> Result<u64> {
+        info!("get_transaction_count rpc request received");
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_transaction_count()
+    }
+
+    fn request_airdrop(&self, meta: Self::Metadata, id: String, tokens: u64) -> Result<String> {
+        trace!("request_airdrop id={} tokens={}", id, tokens);
+        let pubkey = verify_pubkey(id)?;
+
+        let last_id = meta.request_processor.read().unwrap().bank()?.last_id();
+        let transaction = request_airdrop_transaction(&meta.drone_addr, &pubkey, tokens, last_id)
+            .map_err(|err| {
+            info!("request_airdrop_transaction failed: {:?}", err);
+            Error::internal_error()
+        })?;;
+
+        let data = serialize(&transaction).map_err(|err| {
+            info!("request_airdrop: serialize error: {:?}", err);
+            Error::internal_error()
+        })?;
+
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let transactions_addr = get_leader_addr(&meta.cluster_info)?;
+        transactions_socket
+            .send_to(&data, transactions_addr)
+            .map_err(|err| {
+                info!("request_airdrop: send_to error: {:?}", err);
+                Error::internal_error()
+            })?;
+
+        let signature = transaction.signatures[0];
+        let now = Instant::now();
+        let mut signature_status;
+        loop {
+            signature_status = meta
+                .request_processor
+                .read()
+                .unwrap()
+                .get_signature_status(signature);
+
+            if signature_status == Some(Ok(())) {
+                info!("airdrop signature ok");
+                return Ok(bs58::encode(signature).into_string());
+            } else if now.elapsed().as_secs() > 5 {
+                info!("airdrop signature timeout");
+                return Err(Error::internal_error());
+            }
+            sleep(Duration::from_millis(100));
+        }
+    }
+
+    fn send_transaction(&self, meta: Self::Metadata, data: Vec<u8>) -> Result<String> {
+        let tx: Transaction = deserialize(&data).map_err(|err| {
+            info!("send_transaction: deserialize error: {:?}", err);
+            Error::invalid_request()
+        })?;
+        if data.len() >= PACKET_DATA_SIZE {
+            info!(
+                "send_transaction: transaction too large: {} bytes (max: {} bytes)",
+                data.len(),
+                PACKET_DATA_SIZE
+            );
+            return Err(Error::invalid_request());
+        }
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let transactions_addr = get_leader_addr(&meta.cluster_info)?;
+        trace!("send_transaction: leader is {:?}", &transactions_addr);
+        transactions_socket
+            .send_to(&data, transactions_addr)
+            .map_err(|err| {
+                info!("send_transaction: send_to error: {:?}", err);
+                Error::internal_error()
+            })?;
+        let signature = bs58::encode(tx.signatures[0]).into_string();
+        trace!(
+            "send_transaction: sent {} bytes, signature={}",
+            data.len(),
+            signature
+        );
+        Ok(signature)
+    }
+
+    fn get_storage_mining_last_id(&self, meta: Self::Metadata) -> Result<String> {
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_storage_mining_last_id()
+    }
+
+    fn get_storage_mining_entry_height(&self, meta: Self::Metadata) -> Result<u64> {
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_storage_mining_entry_height()
+    }
+
+    fn get_storage_pubkeys_for_entry_height(
+        &self,
+        meta: Self::Metadata,
+        entry_height: u64,
+    ) -> Result<Vec<Pubkey>> {
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_storage_pubkeys_for_entry_height(entry_height)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster_info::NodeInfo;
+    use jsonrpc_core::{MetaIoHandler, Response};
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::{hash, Hash};
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+    use std::thread;
+
+    fn start_rpc_handler_with_tx(pubkey: Pubkey) -> (MetaIoHandler<Meta>, Meta, Hash, Keypair) {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bank = Arc::new(Bank::new(&genesis_block));
+
+        let last_id = bank.last_id();
+        let tx = SystemTransaction::new_move(&alice, pubkey, 20, last_id, 0);
+        bank.process_transaction(&tx).expect("process transaction");
+
+        let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
+            StorageState::default(),
+        )));
+        request_processor.write().unwrap().set_bank(&bank);
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(NodeInfo::default())));
+        let leader = NodeInfo::new_with_socketaddr(&socketaddr!("127.0.0.1:1234"));
+
+        cluster_info.write().unwrap().insert_info(leader.clone());
+        cluster_info.write().unwrap().set_leader(leader.id);
+        let rpc_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
+        let drone_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
+
+        let mut io = MetaIoHandler::default();
+        let rpc = RpcSolImpl;
+        io.extend_with(rpc.to_delegate());
+        let meta = Meta {
+            request_processor,
+            cluster_info,
+            drone_addr,
+            rpc_addr,
+        };
+        (io, meta, last_id, alice)
+    }
+
+    #[test]
+    fn test_rpc_request_processor_new() {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bob_pubkey = Keypair::new().pubkey();
+        let bank = Arc::new(Bank::new(&genesis_block));
+        let mut request_processor = JsonRpcRequestProcessor::new(StorageState::default());
+        request_processor.set_bank(&bank);
+        thread::spawn(move || {
+            let last_id = bank.last_id();
+            let tx = SystemTransaction::new_move(&alice, bob_pubkey, 20, last_id, 0);
+            bank.process_transaction(&tx).expect("process transaction");
+        })
+        .join()
+        .unwrap();
+        assert_eq!(request_processor.get_transaction_count().unwrap(), 1);
+    }
+
+    #[test]
+    fn test_rpc_get_balance() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, _last_id, _alice) = start_rpc_handler_with_tx(bob_pubkey);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["{}"]}}"#,
+            bob_pubkey
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":20,"id":1}}"#);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_tx_count() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, _last_id, _alice) = start_rpc_handler_with_tx(bob_pubkey);
+
+        let req = format!(r#"{{"jsonrpc":"2.0","id":1,"method":"getTransactionCount"}}"#);
+        let res = io.handle_request_sync(&req, meta);
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":1,"id":1}}"#);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_account_info() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, _last_id, _alice) = start_rpc_handler_with_tx(bob_pubkey);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}"]}}"#,
+            bob_pubkey
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected = r#"{
+            "jsonrpc":"2.0",
+            "result":{
+                "owner": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
+                "tokens": 20,
+                "userdata": [],
+                "executable": false
+            },
+            "id":1}
+        "#;
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_confirm_tx() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, last_id, alice) = start_rpc_handler_with_tx(bob_pubkey);
+        let tx = SystemTransaction::new_move(&alice, bob_pubkey, 20, last_id, 0);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"confirmTransaction","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":true,"id":1}}"#);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_signature_status() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, last_id, alice) = start_rpc_handler_with_tx(bob_pubkey);
+        let tx = SystemTransaction::new_move(&alice, bob_pubkey, 20, last_id, 0);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":"Confirmed","id":1}}"#);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+
+        // Test getSignatureStatus request on unprocessed tx
+        let tx = SystemTransaction::new_move(&alice, bob_pubkey, 10, last_id, 0);
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":"SignatureNotFound","id":1}}"#);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_last_id() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, last_id, _alice) = start_rpc_handler_with_tx(bob_pubkey);
+
+        let req = format!(r#"{{"jsonrpc":"2.0","id":1,"method":"getLastId"}}"#);
+        let res = io.handle_request_sync(&req, meta);
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":"{}","id":1}}"#, last_id);
+        let expected: Response =
+            serde_json::from_str(&expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_fail_request_airdrop() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let (io, meta, _last_id, _alice) = start_rpc_handler_with_tx(bob_pubkey);
+
+        // Expect internal error because no leader is running
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"requestAirdrop","params":["{}", 50]}}"#,
+            bob_pubkey
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected =
+            r#"{"jsonrpc":"2.0","error":{"code":-32603,"message":"Internal error"},"id":1}"#;
+        let expected: Response =
+            serde_json::from_str(expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_send_bad_tx() {
+        let (genesis_block, _) = GenesisBlock::new(10_000);
+        let bank = Arc::new(Bank::new(&genesis_block));
+
+        let mut io = MetaIoHandler::default();
+        let rpc = RpcSolImpl;
+        io.extend_with(rpc.to_delegate());
+        let meta = Meta {
+            request_processor: {
+                let mut request_processor = JsonRpcRequestProcessor::new(StorageState::default());
+                request_processor.set_bank(&bank);
+                Arc::new(RwLock::new(request_processor))
+            },
+            cluster_info: Arc::new(RwLock::new(ClusterInfo::new(NodeInfo::default()))),
+            drone_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
+            rpc_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0),
+        };
+
+        let req =
+            r#"{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":[[0,0,0,0,0,0,0,0]]}"#;
+        let res = io.handle_request_sync(req, meta.clone());
+        let expected =
+            r#"{"jsonrpc":"2.0","error":{"code":-32600,"message":"Invalid request"},"id":1}"#;
+        let expected: Response =
+            serde_json::from_str(expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_leader_addr() {
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(NodeInfo::default())));
+        assert_eq!(
+            get_leader_addr(&cluster_info),
+            Err(Error {
+                code: ErrorCode::InternalError,
+                message: "No leader detected".into(),
+                data: None,
+            })
+        );
+        let leader = NodeInfo::new_with_socketaddr(&socketaddr!("127.0.0.1:1234"));
+        cluster_info.write().unwrap().insert_info(leader.clone());
+        cluster_info.write().unwrap().set_leader(leader.id);
+        assert_eq!(
+            get_leader_addr(&cluster_info),
+            Ok(socketaddr!("127.0.0.1:1234"))
+        );
+    }
+
+    #[test]
+    fn test_rpc_verify_pubkey() {
+        let pubkey = Keypair::new().pubkey();
+        assert_eq!(verify_pubkey(pubkey.to_string()).unwrap(), pubkey);
+        let bad_pubkey = "a1b2c3d4";
+        assert_eq!(
+            verify_pubkey(bad_pubkey.to_string()),
+            Err(Error::invalid_request())
+        );
+    }
+
+    #[test]
+    fn test_rpc_verify_signature() {
+        let tx = SystemTransaction::new_move(
+            &Keypair::new(),
+            Keypair::new().pubkey(),
+            20,
+            hash(&[0]),
+            0,
+        );
+        assert_eq!(
+            verify_signature(&tx.signatures[0].to_string()).unwrap(),
+            tx.signatures[0]
+        );
+        let bad_signature = "a1b2c3d4";
+        assert_eq!(
+            verify_signature(&bad_signature.to_string()),
+            Err(Error::invalid_request())
+        );
+    }
+}

core/src/rpc_mock.rs

@@ -0,0 +1,111 @@
+// Implementation of RpcRequestHandler trait for testing Rpc requests without i/o
+
+use crate::rpc_request::{RpcRequest, RpcRequestHandler};
+use serde_json::{Number, Value};
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::system_transaction::SystemTransaction;
+use solana_sdk::transaction::Transaction;
+use std::error;
+use std::io::{Error, ErrorKind};
+use std::net::SocketAddr;
+
+pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
+pub const SIGNATURE: &str =
+    "43yNSFC6fYTuPgTNFFhF4axw7AfWxB2BPdurme8yrsWEYwm8299xh8n6TAHjGymiSub1XtyxTNyd9GBfY2hxoBw8";
+
+#[derive(Clone)]
+pub struct MockRpcClient {
+    pub addr: String,
+}
+
+impl MockRpcClient {
+    pub fn new(addr: String) -> Self {
+        MockRpcClient { addr }
+    }
+
+    pub fn retry_get_balance(
+        &self,
+        id: u64,
+        pubkey: Pubkey,
+        retries: usize,
+    ) -> Result<Option<u64>, Box<dyn error::Error>> {
+        let params = json!([format!("{}", pubkey)]);
+        let res = self
+            .retry_make_rpc_request(id, &RpcRequest::GetBalance, Some(params), retries)?
+            .as_u64();
+        Ok(res)
+    }
+
+    pub fn retry_make_rpc_request(
+        &self,
+        _id: u64,
+        request: &RpcRequest,
+        params: Option<Value>,
+        mut _retries: usize,
+    ) -> Result<Value, Box<dyn error::Error>> {
+        if self.addr == "fails" {
+            return Ok(Value::Null);
+        }
+        let val = match request {
+            RpcRequest::ConfirmTransaction => {
+                if let Some(Value::Array(param_array)) = params {
+                    if let Value::String(param_string) = &param_array[0] {
+                        Value::Bool(param_string == SIGNATURE)
+                    } else {
+                        Value::Null
+                    }
+                } else {
+                    Value::Null
+                }
+            }
+            RpcRequest::GetBalance => {
+                let n = if self.addr == "airdrop" { 0 } else { 50 };
+                Value::Number(Number::from(n))
+            }
+            RpcRequest::GetLastId => Value::String(PUBKEY.to_string()),
+            RpcRequest::GetSignatureStatus => {
+                let str = if self.addr == "account_in_use" {
+                    "AccountInUse"
+                } else if self.addr == "bad_sig_status" {
+                    "Nonexistent"
+                } else {
+                    "Confirmed"
+                };
+                Value::String(str.to_string())
+            }
+            RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
+            RpcRequest::SendTransaction => Value::String(SIGNATURE.to_string()),
+            _ => Value::Null,
+        };
+        Ok(val)
+    }
+}
+
+impl RpcRequestHandler for MockRpcClient {
+    fn make_rpc_request(
+        &self,
+        id: u64,
+        request: RpcRequest,
+        params: Option<Value>,
+    ) -> Result<Value, Box<dyn error::Error>> {
+        self.retry_make_rpc_request(id, &request, params, 0)
+    }
+}
+
+pub fn request_airdrop_transaction(
+    _drone_addr: &SocketAddr,
+    _id: &Pubkey,
+    tokens: u64,
+    _last_id: Hash,
+) -> Result<Transaction, Error> {
+    if tokens == 0 {
+        Err(Error::new(ErrorKind::Other, "Airdrop failed"))?
+    }
+    let key = Keypair::new();
+    let to = Keypair::new().pubkey();
+    let last_id = Hash::default();
+    let tx = SystemTransaction::new_account(&key, to, 50, last_id, 0);
+    Ok(tx)
+}

core/src/rpc_pubsub.rs

@@ -0,0 +1,445 @@
+//! The `pubsub` module implements a threaded subscription service on client RPC request
+
+use crate::rpc_status::RpcSignatureStatus;
+use crate::rpc_subscriptions::RpcSubscriptions;
+use bs58;
+use jsonrpc_core::{Error, ErrorCode, Result};
+use jsonrpc_derive::rpc;
+use jsonrpc_pubsub::typed::Subscriber;
+use jsonrpc_pubsub::{Session, SubscriptionId};
+use solana_sdk::account::Account;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Signature;
+use std::mem;
+use std::sync::{atomic, Arc};
+
+#[rpc]
+pub trait RpcSolPubSub {
+    type Metadata;
+
+    // Get notification every time account userdata is changed
+    // Accepts pubkey parameter as base-58 encoded string
+    #[pubsub(
+        subscription = "accountNotification",
+        subscribe,
+        name = "accountSubscribe"
+    )]
+    fn account_subscribe(&self, _: Self::Metadata, _: Subscriber<Account>, _: String);
+
+    // Unsubscribe from account notification subscription.
+    #[pubsub(
+        subscription = "accountNotification",
+        unsubscribe,
+        name = "accountUnsubscribe"
+    )]
+    fn account_unsubscribe(&self, _: Option<Self::Metadata>, _: SubscriptionId) -> Result<bool>;
+
+    // Get notification when signature is verified
+    // Accepts signature parameter as base-58 encoded string
+    #[pubsub(
+        subscription = "signatureNotification",
+        subscribe,
+        name = "signatureSubscribe"
+    )]
+    fn signature_subscribe(&self, _: Self::Metadata, _: Subscriber<RpcSignatureStatus>, _: String);
+
+    // Unsubscribe from signature notification subscription.
+    #[pubsub(
+        subscription = "signatureNotification",
+        unsubscribe,
+        name = "signatureUnsubscribe"
+    )]
+    fn signature_unsubscribe(&self, _: Option<Self::Metadata>, _: SubscriptionId) -> Result<bool>;
+}
+
+#[derive(Default)]
+pub struct RpcSolPubSubImpl {
+    uid: Arc<atomic::AtomicUsize>,
+    subscriptions: Arc<RpcSubscriptions>,
+}
+
+impl RpcSolPubSubImpl {
+    pub fn new(subscriptions: Arc<RpcSubscriptions>) -> Self {
+        let uid = Arc::new(atomic::AtomicUsize::default());
+        Self { uid, subscriptions }
+    }
+}
+
+impl RpcSolPubSub for RpcSolPubSubImpl {
+    type Metadata = Arc<Session>;
+
+    fn account_subscribe(
+        &self,
+        _meta: Self::Metadata,
+        subscriber: Subscriber<Account>,
+        pubkey_str: String,
+    ) {
+        let pubkey_vec = bs58::decode(pubkey_str).into_vec().unwrap();
+        if pubkey_vec.len() != mem::size_of::<Pubkey>() {
+            subscriber
+                .reject(Error {
+                    code: ErrorCode::InvalidParams,
+                    message: "Invalid Request: Invalid pubkey provided".into(),
+                    data: None,
+                })
+                .unwrap();
+            return;
+        }
+        let pubkey = Pubkey::new(&pubkey_vec);
+
+        let id = self.uid.fetch_add(1, atomic::Ordering::SeqCst);
+        let sub_id = SubscriptionId::Number(id as u64);
+        info!("account_subscribe: account={:?} id={:?}", pubkey, sub_id);
+        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
+
+        self.subscriptions
+            .add_account_subscription(&pubkey, &sub_id, &sink)
+    }
+
+    fn account_unsubscribe(
+        &self,
+        _meta: Option<Self::Metadata>,
+        id: SubscriptionId,
+    ) -> Result<bool> {
+        info!("account_unsubscribe: id={:?}", id);
+        if self.subscriptions.remove_account_subscription(&id) {
+            Ok(true)
+        } else {
+            Err(Error {
+                code: ErrorCode::InvalidParams,
+                message: "Invalid Request: Subscription id does not exist".into(),
+                data: None,
+            })
+        }
+    }
+
+    fn signature_subscribe(
+        &self,
+        _meta: Self::Metadata,
+        subscriber: Subscriber<RpcSignatureStatus>,
+        signature_str: String,
+    ) {
+        info!("signature_subscribe");
+        let signature_vec = bs58::decode(signature_str).into_vec().unwrap();
+        if signature_vec.len() != mem::size_of::<Signature>() {
+            subscriber
+                .reject(Error {
+                    code: ErrorCode::InvalidParams,
+                    message: "Invalid Request: Invalid signature provided".into(),
+                    data: None,
+                })
+                .unwrap();
+            return;
+        }
+        let signature = Signature::new(&signature_vec);
+        let id = self.uid.fetch_add(1, atomic::Ordering::SeqCst);
+        let sub_id = SubscriptionId::Number(id as u64);
+        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
+
+        self.subscriptions
+            .add_signature_subscription(&signature, &sub_id, &sink);
+    }
+
+    fn signature_unsubscribe(
+        &self,
+        _meta: Option<Self::Metadata>,
+        id: SubscriptionId,
+    ) -> Result<bool> {
+        info!("signature_unsubscribe");
+        if self.subscriptions.remove_signature_subscription(&id) {
+            Ok(true)
+        } else {
+            Err(Error {
+                code: ErrorCode::InvalidParams,
+                message: "Invalid Request: Subscription id does not exist".into(),
+                data: None,
+            })
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use jsonrpc_core::futures::sync::mpsc;
+    use jsonrpc_core::Response;
+    use jsonrpc_pubsub::{PubSubHandler, Session};
+    use solana_runtime::bank::{self, Bank};
+    use solana_sdk::budget_program;
+    use solana_sdk::budget_transaction::BudgetTransaction;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use solana_sdk::transaction::Transaction;
+    use std::thread::sleep;
+    use std::time::Duration;
+    use tokio::prelude::{Async, Stream};
+
+    fn process_transaction_and_notify(
+        bank: &Arc<Bank>,
+        tx: &Transaction,
+        subscriptions: &RpcSubscriptions,
+    ) -> bank::Result<Arc<Bank>> {
+        bank.process_transaction(tx)?;
+        subscriptions.notify_subscribers(&bank);
+
+        // Simulate a block boundary
+        Ok(Arc::new(Bank::new_from_parent(
+            &bank,
+            Pubkey::default(),
+            bank.slot() + 1,
+        )))
+    }
+
+    fn create_session() -> Arc<Session> {
+        Arc::new(Session::new(mpsc::channel(1).0))
+    }
+
+    #[test]
+    fn test_signature_subscribe() {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bob = Keypair::new();
+        let bob_pubkey = bob.pubkey();
+        let bank = Bank::new(&genesis_block);
+        let arc_bank = Arc::new(bank);
+        let last_id = arc_bank.last_id();
+
+        let rpc = RpcSolPubSubImpl::default();
+
+        // Test signature subscriptions
+        let tx = SystemTransaction::new_move(&alice, bob_pubkey, 20, last_id, 0);
+
+        let session = create_session();
+        let (subscriber, _id_receiver, mut receiver) =
+            Subscriber::new_test("signatureNotification");
+        rpc.signature_subscribe(session, subscriber, tx.signatures[0].to_string());
+
+        process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+        sleep(Duration::from_millis(200));
+
+        // Test signature confirmation notification
+        let string = receiver.poll();
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            let expected = format!(r#"{{"jsonrpc":"2.0","method":"signatureNotification","params":{{"result":"Confirmed","subscription":0}}}}"#);
+            assert_eq!(expected, response);
+        }
+    }
+
+    #[test]
+    fn test_signature_unsubscribe() {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bob_pubkey = Keypair::new().pubkey();
+        let bank = Bank::new(&genesis_block);
+        let arc_bank = Arc::new(bank);
+        let last_id = arc_bank.last_id();
+
+        let session = create_session();
+
+        let mut io = PubSubHandler::default();
+        let rpc = RpcSolPubSubImpl::default();
+        io.extend_with(rpc.to_delegate());
+
+        let tx = SystemTransaction::new_move(&alice, bob_pubkey, 20, last_id, 0);
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"signatureSubscribe","params":["{}"]}}"#,
+            tx.signatures[0].to_string()
+        );
+        let _res = io.handle_request_sync(&req, session.clone());
+
+        let req =
+            format!(r#"{{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[0]}}"#);
+        let res = io.handle_request_sync(&req, session.clone());
+
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":true,"id":1}}"#);
+        let expected: Response = serde_json::from_str(&expected).unwrap();
+
+        let result: Response = serde_json::from_str(&res.unwrap()).unwrap();
+        assert_eq!(expected, result);
+
+        // Test bad parameter
+        let req =
+            format!(r#"{{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[1]}}"#);
+        let res = io.handle_request_sync(&req, session.clone());
+        let expected = format!(r#"{{"jsonrpc":"2.0","error":{{"code":-32602,"message":"Invalid Request: Subscription id does not exist"}},"id":1}}"#);
+        let expected: Response = serde_json::from_str(&expected).unwrap();
+
+        let result: Response = serde_json::from_str(&res.unwrap()).unwrap();
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_account_subscribe() {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bob_pubkey = Keypair::new().pubkey();
+        let witness = Keypair::new();
+        let contract_funds = Keypair::new();
+        let contract_state = Keypair::new();
+        let budget_program_id = budget_program::id();
+        let executable = false; // TODO
+        let bank = Bank::new(&genesis_block);
+        let arc_bank = Arc::new(bank);
+        let last_id = arc_bank.last_id();
+
+        let rpc = RpcSolPubSubImpl::default();
+        let session = create_session();
+        let (subscriber, _id_receiver, mut receiver) = Subscriber::new_test("accountNotification");
+        rpc.account_subscribe(session, subscriber, contract_state.pubkey().to_string());
+
+        let tx = SystemTransaction::new_program_account(
+            &alice,
+            contract_funds.pubkey(),
+            last_id,
+            50,
+            0,
+            budget_program_id,
+            0,
+        );
+        let arc_bank = process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+
+        let tx = SystemTransaction::new_program_account(
+            &alice,
+            contract_state.pubkey(),
+            last_id,
+            1,
+            196,
+            budget_program_id,
+            0,
+        );
+
+        let arc_bank = process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+
+        // Test signature confirmation notification #1
+        let string = receiver.poll();
+
+        let expected_userdata = arc_bank
+            .get_account(&contract_state.pubkey())
+            .unwrap()
+            .userdata;
+
+        let expected = json!({
+           "jsonrpc": "2.0",
+           "method": "accountNotification",
+           "params": {
+               "result": {
+                   "owner": budget_program_id,
+                   "tokens": 1,
+                   "userdata": expected_userdata,
+                   "executable": executable,
+
+               },
+               "subscription": 0,
+           }
+        });
+
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            assert_eq!(serde_json::to_string(&expected).unwrap(), response);
+        }
+        let tx = BudgetTransaction::new_when_signed(
+            &contract_funds,
+            bob_pubkey,
+            contract_state.pubkey(),
+            witness.pubkey(),
+            None,
+            50,
+            last_id,
+        );
+        let arc_bank = process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+        sleep(Duration::from_millis(200));
+
+        // Test signature confirmation notification #2
+        let string = receiver.poll();
+        let expected_userdata = arc_bank
+            .get_account(&contract_state.pubkey())
+            .unwrap()
+            .userdata;
+        let expected = json!({
+           "jsonrpc": "2.0",
+           "method": "accountNotification",
+           "params": {
+               "result": {
+                   "owner": budget_program_id,
+                   "tokens": 51,
+                   "userdata": expected_userdata,
+                    "executable": executable,
+               },
+               "subscription": 0,
+           }
+        });
+
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            assert_eq!(serde_json::to_string(&expected).unwrap(), response);
+        }
+
+        let tx = SystemTransaction::new_account(&alice, witness.pubkey(), 1, last_id, 0);
+        let arc_bank = process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+        sleep(Duration::from_millis(200));
+        let tx = BudgetTransaction::new_signature(
+            &witness,
+            contract_state.pubkey(),
+            bob_pubkey,
+            last_id,
+        );
+        let arc_bank = process_transaction_and_notify(&arc_bank, &tx, &rpc.subscriptions).unwrap();
+        sleep(Duration::from_millis(200));
+
+        let expected_userdata = arc_bank
+            .get_account(&contract_state.pubkey())
+            .unwrap()
+            .userdata;
+        let expected = json!({
+           "jsonrpc": "2.0",
+           "method": "accountNotification",
+           "params": {
+               "result": {
+                   "owner": budget_program_id,
+                   "tokens": 1,
+                   "userdata": expected_userdata,
+                    "executable": executable,
+               },
+               "subscription": 0,
+           }
+        });
+        let string = receiver.poll();
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            assert_eq!(serde_json::to_string(&expected).unwrap(), response);
+        }
+    }
+
+    #[test]
+    fn test_account_unsubscribe() {
+        let bob_pubkey = Keypair::new().pubkey();
+        let session = create_session();
+
+        let mut io = PubSubHandler::default();
+        let rpc = RpcSolPubSubImpl::default();
+
+        io.extend_with(rpc.to_delegate());
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"accountSubscribe","params":["{}"]}}"#,
+            bob_pubkey.to_string()
+        );
+        let _res = io.handle_request_sync(&req, session.clone());
+
+        let req =
+            format!(r#"{{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[0]}}"#);
+        let res = io.handle_request_sync(&req, session.clone());
+
+        let expected = format!(r#"{{"jsonrpc":"2.0","result":true,"id":1}}"#);
+        let expected: Response = serde_json::from_str(&expected).unwrap();
+
+        let result: Response = serde_json::from_str(&res.unwrap()).unwrap();
+        assert_eq!(expected, result);
+
+        // Test bad parameter
+        let req =
+            format!(r#"{{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[1]}}"#);
+        let res = io.handle_request_sync(&req, session.clone());
+        let expected = format!(r#"{{"jsonrpc":"2.0","error":{{"code":-32602,"message":"Invalid Request: Subscription id does not exist"}},"id":1}}"#);
+        let expected: Response = serde_json::from_str(&expected).unwrap();
+
+        let result: Response = serde_json::from_str(&res.unwrap()).unwrap();
+        assert_eq!(expected, result);
+    }
+}

core/src/rpc_pubsub_service.rs

@@ -0,0 +1,85 @@
+//! The `pubsub` module implements a threaded subscription service on client RPC request
+
+use crate::rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl};
+use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::service::Service;
+use jsonrpc_pubsub::{PubSubHandler, Session};
+use jsonrpc_ws_server::{RequestContext, ServerBuilder};
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct PubSubService {
+    thread_hdl: JoinHandle<()>,
+    exit: Arc<AtomicBool>,
+}
+
+impl Service for PubSubService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+impl PubSubService {
+    pub fn new(subscriptions: &Arc<RpcSubscriptions>, pubsub_addr: SocketAddr) -> Self {
+        info!("rpc_pubsub bound to {:?}", pubsub_addr);
+        let rpc = RpcSolPubSubImpl::new(subscriptions.clone());
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_ = exit.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-pubsub".to_string())
+            .spawn(move || {
+                let mut io = PubSubHandler::default();
+                io.extend_with(rpc.to_delegate());
+
+                let server = ServerBuilder::with_meta_extractor(io, |context: &RequestContext| {
+                        info!("New pubsub connection");
+                        let session = Arc::new(Session::new(context.sender().clone()));
+                        session.on_drop(|| {
+                            info!("Pubsub connection dropped");
+                        });
+                        session
+                })
+                .start(&pubsub_addr);
+
+                if let Err(e) = server {
+                    warn!("Pubsub service unavailable error: {:?}. \nAlso, check that port {} is not already in use by another application", e, pubsub_addr.port());
+                    return;
+                }
+                while !exit_.load(Ordering::Relaxed) {
+                    sleep(Duration::from_millis(100));
+                }
+                server.unwrap().close();
+            })
+            .unwrap();
+        Self { thread_hdl, exit }
+    }
+
+    pub fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::net::{IpAddr, Ipv4Addr};
+
+    #[test]
+    fn test_pubsub_new() {
+        let subscriptions = Arc::new(RpcSubscriptions::default());
+        let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
+        let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr);
+        let thread = pubsub_service.thread_hdl.thread();
+        assert_eq!(thread.name().unwrap(), "solana-pubsub");
+    }
+}

core/src/rpc_request.rs

@@ -0,0 +1,328 @@
+use reqwest;
+use reqwest::header::CONTENT_TYPE;
+use serde_json::{self, Value};
+use solana_sdk::timing::{DEFAULT_TICKS_PER_SLOT, NUM_TICKS_PER_SECOND};
+use std::net::SocketAddr;
+use std::thread::sleep;
+use std::time::Duration;
+use std::{error, fmt};
+
+use solana_sdk::pubkey::Pubkey;
+
+#[derive(Clone)]
+pub struct RpcClient {
+    pub client: reqwest::Client,
+    pub addr: String,
+}
+
+impl RpcClient {
+    pub fn new(addr: String) -> Self {
+        RpcClient {
+            client: reqwest::Client::new(),
+            addr,
+        }
+    }
+
+    pub fn new_with_timeout(addr: SocketAddr, timeout: Duration) -> Self {
+        let addr = get_rpc_request_str(addr, false);
+        let client = reqwest::Client::builder()
+            .timeout(timeout)
+            .build()
+            .expect("build rpc client");
+        RpcClient { client, addr }
+    }
+
+    pub fn new_from_socket(addr: SocketAddr) -> Self {
+        Self::new(get_rpc_request_str(addr, false))
+    }
+
+    pub fn retry_get_balance(
+        &self,
+        id: u64,
+        pubkey: Pubkey,
+        retries: usize,
+    ) -> Result<Option<u64>, Box<dyn error::Error>> {
+        let params = json!([format!("{}", pubkey)]);
+        let res = self
+            .retry_make_rpc_request(id, &RpcRequest::GetBalance, Some(params), retries)?
+            .as_u64();
+        Ok(res)
+    }
+
+    pub fn retry_make_rpc_request(
+        &self,
+        id: u64,
+        request: &RpcRequest,
+        params: Option<Value>,
+        mut retries: usize,
+    ) -> Result<Value, Box<dyn error::Error>> {
+        let request_json = request.build_request_json(id, params);
+
+        loop {
+            match self
+                .client
+                .post(&self.addr)
+                .header(CONTENT_TYPE, "application/json")
+                .body(request_json.to_string())
+                .send()
+            {
+                Ok(mut response) => {
+                    let json: Value = serde_json::from_str(&response.text()?)?;
+                    if json["error"].is_object() {
+                        Err(RpcError::RpcRequestError(format!(
+                            "RPC Error response: {}",
+                            serde_json::to_string(&json["error"]).unwrap()
+                        )))?
+                    }
+                    return Ok(json["result"].clone());
+                }
+                Err(e) => {
+                    info!(
+                        "make_rpc_request() failed, {} retries left: {:?}",
+                        retries, e
+                    );
+                    if retries == 0 {
+                        Err(e)?;
+                    }
+                    retries -= 1;
+
+                    // Sleep for approximately half a slot
+                    sleep(Duration::from_millis(
+                        500 * DEFAULT_TICKS_PER_SLOT / NUM_TICKS_PER_SECOND,
+                    ));
+                }
+            }
+        }
+    }
+}
+
+pub fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
+    if tls {
+        format!("https://{}", rpc_addr)
+    } else {
+        format!("http://{}", rpc_addr)
+    }
+}
+
+pub trait RpcRequestHandler {
+    fn make_rpc_request(
+        &self,
+        id: u64,
+        request: RpcRequest,
+        params: Option<Value>,
+    ) -> Result<Value, Box<dyn error::Error>>;
+}
+
+impl RpcRequestHandler for RpcClient {
+    fn make_rpc_request(
+        &self,
+        id: u64,
+        request: RpcRequest,
+        params: Option<Value>,
+    ) -> Result<Value, Box<dyn error::Error>> {
+        self.retry_make_rpc_request(id, &request, params, 0)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum RpcRequest {
+    ConfirmTransaction,
+    GetAccountInfo,
+    GetBalance,
+    GetLastId,
+    GetSignatureStatus,
+    GetTransactionCount,
+    RequestAirdrop,
+    SendTransaction,
+    RegisterNode,
+    SignVote,
+    DeregisterNode,
+    GetStorageMiningLastId,
+    GetStorageMiningEntryHeight,
+    GetStoragePubkeysForEntryHeight,
+}
+
+impl RpcRequest {
+    fn build_request_json(&self, id: u64, params: Option<Value>) -> Value {
+        let jsonrpc = "2.0";
+        let method = match self {
+            RpcRequest::ConfirmTransaction => "confirmTransaction",
+            RpcRequest::GetAccountInfo => "getAccountInfo",
+            RpcRequest::GetBalance => "getBalance",
+            RpcRequest::GetLastId => "getLastId",
+            RpcRequest::GetSignatureStatus => "getSignatureStatus",
+            RpcRequest::GetTransactionCount => "getTransactionCount",
+            RpcRequest::RequestAirdrop => "requestAirdrop",
+            RpcRequest::SendTransaction => "sendTransaction",
+            RpcRequest::RegisterNode => "registerNode",
+            RpcRequest::SignVote => "signVote",
+            RpcRequest::DeregisterNode => "deregisterNode",
+            RpcRequest::GetStorageMiningLastId => "getStorageMiningLastId",
+            RpcRequest::GetStorageMiningEntryHeight => "getStorageMiningEntryHeight",
+            RpcRequest::GetStoragePubkeysForEntryHeight => "getStoragePubkeysForEntryHeight",
+        };
+        let mut request = json!({
+           "jsonrpc": jsonrpc,
+           "id": id,
+           "method": method,
+        });
+        if let Some(param_string) = params {
+            request["params"] = param_string;
+        }
+        request
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum RpcError {
+    RpcRequestError(String),
+}
+
+impl fmt::Display for RpcError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "invalid")
+    }
+}
+
+impl error::Error for RpcError {
+    fn description(&self) -> &str {
+        "invalid"
+    }
+
+    fn cause(&self) -> Option<&dyn error::Error> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use jsonrpc_core::{Error, IoHandler, Params};
+    use jsonrpc_http_server::{AccessControlAllowOrigin, DomainsValidation, ServerBuilder};
+    use serde_json::Number;
+    use std::net::{Ipv4Addr, SocketAddr};
+    use std::sync::mpsc::channel;
+    use std::thread;
+
+    #[test]
+    fn test_build_request_json() {
+        let test_request = RpcRequest::GetAccountInfo;
+        let addr = json!(["deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhx"]);
+        let request = test_request.build_request_json(1, Some(addr.clone()));
+        assert_eq!(request["method"], "getAccountInfo");
+        assert_eq!(request["params"], addr,);
+
+        let test_request = RpcRequest::GetBalance;
+        let request = test_request.build_request_json(1, Some(addr));
+        assert_eq!(request["method"], "getBalance");
+
+        let test_request = RpcRequest::GetLastId;
+        let request = test_request.build_request_json(1, None);
+        assert_eq!(request["method"], "getLastId");
+
+        let test_request = RpcRequest::GetTransactionCount;
+        let request = test_request.build_request_json(1, None);
+        assert_eq!(request["method"], "getTransactionCount");
+
+        let test_request = RpcRequest::RequestAirdrop;
+        let request = test_request.build_request_json(1, None);
+        assert_eq!(request["method"], "requestAirdrop");
+
+        let test_request = RpcRequest::SendTransaction;
+        let request = test_request.build_request_json(1, None);
+        assert_eq!(request["method"], "sendTransaction");
+    }
+    #[test]
+    fn test_make_rpc_request() {
+        let (sender, receiver) = channel();
+        thread::spawn(move || {
+            let rpc_addr = socketaddr!(0, 0);
+            let mut io = IoHandler::default();
+            // Successful request
+            io.add_method("getBalance", |_params: Params| {
+                Ok(Value::Number(Number::from(50)))
+            });
+            // Failed request
+            io.add_method("getLastId", |params: Params| {
+                if params != Params::None {
+                    Err(Error::invalid_request())
+                } else {
+                    Ok(Value::String(
+                        "deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhx".to_string(),
+                    ))
+                }
+            });
+
+            let server = ServerBuilder::new(io)
+                .threads(1)
+                .cors(DomainsValidation::AllowOnly(vec![
+                    AccessControlAllowOrigin::Any,
+                ]))
+                .start_http(&rpc_addr)
+                .expect("Unable to start RPC server");
+            sender.send(*server.address()).unwrap();
+            server.wait();
+        });
+
+        let rpc_addr = receiver.recv().unwrap();
+        let rpc_client = RpcClient::new_from_socket(rpc_addr);
+
+        let balance = rpc_client.make_rpc_request(
+            1,
+            RpcRequest::GetBalance,
+            Some(json!(["deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhx"])),
+        );
+        assert_eq!(balance.unwrap().as_u64().unwrap(), 50);
+
+        let last_id = rpc_client.make_rpc_request(2, RpcRequest::GetLastId, None);
+        assert_eq!(
+            last_id.unwrap().as_str().unwrap(),
+            "deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhx"
+        );
+
+        // Send erroneous parameter
+        let last_id =
+            rpc_client.make_rpc_request(3, RpcRequest::GetLastId, Some(json!("paramter")));
+        assert_eq!(last_id.is_err(), true);
+    }
+
+    #[test]
+    fn test_retry_make_rpc_request() {
+        solana_logger::setup();
+        let (sender, receiver) = channel();
+        thread::spawn(move || {
+            // 1. Pick a random port
+            // 2. Tell the client to start using it
+            // 3. Delay for 1.5 seconds before starting the server to ensure the client will fail
+            //    and need to retry
+            let rpc_addr = socketaddr!(0, 4242);
+            sender.send(rpc_addr.clone()).unwrap();
+            sleep(Duration::from_millis(1500));
+
+            let mut io = IoHandler::default();
+            io.add_method("getBalance", move |_params: Params| {
+                Ok(Value::Number(Number::from(5)))
+            });
+            let server = ServerBuilder::new(io)
+                .threads(1)
+                .cors(DomainsValidation::AllowOnly(vec![
+                    AccessControlAllowOrigin::Any,
+                ]))
+                .start_http(&rpc_addr)
+                .expect("Unable to start RPC server");
+            server.wait();
+        });
+
+        let rpc_addr = receiver.recv().unwrap();
+        let rpc_client = RpcClient::new_from_socket(rpc_addr);
+
+        let balance = rpc_client.retry_make_rpc_request(
+            1,
+            &RpcRequest::GetBalance,
+            Some(json!(["deadbeefXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNHhw"])),
+            10,
+        );
+        assert_eq!(balance.unwrap().as_u64().unwrap(), 5);
+    }
+}

core/src/rpc_service.rs

@@ -0,0 +1,136 @@
+//! The `rpc_service` module implements the Solana JSON RPC service.
+
+use crate::cluster_info::ClusterInfo;
+use crate::rpc::*;
+use crate::service::Service;
+use crate::storage_stage::StorageState;
+use jsonrpc_core::MetaIoHandler;
+use jsonrpc_http_server::{hyper, AccessControlAllowOrigin, DomainsValidation, ServerBuilder};
+use solana_runtime::bank::Bank;
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub const RPC_PORT: u16 = 8899;
+
+pub struct JsonRpcService {
+    thread_hdl: JoinHandle<()>,
+    exit: Arc<AtomicBool>,
+    pub request_processor: Arc<RwLock<JsonRpcRequestProcessor>>, // Used only by tests...
+}
+
+impl JsonRpcService {
+    pub fn new(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        rpc_addr: SocketAddr,
+        drone_addr: SocketAddr,
+        storage_state: StorageState,
+    ) -> Self {
+        info!("rpc bound to {:?}", rpc_addr);
+        let exit = Arc::new(AtomicBool::new(false));
+        let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(storage_state)));
+        let request_processor_ = request_processor.clone();
+
+        let info = cluster_info.clone();
+        let exit_ = exit.clone();
+
+        let thread_hdl = Builder::new()
+            .name("solana-jsonrpc".to_string())
+            .spawn(move || {
+                let mut io = MetaIoHandler::default();
+                let rpc = RpcSolImpl;
+                io.extend_with(rpc.to_delegate());
+
+                let server =
+                    ServerBuilder::with_meta_extractor(io, move |_req: &hyper::Request<hyper::Body>| Meta {
+                        request_processor: request_processor_.clone(),
+                        cluster_info: info.clone(),
+                        drone_addr,
+                        rpc_addr,
+                    }).threads(4)
+                        .cors(DomainsValidation::AllowOnly(vec![
+                            AccessControlAllowOrigin::Any,
+                        ]))
+                        .start_http(&rpc_addr);
+                if let Err(e) = server {
+                    warn!("JSON RPC service unavailable error: {:?}. \nAlso, check that port {} is not already in use by another application", e, rpc_addr.port());
+                    return;
+                }
+                while !exit_.load(Ordering::Relaxed) {
+                    sleep(Duration::from_millis(100));
+                }
+                server.unwrap().close();
+            })
+            .unwrap();
+        Self {
+            thread_hdl,
+            exit,
+            request_processor,
+        }
+    }
+
+    pub fn set_bank(&mut self, bank: &Arc<Bank>) {
+        self.request_processor.write().unwrap().set_bank(bank);
+    }
+
+    pub fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+impl Service for JsonRpcService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster_info::NodeInfo;
+    use solana_runtime::bank::Bank;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::signature::KeypairUtil;
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+    #[test]
+    fn test_rpc_new() {
+        let (genesis_block, alice) = GenesisBlock::new(10_000);
+        let bank = Bank::new(&genesis_block);
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(NodeInfo::default())));
+        let rpc_addr = SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+            solana_netutil::find_available_port_in_range((10000, 65535)).unwrap(),
+        );
+        let drone_addr = SocketAddr::new(
+            IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+            solana_netutil::find_available_port_in_range((10000, 65535)).unwrap(),
+        );
+        let mut rpc_service =
+            JsonRpcService::new(&cluster_info, rpc_addr, drone_addr, StorageState::default());
+        rpc_service.set_bank(&Arc::new(bank));
+        let thread = rpc_service.thread_hdl.thread();
+        assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
+
+        assert_eq!(
+            10_000,
+            rpc_service
+                .request_processor
+                .read()
+                .unwrap()
+                .get_balance(alice.pubkey())
+                .unwrap()
+        );
+
+        rpc_service.close().unwrap();
+    }
+}

core/src/rpc_status.rs

@@ -0,0 +1,30 @@
+//! The `rpc_status` module defines transaction status codes
+
+use jsonrpc_core::{Error, Result};
+use std::str::FromStr;
+
+#[derive(Copy, Clone, PartialEq, Serialize, Debug)]
+pub enum RpcSignatureStatus {
+    AccountInUse,
+    AccountLoadedTwice,
+    Confirmed,
+    GenericFailure,
+    ProgramRuntimeError,
+    SignatureNotFound,
+}
+
+impl FromStr for RpcSignatureStatus {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<RpcSignatureStatus> {
+        match s {
+            "AccountInUse" => Ok(RpcSignatureStatus::AccountInUse),
+            "AccountLoadedTwice" => Ok(RpcSignatureStatus::AccountLoadedTwice),
+            "Confirmed" => Ok(RpcSignatureStatus::Confirmed),
+            "GenericFailure" => Ok(RpcSignatureStatus::GenericFailure),
+            "ProgramRuntimeError" => Ok(RpcSignatureStatus::ProgramRuntimeError),
+            "SignatureNotFound" => Ok(RpcSignatureStatus::SignatureNotFound),
+            _ => Err(Error::parse_error()),
+        }
+    }
+}

core/src/rpc_subscriptions.rs

@@ -0,0 +1,236 @@
+//! The `pubsub` module implements a threaded subscription service on client RPC request
+
+use crate::rpc_status::RpcSignatureStatus;
+use jsonrpc_core::futures::Future;
+use jsonrpc_pubsub::typed::Sink;
+use jsonrpc_pubsub::SubscriptionId;
+use solana_runtime::bank::{self, Bank, BankError};
+use solana_sdk::account::Account;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Signature;
+use std::collections::HashMap;
+use std::sync::RwLock;
+
+type RpcAccountSubscriptions = RwLock<HashMap<Pubkey, HashMap<SubscriptionId, Sink<Account>>>>;
+type RpcSignatureSubscriptions =
+    RwLock<HashMap<Signature, HashMap<SubscriptionId, Sink<RpcSignatureStatus>>>>;
+pub struct RpcSubscriptions {
+    account_subscriptions: RpcAccountSubscriptions,
+    signature_subscriptions: RpcSignatureSubscriptions,
+}
+
+impl Default for RpcSubscriptions {
+    fn default() -> Self {
+        RpcSubscriptions {
+            account_subscriptions: RpcAccountSubscriptions::default(),
+            signature_subscriptions: RpcSignatureSubscriptions::default(),
+        }
+    }
+}
+
+impl RpcSubscriptions {
+    pub fn check_account(&self, pubkey: &Pubkey, account: &Account) {
+        let subscriptions = self.account_subscriptions.read().unwrap();
+        if let Some(hashmap) = subscriptions.get(pubkey) {
+            for (_bank_sub_id, sink) in hashmap.iter() {
+                sink.notify(Ok(account.clone())).wait().unwrap();
+            }
+        }
+    }
+
+    pub fn check_signature(&self, signature: &Signature, bank_error: &bank::Result<()>) {
+        let status = match bank_error {
+            Ok(_) => RpcSignatureStatus::Confirmed,
+            Err(BankError::AccountInUse) => RpcSignatureStatus::AccountInUse,
+            Err(BankError::ProgramError(_, _)) => RpcSignatureStatus::ProgramRuntimeError,
+            Err(_) => RpcSignatureStatus::GenericFailure,
+        };
+
+        let mut subscriptions = self.signature_subscriptions.write().unwrap();
+        if let Some(hashmap) = subscriptions.get(signature) {
+            for (_bank_sub_id, sink) in hashmap.iter() {
+                sink.notify(Ok(status)).wait().unwrap();
+            }
+        }
+        subscriptions.remove(&signature);
+    }
+
+    pub fn add_account_subscription(
+        &self,
+        pubkey: &Pubkey,
+        sub_id: &SubscriptionId,
+        sink: &Sink<Account>,
+    ) {
+        let mut subscriptions = self.account_subscriptions.write().unwrap();
+        if let Some(current_hashmap) = subscriptions.get_mut(pubkey) {
+            current_hashmap.insert(sub_id.clone(), sink.clone());
+            return;
+        }
+        let mut hashmap = HashMap::new();
+        hashmap.insert(sub_id.clone(), sink.clone());
+        subscriptions.insert(*pubkey, hashmap);
+    }
+
+    pub fn remove_account_subscription(&self, id: &SubscriptionId) -> bool {
+        let mut subscriptions = self.account_subscriptions.write().unwrap();
+        let mut found = false;
+        subscriptions.retain(|_, v| {
+            v.retain(|k, _| {
+                if *k == *id {
+                    found = true;
+                }
+                !found
+            });
+            !v.is_empty()
+        });
+        found
+    }
+
+    pub fn add_signature_subscription(
+        &self,
+        signature: &Signature,
+        sub_id: &SubscriptionId,
+        sink: &Sink<RpcSignatureStatus>,
+    ) {
+        let mut subscriptions = self.signature_subscriptions.write().unwrap();
+        if let Some(current_hashmap) = subscriptions.get_mut(signature) {
+            current_hashmap.insert(sub_id.clone(), sink.clone());
+            return;
+        }
+        let mut hashmap = HashMap::new();
+        hashmap.insert(sub_id.clone(), sink.clone());
+        subscriptions.insert(*signature, hashmap);
+    }
+
+    pub fn remove_signature_subscription(&self, id: &SubscriptionId) -> bool {
+        let mut subscriptions = self.signature_subscriptions.write().unwrap();
+        let mut found = false;
+        subscriptions.retain(|_, v| {
+            v.retain(|k, _| {
+                if *k == *id {
+                    found = true;
+                }
+                !found
+            });
+            !v.is_empty()
+        });
+        found
+    }
+
+    /// Notify subscribers of changes to any accounts or new signatures since
+    /// the bank's last checkpoint.
+    pub fn notify_subscribers(&self, bank: &Bank) {
+        let pubkeys: Vec<_> = {
+            let subs = self.account_subscriptions.read().unwrap();
+            subs.keys().cloned().collect()
+        };
+        for pubkey in &pubkeys {
+            if let Some(account) = &bank.get_account_modified_since_parent(pubkey) {
+                self.check_account(pubkey, account);
+            }
+        }
+
+        let signatures: Vec<_> = {
+            let subs = self.signature_subscriptions.read().unwrap();
+            subs.keys().cloned().collect()
+        };
+        for signature in &signatures {
+            let status = bank.get_signature_status(signature).unwrap();
+            self.check_signature(signature, &status);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use jsonrpc_pubsub::typed::Subscriber;
+    use solana_sdk::budget_program;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+    use tokio::prelude::{Async, Stream};
+
+    #[test]
+    fn test_check_account_subscribe() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(100);
+        let bank = Bank::new(&genesis_block);
+        let alice = Keypair::new();
+        let last_id = bank.last_id();
+        let tx = SystemTransaction::new_program_account(
+            &mint_keypair,
+            alice.pubkey(),
+            last_id,
+            1,
+            16,
+            budget_program::id(),
+            0,
+        );
+        bank.process_transaction(&tx).unwrap();
+
+        let (subscriber, _id_receiver, mut transport_receiver) =
+            Subscriber::new_test("accountNotification");
+        let sub_id = SubscriptionId::Number(0 as u64);
+        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
+        let subscriptions = RpcSubscriptions::default();
+        subscriptions.add_account_subscription(&alice.pubkey(), &sub_id, &sink);
+
+        assert!(subscriptions
+            .account_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&alice.pubkey()));
+
+        let account = bank.get_account(&alice.pubkey()).unwrap();
+        subscriptions.check_account(&alice.pubkey(), &account);
+        let string = transport_receiver.poll();
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            let expected = format!(r#"{{"jsonrpc":"2.0","method":"accountNotification","params":{{"result":{{"executable":false,"owner":[129,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"tokens":1,"userdata":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},"subscription":0}}}}"#);
+            assert_eq!(expected, response);
+        }
+
+        subscriptions.remove_account_subscription(&sub_id);
+        assert!(!subscriptions
+            .account_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&alice.pubkey()));
+    }
+    #[test]
+    fn test_check_signature_subscribe() {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(100);
+        let bank = Bank::new(&genesis_block);
+        let alice = Keypair::new();
+        let last_id = bank.last_id();
+        let tx = SystemTransaction::new_move(&mint_keypair, alice.pubkey(), 20, last_id, 0);
+        let signature = tx.signatures[0];
+        bank.process_transaction(&tx).unwrap();
+
+        let (subscriber, _id_receiver, mut transport_receiver) =
+            Subscriber::new_test("signatureNotification");
+        let sub_id = SubscriptionId::Number(0 as u64);
+        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
+        let subscriptions = RpcSubscriptions::default();
+        subscriptions.add_signature_subscription(&signature, &sub_id, &sink);
+
+        assert!(subscriptions
+            .signature_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&signature));
+
+        subscriptions.check_signature(&signature, &Ok(()));
+        let string = transport_receiver.poll();
+        if let Async::Ready(Some(response)) = string.unwrap() {
+            let expected = format!(r#"{{"jsonrpc":"2.0","method":"signatureNotification","params":{{"result":"Confirmed","subscription":0}}}}"#);
+            assert_eq!(expected, response);
+        }
+
+        subscriptions.remove_signature_subscription(&sub_id);
+        assert!(!subscriptions
+            .signature_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&signature));
+    }
+}

core/src/service.rs

@@ -0,0 +1,30 @@
+//! The `service` module implements a trait used by services and stages.
+//!
+//! A Service is any object that implements its functionality on a separate thread. It implements a
+//! `join()` method, which can be used to wait for that thread to close.
+//!
+//! The Service trait may also be used to implement a pipeline stage. Like a service, its
+//! functionality is also implemented by a thread, but unlike a service, a stage isn't a server
+//! that replies to client requests. Instead, a stage acts more like a pure function. It's a oneway
+//! street. It processes messages from its input channel and then sends the processed data to an
+//! output channel. Stages can be composed to form a linear chain called a pipeline.
+//!
+//! The approach to creating a pipeline stage in Rust may be unique to Solana. We haven't seen the
+//! same technique used in other Rust projects and there may be better ways to do it. The Solana
+//! approach defines a stage as an object that communicates to its previous stage and the next
+//! stage using channels. By convention, each stage accepts a *receiver* for input and creates a
+//! second output channel. The second channel is used to pass data to the next stage, and so its
+//! sender is moved into the stage's thread and the receiver is returned from its constructor.
+//!
+//! A well-written stage should create a thread and call a short `run()` method.  The method should
+//! read input from its input channel, call a function from another module that processes it, and
+//! then send the output to the output channel. The functionality in the second module will likely
+//! not use threads or channels.
+
+use std::thread::Result;
+
+pub trait Service {
+    type JoinReturnType;
+
+    fn join(self) -> Result<Self::JoinReturnType>;
+}

core/src/sigverify.rs

@@ -0,0 +1,541 @@
+//! The `sigverify` module provides digital signature verification functions.
+//! By default, signatures are verified in parallel using all available CPU
+//! cores.  When `--features=cuda` is enabled, signature verification is
+//! offloaded to the GPU.
+//!
+
+use crate::packet::{Packet, SharedPackets};
+use crate::result::Result;
+use solana_metrics::counter::Counter;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::shortvec::decode_len;
+use solana_sdk::signature::Signature;
+#[cfg(test)]
+use solana_sdk::transaction::Transaction;
+use std::io::Cursor;
+use std::mem::size_of;
+
+pub const TX_OFFSET: usize = 0;
+
+type TxOffsets = (Vec<u32>, Vec<u32>, Vec<u32>, Vec<u32>, Vec<Vec<u32>>);
+
+#[cfg(feature = "cuda")]
+#[repr(C)]
+struct Elems {
+    elems: *const Packet,
+    num: u32,
+}
+
+#[cfg(feature = "cuda")]
+#[link(name = "cuda-crypt")]
+extern "C" {
+    fn ed25519_init() -> bool;
+    fn ed25519_set_verbose(val: bool);
+    fn ed25519_verify_many(
+        vecs: *const Elems,
+        num: u32,          //number of vecs
+        message_size: u32, //size of each element inside the elems field of the vec
+        total_packets: u32,
+        total_signatures: u32,
+        message_lens: *const u32,
+        pubkey_offsets: *const u32,
+        signature_offsets: *const u32,
+        signed_message_offsets: *const u32,
+        out: *mut u8, //combined length of all the items in vecs
+    ) -> u32;
+
+    pub fn chacha_cbc_encrypt_many_sample(
+        input: *const u8,
+        sha_state: *mut u8,
+        in_len: usize,
+        keys: *const u8,
+        ivec: *mut u8,
+        num_keys: u32,
+        samples: *const u64,
+        num_samples: u32,
+        starting_block: u64,
+        time_us: *mut f32,
+    );
+
+    pub fn chacha_init_sha_state(sha_state: *mut u8, num_keys: u32);
+    pub fn chacha_end_sha_state(sha_state_in: *const u8, out: *mut u8, num_keys: u32);
+}
+
+#[cfg(not(feature = "cuda"))]
+pub fn init() {
+    // stub
+}
+
+fn verify_packet(packet: &Packet) -> u8 {
+    use ring::signature;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::Signature;
+    use untrusted;
+
+    let (sig_len, sig_start, msg_start, pubkey_start) = get_packet_offsets(packet, 0);
+    let mut sig_start = sig_start as usize;
+    let mut pubkey_start = pubkey_start as usize;
+    let msg_start = msg_start as usize;
+
+    if packet.meta.size <= msg_start {
+        return 0;
+    }
+
+    let msg_end = packet.meta.size;
+    for _ in 0..sig_len {
+        let pubkey_end = pubkey_start as usize + size_of::<Pubkey>();
+        let sig_end = sig_start as usize + size_of::<Signature>();
+
+        if pubkey_end >= packet.meta.size || sig_end >= packet.meta.size {
+            return 0;
+        }
+
+        if signature::verify(
+            &signature::ED25519,
+            untrusted::Input::from(&packet.data[pubkey_start..pubkey_end]),
+            untrusted::Input::from(&packet.data[msg_start..msg_end]),
+            untrusted::Input::from(&packet.data[sig_start..sig_end]),
+        )
+        .is_err()
+        {
+            return 0;
+        }
+        pubkey_start += size_of::<Pubkey>();
+        sig_start += size_of::<Signature>();
+    }
+    1
+}
+
+fn verify_packet_disabled(_packet: &Packet) -> u8 {
+    warn!("signature verification is disabled");
+    1
+}
+
+fn batch_size(batches: &[SharedPackets]) -> usize {
+    batches
+        .iter()
+        .map(|p| p.read().unwrap().packets.len())
+        .sum()
+}
+
+#[cfg(not(feature = "cuda"))]
+pub fn ed25519_verify(batches: &[SharedPackets]) -> Vec<Vec<u8>> {
+    ed25519_verify_cpu(batches)
+}
+
+pub fn get_packet_offsets(packet: &Packet, current_offset: u32) -> (u32, u32, u32, u32) {
+    // Read in the size of signatures array
+    let start_offset = TX_OFFSET + size_of::<u64>();
+    let mut rd = Cursor::new(&packet.data[start_offset..]);
+    let sig_len = decode_len(&mut rd).unwrap();
+    let sig_size = rd.position() as usize;
+    let msg_start_offset = start_offset + sig_size + sig_len * size_of::<Signature>();
+    let mut rd = Cursor::new(&packet.data[msg_start_offset..]);
+    let _ = decode_len(&mut rd).unwrap();
+    let pubkey_size = rd.position() as usize;
+    let pubkey_offset = current_offset as usize + msg_start_offset + pubkey_size;
+
+    let sig_start = start_offset + current_offset as usize + sig_size;
+
+    (
+        sig_len as u32,
+        sig_start as u32,
+        current_offset + msg_start_offset as u32,
+        pubkey_offset as u32,
+    )
+}
+
+pub fn generate_offsets(batches: &[SharedPackets]) -> Result<TxOffsets> {
+    let mut signature_offsets: Vec<_> = Vec::new();
+    let mut pubkey_offsets: Vec<_> = Vec::new();
+    let mut msg_start_offsets: Vec<_> = Vec::new();
+    let mut msg_sizes: Vec<_> = Vec::new();
+    let mut current_packet = 0;
+    let mut v_sig_lens = Vec::new();
+    batches.iter().for_each(|p| {
+        let mut sig_lens = Vec::new();
+        p.read().unwrap().packets.iter().for_each(|packet| {
+            let current_offset = current_packet as u32 * size_of::<Packet>() as u32;
+
+            let (sig_len, sig_start, msg_start_offset, pubkey_offset) =
+                get_packet_offsets(packet, current_offset);
+            let mut pubkey_offset = pubkey_offset;
+
+            sig_lens.push(sig_len);
+
+            trace!("pubkey_offset: {}", pubkey_offset);
+            let mut sig_offset = sig_start;
+            for _ in 0..sig_len {
+                signature_offsets.push(sig_offset);
+                sig_offset += size_of::<Signature>() as u32;
+
+                pubkey_offsets.push(pubkey_offset);
+                pubkey_offset += size_of::<Pubkey>() as u32;
+
+                msg_start_offsets.push(msg_start_offset);
+
+                msg_sizes.push(current_offset + (packet.meta.size as u32) - msg_start_offset);
+            }
+            current_packet += 1;
+        });
+        v_sig_lens.push(sig_lens);
+    });
+    Ok((
+        signature_offsets,
+        pubkey_offsets,
+        msg_start_offsets,
+        msg_sizes,
+        v_sig_lens,
+    ))
+}
+
+pub fn ed25519_verify_cpu(batches: &[SharedPackets]) -> Vec<Vec<u8>> {
+    use rayon::prelude::*;
+    let count = batch_size(batches);
+    info!("CPU ECDSA for {}", batch_size(batches));
+    let rv = batches
+        .into_par_iter()
+        .map(|p| {
+            p.read()
+                .unwrap()
+                .packets
+                .par_iter()
+                .map(verify_packet)
+                .collect()
+        })
+        .collect();
+    inc_new_counter_info!("ed25519_verify_cpu", count);
+    rv
+}
+
+pub fn ed25519_verify_disabled(batches: &[SharedPackets]) -> Vec<Vec<u8>> {
+    use rayon::prelude::*;
+    let count = batch_size(batches);
+    info!("disabled ECDSA for {}", batch_size(batches));
+    let rv = batches
+        .into_par_iter()
+        .map(|p| {
+            p.read()
+                .unwrap()
+                .packets
+                .par_iter()
+                .map(verify_packet_disabled)
+                .collect()
+        })
+        .collect();
+    inc_new_counter_info!("ed25519_verify_disabled", count);
+    rv
+}
+
+#[cfg(feature = "cuda")]
+pub fn init() {
+    unsafe {
+        ed25519_set_verbose(true);
+        if !ed25519_init() {
+            panic!("ed25519_init() failed");
+        }
+        ed25519_set_verbose(false);
+    }
+}
+
+#[cfg(feature = "cuda")]
+pub fn ed25519_verify(batches: &[SharedPackets]) -> Vec<Vec<u8>> {
+    use crate::packet::PACKET_DATA_SIZE;
+    let count = batch_size(batches);
+
+    // micro-benchmarks show GPU time for smallest batch around 15-20ms
+    // and CPU speed for 64-128 sigverifies around 10-20ms. 64 is a nice
+    // power-of-two number around that accounting for the fact that the CPU
+    // may be busy doing other things while being a real fullnode
+    // TODO: dynamically adjust this crossover
+    if count < 64 {
+        return ed25519_verify_cpu(batches);
+    }
+
+    let (signature_offsets, pubkey_offsets, msg_start_offsets, msg_sizes, sig_lens) =
+        generate_offsets(batches).unwrap();
+
+    info!("CUDA ECDSA for {}", batch_size(batches));
+    let mut out = Vec::new();
+    let mut elems = Vec::new();
+    let mut locks = Vec::new();
+    let mut rvs = Vec::new();
+
+    for packets in batches {
+        locks.push(packets.read().unwrap());
+    }
+    let mut num_packets = 0;
+    for p in locks {
+        elems.push(Elems {
+            elems: p.packets.as_ptr(),
+            num: p.packets.len() as u32,
+        });
+        let mut v = Vec::new();
+        v.resize(p.packets.len(), 0);
+        rvs.push(v);
+        num_packets += p.packets.len();
+    }
+    out.resize(signature_offsets.len(), 0);
+    trace!("Starting verify num packets: {}", num_packets);
+    trace!("elem len: {}", elems.len() as u32);
+    trace!("packet sizeof: {}", size_of::<Packet>() as u32);
+    trace!("len offset: {}", PACKET_DATA_SIZE as u32);
+    unsafe {
+        let res = ed25519_verify_many(
+            elems.as_ptr(),
+            elems.len() as u32,
+            size_of::<Packet>() as u32,
+            num_packets as u32,
+            signature_offsets.len() as u32,
+            msg_sizes.as_ptr(),
+            pubkey_offsets.as_ptr(),
+            signature_offsets.as_ptr(),
+            msg_start_offsets.as_ptr(),
+            out.as_mut_ptr(),
+        );
+        if res != 0 {
+            trace!("RETURN!!!: {}", res);
+        }
+    }
+    trace!("done verify");
+    let mut num = 0;
+    for (vs, sig_vs) in rvs.iter_mut().zip(sig_lens.iter()) {
+        for (v, sig_v) in vs.iter_mut().zip(sig_vs.iter()) {
+            let mut vout = 1;
+            for _ in 0..*sig_v {
+                if 0 == out[num] {
+                    vout = 0;
+                }
+                num += 1;
+            }
+            *v = vout;
+            if *v != 0 {
+                trace!("VERIFIED PACKET!!!!!");
+            }
+        }
+    }
+    inc_new_counter_info!("ed25519_verify_gpu", count);
+    rvs
+}
+
+#[cfg(test)]
+pub fn make_packet_from_transaction(tx: Transaction) -> Packet {
+    use bincode::serialize;
+
+    let tx_bytes = serialize(&tx).unwrap();
+    let mut packet = Packet::default();
+    packet.meta.size = tx_bytes.len();
+    packet.data[..packet.meta.size].copy_from_slice(&tx_bytes);
+    return packet;
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::packet::{Packet, SharedPackets};
+    use crate::sigverify;
+    use crate::test_tx::test_tx;
+    use bincode::{deserialize, serialize};
+    use solana_sdk::budget_program;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_instruction::SystemInstruction;
+    use solana_sdk::system_program;
+    use solana_sdk::transaction::{Instruction, Transaction};
+
+    const SIG_OFFSET: usize = std::mem::size_of::<u64>() + 1;
+
+    pub fn memfind<A: Eq>(a: &[A], b: &[A]) -> Option<usize> {
+        assert!(a.len() >= b.len());
+        let end = a.len() - b.len() + 1;
+        for i in 0..end {
+            if a[i..i + b.len()] == b[..] {
+                return Some(i);
+            }
+        }
+        None
+    }
+
+    #[test]
+    fn test_layout() {
+        let tx = test_tx();
+        let tx_bytes = serialize(&tx).unwrap();
+        let packet = serialize(&tx).unwrap();
+        assert_matches!(memfind(&packet, &tx_bytes), Some(sigverify::TX_OFFSET));
+        assert_matches!(memfind(&packet, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), None);
+    }
+
+    #[test]
+    fn test_system_transaction_layout() {
+        let tx = test_tx();
+        let tx_bytes = serialize(&tx).unwrap();
+        let message = tx.message();
+        let packet = sigverify::make_packet_from_transaction(tx.clone());
+
+        let (sig_len, sig_start, msg_start_offset, pubkey_offset) =
+            sigverify::get_packet_offsets(&packet, 0);
+
+        assert_eq!(
+            memfind(&tx_bytes, &tx.signatures[0].as_ref()),
+            Some(SIG_OFFSET)
+        );
+        assert_eq!(
+            memfind(&tx_bytes, &tx.account_keys[0].as_ref()),
+            Some(pubkey_offset as usize)
+        );
+        assert_eq!(
+            memfind(&tx_bytes, &message),
+            Some(msg_start_offset as usize)
+        );
+        assert_eq!(
+            memfind(&tx_bytes, &tx.signatures[0].as_ref()),
+            Some(sig_start as usize)
+        );
+        assert_eq!(sig_len, 1);
+        assert!(tx.verify_signature());
+    }
+
+    #[test]
+    fn test_system_transaction_userdata_layout() {
+        use crate::packet::PACKET_DATA_SIZE;
+        let mut tx0 = test_tx();
+        tx0.instructions[0].userdata = vec![1, 2, 3];
+        let message0a = tx0.message();
+        let tx_bytes = serialize(&tx0).unwrap();
+        assert!(tx_bytes.len() < PACKET_DATA_SIZE);
+        assert_eq!(
+            memfind(&tx_bytes, &tx0.signatures[0].as_ref()),
+            Some(SIG_OFFSET)
+        );
+        let tx1 = deserialize(&tx_bytes).unwrap();
+        assert_eq!(tx0, tx1);
+        assert_eq!(tx1.instructions[0].userdata, vec![1, 2, 3]);
+
+        tx0.instructions[0].userdata = vec![1, 2, 4];
+        let message0b = tx0.message();
+        assert_ne!(message0a, message0b);
+    }
+
+    #[test]
+    fn test_get_packet_offsets() {
+        let tx = test_tx();
+        let packet = sigverify::make_packet_from_transaction(tx);
+        let (sig_len, sig_start, msg_start_offset, pubkey_offset) =
+            sigverify::get_packet_offsets(&packet, 0);
+        assert_eq!(sig_len, 1);
+        assert_eq!(sig_start, 9);
+        assert_eq!(msg_start_offset, 73);
+        assert_eq!(pubkey_offset, 74);
+    }
+
+    fn generate_packet_vec(
+        packet: &Packet,
+        num_packets_per_batch: usize,
+        num_batches: usize,
+    ) -> Vec<SharedPackets> {
+        // generate packet vector
+        let batches: Vec<_> = (0..num_batches)
+            .map(|_| {
+                let packets = SharedPackets::default();
+                packets
+                    .write()
+                    .unwrap()
+                    .packets
+                    .resize(0, Packet::default());
+                for _ in 0..num_packets_per_batch {
+                    packets.write().unwrap().packets.push(packet.clone());
+                }
+                assert_eq!(packets.read().unwrap().packets.len(), num_packets_per_batch);
+                packets
+            })
+            .collect();
+        assert_eq!(batches.len(), num_batches);
+
+        batches
+    }
+
+    fn test_verify_n(n: usize, modify_data: bool) {
+        let tx = test_tx();
+        let mut packet = sigverify::make_packet_from_transaction(tx);
+
+        // jumble some data to test failure
+        if modify_data {
+            packet.data[20] = packet.data[20].wrapping_add(10);
+        }
+
+        let batches = generate_packet_vec(&packet, n, 2);
+
+        // verify packets
+        let ans = sigverify::ed25519_verify(&batches);
+
+        // check result
+        let ref_ans = if modify_data { 0u8 } else { 1u8 };
+        assert_eq!(ans, vec![vec![ref_ans; n], vec![ref_ans; n]]);
+    }
+
+    #[test]
+    fn test_verify_zero() {
+        test_verify_n(0, false);
+    }
+
+    #[test]
+    fn test_verify_one() {
+        test_verify_n(1, false);
+    }
+
+    #[test]
+    fn test_verify_seventy_one() {
+        test_verify_n(71, false);
+    }
+
+    #[test]
+    fn test_verify_multi_sig() {
+        solana_logger::setup();
+        let keypair0 = Keypair::new();
+        let keypair1 = Keypair::new();
+        let keypairs = vec![&keypair0, &keypair1];
+        let tokens = 5;
+        let fee = 2;
+        let last_id = Hash::default();
+
+        let keys = vec![keypair0.pubkey(), keypair1.pubkey()];
+
+        let system_instruction = SystemInstruction::Move { tokens };
+
+        let program_ids = vec![system_program::id(), budget_program::id()];
+
+        let instructions = vec![Instruction::new(0, &system_instruction, vec![0, 1])];
+
+        let tx = Transaction::new_with_instructions(
+            &keypairs,
+            &keys,
+            last_id,
+            fee,
+            program_ids,
+            instructions,
+        );
+
+        let mut packet = sigverify::make_packet_from_transaction(tx);
+
+        let n = 4;
+        let num_batches = 3;
+        let batches = generate_packet_vec(&packet, n, num_batches);
+
+        packet.data[40] = packet.data[40].wrapping_add(8);
+
+        batches[0].write().unwrap().packets.push(packet);
+
+        // verify packets
+        let ans = sigverify::ed25519_verify(&batches);
+
+        // check result
+        let ref_ans = 1u8;
+        let mut ref_vec = vec![vec![ref_ans; n]; num_batches];
+        ref_vec[0].push(0u8);
+        assert_eq!(ans, ref_vec);
+    }
+
+    #[test]
+    fn test_verify_fail() {
+        test_verify_n(5, true);
+    }
+}

core/src/sigverify_stage.rs

@@ -0,0 +1,155 @@
+//! The `sigverify_stage` implements the signature verification stage of the TPU. It
+//! receives a list of lists of packets and outputs the same list, but tags each
+//! top-level list with a list of booleans, telling the next stage whether the
+//! signature in that packet is valid. It assumes each packet contains one
+//! transaction. All processing is done on the CPU by default and on a GPU
+//! if the `cuda` feature is enabled with `--features=cuda`.
+
+use crate::packet::SharedPackets;
+use crate::result::{Error, Result};
+use crate::service::Service;
+use crate::sigverify;
+use crate::streamer::{self, PacketReceiver};
+use rand::{thread_rng, Rng};
+use solana_metrics::counter::Counter;
+use solana_metrics::{influxdb, submit};
+use solana_sdk::timing;
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
+use std::sync::{Arc, Mutex};
+use std::thread::{self, spawn, JoinHandle};
+use std::time::Instant;
+
+pub type VerifiedPackets = Vec<(SharedPackets, Vec<u8>)>;
+
+pub struct SigVerifyStage {
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl SigVerifyStage {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        packet_receiver: Receiver<SharedPackets>,
+        sigverify_disabled: bool,
+    ) -> (Self, Receiver<VerifiedPackets>) {
+        sigverify::init();
+        let (verified_sender, verified_receiver) = channel();
+        let thread_hdls =
+            Self::verifier_services(packet_receiver, verified_sender, sigverify_disabled);
+        (Self { thread_hdls }, verified_receiver)
+    }
+
+    fn verify_batch(batch: Vec<SharedPackets>, sigverify_disabled: bool) -> VerifiedPackets {
+        let r = if sigverify_disabled {
+            sigverify::ed25519_verify_disabled(&batch)
+        } else {
+            sigverify::ed25519_verify(&batch)
+        };
+        batch.into_iter().zip(r).collect()
+    }
+
+    fn verifier(
+        recvr: &Arc<Mutex<PacketReceiver>>,
+        sendr: &Arc<Mutex<Sender<VerifiedPackets>>>,
+        sigverify_disabled: bool,
+    ) -> Result<()> {
+        let (batch, len, recv_time) =
+            streamer::recv_batch(&recvr.lock().expect("'recvr' lock in fn verifier"))?;
+        inc_new_counter_info!("sigverify_stage-entries_received", len);
+
+        let now = Instant::now();
+        let batch_len = batch.len();
+        let rand_id = thread_rng().gen_range(0, 100);
+        info!(
+            "@{:?} verifier: verifying: {} id: {}",
+            timing::timestamp(),
+            batch.len(),
+            rand_id
+        );
+
+        let verified_batch = Self::verify_batch(batch, sigverify_disabled);
+        inc_new_counter_info!(
+            "sigverify_stage-verified_entries_send",
+            verified_batch.len()
+        );
+
+        if sendr
+            .lock()
+            .expect("lock in fn verify_batch in tpu")
+            .send(verified_batch)
+            .is_err()
+        {
+            return Err(Error::SendError);
+        }
+
+        let total_time_ms = timing::duration_as_ms(&now.elapsed());
+        let total_time_s = timing::duration_as_s(&now.elapsed());
+        inc_new_counter_info!(
+            "sigverify_stage-time_ms",
+            (total_time_ms + recv_time) as usize
+        );
+        info!(
+            "@{:?} verifier: done. batches: {} total verify time: {:?} id: {} verified: {} v/s {}",
+            timing::timestamp(),
+            batch_len,
+            total_time_ms,
+            rand_id,
+            len,
+            (len as f32 / total_time_s)
+        );
+
+        submit(
+            influxdb::Point::new("sigverify_stage-total_verify_time")
+                .add_field("batch_len", influxdb::Value::Integer(batch_len as i64))
+                .add_field("len", influxdb::Value::Integer(len as i64))
+                .add_field(
+                    "total_time_ms",
+                    influxdb::Value::Integer(total_time_ms as i64),
+                )
+                .to_owned(),
+        );
+
+        Ok(())
+    }
+
+    fn verifier_service(
+        packet_receiver: Arc<Mutex<PacketReceiver>>,
+        verified_sender: Arc<Mutex<Sender<VerifiedPackets>>>,
+        sigverify_disabled: bool,
+    ) -> JoinHandle<()> {
+        spawn(move || loop {
+            if let Err(e) = Self::verifier(&packet_receiver, &verified_sender, sigverify_disabled) {
+                match e {
+                    Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                    Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                    Error::SendError => {
+                        break;
+                    }
+                    _ => error!("{:?}", e),
+                }
+            }
+        })
+    }
+
+    fn verifier_services(
+        packet_receiver: PacketReceiver,
+        verified_sender: Sender<VerifiedPackets>,
+        sigverify_disabled: bool,
+    ) -> Vec<JoinHandle<()>> {
+        let sender = Arc::new(Mutex::new(verified_sender));
+        let receiver = Arc::new(Mutex::new(packet_receiver));
+        (0..4)
+            .map(|_| Self::verifier_service(receiver.clone(), sender.clone(), sigverify_disabled))
+            .collect()
+    }
+}
+
+impl Service for SigVerifyStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

core/src/staking_utils.rs

@@ -0,0 +1,259 @@
+use hashbrown::HashMap;
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::vote_program::VoteState;
+
+/// Looks through vote accounts, and finds the latest slot that has achieved
+/// supermajority lockout
+pub fn get_supermajority_slot(bank: &Bank, epoch_height: u64) -> Option<u64> {
+    // Find the amount of stake needed for supermajority
+    let stakes_and_lockouts = epoch_stakes_and_lockouts(bank, epoch_height);
+    let total_stake: u64 = stakes_and_lockouts.iter().map(|s| s.0).sum();
+    let supermajority_stake = total_stake * 2 / 3;
+
+    // Filter out the states that don't have a max lockout
+    find_supermajority_slot(supermajority_stake, stakes_and_lockouts.iter())
+}
+
+/// Collect the node Pubkey and staker account balance for nodes
+/// that have non-zero balance in their corresponding staking accounts
+pub fn node_stakes(bank: &Bank) -> HashMap<Pubkey, u64> {
+    sum_node_stakes(&node_stakes_extractor(bank, |stake, _| stake))
+}
+
+/// Return the checkpointed stakes that should be used to generate a leader schedule.
+pub fn node_stakes_at_epoch(bank: &Bank, epoch_height: u64) -> HashMap<Pubkey, u64> {
+    sum_node_stakes(&node_stakes_at_epoch_extractor(
+        bank,
+        epoch_height,
+        |stake, _| stake,
+    ))
+}
+
+/// Sum up all the staking accounts for each delegate
+fn sum_node_stakes(stakes: &HashMap<Pubkey, Vec<u64>>) -> HashMap<Pubkey, u64> {
+    stakes
+        .iter()
+        .map(|(delegate, stakes)| (*delegate, stakes.iter().sum()))
+        .collect()
+}
+
+/// Return the checkpointed stakes that should be used to generate a leader schedule.
+/// state_extractor takes (stake, vote_state) and maps to an output.
+fn node_stakes_at_epoch_extractor<F, T: Clone>(
+    bank: &Bank,
+    epoch_height: u64,
+    state_extractor: F,
+) -> HashMap<Pubkey, Vec<T>>
+where
+    F: Fn(u64, &VoteState) -> T,
+{
+    let epoch_slot_height = epoch_height * bank.slots_per_epoch();
+    node_stakes_at_slot_extractor(bank, epoch_slot_height, state_extractor)
+}
+
+/// Return the checkpointed stakes that should be used to generate a leader schedule.
+/// state_extractor takes (stake, vote_state) and maps to an output
+fn node_stakes_at_slot_extractor<F, T: Clone>(
+    bank: &Bank,
+    current_slot_height: u64,
+    state_extractor: F,
+) -> HashMap<Pubkey, Vec<T>>
+where
+    F: Fn(u64, &VoteState) -> T,
+{
+    let slot_height = current_slot_height.saturating_sub(bank.stakers_slot_offset());
+
+    let parents = bank.parents();
+    let mut banks = vec![bank];
+    banks.extend(parents.iter().map(|x| x.as_ref()));
+
+    let bank = banks
+        .iter()
+        .find(|bank| bank.slot() <= slot_height)
+        .unwrap_or_else(|| banks.last().unwrap());
+
+    node_stakes_extractor(bank, state_extractor)
+}
+
+/// Collect the node Pubkey and staker account balance for nodes
+/// that have non-zero balance in their corresponding staker accounts.
+/// state_extractor takes (stake, vote_state) and maps to an output
+fn node_stakes_extractor<F, T: Clone>(bank: &Bank, state_extractor: F) -> HashMap<Pubkey, Vec<T>>
+where
+    F: Fn(u64, &VoteState) -> T,
+{
+    let mut map: HashMap<Pubkey, Vec<T>> = HashMap::new();
+    let vote_states = bank.vote_states(|account_id, _| bank.get_balance(&account_id) > 0);
+    vote_states.into_iter().for_each(|(account_id, state)| {
+        if map.contains_key(&state.delegate_id) {
+            let entry = map.get_mut(&state.delegate_id).unwrap();
+            entry.push(state_extractor(bank.get_balance(&account_id), &state));
+        } else {
+            map.insert(
+                state.delegate_id,
+                vec![state_extractor(bank.get_balance(&account_id), &state)],
+            );
+        }
+    });
+    map
+}
+
+fn epoch_stakes_and_lockouts(bank: &Bank, epoch_height: u64) -> Vec<(u64, Option<u64>)> {
+    node_stakes_at_epoch_extractor(bank, epoch_height, |stake, states| {
+        (stake, states.root_slot)
+    })
+    .into_iter()
+    .flat_map(|(_, stake_and_states)| stake_and_states)
+    .collect()
+}
+
+fn find_supermajority_slot<'a, I>(supermajority_stake: u64, stakes_and_lockouts: I) -> Option<u64>
+where
+    I: Iterator<Item = &'a (u64, Option<u64>)>,
+{
+    // Filter out the states that don't have a max lockout
+    let mut stakes_and_lockouts: Vec<_> = stakes_and_lockouts
+        .filter_map(|(stake, slot)| slot.map(|s| (stake, s)))
+        .collect();
+
+    // Sort by the root slot, in descending order
+    stakes_and_lockouts.sort_unstable_by(|s1, s2| s1.1.cmp(&s2.1).reverse());
+
+    // Find if any slot has achieved sufficient votes for supermajority lockout
+    let mut total = 0;
+    for (stake, slot) in stakes_and_lockouts {
+        total += stake;
+        if total > supermajority_stake {
+            return Some(slot);
+        }
+    }
+
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::voting_keypair::tests as voting_keypair_tests;
+    use hashbrown::HashSet;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use std::iter::FromIterator;
+    use std::sync::Arc;
+
+    fn register_ticks(bank: &Bank, n: u64) -> (u64, u64, u64) {
+        for _ in 0..n {
+            bank.register_tick(&Hash::default());
+        }
+        (bank.tick_index(), bank.slot_index(), bank.epoch_height())
+    }
+
+    fn new_from_parent(parent: &Arc<Bank>) -> Bank {
+        Bank::new_from_parent(parent, Pubkey::default(), parent.slot() + 1)
+    }
+
+    #[test]
+    fn test_bank_staked_nodes_at_epoch() {
+        let pubkey = Keypair::new().pubkey();
+        let bootstrap_tokens = 2;
+        let (genesis_block, _) = GenesisBlock::new_with_leader(2, pubkey, bootstrap_tokens);
+        let bank = Bank::new(&genesis_block);
+        let bank = new_from_parent(&Arc::new(bank));
+        let ticks_per_offset = bank.stakers_slot_offset() * bank.ticks_per_slot();
+        register_ticks(&bank, ticks_per_offset);
+        assert_eq!(bank.slot_height(), bank.stakers_slot_offset());
+
+        let mut expected = HashMap::new();
+        expected.insert(pubkey, vec![bootstrap_tokens - 1]);
+        let bank = new_from_parent(&Arc::new(bank));
+        assert_eq!(
+            node_stakes_at_slot_extractor(&bank, bank.slot_height(), |s, _| s),
+            expected
+        );
+    }
+
+    #[test]
+    fn test_epoch_stakes_and_lockouts() {
+        let validator = Keypair::new();
+
+        let (genesis_block, mint_keypair) = GenesisBlock::new(500);
+        let bank = Bank::new(&genesis_block);
+        let bank_voter = Keypair::new();
+
+        // Give the validator some stake but don't setup a staking account
+        bank.transfer(1, &mint_keypair, validator.pubkey(), genesis_block.hash())
+            .unwrap();
+
+        // Validator has no token staked, so they get filtered out. Only the bootstrap leader
+        // created by the genesis block will get included
+        let expected: Vec<_> = epoch_stakes_and_lockouts(&bank, 0);
+        assert_eq!(expected, vec![(1, None)]);
+
+        voting_keypair_tests::new_vote_account_with_vote(&mint_keypair, &bank_voter, &bank, 499, 0);
+
+        let result: HashSet<_> = HashSet::from_iter(epoch_stakes_and_lockouts(&bank, 0));
+        let expected: HashSet<_> = HashSet::from_iter(vec![(1, None), (499, None)]);
+        assert_eq!(result, expected);
+    }
+
+    #[test]
+    fn test_find_supermajority_slot() {
+        let supermajority = 10;
+
+        let stakes_and_slots = vec![];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            None
+        );
+
+        let stakes_and_slots = vec![(5, None), (5, None)];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            None
+        );
+
+        let stakes_and_slots = vec![(5, None), (5, None), (9, Some(2))];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            None
+        );
+
+        let stakes_and_slots = vec![(5, None), (5, None), (9, Some(2)), (1, Some(3))];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            None
+        );
+
+        let stakes_and_slots = vec![(5, None), (5, None), (9, Some(2)), (2, Some(3))];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            Some(2)
+        );
+
+        let stakes_and_slots = vec![(9, Some(2)), (2, Some(3)), (9, None)];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            Some(2)
+        );
+
+        let stakes_and_slots = vec![(9, Some(2)), (2, Some(3)), (9, Some(3))];
+        assert_eq!(
+            find_supermajority_slot(supermajority, stakes_and_slots.iter()),
+            Some(3)
+        );
+    }
+
+    #[test]
+    fn test_sum_node_stakes() {
+        let mut stakes = HashMap::new();
+        stakes.insert(Pubkey::default(), vec![1, 2, 3, 4, 5]);
+        assert_eq!(sum_node_stakes(&stakes).len(), 1);
+        assert_eq!(
+            sum_node_stakes(&stakes).get(&Pubkey::default()),
+            Some(&15_u64)
+        );
+    }
+}

core/src/storage_stage.rs

@@ -0,0 +1,659 @@
+// A stage that handles generating the keys used to encrypt the ledger and sample it
+// for storage mining. Replicators submit storage proofs, validator then bundles them
+// to submit its proof for mining to be rewarded.
+
+use crate::blocktree::Blocktree;
+#[cfg(all(feature = "chacha", feature = "cuda"))]
+use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
+use crate::client::mk_client_with_timeout;
+use crate::cluster_info::ClusterInfo;
+use crate::entry::{Entry, EntryReceiver};
+use crate::result::{Error, Result};
+use crate::service::Service;
+use bincode::deserialize;
+use rand::{Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, Signature};
+use solana_sdk::storage_program::{self, StorageProgram, StorageTransaction};
+use solana_sdk::transaction::Transaction;
+use solana_sdk::vote_program;
+use std::collections::HashSet;
+use std::io;
+use std::mem::size_of;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, RecvTimeoutError, Sender};
+use std::sync::{Arc, RwLock};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+// Block of hash answers to validate against
+// Vec of [ledger blocks] x [keys]
+type StorageResults = Vec<Hash>;
+type StorageKeys = Vec<u8>;
+type ReplicatorMap = Vec<HashSet<Pubkey>>;
+
+#[derive(Default)]
+pub struct StorageStateInner {
+    storage_results: StorageResults,
+    storage_keys: StorageKeys,
+    replicator_map: ReplicatorMap,
+    storage_last_id: Hash,
+    entry_height: u64,
+}
+
+#[derive(Clone, Default)]
+pub struct StorageState {
+    state: Arc<RwLock<StorageStateInner>>,
+}
+
+pub struct StorageStage {
+    t_storage_mining_verifier: JoinHandle<()>,
+    t_storage_create_accounts: JoinHandle<()>,
+}
+
+macro_rules! cross_boundary {
+    ($start:expr, $len:expr, $boundary:expr) => {
+        (($start + $len) & !($boundary - 1)) > $start & !($boundary - 1)
+    };
+}
+
+pub const STORAGE_ROTATE_TEST_COUNT: u64 = 128;
+// TODO: some way to dynamically size NUM_IDENTITIES
+const NUM_IDENTITIES: usize = 1024;
+pub const NUM_STORAGE_SAMPLES: usize = 4;
+pub const ENTRIES_PER_SEGMENT: u64 = 16;
+const KEY_SIZE: usize = 64;
+
+type TransactionSender = Sender<Transaction>;
+
+pub fn get_segment_from_entry(entry_height: u64) -> u64 {
+    entry_height / ENTRIES_PER_SEGMENT
+}
+
+fn get_identity_index_from_signature(key: &Signature) -> usize {
+    let rkey = key.as_ref();
+    let mut res: usize = (rkey[0] as usize)
+        | ((rkey[1] as usize) << 8)
+        | ((rkey[2] as usize) << 16)
+        | ((rkey[3] as usize) << 24);
+    res &= NUM_IDENTITIES - 1;
+    res
+}
+
+impl StorageState {
+    pub fn new() -> Self {
+        let storage_keys = vec![0u8; KEY_SIZE * NUM_IDENTITIES];
+        let storage_results = vec![Hash::default(); NUM_IDENTITIES];
+        let replicator_map = vec![];
+
+        let state = StorageStateInner {
+            storage_keys,
+            storage_results,
+            replicator_map,
+            entry_height: 0,
+            storage_last_id: Hash::default(),
+        };
+
+        StorageState {
+            state: Arc::new(RwLock::new(state)),
+        }
+    }
+
+    pub fn get_mining_key(&self, key: &Signature) -> Vec<u8> {
+        let idx = get_identity_index_from_signature(key);
+        self.state.read().unwrap().storage_keys[idx..idx + KEY_SIZE].to_vec()
+    }
+
+    pub fn get_mining_result(&self, key: &Signature) -> Hash {
+        let idx = get_identity_index_from_signature(key);
+        self.state.read().unwrap().storage_results[idx]
+    }
+
+    pub fn get_last_id(&self) -> Hash {
+        self.state.read().unwrap().storage_last_id
+    }
+
+    pub fn get_entry_height(&self) -> u64 {
+        self.state.read().unwrap().entry_height
+    }
+
+    pub fn get_pubkeys_for_entry_height(&self, entry_height: u64) -> Vec<Pubkey> {
+        // TODO: keep track of age?
+        const MAX_PUBKEYS_TO_RETURN: usize = 5;
+        let index = (entry_height / ENTRIES_PER_SEGMENT) as usize;
+        let replicator_map = &self.state.read().unwrap().replicator_map;
+        if index < replicator_map.len() {
+            replicator_map[index]
+                .iter()
+                .cloned()
+                .take(MAX_PUBKEYS_TO_RETURN)
+                .collect::<Vec<_>>()
+        } else {
+            vec![]
+        }
+    }
+}
+
+impl StorageStage {
+    pub fn new(
+        storage_state: &StorageState,
+        storage_entry_receiver: EntryReceiver,
+        blocktree: Option<Arc<Blocktree>>,
+        keypair: &Arc<Keypair>,
+        exit: &Arc<AtomicBool>,
+        entry_height: u64,
+        storage_rotate_count: u64,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+    ) -> Self {
+        debug!("storage_stage::new: entry_height: {}", entry_height);
+        storage_state.state.write().unwrap().entry_height = entry_height;
+        let storage_state_inner = storage_state.state.clone();
+        let exit0 = exit.clone();
+        let keypair0 = keypair.clone();
+
+        let (tx_sender, tx_receiver) = channel();
+
+        let t_storage_mining_verifier = Builder::new()
+            .name("solana-storage-mining-verify-stage".to_string())
+            .spawn(move || {
+                let mut poh_height = 0;
+                let mut current_key = 0;
+                let mut entry_height = entry_height;
+                loop {
+                    if let Some(ref some_blocktree) = blocktree {
+                        if let Err(e) = Self::process_entries(
+                            &keypair0,
+                            &storage_state_inner,
+                            &storage_entry_receiver,
+                            &some_blocktree,
+                            &mut poh_height,
+                            &mut entry_height,
+                            &mut current_key,
+                            storage_rotate_count,
+                            &tx_sender,
+                        ) {
+                            match e {
+                                Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                                Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                                _ => info!("Error from process_entries: {:?}", e),
+                            }
+                        }
+                    }
+                    if exit0.load(Ordering::Relaxed) {
+                        break;
+                    }
+                }
+            })
+            .unwrap();
+
+        let cluster_info0 = cluster_info.clone();
+        let exit1 = exit.clone();
+        let keypair1 = keypair.clone();
+        let t_storage_create_accounts = Builder::new()
+            .name("solana-storage-create-accounts".to_string())
+            .spawn(move || loop {
+                match tx_receiver.recv_timeout(Duration::from_secs(1)) {
+                    Ok(mut tx) => {
+                        if Self::send_tx(&cluster_info0, &mut tx, &exit1, &keypair1, None).is_ok() {
+                            debug!("sent tx: {:?}", tx);
+                        }
+                    }
+                    Err(e) => match e {
+                        RecvTimeoutError::Disconnected => break,
+                        RecvTimeoutError::Timeout => (),
+                    },
+                };
+
+                if exit1.load(Ordering::Relaxed) {
+                    break;
+                }
+                sleep(Duration::from_millis(100));
+            })
+            .unwrap();
+
+        StorageStage {
+            t_storage_mining_verifier,
+            t_storage_create_accounts,
+        }
+    }
+
+    fn send_tx(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        tx: &mut Transaction,
+        exit: &Arc<AtomicBool>,
+        keypair: &Arc<Keypair>,
+        account_to_create: Option<Pubkey>,
+    ) -> io::Result<()> {
+        if let Some(leader_info) = cluster_info.read().unwrap().leader_data() {
+            let mut client = mk_client_with_timeout(leader_info, Duration::from_secs(5));
+
+            if let Some(account) = account_to_create {
+                if client.get_account_userdata(&account).is_ok() {
+                    return Ok(());
+                }
+            }
+
+            let mut last_id = None;
+            for _ in 0..10 {
+                if let Some(new_last_id) = client.try_get_last_id(1) {
+                    last_id = Some(new_last_id);
+                    break;
+                }
+
+                if exit.load(Ordering::Relaxed) {
+                    Err(io::Error::new(io::ErrorKind::Other, "exit signaled"))?;
+                }
+            }
+
+            if let Some(last_id) = last_id {
+                tx.sign(&[keypair.as_ref()], last_id);
+
+                if exit.load(Ordering::Relaxed) {
+                    Err(io::Error::new(io::ErrorKind::Other, "exit signaled"))?;
+                }
+
+                if let Ok(signature) = client.transfer_signed(&tx) {
+                    for _ in 0..10 {
+                        if client.check_signature(&signature) {
+                            return Ok(());
+                        }
+
+                        if exit.load(Ordering::Relaxed) {
+                            Err(io::Error::new(io::ErrorKind::Other, "exit signaled"))?;
+                        }
+
+                        sleep(Duration::from_millis(200));
+                    }
+                }
+            }
+        }
+
+        Err(io::Error::new(io::ErrorKind::Other, "leader not found"))
+    }
+
+    pub fn process_entry_crossing(
+        state: &Arc<RwLock<StorageStateInner>>,
+        keypair: &Arc<Keypair>,
+        _blocktree: &Arc<Blocktree>,
+        entry_id: Hash,
+        entry_height: u64,
+        tx_sender: &TransactionSender,
+    ) -> Result<()> {
+        let mut seed = [0u8; 32];
+        let signature = keypair.sign(&entry_id.as_ref());
+
+        let tx = StorageTransaction::new_advertise_last_id(
+            keypair,
+            entry_id,
+            Hash::default(),
+            entry_height,
+        );
+        tx_sender.send(tx)?;
+
+        seed.copy_from_slice(&signature.as_ref()[..32]);
+
+        let mut rng = ChaChaRng::from_seed(seed);
+
+        state.write().unwrap().entry_height = entry_height;
+
+        // Regenerate the answers
+        let num_segments = (entry_height / ENTRIES_PER_SEGMENT) as usize;
+        if num_segments == 0 {
+            info!("Ledger has 0 segments!");
+            return Ok(());
+        }
+        // TODO: what if the validator does not have this segment
+        let segment = signature.as_ref()[0] as usize % num_segments;
+
+        debug!(
+            "storage verifying: segment: {} identities: {}",
+            segment, NUM_IDENTITIES,
+        );
+
+        let mut samples = vec![];
+        for _ in 0..NUM_STORAGE_SAMPLES {
+            samples.push(rng.gen_range(0, 10));
+        }
+        debug!("generated samples: {:?}", samples);
+        // TODO: cuda required to generate the reference values
+        // but if it is missing, then we need to take care not to
+        // process storage mining results.
+        #[cfg(all(feature = "chacha", feature = "cuda"))]
+        {
+            // Lock the keys, since this is the IV memory,
+            // it will be updated in-place by the encryption.
+            // Should be overwritten by the vote signatures which replace the
+            // key values by the time it runs again.
+
+            let mut statew = state.write().unwrap();
+
+            match chacha_cbc_encrypt_file_many_keys(
+                _blocktree,
+                segment as u64,
+                &mut statew.storage_keys,
+                &samples,
+            ) {
+                Ok(hashes) => {
+                    debug!("Success! encrypted ledger segment: {}", segment);
+                    statew.storage_results.copy_from_slice(&hashes);
+                }
+                Err(e) => {
+                    info!("error encrypting file: {:?}", e);
+                    Err(e)?;
+                }
+            }
+        }
+        // TODO: bundle up mining submissions from replicators
+        // and submit them in a tx to the leader to get reward.
+        Ok(())
+    }
+
+    pub fn process_entries(
+        keypair: &Arc<Keypair>,
+        storage_state: &Arc<RwLock<StorageStateInner>>,
+        entry_receiver: &EntryReceiver,
+        blocktree: &Arc<Blocktree>,
+        poh_height: &mut u64,
+        entry_height: &mut u64,
+        current_key_idx: &mut usize,
+        storage_rotate_count: u64,
+        tx_sender: &TransactionSender,
+    ) -> Result<()> {
+        let timeout = Duration::new(1, 0);
+        let entries: Vec<Entry> = entry_receiver.recv_timeout(timeout)?;
+        for entry in entries {
+            // Go through the transactions, find votes, and use them to update
+            // the storage_keys with their signatures.
+            for tx in entry.transactions {
+                for (i, program_id) in tx.program_ids.iter().enumerate() {
+                    if vote_program::check_id(&program_id) {
+                        debug!(
+                            "generating storage_keys from votes current_key_idx: {}",
+                            *current_key_idx
+                        );
+                        let storage_keys = &mut storage_state.write().unwrap().storage_keys;
+                        storage_keys[*current_key_idx..*current_key_idx + size_of::<Signature>()]
+                            .copy_from_slice(tx.signatures[0].as_ref());
+                        *current_key_idx += size_of::<Signature>();
+                        *current_key_idx %= storage_keys.len();
+                    } else if storage_program::check_id(&program_id) {
+                        match deserialize(&tx.instructions[i].userdata) {
+                            Ok(StorageProgram::SubmitMiningProof {
+                                entry_height: proof_entry_height,
+                                ..
+                            }) => {
+                                if proof_entry_height < *entry_height {
+                                    let mut statew = storage_state.write().unwrap();
+                                    let max_segment_index =
+                                        (*entry_height / ENTRIES_PER_SEGMENT) as usize;
+                                    if statew.replicator_map.len() <= max_segment_index {
+                                        statew
+                                            .replicator_map
+                                            .resize(max_segment_index, HashSet::new());
+                                    }
+                                    let proof_segment_index =
+                                        (proof_entry_height / ENTRIES_PER_SEGMENT) as usize;
+                                    if proof_segment_index < statew.replicator_map.len() {
+                                        statew.replicator_map[proof_segment_index]
+                                            .insert(tx.account_keys[0]);
+                                    }
+                                }
+                                debug!("storage proof: entry_height: {}", entry_height);
+                            }
+                            Ok(_) => {}
+                            Err(e) => {
+                                info!("error: {:?}", e);
+                            }
+                        }
+                    }
+                }
+            }
+            if cross_boundary!(*poh_height, entry.num_hashes, storage_rotate_count) {
+                trace!(
+                    "crosses sending at poh_height: {} entry_height: {}! hashes: {}",
+                    *poh_height,
+                    entry_height,
+                    entry.num_hashes
+                );
+                Self::process_entry_crossing(
+                    &storage_state,
+                    &keypair,
+                    &blocktree,
+                    entry.hash,
+                    *entry_height,
+                    tx_sender,
+                )?;
+            }
+            *entry_height += 1;
+            *poh_height += entry.num_hashes;
+        }
+        Ok(())
+    }
+}
+
+impl Service for StorageStage {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.t_storage_create_accounts.join().unwrap();
+        self.t_storage_mining_verifier.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::blocktree::{create_new_tmp_ledger, Blocktree};
+    use crate::cluster_info::{ClusterInfo, NodeInfo};
+    use crate::entry::{make_tiny_test_entries, Entry};
+    use crate::service::Service;
+    use crate::storage_stage::StorageState;
+    use crate::storage_stage::NUM_IDENTITIES;
+    use crate::storage_stage::{
+        get_identity_index_from_signature, StorageStage, STORAGE_ROTATE_TEST_COUNT,
+    };
+    use rayon::prelude::*;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::hash::Hasher;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::vote_transaction::VoteTransaction;
+    use std::cmp::{max, min};
+    use std::fs::remove_dir_all;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    #[test]
+    fn test_storage_stage_none_ledger() {
+        let keypair = Arc::new(Keypair::new());
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let cluster_info = test_cluster_info(keypair.pubkey());
+
+        let (_storage_entry_sender, storage_entry_receiver) = channel();
+        let storage_state = StorageState::new();
+        let storage_stage = StorageStage::new(
+            &storage_state,
+            storage_entry_receiver,
+            None,
+            &keypair,
+            &exit.clone(),
+            0,
+            STORAGE_ROTATE_TEST_COUNT,
+            &cluster_info,
+        );
+        exit.store(true, Ordering::Relaxed);
+        storage_stage.join().unwrap();
+    }
+
+    fn test_cluster_info(id: Pubkey) -> Arc<RwLock<ClusterInfo>> {
+        let node_info = NodeInfo::new_localhost(id, 0);
+        let cluster_info = ClusterInfo::new(node_info);
+        Arc::new(RwLock::new(cluster_info))
+    }
+
+    #[test]
+    fn test_storage_stage_process_entries() {
+        solana_logger::setup();
+        let keypair = Arc::new(Keypair::new());
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(1000);
+        let ticks_per_slot = genesis_block.ticks_per_slot;
+        let (ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        let entries = make_tiny_test_entries(64);
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap();
+        blocktree.write_entries(1, 0, 0, &entries).unwrap();
+
+        let cluster_info = test_cluster_info(keypair.pubkey());
+
+        let (storage_entry_sender, storage_entry_receiver) = channel();
+        let storage_state = StorageState::new();
+        let storage_stage = StorageStage::new(
+            &storage_state,
+            storage_entry_receiver,
+            Some(Arc::new(blocktree)),
+            &keypair,
+            &exit.clone(),
+            0,
+            STORAGE_ROTATE_TEST_COUNT,
+            &cluster_info,
+        );
+        storage_entry_sender.send(entries.clone()).unwrap();
+
+        let keypair = Keypair::new();
+        let hash = Hash::default();
+        let signature = keypair.sign_message(&hash.as_ref());
+        let mut result = storage_state.get_mining_result(&signature);
+        assert_eq!(result, Hash::default());
+
+        for _ in 0..9 {
+            storage_entry_sender.send(entries.clone()).unwrap();
+        }
+        for _ in 0..5 {
+            result = storage_state.get_mining_result(&signature);
+            if result != Hash::default() {
+                info!("found result = {:?} sleeping..", result);
+                break;
+            }
+            info!("result = {:?} sleeping..", result);
+            sleep(Duration::new(1, 0));
+        }
+
+        info!("joining..?");
+        exit.store(true, Ordering::Relaxed);
+        storage_stage.join().unwrap();
+
+        #[cfg(not(all(feature = "cuda", feature = "chacha")))]
+        assert_eq!(result, Hash::default());
+
+        #[cfg(all(feature = "cuda", feature = "chacha"))]
+        assert_ne!(result, Hash::default());
+
+        remove_dir_all(ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_storage_stage_process_vote_entries() {
+        solana_logger::setup();
+        let keypair = Arc::new(Keypair::new());
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(1000);
+        let ticks_per_slot = genesis_block.ticks_per_slot;;
+        let (ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+        let entries = make_tiny_test_entries(128);
+        let blocktree = Blocktree::open_config(&ledger_path, ticks_per_slot).unwrap();
+        blocktree.write_entries(1, 0, 0, &entries).unwrap();
+
+        let cluster_info = test_cluster_info(keypair.pubkey());
+
+        let (storage_entry_sender, storage_entry_receiver) = channel();
+        let storage_state = StorageState::new();
+        let storage_stage = StorageStage::new(
+            &storage_state,
+            storage_entry_receiver,
+            Some(Arc::new(blocktree)),
+            &keypair,
+            &exit.clone(),
+            0,
+            STORAGE_ROTATE_TEST_COUNT,
+            &cluster_info,
+        );
+        storage_entry_sender.send(entries.clone()).unwrap();
+
+        let mut reference_keys;
+        {
+            let keys = &storage_state.state.read().unwrap().storage_keys;
+            reference_keys = vec![0; keys.len()];
+            reference_keys.copy_from_slice(keys);
+        }
+        let mut vote_txs: Vec<_> = Vec::new();
+        let keypair = Keypair::new();
+        let vote_tx = VoteTransaction::new_vote(&keypair, 123456, Hash::default(), 1);
+        vote_txs.push(vote_tx);
+        let vote_entries = vec![Entry::new(&Hash::default(), 1, vote_txs)];
+        storage_entry_sender.send(vote_entries).unwrap();
+
+        for _ in 0..5 {
+            {
+                let keys = &storage_state.state.read().unwrap().storage_keys;
+                if keys[..] != *reference_keys.as_slice() {
+                    break;
+                }
+            }
+
+            sleep(Duration::new(1, 0));
+        }
+
+        debug!("joining..?");
+        exit.store(true, Ordering::Relaxed);
+        storage_stage.join().unwrap();
+
+        {
+            let keys = &storage_state.state.read().unwrap().storage_keys;
+            assert_ne!(keys[..], *reference_keys);
+        }
+
+        remove_dir_all(ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_signature_distribution() {
+        // See that signatures have an even-ish distribution..
+        let mut hist = Arc::new(vec![]);
+        for _ in 0..NUM_IDENTITIES {
+            Arc::get_mut(&mut hist).unwrap().push(AtomicUsize::new(0));
+        }
+        let hasher = Hasher::default();
+        {
+            let hist = hist.clone();
+            (0..(32 * NUM_IDENTITIES))
+                .into_par_iter()
+                .for_each(move |_| {
+                    let keypair = Keypair::new();
+                    let hash = hasher.clone().result();
+                    let signature = keypair.sign_message(&hash.as_ref());
+                    let ix = get_identity_index_from_signature(&signature);
+                    hist[ix].fetch_add(1, Ordering::Relaxed);
+                });
+        }
+
+        let mut hist_max = 0;
+        let mut hist_min = NUM_IDENTITIES;
+        for x in hist.iter() {
+            let val = x.load(Ordering::Relaxed);
+            hist_max = max(val, hist_max);
+            hist_min = min(val, hist_min);
+        }
+        info!("min: {} max: {}", hist_min, hist_max);
+        assert_ne!(hist_min, 0);
+    }
+}

core/src/streamer.rs

@@ -0,0 +1,202 @@
+//! The `streamer` module defines a set of services for efficiently pulling data from UDP sockets.
+//!
+
+use crate::packet::{Blob, SharedBlobs, SharedPackets};
+use crate::result::{Error, Result};
+use solana_metrics::{influxdb, submit};
+use solana_sdk::timing::duration_as_ms;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
+use std::sync::Arc;
+use std::thread::{Builder, JoinHandle};
+use std::time::{Duration, Instant};
+
+pub type PacketReceiver = Receiver<SharedPackets>;
+pub type PacketSender = Sender<SharedPackets>;
+pub type BlobSender = Sender<SharedBlobs>;
+pub type BlobReceiver = Receiver<SharedBlobs>;
+
+fn recv_loop(
+    sock: &UdpSocket,
+    exit: &Arc<AtomicBool>,
+    channel: &PacketSender,
+    channel_tag: &'static str,
+) -> Result<()> {
+    loop {
+        let msgs = SharedPackets::default();
+        loop {
+            // Check for exit signal, even if socket is busy
+            // (for instance the leader trasaction socket)
+            if exit.load(Ordering::Relaxed) {
+                return Ok(());
+            }
+            if msgs.write().unwrap().recv_from(sock).is_ok() {
+                let len = msgs.read().unwrap().packets.len();
+                submit(
+                    influxdb::Point::new(channel_tag)
+                        .add_field("count", influxdb::Value::Integer(len as i64))
+                        .to_owned(),
+                );
+                channel.send(msgs)?;
+                break;
+            }
+        }
+    }
+}
+
+pub fn receiver(
+    sock: Arc<UdpSocket>,
+    exit: Arc<AtomicBool>,
+    packet_sender: PacketSender,
+    sender_tag: &'static str,
+) -> JoinHandle<()> {
+    let res = sock.set_read_timeout(Some(Duration::new(1, 0)));
+    if res.is_err() {
+        panic!("streamer::receiver set_read_timeout error");
+    }
+    Builder::new()
+        .name("solana-receiver".to_string())
+        .spawn(move || {
+            let _ = recv_loop(&sock, &exit, &packet_sender, sender_tag);
+        })
+        .unwrap()
+}
+
+fn recv_send(sock: &UdpSocket, r: &BlobReceiver) -> Result<()> {
+    let timer = Duration::new(1, 0);
+    let msgs = r.recv_timeout(timer)?;
+    Blob::send_to(sock, msgs)?;
+    Ok(())
+}
+
+pub fn recv_batch(recvr: &PacketReceiver) -> Result<(Vec<SharedPackets>, usize, u64)> {
+    let timer = Duration::new(1, 0);
+    let msgs = recvr.recv_timeout(timer)?;
+    let recv_start = Instant::now();
+    trace!("got msgs");
+    let mut len = msgs.read().unwrap().packets.len();
+    let mut batch = vec![msgs];
+    while let Ok(more) = recvr.try_recv() {
+        trace!("got more msgs");
+        len += more.read().unwrap().packets.len();
+        batch.push(more);
+
+        if len > 100_000 {
+            break;
+        }
+    }
+    trace!("batch len {}", batch.len());
+    Ok((batch, len, duration_as_ms(&recv_start.elapsed())))
+}
+
+pub fn responder(name: &'static str, sock: Arc<UdpSocket>, r: BlobReceiver) -> JoinHandle<()> {
+    Builder::new()
+        .name(format!("solana-responder-{}", name))
+        .spawn(move || loop {
+            if let Err(e) = recv_send(&sock, &r) {
+                match e {
+                    Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                    Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                    _ => warn!("{} responder error: {:?}", name, e),
+                }
+            }
+        })
+        .unwrap()
+}
+
+//TODO, we would need to stick block authentication before we create the
+//window.
+fn recv_blobs(sock: &UdpSocket, s: &BlobSender) -> Result<()> {
+    trace!("recv_blobs: receiving on {}", sock.local_addr().unwrap());
+    let dq = Blob::recv_from(sock)?;
+    if !dq.is_empty() {
+        s.send(dq)?;
+    }
+    Ok(())
+}
+
+pub fn blob_receiver(sock: Arc<UdpSocket>, exit: Arc<AtomicBool>, s: BlobSender) -> JoinHandle<()> {
+    //DOCUMENTED SIDE-EFFECT
+    //1 second timeout on socket read
+    let timer = Duration::new(1, 0);
+    sock.set_read_timeout(Some(timer))
+        .expect("set socket timeout");
+    Builder::new()
+        .name("solana-blob_receiver".to_string())
+        .spawn(move || loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+            let _ = recv_blobs(&sock, &s);
+        })
+        .unwrap()
+}
+
+#[cfg(test)]
+mod test {
+    use crate::packet::{Blob, Packet, Packets, SharedBlob, PACKET_DATA_SIZE};
+    use crate::streamer::PacketReceiver;
+    use crate::streamer::{receiver, responder};
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    fn get_msgs(r: PacketReceiver, num: &mut usize) {
+        for _t in 0..5 {
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(m) => *num += m.read().unwrap().packets.len(),
+                _ => info!("get_msgs error"),
+            }
+            if *num == 10 {
+                break;
+            }
+        }
+    }
+    #[test]
+    pub fn streamer_debug() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn streamer_send_test() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
+
+        let addr = read.local_addr().unwrap();
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let (s_reader, r_reader) = channel();
+        let t_receiver = receiver(Arc::new(read), exit.clone(), s_reader, "streamer-test");
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let t_responder = responder("streamer_send_test", Arc::new(send), r_responder);
+            let mut msgs = Vec::new();
+            for i in 0..10 {
+                let b = SharedBlob::default();
+                {
+                    let mut w = b.write().unwrap();
+                    w.data[0] = i as u8;
+                    w.meta.size = PACKET_DATA_SIZE;
+                    w.meta.set_addr(&addr);
+                }
+                msgs.push(b);
+            }
+            s_responder.send(msgs).expect("send");
+            t_responder
+        };
+
+        let mut num = 0;
+        get_msgs(r_reader, &mut num);
+        assert_eq!(num, 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+}

core/src/test_tx.rs

@@ -0,0 +1,11 @@
+use solana_sdk::hash::Hash;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::system_transaction::SystemTransaction;
+use solana_sdk::transaction::Transaction;
+
+pub fn test_tx() -> Transaction {
+    let keypair1 = Keypair::new();
+    let pubkey1 = keypair1.pubkey();
+    let zero = Hash::default();
+    SystemTransaction::new_account(&keypair1, pubkey1, 42, zero, 0)
+}

core/src/thin_client.rs

@@ -0,0 +1,686 @@
+//! The `thin_client` module is a client-side object that interfaces with
+//! a server-side TPU.  Client code should use this object instead of writing
+//! messages to the network directly. The binary encoding of its messages are
+//! unstable and may change in future releases.
+
+use crate::cluster_info::{ClusterInfo, ClusterInfoError, NodeInfo};
+use crate::fullnode::{Fullnode, FullnodeConfig};
+use crate::gossip_service::GossipService;
+use crate::packet::PACKET_DATA_SIZE;
+use crate::result::{Error, Result};
+use crate::rpc_request::{RpcClient, RpcRequest, RpcRequestHandler};
+use bincode::serialize_into;
+use bs58;
+use serde_json;
+use solana_metrics;
+use solana_metrics::influxdb;
+use solana_sdk::account::Account;
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
+use solana_sdk::system_transaction::SystemTransaction;
+use solana_sdk::timing;
+use solana_sdk::transaction::Transaction;
+use std;
+use std::io;
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::time::Duration;
+use std::time::Instant;
+
+/// An object for querying and sending transactions to the network.
+pub struct ThinClient {
+    rpc_addr: SocketAddr,
+    transactions_addr: SocketAddr,
+    transactions_socket: UdpSocket,
+    rpc_client: RpcClient,
+}
+
+impl ThinClient {
+    /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP
+    /// and the Tpu at `transactions_addr` over `transactions_socket` using UDP.
+    pub fn new(
+        rpc_addr: SocketAddr,
+        transactions_addr: SocketAddr,
+        transactions_socket: UdpSocket,
+    ) -> Self {
+        Self::new_from_client(
+            rpc_addr,
+            transactions_addr,
+            transactions_socket,
+            RpcClient::new_from_socket(rpc_addr),
+        )
+    }
+
+    pub fn new_with_timeout(
+        rpc_addr: SocketAddr,
+        transactions_addr: SocketAddr,
+        transactions_socket: UdpSocket,
+        timeout: Duration,
+    ) -> Self {
+        let rpc_client = RpcClient::new_with_timeout(rpc_addr, timeout);
+        Self::new_from_client(rpc_addr, transactions_addr, transactions_socket, rpc_client)
+    }
+
+    fn new_from_client(
+        rpc_addr: SocketAddr,
+        transactions_addr: SocketAddr,
+        transactions_socket: UdpSocket,
+        rpc_client: RpcClient,
+    ) -> Self {
+        ThinClient {
+            rpc_client,
+            rpc_addr,
+            transactions_addr,
+            transactions_socket,
+        }
+    }
+
+    /// Send a signed Transaction to the server for processing. This method
+    /// does not wait for a response.
+    pub fn transfer_signed(&self, transaction: &Transaction) -> io::Result<Signature> {
+        let mut buf = vec![0; transaction.serialized_size().unwrap() as usize];
+        let mut wr = std::io::Cursor::new(&mut buf[..]);
+        serialize_into(&mut wr, &transaction)
+            .expect("serialize Transaction in pub fn transfer_signed");
+        assert!(buf.len() < PACKET_DATA_SIZE);
+        self.transactions_socket
+            .send_to(&buf[..], &self.transactions_addr)?;
+        Ok(transaction.signatures[0])
+    }
+
+    /// Retry a sending a signed Transaction to the server for processing.
+    pub fn retry_transfer(
+        &mut self,
+        keypair: &Keypair,
+        transaction: &mut Transaction,
+        tries: usize,
+    ) -> io::Result<Signature> {
+        for x in 0..tries {
+            transaction.sign(&[keypair], self.get_last_id());
+            let mut buf = vec![0; transaction.serialized_size().unwrap() as usize];
+            let mut wr = std::io::Cursor::new(&mut buf[..]);
+            serialize_into(&mut wr, &transaction)
+                .expect("serialize Transaction in pub fn transfer_signed");
+            self.transactions_socket
+                .send_to(&buf[..], &self.transactions_addr)?;
+            if self.poll_for_signature(&transaction.signatures[0]).is_ok() {
+                return Ok(transaction.signatures[0]);
+            }
+            info!("{} tries failed transfer to {}", x, self.transactions_addr);
+        }
+        Err(io::Error::new(
+            io::ErrorKind::Other,
+            "retry_transfer failed",
+        ))
+    }
+
+    /// Creates, signs, and processes a Transaction. Useful for writing unit-tests.
+    pub fn transfer(
+        &self,
+        tokens: u64,
+        keypair: &Keypair,
+        to: Pubkey,
+        last_id: &Hash,
+    ) -> io::Result<Signature> {
+        debug!(
+            "transfer: tokens={} from={:?} to={:?} last_id={:?}",
+            tokens,
+            keypair.pubkey(),
+            to,
+            last_id
+        );
+        let now = Instant::now();
+        let transaction = SystemTransaction::new_account(keypair, to, tokens, *last_id, 0);
+        let result = self.transfer_signed(&transaction);
+        solana_metrics::submit(
+            influxdb::Point::new("thinclient")
+                .add_tag("op", influxdb::Value::String("transfer".to_string()))
+                .add_field(
+                    "duration_ms",
+                    influxdb::Value::Integer(timing::duration_as_ms(&now.elapsed()) as i64),
+                )
+                .to_owned(),
+        );
+        result
+    }
+
+    pub fn get_account_userdata(&mut self, pubkey: &Pubkey) -> io::Result<Option<Vec<u8>>> {
+        let params = json!([format!("{}", pubkey)]);
+        let response =
+            self.rpc_client
+                .make_rpc_request(1, RpcRequest::GetAccountInfo, Some(params));
+        match response {
+            Ok(account_json) => {
+                let account: Account =
+                    serde_json::from_value(account_json).expect("deserialize account");
+                Ok(Some(account.userdata))
+            }
+            Err(error) => {
+                debug!("get_account_userdata failed: {:?}", error);
+                Err(io::Error::new(
+                    io::ErrorKind::Other,
+                    "get_account_userdata failed",
+                ))
+            }
+        }
+    }
+
+    /// Request the balance of the user holding `pubkey`. This method blocks
+    /// until the server sends a response. If the response packet is dropped
+    /// by the network, this method will hang indefinitely.
+    pub fn get_balance(&mut self, pubkey: &Pubkey) -> io::Result<u64> {
+        trace!("get_balance sending request to {}", self.rpc_addr);
+        let params = json!([format!("{}", pubkey)]);
+        let response =
+            self.rpc_client
+                .make_rpc_request(1, RpcRequest::GetAccountInfo, Some(params));
+
+        response
+            .and_then(|account_json| {
+                let account: Account =
+                    serde_json::from_value(account_json).expect("deserialize account");
+                trace!("Response account {:?} {:?}", pubkey, account);
+                trace!("get_balance {:?}", account.tokens);
+                Ok(account.tokens)
+            })
+            .map_err(|error| {
+                debug!("Response account {}: None (error: {:?})", pubkey, error);
+                io::Error::new(io::ErrorKind::Other, "AccountNotFound")
+            })
+    }
+
+    /// Request the transaction count.  If the response packet is dropped by the network,
+    /// this method will try again 5 times.
+    pub fn transaction_count(&mut self) -> u64 {
+        debug!("transaction_count");
+        for _tries in 0..5 {
+            let response =
+                self.rpc_client
+                    .make_rpc_request(1, RpcRequest::GetTransactionCount, None);
+
+            match response {
+                Ok(value) => {
+                    debug!("transaction_count response: {:?}", value);
+                    let transaction_count = value.as_u64().unwrap();
+                    return transaction_count;
+                }
+                Err(error) => {
+                    debug!("transaction_count failed: {:?}", error);
+                }
+            };
+        }
+        0
+    }
+
+    /// Request the last Entry ID from the server without blocking.
+    /// Returns the last_id Hash or None if there was no response from the server.
+    pub fn try_get_last_id(&mut self, mut num_retries: u64) -> Option<Hash> {
+        loop {
+            trace!("try_get_last_id send_to {}", &self.rpc_addr);
+            let response = self
+                .rpc_client
+                .make_rpc_request(1, RpcRequest::GetLastId, None);
+
+            match response {
+                Ok(value) => {
+                    let last_id_str = value.as_str().unwrap();
+                    let last_id_vec = bs58::decode(last_id_str).into_vec().unwrap();
+                    return Some(Hash::new(&last_id_vec));
+                }
+                Err(error) => {
+                    debug!("thin_client get_last_id error: {:?}", error);
+                    num_retries -= 1;
+                    if num_retries == 0 {
+                        return None;
+                    }
+                }
+            }
+        }
+    }
+
+    /// Request the last Entry ID from the server. This method blocks
+    /// until the server sends a response.
+    pub fn get_last_id(&mut self) -> Hash {
+        loop {
+            trace!("get_last_id send_to {}", &self.rpc_addr);
+            if let Some(hash) = self.try_get_last_id(10) {
+                return hash;
+            }
+        }
+    }
+
+    /// Request a new last Entry ID from the server. This method blocks
+    /// until the server sends a response.
+    pub fn get_next_last_id(&mut self, previous_last_id: &Hash) -> Hash {
+        self.get_next_last_id_ext(previous_last_id, &|| {
+            sleep(Duration::from_millis(100));
+        })
+    }
+    pub fn get_next_last_id_ext(&mut self, previous_last_id: &Hash, func: &Fn()) -> Hash {
+        loop {
+            let last_id = self.get_last_id();
+            if last_id != *previous_last_id {
+                break last_id;
+            }
+            debug!("Got same last_id ({:?}), will retry...", last_id);
+            func()
+        }
+    }
+
+    pub fn submit_poll_balance_metrics(elapsed: &Duration) {
+        solana_metrics::submit(
+            influxdb::Point::new("thinclient")
+                .add_tag("op", influxdb::Value::String("get_balance".to_string()))
+                .add_field(
+                    "duration_ms",
+                    influxdb::Value::Integer(timing::duration_as_ms(elapsed) as i64),
+                )
+                .to_owned(),
+        );
+    }
+
+    pub fn poll_balance_with_timeout(
+        &mut self,
+        pubkey: &Pubkey,
+        polling_frequency: &Duration,
+        timeout: &Duration,
+    ) -> io::Result<u64> {
+        let now = Instant::now();
+        loop {
+            match self.get_balance(&pubkey) {
+                Ok(bal) => {
+                    ThinClient::submit_poll_balance_metrics(&now.elapsed());
+                    return Ok(bal);
+                }
+                Err(e) => {
+                    sleep(*polling_frequency);
+                    if now.elapsed() > *timeout {
+                        ThinClient::submit_poll_balance_metrics(&now.elapsed());
+                        return Err(e);
+                    }
+                }
+            };
+        }
+    }
+
+    pub fn poll_get_balance(&mut self, pubkey: &Pubkey) -> io::Result<u64> {
+        self.poll_balance_with_timeout(pubkey, &Duration::from_millis(100), &Duration::from_secs(1))
+    }
+
+    /// Poll the server to confirm a transaction.
+    pub fn poll_for_signature(&mut self, signature: &Signature) -> io::Result<()> {
+        let now = Instant::now();
+        while !self.check_signature(signature) {
+            if now.elapsed().as_secs() > 15 {
+                // TODO: Return a better error.
+                return Err(io::Error::new(io::ErrorKind::Other, "signature not found"));
+            }
+            sleep(Duration::from_millis(250));
+        }
+        Ok(())
+    }
+
+    /// Check a signature in the bank. This method blocks
+    /// until the server sends a response.
+    pub fn check_signature(&mut self, signature: &Signature) -> bool {
+        trace!("check_signature: {:?}", signature);
+        let params = json!([format!("{}", signature)]);
+        let now = Instant::now();
+
+        loop {
+            let response = self.rpc_client.make_rpc_request(
+                1,
+                RpcRequest::ConfirmTransaction,
+                Some(params.clone()),
+            );
+
+            match response {
+                Ok(confirmation) => {
+                    let signature_status = confirmation.as_bool().unwrap();
+                    if signature_status {
+                        trace!("Response found signature");
+                    } else {
+                        trace!("Response signature not found");
+                    }
+                    solana_metrics::submit(
+                        influxdb::Point::new("thinclient")
+                            .add_tag("op", influxdb::Value::String("check_signature".to_string()))
+                            .add_field(
+                                "duration_ms",
+                                influxdb::Value::Integer(
+                                    timing::duration_as_ms(&now.elapsed()) as i64
+                                ),
+                            )
+                            .to_owned(),
+                    );
+                    return signature_status;
+                }
+                Err(err) => {
+                    debug!("check_signature request failed: {:?}", err);
+                }
+            };
+        }
+    }
+}
+
+impl Drop for ThinClient {
+    fn drop(&mut self) {
+        solana_metrics::flush();
+    }
+}
+
+pub fn poll_gossip_for_leader(leader_gossip: SocketAddr, timeout: Option<u64>) -> Result<NodeInfo> {
+    let exit = Arc::new(AtomicBool::new(false));
+    let (node, gossip_socket) = ClusterInfo::spy_node();
+    let my_addr = gossip_socket.local_addr().unwrap();
+    let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(node)));
+    let gossip_service = GossipService::new(
+        &cluster_info.clone(),
+        None,
+        None,
+        gossip_socket,
+        exit.clone(),
+    );
+
+    let leader_entry_point = NodeInfo::new_entry_point(&leader_gossip);
+    cluster_info
+        .write()
+        .unwrap()
+        .insert_info(leader_entry_point);
+
+    sleep(Duration::from_millis(100));
+
+    let deadline = match timeout {
+        Some(timeout) => Duration::new(timeout, 0),
+        None => Duration::new(std::u64::MAX, 0),
+    };
+    let now = Instant::now();
+    // Block until leader's correct contact info is received
+    let leader;
+
+    loop {
+        trace!("polling {:?} for leader from {:?}", leader_gossip, my_addr);
+
+        if let Some(l) = cluster_info.read().unwrap().get_gossip_top_leader() {
+            leader = Some(l.clone());
+            break;
+        }
+
+        if log_enabled!(log::Level::Trace) {
+            trace!("{}", cluster_info.read().unwrap().node_info_trace());
+        }
+
+        if now.elapsed() > deadline {
+            return Err(Error::ClusterInfoError(ClusterInfoError::NoLeader));
+        }
+
+        sleep(Duration::from_millis(100));
+    }
+
+    gossip_service.close()?;
+
+    if log_enabled!(log::Level::Trace) {
+        trace!("{}", cluster_info.read().unwrap().node_info_trace());
+    }
+
+    Ok(leader.unwrap().clone())
+}
+
+pub fn retry_get_balance(
+    client: &mut ThinClient,
+    bob_pubkey: &Pubkey,
+    expected_balance: Option<u64>,
+) -> Option<u64> {
+    const LAST: usize = 30;
+    for run in 0..LAST {
+        let balance_result = client.poll_get_balance(bob_pubkey);
+        if expected_balance.is_none() {
+            return balance_result.ok();
+        }
+        trace!(
+            "retry_get_balance[{}] {:?} {:?}",
+            run,
+            balance_result,
+            expected_balance
+        );
+        if let (Some(expected_balance), Ok(balance_result)) = (expected_balance, balance_result) {
+            if expected_balance == balance_result {
+                return Some(balance_result);
+            }
+        }
+    }
+    None
+}
+
+pub fn new_fullnode() -> (Fullnode, NodeInfo, Keypair, String) {
+    use crate::blocktree::create_new_tmp_ledger;
+    use crate::cluster_info::Node;
+    use crate::fullnode::Fullnode;
+    use crate::voting_keypair::VotingKeypair;
+    use solana_sdk::genesis_block::GenesisBlock;
+    use solana_sdk::signature::KeypairUtil;
+
+    let node_keypair = Arc::new(Keypair::new());
+    let node = Node::new_localhost_with_pubkey(node_keypair.pubkey());
+    let node_info = node.info.clone();
+
+    let (genesis_block, mint_keypair) = GenesisBlock::new_with_leader(10_000, node_info.id, 42);
+    let (ledger_path, _last_id) = create_new_tmp_ledger!(&genesis_block);
+
+    let vote_account_keypair = Arc::new(Keypair::new());
+    let voting_keypair = VotingKeypair::new_local(&vote_account_keypair);
+    let node = Fullnode::new(
+        node,
+        &node_keypair,
+        &ledger_path,
+        voting_keypair,
+        None,
+        &FullnodeConfig::default(),
+    );
+
+    (node, node_info, mint_keypair, ledger_path)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::client::mk_client;
+    use bincode::{deserialize, serialize};
+    use solana_sdk::system_instruction::SystemInstruction;
+    use solana_sdk::vote_program::VoteState;
+    use solana_sdk::vote_transaction::VoteTransaction;
+    use std::fs::remove_dir_all;
+
+    #[test]
+    fn test_thin_client_basic() {
+        solana_logger::setup();
+        let (server, leader_data, alice, ledger_path) = new_fullnode();
+        let server_exit = server.run(None);
+        let bob_pubkey = Keypair::new().pubkey();
+
+        info!(
+            "found leader: {:?}",
+            poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap()
+        );
+
+        let mut client = mk_client(&leader_data);
+
+        let transaction_count = client.transaction_count();
+        assert_eq!(transaction_count, 0);
+
+        let last_id = client.get_last_id();
+        info!("test_thin_client last_id: {:?}", last_id);
+
+        let signature = client.transfer(500, &alice, bob_pubkey, &last_id).unwrap();
+        info!("test_thin_client signature: {:?}", signature);
+        client.poll_for_signature(&signature).unwrap();
+
+        let balance = client.get_balance(&bob_pubkey);
+        assert_eq!(balance.unwrap(), 500);
+
+        let transaction_count = client.transaction_count();
+        assert_eq!(transaction_count, 1);
+        server_exit();
+        remove_dir_all(ledger_path).unwrap();
+    }
+
+    #[test]
+    #[ignore]
+    fn test_bad_sig() {
+        solana_logger::setup();
+        let (server, leader_data, alice, ledger_path) = new_fullnode();
+        let server_exit = server.run(None);
+        let bob_pubkey = Keypair::new().pubkey();
+        info!(
+            "found leader: {:?}",
+            poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap()
+        );
+
+        let mut client = mk_client(&leader_data);
+
+        let last_id = client.get_last_id();
+
+        let tx = SystemTransaction::new_account(&alice, bob_pubkey, 500, last_id, 0);
+
+        let _sig = client.transfer_signed(&tx).unwrap();
+
+        let last_id = client.get_last_id();
+
+        let mut tr2 = SystemTransaction::new_account(&alice, bob_pubkey, 501, last_id, 0);
+        let mut instruction2 = deserialize(tr2.userdata(0)).unwrap();
+        if let SystemInstruction::Move { ref mut tokens } = instruction2 {
+            *tokens = 502;
+        }
+        tr2.instructions[0].userdata = serialize(&instruction2).unwrap();
+        let signature = client.transfer_signed(&tr2).unwrap();
+        client.poll_for_signature(&signature).unwrap();
+
+        let balance = client.get_balance(&bob_pubkey);
+        assert_eq!(balance.unwrap(), 1001);
+        server_exit();
+        remove_dir_all(ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_register_vote_account() {
+        solana_logger::setup();
+        let (server, leader_data, alice, ledger_path) = new_fullnode();
+        let server_exit = server.run(None);
+        info!(
+            "found leader: {:?}",
+            poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap()
+        );
+
+        let mut client = mk_client(&leader_data);
+
+        // Create the validator account, transfer some tokens to that account
+        let validator_keypair = Keypair::new();
+        let last_id = client.get_last_id();
+        let signature = client
+            .transfer(500, &alice, validator_keypair.pubkey(), &last_id)
+            .unwrap();
+
+        client.poll_for_signature(&signature).unwrap();
+
+        // Create and register the vote account
+        let validator_vote_account_keypair = Keypair::new();
+        let vote_account_id = validator_vote_account_keypair.pubkey();
+        let last_id = client.get_last_id();
+
+        let transaction = VoteTransaction::fund_staking_account(
+            &validator_keypair,
+            vote_account_id,
+            last_id,
+            1,
+            1,
+        );
+        let signature = client.transfer_signed(&transaction).unwrap();
+        client.poll_for_signature(&signature).unwrap();
+
+        let balance = retry_get_balance(&mut client, &vote_account_id, Some(1))
+            .expect("Expected balance for new account to exist");
+        assert_eq!(balance, 1);
+
+        const LAST: usize = 30;
+        for run in 0..=LAST {
+            let account_user_data = client
+                .get_account_userdata(&vote_account_id)
+                .expect("Expected valid response for account userdata")
+                .expect("Expected valid account userdata to exist after account creation");
+
+            let vote_state = VoteState::deserialize(&account_user_data);
+
+            if vote_state.map(|vote_state| vote_state.delegate_id) == Ok(validator_keypair.pubkey())
+            {
+                break;
+            }
+
+            if run == LAST {
+                panic!("Expected successful vote account registration");
+            }
+            sleep(Duration::from_millis(900));
+        }
+
+        server_exit();
+        remove_dir_all(ledger_path).unwrap();
+    }
+
+    #[test]
+    fn test_transaction_count() {
+        // set a bogus address, see that we don't hang
+        solana_logger::setup();
+        let addr = "0.0.0.0:1234".parse().unwrap();
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let mut client =
+            ThinClient::new_with_timeout(addr, addr, transactions_socket, Duration::from_secs(2));
+        assert_eq!(client.transaction_count(), 0);
+    }
+
+    #[test]
+    fn test_zero_balance_after_nonzero() {
+        solana_logger::setup();
+        let (server, leader_data, alice, ledger_path) = new_fullnode();
+        let server_exit = server.run(None);
+        let bob_keypair = Keypair::new();
+
+        info!(
+            "found leader: {:?}",
+            poll_gossip_for_leader(leader_data.gossip, Some(5)).unwrap()
+        );
+
+        let mut client = mk_client(&leader_data);
+        let last_id = client.get_last_id();
+        info!("test_thin_client last_id: {:?}", last_id);
+
+        let starting_alice_balance = client.poll_get_balance(&alice.pubkey()).unwrap();
+        info!("Alice has {} tokens", starting_alice_balance);
+
+        info!("Give Bob 500 tokens");
+        let signature = client
+            .transfer(500, &alice, bob_keypair.pubkey(), &last_id)
+            .unwrap();
+        client.poll_for_signature(&signature).unwrap();
+
+        let bob_balance = client.poll_get_balance(&bob_keypair.pubkey());
+        assert_eq!(bob_balance.unwrap(), 500);
+
+        info!("Take Bob's 500 tokens away");
+        let signature = client
+            .transfer(500, &bob_keypair, alice.pubkey(), &last_id)
+            .unwrap();
+        client.poll_for_signature(&signature).unwrap();
+        let alice_balance = client.poll_get_balance(&alice.pubkey()).unwrap();
+        assert_eq!(alice_balance, starting_alice_balance);
+
+        info!("Should get an error when Bob's balance hits zero and is purged");
+        let bob_balance = client.poll_get_balance(&bob_keypair.pubkey());
+        info!("Bob's balance is {:?}", bob_balance);
+        assert!(bob_balance.is_err(),);
+
+        server_exit();
+        remove_dir_all(ledger_path).unwrap();
+    }
+}

core/src/tpu.rs

@@ -0,0 +1,283 @@
+//! The `tpu` module implements the Transaction Processing Unit, a
+//! multi-stage transaction processing pipeline in software.
+
+use crate::banking_stage::{BankingStage, UnprocessedPackets};
+use crate::blocktree::Blocktree;
+use crate::broadcast_stage::BroadcastStage;
+use crate::cluster_info::ClusterInfo;
+use crate::cluster_info_vote_listener::ClusterInfoVoteListener;
+use crate::fetch_stage::FetchStage;
+use crate::poh_recorder::PohRecorder;
+use crate::service::Service;
+use crate::sigverify_stage::SigVerifyStage;
+use crate::tpu_forwarder::TpuForwarder;
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread;
+
+pub enum TpuMode {
+    Leader(LeaderServices),
+    Forwarder(ForwarderServices),
+}
+
+pub struct LeaderServices {
+    fetch_stage: FetchStage,
+    sigverify_stage: SigVerifyStage,
+    banking_stage: BankingStage,
+    cluster_info_vote_listener: ClusterInfoVoteListener,
+    broadcast_stage: BroadcastStage,
+}
+
+impl LeaderServices {
+    fn new(
+        fetch_stage: FetchStage,
+        sigverify_stage: SigVerifyStage,
+        banking_stage: BankingStage,
+        cluster_info_vote_listener: ClusterInfoVoteListener,
+        broadcast_stage: BroadcastStage,
+    ) -> Self {
+        LeaderServices {
+            fetch_stage,
+            sigverify_stage,
+            banking_stage,
+            cluster_info_vote_listener,
+            broadcast_stage,
+        }
+    }
+
+    fn exit(&self) {
+        self.fetch_stage.close();
+    }
+
+    fn join(self) -> thread::Result<()> {
+        let mut results = vec![];
+        results.push(self.fetch_stage.join());
+        results.push(self.sigverify_stage.join());
+        results.push(self.cluster_info_vote_listener.join());
+        results.push(self.banking_stage.join());
+        let broadcast_result = self.broadcast_stage.join();
+        for result in results {
+            result?;
+        }
+        let _ = broadcast_result?;
+        Ok(())
+    }
+
+    fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+pub struct ForwarderServices {
+    tpu_forwarder: TpuForwarder,
+}
+
+impl ForwarderServices {
+    fn new(tpu_forwarder: TpuForwarder) -> Self {
+        ForwarderServices { tpu_forwarder }
+    }
+
+    fn exit(&self) {
+        self.tpu_forwarder.close();
+    }
+
+    fn join(self) -> thread::Result<()> {
+        self.tpu_forwarder.join()
+    }
+
+    fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+pub struct Tpu {
+    tpu_mode: Option<TpuMode>,
+    exit: Arc<AtomicBool>,
+    id: Pubkey,
+    cluster_info: Arc<RwLock<ClusterInfo>>,
+}
+
+impl Tpu {
+    pub fn new(id: Pubkey, cluster_info: &Arc<RwLock<ClusterInfo>>) -> Self {
+        Self {
+            tpu_mode: None,
+            exit: Arc::new(AtomicBool::new(false)),
+            id,
+            cluster_info: cluster_info.clone(),
+        }
+    }
+
+    fn mode_exit(&mut self) {
+        match &mut self.tpu_mode {
+            Some(TpuMode::Leader(svcs)) => {
+                svcs.exit();
+            }
+            Some(TpuMode::Forwarder(svcs)) => {
+                svcs.exit();
+            }
+            None => (),
+        }
+    }
+
+    fn mode_close(&mut self) {
+        let tpu_mode = self.tpu_mode.take();
+        if let Some(tpu_mode) = tpu_mode {
+            match tpu_mode {
+                TpuMode::Leader(svcs) => {
+                    let _ = svcs.close();
+                }
+                TpuMode::Forwarder(svcs) => {
+                    let _ = svcs.close();
+                }
+            }
+        }
+    }
+
+    fn forward_unprocessed_packets(
+        tpu: &std::net::SocketAddr,
+        unprocessed_packets: UnprocessedPackets,
+    ) -> std::io::Result<()> {
+        let socket = UdpSocket::bind("0.0.0.0:0")?;
+        for (packets, start_index) in unprocessed_packets {
+            let packets = packets.read().unwrap();
+            for packet in packets.packets.iter().skip(start_index) {
+                socket.send_to(&packet.data[..packet.meta.size], tpu)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn close_and_forward_unprocessed_packets(&mut self) {
+        self.mode_exit();
+
+        let unprocessed_packets = match self.tpu_mode.as_mut() {
+            Some(TpuMode::Leader(svcs)) => {
+                svcs.banking_stage.join_and_collect_unprocessed_packets()
+            }
+            Some(TpuMode::Forwarder(svcs)) => {
+                svcs.tpu_forwarder.join_and_collect_unprocessed_packets()
+            }
+            None => vec![],
+        };
+
+        if !unprocessed_packets.is_empty() {
+            let tpu = self.cluster_info.read().unwrap().leader_data().unwrap().tpu;
+            info!("forwarding unprocessed packets to new leader at {:?}", tpu);
+            Tpu::forward_unprocessed_packets(&tpu, unprocessed_packets).unwrap_or_else(|err| {
+                warn!("Failed to forward unprocessed transactions: {:?}", err)
+            });
+        }
+
+        self.mode_close();
+    }
+
+    pub fn switch_to_forwarder(&mut self, leader_id: Pubkey, transactions_sockets: Vec<UdpSocket>) {
+        self.close_and_forward_unprocessed_packets();
+
+        self.cluster_info.write().unwrap().set_leader(leader_id);
+
+        let tpu_forwarder = TpuForwarder::new(transactions_sockets, self.cluster_info.clone());
+        self.tpu_mode = Some(TpuMode::Forwarder(ForwarderServices::new(tpu_forwarder)));
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub fn switch_to_leader(
+        &mut self,
+        bank: &Arc<Bank>,
+        poh_recorder: &Arc<Mutex<PohRecorder>>,
+        transactions_sockets: Vec<UdpSocket>,
+        broadcast_socket: UdpSocket,
+        sigverify_disabled: bool,
+        slot: u64,
+        blocktree: &Arc<Blocktree>,
+    ) {
+        self.close_and_forward_unprocessed_packets();
+
+        self.cluster_info.write().unwrap().set_leader(self.id);
+
+        self.exit = Arc::new(AtomicBool::new(false));
+        let (packet_sender, packet_receiver) = channel();
+        let fetch_stage = FetchStage::new_with_sender(
+            transactions_sockets,
+            self.exit.clone(),
+            &packet_sender.clone(),
+        );
+        let cluster_info_vote_listener = ClusterInfoVoteListener::new(
+            self.exit.clone(),
+            self.cluster_info.clone(),
+            packet_sender,
+        );
+
+        let (sigverify_stage, verified_receiver) =
+            SigVerifyStage::new(packet_receiver, sigverify_disabled);
+
+        // TODO: Fix BankingStage/BroadcastStage to operate on `slot` directly instead of
+        // `max_tick_height`
+        let max_tick_height = (slot + 1) * bank.ticks_per_slot() - 1;
+        let blob_index = blocktree
+            .meta(slot)
+            .expect("Database error")
+            .map(|meta| meta.consumed)
+            .unwrap_or(0);
+
+        let (banking_stage, entry_receiver) = BankingStage::new(
+            &bank,
+            poh_recorder,
+            verified_receiver,
+            max_tick_height,
+            self.id,
+        );
+
+        let broadcast_stage = BroadcastStage::new(
+            slot,
+            bank,
+            broadcast_socket,
+            self.cluster_info.clone(),
+            blob_index,
+            entry_receiver,
+            self.exit.clone(),
+            blocktree,
+        );
+
+        let svcs = LeaderServices::new(
+            fetch_stage,
+            sigverify_stage,
+            banking_stage,
+            cluster_info_vote_listener,
+            broadcast_stage,
+        );
+        self.tpu_mode = Some(TpuMode::Leader(svcs));
+    }
+
+    pub fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+
+    pub fn is_exited(&self) -> bool {
+        self.exit.load(Ordering::Relaxed)
+    }
+
+    pub fn close(mut self) -> thread::Result<()> {
+        self.mode_close();
+        self.join()
+    }
+}
+
+impl Service for Tpu {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        match self.tpu_mode {
+            Some(TpuMode::Leader(svcs)) => svcs.join()?,
+            Some(TpuMode::Forwarder(svcs)) => svcs.join()?,
+            None => (),
+        }
+        Ok(())
+    }
+}

core/src/tpu_forwarder.rs

@@ -0,0 +1,164 @@
+//! The `tpu_forwarder` module implements a validator's
+//!  transaction processing unit responsibility, which
+//!  forwards received packets to the current leader
+
+use crate::banking_stage::UnprocessedPackets;
+use crate::cluster_info::ClusterInfo;
+use crate::contact_info::ContactInfo;
+use crate::service::Service;
+use crate::streamer::{self, PacketReceiver};
+use solana_metrics::counter::Counter;
+use solana_sdk::pubkey::Pubkey;
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::{Arc, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+
+fn get_forwarding_addr(leader_data: Option<&ContactInfo>, my_id: &Pubkey) -> Option<SocketAddr> {
+    let leader_data = leader_data?;
+    if leader_data.id == *my_id {
+        info!("I may be stuck in a loop"); // Should never try to forward to ourselves
+        return None;
+    }
+    if !ContactInfo::is_valid_address(&leader_data.tpu) {
+        return None;
+    }
+    Some(leader_data.tpu)
+}
+
+pub struct TpuForwarder {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+    forwarder_thread: Option<JoinHandle<UnprocessedPackets>>,
+}
+
+impl TpuForwarder {
+    fn forward(
+        receiver: &PacketReceiver,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        exit: &Arc<AtomicBool>,
+    ) -> UnprocessedPackets {
+        let socket = UdpSocket::bind("0.0.0.0:0").expect("Unable to bind");
+        let my_id = cluster_info.read().unwrap().id();
+
+        let mut unprocessed_packets = vec![];
+        loop {
+            match receiver.recv() {
+                Ok(msgs) => {
+                    inc_new_counter_info!(
+                        "tpu_forwarder-msgs_received",
+                        msgs.read().unwrap().packets.len()
+                    );
+
+                    if exit.load(Ordering::Relaxed) {
+                        // Collect all remaining packets on exit signaled
+                        unprocessed_packets.push((msgs, 0));
+                        continue;
+                    }
+
+                    match get_forwarding_addr(cluster_info.read().unwrap().leader_data(), &my_id) {
+                        Some(send_addr) => {
+                            msgs.write().unwrap().set_addr(&send_addr);
+                            msgs.read().unwrap().send_to(&socket).unwrap_or_else(|err| {
+                                info!("Failed to forward packet to {:?}: {:?}", send_addr, err)
+                            });
+                        }
+                        None => warn!("Packets dropped due to no forwarding address"),
+                    }
+                }
+                Err(err) => {
+                    trace!("Exiting forwarder due to {:?}", err);
+                    break;
+                }
+            }
+        }
+        unprocessed_packets
+    }
+
+    pub fn join_and_collect_unprocessed_packets(&mut self) -> UnprocessedPackets {
+        let forwarder_thread = self.forwarder_thread.take().unwrap();
+        forwarder_thread.join().unwrap_or_else(|err| {
+            warn!("forwarder_thread join failed: {:?}", err);
+            vec![]
+        })
+    }
+
+    pub fn new(sockets: Vec<UdpSocket>, cluster_info: Arc<RwLock<ClusterInfo>>) -> Self {
+        let exit = Arc::new(AtomicBool::new(false));
+        let (sender, receiver) = channel();
+
+        let thread_hdls: Vec<_> = sockets
+            .into_iter()
+            .map(|socket| {
+                streamer::receiver(
+                    Arc::new(socket),
+                    exit.clone(),
+                    sender.clone(),
+                    "tpu-forwarder",
+                )
+            })
+            .collect();
+
+        let thread_exit = exit.clone();
+        let forwarder_thread = Some(
+            Builder::new()
+                .name("solana-tpu_forwarder".to_string())
+                .spawn(move || Self::forward(&receiver, &cluster_info, &thread_exit))
+                .unwrap(),
+        );
+
+        TpuForwarder {
+            exit,
+            thread_hdls,
+            forwarder_thread,
+        }
+    }
+
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for TpuForwarder {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.close();
+        for thread_hdl in self.thread_hdls {
+            thread_hdl.join()?;
+        }
+        if let Some(forwarder_thread) = self.forwarder_thread {
+            forwarder_thread.join()?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::contact_info::ContactInfo;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+
+    #[test]
+    fn test_get_forwarding_addr() {
+        let my_id = Keypair::new().pubkey();
+
+        // Test with no leader
+        assert_eq!(get_forwarding_addr(None, &my_id,), None);
+
+        // Test with no TPU
+        let leader_data = ContactInfo::default();
+        assert_eq!(get_forwarding_addr(Some(&leader_data), &my_id,), None);
+
+        // Test with my pubkey
+        let leader_data = ContactInfo::new_localhost(my_id, 0);
+        assert_eq!(get_forwarding_addr(Some(&leader_data), &my_id,), None);
+
+        // Test with pubkey other than mine
+        let alice_id = Keypair::new().pubkey();
+        let leader_data = ContactInfo::new_localhost(alice_id, 0);
+        assert!(get_forwarding_addr(Some(&leader_data), &my_id,).is_some());
+    }
+}

core/src/tvu.rs

@@ -0,0 +1,254 @@
+//! The `tvu` module implements the Transaction Validation Unit, a
+//! multi-stage transaction validation pipeline in software.
+//!
+//! 1. BlobFetchStage
+//! - Incoming blobs are picked up from the TVU sockets and repair socket.
+//! 2. RetransmitStage
+//! - Blobs are windowed until a contiguous chunk is available.  This stage also repairs and
+//! retransmits blobs that are in the queue.
+//! 3. ReplayStage
+//! - Transactions in blobs are processed and applied to the bank.
+//! - TODO We need to verify the signatures in the blobs.
+//! 4. StorageStage
+//! - Generating the keys used to encrypt the ledger and sample it for storage mining.
+
+use crate::bank_forks::BankForks;
+use crate::blob_fetch_stage::BlobFetchStage;
+use crate::blockstream_service::BlockstreamService;
+use crate::blocktree::Blocktree;
+use crate::blocktree_processor::BankForksInfo;
+use crate::cluster_info::ClusterInfo;
+use crate::replay_stage::ReplayStage;
+use crate::retransmit_stage::RetransmitStage;
+use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::service::Service;
+use crate::storage_stage::{StorageStage, StorageState};
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil};
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::{Arc, RwLock};
+use std::thread;
+
+pub struct TvuRotationInfo {
+    pub tick_height: u64,  // tick height, bank might not exist yet
+    pub last_id: Hash,     // last_id that was voted on
+    pub slot: u64,         // slot height to initiate a rotation
+    pub leader_id: Pubkey, // leader upon rotation
+}
+pub type TvuRotationSender = Sender<TvuRotationInfo>;
+pub type TvuRotationReceiver = Receiver<TvuRotationInfo>;
+
+pub struct Tvu {
+    fetch_stage: BlobFetchStage,
+    retransmit_stage: RetransmitStage,
+    replay_stage: ReplayStage,
+    blockstream_service: Option<BlockstreamService>,
+    storage_stage: StorageStage,
+    exit: Arc<AtomicBool>,
+}
+
+pub struct Sockets {
+    pub fetch: Vec<UdpSocket>,
+    pub repair: UdpSocket,
+    pub retransmit: UdpSocket,
+}
+
+impl Tvu {
+    /// This service receives messages from a leader in the network and processes the transactions
+    /// on the bank state.
+    /// # Arguments
+    /// * `cluster_info` - The cluster_info state.
+    /// * `sockets` - fetch, repair, and retransmit sockets
+    /// * `blocktree` - the ledger itself
+    #[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
+    pub fn new<T>(
+        voting_keypair: Option<Arc<T>>,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        bank_forks_info: &[BankForksInfo],
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        sockets: Sockets,
+        blocktree: Arc<Blocktree>,
+        storage_rotate_count: u64,
+        to_leader_sender: &TvuRotationSender,
+        storage_state: &StorageState,
+        blockstream: Option<&String>,
+        ledger_signal_receiver: Receiver<bool>,
+        subscriptions: &Arc<RpcSubscriptions>,
+    ) -> Self
+    where
+        T: 'static + KeypairUtil + Sync + Send,
+    {
+        let exit = Arc::new(AtomicBool::new(false));
+        let keypair: Arc<Keypair> = cluster_info
+            .read()
+            .expect("Unable to read from cluster_info during Tvu creation")
+            .keypair
+            .clone();
+
+        let Sockets {
+            repair: repair_socket,
+            fetch: fetch_sockets,
+            retransmit: retransmit_socket,
+        } = sockets;
+
+        let (blob_fetch_sender, blob_fetch_receiver) = channel();
+
+        let repair_socket = Arc::new(repair_socket);
+        let mut blob_sockets: Vec<Arc<UdpSocket>> =
+            fetch_sockets.into_iter().map(Arc::new).collect();
+        blob_sockets.push(repair_socket.clone());
+        let fetch_stage =
+            BlobFetchStage::new_multi_socket(blob_sockets, &blob_fetch_sender, exit.clone());
+
+        //TODO
+        //the packets coming out of blob_receiver need to be sent to the GPU and verified
+        //then sent to the window, which does the erasure coding reconstruction
+        let retransmit_stage = RetransmitStage::new(
+            &bank_forks,
+            blocktree.clone(),
+            &cluster_info,
+            Arc::new(retransmit_socket),
+            repair_socket,
+            blob_fetch_receiver,
+            exit.clone(),
+        );
+
+        let (replay_stage, slot_full_receiver, forward_entry_receiver) = ReplayStage::new(
+            keypair.pubkey(),
+            voting_keypair,
+            blocktree.clone(),
+            &bank_forks,
+            &bank_forks_info,
+            cluster_info.clone(),
+            exit.clone(),
+            to_leader_sender,
+            ledger_signal_receiver,
+            subscriptions,
+        );
+
+        let blockstream_service = if blockstream.is_some() {
+            let blockstream_service = BlockstreamService::new(
+                slot_full_receiver,
+                blocktree.clone(),
+                blockstream.unwrap().to_string(),
+                exit.clone(),
+            );
+            Some(blockstream_service)
+        } else {
+            None
+        };
+
+        let storage_stage = StorageStage::new(
+            storage_state,
+            forward_entry_receiver,
+            Some(blocktree),
+            &keypair,
+            &exit.clone(),
+            bank_forks_info[0].entry_height, // TODO: StorageStage needs to deal with BankForks somehow still
+            storage_rotate_count,
+            &cluster_info,
+        );
+
+        Tvu {
+            fetch_stage,
+            retransmit_stage,
+            replay_stage,
+            blockstream_service,
+            storage_stage,
+            exit,
+        }
+    }
+
+    pub fn is_exited(&self) -> bool {
+        self.exit.load(Ordering::Relaxed)
+    }
+
+    pub fn exit(&self) {
+        // Call exit to make sure replay stage is unblocked from a channel it may be blocked on.
+        // Then replay stage will set the self.exit variable and cause the rest of the
+        // pipeline to exit
+        self.replay_stage.exit();
+    }
+
+    pub fn close(self) -> thread::Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+impl Service for Tvu {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.retransmit_stage.join()?;
+        self.fetch_stage.join()?;
+        self.storage_stage.join()?;
+        if self.blockstream_service.is_some() {
+            self.blockstream_service.unwrap().join()?;
+        }
+        self.replay_stage.join()?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use crate::blocktree::get_tmp_ledger_path;
+    use crate::cluster_info::{ClusterInfo, Node};
+    use crate::storage_stage::STORAGE_ROTATE_TEST_COUNT;
+    use solana_runtime::bank::Bank;
+    use solana_sdk::genesis_block::GenesisBlock;
+
+    #[test]
+    fn test_tvu_exit() {
+        solana_logger::setup();
+        let leader = Node::new_localhost();
+        let target1_keypair = Keypair::new();
+        let target1 = Node::new_localhost_with_pubkey(target1_keypair.pubkey());
+
+        let starting_balance = 10_000;
+        let (genesis_block, _mint_keypair) = GenesisBlock::new(starting_balance);
+
+        let bank_forks = BankForks::new(0, Bank::new(&genesis_block));
+        let bank_forks_info = vec![BankForksInfo {
+            bank_id: 0,
+            entry_height: 0,
+        }];
+
+        //start cluster_info1
+        let mut cluster_info1 = ClusterInfo::new(target1.info.clone());
+        cluster_info1.insert_info(leader.info.clone());
+        cluster_info1.set_leader(leader.info.id);
+        let cref1 = Arc::new(RwLock::new(cluster_info1));
+
+        let blocktree_path = get_tmp_ledger_path!();
+        let (blocktree, l_receiver) = Blocktree::open_with_signal(&blocktree_path)
+            .expect("Expected to successfully open ledger");
+        let (sender, _receiver) = channel();
+        let tvu = Tvu::new(
+            Some(Arc::new(Keypair::new())),
+            &Arc::new(RwLock::new(bank_forks)),
+            &bank_forks_info,
+            &cref1,
+            {
+                Sockets {
+                    repair: target1.sockets.repair,
+                    retransmit: target1.sockets.retransmit,
+                    fetch: target1.sockets.tvu,
+                }
+            },
+            Arc::new(blocktree),
+            STORAGE_ROTATE_TEST_COUNT,
+            &sender,
+            &StorageState::default(),
+            None,
+            l_receiver,
+            &Arc::new(RpcSubscriptions::default()),
+        );
+        tvu.close().expect("close");
+    }
+}

core/src/voting_keypair.rs

@@ -0,0 +1,139 @@
+//! The `vote_signer_proxy` votes on the `last_id` of the bank at a regular cadence
+
+use crate::rpc_request::{RpcClient, RpcRequest};
+use jsonrpc_core;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
+use solana_vote_signer::rpc::LocalVoteSigner;
+use solana_vote_signer::rpc::VoteSigner;
+use std::net::SocketAddr;
+use std::sync::Arc;
+
+pub struct RemoteVoteSigner {
+    rpc_client: RpcClient,
+}
+
+impl RemoteVoteSigner {
+    pub fn new(signer: SocketAddr) -> Self {
+        let rpc_client = RpcClient::new_from_socket(signer);
+        Self { rpc_client }
+    }
+}
+
+impl VoteSigner for RemoteVoteSigner {
+    fn register(
+        &self,
+        pubkey: Pubkey,
+        sig: &Signature,
+        msg: &[u8],
+    ) -> jsonrpc_core::Result<Pubkey> {
+        let params = json!([pubkey, sig, msg]);
+        let resp = self
+            .rpc_client
+            .retry_make_rpc_request(1, &RpcRequest::RegisterNode, Some(params), 5)
+            .unwrap();
+        let vote_account: Pubkey = serde_json::from_value(resp).unwrap();
+        Ok(vote_account)
+    }
+    fn sign(&self, pubkey: Pubkey, sig: &Signature, msg: &[u8]) -> jsonrpc_core::Result<Signature> {
+        let params = json!([pubkey, sig, msg]);
+        let resp = self
+            .rpc_client
+            .retry_make_rpc_request(1, &RpcRequest::SignVote, Some(params), 0)
+            .unwrap();
+        let vote_signature: Signature = serde_json::from_value(resp).unwrap();
+        Ok(vote_signature)
+    }
+    fn deregister(&self, pubkey: Pubkey, sig: &Signature, msg: &[u8]) -> jsonrpc_core::Result<()> {
+        let params = json!([pubkey, sig, msg]);
+        let _resp = self
+            .rpc_client
+            .retry_make_rpc_request(1, &RpcRequest::DeregisterNode, Some(params), 5)
+            .unwrap();
+        Ok(())
+    }
+}
+
+impl KeypairUtil for VotingKeypair {
+    /// Return a local VotingKeypair with a new keypair. Used for unit-tests.
+    fn new() -> Self {
+        Self::new_local(&Arc::new(Keypair::new()))
+    }
+
+    /// Return the public key of the keypair used to sign votes
+    fn pubkey(&self) -> Pubkey {
+        self.vote_account
+    }
+
+    fn sign_message(&self, msg: &[u8]) -> Signature {
+        let sig = self.keypair.sign_message(msg);
+        self.signer.sign(self.keypair.pubkey(), &sig, &msg).unwrap()
+    }
+}
+
+pub struct VotingKeypair {
+    keypair: Arc<Keypair>,
+    signer: Box<VoteSigner + Send + Sync>,
+    vote_account: Pubkey,
+}
+
+impl VotingKeypair {
+    pub fn new_with_signer(keypair: &Arc<Keypair>, signer: Box<VoteSigner + Send + Sync>) -> Self {
+        let msg = "Registering a new node";
+        let sig = keypair.sign_message(msg.as_bytes());
+        let vote_account = signer
+            .register(keypair.pubkey(), &sig, msg.as_bytes())
+            .unwrap();
+        Self {
+            keypair: keypair.clone(),
+            signer,
+            vote_account,
+        }
+    }
+
+    pub fn new_local(keypair: &Arc<Keypair>) -> Self {
+        Self::new_with_signer(keypair, Box::new(LocalVoteSigner::default()))
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use solana_runtime::bank::Bank;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::vote_transaction::VoteTransaction;
+
+    pub fn new_vote_account(
+        from_keypair: &Keypair,
+        voting_pubkey: &Pubkey,
+        bank: &Bank,
+        num_tokens: u64,
+    ) {
+        let last_id = bank.last_id();
+        let tx = VoteTransaction::fund_staking_account(
+            from_keypair,
+            *voting_pubkey,
+            last_id,
+            num_tokens,
+            0,
+        );
+        bank.process_transaction(&tx).unwrap();
+    }
+
+    pub fn push_vote<T: KeypairUtil>(voting_keypair: &T, bank: &Bank, slot_height: u64) {
+        let last_id = bank.last_id();
+        let tx = VoteTransaction::new_vote(voting_keypair, slot_height, last_id, 0);
+        bank.process_transaction(&tx).unwrap();
+    }
+
+    pub fn new_vote_account_with_vote<T: KeypairUtil>(
+        from_keypair: &Keypair,
+        voting_keypair: &T,
+        bank: &Bank,
+        num_tokens: u64,
+        slot_height: u64,
+    ) {
+        new_vote_account(from_keypair, &voting_keypair.pubkey(), bank, num_tokens);
+        push_vote(voting_keypair, bank, slot_height);
+    }
+}

core/src/window.rs

@@ -0,0 +1,325 @@
+//! The `window` module defines data structure for storing the tail of the ledger.
+//!
+use crate::packet::SharedBlob;
+use solana_sdk::pubkey::Pubkey;
+use std::cmp;
+use std::sync::{Arc, RwLock};
+
+#[derive(Default, Clone)]
+pub struct WindowSlot {
+    pub data: Option<SharedBlob>,
+    pub coding: Option<SharedBlob>,
+    pub leader_unknown: bool,
+}
+
+impl WindowSlot {
+    fn blob_index(&self) -> Option<u64> {
+        match self.data {
+            Some(ref blob) => Some(blob.read().unwrap().index()),
+            None => None,
+        }
+    }
+
+    fn clear_data(&mut self) {
+        self.data.take();
+    }
+}
+
+type Window = Vec<WindowSlot>;
+pub type SharedWindow = Arc<RwLock<Window>>;
+
+#[derive(Debug)]
+pub struct WindowIndex {
+    pub data: u64,
+    pub coding: u64,
+}
+
+pub trait WindowUtil {
+    /// Finds available slots, clears them, and returns their indices.
+    fn clear_slots(&mut self, consumed: u64, received: u64) -> Vec<u64>;
+
+    fn window_size(&self) -> u64;
+
+    fn print(&self, id: &Pubkey, consumed: u64) -> String;
+
+    fn blob_idx_in_window(&self, id: &Pubkey, pix: u64, consumed: u64, received: &mut u64) -> bool;
+}
+
+impl WindowUtil for Window {
+    fn clear_slots(&mut self, consumed: u64, received: u64) -> Vec<u64> {
+        (consumed..received)
+            .filter_map(|pix| {
+                let i = (pix % self.window_size()) as usize;
+                if let Some(blob_idx) = self[i].blob_index() {
+                    if blob_idx == pix {
+                        return None;
+                    }
+                }
+                self[i].clear_data();
+                Some(pix)
+            })
+            .collect()
+    }
+
+    fn blob_idx_in_window(&self, id: &Pubkey, pix: u64, consumed: u64, received: &mut u64) -> bool {
+        // Prevent receive window from running over
+        // Got a blob which has already been consumed, skip it
+        // probably from a repair window request
+        if pix < consumed {
+            trace!(
+                "{}: received: {} but older than consumed: {} skipping..",
+                id,
+                pix,
+                consumed
+            );
+            false
+        } else {
+            // received always has to be updated even if we don't accept the packet into
+            //  the window.  The worst case here is the server *starts* outside
+            //  the window, none of the packets it receives fits in the window
+            //  and repair requests (which are based on received) are never generated
+            *received = cmp::max(pix, *received);
+
+            if pix >= consumed + self.window_size() {
+                trace!(
+                    "{}: received: {} will overrun window: {} skipping..",
+                    id,
+                    pix,
+                    consumed + self.window_size()
+                );
+                false
+            } else {
+                true
+            }
+        }
+    }
+
+    fn window_size(&self) -> u64 {
+        self.len() as u64
+    }
+
+    fn print(&self, id: &Pubkey, consumed: u64) -> String {
+        let pointer: Vec<_> = self
+            .iter()
+            .enumerate()
+            .map(|(i, _v)| {
+                if i == (consumed % self.window_size()) as usize {
+                    "V"
+                } else {
+                    " "
+                }
+            })
+            .collect();
+
+        let buf: Vec<_> = self
+            .iter()
+            .map(|v| {
+                if v.data.is_none() && v.coding.is_none() {
+                    "O"
+                } else if v.data.is_some() && v.coding.is_some() {
+                    "D"
+                } else if v.data.is_some() {
+                    // coding.is_none()
+                    "d"
+                } else {
+                    // data.is_none()
+                    "c"
+                }
+            })
+            .collect();
+        format!(
+            "\n{}: WINDOW ({}): {}\n{}: WINDOW ({}): {}",
+            id,
+            consumed,
+            pointer.join(""),
+            id,
+            consumed,
+            buf.join("")
+        )
+    }
+}
+
+fn calculate_max_repair(
+    num_peers: u64,
+    consumed: u64,
+    received: u64,
+    times: usize,
+    is_next_leader: bool,
+    window_size: u64,
+) -> u64 {
+    // Calculate the highest blob index that this node should have already received
+    // via avalanche. The avalanche splits data stream into nodes and each node retransmits
+    // the data to their peer nodes. So there's a possibility that a blob (with index lower
+    // than current received index) is being retransmitted by a peer node.
+    let max_repair = if times >= 8 || is_next_leader {
+        // if repair backoff is getting high, or if we are the next leader,
+        // don't wait for avalanche
+        cmp::max(consumed, received)
+    } else {
+        cmp::max(consumed, received.saturating_sub(num_peers))
+    };
+
+    // This check prevents repairing a blob that will cause window to roll over. Even if
+    // the highes_lost blob is actually missing, asking to repair it might cause our
+    // current window to move past other missing blobs
+    cmp::min(consumed + window_size - 1, max_repair)
+}
+
+pub fn new_window(window_size: usize) -> Window {
+    (0..window_size).map(|_| WindowSlot::default()).collect()
+}
+
+pub fn default_window() -> Window {
+    (0..2048).map(|_| WindowSlot::default()).collect()
+}
+
+#[cfg(test)]
+mod test {
+    use crate::packet::{Blob, Packet, Packets, SharedBlob, PACKET_DATA_SIZE};
+    use crate::streamer::{receiver, responder, PacketReceiver};
+    use crate::window::{calculate_max_repair, new_window, Window, WindowUtil};
+    use solana_sdk::pubkey::Pubkey;
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    fn get_msgs(r: PacketReceiver, num: &mut usize) {
+        for _t in 0..5 {
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(m) => *num += m.read().unwrap().packets.len(),
+                e => info!("error {:?}", e),
+            }
+            if *num == 10 {
+                break;
+            }
+        }
+    }
+    #[test]
+    pub fn streamer_debug() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn streamer_send_test() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
+
+        let addr = read.local_addr().unwrap();
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let (s_reader, r_reader) = channel();
+        let t_receiver = receiver(
+            Arc::new(read),
+            exit.clone(),
+            s_reader,
+            "window-streamer-test",
+        );
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let t_responder = responder("streamer_send_test", Arc::new(send), r_responder);
+            let mut msgs = Vec::new();
+            for i in 0..10 {
+                let b = SharedBlob::default();
+                {
+                    let mut w = b.write().unwrap();
+                    w.data[0] = i as u8;
+                    w.meta.size = PACKET_DATA_SIZE;
+                    w.meta.set_addr(&addr);
+                }
+                msgs.push(b);
+            }
+            s_responder.send(msgs).expect("send");
+            t_responder
+        };
+
+        let mut num = 0;
+        get_msgs(r_reader, &mut num);
+        assert_eq!(num, 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+
+    #[test]
+    pub fn test_calculate_max_repair() {
+        const WINDOW_SIZE: u64 = 200;
+
+        assert_eq!(calculate_max_repair(0, 10, 90, 0, false, WINDOW_SIZE), 90);
+        assert_eq!(calculate_max_repair(15, 10, 90, 32, false, WINDOW_SIZE), 90);
+        assert_eq!(calculate_max_repair(15, 10, 90, 0, false, WINDOW_SIZE), 75);
+        assert_eq!(calculate_max_repair(90, 10, 90, 0, false, WINDOW_SIZE), 10);
+        assert_eq!(calculate_max_repair(90, 10, 50, 0, false, WINDOW_SIZE), 10);
+        assert_eq!(calculate_max_repair(90, 10, 99, 0, false, WINDOW_SIZE), 10);
+        assert_eq!(calculate_max_repair(90, 10, 101, 0, false, WINDOW_SIZE), 11);
+        assert_eq!(
+            calculate_max_repair(90, 10, 95 + WINDOW_SIZE, 0, false, WINDOW_SIZE),
+            WINDOW_SIZE + 5
+        );
+        assert_eq!(
+            calculate_max_repair(90, 10, 99 + WINDOW_SIZE, 0, false, WINDOW_SIZE),
+            WINDOW_SIZE + 9
+        );
+        assert_eq!(
+            calculate_max_repair(90, 10, 100 + WINDOW_SIZE, 0, false, WINDOW_SIZE),
+            WINDOW_SIZE + 9
+        );
+        assert_eq!(
+            calculate_max_repair(90, 10, 120 + WINDOW_SIZE, 0, false, WINDOW_SIZE),
+            WINDOW_SIZE + 9
+        );
+        assert_eq!(
+            calculate_max_repair(50, 100, 50 + WINDOW_SIZE, 0, false, WINDOW_SIZE),
+            WINDOW_SIZE
+        );
+        assert_eq!(
+            calculate_max_repair(50, 100, 50 + WINDOW_SIZE, 0, true, WINDOW_SIZE),
+            50 + WINDOW_SIZE
+        );
+    }
+
+    fn wrap_blob_idx_in_window(
+        window: &Window,
+        id: &Pubkey,
+        pix: u64,
+        consumed: u64,
+        received: u64,
+    ) -> (bool, u64) {
+        let mut received = received;
+        let is_in_window = window.blob_idx_in_window(&id, pix, consumed, &mut received);
+        (is_in_window, received)
+    }
+    #[test]
+    pub fn test_blob_idx_in_window() {
+        let id = Pubkey::default();
+        const WINDOW_SIZE: u64 = 200;
+        let window = new_window(WINDOW_SIZE as usize);
+
+        assert_eq!(
+            wrap_blob_idx_in_window(&window, &id, 90 + WINDOW_SIZE, 90, 100),
+            (false, 90 + WINDOW_SIZE)
+        );
+        assert_eq!(
+            wrap_blob_idx_in_window(&window, &id, 91 + WINDOW_SIZE, 90, 100),
+            (false, 91 + WINDOW_SIZE)
+        );
+        assert_eq!(
+            wrap_blob_idx_in_window(&window, &id, 89, 90, 100),
+            (false, 100)
+        );
+
+        assert_eq!(
+            wrap_blob_idx_in_window(&window, &id, 91, 90, 100),
+            (true, 100)
+        );
+        assert_eq!(
+            wrap_blob_idx_in_window(&window, &id, 101, 90, 100),
+            (true, 101)
+        );
+    }
+}

core/src/window_service.rs

@@ -0,0 +1,308 @@
+//! The `window_service` provides a thread for maintaining a window (tail of the ledger).
+//!
+use crate::blocktree::Blocktree;
+use crate::cluster_info::ClusterInfo;
+use crate::db_window::*;
+use crate::repair_service::RepairService;
+use crate::result::{Error, Result};
+use crate::service::Service;
+use crate::streamer::{BlobReceiver, BlobSender};
+use solana_metrics::counter::Counter;
+use solana_metrics::{influxdb, submit};
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::timing::duration_as_ms;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::RecvTimeoutError;
+use std::sync::{Arc, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::{Duration, Instant};
+
+pub const MAX_REPAIR_BACKOFF: usize = 128;
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum WindowServiceReturnType {
+    LeaderRotation(u64),
+}
+
+#[allow(clippy::too_many_arguments)]
+fn recv_window(
+    blocktree: &Arc<Blocktree>,
+    id: &Pubkey,
+    r: &BlobReceiver,
+    retransmit: &BlobSender,
+) -> Result<()> {
+    let timer = Duration::from_millis(200);
+    let mut dq = r.recv_timeout(timer)?;
+
+    while let Ok(mut nq) = r.try_recv() {
+        dq.append(&mut nq)
+    }
+    let now = Instant::now();
+    inc_new_counter_info!("streamer-recv_window-recv", dq.len(), 100);
+
+    submit(
+        influxdb::Point::new("recv-window")
+            .add_field("count", influxdb::Value::Integer(dq.len() as i64))
+            .to_owned(),
+    );
+
+    retransmit_blobs(&dq, retransmit, id)?;
+
+    //send a contiguous set of blocks
+    trace!("{} num blobs received: {}", id, dq.len());
+
+    for b in dq {
+        let (pix, meta_size) = {
+            let p = b.read().unwrap();
+            (p.index(), p.meta.size)
+        };
+
+        trace!("{} window pix: {} size: {}", id, pix, meta_size);
+
+        let _ = process_blob(blocktree, &b);
+    }
+
+    trace!(
+        "Elapsed processing time in recv_window(): {}",
+        duration_as_ms(&now.elapsed())
+    );
+
+    Ok(())
+}
+
+// Implement a destructor for the window_service thread to signal it exited
+// even on panics
+struct Finalizer {
+    exit_sender: Arc<AtomicBool>,
+}
+
+impl Finalizer {
+    fn new(exit_sender: Arc<AtomicBool>) -> Self {
+        Finalizer { exit_sender }
+    }
+}
+// Implement a destructor for Finalizer.
+impl Drop for Finalizer {
+    fn drop(&mut self) {
+        self.exit_sender.clone().store(true, Ordering::Relaxed);
+    }
+}
+
+pub struct WindowService {
+    t_window: JoinHandle<()>,
+    repair_service: RepairService,
+}
+
+impl WindowService {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        blocktree: Arc<Blocktree>,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        r: BlobReceiver,
+        retransmit: BlobSender,
+        repair_socket: Arc<UdpSocket>,
+        exit: Arc<AtomicBool>,
+    ) -> WindowService {
+        let exit_ = exit.clone();
+        let repair_service = RepairService::new(
+            blocktree.clone(),
+            exit.clone(),
+            repair_socket,
+            cluster_info.clone(),
+        );
+        let t_window = Builder::new()
+            .name("solana-window".to_string())
+            .spawn(move || {
+                let _exit = Finalizer::new(exit_);
+                let id = cluster_info.read().unwrap().id();
+                trace!("{}: RECV_WINDOW started", id);
+                loop {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+                    if let Err(e) = recv_window(&blocktree, &id, &r, &retransmit) {
+                        match e {
+                            Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                            Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                            _ => {
+                                inc_new_counter_info!("streamer-window-error", 1, 1);
+                                error!("window error: {:?}", e);
+                            }
+                        }
+                    }
+                }
+            })
+            .unwrap();
+
+        WindowService {
+            t_window,
+            repair_service,
+        }
+    }
+}
+
+impl Service for WindowService {
+    type JoinReturnType = ();
+
+    fn join(self) -> thread::Result<()> {
+        self.t_window.join()?;
+        self.repair_service.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::blocktree::get_tmp_ledger_path;
+    use crate::blocktree::Blocktree;
+    use crate::cluster_info::{ClusterInfo, Node};
+    use crate::entry::make_consecutive_blobs;
+    use crate::service::Service;
+    use crate::streamer::{blob_receiver, responder};
+    use crate::window_service::WindowService;
+    use solana_sdk::hash::Hash;
+    use std::fs::remove_dir_all;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::time::Duration;
+
+    #[test]
+    pub fn window_send_test() {
+        solana_logger::setup();
+        // setup a leader whose id is used to generates blobs and a validator
+        // node whose window service will retransmit leader blobs.
+        let leader_node = Node::new_localhost();
+        let validator_node = Node::new_localhost();
+        let exit = Arc::new(AtomicBool::new(false));
+        let mut cluster_info_me = ClusterInfo::new(validator_node.info.clone());
+        let me_id = leader_node.info.id;
+        cluster_info_me.set_leader(me_id);
+        let subs = Arc::new(RwLock::new(cluster_info_me));
+
+        let (s_reader, r_reader) = channel();
+        let t_receiver =
+            blob_receiver(Arc::new(leader_node.sockets.gossip), exit.clone(), s_reader);
+        let (s_retransmit, r_retransmit) = channel();
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Arc::new(
+            Blocktree::open(&blocktree_path).expect("Expected to be able to open database ledger"),
+        );
+        let t_window = WindowService::new(
+            blocktree,
+            subs,
+            r_reader,
+            s_retransmit,
+            Arc::new(leader_node.sockets.repair),
+            exit.clone(),
+        );
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let blob_sockets: Vec<Arc<UdpSocket>> =
+                leader_node.sockets.tvu.into_iter().map(Arc::new).collect();
+
+            let t_responder = responder("window_send_test", blob_sockets[0].clone(), r_responder);
+            let num_blobs_to_make = 10;
+            let gossip_address = &leader_node.info.gossip;
+            let msgs = make_consecutive_blobs(
+                &me_id,
+                num_blobs_to_make,
+                0,
+                Hash::default(),
+                &gossip_address,
+            )
+            .into_iter()
+            .rev()
+            .collect();;
+            s_responder.send(msgs).expect("send");
+            t_responder
+        };
+
+        let max_attempts = 10;
+        let mut num_attempts = 0;
+        let mut q = Vec::new();
+        loop {
+            assert!(num_attempts != max_attempts);
+            while let Ok(mut nq) = r_retransmit.recv_timeout(Duration::from_millis(500)) {
+                q.append(&mut nq);
+            }
+            if q.len() == 10 {
+                break;
+            }
+            num_attempts += 1;
+        }
+
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+        t_window.join().expect("join");
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+        let _ignored = remove_dir_all(&blocktree_path);
+    }
+
+    #[test]
+    pub fn window_send_leader_test2() {
+        solana_logger::setup();
+        // setup a leader whose id is used to generates blobs and a validator
+        // node whose window service will retransmit leader blobs.
+        let leader_node = Node::new_localhost();
+        let validator_node = Node::new_localhost();
+        let exit = Arc::new(AtomicBool::new(false));
+        let cluster_info_me = ClusterInfo::new(validator_node.info.clone());
+        let me_id = leader_node.info.id;
+        let subs = Arc::new(RwLock::new(cluster_info_me));
+
+        let (s_reader, r_reader) = channel();
+        let t_receiver =
+            blob_receiver(Arc::new(leader_node.sockets.gossip), exit.clone(), s_reader);
+        let (s_retransmit, r_retransmit) = channel();
+        let blocktree_path = get_tmp_ledger_path!();
+        let blocktree = Arc::new(
+            Blocktree::open(&blocktree_path).expect("Expected to be able to open database ledger"),
+        );
+        let t_window = WindowService::new(
+            blocktree,
+            subs.clone(),
+            r_reader,
+            s_retransmit,
+            Arc::new(leader_node.sockets.repair),
+            exit.clone(),
+        );
+        let t_responder = {
+            let (s_responder, r_responder) = channel();
+            let blob_sockets: Vec<Arc<UdpSocket>> =
+                leader_node.sockets.tvu.into_iter().map(Arc::new).collect();
+            let t_responder = responder("window_send_test", blob_sockets[0].clone(), r_responder);
+            let mut msgs = Vec::new();
+            let blobs =
+                make_consecutive_blobs(&me_id, 14u64, 0, Hash::default(), &leader_node.info.gossip);
+
+            for v in 0..10 {
+                let i = 9 - v;
+                msgs.push(blobs[i].clone());
+            }
+            s_responder.send(msgs).expect("send");
+
+            subs.write().unwrap().set_leader(me_id);
+            let mut msgs1 = Vec::new();
+            for v in 1..5 {
+                let i = 9 + v;
+                msgs1.push(blobs[i].clone());
+            }
+            s_responder.send(msgs1).expect("send");
+            t_responder
+        };
+        let mut q = Vec::new();
+        while let Ok(mut nq) = r_retransmit.recv_timeout(Duration::from_millis(500)) {
+            q.append(&mut nq);
+        }
+        assert!(q.len() > 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+        t_window.join().expect("join");
+        Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
+        let _ignored = remove_dir_all(&blocktree_path);
+    }
+}