Add StakeInstruction::Merge (#10503) (#10507)

automerge

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,10 +10,10 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-measure = { path = "../measure", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,16 +13,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-core = { path = "../core", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-perf = { path = "../perf", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-streamer = { path = "../streamer", version = "1.2.1" }
+solana-perf = { path = "../perf", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-measure = { path = "../measure", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-genesis = { path = "../genesis", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-genesis = { path = "../genesis", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-faucet = { path = "../faucet", version = "1.2.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.2.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -449,7 +449,7 @@ fn swapper<T>(
             }
             account_group = (account_group + 1) % account_groups as usize;
 
-            let (blockhash, _fee_calculator) = client
+            let (blockhash, _fee_calculator, _last_valid_slot) = client
                 .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                 .expect("Failed to get blockhash");
             let to_swap_txs: Vec<_> = to_swap
@@ -577,7 +577,7 @@ fn trader<T>(
         }
         account_group = (account_group + 1) % account_groups as usize;
 
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
 
@@ -776,7 +776,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
                     to_fund_txs.len(),
                 );
 
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("blockhash");
                 to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
@@ -868,7 +868,7 @@ pub fn create_token_accounts<T: Client>(
 
             let mut retries = 0;
             while !to_create_txs.is_empty() {
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("Failed to get blockhash");
                 to_create_txs
@@ -997,7 +997,7 @@ pub fn airdrop_lamports<T: Client>(
 
     let mut tries = 0;
     loop {
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
         match request_airdrop_transaction(&faucet_addr, &id.pubkey(), amount_to_drop, blockhash) {

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-streamer = { path = "../streamer", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,25 +14,25 @@ log = "0.4.8"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-genesis = { path = "../genesis", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-librapay = { path = "../programs/librapay", version = "1.2.0", optional = true }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.2.0", optional = true }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-genesis = { path = "../genesis", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-faucet = { path = "../faucet", version = "1.2.1" }
+solana-librapay = { path = "../programs/librapay", version = "1.2.1", optional = true }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
+solana-measure = { path = "../measure", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-move-loader-program = { path = "../programs/move_loader", version = "1.2.1", optional = true }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.2.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.1" }
 
 [features]
 move = ["solana-librapay", "solana-move-loader-program"]

bench-tps/src/bench.rs

@@ -55,7 +55,9 @@ type LibraKeys = (Keypair, Pubkey, Pubkey, Vec<Keypair>);
 fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
         match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
-            Ok((blockhash, fee_calculator)) => return (blockhash, fee_calculator),
+            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
+                return (blockhash, fee_calculator)
+            }
             Err(err) => {
                 info!("Couldn't get recent blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));

ci/run-sanity.sh

@@ -19,6 +19,9 @@ while [[ ! -f config/run/init-completed ]]; do
   fi
 done
 
+while [[ $($solana_cli --url http://localhost:8899 slot --commitment recent) -eq 0 ]]; do
+  sleep 1
+done
 curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"validatorExit"}' http://localhost:8899
 
 wait $pid

ci/test-stable.sh

@@ -39,9 +39,9 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 rm -rf target/xargo # Issue #3105
 
 # Limit compiler jobs to reduce memory usage
-# on machines with 1gb/thread of memory
+# on machines with 2gb/thread of memory
 NPROC=$(nproc)
-NPROC=$((NPROC>16 ? 16 : NPROC))
+NPROC=$((NPROC>14 ? 14 : NPROC))
 
 echo "Executing $testName"
 case $testName in

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,28 +27,28 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-cli-config = { path = "../cli-config", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-config-program = { path = "../programs/config", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.0" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-cli-config = { path = "../cli-config", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-config-program = { path = "../programs/config", version = "1.2.1" }
+solana-faucet = { path = "../faucet", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.1" }
 thiserror = "1.0.19"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.2.0" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.1" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -2,7 +2,7 @@ use crate::{
     checks::*,
     cli_output::{CliAccount, CliSignOnlyData, CliSignature, OutputFormat},
     cluster_query::*,
-    display::println_name_value,
+    display::{new_spinner_progress_bar, println_name_value, println_transaction},
     nonce::{self, *},
     offline::{blockhash_query::BlockhashQuery, *},
     spend_utils::*,
@@ -27,7 +27,7 @@ use solana_clap_utils::{
 use solana_client::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     rpc_client::RpcClient,
-    rpc_config::RpcLargestAccountsFilter,
+    rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
     rpc_response::{RpcAccount, RpcKeyedAccount},
 };
 #[cfg(not(test))]
@@ -37,7 +37,7 @@ use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     bpf_loader,
-    clock::{Epoch, Slot},
+    clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
     commitment_config::CommitmentConfig,
     fee_calculator::FeeCalculator,
     hash::Hash,
@@ -48,6 +48,7 @@ use solana_sdk::{
     program_utils::DecodeError,
     pubkey::{Pubkey, MAX_SEED_LEN},
     signature::{Keypair, Signature, Signer, SignerError},
+    signers::Signers,
     system_instruction::{self, SystemError},
     system_program,
     transaction::{Transaction, TransactionError},
@@ -396,6 +397,10 @@ pub enum CliCommand {
         vote_account_pubkey: Pubkey,
         new_identity_account: SignerIndex,
     },
+    VoteUpdateCommission {
+        vote_account_pubkey: Pubkey,
+        commission: u8,
+    },
     // Wallet Commands
     Address,
     Airdrop {
@@ -709,6 +714,9 @@ pub fn parse_command(
         ("vote-update-validator", Some(matches)) => {
             parse_vote_update_validator(matches, default_signer_path, wallet_manager)
         }
+        ("vote-update-commission", Some(matches)) => {
+            parse_vote_update_commission(matches, default_signer_path, wallet_manager)
+        }
         ("vote-authorize-voter", Some(matches)) => parse_vote_authorize(
             matches,
             default_signer_path,
@@ -1159,7 +1167,7 @@ fn process_confirm(
                                 "\nTransaction executed in slot {}:",
                                 confirmed_transaction.slot
                             );
-                            crate::display::println_transaction(
+                            println_transaction(
                                 &confirmed_transaction
                                     .transaction
                                     .transaction
@@ -1189,7 +1197,7 @@ fn process_confirm(
 }
 
 fn process_decode_transaction(transaction: &Transaction) -> ProcessResult {
-    crate::display::println_transaction(transaction, &None, "");
+    println_transaction(transaction, &None, "");
     Ok("".to_string())
 }
 
@@ -1227,6 +1235,103 @@ fn process_show_account(
     Ok(account_string)
 }
 
+fn send_and_confirm_transactions_with_spinner<T: Signers>(
+    rpc_client: &RpcClient,
+    mut transactions: Vec<Transaction>,
+    signer_keys: &T,
+) -> Result<(), Box<dyn error::Error>> {
+    let progress_bar = new_spinner_progress_bar();
+    let mut send_retries = 5;
+    loop {
+        let mut status_retries = 15;
+
+        // Send all transactions
+        let mut transactions_signatures = vec![];
+        let num_transactions = transactions.len();
+        for transaction in transactions {
+            if cfg!(not(test)) {
+                // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
+                // when all the write transactions modify the same program account (eg, deploying a
+                // new program)
+                sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
+            }
+
+            let signature = rpc_client
+                .send_transaction_with_config(
+                    &transaction,
+                    RpcSendTransactionConfig {
+                        skip_preflight: true,
+                    },
+                )
+                .ok();
+            transactions_signatures.push((transaction, signature));
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions sent",
+                transactions_signatures.len(),
+                num_transactions
+            ));
+        }
+
+        // Collect statuses for all the transactions, drop those that are confirmed
+        while status_retries > 0 {
+            status_retries -= 1;
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Transactions confirmed",
+                num_transactions - transactions_signatures.len(),
+                num_transactions
+            ));
+
+            if cfg!(not(test)) {
+                // Retry twice a second
+                sleep(Duration::from_millis(500));
+            }
+
+            transactions_signatures = transactions_signatures
+                .into_iter()
+                .filter(|(_transaction, signature)| {
+                    if let Some(signature) = signature {
+                        if let Ok(status) = rpc_client.get_signature_status(&signature) {
+                            if rpc_client
+                                .get_num_blocks_since_signature_confirmation(&signature)
+                                .unwrap_or(0)
+                                > 1
+                            {
+                                return false;
+                            } else {
+                                return match status {
+                                    None => true,
+                                    Some(result) => result.is_err(),
+                                };
+                            }
+                        }
+                    }
+                    true
+                })
+                .collect();
+
+            if transactions_signatures.is_empty() {
+                return Ok(());
+            }
+        }
+
+        if send_retries == 0 {
+            return Err("Transactions failed".into());
+        }
+        send_retries -= 1;
+
+        // Re-sign any failed transactions with a new blockhash and retry
+        let (blockhash, _fee_calculator) = rpc_client
+            .get_new_blockhash(&transactions_signatures[0].0.message().recent_blockhash)?;
+        transactions = vec![];
+        for (mut transaction, _) in transactions_signatures.into_iter() {
+            transaction.try_sign(signer_keys, blockhash)?;
+            transactions.push(transaction);
+        }
+    }
+}
+
 fn process_deploy(
     rpc_client: &RpcClient,
     config: &CliConfig,
@@ -1294,15 +1399,18 @@ fn process_deploy(
     })?;
 
     trace!("Writing program data");
-    rpc_client
-        .send_and_confirm_transactions_with_spinner(write_transactions, &signers)
-        .map_err(|_| {
-            CliError::DynamicProgramError("Data writes to program account failed".to_string())
-        })?;
+    send_and_confirm_transactions_with_spinner(&rpc_client, write_transactions, &signers).map_err(
+        |_| CliError::DynamicProgramError("Data writes to program account failed".to_string()),
+    )?;
 
     trace!("Finalizing program account");
     rpc_client
-        .send_and_confirm_transaction_with_spinner(&finalize_tx)
+        .send_and_confirm_transaction_with_spinner_and_config(
+            &finalize_tx,
+            RpcSendTransactionConfig {
+                skip_preflight: true,
+            },
+        )
         .map_err(|e| {
             CliError::DynamicProgramError(format!("Finalizing program account failed: {}", e))
         })?;
@@ -1530,11 +1638,6 @@ fn process_transfer(
 ) -> ProcessResult {
     let from = config.signers[from];
 
-    check_unique_pubkeys(
-        (&from.pubkey(), "cli keypair".to_string()),
-        (to, "to".to_string()),
-    )?;
-
     let (recent_blockhash, fee_calculator) =
         blockhash_query.get_blockhash_and_fee_calculator(rpc_client)?;
 
@@ -1655,7 +1758,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             seed,
             program_id,
         } => process_create_address_with_seed(config, from_pubkey.as_ref(), &seed, &program_id),
-        CliCommand::Fees => process_fees(&rpc_client),
+        CliCommand::Fees => process_fees(&rpc_client, config),
         CliCommand::GetBlockTime { slot } => process_get_block_time(&rpc_client, config, *slot),
         CliCommand::GetGenesisHash => process_get_genesis_hash(&rpc_client),
         CliCommand::GetEpochInfo { commitment_config } => {
@@ -2048,6 +2151,10 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             &vote_account_pubkey,
             *new_identity_account,
         ),
+        CliCommand::VoteUpdateCommission {
+            vote_account_pubkey,
+            commission,
+        } => process_vote_update_commission(&rpc_client, config, &vote_account_pubkey, *commission),
 
         // Wallet Commands
 
@@ -3241,6 +3348,7 @@ mod tests {
         // Success cases
         let mut config = CliConfig::default();
         config.rpc_client = Some(RpcClient::new_mock("succeeds".to_string()));
+        config.json_rpc_url = "http://127.0.0.1:8899".to_string();
 
         let keypair = Keypair::new();
         let pubkey = keypair.pubkey().to_string();

cli/src/cli_output.rs

@@ -482,7 +482,7 @@ impl fmt::Display for CliKeyedStakeState {
 #[serde(rename_all = "camelCase")]
 pub struct CliStakeState {
     pub stake_type: CliStakeType,
-    pub total_stake: u64,
+    pub account_balance: u64,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub delegated_stake: Option<u64>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -497,6 +497,16 @@ pub struct CliStakeState {
     pub lockup: Option<CliLockup>,
     #[serde(skip_serializing)]
     pub use_lamports_unit: bool,
+    #[serde(skip_serializing)]
+    pub current_epoch: Epoch,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rent_exempt_reserve: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub active_stake: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub activating_stake: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub deactivating_stake: Option<u64>,
 }
 
 impl fmt::Display for CliStakeState {
@@ -522,52 +532,122 @@ impl fmt::Display for CliStakeState {
             Ok(())
         }
 
+        writeln!(
+            f,
+            "Balance: {}",
+            build_balance_message(self.account_balance, self.use_lamports_unit, true)
+        )?;
+
+        if let Some(rent_exempt_reserve) = self.rent_exempt_reserve {
+            writeln!(
+                f,
+                "Rent Exempt Reserve: {}",
+                build_balance_message(rent_exempt_reserve, self.use_lamports_unit, true)
+            )?;
+        }
+
         match self.stake_type {
             CliStakeType::RewardsPool => writeln!(f, "Stake account is a rewards pool")?,
             CliStakeType::Uninitialized => writeln!(f, "Stake account is uninitialized")?,
             CliStakeType::Initialized => {
-                writeln!(
-                    f,
-                    "Total Stake: {}",
-                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
-                )?;
                 writeln!(f, "Stake account is undelegated")?;
                 show_authorized(f, self.authorized.as_ref().unwrap())?;
                 show_lockup(f, self.lockup.as_ref().unwrap())?;
             }
             CliStakeType::Stake => {
-                writeln!(
-                    f,
-                    "Total Stake: {}",
-                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
-                )?;
-                writeln!(
-                    f,
-                    "Delegated Stake: {}",
-                    build_balance_message(
-                        self.delegated_stake.unwrap(),
-                        self.use_lamports_unit,
-                        true
-                    )
-                )?;
-                if let Some(delegated_vote_account_address) = &self.delegated_vote_account_address {
+                let show_delegation = {
+                    self.active_stake.is_some()
+                        || self.activating_stake.is_some()
+                        || self.deactivating_stake.is_some()
+                        || self
+                            .deactivation_epoch
+                            .map(|de| de > self.current_epoch)
+                            .unwrap_or(true)
+                };
+                if show_delegation {
+                    let delegated_stake = self.delegated_stake.unwrap();
                     writeln!(
                         f,
-                        "Delegated Vote Account Address: {}",
-                        delegated_vote_account_address
-                    )?;
-                }
-                writeln!(
-                    f,
-                    "Stake activates starting from epoch: {}",
-                    self.activation_epoch.unwrap()
-                )?;
-                if let Some(deactivation_epoch) = self.deactivation_epoch {
-                    writeln!(
-                        f,
-                        "Stake deactivates starting from epoch: {}",
-                        deactivation_epoch
+                        "Delegated Stake: {}",
+                        build_balance_message(delegated_stake, self.use_lamports_unit, true)
                     )?;
+                    if self
+                        .deactivation_epoch
+                        .map(|d| self.current_epoch <= d)
+                        .unwrap_or(true)
+                    {
+                        let active_stake = self.active_stake.unwrap_or(0);
+                        writeln!(
+                            f,
+                            "Active Stake: {}",
+                            build_balance_message(active_stake, self.use_lamports_unit, true),
+                        )?;
+                        let activating_stake = self.activating_stake.or_else(|| {
+                            if self.active_stake.is_none() {
+                                Some(delegated_stake)
+                            } else {
+                                None
+                            }
+                        });
+                        if let Some(activating_stake) = activating_stake {
+                            writeln!(
+                                f,
+                                "Activating Stake: {}",
+                                build_balance_message(
+                                    activating_stake,
+                                    self.use_lamports_unit,
+                                    true
+                                ),
+                            )?;
+                            writeln!(
+                                f,
+                                "Stake activates starting from epoch: {}",
+                                self.activation_epoch.unwrap()
+                            )?;
+                        }
+                    }
+
+                    if let Some(deactivation_epoch) = self.deactivation_epoch {
+                        if self.current_epoch > deactivation_epoch {
+                            let deactivating_stake = self.deactivating_stake.or(self.active_stake);
+                            if let Some(deactivating_stake) = deactivating_stake {
+                                writeln!(
+                                    f,
+                                    "Inactive Stake: {}",
+                                    build_balance_message(
+                                        delegated_stake - deactivating_stake,
+                                        self.use_lamports_unit,
+                                        true
+                                    ),
+                                )?;
+                                writeln!(
+                                    f,
+                                    "Deactivating Stake: {}",
+                                    build_balance_message(
+                                        deactivating_stake,
+                                        self.use_lamports_unit,
+                                        true
+                                    ),
+                                )?;
+                            }
+                        }
+                        writeln!(
+                            f,
+                            "Stake deactivates starting from epoch: {}",
+                            deactivation_epoch
+                        )?;
+                    }
+                    if let Some(delegated_vote_account_address) =
+                        &self.delegated_vote_account_address
+                    {
+                        writeln!(
+                            f,
+                            "Delegated Vote Account Address: {}",
+                            delegated_vote_account_address
+                        )?;
+                    }
+                } else {
+                    writeln!(f, "Stake account is undelegated")?;
                 }
                 show_authorized(f, self.authorized.as_ref().unwrap())?;
                 show_lockup(f, self.lockup.as_ref().unwrap())?;
@@ -900,6 +980,7 @@ impl fmt::Display for CliSignOnlyData {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliSignature {
     pub signature: String,
 }
@@ -913,6 +994,7 @@ impl fmt::Display for CliSignature {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliAccountBalances {
     pub accounts: Vec<RpcAccountBalance>,
 }
@@ -937,6 +1019,7 @@ impl fmt::Display for CliAccountBalances {
 }
 
 #[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct CliSupply {
     pub total: u64,
     pub circulating: u64,
@@ -981,3 +1064,25 @@ impl fmt::Display for CliSupply {
         Ok(())
     }
 }
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliFees {
+    pub slot: Slot,
+    pub blockhash: String,
+    pub lamports_per_signature: u64,
+    pub last_valid_slot: Slot,
+}
+
+impl fmt::Display for CliFees {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Blockhash:", &self.blockhash)?;
+        writeln_name_value(
+            f,
+            "Lamports per signature:",
+            &self.lamports_per_signature.to_string(),
+        )?;
+        writeln_name_value(f, "Last valid slot:", &self.last_valid_slot.to_string())?;
+        Ok(())
+    }
+}

cli/src/cluster_query.rs

@@ -1,12 +1,11 @@
 use crate::{
     cli::{CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
     cli_output::*,
-    display::println_name_value,
+    display::{new_spinner_progress_bar, println_name_value},
     spend_utils::{resolve_spend_tx_and_check_account_balance, SpendAmount},
 };
 use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
-use indicatif::{ProgressBar, ProgressStyle};
 use solana_clap_utils::{
     commitment::{commitment_arg, COMMITMENT_ARG},
     input_parsers::*,
@@ -29,7 +28,11 @@ use solana_sdk::{
     native_token::lamports_to_sol,
     pubkey::{self, Pubkey},
     system_instruction, system_program,
-    sysvar::{self, Sysvar},
+    sysvar::{
+        self,
+        stake_history::{self, StakeHistory},
+        Sysvar,
+    },
     transaction::Transaction,
 };
 use std::{
@@ -463,15 +466,6 @@ pub fn parse_transaction_history(
     })
 }
 
-/// Creates a new process bar for processing that will take an unknown amount of time
-fn new_spinner_progress_bar() -> ProgressBar {
-    let progress_bar = ProgressBar::new(42);
-    progress_bar
-        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
-    progress_bar.enable_steady_tick(100);
-    progress_bar
-}
-
 pub fn process_catchup(
     rpc_client: &RpcClient,
     node_pubkey: &Pubkey,
@@ -597,13 +591,16 @@ pub fn process_cluster_version(rpc_client: &RpcClient) -> ProcessResult {
     Ok(remote_version.solana_core)
 }
 
-pub fn process_fees(rpc_client: &RpcClient) -> ProcessResult {
-    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-
-    Ok(format!(
-        "blockhash: {}\nlamports per signature: {}",
-        recent_blockhash, fee_calculator.lamports_per_signature
-    ))
+pub fn process_fees(rpc_client: &RpcClient, config: &CliConfig) -> ProcessResult {
+    let result = rpc_client.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;
+    let (recent_blockhash, fee_calculator, last_valid_slot) = result.value;
+    let fees = CliFees {
+        slot: result.context.slot,
+        blockhash: recent_blockhash.to_string(),
+        lamports_per_signature: fee_calculator.lamports_per_signature,
+        last_valid_slot,
+    };
+    Ok(config.output_format.formatted_string(&fees))
 }
 
 pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
@@ -1181,8 +1178,13 @@ pub fn process_show_stakes(
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Fetching stake accounts...");
     let all_stake_accounts = rpc_client.get_program_accounts(&solana_stake_program::id())?;
+    let stake_history_account = rpc_client.get_account(&stake_history::id())?;
     progress_bar.finish_and_clear();
 
+    let stake_history = StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+        CliError::RpcRequestError("Failed to deserialize stake history".to_string())
+    })?;
+
     let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
         if let Ok(stake_state) = stake_account.state() {
@@ -1195,6 +1197,7 @@ pub fn process_show_stakes(
                                 stake_account.lamports,
                                 &stake_state,
                                 use_lamports_unit,
+                                &stake_history,
                             ),
                         });
                     }
@@ -1211,6 +1214,7 @@ pub fn process_show_stakes(
                                 stake_account.lamports,
                                 &stake_state,
                                 use_lamports_unit,
+                                &stake_history,
                             ),
                         });
                     }

cli/src/display.rs

@@ -1,5 +1,6 @@
 use crate::cli::SettingType;
 use console::style;
+use indicatif::{ProgressBar, ProgressStyle};
 use solana_sdk::{
     hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
     transaction::Transaction,
@@ -200,3 +201,12 @@ pub fn println_transaction(
         }
     }
 }
+
+/// Creates a new process bar for processing that will take an unknown amount of time
+pub fn new_spinner_progress_bar() -> ProgressBar {
+    let progress_bar = ProgressBar::new(42);
+    progress_bar
+        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
+    progress_bar.enable_steady_tick(100);
+    progress_bar
+}

cli/src/stake.rs

@@ -12,7 +12,7 @@ use crate::{
 };
 use clap::{App, Arg, ArgGroup, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*, offline::*, ArgConstant};
-use solana_client::rpc_client::RpcClient;
+use solana_client::{rpc_client::RpcClient, rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE};
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account_utils::StateMut,
@@ -1256,53 +1256,85 @@ pub fn process_stake_set_lockup(
     }
 }
 
+fn u64_some_if_not_zero(n: u64) -> Option<u64> {
+    if n > 0 {
+        Some(n)
+    } else {
+        None
+    }
+}
+
 pub fn build_stake_state(
-    stake_lamports: u64,
+    account_balance: u64,
     stake_state: &StakeState,
     use_lamports_unit: bool,
+    stake_history: &StakeHistory,
 ) -> CliStakeState {
     match stake_state {
         StakeState::Stake(
             Meta {
-                authorized, lockup, ..
+                rent_exempt_reserve,
+                authorized,
+                lockup,
             },
             stake,
-        ) => CliStakeState {
-            stake_type: CliStakeType::Stake,
-            total_stake: stake_lamports,
-            delegated_stake: Some(stake.delegation.stake),
-            delegated_vote_account_address: if stake.delegation.voter_pubkey != Pubkey::default() {
-                Some(stake.delegation.voter_pubkey.to_string())
-            } else {
-                None
-            },
-            activation_epoch: Some(if stake.delegation.activation_epoch < std::u64::MAX {
-                stake.delegation.activation_epoch
-            } else {
-                0
-            }),
-            deactivation_epoch: if stake.delegation.deactivation_epoch < std::u64::MAX {
-                Some(stake.delegation.deactivation_epoch)
-            } else {
-                None
-            },
-            authorized: Some(authorized.into()),
-            lockup: Some(lockup.into()),
-            use_lamports_unit,
-        },
+        ) => {
+            // The first entry in stake history is the previous epoch, so +1 for current
+            let current_epoch = stake_history.iter().next().unwrap().0 + 1;
+            let (active_stake, activating_stake, deactivating_stake) = stake
+                .delegation
+                .stake_activating_and_deactivating(current_epoch, Some(stake_history));
+            CliStakeState {
+                stake_type: CliStakeType::Stake,
+                account_balance,
+                delegated_stake: Some(stake.delegation.stake),
+                delegated_vote_account_address: if stake.delegation.voter_pubkey
+                    != Pubkey::default()
+                {
+                    Some(stake.delegation.voter_pubkey.to_string())
+                } else {
+                    None
+                },
+                activation_epoch: Some(if stake.delegation.activation_epoch < std::u64::MAX {
+                    stake.delegation.activation_epoch
+                } else {
+                    0
+                }),
+                deactivation_epoch: if stake.delegation.deactivation_epoch < std::u64::MAX {
+                    Some(stake.delegation.deactivation_epoch)
+                } else {
+                    None
+                },
+                authorized: Some(authorized.into()),
+                lockup: Some(lockup.into()),
+                use_lamports_unit,
+                current_epoch,
+                rent_exempt_reserve: Some(*rent_exempt_reserve),
+                active_stake: u64_some_if_not_zero(active_stake),
+                activating_stake: u64_some_if_not_zero(activating_stake),
+                deactivating_stake: u64_some_if_not_zero(deactivating_stake),
+            }
+        }
         StakeState::RewardsPool => CliStakeState {
             stake_type: CliStakeType::RewardsPool,
+            account_balance,
+            ..CliStakeState::default()
+        },
+        StakeState::Uninitialized => CliStakeState {
+            account_balance,
             ..CliStakeState::default()
         },
-        StakeState::Uninitialized => CliStakeState::default(),
         StakeState::Initialized(Meta {
-            authorized, lockup, ..
+            rent_exempt_reserve,
+            authorized,
+            lockup,
         }) => CliStakeState {
             stake_type: CliStakeType::Initialized,
-            total_stake: stake_lamports,
+            account_balance,
             authorized: Some(authorized.into()),
             lockup: Some(lockup.into()),
             use_lamports_unit,
+            rent_exempt_reserve: Some(*rent_exempt_reserve),
             ..CliStakeState::default()
         },
     }
@@ -1324,7 +1356,18 @@ pub fn process_show_stake_account(
     }
     match stake_account.state() {
         Ok(stake_state) => {
-            let state = build_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+            let stake_history_account = rpc_client.get_account(&stake_history::id())?;
+            let stake_history =
+                StakeHistory::from_account(&stake_history_account).ok_or_else(|| {
+                    CliError::RpcRequestError("Failed to deserialize stake history".to_string())
+                })?;
+
+            let state = build_stake_state(
+                stake_account.lamports,
+                &stake_state,
+                use_lamports_unit,
+                &stake_history,
+            );
             Ok(config.output_format.formatted_string(&state))
         }
         Err(err) => Err(CliError::RpcRequestError(format!(
@@ -1399,13 +1442,15 @@ pub fn process_delegate_stake(
                 "Unable to delegate. Vote account has no root slot".to_string(),
             )),
             Some(root_slot) => {
-                let slot = rpc_client.get_slot()?;
-                if root_slot + solana_sdk::clock::DEFAULT_SLOTS_PER_TURN < slot {
-                    Err(CliError::BadParameter(
-                        format!(
-                        "Unable to delegate. Vote account root slot ({}) is too old, the current slot is {}", root_slot, slot
-                        )
-                    ))
+                let min_root_slot = rpc_client
+                    .get_slot()?
+                    .saturating_sub(DELINQUENT_VALIDATOR_SLOT_DISTANCE);
+                if root_slot < min_root_slot {
+                    Err(CliError::DynamicProgramError(format!(
+                        "Unable to delegate.  Vote account appears delinquent \
+                                 because its current root slot, {}, is less than {}",
+                        root_slot, min_root_slot
+                    )))
                 } else {
                     Ok(())
                 }

cli/src/vote.rs

@@ -161,6 +161,35 @@ impl VoteSubCommands for App<'_, '_> {
                         .help("Authorized withdrawer keypair"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("vote-update-commission")
+                .about("Update the vote account's commission")
+                .arg(
+                    pubkey!(Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .required(true),
+                        "Vote account to update. "),
+                )
+                .arg(
+                    Arg::with_name("commission")
+                        .index(2)
+                        .value_name("PERCENTAGE")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_percentage)
+                        .help("The new commission")
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .index(3)
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer keypair"),
+                )
+        )
         .subcommand(
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
@@ -309,6 +338,33 @@ pub fn parse_vote_update_validator(
     })
 }
 
+pub fn parse_vote_update_commission(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let (authorized_withdrawer, _) = signer_of(matches, "authorized_withdrawer", wallet_manager)?;
+    let commission = value_t_or_exit!(matches, "commission", u8);
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, authorized_withdrawer],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
+    Ok(CliCommandInfo {
+        command: CliCommand::VoteUpdateCommission {
+            vote_account_pubkey,
+            commission,
+        },
+        signers: signer_info.signers,
+    })
+}
+
 pub fn parse_vote_get_account_command(
     matches: &ArgMatches<'_>,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
@@ -521,6 +577,33 @@ pub fn process_vote_update_validator(
     log_instruction_custom_error::<VoteError>(result, &config)
 }
 
+pub fn process_vote_update_commission(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    vote_account_pubkey: &Pubkey,
+    commission: u8,
+) -> ProcessResult {
+    let authorized_withdrawer = config.signers[1];
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let ixs = vec![vote_instruction::update_commission(
+        vote_account_pubkey,
+        &authorized_withdrawer.pubkey(),
+        commission,
+    )];
+
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
+    check_account_for_fee(
+        rpc_client,
+        &config.signers[0].pubkey(),
+        &fee_calculator,
+        &tx.message,
+    )?;
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&tx);
+    log_instruction_custom_error::<VoteError>(result, &config)
+}
+
 fn get_vote_account(
     rpc_client: &RpcClient,
     vote_account_pubkey: &Pubkey,
@@ -842,6 +925,27 @@ mod tests {
             }
         );
 
+        let test_update_commission = test_commands.clone().get_matches_from(vec![
+            "test",
+            "vote-update-commission",
+            &pubkey_string,
+            "42",
+            &keypair_file,
+        ]);
+        assert_eq!(
+            parse_command(&test_update_commission, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::VoteUpdateCommission {
+                    vote_account_pubkey: pubkey,
+                    commission: 42,
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(read_keypair_file(&keypair_file).unwrap()),
+                ],
+            }
+        );
+
         // Test WithdrawFromVoteAccount subcommand
         let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
             "test",

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -31,7 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.1.0"
 jsonrpc-http-server = "14.1.0"
-solana-logger = { path = "../logger", version = "1.2.0" }
+solana-logger = { path = "../logger", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -2,7 +2,7 @@ use crate::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     http_sender::HttpSender,
     mock_sender::{MockSender, Mocks},
-    rpc_config::RpcLargestAccountsConfig,
+    rpc_config::{RpcLargestAccountsConfig, RpcSendTransactionConfig},
     rpc_request::{RpcError, RpcRequest},
     rpc_response::*,
     rpc_sender::RpcSender,
@@ -22,7 +22,6 @@ use solana_sdk::{
     epoch_schedule::EpochSchedule,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     hash::Hash,
-    inflation::Inflation,
     pubkey::Pubkey,
     signature::Signature,
     signers::Signers,
@@ -33,7 +32,6 @@ use solana_transaction_status::{
 };
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
-    error,
     net::SocketAddr,
     thread::sleep,
     time::{Duration, Instant},
@@ -95,10 +93,20 @@ impl RpcClient {
     }
 
     pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
+        self.send_transaction_with_config(transaction, RpcSendTransactionConfig::default())
+    }
+
+    pub fn send_transaction_with_config(
+        &self,
+        transaction: &Transaction,
+        config: RpcSendTransactionConfig,
+    ) -> ClientResult<Signature> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
 
-        let signature_base58_str: String =
-            self.send(RpcRequest::SendTransaction, json!([serialized_encoded]))?;
+        let signature_base58_str: String = self.send(
+            RpcRequest::SendTransaction,
+            json!([serialized_encoded, config]),
+        )?;
 
         let signature = signature_base58_str
             .parse::<Signature>()
@@ -122,7 +130,7 @@ impl RpcClient {
         &self,
         transaction: &Transaction,
         sig_verify: bool,
-    ) -> RpcResult<TransactionStatus> {
+    ) -> RpcResult<RpcSimulateTransactionResult> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
         self.send(
             RpcRequest::SimulateTransaction,
@@ -346,8 +354,12 @@ impl RpcClient {
         })
     }
 
-    pub fn get_inflation(&self) -> ClientResult<Inflation> {
-        self.send(RpcRequest::GetInflation, Value::Null)
+    pub fn get_inflation_governor(&self) -> ClientResult<RpcInflationGovernor> {
+        self.send(RpcRequest::GetInflationGovernor, Value::Null)
+    }
+
+    pub fn get_inflation_rate(&self) -> ClientResult<RpcInflationRate> {
+        self.send(RpcRequest::GetInflationRate, Value::Null)
     }
 
     pub fn get_version(&self) -> ClientResult<RpcVersionInfo> {
@@ -403,96 +415,6 @@ impl RpcClient {
         }
     }
 
-    pub fn send_and_confirm_transactions_with_spinner<T: Signers>(
-        &self,
-        mut transactions: Vec<Transaction>,
-        signer_keys: &T,
-    ) -> Result<(), Box<dyn error::Error>> {
-        let progress_bar = new_spinner_progress_bar();
-        let mut send_retries = 5;
-        loop {
-            let mut status_retries = 15;
-
-            // Send all transactions
-            let mut transactions_signatures = vec![];
-            let num_transactions = transactions.len();
-            for transaction in transactions {
-                if cfg!(not(test)) {
-                    // Delay ~1 tick between write transactions in an attempt to reduce AccountInUse errors
-                    // when all the write transactions modify the same program account (eg, deploying a
-                    // new program)
-                    sleep(Duration::from_millis(1000 / DEFAULT_TICKS_PER_SECOND));
-                }
-
-                let signature = self.send_transaction(&transaction).ok();
-                transactions_signatures.push((transaction, signature));
-
-                progress_bar.set_message(&format!(
-                    "[{}/{}] Transactions sent",
-                    transactions_signatures.len(),
-                    num_transactions
-                ));
-            }
-
-            // Collect statuses for all the transactions, drop those that are confirmed
-            while status_retries > 0 {
-                status_retries -= 1;
-
-                progress_bar.set_message(&format!(
-                    "[{}/{}] Transactions confirmed",
-                    num_transactions - transactions_signatures.len(),
-                    num_transactions
-                ));
-
-                if cfg!(not(test)) {
-                    // Retry twice a second
-                    sleep(Duration::from_millis(500));
-                }
-
-                transactions_signatures = transactions_signatures
-                    .into_iter()
-                    .filter(|(_transaction, signature)| {
-                        if let Some(signature) = signature {
-                            if let Ok(status) = self.get_signature_status(&signature) {
-                                if self
-                                    .get_num_blocks_since_signature_confirmation(&signature)
-                                    .unwrap_or(0)
-                                    > 1
-                                {
-                                    return false;
-                                } else {
-                                    return match status {
-                                        None => true,
-                                        Some(result) => result.is_err(),
-                                    };
-                                }
-                            }
-                        }
-                        true
-                    })
-                    .collect();
-
-                if transactions_signatures.is_empty() {
-                    return Ok(());
-                }
-            }
-
-            if send_retries == 0 {
-                return Err(RpcError::ForUser("Transactions failed".to_string()).into());
-            }
-            send_retries -= 1;
-
-            // Re-sign any failed transactions with a new blockhash and retry
-            let (blockhash, _fee_calculator) =
-                self.get_new_blockhash(&transactions_signatures[0].0.message().recent_blockhash)?;
-            transactions = vec![];
-            for (mut transaction, _) in transactions_signatures.into_iter() {
-                transaction.try_sign(signer_keys, blockhash)?;
-                transactions.push(transaction);
-            }
-        }
-    }
-
     pub fn resign_transaction<T: Signers>(
         &self,
         tx: &mut Transaction,
@@ -614,26 +536,46 @@ impl RpcClient {
     }
 
     pub fn get_recent_blockhash(&self) -> ClientResult<(Hash, FeeCalculator)> {
-        Ok(self
+        let (blockhash, fee_calculator, _last_valid_slot) = self
             .get_recent_blockhash_with_commitment(CommitmentConfig::default())?
-            .value)
+            .value;
+        Ok((blockhash, fee_calculator))
     }
 
     pub fn get_recent_blockhash_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<(Hash, FeeCalculator)> {
-        let Response {
+    ) -> RpcResult<(Hash, FeeCalculator, Slot)> {
+        let (context, blockhash, fee_calculator, last_valid_slot) = if let Ok(Response {
+            context,
+            value:
+                RpcFees {
+                    blockhash,
+                    fee_calculator,
+                    last_valid_slot,
+                },
+        }) =
+            self.send::<Response<RpcFees>>(RpcRequest::GetFees, json!([commitment_config]))
+        {
+            (context, blockhash, fee_calculator, last_valid_slot)
+        } else if let Ok(Response {
             context,
             value:
                 RpcBlockhashFeeCalculator {
                     blockhash,
                     fee_calculator,
                 },
-        } = self.send::<Response<RpcBlockhashFeeCalculator>>(
+        }) = self.send::<Response<RpcBlockhashFeeCalculator>>(
             RpcRequest::GetRecentBlockhash,
             json!([commitment_config]),
-        )?;
+        ) {
+            (context, blockhash, fee_calculator, 0)
+        } else {
+            return Err(ClientError::new_with_request(
+                RpcError::ParseError("RpcBlockhashFeeCalculator or RpcFees".to_string()).into(),
+                RpcRequest::GetRecentBlockhash,
+            ));
+        };
 
         let blockhash = blockhash.parse().map_err(|_| {
             ClientError::new_with_request(
@@ -643,7 +585,7 @@ impl RpcClient {
         })?;
         Ok(Response {
             context,
-            value: (blockhash, fee_calculator),
+            value: (blockhash, fee_calculator, last_valid_slot),
         })
     }
 
@@ -651,12 +593,28 @@ impl RpcClient {
         &self,
         blockhash: &Hash,
     ) -> ClientResult<Option<FeeCalculator>> {
-        let Response { value, .. } = self.send::<Response<Option<RpcFeeCalculator>>>(
+        Ok(self
+            .get_fee_calculator_for_blockhash_with_commitment(
+                blockhash,
+                CommitmentConfig::default(),
+            )?
+            .value)
+    }
+
+    pub fn get_fee_calculator_for_blockhash_with_commitment(
+        &self,
+        blockhash: &Hash,
+        commitment_config: CommitmentConfig,
+    ) -> RpcResult<Option<FeeCalculator>> {
+        let Response { context, value } = self.send::<Response<Option<RpcFeeCalculator>>>(
             RpcRequest::GetFeeCalculatorForBlockhash,
-            json!([blockhash.to_string()]),
+            json!([blockhash.to_string(), commitment_config]),
         )?;
 
-        Ok(value.map(|rf| rf.fee_calculator))
+        Ok(Response {
+            context,
+            value: value.map(|rf| rf.fee_calculator),
+        })
     }
 
     pub fn get_fee_rate_governor(&self) -> RpcResult<FeeRateGovernor> {
@@ -913,6 +871,17 @@ impl RpcClient {
     pub fn send_and_confirm_transaction_with_spinner(
         &self,
         transaction: &Transaction,
+    ) -> ClientResult<Signature> {
+        self.send_and_confirm_transaction_with_spinner_and_config(
+            transaction,
+            RpcSendTransactionConfig::default(),
+        )
+    }
+
+    pub fn send_and_confirm_transaction_with_spinner_and_config(
+        &self,
+        transaction: &Transaction,
+        config: RpcSendTransactionConfig,
     ) -> ClientResult<Signature> {
         let mut confirmations = 0;
 
@@ -928,7 +897,7 @@ impl RpcClient {
             ));
             let mut status_retries = 15;
             let (signature, status) = loop {
-                let signature = self.send_transaction(transaction)?;
+                let signature = self.send_transaction_with_config(transaction, config.clone())?;
 
                 // Get recent commitment in order to count confirmations for successful transactions
                 let status = self

client/src/rpc_config.rs

@@ -1,4 +1,4 @@
-use solana_sdk::commitment_config::CommitmentConfig;
+use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig};
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -6,7 +6,13 @@ pub struct RpcSignatureStatusConfig {
     pub search_transaction_history: bool,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSendTransactionConfig {
+    pub skip_preflight: bool,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSimulateTransactionConfig {
     pub sig_verify: bool,
@@ -26,3 +32,11 @@ pub struct RpcLargestAccountsConfig {
     pub commitment: Option<CommitmentConfig>,
     pub filter: Option<RpcLargestAccountsFilter>,
 }
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationConfig {
+    pub epoch: Option<Epoch>,
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+}

client/src/rpc_request.rs

@@ -16,15 +16,18 @@ pub enum RpcRequest {
     GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
-    GetGenesisHash,
-    GetIdentity,
-    GetInflation,
-    GetLargestAccounts,
-    GetLeaderSchedule,
-    GetProgramAccounts,
-    GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
+    GetFees,
+    GetGenesisHash,
+    GetIdentity,
+    GetInflationGovernor,
+    GetInflationRate,
+    GetLargestAccounts,
+    GetLeaderSchedule,
+    GetMinimumBalanceForRentExemption,
+    GetProgramAccounts,
+    GetRecentBlockhash,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
@@ -37,13 +40,12 @@ pub enum RpcRequest {
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
+    MinimumLedgerSlot,
     RegisterNode,
     RequestAirdrop,
     SendTransaction,
     SimulateTransaction,
     SignVote,
-    GetMinimumBalanceForRentExemption,
-    MinimumLedgerSlot,
 }
 
 impl fmt::Display for RpcRequest {
@@ -61,15 +63,18 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
-            RpcRequest::GetGenesisHash => "getGenesisHash",
-            RpcRequest::GetIdentity => "getIdentity",
-            RpcRequest::GetInflation => "getInflation",
-            RpcRequest::GetLargestAccounts => "getLargestAccounts",
-            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetProgramAccounts => "getProgramAccounts",
-            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
+            RpcRequest::GetFees => "getFees",
+            RpcRequest::GetGenesisHash => "getGenesisHash",
+            RpcRequest::GetIdentity => "getIdentity",
+            RpcRequest::GetInflationGovernor => "getInflationGovernor",
+            RpcRequest::GetInflationRate => "getInflationRate",
+            RpcRequest::GetLargestAccounts => "getLargestAccounts",
+            RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
+            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
+            RpcRequest::GetProgramAccounts => "getProgramAccounts",
+            RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
@@ -82,13 +87,12 @@ impl fmt::Display for RpcRequest {
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",
+            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
             RpcRequest::RegisterNode => "registerNode",
             RpcRequest::RequestAirdrop => "requestAirdrop",
             RpcRequest::SendTransaction => "sendTransaction",
             RpcRequest::SimulateTransaction => "simulateTransaction",
             RpcRequest::SignVote => "signVote",
-            RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
-            RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",
         };
 
         write!(f, "{}", method)
@@ -99,6 +103,9 @@ pub const NUM_LARGEST_ACCOUNTS: usize = 20;
 pub const MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS: usize = 256;
 pub const MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS_SLOT_RANGE: u64 = 10_000;
 
+// Validators that are this number of slots behind are considered delinquent
+pub const DELINQUENT_VALIDATOR_SLOT_DISTANCE: u64 = 128;
+
 impl RpcRequest {
     pub(crate) fn build_request_json(self, id: u64, params: Value) -> Value {
         let jsonrpc = "2.0";
@@ -144,10 +151,6 @@ mod tests {
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getEpochInfo");
 
-        let test_request = RpcRequest::GetInflation;
-        let request = test_request.build_request_json(1, Value::Null);
-        assert_eq!(request["method"], "getInflation");
-
         let test_request = RpcRequest::GetRecentBlockhash;
         let request = test_request.build_request_json(1, Value::Null);
         assert_eq!(request["method"], "getRecentBlockhash");

client/src/rpc_response.rs

@@ -3,6 +3,7 @@ use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
+    inflation::Inflation,
     pubkey::Pubkey,
     transaction::{Result, TransactionError},
 };
@@ -35,6 +36,14 @@ pub struct RpcBlockhashFeeCalculator {
     pub fee_calculator: FeeCalculator,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcFees {
+    pub blockhash: String,
+    pub fee_calculator: FeeCalculator,
+    pub last_valid_slot: Slot,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcFeeCalculator {
@@ -47,6 +56,37 @@ pub struct RpcFeeRateGovernor {
     pub fee_rate_governor: FeeRateGovernor,
 }
 
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationGovernor {
+    pub initial: f64,
+    pub terminal: f64,
+    pub taper: f64,
+    pub foundation: f64,
+    pub foundation_term: f64,
+}
+
+impl From<Inflation> for RpcInflationGovernor {
+    fn from(inflation: Inflation) -> Self {
+        Self {
+            initial: inflation.initial,
+            terminal: inflation.terminal,
+            taper: inflation.taper,
+            foundation: inflation.foundation,
+            foundation_term: inflation.foundation_term,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcInflationRate {
+    pub total: f64,
+    pub validator: f64,
+    pub foundation: f64,
+    pub epoch: Epoch,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcKeyedAccount {
@@ -171,6 +211,13 @@ pub struct RpcSignatureConfirmation {
     pub status: Result<()>,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionResult {
+    pub err: Option<TransactionError>,
+    pub logs: Option<Vec<String>>,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcStorageTurn {

client/src/thin_client.rs

@@ -9,7 +9,7 @@ use log::*;
 use solana_sdk::{
     account::Account,
     client::{AsyncClient, Client, SyncClient},
-    clock::MAX_PROCESSING_AGE,
+    clock::{Slot, MAX_PROCESSING_AGE},
     commitment_config::CommitmentConfig,
     epoch_info::EpochInfo,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
@@ -427,13 +427,15 @@ impl SyncClient for ThinClient {
     }
 
     fn get_recent_blockhash(&self) -> TransportResult<(Hash, FeeCalculator)> {
-        self.get_recent_blockhash_with_commitment(CommitmentConfig::default())
+        let (blockhash, fee_calculator, _last_valid_slot) =
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::default())?;
+        Ok((blockhash, fee_calculator))
     }
 
     fn get_recent_blockhash_with_commitment(
         &self,
         commitment_config: CommitmentConfig,
-    ) -> TransportResult<(Hash, FeeCalculator)> {
+    ) -> TransportResult<(Hash, FeeCalculator, Slot)> {
         let index = self.optimizer.experiment();
         let now = Instant::now();
         let recent_blockhash =
@@ -441,7 +443,7 @@ impl SyncClient for ThinClient {
         match recent_blockhash {
             Ok(Response { value, .. }) => {
                 self.optimizer.report(index, duration_as_ms(&now.elapsed()));
-                Ok(value)
+                Ok((value.0, value.1, value.2))
             }
             Err(e) => {
                 self.optimizer.report(index, std::u64::MAX);

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -21,6 +21,7 @@ byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
 crossbeam-channel = "0.4"
+ed25519-dalek = "=1.0.0-pre.3"
 fs_extra = "1.1.0"
 flate2 = "1.0"
 indexmap = "1.3"
@@ -41,35 +42,35 @@ regex = "1.3.7"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.0" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-ed25519-dalek = "=1.0.0-pre.3"
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-perf = { path = "../perf", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-streamer = { path = "../streamer", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.2.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.1" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.1" }
+solana-faucet = { path = "../faucet", version = "1.2.1" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
+solana-measure = { path = "../measure", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-perf = { path = "../perf", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-streamer = { path = "../streamer", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.1" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.2.1" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.1" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/cluster_info.rs

@@ -3,6 +3,7 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
+use solana_core::broadcast_stage::broadcast_metrics::TransmitShredsStats;
 use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
@@ -47,7 +48,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
             &peers_and_stakes,
             &peers,
             &last_datapoint,
-            &mut 0,
+            &mut TransmitShredsStats::default(),
         )
         .unwrap();
     });

core/src/banking_stage.rs

@@ -514,7 +514,7 @@ impl BankingStage {
             vec![]
         };
         let (mut loaded_accounts, results, mut retryable_txs, tx_count, signature_count) =
-            bank.load_and_execute_transactions(batch, MAX_PROCESSING_AGE);
+            bank.load_and_execute_transactions(batch, MAX_PROCESSING_AGE, None);
         load_execute_time.stop();
 
         let freeze_lock = bank.freeze_lock();

core/src/broadcast_stage.rs

@@ -35,7 +35,7 @@ use std::{
 };
 
 mod broadcast_fake_shreds_run;
-pub(crate) mod broadcast_metrics;
+pub mod broadcast_metrics;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
@@ -374,13 +374,14 @@ pub fn broadcast_shreds(
     peers_and_stakes: &[(u64, usize)],
     peers: &[ContactInfo],
     last_datapoint_submit: &Arc<AtomicU64>,
-    send_mmsg_total: &mut u64,
+    transmit_stats: &mut TransmitShredsStats,
 ) -> Result<()> {
     let broadcast_len = peers_and_stakes.len();
     if broadcast_len == 0 {
         update_peer_stats(1, 1, last_datapoint_submit);
         return Ok(());
     }
+    let mut shred_select = Measure::start("shred_select");
     let packets: Vec<_> = shreds
         .iter()
         .map(|shred| {
@@ -389,6 +390,8 @@ pub fn broadcast_shreds(
             (&shred.payload, &peers[broadcast_index].tvu)
         })
         .collect();
+    shred_select.stop();
+    transmit_stats.shred_select += shred_select.as_us();
 
     let mut sent = 0;
     let mut send_mmsg_time = Measure::start("send_mmsg");
@@ -401,7 +404,7 @@ pub fn broadcast_shreds(
         }
     }
     send_mmsg_time.stop();
-    *send_mmsg_total += send_mmsg_time.as_us();
+    transmit_stats.send_mmsg_elapsed += send_mmsg_time.as_us();
 
     let num_live_peers = num_live_peers(&peers);
     update_peer_stats(

core/src/broadcast_stage/broadcast_metrics.rs

@@ -29,11 +29,12 @@ impl ProcessShredsStats {
 }
 
 #[derive(Default, Clone)]
-pub(crate) struct TransmitShredsStats {
-    pub(crate) transmit_elapsed: u64,
-    pub(crate) send_mmsg_elapsed: u64,
-    pub(crate) get_peers_elapsed: u64,
-    pub(crate) num_shreds: usize,
+pub struct TransmitShredsStats {
+    pub transmit_elapsed: u64,
+    pub send_mmsg_elapsed: u64,
+    pub get_peers_elapsed: u64,
+    pub shred_select: u64,
+    pub num_shreds: usize,
 }
 
 impl BroadcastStats for TransmitShredsStats {
@@ -42,6 +43,7 @@ impl BroadcastStats for TransmitShredsStats {
         self.send_mmsg_elapsed += new_stats.send_mmsg_elapsed;
         self.get_peers_elapsed += new_stats.get_peers_elapsed;
         self.num_shreds += new_stats.num_shreds;
+        self.shred_select += new_stats.shred_select;
     }
     fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
         datapoint_info!(
@@ -58,6 +60,7 @@ impl BroadcastStats for TransmitShredsStats {
             ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
             ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
             ("num_shreds", self.num_shreds as i64, i64),
+            ("shred_select", self.shred_select as i64, i64),
         );
     }
 }
@@ -176,15 +179,16 @@ mod test {
     }
 
     #[test]
-    fn test_update() {
+    fn test_update_broadcast() {
         let start = Instant::now();
         let mut slot_broadcast_stats = SlotBroadcastStats::default();
         slot_broadcast_stats.update(
             &TransmitShredsStats {
                 transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                get_peers_elapsed: 2,
+                send_mmsg_elapsed: 3,
+                shred_select: 4,
+                num_shreds: 5,
             },
             &Some(BroadcastShredBatchInfo {
                 slot: 0,
@@ -198,16 +202,18 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         slot_broadcast_stats.update(
             &TransmitShredsStats {
-                transmit_elapsed: 1,
-                get_peers_elapsed: 1,
-                send_mmsg_elapsed: 1,
-                num_shreds: 1,
+                transmit_elapsed: 7,
+                get_peers_elapsed: 8,
+                send_mmsg_elapsed: 9,
+                shred_select: 10,
+                num_shreds: 11,
             },
             &None,
         );
@@ -217,9 +223,10 @@ mod test {
         assert_eq!(slot_0_stats.num_batches, 1);
         assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
         assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
-        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 3);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.shred_select, 4);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 5);
 
         // If another batch is given, then total number of batches == num_expected_batches == 2,
         // so the batch should be purged from the HashMap
@@ -228,6 +235,7 @@ mod test {
                 transmit_elapsed: 1,
                 get_peers_elapsed: 1,
                 send_mmsg_elapsed: 1,
+                shred_select: 1,
                 num_shreds: 1,
             },
             &Some(BroadcastShredBatchInfo {

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -137,14 +137,13 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         // Broadcast data
         let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
 
-        let mut send_mmsg_total = 0;
         broadcast_shreds(
             sock,
             &shreds,
             &peers_and_stakes,
             &peers,
             &Arc::new(AtomicU64::new(0)),
-            &mut send_mmsg_total,
+            &mut TransmitShredsStats::default(),
         )?;
 
         Ok(())

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -9,6 +9,7 @@ use solana_ledger::{
 };
 use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
 use std::collections::HashMap;
+use std::sync::RwLock;
 use std::time::Duration;
 
 #[derive(Clone)]
@@ -23,6 +24,14 @@ pub struct StandardBroadcastRun {
     shred_version: u16,
     last_datapoint_submit: Arc<AtomicU64>,
     num_batches: usize,
+    broadcast_peer_cache: Arc<RwLock<BroadcastPeerCache>>,
+    last_peer_update: Arc<AtomicU64>,
+}
+
+#[derive(Default)]
+struct BroadcastPeerCache {
+    peers: Vec<ContactInfo>,
+    peers_and_stakes: Vec<(u64, usize)>,
 }
 
 impl StandardBroadcastRun {
@@ -38,6 +47,8 @@ impl StandardBroadcastRun {
             shred_version,
             last_datapoint_submit: Arc::new(AtomicU64::new(0)),
             num_batches: 0,
+            broadcast_peer_cache: Arc::new(RwLock::new(BroadcastPeerCache::default())),
+            last_peer_update: Arc::new(AtomicU64::new(0)),
         }
     }
 
@@ -293,33 +304,46 @@ impl StandardBroadcastRun {
         shreds: Arc<Vec<Shred>>,
         broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
     ) -> Result<()> {
+        const BROADCAST_PEER_UPDATE_INTERVAL_MS: u64 = 1000;
         trace!("Broadcasting {:?} shreds", shreds.len());
         // Get the list of peers to broadcast to
-        let get_peers_start = Instant::now();
-        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
-        let get_peers_elapsed = get_peers_start.elapsed();
+        let mut get_peers_time = Measure::start("broadcast::get_peers");
+        let now = timestamp();
+        let last = self.last_peer_update.load(Ordering::Relaxed);
+        if now - last > BROADCAST_PEER_UPDATE_INTERVAL_MS
+            && self
+                .last_peer_update
+                .compare_and_swap(now, last, Ordering::Relaxed)
+                == last
+        {
+            let mut w_broadcast_peer_cache = self.broadcast_peer_cache.write().unwrap();
+            let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+            w_broadcast_peer_cache.peers = peers;
+            w_broadcast_peer_cache.peers_and_stakes = peers_and_stakes;
+        }
+        get_peers_time.stop();
+        let r_broadcast_peer_cache = self.broadcast_peer_cache.read().unwrap();
 
+        let mut transmit_stats = TransmitShredsStats::default();
         // Broadcast the shreds
-        let transmit_start = Instant::now();
-        let mut send_mmsg_total = 0;
+        let mut transmit_time = Measure::start("broadcast_shreds");
         broadcast_shreds(
             sock,
             &shreds,
-            &peers_and_stakes,
-            &peers,
+            &r_broadcast_peer_cache.peers_and_stakes,
+            &r_broadcast_peer_cache.peers,
             &self.last_datapoint_submit,
-            &mut send_mmsg_total,
+            &mut transmit_stats,
         )?;
-        let transmit_elapsed = transmit_start.elapsed();
-        let new_transmit_shreds_stats = TransmitShredsStats {
-            transmit_elapsed: duration_as_us(&transmit_elapsed),
-            get_peers_elapsed: duration_as_us(&get_peers_elapsed),
-            send_mmsg_elapsed: send_mmsg_total,
-            num_shreds: shreds.len(),
-        };
+        drop(r_broadcast_peer_cache);
+        transmit_time.stop();
+
+        transmit_stats.transmit_elapsed = transmit_time.as_us();
+        transmit_stats.get_peers_elapsed = get_peers_time.as_us();
+        transmit_stats.num_shreds = shreds.len();
 
         // Process metrics
-        self.update_transmit_metrics(&new_transmit_shreds_stats, &broadcast_shred_batch_info);
+        self.update_transmit_metrics(&transmit_stats, &broadcast_shred_batch_info);
         Ok(())
     }
 

core/src/crds.rs

@@ -93,6 +93,24 @@ impl Crds {
     pub fn new_versioned(&self, local_timestamp: u64, value: CrdsValue) -> VersionedCrdsValue {
         VersionedCrdsValue::new(local_timestamp, value)
     }
+    pub fn would_insert(
+        &self,
+        value: CrdsValue,
+        local_timestamp: u64,
+    ) -> Option<VersionedCrdsValue> {
+        let new_value = self.new_versioned(local_timestamp, value);
+        let label = new_value.value.label();
+        let would_insert = self
+            .table
+            .get(&label)
+            .map(|current| new_value > *current)
+            .unwrap_or(true);
+        if would_insert {
+            Some(new_value)
+        } else {
+            None
+        }
+    }
     /// insert the new value, returns the old value if insert succeeds
     pub fn insert_versioned(
         &mut self,

core/src/crds_gossip.rs

@@ -6,7 +6,7 @@
 use crate::{
     crds::{Crds, VersionedCrdsValue},
     crds_gossip_error::CrdsGossipError,
-    crds_gossip_pull::{CrdsFilter, CrdsGossipPull},
+    crds_gossip_pull::{CrdsFilter, CrdsGossipPull, ProcessPullStats},
     crds_gossip_push::{CrdsGossipPush, CRDS_GOSSIP_NUM_ACTIVE},
     crds_value::{CrdsValue, CrdsValueLabel},
 };
@@ -158,24 +158,47 @@ impl CrdsGossip {
         self.pull.mark_pull_request_creation_time(from, now)
     }
     /// process a pull request and create a response
-    pub fn process_pull_requests(
-        &mut self,
-        filters: Vec<(CrdsValue, CrdsFilter)>,
-        now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
+    pub fn process_pull_requests(&mut self, filters: Vec<(CrdsValue, CrdsFilter)>, now: u64) {
         self.pull
-            .process_pull_requests(&mut self.crds, filters, now)
+            .process_pull_requests(&mut self.crds, filters, now);
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        from: &Pubkey,
+
+    pub fn generate_pull_responses(
+        &self,
+        filters: &[(CrdsValue, CrdsFilter)],
+        now: Option<u64>,
+    ) -> Vec<Vec<CrdsValue>> {
+        self.pull.generate_pull_responses(&self.crds, filters, now)
+    }
+
+    pub fn filter_pull_responses(
+        &self,
         timeouts: &HashMap<Pubkey, u64>,
         response: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
+        process_pull_stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
         self.pull
-            .process_pull_response(&mut self.crds, from, timeouts, response, now)
+            .filter_pull_responses(&self.crds, timeouts, response, now, process_pull_stats)
+    }
+
+    /// process a pull response
+    pub fn process_pull_responses(
+        &mut self,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        process_pull_stats: &mut ProcessPullStats,
+    ) {
+        self.pull.process_pull_responses(
+            &mut self.crds,
+            from,
+            responses,
+            responses_expired_timeout,
+            now,
+            process_pull_stats,
+        )
     }
 
     pub fn make_timeouts_test(&self) -> HashMap<Pubkey, u64> {

core/src/crds_gossip_pull.rs

@@ -10,7 +10,7 @@
 //! of false positives.
 
 use crate::contact_info::ContactInfo;
-use crate::crds::Crds;
+use crate::crds::{Crds, VersionedCrdsValue};
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
@@ -20,8 +20,8 @@ use solana_runtime::bloom::Bloom;
 use solana_sdk::hash::Hash;
 use solana_sdk::pubkey::Pubkey;
 use std::cmp;
-use std::collections::HashMap;
 use std::collections::VecDeque;
+use std::collections::{HashMap, HashSet};
 
 pub const CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS: u64 = 15000;
 // The maximum age of a value received over pull responses
@@ -100,6 +100,9 @@ impl CrdsFilter {
         accum
     }
     pub fn test_mask(&self, item: &Hash) -> bool {
+        if self.mask_bits == 0 {
+            return true;
+        }
         // only consider the highest mask_bits bits from the hash and set the rest to 1.
         let ones = (!0u64).checked_shr(self.mask_bits).unwrap_or(!0u64);
         let bits = Self::hash_as_u64(item) | ones;
@@ -118,6 +121,14 @@ impl CrdsFilter {
     }
 }
 
+#[derive(Default)]
+pub struct ProcessPullStats {
+    pub success: usize,
+    pub failed_insert: usize,
+    pub failed_timeout: usize,
+    pub timeout_count: usize,
+}
+
 #[derive(Clone)]
 pub struct CrdsGossipPull {
     /// timestamp of last request
@@ -204,14 +215,13 @@ impl CrdsGossipPull {
         self.purged_values.push_back((hash, timestamp))
     }
 
-    /// process a pull request and create a response
+    /// process a pull request
     pub fn process_pull_requests(
         &mut self,
         crds: &mut Crds,
         requests: Vec<(CrdsValue, CrdsFilter)>,
         now: u64,
-    ) -> Vec<Vec<CrdsValue>> {
-        let rv = self.filter_crds_values(crds, &requests);
+    ) {
         requests.into_iter().for_each(|(caller, _)| {
             let key = caller.label().pubkey();
             let old = crds.insert(caller, now);
@@ -221,19 +231,34 @@ impl CrdsGossipPull {
             }
             crds.update_record_timestamp(&key, now);
         });
-        rv
     }
-    /// process a pull response
-    pub fn process_pull_response(
-        &mut self,
-        crds: &mut Crds,
-        from: &Pubkey,
+
+    /// Create gossip responses to pull requests
+    pub fn generate_pull_responses(
+        &self,
+        crds: &Crds,
+        requests: &[(CrdsValue, CrdsFilter)],
+        now: Option<u64>,
+    ) -> Vec<Vec<CrdsValue>> {
+        Self::filter_crds_values(crds, requests, now)
+    }
+
+    // Checks if responses should be inserted and
+    // returns those responses converted to VersionedCrdsValue
+    // Separated in two vecs as:
+    //  .0 => responses that update the owner timestamp
+    //  .1 => responses that do not update the owner timestamp
+    pub fn filter_pull_responses(
+        &self,
+        crds: &Crds,
         timeouts: &HashMap<Pubkey, u64>,
-        response: Vec<CrdsValue>,
+        responses: Vec<CrdsValue>,
         now: u64,
-    ) -> usize {
-        let mut failed = 0;
-        for r in response {
+        stats: &mut ProcessPullStats,
+    ) -> (Vec<VersionedCrdsValue>, Vec<VersionedCrdsValue>) {
+        let mut versioned = vec![];
+        let mut versioned_expired_timestamp = vec![];
+        for r in responses {
             let owner = r.label().pubkey();
             // Check if the crds value is older than the msg_timeout
             if now
@@ -252,11 +277,8 @@ impl CrdsGossipPull {
                         if now > r.wallclock().checked_add(timeout).unwrap_or_else(|| 0)
                             || now + timeout < r.wallclock()
                         {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         }
                     }
@@ -264,32 +286,62 @@ impl CrdsGossipPull {
                         // Before discarding this value, check if a ContactInfo for the owner
                         // exists in the table. If it doesn't, that implies that this value can be discarded
                         if crds.lookup(&CrdsValueLabel::ContactInfo(owner)).is_none() {
-                            inc_new_counter_warn!(
-                                "cluster_info-gossip_pull_response_value_timeout",
-                                1
-                            );
-                            failed += 1;
+                            stats.timeout_count += 1;
+                            stats.failed_timeout += 1;
                             continue;
                         } else {
                             // Silently insert this old value without bumping record timestamps
-                            failed += crds.insert(r, now).is_err() as usize;
+                            match crds.would_insert(r, now) {
+                                Some(resp) => versioned_expired_timestamp.push(resp),
+                                None => stats.failed_insert += 1,
+                            }
                             continue;
                         }
                     }
                 }
             }
-            let old = crds.insert(r, now);
-            failed += old.is_err() as usize;
+            match crds.would_insert(r, now) {
+                Some(resp) => versioned.push(resp),
+                None => stats.failed_insert += 1,
+            }
+        }
+        (versioned, versioned_expired_timestamp)
+    }
+
+    /// process a vec of pull responses
+    pub fn process_pull_responses(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        responses: Vec<VersionedCrdsValue>,
+        responses_expired_timeout: Vec<VersionedCrdsValue>,
+        now: u64,
+        stats: &mut ProcessPullStats,
+    ) {
+        let mut owners = HashSet::new();
+        for r in responses_expired_timeout {
+            stats.failed_insert += crds.insert_versioned(r).is_err() as usize;
+        }
+        for r in responses {
+            let owner = r.value.label().pubkey();
+            let old = crds.insert_versioned(r);
+            if old.is_err() {
+                stats.failed_insert += 1;
+            } else {
+                stats.success += 1;
+            }
             old.ok().map(|opt| {
-                crds.update_record_timestamp(&owner, now);
+                owners.insert(owner);
                 opt.map(|val| {
                     self.purged_values
                         .push_back((val.value_hash, val.local_timestamp))
                 })
             });
         }
-        crds.update_record_timestamp(from, now);
-        failed
+        owners.insert(*from);
+        for owner in owners {
+            crds.update_record_timestamp(&owner, now);
+        }
     }
     // build a set of filters of the current crds table
     // num_filters - used to increase the likelyhood of a value in crds being added to some filter
@@ -311,12 +363,17 @@ impl CrdsGossipPull {
     }
     /// filter values that fail the bloom filter up to max_bytes
     fn filter_crds_values(
-        &self,
         crds: &Crds,
         filters: &[(CrdsValue, CrdsFilter)],
+        now: Option<u64>,
     ) -> Vec<Vec<CrdsValue>> {
         let mut ret = vec![vec![]; filters.len()];
+        let now = now.unwrap_or(u64::MAX);
         for v in crds.table.values() {
+            //skip messages that are newer than now
+            if v.insert_timestamp > now {
+                continue;
+            }
             filters.iter().enumerate().for_each(|(i, (_, filter))| {
                 if !filter.contains(&v.value_hash) {
                     ret[i].push(v.value.clone());
@@ -379,6 +436,34 @@ impl CrdsGossipPull {
             .count();
         self.purged_values.drain(..cnt);
     }
+
+    /// For legacy tests
+    #[cfg(test)]
+    pub fn process_pull_response(
+        &mut self,
+        crds: &mut Crds,
+        from: &Pubkey,
+        timeouts: &HashMap<Pubkey, u64>,
+        response: Vec<CrdsValue>,
+        now: u64,
+    ) -> (usize, usize, usize) {
+        let mut stats = ProcessPullStats::default();
+        let (versioned, versioned_expired_timeout) =
+            self.filter_pull_responses(crds, timeouts, response, now, &mut stats);
+        self.process_pull_responses(
+            crds,
+            from,
+            versioned,
+            versioned_expired_timeout,
+            now,
+            &mut stats,
+        );
+        (
+            stats.failed_timeout + stats.failed_insert,
+            stats.timeout_count,
+            stats.success,
+        )
+    }
 }
 #[cfg(test)]
 mod test {
@@ -578,8 +663,9 @@ mod test {
         let mut dest_crds = Crds::default();
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
-        let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.process_pull_requests(&mut dest_crds, filters, 1);
+        let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+        let rsp = dest.generate_pull_responses(&dest_crds, &filters, None);
+        dest.process_pull_requests(&mut dest_crds, filters, 1);
         assert!(rsp.iter().all(|rsp| rsp.is_empty()));
         assert!(dest_crds.lookup(&caller.label()).is_some());
         assert_eq!(
@@ -648,8 +734,9 @@ mod test {
                 PACKET_DATA_SIZE,
             );
             let (_, filters, caller) = req.unwrap();
-            let filters = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.process_pull_requests(&mut dest_crds, filters, 0);
+            let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters, None);
+            dest.process_pull_requests(&mut dest_crds, filters, 0);
             // if there is a false positive this is empty
             // prob should be around 0.1 per iteration
             if rsp.is_empty() {
@@ -660,13 +747,15 @@ mod test {
                 continue;
             }
             assert_eq!(rsp.len(), 1);
-            let failed = node.process_pull_response(
-                &mut node_crds,
-                &node_pubkey,
-                &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
-                rsp.pop().unwrap(),
-                1,
-            );
+            let failed = node
+                .process_pull_response(
+                    &mut node_crds,
+                    &node_pubkey,
+                    &node.make_timeouts_def(&node_pubkey, &HashMap::new(), 0, 1),
+                    rsp.pop().unwrap(),
+                    1,
+                )
+                .0;
             assert_eq!(failed, 0);
             assert_eq!(
                 node_crds
@@ -688,6 +777,37 @@ mod test {
         }
         assert!(done);
     }
+    #[test]
+    fn test_filter_crds_values() {
+        let mut node_crds = Crds::default();
+        let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            0,
+        )));
+
+        let caller = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            0,
+        )));
+
+        let node_label = entry.label();
+        let node_pubkey = node_label.pubkey();
+        node_crds.insert(entry, 1).unwrap();
+        let mut filter = CrdsFilter::new_rand(1, 128);
+        filter.mask_bits = 0;
+        let requests = [(caller, filter)];
+
+        let rsp = CrdsGossipPull::filter_crds_values(&node_crds, &requests, None);
+        assert_eq!(rsp[0][0].pubkey(), node_pubkey);
+
+        //skip 1 since its newer than 0
+        let rsp = CrdsGossipPull::filter_crds_values(&node_crds, &requests, Some(0));
+        assert!(rsp[0].is_empty());
+
+        let rsp = CrdsGossipPull::filter_crds_values(&node_crds, &requests, Some(1));
+        assert_eq!(rsp[0][0].pubkey(), node_pubkey);
+    }
+
     #[test]
     fn test_gossip_purge() {
         let mut node_crds = Crds::default();
@@ -751,6 +871,15 @@ mod test {
         let h: Hash = hash(h.as_ref());
         assert!(!filter.contains(&h));
     }
+
+    #[test]
+    fn test_crds_filter_test_mask_default() {
+        let filter = CrdsFilter::default();
+        assert_eq!(filter.mask_bits, 0);
+        let h: Hash = Hash::default();
+        assert_eq!(filter.test_mask(&h), true);
+    }
+
     #[test]
     fn test_crds_filter_add_mask() {
         let mut filter = CrdsFilter::new_rand(1000, 10);
@@ -827,7 +956,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone()],
                 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -843,7 +973,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry.clone(), unstaked_peer_entry],
                 node.msg_timeout + 100,
-            ),
+            )
+            .0,
             2
         );
 
@@ -856,7 +987,8 @@ mod test {
                 &timeouts,
                 vec![peer_entry],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -872,7 +1004,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote.clone()],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             0
         );
 
@@ -885,7 +1018,8 @@ mod test {
                 &timeouts,
                 vec![peer_vote],
                 node.msg_timeout + 1,
-            ),
+            )
+            .0,
             1
         );
     }

core/src/ledger_cleanup_service.rs

@@ -1,9 +1,9 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
-use solana_ledger::blockstore::Blockstore;
+use solana_ledger::blockstore::{Blockstore, PurgeType};
 use solana_ledger::blockstore_db::Result as BlockstoreResult;
 use solana_measure::measure::Measure;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Slot, DEFAULT_TICKS_PER_SLOT, TICKS_PER_DAY};
 use std::string::ToString;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
@@ -32,6 +32,10 @@ pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
 // Delay between purges to cooperate with other blockstore users
 pub const DEFAULT_DELAY_BETWEEN_PURGES: Duration = Duration::from_millis(500);
 
+// Compacting at a slower interval than purging helps keep IOPS down.
+// Once a day should be ample
+const DEFAULT_COMPACTION_SLOT_INTERVAL: u64 = TICKS_PER_DAY / DEFAULT_TICKS_PER_SLOT;
+
 pub struct LedgerCleanupService {
     t_cleanup: JoinHandle<()>,
 }
@@ -49,6 +53,8 @@ impl LedgerCleanupService {
         );
         let exit = exit.clone();
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
+
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -62,6 +68,8 @@ impl LedgerCleanupService {
                     &mut last_purge_slot,
                     DEFAULT_PURGE_SLOT_INTERVAL,
                     Some(DEFAULT_DELAY_BETWEEN_PURGES),
+                    &mut last_compaction_slot,
+                    DEFAULT_COMPACTION_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -116,7 +124,7 @@ impl LedgerCleanupService {
             }
         }
 
-        (true, lowest_cleanup_slot, first_slot, total_shreds)
+        (true, first_slot, lowest_cleanup_slot, total_shreds)
     }
 
     fn receive_new_roots(new_root_receiver: &Receiver<Slot>) -> Result<Slot, RecvTimeoutError> {
@@ -135,6 +143,8 @@ impl LedgerCleanupService {
         last_purge_slot: &mut u64,
         purge_interval: u64,
         delay_between_purges: Option<Duration>,
+        last_compaction_slot: &mut u64,
+        compaction_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
         let root = Self::receive_new_roots(new_root_receiver)?;
         if root - *last_purge_slot <= purge_interval {
@@ -143,19 +153,20 @@ impl LedgerCleanupService {
 
         let disk_utilization_pre = blockstore.storage_size();
         info!(
-            "purge: last_root={}, last_purge_slot={}, purge_interval={}, disk_utilization={:?}",
-            root, last_purge_slot, purge_interval, disk_utilization_pre
+            "purge: last_root={}, last_purge_slot={}, purge_interval={}, last_compaction_slot={}, disk_utilization={:?}",
+            root, last_purge_slot, purge_interval, last_compaction_slot, disk_utilization_pre
         );
         *last_purge_slot = root;
 
-        let (slots_to_clean, lowest_cleanup_slot, first_slot, total_shreds) =
+        let (slots_to_clean, purge_first_slot, lowest_cleanup_slot, total_shreds) =
             Self::find_slots_to_clean(&blockstore, root, max_ledger_shreds);
 
         if slots_to_clean {
-            info!(
-                "purging data from slots {} to {}",
-                first_slot, lowest_cleanup_slot
-            );
+            let mut compact_first_slot = std::u64::MAX;
+            if lowest_cleanup_slot.saturating_sub(*last_compaction_slot) > compaction_interval {
+                compact_first_slot = *last_compaction_slot;
+                *last_compaction_slot = lowest_cleanup_slot;
+            }
 
             let purge_complete = Arc::new(AtomicBool::new(false));
             let blockstore = blockstore.clone();
@@ -167,14 +178,37 @@ impl LedgerCleanupService {
                     *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot;
                     slot_update_time.stop();
 
+                    info!(
+                        "purging data from slots {} to {}",
+                        purge_first_slot, lowest_cleanup_slot
+                    );
+
                     let mut purge_time = Measure::start("purge_slots_with_delay");
                     blockstore.purge_slots_with_delay(
-                        first_slot,
+                        purge_first_slot,
                         lowest_cleanup_slot,
                         delay_between_purges,
+                        PurgeType::PrimaryIndex,
                     );
                     purge_time.stop();
                     info!("{}", purge_time);
+
+                    if compact_first_slot < lowest_cleanup_slot {
+                        info!(
+                            "compacting data from slots {} to {}",
+                            compact_first_slot, lowest_cleanup_slot
+                        );
+                        if let Err(err) =
+                            blockstore.compact_storage(compact_first_slot, lowest_cleanup_slot)
+                        {
+                            // This error is not fatal and indicates an internal error?
+                            error!(
+                                "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
+                                err, compact_first_slot, lowest_cleanup_slot
+                            );
+                        }
+                    }
+
                     purge_complete1.store(true, Ordering::Relaxed);
                 })
                 .unwrap();
@@ -233,6 +267,7 @@ mod tests {
 
         //send a signal to kill all but 5 shreds, which will be in the newest slots
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         sender.send(50).unwrap();
         LedgerCleanupService::cleanup_ledger(
             &receiver,
@@ -241,6 +276,8 @@ mod tests {
             &mut last_purge_slot,
             10,
             None,
+            &mut last_compaction_slot,
+            10,
         )
         .unwrap();
 
@@ -272,6 +309,7 @@ mod tests {
         info!("{}", first_insert);
 
         let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         let mut slot = initial_slots;
         let mut num_slots = 6;
         for _ in 0..5 {
@@ -296,6 +334,8 @@ mod tests {
                 &mut last_purge_slot,
                 10,
                 None,
+                &mut last_compaction_slot,
+                10,
             )
             .unwrap();
             time.stop();

core/src/lib.rs

@@ -43,10 +43,12 @@ pub mod retransmit_stage;
 pub mod rewards_recorder_service;
 pub mod rpc;
 pub mod rpc_error;
+pub mod rpc_health;
 pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
+pub mod send_transaction_service;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;

core/src/poh_recorder.rs

@@ -354,7 +354,7 @@ impl PohRecorder {
     pub fn tick(&mut self) {
         let now = Instant::now();
         let poh_entry = self.poh.lock().unwrap().tick();
-        inc_new_counter_warn!(
+        inc_new_counter_info!(
             "poh_recorder-tick_lock_contention",
             timing::duration_as_us(&now.elapsed()) as usize
         );
@@ -364,7 +364,7 @@ impl PohRecorder {
             trace!("tick_height {}", self.tick_height);
 
             if self.leader_first_tick_height.is_none() {
-                inc_new_counter_warn!(
+                inc_new_counter_info!(
                     "poh_recorder-tick_overhead",
                     timing::duration_as_us(&now.elapsed()) as usize
                 );
@@ -380,7 +380,7 @@ impl PohRecorder {
             self.tick_cache.push((entry, self.tick_height));
             let _ = self.flush_cache(true);
         }
-        inc_new_counter_warn!(
+        inc_new_counter_info!(
             "poh_recorder-tick_overhead",
             timing::duration_as_us(&now.elapsed()) as usize
         );

core/src/poh_service.rs

@@ -89,18 +89,22 @@ impl PohService {
         let poh = poh_recorder.lock().unwrap().poh.clone();
         let mut now = Instant::now();
         let mut num_ticks = 0;
+        let mut num_hashes = 0;
         loop {
+            num_hashes += NUM_HASHES_PER_BATCH;
             if poh.lock().unwrap().hash(NUM_HASHES_PER_BATCH) {
                 // Lock PohRecorder only for the final hash...
                 poh_recorder.lock().unwrap().tick();
                 num_ticks += 1;
                 if num_ticks >= DEFAULT_TICKS_PER_SLOT * 2 {
-                    datapoint_debug!(
+                    datapoint_info!(
                         "poh-service",
                         ("ticks", num_ticks as i64, i64),
+                        ("hashes", num_hashes as i64, i64),
                         ("elapsed_ms", now.elapsed().as_millis() as i64, i64),
                     );
                     num_ticks = 0;
+                    num_hashes = 0;
                     now = Instant::now();
                 }
                 if poh_exit.load(Ordering::Relaxed) {

core/src/retransmit_stage.rs

@@ -3,6 +3,7 @@
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_slots::ClusterSlots,
+    contact_info::ContactInfo,
     repair_service::DuplicateSlotsResetSender,
     repair_service::RepairInfo,
     result::{Error, Result},
@@ -18,8 +19,9 @@ use solana_ledger::{
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
 use solana_perf::packet::Packets;
-use solana_sdk::clock::Slot;
+use solana_sdk::clock::{Epoch, Slot};
 use solana_sdk::epoch_schedule::EpochSchedule;
+use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
 use std::{
@@ -44,6 +46,8 @@ struct RetransmitStats {
     total_packets: AtomicU64,
     total_batches: AtomicU64,
     total_time: AtomicU64,
+    epoch_fetch: AtomicU64,
+    epoch_cache_update: AtomicU64,
     repair_total: AtomicU64,
     discard_total: AtomicU64,
     retransmit_total: AtomicU64,
@@ -65,6 +69,8 @@ fn update_retransmit_stats(
     peers_len: usize,
     packets_by_slot: HashMap<Slot, usize>,
     packets_by_source: HashMap<String, usize>,
+    epoch_fetch: u64,
+    epoch_cach_update: u64,
 ) {
     stats.total_time.fetch_add(total_time, Ordering::Relaxed);
     stats
@@ -83,6 +89,10 @@ fn update_retransmit_stats(
         .compute_turbine_peers_total
         .fetch_add(compute_turbine_peers_total, Ordering::Relaxed);
     stats.total_batches.fetch_add(1, Ordering::Relaxed);
+    stats.epoch_fetch.fetch_add(epoch_fetch, Ordering::Relaxed);
+    stats
+        .epoch_cache_update
+        .fetch_add(epoch_cach_update, Ordering::Relaxed);
     {
         let mut stats_packets_by_slot = stats.packets_by_slot.lock().unwrap();
         for (slot, count) in packets_by_slot {
@@ -107,6 +117,16 @@ fn update_retransmit_stats(
                 stats.total_time.swap(0, Ordering::Relaxed) as i64,
                 i64
             ),
+            (
+                "epoch_fetch",
+                stats.epoch_fetch.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "epoch_cache_update",
+                stats.epoch_cache_update.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
             (
                 "total_batches",
                 stats.total_batches.swap(0, Ordering::Relaxed) as i64,
@@ -148,6 +168,14 @@ fn update_retransmit_stats(
     }
 }
 
+#[derive(Default)]
+struct EpochStakesCache {
+    epoch: Epoch,
+    stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+    peers: Vec<ContactInfo>,
+    stakes_and_index: Vec<(u64, usize)>,
+}
+
 fn retransmit(
     bank_forks: &Arc<RwLock<BankForks>>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
@@ -156,6 +184,8 @@ fn retransmit(
     sock: &UdpSocket,
     id: u32,
     stats: &Arc<RetransmitStats>,
+    epoch_stakes_cache: &Arc<RwLock<EpochStakesCache>>,
+    last_peer_update: &Arc<AtomicU64>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -172,12 +202,42 @@ fn retransmit(
     }
     drop(r_lock);
 
+    let mut epoch_fetch = Measure::start("retransmit_epoch_fetch");
     let r_bank = bank_forks.read().unwrap().working_bank();
     let bank_epoch = r_bank.get_leader_schedule_epoch(r_bank.slot());
+    epoch_fetch.stop();
+
+    let mut epoch_cache_update = Measure::start("retransmit_epoch_cach_update");
+    let mut r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    if r_epoch_stakes_cache.epoch != bank_epoch {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        if w_epoch_stakes_cache.epoch != bank_epoch {
+            let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
+            let stakes = stakes.map(Arc::new);
+            w_epoch_stakes_cache.stakes = stakes;
+            w_epoch_stakes_cache.epoch = bank_epoch;
+        }
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
+
+    let now = timestamp();
+    let last = last_peer_update.load(Ordering::Relaxed);
+    if now - last > 1000 && last_peer_update.compare_and_swap(last, now, Ordering::Relaxed) == last
+    {
+        drop(r_epoch_stakes_cache);
+        let mut w_epoch_stakes_cache = epoch_stakes_cache.write().unwrap();
+        let (peers, stakes_and_index) =
+            cluster_info.sorted_retransmit_peers_and_stakes(w_epoch_stakes_cache.stakes.clone());
+        w_epoch_stakes_cache.peers = peers;
+        w_epoch_stakes_cache.stakes_and_index = stakes_and_index;
+        drop(w_epoch_stakes_cache);
+        r_epoch_stakes_cache = epoch_stakes_cache.read().unwrap();
+    }
     let mut peers_len = 0;
-    let stakes = staking_utils::staked_nodes_at_epoch(&r_bank, bank_epoch);
-    let stakes = stakes.map(Arc::new);
-    let (peers, stakes_and_index) = cluster_info.sorted_retransmit_peers_and_stakes(stakes);
+    epoch_cache_update.stop();
+
     let my_id = cluster_info.id();
     let mut discard_total = 0;
     let mut repair_total = 0;
@@ -202,8 +262,8 @@ fn retransmit(
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
                 &my_id,
-                &peers,
-                &stakes_and_index,
+                &r_epoch_stakes_cache.peers,
+                &r_epoch_stakes_cache.stakes_and_index,
                 packet.meta.seed,
             );
             peers_len = cmp::max(peers_len, shuffled_stakes_and_index.len());
@@ -216,8 +276,14 @@ fn retransmit(
 
             let (neighbors, children) =
                 compute_retransmit_peers(DATA_PLANE_FANOUT, my_index, indexes);
-            let neighbors: Vec<_> = neighbors.into_iter().map(|index| &peers[index]).collect();
-            let children: Vec<_> = children.into_iter().map(|index| &peers[index]).collect();
+            let neighbors: Vec<_> = neighbors
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
+            let children: Vec<_> = children
+                .into_iter()
+                .map(|index| &r_epoch_stakes_cache.peers[index])
+                .collect();
             compute_turbine_peers.stop();
             compute_turbine_peers_total += compute_turbine_peers.as_us();
 
@@ -258,6 +324,8 @@ fn retransmit(
         peers_len,
         packets_by_slot,
         packets_by_source,
+        epoch_fetch.as_us(),
+        epoch_cache_update.as_us(),
     );
 
     Ok(())
@@ -287,6 +355,8 @@ pub fn retransmitter(
             let r = r.clone();
             let cluster_info = cluster_info.clone();
             let stats = stats.clone();
+            let epoch_stakes_cache = Arc::new(RwLock::new(EpochStakesCache::default()));
+            let last_peer_update = Arc::new(AtomicU64::new(0));
 
             Builder::new()
                 .name("solana-retransmitter".to_string())
@@ -301,6 +371,8 @@ pub fn retransmitter(
                             &sockets[s],
                             s as u32,
                             &stats,
+                            &epoch_stakes_cache,
+                            &last_peer_update,
                         ) {
                             match e {
                                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,

core/src/rpc_error.rs

@@ -3,6 +3,7 @@ use solana_sdk::clock::Slot;
 
 const JSON_RPC_SERVER_ERROR_0: i64 = -32000;
 const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
+const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
 
 pub enum RpcCustomError {
     NonexistentClusterRoot {
@@ -13,6 +14,9 @@ pub enum RpcCustomError {
         slot: Slot,
         first_available_block: Slot,
     },
+    SendTransactionPreflightFailure {
+        message: String,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -40,6 +44,11 @@ impl From<RpcCustomError> for Error {
                 ),
                 data: None,
             },
+            RpcCustomError::SendTransactionPreflightFailure { message } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_2),
+                message,
+                data: None,
+            },
         }
     }
 }

core/src/rpc_health.rs

@@ -0,0 +1,118 @@
+use crate::cluster_info::ClusterInfo;
+use solana_sdk::pubkey::Pubkey;
+use std::{
+    collections::HashSet,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::Arc,
+};
+
+#[derive(PartialEq, Clone, Copy)]
+pub enum RpcHealthStatus {
+    Ok,
+    Behind, // Validator is behind its trusted validators
+}
+
+pub struct RpcHealth {
+    cluster_info: Arc<ClusterInfo>,
+    trusted_validators: Option<HashSet<Pubkey>>,
+    health_check_slot_distance: u64,
+    override_health_check: Arc<AtomicBool>,
+    #[cfg(test)]
+    stub_health_status: std::sync::RwLock<Option<RpcHealthStatus>>,
+}
+
+impl RpcHealth {
+    pub fn new(
+        cluster_info: Arc<ClusterInfo>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+        health_check_slot_distance: u64,
+        override_health_check: Arc<AtomicBool>,
+    ) -> Self {
+        Self {
+            cluster_info,
+            trusted_validators,
+            health_check_slot_distance,
+            override_health_check,
+            #[cfg(test)]
+            stub_health_status: std::sync::RwLock::new(None),
+        }
+    }
+
+    pub fn check(&self) -> RpcHealthStatus {
+        #[cfg(test)]
+        {
+            if let Some(stub_health_status) = *self.stub_health_status.read().unwrap() {
+                return stub_health_status;
+            }
+        }
+
+        if self.override_health_check.load(Ordering::Relaxed) {
+            RpcHealthStatus::Ok
+        } else if let Some(trusted_validators) = &self.trusted_validators {
+            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
+                (
+                    self.cluster_info
+                        .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
+                            hashes
+                                .iter()
+                                .max_by(|a, b| a.0.cmp(&b.0))
+                                .map(|slot_hash| slot_hash.0)
+                        })
+                        .flatten()
+                        .unwrap_or(0),
+                    trusted_validators
+                        .iter()
+                        .map(|trusted_validator| {
+                            self.cluster_info
+                                .get_accounts_hash_for_node(&trusted_validator, |hashes| {
+                                    hashes
+                                        .iter()
+                                        .max_by(|a, b| a.0.cmp(&b.0))
+                                        .map(|slot_hash| slot_hash.0)
+                                })
+                                .flatten()
+                                .unwrap_or(0)
+                        })
+                        .max()
+                        .unwrap_or(0),
+                )
+            };
+
+            // This validator is considered healthy if its latest account hash slot is within
+            // `health_check_slot_distance` of the latest trusted validator's account hash slot
+            if latest_account_hash_slot > 0
+                && latest_trusted_validator_account_hash_slot > 0
+                && latest_account_hash_slot
+                    > latest_trusted_validator_account_hash_slot
+                        .saturating_sub(self.health_check_slot_distance)
+            {
+                RpcHealthStatus::Ok
+            } else {
+                warn!(
+                    "health check: me={}, latest trusted_validator={}",
+                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
+                );
+                RpcHealthStatus::Behind
+            }
+        } else {
+            // No trusted validator point of reference available, so this validator is healthy
+            // because it's running
+            RpcHealthStatus::Ok
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub() -> Arc<Self> {
+        Arc::new(Self::new(
+            Arc::new(ClusterInfo::default()),
+            None,
+            42,
+            Arc::new(AtomicBool::new(false)),
+        ))
+    }
+
+    #[cfg(test)]
+    pub(crate) fn stub_set_health_status(&self, stub_health_status: Option<RpcHealthStatus>) {
+        *self.stub_health_status.write().unwrap() = stub_health_status;
+    }
+}

core/src/rpc_service.rs

@@ -1,7 +1,8 @@
 //! The `rpc_service` module implements the Solana JSON RPC service.
 
 use crate::{
-    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*, validator::ValidatorExit,
+    cluster_info::ClusterInfo, commitment::BlockCommitmentCache, rpc::*, rpc_health::*,
+    send_transaction_service::SendTransactionService, validator::ValidatorExit,
 };
 use jsonrpc_core::MetaIoHandler;
 use jsonrpc_http_server::{
@@ -19,20 +20,17 @@ use std::{
     collections::HashSet,
     net::SocketAddr,
     path::{Path, PathBuf},
+    sync::atomic::{AtomicBool, Ordering},
     sync::{mpsc::channel, Arc, RwLock},
     thread::{self, Builder, JoinHandle},
 };
 use tokio::prelude::Future;
 
-// If trusted validators are specified, consider this validator healthy if its latest account hash
-// is no further behind than this distance from the latest trusted validator account hash
-const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
-
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
     #[cfg(test)]
-    pub request_processor: Arc<RwLock<JsonRpcRequestProcessor>>, // Used only by test_rpc_new()...
+    pub request_processor: JsonRpcRequestProcessor, // Used only by test_rpc_new()...
 
     close_handle: Option<CloseHandle>,
 }
@@ -41,18 +39,16 @@ struct RpcRequestMiddleware {
     ledger_path: PathBuf,
     snapshot_archive_path_regex: Regex,
     snapshot_config: Option<SnapshotConfig>,
-    cluster_info: Arc<ClusterInfo>,
-    trusted_validators: Option<HashSet<Pubkey>>,
     bank_forks: Arc<RwLock<BankForks>>,
+    health: Arc<RpcHealth>,
 }
 
 impl RpcRequestMiddleware {
     pub fn new(
         ledger_path: PathBuf,
         snapshot_config: Option<SnapshotConfig>,
-        cluster_info: Arc<ClusterInfo>,
-        trusted_validators: Option<HashSet<Pubkey>>,
         bank_forks: Arc<RwLock<BankForks>>,
+        health: Arc<RpcHealth>,
     ) -> Self {
         Self {
             ledger_path,
@@ -61,9 +57,8 @@ impl RpcRequestMiddleware {
             )
             .unwrap(),
             snapshot_config,
-            cluster_info,
-            trusted_validators,
             bank_forks,
+            health,
         }
     }
 
@@ -134,58 +129,10 @@ impl RpcRequestMiddleware {
     }
 
     fn health_check(&self) -> &'static str {
-        let response = if let Some(trusted_validators) = &self.trusted_validators {
-            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
-                (
-                    self.cluster_info
-                        .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
-                            hashes
-                                .iter()
-                                .max_by(|a, b| a.0.cmp(&b.0))
-                                .map(|slot_hash| slot_hash.0)
-                        })
-                        .flatten()
-                        .unwrap_or(0),
-                    trusted_validators
-                        .iter()
-                        .map(|trusted_validator| {
-                            self.cluster_info
-                                .get_accounts_hash_for_node(&trusted_validator, |hashes| {
-                                    hashes
-                                        .iter()
-                                        .max_by(|a, b| a.0.cmp(&b.0))
-                                        .map(|slot_hash| slot_hash.0)
-                                })
-                                .flatten()
-                                .unwrap_or(0)
-                        })
-                        .max()
-                        .unwrap_or(0),
-                )
-            };
-
-            // This validator is considered healthy if its latest account hash slot is within
-            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
-            if latest_account_hash_slot > 0
-                && latest_trusted_validator_account_hash_slot > 0
-                && latest_account_hash_slot
-                    > latest_trusted_validator_account_hash_slot
-                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
-            {
-                "ok"
-            } else {
-                warn!(
-                    "health check: me={}, latest trusted_validator={}",
-                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
-                );
-                "behind"
-            }
-        } else {
-            // No trusted validator point of reference available, so this validator is healthy
-            // because it's running
-            "ok"
+        let response = match self.health.check() {
+            RpcHealthStatus::Ok => "ok",
+            RpcHealthStatus::Behind => "behind",
         };
-
         info!("health check: {}", response);
         response
     }
@@ -290,16 +237,36 @@ impl JsonRpcService {
         ledger_path: &Path,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
         trusted_validators: Option<HashSet<Pubkey>>,
+        override_health_check: Arc<AtomicBool>,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
-        let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
+
+        let health = Arc::new(RpcHealth::new(
+            cluster_info.clone(),
+            trusted_validators,
+            config.health_check_slot_distance,
+            override_health_check,
+        ));
+
+        let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
+        let send_transaction_service = Arc::new(SendTransactionService::new(
+            &cluster_info,
+            &bank_forks,
+            &exit_send_transaction_service,
+        ));
+
+        let request_processor = JsonRpcRequestProcessor::new(
             config,
             bank_forks.clone(),
             block_commitment_cache,
             blockstore,
             validator_exit.clone(),
-        )));
+            health.clone(),
+            cluster_info,
+            genesis_hash,
+            send_transaction_service,
+        );
 
         #[cfg(test)]
         let test_request_processor = request_processor.clone();
@@ -317,17 +284,12 @@ impl JsonRpcService {
                 let request_middleware = RpcRequestMiddleware::new(
                     ledger_path,
                     snapshot_config,
-                    cluster_info.clone(),
-                    trusted_validators,
                     bank_forks.clone(),
+                    health.clone(),
                 );
                 let server = ServerBuilder::with_meta_extractor(
                     io,
-                    move |_req: &hyper::Request<hyper::Body>| Meta {
-                        request_processor: request_processor.clone(),
-                        cluster_info: cluster_info.clone(),
-                        genesis_hash,
-                    },
+                    move |_req: &hyper::Request<hyper::Body>| request_processor.clone(),
                 )
                 .threads(num_cpus::get())
                 .cors(DomainsValidation::AllowOnly(vec![
@@ -350,6 +312,7 @@ impl JsonRpcService {
                 let server = server.unwrap();
                 close_handle_sender.send(server.close_handle()).unwrap();
                 server.wait();
+                exit_send_transaction_service.store(true, Ordering::Relaxed);
             })
             .unwrap();
 
@@ -383,7 +346,6 @@ impl JsonRpcService {
 mod tests {
     use super::*;
     use crate::{
-        contact_info::ContactInfo,
         crds_value::{CrdsData, CrdsValue, SnapshotHash},
         rpc::tests::create_validator_exit,
     };
@@ -394,8 +356,7 @@ mod tests {
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::Signer;
-    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-    use std::sync::atomic::AtomicBool;
+    use std::net::{IpAddr, Ipv4Addr};
 
     #[test]
     fn test_rpc_new() {
@@ -407,7 +368,7 @@ mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let validator_exit = create_validator_exit(&exit);
         let bank = Bank::new(&genesis_config);
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
+        let cluster_info = Arc::new(ClusterInfo::default());
         let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         let rpc_addr = SocketAddr::new(
             ip_addr,
@@ -431,6 +392,7 @@ mod tests {
             &PathBuf::from("farf"),
             validator_exit,
             None,
+            Arc::new(AtomicBool::new(false)),
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -439,8 +401,6 @@ mod tests {
             10_000,
             rpc_service
                 .request_processor
-                .read()
-                .unwrap()
                 .get_balance(Ok(mint_keypair.pubkey()), None)
                 .unwrap()
                 .value
@@ -472,15 +432,12 @@ mod tests {
 
     #[test]
     fn test_is_file_get_path() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
         let bank_forks = create_bank_forks();
-
         let rrm = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             None,
-            cluster_info.clone(),
-            None,
             bank_forks.clone(),
+            RpcHealth::stub(),
         );
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
@@ -490,9 +447,8 @@ mod tests {
                 snapshot_path: PathBuf::from("/"),
                 compression: CompressionType::Bzip2,
             }),
-            cluster_info,
-            None,
             bank_forks,
+            RpcHealth::stub(),
         );
 
         assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
@@ -518,30 +474,30 @@ mod tests {
 
     #[test]
     fn test_health_check_with_no_trusted_validators() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
-
         let rm = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             None,
-            cluster_info,
-            None,
             create_bank_forks(),
+            RpcHealth::stub(),
         );
         assert_eq!(rm.health_check(), "ok");
     }
 
     #[test]
     fn test_health_check_with_trusted_validators() {
-        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(ContactInfo::default()));
-
+        let cluster_info = Arc::new(ClusterInfo::default());
+        let health_check_slot_distance = 123;
+        let override_health_check = Arc::new(AtomicBool::new(false));
         let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
-        let rm = RpcRequestMiddleware::new(
-            PathBuf::from("/"),
-            None,
+
+        let health = Arc::new(RpcHealth::new(
             cluster_info.clone(),
             Some(trusted_validators.clone().into_iter().collect()),
-            create_bank_forks(),
-        );
+            health_check_slot_distance,
+            override_health_check.clone(),
+        ));
+
+        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, create_bank_forks(), health);
 
         // No account hashes for this node or any trusted validators == "behind"
         assert_eq!(rm.health_check(), "behind");
@@ -549,6 +505,9 @@ mod tests {
         // No account hashes for any trusted validators == "behind"
         cluster_info.push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
         assert_eq!(rm.health_check(), "behind");
+        override_health_check.store(true, Ordering::Relaxed);
+        assert_eq!(rm.health_check(), "ok");
+        override_health_check.store(false, Ordering::Relaxed);
 
         // This node is ahead of the trusted validators == "ok"
         cluster_info
@@ -579,7 +538,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[1],
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
+                    vec![(1000 + health_check_slot_distance - 1, Hash::default())],
                 ))),
                 1,
             )
@@ -595,7 +554,7 @@ mod tests {
             .insert(
                 CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
                     trusted_validators[2],
-                    vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
+                    vec![(1000 + health_check_slot_distance, Hash::default())],
                 ))),
                 1,
             )

core/src/send_transaction_service.rs

@@ -0,0 +1,377 @@
+use crate::cluster_info::ClusterInfo;
+use solana_ledger::bank_forks::BankForks;
+use solana_metrics::{datapoint_warn, inc_new_counter_info};
+use solana_runtime::bank::Bank;
+use solana_sdk::{clock::Slot, signature::Signature};
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, UdpSocket},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::{channel, Receiver, Sender},
+        Arc, Mutex, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+/// Maximum size of the transaction queue
+const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
+
+pub struct SendTransactionService {
+    thread: JoinHandle<()>,
+    sender: Mutex<Sender<TransactionInfo>>,
+    send_socket: UdpSocket,
+    tpu_address: SocketAddr,
+}
+
+struct TransactionInfo {
+    signature: Signature,
+    wire_transaction: Vec<u8>,
+    last_valid_slot: Slot,
+}
+
+#[derive(Default, Debug, PartialEq)]
+struct ProcessTransactionsResult {
+    rooted: u64,
+    expired: u64,
+    retried: u64,
+    failed: u64,
+    retained: u64,
+}
+
+impl SendTransactionService {
+    pub fn new(
+        cluster_info: &Arc<ClusterInfo>,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let (sender, receiver) = channel::<TransactionInfo>();
+        let tpu_address = cluster_info.my_contact_info().tpu;
+
+        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address, exit.clone());
+        Self {
+            thread,
+            sender: Mutex::new(sender),
+            send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
+            tpu_address,
+        }
+    }
+
+    fn retry_thread(
+        receiver: Receiver<TransactionInfo>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        tpu_address: SocketAddr,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        let mut last_status_check = Instant::now();
+        let mut transactions = HashMap::new();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        Builder::new()
+            .name("send-tx-svc".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+
+                if let Ok(transaction_info) = receiver.recv_timeout(Duration::from_secs(1)) {
+                    if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
+                        transactions.insert(transaction_info.signature, transaction_info);
+                    } else {
+                        datapoint_warn!("send_transaction_service-queue-overflow");
+                    }
+                }
+
+                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                    if !transactions.is_empty() {
+                        datapoint_info!(
+                            "send_transaction_service-queue-size",
+                            ("len", transactions.len(), i64)
+                        );
+                        let bank_forks = bank_forks.read().unwrap();
+                        let root_bank = bank_forks.root_bank();
+                        let working_bank = bank_forks.working_bank();
+
+                        let _result = Self::process_transactions(
+                            &working_bank,
+                            &root_bank,
+                            &send_socket,
+                            &tpu_address,
+                            &mut transactions,
+                        );
+                    }
+                    last_status_check = Instant::now();
+                }
+            })
+            .unwrap()
+    }
+
+    fn process_transactions(
+        working_bank: &Arc<Bank>,
+        root_bank: &Arc<Bank>,
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        transactions: &mut HashMap<Signature, TransactionInfo>,
+    ) -> ProcessTransactionsResult {
+        let mut result = ProcessTransactionsResult::default();
+
+        transactions.retain(|signature, transaction_info| {
+            if root_bank.has_signature(signature) {
+                info!("Transaction is rooted: {}", signature);
+                result.rooted += 1;
+                inc_new_counter_info!("send_transaction_service-rooted", 1);
+                false
+            } else if transaction_info.last_valid_slot < root_bank.slot() {
+                info!("Dropping expired transaction: {}", signature);
+                result.expired += 1;
+                inc_new_counter_info!("send_transaction_service-expired", 1);
+                false
+            } else {
+                match working_bank.get_signature_status_slot(signature) {
+                    None => {
+                        // Transaction is unknown to the working bank, it might have been
+                        // dropped or landed in another fork.  Re-send it
+                        info!("Retrying transaction: {}", signature);
+                        result.retried += 1;
+                        inc_new_counter_info!("send_transaction_service-retry", 1);
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        true
+                    }
+                    Some((_slot, status)) => {
+                        if status.is_err() {
+                            info!("Dropping failed transaction: {}", signature);
+                            result.failed += 1;
+                            inc_new_counter_info!("send_transaction_service-failed", 1);
+                            false
+                        } else {
+                            result.retained += 1;
+                            true
+                        }
+                    }
+                }
+            }
+        });
+
+        result
+    }
+
+    fn send_transaction(
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        wire_transaction: &[u8],
+    ) {
+        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+        }
+    }
+
+    pub fn send(&self, signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) {
+        inc_new_counter_info!("send_transaction_service-enqueue", 1, 1);
+        Self::send_transaction(&self.send_socket, &self.tpu_address, &wire_transaction);
+
+        self.sender
+            .lock()
+            .unwrap()
+            .send(TransactionInfo {
+                signature,
+                wire_transaction,
+                last_valid_slot,
+            })
+            .unwrap_or_else(|err| warn!("Failed to enqueue transaction: {}", err));
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::rpc::tests::new_bank_forks;
+    use solana_sdk::{pubkey::Pubkey, signature::Signer};
+
+    #[test]
+    fn service_exit() {
+        let cluster_info = Arc::new(ClusterInfo::default());
+        let bank_forks = new_bank_forks().0;
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let send_tranaction_service =
+            SendTransactionService::new(&cluster_info, &bank_forks, &exit);
+
+        exit.store(true, Ordering::Relaxed);
+        send_tranaction_service.join().unwrap();
+    }
+
+    #[test]
+    fn process_transactions() {
+        solana_logger::setup();
+
+        let (bank_forks, mint_keypair, _voting_keypair) = new_bank_forks();
+        let cluster_info = ClusterInfo::default();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let tpu_address = cluster_info.my_contact_info().tpu;
+
+        let root_bank = Arc::new(Bank::new_from_parent(
+            &bank_forks.read().unwrap().working_bank(),
+            &Pubkey::default(),
+            1,
+        ));
+        let rooted_signature = root_bank
+            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
+
+        let non_rooted_signature = working_bank
+            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let failed_signature = {
+            let blockhash = working_bank.last_blockhash();
+            let transaction = solana_sdk::system_transaction::transfer(
+                &mint_keypair,
+                &Pubkey::default(),
+                1,
+                blockhash,
+            );
+            let signature = transaction.signatures[0];
+            working_bank.process_transaction(&transaction).unwrap_err();
+            signature
+        };
+
+        let mut transactions = HashMap::new();
+
+        info!("Expired transactions are dropped..");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo {
+                signature: Signature::default(),
+                wire_transaction: vec![],
+                last_valid_slot: root_bank.slot() - 1,
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Rooted transactions are dropped...");
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo {
+                signature: rooted_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Failed transactions are dropped...");
+        transactions.insert(
+            failed_signature,
+            TransactionInfo {
+                signature: failed_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                failed: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Non-rooted transactions are kept...");
+        transactions.insert(
+            non_rooted_signature,
+            TransactionInfo {
+                signature: non_rooted_signature,
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retained: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        transactions.clear();
+
+        info!("Unknown transactions are retried...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo {
+                signature: Signature::default(),
+                wire_transaction: vec![],
+                last_valid_slot: working_bank.slot(),
+            },
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retried: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+    }
+}

core/src/serve_repair.rs

@@ -532,6 +532,8 @@ impl ServeRepair {
                 );
                 if let Some(packet) = packet {
                     res.packets.push(packet);
+                } else {
+                    break;
                 }
                 if meta.is_parent_set() && res.packets.len() <= max_responses {
                     slot = meta.parent_slot;
@@ -864,4 +866,64 @@ mod tests {
 
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
     }
+
+    #[test]
+    fn run_orphan_corrupted_shred_size() {
+        solana_logger::setup();
+        let recycler = PacketsRecycler::default();
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+            // Create slots [1, 2] with 1 shred apiece
+            let (mut shreds, _) = make_many_slot_entries(1, 2, 1);
+
+            // Make shred for slot 1 too large
+            assert_eq!(shreds[0].slot(), 1);
+            assert_eq!(shreds[0].index(), 0);
+            shreds[0].payload.push(10);
+            blockstore
+                .insert_shreds(shreds, None, false)
+                .expect("Expect successful ledger write");
+            let nonce = 42;
+            // Make sure repair response is corrupted
+            assert!(repair_response::repair_response_packet(
+                &blockstore,
+                1,
+                0,
+                &socketaddr_any!(),
+                nonce,
+            )
+            .is_none());
+
+            // Orphan request for slot 2 should only return slot 1 since
+            // calling `repair_response_packet` on slot 1's shred will
+            // be corrupted
+            let rv: Vec<_> = ServeRepair::run_orphan(
+                &recycler,
+                &socketaddr_any!(),
+                Some(&blockstore),
+                2,
+                5,
+                nonce,
+            )
+            .expect("run_orphan packets")
+            .packets
+            .iter()
+            .cloned()
+            .collect();
+
+            // Verify responses
+            let expected = vec![repair_response::repair_response_packet(
+                &blockstore,
+                2,
+                0,
+                &socketaddr_any!(),
+                nonce,
+            )
+            .unwrap()];
+            assert_eq!(rv, expected);
+        }
+
+        Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
+    }
 }

core/src/validator.rs

@@ -234,6 +234,7 @@ impl Validator {
             block_commitment_cache.clone(),
         ));
 
+        let rpc_override_health_check = Arc::new(AtomicBool::new(false));
         let rpc_service = config.rpc_ports.map(|(rpc_port, rpc_pubsub_port)| {
             if ContactInfo::is_valid_address(&node.info.rpc) {
                 assert!(ContactInfo::is_valid_address(&node.info.rpc_pubsub));
@@ -255,6 +256,7 @@ impl Validator {
                     ledger_path,
                     validator_exit.clone(),
                     config.trusted_validators.clone(),
+                    rpc_override_health_check.clone(),
                 ),
                 PubSubService::new(
                     &subscriptions,
@@ -374,7 +376,7 @@ impl Validator {
                 (None, None)
             };
 
-        wait_for_supermajority(config, &bank, &cluster_info);
+        wait_for_supermajority(config, &bank, &cluster_info, rpc_override_health_check);
 
         let poh_service = PohService::new(poh_recorder.clone(), &poh_config, &exit);
         assert_eq!(
@@ -620,7 +622,12 @@ fn new_banks_from_blockstore(
     )
 }
 
-fn wait_for_supermajority(config: &ValidatorConfig, bank: &Bank, cluster_info: &ClusterInfo) {
+fn wait_for_supermajority(
+    config: &ValidatorConfig,
+    bank: &Bank,
+    cluster_info: &ClusterInfo,
+    rpc_override_health_check: Arc<AtomicBool>,
+) {
     if config.wait_for_supermajority != Some(bank.slot()) {
         return;
     }
@@ -635,8 +642,13 @@ fn wait_for_supermajority(config: &ValidatorConfig, bank: &Bank, cluster_info: &
         if gossip_stake_percent >= 80 {
             break;
         }
+        // The normal RPC health checks don't apply as the node is waiting, so feign health to
+        // prevent load balancers from removing the node from their list of candidates during a
+        // manual restart.
+        rpc_override_health_check.store(true, Ordering::Relaxed);
         sleep(Duration::new(1, 0));
     }
+    rpc_override_health_check.store(false, Ordering::Relaxed);
 }
 
 pub struct TestValidator {

core/tests/crds_gossip.rs

@@ -5,7 +5,7 @@ use solana_core::cluster_info;
 use solana_core::contact_info::ContactInfo;
 use solana_core::crds_gossip::*;
 use solana_core::crds_gossip_error::CrdsGossipError;
-use solana_core::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+use solana_core::crds_gossip_pull::{ProcessPullStats, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS};
 use solana_core::crds_gossip_push::CRDS_GOSSIP_PUSH_MSG_TIMEOUT_MS;
 use solana_core::crds_value::CrdsValueLabel;
 use solana_core::crds_value::{CrdsData, CrdsValue};
@@ -426,36 +426,35 @@ fn network_run_pull(
                     .map(|f| f.filter.bits.len() as usize / 8)
                     .sum::<usize>();
                 bytes += serialized_size(&caller_info).unwrap() as usize;
-                let filters = filters
+                let filters: Vec<_> = filters
                     .into_iter()
                     .map(|f| (caller_info.clone(), f))
                     .collect();
-                let rsp = network
+                let rsp: Vec<_> = network
                     .get(&to)
                     .map(|node| {
-                        let mut rsp = vec![];
-                        rsp.append(
-                            &mut node
-                                .lock()
-                                .unwrap()
-                                .process_pull_requests(filters, now)
-                                .into_iter()
-                                .flatten()
-                                .collect(),
-                        );
+                        let rsp = node
+                            .lock()
+                            .unwrap()
+                            .generate_pull_responses(&filters, None)
+                            .into_iter()
+                            .flatten()
+                            .collect();
+                        node.lock().unwrap().process_pull_requests(filters, now);
                         rsp
                     })
                     .unwrap();
                 bytes += serialized_size(&rsp).unwrap() as usize;
                 msgs += rsp.len();
                 if let Some(node) = network.get(&from) {
-                    node.lock()
-                        .unwrap()
-                        .mark_pull_request_creation_time(&from, now);
-                    overhead += node
-                        .lock()
-                        .unwrap()
-                        .process_pull_response(&from, &timeouts, rsp, now);
+                    let mut node = node.lock().unwrap();
+                    node.mark_pull_request_creation_time(&from, now);
+                    let mut stats = ProcessPullStats::default();
+                    let (vers, vers_expired_timeout) =
+                        node.filter_pull_responses(&timeouts, rsp, now, &mut stats);
+                    node.process_pull_responses(&from, vers, vers_expired_timeout, now, &mut stats);
+                    overhead += stats.failed_insert;
+                    overhead += stats.failed_timeout;
                 }
                 (bytes, msgs, overhead)
             })

core/tests/ledger_cleanup.rs

@@ -374,14 +374,17 @@ mod tests {
         // send signal to cleanup slots
         let (sender, receiver) = channel();
         sender.send(n).unwrap();
-        let mut next_purge_batch = 0;
+        let mut last_purge_slot = 0;
+        let mut last_compaction_slot = 0;
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
             max_ledger_shreds,
-            &mut next_purge_batch,
+            &mut last_purge_slot,
             10,
             None,
+            &mut last_compaction_slot,
+            10,
         )
         .unwrap();
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

docs/art/sdk-tools.bob

@@ -1,20 +1,20 @@
 
-                            .----------------------------------------.
-                            |            Solana Runtime              |
-                            |                                        |
-                            |   .------------.   .------------.      |
-                            |   |            |   |            |      |
-                      .-------->|  Verifier  +-->|  Accounts  |      |
-                      |     |   |            |   |            |      |
-  .----------.        |     |   `------------`   `------------`      |
-  |          +--------`     |                           ^            |
-  |  Client  |              |             LoadAccounts  |            |
-  |          +--------.     |          .----------------`            |
-  `----------`        |     |          |                             |
-                      |     |   .------+-----.   .-------------.     |
-                      |     |   |            |   |             |     |
-                      `-------->|  Loader    +-->| Interpreter |     |
-                            |   |            |   |             |     |
-                            |   `------------`   `-------------`     |
-                            |                                        |
-                            `----------------------------------------`
+                 .-------------------------------------.
+                 | Solana Runtime                      |
+                 |                                     |
+  .----------.   |   .------------.   .------------.   |
+  | Program  |   |   | BPF        |   | Executable |   |
+  | Author   +------>| Bytecode   +-->| Account    |   |
+  |          |   |   | Verifier   |   |            |   |
+  `----------`   |   `------------`   `------------`   |
+                 |                           |         |
+                 |          .----------------`         |
+                 |          |  LoadAccounts            |
+                 |          V                          |
+  .----------.   |   .------------.   .-------------.  |
+  |          |   |   |  BPF       |   | BPF         |  |
+  |  Client  +------>|  Loader    +-->| Interpreter |  |
+  |          |   |   |            |   |             |  |
+  `----------`   |   `------------`   `-------------`  |
+                 |                                     |
+                 `-------------------------------------`

docs/set-solana-release-tag.sh

@@ -13,6 +13,11 @@ else
   )
 fi
 
+if [[ -z "$LATEST_SOLANA_RELEASE_VERSION" ]]; then
+  echo Error: release version not defined
+  exit 1
+fi
+
 set -x
 find html/ -name \*.html -exec sed -i "s/LATEST_SOLANA_RELEASE_VERSION/$LATEST_SOLANA_RELEASE_VERSION/g" {} \;
 if [[ -n $CI ]]; then

docs/src/SUMMARY.md

@@ -24,12 +24,16 @@
   * [Command-line Reference](cli/usage.md)
 * [Solana Clusters](clusters.md)
 * [Develop Applications](apps/README.md)
+  * [Accounts and Rent](apps/rent.md)
   * [Example: Web Wallet](apps/webwallet.md)
   * [Example: Tic-Tac-Toe](apps/tictactoe.md)
   * [Drones](apps/drones.md)
   * [Anatomy of a Transaction](transaction.md)
   * [JSON RPC API](apps/jsonrpc-api.md)
   * [JavaScript API](apps/javascript-api.md)
+  * [Builtin Programs](apps/builtins/README.md)
+* [Integration Guides](integrations/README.md)
+  * [Exchange](integrations/exchange.md)
 * [Run a Validator](running-validator/README.md)
   * [Validator Requirements](running-validator/validator-reqs.md)
   * [Start a Validator](running-validator/validator-start.md)
@@ -95,14 +99,15 @@
   * [Validator Timestamp Oracle](implemented-proposals/validator-timestamp-oracle.md)
   * [Commitment](implemented-proposals/commitment.md)
   * [Snapshot Verification](implemented-proposals/snapshot-verification.md)
+  * [Cross-Program Invocation](implemented-proposals/cross-program-invocation.md)
+  * [Program Derived Addresses](implemented-proposals/program-derived-addresses.md)
 * [Accepted Design Proposals](proposals/README.md)
+  * [Ledger Replication](proposals/ledger-replication-to-implement.md)
   * [Optimistic Confirmation and Slashing](proposals/optimistic-confirmation-and-slashing.md)
   * [Secure Vote Signing](proposals/vote-signing-to-implement.md)
   * [Cluster Test Framework](proposals/cluster-test-framework.md)
   * [Validator](proposals/validator-proposal.md)
   * [Simple Payment and State Verification](proposals/simple-payment-and-state-verification.md)
-  * [Cross-Program Invocation](proposals/cross-program-invocation.md)
-  * [Program Keys and Signatures](proposals/program-keys-and-signatures.md)
   * [Inter-chain Transaction Verification](proposals/interchain-transaction-verification.md)
   * [Snapshot Verification](proposals/snapshot-verification.md)
   * [Bankless Leader](proposals/bankless-leader.md)

docs/src/apps/README.md

@@ -1,6 +1,6 @@
 # Programming Model
 
-An _app_ interacts with a Solana cluster by sending it _transactions_ with one or more _instructions_. The Solana _runtime_ passes those instructions to user-contributed _programs_. An instruction might, for example, tell a program to transfer _lamports_ from one _account_ to another or create an interactive contract that governs how lamports are transfered. Instructions are executed sequentially and atomically. If any instruction is invalid, any changes made within the transaction are discarded.
+An _app_ interacts with a Solana cluster by sending it _transactions_ with one or more _instructions_. The Solana _runtime_ passes those instructions to _programs_ deployed by app developers beforehand. An instruction might, for example, tell a program to transfer _lamports_ from one _account_ to another or create an interactive contract that governs how lamports are transferred. Instructions are executed sequentially and atomically for each transaction. If any instruction is invalid, all account changes in the transaction are discarded.
 
 ### Accounts and Signatures
 
@@ -10,7 +10,7 @@ The transaction also marks some accounts as _read-only accounts_. The runtime pe
 
 ### Recent Blockhash
 
-A Transaction includes a recent blockhash to prevent duplication and to give transactions lifetimes. Any transaction that is completely identical to a previous one is rejected, so adding a newer blockhash allows multiple transactions to repeat the exact same action. Transactions also have lifetimes that are defined by the blockhash, as any transaction whose blockhash is too old will be rejected.
+A transaction includes a recent blockhash to prevent duplication and to give transactions lifetimes. Any transaction that is completely identical to a previous one is rejected, so adding a newer blockhash allows multiple transactions to repeat the exact same action. Transactions also have lifetimes that are defined by the blockhash, as any transaction whose blockhash is too old will be rejected.
 
 ### Instructions
 
@@ -20,25 +20,32 @@ Each instruction specifies a single program account \(which must be marked execu
 
 ![SDK tools](../.gitbook/assets/sdk-tools.svg)
 
-As shown in the diagram above a client creates a program and compiles it to an ELF shared object containing BPF bytecode and sends it to the Solana cluster. The cluster stores the program locally and makes it available to clients via a _program ID_. The program ID is a _public key_ generated by the client and is used to reference the program in subsequent transactions.
+As shown in the diagram above, a program author creates a program and compiles it to an ELF shared object containing BPF bytecode and uploads it to the Solana cluster with a special _deploy_ transaction. The cluster makes it available to clients via a _program ID_. The program ID is a _address_ specified when deploying and is used to reference the program in subsequent transactions.
 
-A program may be written in any programming language that can target the Berkley Packet Filter \(BPF\) safe execution environment. The Solana SDK offers the best support for C programs, which is compiled to BPF using the [LLVM compiler infrastructure](https://llvm.org).
+A program may be written in any programming language that can target the Berkley Packet Filter \(BPF\) safe execution environment. The Solana SDK offers the best support for C/C++ and Rust programs, which are compiled to BPF using the [LLVM compiler infrastructure](https://llvm.org).
 
 ## Storing State between Transactions
 
-If the program needs to store state between transactions, it does so using _accounts_. Accounts are similar to files in operating systems such as Linux. Like a file, an account may hold arbitrary data and that data persists beyond the lifetime of a program. Also like a file, an account includes metadata that tells the runtime who is allowed to access the data and how. Unlike a file, the account includes metadata for the lifetime of the file. That lifetime is expressed in "tokens", which is a number of fractional native tokens, called _lamports_. Accounts are held in validator memory and pay "rent" to stay there. Each validator periodically scan all accounts and collects rent. Any account that drops to zero lamports is purged.
+If the program needs to store state between transactions, it does so using _accounts_. Accounts are similar to files in operating systems such as Linux. Like a file, an account may hold arbitrary data and that data persists beyond the lifetime of a program. Also like a file, an account includes metadata that tells the runtime who is allowed to access the data and how.
 
-If an account is marked "executable", it will only be used by a _loader_ to run programs. For example, a BPF-compiled program is marked executable and loaded by the BPF loader. No program is allowed to modify the contents of an executable account.
+Unlike a file, the account includes metadata for the lifetime of the file. That lifetime is expressed in "tokens", which is a number of fractional native tokens, called _lamports_. Accounts are held in validator memory and pay ["rent"](rent.md) to stay there. Each validator periodically scans all accounts and collects rent. Any account that drops to zero lamports is purged.
 
-An account also includes "owner" metadata. The owner is a program ID. The runtime grants the program write access to the account if its ID matches the owner. If an account is not owned by a program, the program is permitted to read its data and credit the account.
+In the same way that a Linux user uses a path to look up a file, a Solana client uses an _address_ to look up an account. The address is usually a 256-bit public key. To create an account with a public-key address, the client generates a _keypair_ and registers its public key using the `CreateAccount` instruction with preallocated fixed storage size in bytes. In fact, the account address can be an arbitrary 32 bytes, and there is a mechanism for advanced users to create derived addresses (`CreateAccountWithSeed`). Addresses are presented in Base58 encoding on user interfaces.
 
-In the same way that a Linux user uses a path to look up a file, a Solana client uses public keys to look up accounts. To create an account, the client generates a _keypair_ and registers its public key using the `CreateAccount` instruction. The account created by `CreateAccount` is called a _system account_ and is owned by a built-in program called the System program. The System program allows clients to transfer lamports and assign account ownership.
+## Ownership of Accounts and Assignment to Programs
 
-The runtime only permits the owner to debit the account or modify its data. The program then defines additional rules for whether the client can modify accounts it owns. In the case of the System program, it allows users to transfer lamports by recognizing transaction signatures. If it sees the client signed the transaction using the keypair's _private key_, it knows the client authorized the token transfer.
+The created account is initialized to be _owned_ by a built-in program called the System program and is called a _system account_ aptly. An account includes "owner" metadata. The owner is a program ID. The runtime grants the program write access to the account if its ID matches the owner. For the case of the System program, the runtime allows clients to transfer lamports and importantly _assign_ account ownership, meaning changing owner to different program ID. If an account is not owned by a program, the program is only permitted to read its data and credit the account.
+
+Also, if an account is marked "executable" in metadata, it will only be used by a _loader_ to run programs. For example, a BPF-compiled program is marked executable and loaded by the BPF loader when executing its transactions. No program is allowed to modify the contents of an executable account once deployed.
+
+## Runtime Capability of Programs on Accounts
+
+The runtime only permits the owner program to debit the account or modify its data. The program then defines additional rules for whether the client can modify accounts it owns. In the case of the System program, it allows users to transfer lamports by recognizing transaction signatures. If it sees the client signed the transaction using the keypair's _private key_, it knows the client authorized the token transfer.
+
+In other words, the entire set of accounts owned by a given program can be regarded as a key-value store where a key is the account address and value is program-specific arbitrary binary data. A program author can decide how to manage the program's whole state as possibly many accounts.
 
 After the runtime executes each of the transaction's instructions, it uses the account metadata to verify that none of the access rules were violated. If a program violates an access rule, the runtime discards all account changes made by all instructions and marks the transaction as failed.
 
 ## Smart Contracts
 
 Programs don't always require transaction signatures, as the System program does. Instead, the program may manage _smart contracts_. A smart contract is a set of constraints that once satisfied, signal to a program that a token transfer or account update is permitted. For example, one could use the Budget program to create a smart contract that authorizes a token transfer only after some date. Once evidence that the date has past, the contract progresses, and token transfer completes.
-

docs/src/apps/builtins/README.md

@@ -0,0 +1,61 @@
+# Builtin Programs
+
+Solana contains a small handful of builtin programs, which are required to run
+validator nodes. Unlike third-party programs, the builtin programs are part of
+the validator implementation and can be upgraded as part of cluster upgrades.
+Upgrades may occur to add features, fix bugs, or improve performance. Interface
+changes to individual instructions should rarely, if ever, occur. Instead, when
+change is needed, new instructions are added and previous ones are marked
+deprecated. Apps can upgrade on their own timeline without concern of breakages
+across upgrades.
+
+The builtin programs include the System, Config, Stake, Vote, and BPFLoader
+programs. For each, we provide the program ID and describe each supported
+instruction. A transaction can mix and match instructions from different
+programs, as well include instructions from third-party programs.
+
+## System Program
+
+Create accounts and transfer lamports between them
+
+* Program ID: `11111111111111111111111111111111`
+* Instructions: [SystemInstruction](https://docs.rs/solana-sdk/LATEST_SOLANA_RELEASE_VERSION/solana_sdk/system_instruction/enum.SystemInstruction.html)
+
+## Config Program
+
+Add configuration data to the chain and the list of public keys that are permitted to modify it
+
+* Program ID: `Config1111111111111111111111111111111111111`
+* Instructions: [config_instruction](https://docs.rs/solana-config-program/LATEST_SOLANA_RELEASE_VERSION/solana_config_program/config_instruction/index.html)
+
+Unlike the other programs, the Config program does not define any individual
+instructions. It has just one implicit instruction, a "store" instruction. Its
+instruction data is a set of keys that gate access to the account, and the
+data to store in it.
+
+## Stake Program
+
+Create stake accounts and delegate it to validators
+
+* Program ID: `Stake11111111111111111111111111111111111111`
+* Instructions: [StakeInstruction](https://docs.rs/solana-stake-program/LATEST_SOLANA_RELEASE_VERSION/solana_stake_program/stake_instruction/enum.StakeInstruction.html)
+
+## Vote Program
+
+Create vote accounts and vote on blocks
+
+* Program ID: `Vote111111111111111111111111111111111111111`
+* Instructions: [VoteInstruction](https://docs.rs/solana-vote-program/LATEST_SOLANA_RELEASE_VERSION/solana_vote_program/vote_instruction/enum.VoteInstruction.html)
+
+## BPF Loader
+
+Add programs to the chain.
+
+* Program ID: `BPFLoader1111111111111111111111111111111111`
+* Instructions: [LoaderInstruction](https://docs.rs/solana-sdk/LATEST_SOLANA_RELEASE_VERSION/solana_sdk/loader_instruction/enum.LoaderInstruction.html)
+
+The BPF Loader marks itself as its "owner" of the executable account it
+creates to store your program. When a user invokes an instruction via a
+program ID, the Solana runtime will load both your executable account and its
+owner, the BPF Loader. The runtime then passes your program to the BPF Loader
+to process the instruction.

docs/src/apps/jsonrpc-api.md

@@ -27,10 +27,12 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
 * [getEpochSchedule](jsonrpc-api.md#getepochschedule)
 * [getFeeCalculatorForBlockhash](jsonrpc-api.md#getfeecalculatorforblockhash)
 * [getFeeRateGovernor](jsonrpc-api.md#getfeerategovernor)
+* [getFees](jsonrpc-api.md#getfees)
 * [getFirstAvailableBlock](jsonrpc-api.md#getfirstavailableblock)
 * [getGenesisHash](jsonrpc-api.md#getgenesishash)
 * [getIdentity](jsonrpc-api.md#getidentity)
-* [getInflation](jsonrpc-api.md#getinflation)
+* [getInflationGovernor](jsonrpc-api.md#getinflationgovernor)
+* [getInflationRate](jsonrpc-api.md#getinflationrate)
 * [getLargestAccounts](jsonrpc-api.md#getlargestaccounts)
 * [getLeaderSchedule](jsonrpc-api.md#getleaderschedule)
 * [getMinimumBalanceForRentExemption](jsonrpc-api.md#getminimumbalanceforrentexemption)
@@ -39,10 +41,6 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
 * [getSignatureStatuses](jsonrpc-api.md#getsignaturestatuses)
 * [getSlot](jsonrpc-api.md#getslot)
 * [getSlotLeader](jsonrpc-api.md#getslotleader)
-* [getSlotsPerSegment](jsonrpc-api.md#getslotspersegment)
-* [getStoragePubkeysForSlot](jsonrpc-api.md#getstoragepubkeysforslot)
-* [getStorageTurn](jsonrpc-api.md#getstorageturn)
-* [getStorageTurnRate](jsonrpc-api.md#getstorageturnrate)
 * [getSupply](jsonrpc-api.md#getsupply)
 * [getTransactionCount](jsonrpc-api.md#gettransactioncount)
 * [getVersion](jsonrpc-api.md#getversion)
@@ -494,7 +492,8 @@ Returns the fee calculator associated with the query blockhash, or `null` if the
 
 #### Parameters:
 
-* `blockhash: <string>`, query blockhash as a Base58 encoded string
+* `<string>` - query blockhash as a Base58 encoded string
+* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
 
 #### Results:
 
@@ -542,6 +541,34 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":{"context":{"slot":54},"value":{"feeRateGovernor":{"burnPercent":50,"maxLamportsPerSignature":100000,"minLamportsPerSignature":5000,"targetLamportsPerSignature":10000,"targetSignaturesPerSlot":20000}}},"id":1}
 ```
 
+### getFees
+
+Returns a recent block hash from the ledger, a fee schedule that can be used to
+compute the cost of submitting a transaction using it, and the last slot in
+which the blockhash will be valid.
+
+#### Parameters:
+
+* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
+
+#### Results:
+
+The result will be an RpcResponse JSON object with `value` set to a JSON object with the following fields:
+
+* `blockhash: <string>` - a Hash as base-58 encoded string
+* `feeCalculator: <object>` - FeeCalculator object, the fee schedule for this block hash
+* `lastValidSlot: <u64>` - last slot in which a blockhash will be valid
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getFees"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":{"blockhash":"CSymwgTNX1j3E4qhKfJAUE41nBWEwXufoYryPbkde5RR","feeCalculator":{"lamportsPerSignature":5000},"lastValidSlot":297}},"id":1}
+```
+
 ### getFirstAvailableBlock
 
 Returns the slot of the lowest confirmed block that has not been purged from the ledger
@@ -609,33 +636,59 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":{"identity": "2r1F4iWqVcb8M1DbAjQuFpebkQHY9hcVU4WuW2DJBppN"},"id":1}
 ```
 
-### getInflation
+### getInflationGovernor
 
-Returns the inflation configuration of the cluster
+Returns the current inflation governor
 
 #### Parameters:
 
-None
+* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
 
 #### Results:
 
-The result field will be an Inflation object with the following fields:
+The result field will be a JSON object with the following fields:
 
 * `initial: <f64>`, the initial inflation percentage from time 0
 * `terminal: <f64>`, terminal inflation percentage
 * `taper: <f64>`, rate per year at which inflation is lowered
 * `foundation: <f64>`, percentage of total inflation allocated to the foundation
 * `foundationTerm: <f64>`, duration of foundation pool inflation in years
-* `storage: <f64>`, percentage of total inflation allocated to storage rewards
 
 #### Example:
 
 ```bash
 // Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getInflation"}' http://localhost:8899
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getInflationGovernor"}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":{"foundation":0.05,"foundationTerm":7.0,"initial":0.15,"storage":0.1,"taper":0.15,"terminal":0.015},"id":1}
+{"jsonrpc":"2.0","result":{"foundation":0.05,"foundationTerm":7.0,"initial":0.15,"taper":0.15,"terminal":0.015},"id":1}
+```
+
+### getInflationRate
+
+Returns the specific inflation values for a particular epoch
+
+#### Parameters:
+
+* `<u64>` - (optional) Epoch, default is the current epoch
+
+#### Results:
+
+The result field will be a JSON object with the following fields:
+
+* `total: <f64>`, total inflation
+* `validator: <f64>`, inflation allocated to validators
+* `foundation: <f64>`, inflation allocated to the foundation
+* `epoch: <f64>`, epoch for which these values are valid
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getInflationRate"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":{"epoch":100,"foundation":0.001,"total":0.149,"validator":0.148},"id":1}
 ```
 
 ### getLargestAccounts
@@ -769,7 +822,7 @@ An RpcResponse containing a JSON object consisting of a string blockhash and Fee
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getRecentBlockhash"}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":{"blockhash":"CSymwgTNX1j3E4qhKfJAUE41nBWEwXufoYryPbkde5RR","feeCalculator":{"burnPercent":50,"lamportsPerSignature":5000,"maxLamportsPerSignature":100000,"minLamportsPerSignature":5000,"targetLamportsPerSignature":10000,"targetSignaturesPerSlot":20000}}},"id":1}
+{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":{"blockhash":"CSymwgTNX1j3E4qhKfJAUE41nBWEwXufoYryPbkde5RR","feeCalculator":{"lamportsPerSignature":5000}}},"id":1}
 ```
 
 ### getSignatureStatuses
@@ -862,93 +915,6 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":"ENvAW7JScgYq6o4zKZwewtkzzJgDzuJAFxYasvmEQdpS","id":1}
 ```
 
-### getSlotsPerSegment
-
-Returns the current storage segment size in terms of slots
-
-#### Parameters:
-
-* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `<u64>` - Number of slots in a storage segment
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getSlotsPerSegment"}' http://localhost:8899
-// Result
-{"jsonrpc":"2.0","result":1024,"id":1}
-```
-
-### getStoragePubkeysForSlot
-
-Returns the storage Pubkeys for a particular slot
-
-#### Parameters:
-
-None
-
-#### Results:
-
-An array of Pubkeys, as base-58 encoded strings
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getStoragePubkeysForSlot","params":[1]}' http://localhost:8899
- // Result
-{"jsonrpc":"2.0","result":["GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC"],"id":1}
-```
-
-### getStorageTurn
-
-Returns the current storage turn's blockhash and slot
-
-#### Parameters:
-
-None
-
-#### Results:
-
-A JSON object consisting of
-
-* `blockhash: <string>` - a Hash as base-58 encoded string indicating the blockhash of the turn slot
-* `slot: <u64>` - the current storage turn slot
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getStorageTurn"}' http://localhost:8899
- // Result
-{"jsonrpc":"2.0","result":{"blockhash": "GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC", "slot": 2048},"id":1}
-```
-
-### getStorageTurnRate
-
-Returns the current storage turn rate in terms of slots per turn
-
-#### Parameters:
-
-None
-
-#### Results:
-
-* `<u64>` - Number of slots in storage turn
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getStorageTurnRate"}' http://localhost:8899
- // Result
-{"jsonrpc":"2.0","result":1024,"id":1}
-```
-
 ### getSupply
 
 Returns information about the current supply.
@@ -1017,7 +983,7 @@ The result field will be a JSON object with the following fields:
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"solana-core": "1.2.0"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.2.1"},"id":1}
 ```
 
 ### getVoteAccounts
@@ -1099,11 +1065,20 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 
 ### sendTransaction
 
-Creates new transaction
+Submits a signed transaction to the cluster for processing.
+
+Before submitting, the following preflight checks are performed:
+1. The transaction signatures are verified
+2. The transaction is simulated against the latest max confirmed bank
+and on failure an error will be returned.  Preflight checks may be disabled if
+desired.
 
 #### Parameters:
 
 * `<string>` - fully-signed Transaction, as base-58 encoded string
+* `<object>` - (optional) Configuration object containing the following field:
+  * `skipPreflight: <bool>` - if true, skip the preflight transaction checks (default: false)
+
 
 #### Results:
 
@@ -1132,6 +1107,10 @@ Simulate sending a transaction
 #### Results:
 
 An RpcResponse containing a TransactionStatus object
+The result will be an RpcResponse JSON object with `value` set to a JSON object with the following fields:
+
+* `err: <object | null>` - Error if transaction failed, null if transaction succeeded. [TransactionError definitions](https://github.com/solana-labs/solana/blob/master/sdk/src/transaction.rs#L14)
+* `logs: <array | null>` - Array of log messages the transaction instructions output during execution, null if simulation failed before the transaction was able to execute (for example due to an invalid blockhash or signature verification failure)
 
 #### Example:
 

docs/src/apps/rent.md

@@ -0,0 +1,55 @@
+# Storage Rent for Accounts
+
+Keeping accounts alive on Solana incurs a storage cost called _rent_ because the cluster must actively maintain the data to process any future transactions on it. This is different from Bitcoin and Ethereum, where storing accounts doesn't incur any costs.
+
+The rent is debited from an account's balance by the runtime upon the first access (including the initial account creation) in the current epoch by transactions or once per an epoch if there are no transactions. The fee is currently a fixed rate, measured in bytes-times-epochs. The fee may change in the future.
+
+For the sake of simple rent calculation, rent is always collected for a single, full epoch. Rent is not pro-rated, meaning there are neither fees nor refunds for partial epochs. This means that, on account creation, the first rent collected isn't for the current partial epoch, but collected up front for the next full epoch. Subsequent rent collections are for further future epochs. On the other end, if the balance of an already-rent-collected account drops below another rent fee mid-epoch, the account will continue to exist through the current epoch and be purged immediately at the start of the upcoming epoch.
+
+Accounts can be exempt from paying rent if they maintain a minimum balance. This rent-exemption is described below.
+
+## Calculation of rent
+
+Note: The rent rate can change in the future.
+
+As of writing, the fixed rent fee is 19.055441478439427 lamports per byte-epoch on the testnet and mainnet-beta clusters. An [epoch](../terminology.md#epoch) is targeted to be 2 days (For devnet, the rent fee is 0.3608183131797095 lamports per byte-epoch with its 54m36s-long epoch).
+
+This value is calculated to target 0.01 SOL per mebibyte-day (exactly matching to 3.56 SOL per mebibyte-year):
+
+```
+Rent fee: 19.055441478439427 = 10_000_000 (0.01 SOL) * 365(approx. day in a year) / (1024 * 1024)(1 MiB) / (365.25/2)(epochs in 1 year)
+```
+
+And rent calculation is done with the `f64` precision and the final result is truncated to `u64` in lamports.
+
+The rent calculation includes account metadata (address, owner, lamports, etc) in the size of an account. Therefore the smallest an account can be for rent calculations is 128 bytes.
+
+For example, an account is created with the initial transfer of 10,000 lamports and no additional data. Rent is immediately debited from it on creation, resulting in a balance of 7,561 lamports:
+
+
+```
+Rent: 2,439 = 19.055441478439427 (rent rate) * 128 bytes (minimum account size) * 1 (epoch)
+Account Balance: 7,561 = 10,000 (transfered lamports) - 2,439 (this account's rent fee for an epoch)
+```
+
+The account balance will be reduced to 5,122 lamports at the next epoch even if there is no activity:
+
+```
+Account Balance: 5,122 = 7,561 (current balance) - 2,439 (this account's rent fee for an epoch)
+```
+
+Accordingly, a minimum-size account will be immediately removed after creation if the transferred lamports are less than or equal to 2,439.
+
+## Rent exemption
+
+Alternatively, an account can be made entirely exempt from rent collection by depositing at least 2 years-worth of rent. This is checked every time an account's balance is reduced and rent is immediately debited once the balance goes below the minimum amount.
+
+Program executable accounts are required by the runtime to be rent-exempt to avoid being purged.
+
+Note: Use the [`getMinimumBalanceForRentExemption` RPC endpoint](jsonrpc-api.md#getminimumbalanceforrentexemption) to calculate the minimum balance for a particular account size. The following calculation is illustrative only.
+
+For example, a program executable with the size of 15,000 bytes requires a balance of 105,290,880 lamports (=~ 0.105 SOL) to be rent-exempt:
+
+```
+105,290,880 = 19.055441478439427 (fee rate) * (128 + 15_000)(account size including metadata) * ((365.25/2) * 2)(epochs in 2 years)
+```

docs/src/clusters.md

@@ -44,6 +44,8 @@ $ solana-validator \
     --limit-ledger-size
 ```
 
+The `--trusted-validator`s is operated by Solana
+
 
 ## Testnet
 * Testnet is where we stress test recent release features on a live
@@ -72,6 +74,8 @@ $ solana-validator \
     --identity ~/validator-keypair.json \
     --vote-account ~/vote-account-keypair.json \
     --trusted-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \
+    --trusted-validator Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQAD \
+    --trusted-validator 9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv \
     --no-untrusted-rpc \
     --ledger ~/validator-ledger \
     --rpc-port 8899 \
@@ -82,6 +86,11 @@ $ solana-validator \
     --limit-ledger-size
 ```
 
+The identity of the `--trusted-validator`s are:
+* `5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on` - testnet.solana.com (Solana)
+* `Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQADN` - Certus One
+* `9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv` - Algo|Stake
+
 ## Mainnet Beta
 A permissionless, persistent cluster for early token holders and launch partners.
 Currently smart contracts, rewards, and inflation are disabled.
@@ -114,6 +123,8 @@ $ solana-validator \
     --dynamic-port-range 8000-8010 \
     --entrypoint mainnet-beta.solana.com:8001 \
     --expected-genesis-hash 5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d \
-    --expected-shred-version 54208 \
+    --expected-shred-version 64864 \
     --limit-ledger-size
 ```
+
+All four `--trusted-validator`s are operated by Solana

docs/src/implemented-proposals/cross-program-invocation.md

@@ -0,0 +1,95 @@
+# Cross-Program Invocation
+
+## Problem
+
+In today's implementation, a client can create a transaction that modifies two accounts, each owned by a separate on-chain program:
+
+```rust,ignore
+let message = Message::new(vec![
+    token_instruction::pay(&alice_pubkey),
+    acme_instruction::launch_missiles(&bob_pubkey),
+]);
+client.send_message(&[&alice_keypair, &bob_keypair], &message);
+```
+
+However, the current implementation does not allow the `acme` program to conveniently invoke `token` instructions on the client's behalf:
+
+```rust,ignore
+let message = Message::new(vec![
+    acme_instruction::pay_and_launch_missiles(&alice_pubkey, &bob_pubkey),
+]);
+client.send_message(&[&alice_keypair, &bob_keypair], &message);
+```
+
+Currently, there is no way to create instruction `pay_and_launch_missiles` that executes `token_instruction::pay` from the `acme` program. A possible workaround is to extend the `acme` program with the implementation of the `token` program and create `token` accounts with `ACME_PROGRAM_ID`, which the `acme` program is permitted to modify. With that workaround, `acme` can modify token-like accounts created by the `acme` program, but not token accounts created by the `token` program.
+
+## Proposed Solution
+
+The goal of this design is to modify Solana's runtime such that an on-chain program can invoke an instruction from another program.
+
+Given two on-chain programs `token` and `acme`, each implementing instructions `pay()` and `launch_missiles()` respectively, we would ideally like to implement the `acme` module with a call to a function defined in the `token` module:
+
+```rust,ignore
+mod acme {
+    use token;
+
+    fn launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
+        ...
+    }
+
+    fn pay_and_launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
+        token::pay(&keyed_accounts[1..])?;
+
+        launch_missiles(keyed_accounts)?;
+    }
+```
+
+The above code would require that the `token` crate be dynamically linked so that a custom linker could intercept calls and validate accesses to `keyed_accounts`. Even though the client intends to modify both `token` and `acme` accounts, only `token` program is permitted to modify the `token` account, and only the `acme` program is allowed to modify the `acme` account.
+
+Backing off from that ideal direct cross-program call, a slightly more verbose solution is to allow `acme` to invoke `token` by issuing a token instruction via the runtime.
+
+```rust,ignore
+mod acme {
+    use token_instruction;
+
+    fn launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
+        ...
+    }
+
+    fn pay_and_launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
+        let alice_pubkey = keyed_accounts[1].key;
+        let instruction = token_instruction::pay(&alice_pubkey);
+        invoke(&instruction, accounts)?;
+
+        launch_missiles(keyed_accounts)?;
+    }
+```
+
+`invoke()` is built into Solana's runtime and is responsible for routing the given instruction to the `token` program via the instruction's `program_id` field.
+
+Before invoking `pay()`, the runtime must ensure that `acme` didn't modify any accounts owned by `token`.  It does this by applying the runtime's policy to the current state of the accounts at the time `acme` calls `invoke` vs. the initial state of the accounts at the beginning of the `acme`'s instruction.  After `pay()` completes, the runtime must again ensure that `token` didn't modify any accounts owned by `acme` by again applying the runtime's policy, but this time with the `token` program ID.  Lastly, after `pay_and_launch_missiles()` completes, the runtime must apply the runtime policy one more time, where it normally would, but using all updated `pre_*` variables. If executing `pay_and_launch_missiles()` up to `pay()` made no invalid account changes, `pay()` made no invalid changes, and executing from `pay()` until `pay_and_launch_missiles()` returns made no invalid changes, then the runtime can transitively assume `pay_and_launch_missiles()` as whole made no invalid account changes, and therefore commit all these account modifications.
+
+### Instructions that require privileges
+
+The runtime uses the privileges granted to the caller program to determine what privileges can be extended to the callee.  Privileges in this context refer to signers and writable accounts.  For example, if the instruction the caller is processing contains a signer or writable account, then the caller can invoke an instruction that also contains that signer and/or writable account.
+
+This privilege extension relies on the fact that programs are immutable.  In the case of the `acme` program, the runtime can safely treat the transaction's signature as a signature of a `token` instruction.  When the runtime sees the `token` instruction references `alice_pubkey`, it looks up the key in the `acme` instruction to see if that key corresponds to a signed account. In this case, it does and thereby authorizes the `token` program to modify Alice's account.
+
+### Program signed accounts
+
+Programs can issue instructions that contain signed accounts that were not signed in the original transaction by
+using [Program derived addresses](program-derived-addresses.md).
+
+To sign an account with program derived addresses, a program may `invoke_signed()`.
+
+```rust,ignore
+        invoke_signed(
+            &instruction,
+            accounts,
+            &[&["First addresses seed"],
+              &["Second addresses first seed", "Second addresses second seed"]],
+        )?;
+
+### Reentrancy
+
+Reentrancy is currently limited to direct self recursion capped at a fixed depth.  This restriction prevents situations where a program might invoke another from an intermediary state without the knowledge that it might later be called back into.  Direct recursion gives the program full control of its state at the point that it gets called back.

docs/src/implemented-proposals/ed_overview/ed_storage_rent_economics.md

@@ -10,7 +10,7 @@ With this approach, accounts with two-years worth of rent deposits secured are e
 
 Method 2: Pay per byte
 
-If an account has less than two-years worth of deposited rent the network charges rent on a per-epoch basis, in credit for the next epoch (but in arrears when necessary). This rent is deducted at a rate specified in genesis, in lamports per kilobyte-year.
+If an account has less than two-years worth of deposited rent the network charges rent on a per-epoch basis, in credit for the next epoch. This rent is deducted at a rate specified in genesis, in lamports per kilobyte-year.
 
 For information on the technical implementation details of this design, see the [Rent](../rent.md) section.
 

docs/src/implemented-proposals/program-derived-addresses.md

@@ -0,0 +1,154 @@
+# Program Derived Addresses
+
+## Problem
+
+Programs cannot generate signatures when issuing instructions to
+other programs as defined in the [Cross-Program Invocations](cross-program-invocation.md)
+design.
+
+The lack of programmatic signature generation limits the kinds of programs
+that can be implemented in Solana.  A program may be given the
+authority over an account and later want to transfer that authority to another.
+This is impossible today because the program cannot act as the signer in the transaction that gives authority.
+
+For example, if two users want
+to make a wager on the outcome of a game in Solana, they must each
+transfer their wager's assets to some intermediary that will honor
+their agreement.  Currently, there is no way to implement this intermediary
+as a program in Solana because the intermediary program cannot transfer the
+assets to the winner.
+
+This capability is necessary for many DeFi applications since they
+require assets to be transferred to an escrow agent until some event
+occurs that determines the new owner.
+
+* Decentralized Exchanges that transfer assets between matching bid and
+ask orders.
+
+* Auctions that transfer assets to the winner.
+
+* Games or prediction markets that collect and redistribute prizes to
+the winners.
+
+## Proposed Solution
+
+The key to the design is two-fold:
+
+1. Allow programs to control specific addresses, called Program-Addresses, in such a way that no external
+user can generate valid transactions with signatures for those
+addresses.
+
+2. Allow programs to programmatically sign for Program-Addresses that are
+present in instructions invoked via [Cross-Program Invocations](cross-program-invocation.md).
+
+Given the two conditions, users can securely transfer or assign
+the authority of on-chain assets to Program-Addresses and the program
+can then assign that authority elsewhere at its discretion.
+
+### Private keys for Program Addresses
+
+A Program -Address has no private key associated with it, and generating
+a signature for it is impossible.  While it has no private key of
+its own, it can issue an instruction that includes the Program-Address as a signer.
+
+### Hash-based generated Program Addresses
+
+All 256-bit values are valid ed25519 curve points and valid ed25519 public
+keys.  All are equally secure and equally as hard to break.
+Based on this assumption, Program Addresses can be deterministically
+derived from a base seed using a 256-bit preimage resistant hash function.
+
+Deterministic Program Addresses for programs follow a similar derivation
+path as Accounts created with `SystemInstruction::CreateAccountWithSeed`
+which is implemented with `system_instruction::create_address_with_seed`.
+
+For reference that implementation is as follows:
+
+```rust,ignore
+pub fn create_address_with_seed(
+    base: &Pubkey,
+    seed: &str,
+    program_id: &Pubkey,
+) -> Result<Pubkey, SystemError> {
+    if seed.len() > MAX_ADDRESS_SEED_LEN {
+        return Err(SystemError::MaxSeedLengthExceeded);
+    }
+
+    Ok(Pubkey::new(
+        hashv(&[base.as_ref(), seed.as_ref(), program_id.as_ref()]).as_ref(),
+    ))
+}
+```
+
+Programs can deterministically derive any number of addresses by
+using keywords.  These keywords can symbolically identify how the addresses are used.
+
+```rust,ignore
+//! Generate a derived program address
+//!     * seeds, symbolic keywords used to derive the key
+//!     * owner, program that the key is derived for
+pub fn create_program_address(seeds: &[&str], owner: &Pubkey) -> Result<Pubkey, PubkeyError> {
+  let mut hasher = Hasher::default();
+  for seed in seeds.iter() {
+      if seed.len() > MAX_SEED_LEN {
+          return Err(PubkeyError::MaxSeedLengthExceeded);
+      }
+      hasher.hash(seed.as_ref());
+  }
+  hasher.hashv(&[owner.as_ref(), "ProgramDerivedAddress".as_ref()]);
+
+  Ok(Pubkey::new(hashv(&[hasher.result().as_ref()]).as_ref()))
+}
+```
+
+### Using Program Addresses
+
+Clients can use the `create_program_address` function to generate
+a destination address.
+
+```rust,ignore
+//deterministically derive the escrow key
+let escrow_pubkey = create_program_address(&[&["escrow"]], &escrow_program_id);
+let message = Message::new(vec![
+    token_instruction::transfer(&alice_pubkey, &escrow_pubkey, 1),
+]);
+//transfer 1 token to escrow
+client.send_message(&[&alice_keypair], &message);
+```
+
+Programs can use the same function to generate the same address.
+In the function below the program issues a `token_instruction::transfer` from
+Program Address as if it had the private key to sign the transaction.
+
+```rust,ignore
+fn transfer_one_token_from_escrow(
+    program_id: &Pubkey,
+    keyed_accounts: &[KeyedAccount]
+) -> Result<()> {
+
+    // User supplies the destination
+    let alice_pubkey = keyed_accounts[1].unsigned_key();
+
+    // Deterministically derive the escrow pubkey.
+    let escrow_pubkey = create_program_address(&[&["escrow"]], program_id);
+
+    // Create the transfer instruction
+    let instruction = token_instruction::transfer(&escrow_pubkey, &alice_pubkey, 1);
+
+    // The runtime deterministically derives the key from the currently
+    // executing program ID and the supplied keywords.
+    // If the derived key matches a key marked as signed in the instruction
+    // then that key is accepted as signed.
+    invoke_signed(&instruction,  &[&["escrow"]])?
+}
+```
+
+### Instructions that require signers
+
+The addresses generated with `create_program_address` are indistinguishable
+from any other public key.  The only way for the runtime to verify that the
+address belongs to a program is for the program to supply the keywords used
+to generate the address.
+
+The runtime will internally call `create_program_address`, and compare the
+result against the addresses supplied in the instruction.

docs/src/implemented-proposals/rent.md

@@ -4,11 +4,17 @@ Accounts on Solana may have owner-controlled state \(`Account::data`\) that's se
 
 ## Two-tiered rent regime
 
-Accounts which maintain a minimum balance equivalent to 2 years of rent payments are exempt. Accounts whose balance falls below this threshold are charged rent at a rate specified in genesis, in lamports per kilobyte-year. The network charges rent on a per-epoch basis, in credit for the next epoch \(but in arrears when necessary\), and `Account::rent_epoch` keeps track of the next time rent should be collected from the account.
+Accounts which maintain a minimum balance equivalent to 2 years of rent payments are exempt. The _2 years_ is drawn from the fact hardware cost drops by 50% in price every 2 years and the resulting convergence due to being a geometric series. Accounts whose balance falls below this threshold are charged rent at a rate specified in genesis, in lamports per byte-year. The network charges rent on a per-epoch basis, in credit for the next epoch, and `Account::rent_epoch` keeps track of the next time rent should be collected from the account.
 
-## Collecting rent
+Currently, the rent cost is fixed at the genesis. However, it's anticipated to be dynamic, reflecting the underlying hardware storage cost at the time. So the price is generally expected to decrease as the hardware cost declines as the technology advances.
 
-Rent is due at account creation time for one epoch's worth of time, and the new account has `Account::rent_epoch` of `current_epoch + 1`. After that, the bank deducts rent from accounts during normal transaction processing as part of the load phase.
+## Timings of collecting rent
+
+There are two timings of collecting rent from accounts: \(1\) when referenced by a transaction, \(2\) periodically once an epoch. \(1\) includes the transaction to create the new account itself, and it happens during the normal transaction processing by the bank as part of the load phase. \(2\) exists to ensure to collect rents from stale accounts, which aren't referenced in recent epochs at all. \(2\) requires the whole scan of accounts and is spread over an epoch based on account address prefix to avoid load spikes due to this rent collection.
+
+## Actual processing of collecting rent
+
+Rent is due for one epoch's worth of time, and accounts always have `Account::rent_epoch` of `current_epoch + 1`.
 
 If the account is in the exempt regime, `Account::rent_epoch` is simply pushed to `current_epoch + 1`.
 
@@ -18,13 +24,19 @@ Accounts whose balance is insufficient to satisfy the rent that would be due sim
 
 A percentage of the rent collected is destroyed. The rest is distributed to validator accounts by stake weight, a la transaction fees, at the end of every slot.
 
-## Read-only accounts
+Finally, rent collection happens according to the protocol-level account updates like the rent distribution to validators, meaning there is no corresponding transaction for rent deductions. So, rent collection is rather invisible, only implicitly observable by a recent transaction or predetermined timing given its account address prefix.
 
-Read-only accounts are not being charged rent in current implementation.
+## Design considerations
 
-## Design considerations, others considered
+### Current design rationale
 
-Under this design, it is possible to have accounts that linger, never get touched, and never have to pay rent. `Noop` instructions that name these accounts can be used to "garbage collect", but it'd also be possible for accounts that never get touched to migrate out of a validator's working set, thereby reducing memory consumption and obviating the need to charge rent.
+Under the preceding design, it is NOT possible to have accounts that linger, never get touched, and never have to pay rent. Accounts are always pay rent exactly once for each epoch, except rent-exempt, sysvar and executable accounts.
+
+This is intended design choice. Otherwise, it would be possible to trigger unauthorized rent collection with `Noop` instruction by anyone who may unfairly profit the rent (a leader at the moment) or save the rent given anticipated fluctuating rent cost.
+
+As another side-effect of this choice, also note that this periodic rent collection effectively forces validator not to store stale accounts into a cold storage optimistically and save the storage cost, which is unfavorable for account owners and may cause transactions on them to stall longer than the others. On the flip side, this prevents malicious users from piling significant amount of garbage accounts, burdening validators.
+
+As the overall consequence of this design, all of accounts is stored equally as validator's working set with same performance characteristics, straightly reflecting the uniform rent pricing structure.
 
 ### Ad-hoc collection
 
@@ -45,8 +57,3 @@ Collecting rent via a system instruction was considered, as it would naturally h
 * it would have adversely affected network throughput
 * it would require special-casing by the runtime, as accounts with non-SystemProgram owners may be debited by this instruction
 * someone would have to issue the transactions
-
-### Account scans on every epoch
-
-Scanning the entire Bank for accounts that owe rent at the beginning of each epoch was considered. This would have been an expensive operation, and would require that the entire current state of the network be present on every validator at the beginning of each epoch.
-

docs/src/integrations/exchange.md

@@ -0,0 +1,390 @@
+# Add Solana to Your Exchange
+
+This guide describes how to add Solana's native token SOL to your cryptocurrency
+exchange.
+
+## Node Setup
+
+We highly recommend setting up at least two of your own Solana api nodes to
+give you a trusted entrypoint to the network, allow you full control over how
+much data is retained, and ensure you do not miss any data if one node fails.
+
+To run an api node:
+
+1. [Install the Solana command-line tool suite](../cli/install-solana-cli-tools.md)
+2. Boot the node with at least the following parameters:
+```bash
+solana-validator \
+  --ledger <LEDGER_PATH> \
+  --entrypoint <CLUSTER_ENTRYPOINT> \
+  --expected-genesis-hash <EXPECTED_GENESIS_HASH> \
+  --expected-shred-version <EXPECTED_SHRED_VERSION> \
+  --rpc-port 8899 \
+  --no-voting \
+  --enable-rpc-transaction-history \
+  --limit-ledger-size <SHRED_COUNT> \
+  --trusted-validator <VALIDATOR_ADDRESS> \
+  --no-untrusted-rpc
+```
+
+  Customize `--ledger` to your desired ledger storage location, and `--rpc-port` to the port you want to expose.
+
+  The `--entrypoint`, `--expected-genesis-hash`, and `--expected-shred-version` parameters are all specific to the cluster you are joining. The shred version will change on any hard forks in the cluster, so including `--expected-shred-version` ensures you are receiving current data from the cluster you expect.
+  [Current parameters for Mainnet Beta](../clusters.md#example-solana-validator-command-line-2)
+
+  The `--limit-ledger-size` parameter allows you to specify how many ledger [shreds](../terminology.md#shred) your node retains on disk. If you do not include this parameter, the ledger will keep the entire ledger until it runs out of disk space. A larger value like `--limit-ledger-size 250000000000` is good for a couple days
+
+  Specifying one or more `--trusted-validator` parameters can protect you from booting from a malicious snapshot. [More on the value of booting with trusted validators](../running-validator/validator-start.md#trusted-validators)
+
+  Optional parameters to consider:
+  - `--private-rpc` prevents your RPC port from being published for use by other nodes
+  - `--rpc-bind-address` allows you to specify a different IP address to bind the RPC port
+
+### Automatic Restarts
+
+We recommend configuring each of your nodes to restart automatically on exit, to
+ensure you miss as little data as possible. Running the solana software as a
+systemd service is one great option.
+
+### Ledger Continuity
+
+By default, each of your nodes will boot from a snapshot provided by one of your
+trusted validators. This snapshot reflects the current state of the chain, but
+does not contain the complete historical ledger. If one of your node exits and
+boots from a new snapshot, there may be a gap in the ledger on that node. In
+order to prevent this issue, add the `--no-snapshot-fetch` parameter to your
+`solana-validator` command to receive historical ledger data instead of a
+snapshot.
+
+If you pass the `--no-snapshot-fetch` parameter on your initial boot, it will
+take your node a very long time to catch up. We recommend booting from a
+snapshot first, and then using the `--no-snapshot-fetch` parameter for reboots.
+
+It is important to note that the amount of historical ledger available to your
+nodes is limited to what your trusted validators retain. You will need to ensure
+your nodes do not experience downtimes longer than this span, if ledger
+continuity is crucial for you.
+
+## Setting up Deposit Accounts
+
+Solana accounts do not require any on-chain initialization; once they contain
+some SOL, they exist. To set up a deposit account for your exchange, simply
+generate a Solana keypair using any of our [wallet tools](../wallet-guide/cli.md).
+
+We recommend using a unique deposit account for each of your users.
+
+Solana accounts are charged [rent](../apps/rent.md) on creation and once per
+epoch, but they can be made rent-exempt if they contain 2-years worth of rent in
+SOL. In order to find the minimum rent-exempt balance for your deposit accounts,
+query the
+[`getMinimumBalanceForRentExemption` endpoint](../apps/jsonrpc-api.md#getminimumbalanceforrentexemption):
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getMinimumBalanceForRentExemption","params":[0]}' localhost:8899
+
+{"jsonrpc":"2.0","result":890880,"id":1}
+```
+
+### Offline Accounts
+
+You may wish to keep the keys for one or more collection accounts offline for
+greater security. If so, you will need to move SOL to hot accounts using our
+[offline methods](../offline-signing/README.md).
+
+## Listening for Deposits
+
+When a user wants to deposit SOL into your exchange, instruct them to send a
+transfer to the appropriate deposit address.
+
+### Poll for Blocks
+
+The easiest way to track all the deposit accounts for your exchange is to poll
+for each confirmed block and inspect for addresses of interest, using the
+JSON-RPC service of your Solana api node.
+
+* To identify which blocks are available, send a [`getConfirmedBlocks` request](../apps/jsonrpc-api.md#getconfirmedblocks),
+passing the last block you have already processed as the start-slot parameter:
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedBlocks","params":[5]}' localhost:8899
+
+{"jsonrpc":"2.0","result":[5,6,8,9,11],"id":1}
+```
+Not every slot produces a block, so there may be gaps in the sequence of integers.
+
+* For each block, request its contents with a [`getConfirmedBlock` request](../apps/jsonrpc-api.md#getconfirmedblock):
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedBlock","params":[5, "json"]}' localhost:8899
+
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "blockhash": "2WcrsKSVANoe6xQHKtCcqNdUpCQPQ3vb6QTgi1dcE2oL",
+    "parentSlot": 4,
+    "previousBlockhash": "7ZDoGW83nXgP14vnn9XhGSaGjbuLdLWkQAoUQ7pg6qDZ",
+    "rewards": [],
+    "transactions": [
+      {
+        "meta": {
+          "err": null,
+          "fee": 5000,
+          "postBalances": [
+            2033973061360,
+            218099990000,
+            42000000003
+          ],
+          "preBalances": [
+            2044973066360,
+            207099990000,
+            42000000003
+          ],
+          "status": {
+            "Ok": null
+          }
+        },
+        "transaction": {
+          "message": {
+            "accountKeys": [
+              "Bbqg1M4YVVfbhEzwA9SpC9FhsaG83YMTYoR4a8oTDLX",
+              "47Sbuv6jL7CViK9F2NMW51aQGhfdpUu7WNvKyH645Rfi",
+              "11111111111111111111111111111111"
+            ],
+            "header": {
+              "numReadonlySignedAccounts": 0,
+              "numReadonlyUnsignedAccounts": 1,
+              "numRequiredSignatures": 1
+            },
+            "instructions": [
+              {
+                "accounts": [
+                  0,
+                  1
+                ],
+                "data": "3Bxs3zyH82bhpB8j",
+                "programIdIndex": 2
+              }
+            ],
+            "recentBlockhash": "7GytRgrWXncJWKhzovVoP9kjfLwoiuDb3cWjpXGnmxWh"
+          },
+          "signatures": [
+            "dhjhJp2V2ybQGVfELWM1aZy98guVVsxRCB5KhNiXFjCBMK5KEyzV8smhkVvs3xwkAug31KnpzJpiNPtcD5bG1t6"
+          ]
+        }
+      }
+    ]
+  },
+  "id": 1
+}
+```
+
+The `preBalances` and `postBalances` fields allow you to track the balance
+changes in every account without having to parse the entire transaction. They
+list the starting and ending balances of each account in
+[lamports](../terminology.md#lamport), indexed to the `accountKeys` list. For
+example, if the deposit address if interest is
+`47Sbuv6jL7CViK9F2NMW51aQGhfdpUu7WNvKyH645Rfi`, this transaction represents a
+transfer of 218099990000 - 207099990000 = 11000000000 lamports = 11 SOL
+
+If you need more information about the transaction type or other specifics, you
+can request the block from RPC in binary format, and parse it using either our
+[Rust SDK](https://github.com/solana-labs/solana) or
+[Javascript SDK](https://github.com/solana-labs/solana-web3.js).
+
+### Address History
+
+You can also query the transaction history of a specific address.
+
+* Send a [`getConfirmedSignaturesForAddress`](../apps/jsonrpc-api.md#getconfirmedsignaturesforaddress)
+request to the api node, specifying a range of recent slots:
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedSignaturesForAddress","params":["6H94zdiaYfRfPfKjYLjyr2VFBg6JHXygy84r3qhc3NsC", 0, 10]}' localhost:8899
+
+{
+  "jsonrpc": "2.0",
+  "result": [
+    "35YGay1Lwjwgxe9zaH6APSHbt9gYQUCtBWTNL3aVwVGn9xTFw2fgds7qK5AL29mP63A9j3rh8KpN1TgSR62XCaby",
+    "4bJdGN8Tt2kLWZ3Fa1dpwPSEkXWWTSszPSf1rRVsCwNjxbbUdwTeiWtmi8soA26YmwnKD4aAxNp8ci1Gjpdv4gsr",
+    "dhjhJp2V2ybQGVfELWM1aZy98guVVsxRCB5KhNiXFjCBMK5KEyzV8smhkVvs3xwkAug31KnpzJpiNPtcD5bG1t6"
+  ],
+  "id": 1
+}
+```
+
+* For each signature returned, get the transaction details by sending a
+[`getConfirmedTransaction`](../apps/jsonrpc-api.md#getconfirmedtransaction) request:
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedTransaction","params":["dhjhJp2V2ybQGVfELWM1aZy98guVVsxRCB5KhNiXFjCBMK5KEyzV8smhkVvs3xwkAug31KnpzJpiNPtcD5bG1t6", "json"]}' localhost:8899
+
+// Result
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "slot": 5,
+    "transaction": {
+      "message": {
+        "accountKeys": [
+          "Bbqg1M4YVVfbhEzwA9SpC9FhsaG83YMTYoR4a8oTDLX",
+          "47Sbuv6jL7CViK9F2NMW51aQGhfdpUu7WNvKyH645Rfi",
+          "11111111111111111111111111111111"
+        ],
+        "header": {
+          "numReadonlySignedAccounts": 0,
+          "numReadonlyUnsignedAccounts": 1,
+          "numRequiredSignatures": 1
+        },
+        "instructions": [
+          {
+            "accounts": [
+              0,
+              1
+            ],
+            "data": "3Bxs3zyH82bhpB8j",
+            "programIdIndex": 2
+          }
+        ],
+        "recentBlockhash": "7GytRgrWXncJWKhzovVoP9kjfLwoiuDb3cWjpXGnmxWh"
+      },
+      "signatures": [
+        "dhjhJp2V2ybQGVfELWM1aZy98guVVsxRCB5KhNiXFjCBMK5KEyzV8smhkVvs3xwkAug31KnpzJpiNPtcD5bG1t6"
+      ]
+    },
+    "meta": {
+      "err": null,
+      "fee": 5000,
+      "postBalances": [
+        2033973061360,
+        218099990000,
+        42000000003
+      ],
+      "preBalances": [
+        2044973066360,
+        207099990000,
+        42000000003
+      ],
+      "status": {
+        "Ok": null
+      }
+    }
+  },
+  "id": 1
+}
+```
+
+## Sending Withdrawals
+
+To accommodate a user's request to withdraw SOL, you must generate a Solana
+transfer transaction, and send it to the api node to be forwarded to your
+cluster.
+
+### Synchronous
+
+Sending a synchronous transfer to the Solana cluster allows you to easily ensure
+that a transfer is successful and finalized by the cluster.
+
+Solana's command-line tool offers a simple command, `solana transfer`, to
+generate, submit, and confirm transfer transactions. By default, this method
+will wait and track progress on stderr until the transaction has been finalized
+by the cluster. If the transaction fails, it will report any transaction errors.
+
+```bash
+solana transfer <USER_ADDRESS> <AMOUNT> --keypair <KEYPAIR> --url http://localhost:8899
+```
+
+The [Solana Javascript SDK](https://github.com/solana-labs/solana-web3.js)
+offers a similar approach for the JS ecosystem. Use the `SystemProgram` to build
+a transfer transaction, and submit it using the `sendAndConfirmTransaction`
+method.
+
+### Asynchronous
+
+For greater flexibility, you can submit withdrawal transfers asynchronously. In
+these cases, it is your responsibility to verify that the transaction succeeded
+and was finalized by the cluster.
+
+**Note:** Each transaction contains a [recent blockhash](../transaction.md#blockhash-format)
+to indicate its liveness. It is **critical** to wait until this blockhash
+expires before retrying a withdrawal transfer that does not appear to have been
+confirmed or finalized by the cluster. Otherwise, you risk a double spend. See
+more on [blockhash expiration](#blockhash-expiration) below.
+
+First, get a recent blockhash using the [`getFees` endpoint](../apps/jsonrpc-api.md#getfees)
+or the CLI command:
+```bash
+solana fees --url http://localhost:8899
+```
+
+In the command-line tool, pass the `--no-wait` argument to send a transfer
+asynchronously, and include your recent blockhash with the `--blockhash` argument:
+
+```bash
+solana transfer <USER_ADDRESS> <AMOUNT> --no-wait --blockhash <RECENT_BLOCKHASH> --keypair <KEYPAIR> --url http://localhost:8899
+```
+
+You can also build, sign, and serialize the transaction manually, and fire it off to
+the cluster using the JSON-RPC [`sendTransaction` endpoint](../apps/jsonrpc-api.md#sendtransaction).
+
+#### Transaction Confirmations & Finality
+
+Get the status of a batch of transactions using the
+[`getSignatureStatuses` JSON-RPC endpoint](../apps/jsonrpc-api.md#getsignaturestatuses).
+The `confirmations` field reports how many
+[confirmed blocks](../terminology.md#confirmed-block) have elapsed since the
+transaction was processed. If `confirmations: null`, it is [finalized](../terminology.md#finality).
+
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getSignatureStatuses", "params":[["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW", "5j7s6NiJS3JAkvgkoc18WVAsiSaci2pxB2A6ueCJP4tprA2TFg9wSyTLeYouxPBJEMzJinENTkpA52YStRW5Dia7"]]}' http://localhost:8899
+
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "context": {
+      "slot": 82
+    },
+    "value": [
+      {
+        "slot": 72,
+        "confirmations": 10,
+        "err": null,
+        "status": {
+          "Ok": null
+        }
+      },
+      {
+        "slot": 48,
+        "confirmations": null,
+        "err": null,
+        "status": {
+          "Ok": null
+        }
+      }
+    ]
+  },
+  "id": 1
+}
+```
+
+#### Blockhash Expiration
+
+When you request a recent blockhash for your withdrawal transaction using the
+[`getFees` endpoint](../apps/jsonrpc-api.md#getfees) or `solana fees`, the
+response will include the `lastValidSlot`, the last slot in which the blockhash
+will be valid. You can check the cluster slot with a
+[`getSlot` query](../apps/jsonrpc-api.md#getslot); once the cluster slot is
+greater than `lastValidSlot`, the withdrawal transaction using that blockhash
+should never succeed.
+
+You can also doublecheck whether a particular blockhash is still valid by sending a
+[`getFeeCalculatorForBlockhash`](../apps/jsonrpc-api.md#getfeecalculatorforblockhash)
+request with the blockhash as a parameter. If the response value is null, the
+blockhash is expired, and the withdrawal transaction should never succeed.
+
+## Testing the Integration
+
+Be sure to test your complete workflow on Solana devnet and testnet
+[clusters](../clusters.md) before moving to production on mainnet-beta. Devnet
+is the most open and flexible, and ideal for initial development, while testnet
+offers more realistic cluster configuration. Devnet features a token faucet, but
+you will need to request some testnet SOL to get going on testnet.

docs/src/proposals/cross-program-invocation.md

@@ -1,71 +0,0 @@
-# Cross-Program Invocation
-
-## Problem
-
-In today's implementation a client can create a transaction that modifies two accounts, each owned by a separate on-chain program:
-
-```text
-let message = Message::new(vec![
-    token_instruction::pay(&alice_pubkey),
-    acme_instruction::launch_missiles(&bob_pubkey),
-]);
-client.send_message(&[&alice_keypair, &bob_keypair], &message);
-```
-
-The current implementation does not, however, allow the `acme` program to conveniently invoke `token` instructions on the client's behalf:
-
-```text
-let message = Message::new(vec![
-    acme_instruction::pay_and_launch_missiles(&alice_pubkey, &bob_pubkey),
-]);
-client.send_message(&[&alice_keypair, &bob_keypair], &message);
-```
-
-Currently, there is no way to create instruction `pay_and_launch_missiles` that executes `token_instruction::pay` from the `acme` program. The workaround is to extend the `acme` program with the implementation of the `token` program, and create `token` accounts with `ACME_PROGRAM_ID`, which the `acme` program is permitted to modify. With that workaround, `acme` can modify token-like accounts created by the `acme` program, but not token accounts created by the `token` program.
-
-## Proposed Solution
-
-The goal of this design is to modify Solana's runtime such that an on-chain program can invoke an instruction from another program.
-
-Given two on-chain programs `token` and `acme`, each implementing instructions `pay()` and `launch_missiles()` respectively, we would ideally like to implement the `acme` module with a call to a function defined in the `token` module:
-
-```text
-use token;
-
-fn launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
-    ...
-}
-
-fn pay_and_launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
-    token::pay(&keyed_accounts[1..])?;
-
-    launch_missiles(keyed_accounts)?;
-}
-```
-
-The above code would require that the `token` crate be dynamically linked, so that a custom linker could intercept calls and validate accesses to `keyed_accounts`. That is, even though the client intends to modify both `token` and `acme` accounts, only `token` program is permitted to modify the `token` account, and only the `acme` program is permitted to modify the `acme` account.
-
-Backing off from that ideal cross-program call, a slightly more verbose solution is to expose token's existing `process_instruction()` entrypoint to the acme program:
-
-```text
-use token_instruction;
-
-fn launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
-    ...
-}
-
-fn pay_and_launch_missiles(keyed_accounts: &[KeyedAccount]) -> Result<()> {
-    let alice_pubkey = keyed_accounts[1].key;
-    let instruction = token_instruction::pay(&alice_pubkey);
-    process_instruction(&instruction)?;
-
-    launch_missiles(keyed_accounts)?;
-}
-```
-
-where `process_instruction()` is built into Solana's runtime and responsible for routing the given instruction to the `token` program via the instruction's `program_id` field. Before invoking `pay()`, the runtime must also ensure that `acme` didn't modify any accounts owned by `token`. It does this by calling `runtime::verify_account_changes()` and then afterward updating all the `pre_*` variables to tentatively commit `acme`'s account modifications. After `pay()` completes, the runtime must again ensure that `token` didn't modify any accounts owned by `acme`. It should call `verify_account_changes()` again, but this time with the `token` program ID. Lastly, after `pay_and_launch_missiles()` completes, the runtime must call `verify_account_changes()` one more time, where it normally would, but using all updated `pre_*` variables. If executing `pay_and_launch_missiles()` up to `pay()` made no invalid account changes, `pay()` made no invalid changes, and executing from `pay()` until `pay_and_launch_missiles()` returns made no invalid changes, then the runtime can transitively assume `pay_and_launch_missiles()` as whole made no invalid account changes, and therefore commit all account modifications.
-
-### Setting `KeyedAccount.is_signer`
-
-When `process_instruction()` is invoked, the runtime must create a new `KeyedAccounts` parameter using the signatures from the _original_ transaction data. Since the `token` program is immutable and existed on-chain prior to the `acme` program, the runtime can safely treat the transaction signature as a signature of a transaction with a `token` instruction. When the runtime sees the given instruction references `alice_pubkey`, it looks up the key in the transaction to see if that key corresponds to a transaction signature. In this case it does and so sets `KeyedAccount.is_signer`, thereby authorizing the `token` program to modify Alice's account.
-

docs/src/proposals/ledger-replication-to-implement.md

@@ -0,0 +1,415 @@
+# Ledger Replication
+
+Note: this ledger replication solution was partially implemented, but not
+completed. The partial implementation was removed by
+https://github.com/solana-labs/solana/pull/9992 in order to prevent the security
+risk of unused code. The first part of this design document reflects the
+once-implemented parts of ledger replication. The
+[second part of this document](#ledger-replication-not-implemented) describes the
+parts of the solution never implemented.
+
+## Proof of Replication
+
+At full capacity on a 1gbps network solana will generate 4 petabytes of data per year. To prevent the network from centralizing around validators that have to store the full data set this protocol proposes a way for mining nodes to provide storage capacity for pieces of the data.
+
+The basic idea to Proof of Replication is encrypting a dataset with a public symmetric key using CBC encryption, then hash the encrypted dataset. The main problem with the naive approach is that a dishonest storage node can stream the encryption and delete the data as it's hashed. The simple solution is to periodically regenerate the hash based on a signed PoH value. This ensures that all the data is present during the generation of the proof and it also requires validators to have the entirety of the encrypted data present for verification of every proof of every identity. So the space required to validate is `number_of_proofs * data_size`
+
+## Optimization with PoH
+
+Our improvement on this approach is to randomly sample the encrypted segments faster than it takes to encrypt, and record the hash of those samples into the PoH ledger. Thus the segments stay in the exact same order for every PoRep and verification can stream the data and verify all the proofs in a single batch. This way we can verify multiple proofs concurrently, each one on its own CUDA core. The total space required for verification is `1_ledger_segment + 2_cbc_blocks * number_of_identities` with core count equal to `number_of_identities`. We use a 64-byte chacha CBC block size.
+
+## Network
+
+Validators for PoRep are the same validators that are verifying transactions. If an archiver can prove that a validator verified a fake PoRep, then the validator will not receive a reward for that storage epoch.
+
+Archivers are specialized _light clients_. They download a part of the ledger \(a.k.a Segment\) and store it, and provide PoReps of storing the ledger. For each verified PoRep archivers earn a reward of sol from the mining pool.
+
+## Constraints
+
+We have the following constraints:
+
+* Verification requires generating the CBC blocks. That requires space of 2
+
+  blocks per identity, and 1 CUDA core per identity for the same dataset. So as
+
+  many identities at once should be batched with as many proofs for those
+
+  identities verified concurrently for the same dataset.
+
+* Validators will randomly sample the set of storage proofs to the set that
+
+  they can handle, and only the creators of those chosen proofs will be
+
+  rewarded. The validator can run a benchmark whenever its hardware configuration
+
+  changes to determine what rate it can validate storage proofs.
+
+## Validation and Replication Protocol
+
+### Constants
+
+1. SLOTS\_PER\_SEGMENT: Number of slots in a segment of ledger data. The
+
+   unit of storage for an archiver.
+
+2. NUM\_KEY\_ROTATION\_SEGMENTS: Number of segments after which archivers
+
+   regenerate their encryption keys and select a new dataset to store.
+
+3. NUM\_STORAGE\_PROOFS: Number of storage proofs required for a storage proof
+
+   claim to be successfully rewarded.
+
+4. RATIO\_OF\_FAKE\_PROOFS: Ratio of fake proofs to real proofs that a storage
+
+   mining proof claim has to contain to be valid for a reward.
+
+5. NUM\_STORAGE\_SAMPLES: Number of samples required for a storage mining
+
+   proof.
+
+6. NUM\_CHACHA\_ROUNDS: Number of encryption rounds performed to generate
+
+   encrypted state.
+
+7. NUM\_SLOTS\_PER\_TURN: Number of slots that define a single storage epoch or
+
+   a "turn" of the PoRep game.
+
+### Validator behavior
+
+1. Validators join the network and begin looking for archiver accounts at each
+
+   storage epoch/turn boundary.
+
+2. Every turn, Validators sign the PoH value at the boundary and use that signature
+
+   to randomly pick proofs to verify from each storage account found in the turn boundary.
+
+   This signed value is also submitted to the validator's storage account and will be used by
+
+   archivers at a later stage to cross-verify.
+
+3. Every `NUM_SLOTS_PER_TURN` slots the validator advertises the PoH value. This is value
+
+   is also served to Archivers via RPC interfaces.
+
+4. For a given turn N, all validations get locked out until turn N+3 \(a gap of 2 turn/epoch\).
+
+   At which point all validations during that turn are available for reward collection.
+
+5. Any incorrect validations will be marked during the turn in between.
+
+### Archiver behavior
+
+1. Since an archiver is somewhat of a light client and not downloading all the
+
+   ledger data, they have to rely on other validators and archivers for information.
+
+   Any given validator may or may not be malicious and give incorrect information, although
+
+   there are not any obvious attack vectors that this could accomplish besides having the
+
+   archiver do extra wasted work. For many of the operations there are a number of options
+
+   depending on how paranoid an archiver is:
+
+   * \(a\) archiver can ask a validator
+   * \(b\) archiver can ask multiple validators
+   * \(c\) archiver can ask other archivers
+   * \(d\) archiver can subscribe to the full transaction stream and generate
+
+     the information itself \(assuming the slot is recent enough\)
+
+   * \(e\) archiver can subscribe to an abbreviated transaction stream to
+
+     generate the information itself \(assuming the slot is recent enough\)
+
+2. An archiver obtains the PoH hash corresponding to the last turn with its slot.
+3. The archiver signs the PoH hash with its keypair. That signature is the
+
+   seed used to pick the segment to replicate and also the encryption key. The
+
+   archiver mods the signature with the slot to get which segment to
+
+   replicate.
+
+4. The archiver retrives the ledger by asking peer validators and
+
+   archivers. See 6.5.
+
+5. The archiver then encrypts that segment with the key with chacha algorithm
+
+   in CBC mode with `NUM_CHACHA_ROUNDS` of encryption.
+
+6. The archiver initializes a chacha rng with the a signed recent PoH value as
+
+   the seed.
+
+7. The archiver generates `NUM_STORAGE_SAMPLES` samples in the range of the
+
+   entry size and samples the encrypted segment with sha256 for 32-bytes at each
+
+   offset value. Sampling the state should be faster than generating the encrypted
+
+   segment.
+
+8. The archiver sends a PoRep proof transaction which contains its sha state
+
+   at the end of the sampling operation, its seed and the samples it used to the
+
+   current leader and it is put onto the ledger.
+
+9. During a given turn the archiver should submit many proofs for the same segment
+
+   and based on the `RATIO_OF_FAKE_PROOFS` some of those proofs must be fake.
+
+10. As the PoRep game enters the next turn, the archiver must submit a
+
+    transaction with the mask of which proofs were fake during the last turn. This
+
+    transaction will define the rewards for both archivers and validators.
+
+11. Finally for a turn N, as the PoRep game enters turn N + 3, archiver's proofs for
+
+    turn N will be counted towards their rewards.
+
+### The PoRep Game
+
+The Proof of Replication game has 4 primary stages. For each "turn" multiple PoRep games can be in progress but each in a different stage.
+
+The 4 stages of the PoRep Game are as follows:
+
+1. Proof submission stage
+   * Archivers: submit as many proofs as possible during this stage
+   * Validators: No-op
+2. Proof verification stage
+   * Archivers: No-op
+   * Validators: Select archivers and verify their proofs from the previous turn
+3. Proof challenge stage
+   * Archivers: Submit the proof mask with justifications \(for fake proofs submitted 2 turns ago\)
+   * Validators: No-op
+4. Reward collection stage
+   * Archivers: Collect rewards for 3 turns ago
+   * Validators:  Collect rewards for 3 turns ago
+
+For each turn of the PoRep game, both Validators and Archivers evaluate each stage. The stages are run as separate transactions on the storage program.
+
+### Finding who has a given block of ledger
+
+1. Validators monitor the turns in the PoRep game and look at the rooted bank
+
+   at turn boundaries for any proofs.
+
+2. Validators maintain a map of ledger segments and corresponding archiver public keys.
+
+   The map is updated when a Validator processes an archiver's proofs for a segment.
+
+   The validator provides an RPC interface to access the this map. Using this API, clients
+
+   can map a segment to an archiver's network address \(correlating it via cluster\_info table\).
+
+   The clients can then send repair requests to the archiver to retrieve segments.
+
+3. Validators would need to invalidate this list every N turns.
+
+## Sybil attacks
+
+For any random seed, we force everyone to use a signature that is derived from a PoH hash at the turn boundary. Everyone uses the same count, so the same PoH hash is signed by every participant. The signatures are then each cryptographically tied to the keypair, which prevents a leader from grinding on the resulting value for more than 1 identity.
+
+Since there are many more client identities then encryption identities, we need to split the reward for multiple clients, and prevent Sybil attacks from generating many clients to acquire the same block of data. To remain BFT we want to avoid a single human entity from storing all the replications of a single chunk of the ledger.
+
+Our solution to this is to force the clients to continue using the same identity. If the first round is used to acquire the same block for many client identities, the second round for the same client identities will force a redistribution of the signatures, and therefore PoRep identities and blocks. Thus to get a reward for archivers need to store the first block for free and the network can reward long lived client identities more than new ones.
+
+## Validator attacks
+
+* If a validator approves fake proofs, archiver can easily out them by
+
+  showing the initial state for the hash.
+
+* If a validator marks real proofs as fake, no on-chain computation can be done
+
+  to distinguish who is correct. Rewards would have to rely on the results from
+
+  multiple validators to catch bad actors and archivers from being denied rewards.
+
+* Validator stealing mining proof results for itself. The proofs are derived
+
+  from a signature from an archiver, since the validator does not know the
+
+  private key used to generate the encryption key, it cannot be the generator of
+
+  the proof.
+
+## Reward incentives
+
+Fake proofs are easy to generate but difficult to verify. For this reason, PoRep proof transactions generated by archivers may require a higher fee than a normal transaction to represent the computational cost required by validators.
+
+Some percentage of fake proofs are also necessary to receive a reward from storage mining.
+
+## Notes
+
+* We can reduce the costs of verification of PoRep by using PoH, and actually
+
+  make it feasible to verify a large number of proofs for a global dataset.
+
+* We can eliminate grinding by forcing everyone to sign the same PoH hash and
+
+  use the signatures as the seed
+
+* The game between validators and archivers is over random blocks and random
+
+  encryption identities and random data samples. The goal of randomization is
+
+  to prevent colluding groups from having overlap on data or validation.
+
+* Archiver clients fish for lazy validators by submitting fake proofs that
+
+  they can prove are fake.
+
+* To defend against Sybil client identities that try to store the same block we
+
+  force the clients to store for multiple rounds before receiving a reward.
+
+* Validators should also get rewarded for validating submitted storage proofs
+
+  as incentive for storing the ledger. They can only validate proofs if they
+
+  are storing that slice of the ledger.
+
+# Ledger Replication Not Implemented
+
+Replication behavior yet to be implemented.
+
+## Storage epoch
+
+The storage epoch should be the number of slots which results in around 100GB-1TB of ledger to be generated for archivers to store. Archivers will start storing ledger when a given fork has a high probability of not being rolled back.
+
+## Validator behavior
+
+1. Every NUM\_KEY\_ROTATION\_TICKS it also validates samples received from
+
+   archivers. It signs the PoH hash at that point and uses the following
+
+   algorithm with the signature as the input:
+
+   * The low 5 bits of the first byte of the signature creates an index into
+
+     another starting byte of the signature.
+
+   * The validator then looks at the set of storage proofs where the byte of
+
+     the proof's sha state vector starting from the low byte matches exactly
+
+     with the chosen byte\(s\) of the signature.
+
+   * If the set of proofs is larger than the validator can handle, then it
+
+     increases to matching 2 bytes in the signature.
+
+   * Validator continues to increase the number of matching bytes until a
+
+     workable set is found.
+
+   * It then creates a mask of valid proofs and fake proofs and sends it to
+
+     the leader. This is a storage proof confirmation transaction.
+
+2. After a lockout period of NUM\_SECONDS\_STORAGE\_LOCKOUT seconds, the
+
+   validator then submits a storage proof claim transaction which then causes the
+
+   distribution of the storage reward if no challenges were seen for the proof to
+
+   the validators and archivers party to the proofs.
+
+## Archiver behavior
+
+1. The archiver then generates another set of offsets which it submits a fake
+
+   proof with an incorrect sha state. It can be proven to be fake by providing the
+
+   seed for the hash result.
+
+   * A fake proof should consist of an archiver hash of a signature of a PoH
+
+     value. That way when the archiver reveals the fake proof, it can be
+
+     verified on chain.
+
+2. The archiver monitors the ledger, if it sees a fake proof integrated, it
+
+   creates a challenge transaction and submits it to the current leader. The
+
+   transacation proves the validator incorrectly validated a fake storage proof.
+
+   The archiver is rewarded and the validator's staking balance is slashed or
+
+   frozen.
+
+## Storage proof contract logic
+
+Each archiver and validator will have their own storage account. The validator's account would be separate from their gossip id similiar to their vote account. These should be implemented as two programs one which handles the validator as the keysigner and one for the archiver. In that way when the programs reference other accounts, they can check the program id to ensure it is a validator or archiver account they are referencing.
+
+### SubmitMiningProof
+
+```text
+SubmitMiningProof {
+    slot: u64,
+    sha_state: Hash,
+    signature: Signature,
+};
+keys = [archiver_keypair]
+```
+
+Archivers create these after mining their stored ledger data for a certain hash value. The slot is the end slot of the segment of ledger they are storing, the sha\_state the result of the archiver using the hash function to sample their encrypted ledger segment. The signature is the signature that was created when they signed a PoH value for the current storage epoch. The list of proofs from the current storage epoch should be saved in the account state, and then transfered to a list of proofs for the previous epoch when the epoch passes. In a given storage epoch a given archiver should only submit proofs for one segment.
+
+The program should have a list of slots which are valid storage mining slots. This list should be maintained by keeping track of slots which are rooted slots in which a significant portion of the network has voted on with a high lockout value, maybe 32-votes old. Every SLOTS\_PER\_SEGMENT number of slots would be added to this set. The program should check that the slot is in this set. The set can be maintained by receiving a AdvertiseStorageRecentBlockHash and checking with its bank/Tower BFT state.
+
+The program should do a signature verify check on the signature, public key from the transaction submitter and the message of the previous storage epoch PoH value.
+
+### ProofValidation
+
+```text
+ProofValidation {
+   proof_mask: Vec<ProofStatus>,
+}
+keys = [validator_keypair, archiver_keypair(s) (unsigned)]
+```
+
+A validator will submit this transaction to indicate that a set of proofs for a given segment are valid/not-valid or skipped where the validator did not look at it. The keypairs for the archivers that it looked at should be referenced in the keys so the program logic can go to those accounts and see that the proofs are generated in the previous epoch. The sampling of the storage proofs should be verified ensuring that the correct proofs are skipped by the validator according to the logic outlined in the validator behavior of sampling.
+
+The included archiver keys will indicate the the storage samples which are being referenced; the length of the proof\_mask should be verified against the set of storage proofs in the referenced archiver account\(s\), and should match with the number of proofs submitted in the previous storage epoch in the state of said archiver account.
+
+### ClaimStorageReward
+
+```text
+ClaimStorageReward {
+}
+keys = [validator_keypair or archiver_keypair, validator/archiver_keypairs (unsigned)]
+```
+
+Archivers and validators will use this transaction to get paid tokens from a program state where SubmitStorageProof, ProofValidation and ChallengeProofValidations are in a state where proofs have been submitted and validated and there are no ChallengeProofValidations referencing those proofs. For a validator, it should reference the archiver keypairs to which it has validated proofs in the relevant epoch. And for an archiver it should reference validator keypairs for which it has validated and wants to be rewarded.
+
+### ChallengeProofValidation
+
+```text
+ChallengeProofValidation {
+    proof_index: u64,
+    hash_seed_value: Vec<u8>,
+}
+keys = [archiver_keypair, validator_keypair]
+```
+
+This transaction is for catching lazy validators who are not doing the work to validate proofs. An archiver will submit this transaction when it sees a validator has approved a fake SubmitMiningProof transaction. Since the archiver is a light client not looking at the full chain, it will have to ask a validator or some set of validators for this information maybe via RPC call to obtain all ProofValidations for a certain segment in the previous storage epoch. The program will look in the validator account state see that a ProofValidation is submitted in the previous storage epoch and hash the hash\_seed\_value and see that the hash matches the SubmitMiningProof transaction and that the validator marked it as valid. If so, then it will save the challenge to the list of challenges that it has in its state.
+
+### AdvertiseStorageRecentBlockhash
+
+```text
+AdvertiseStorageRecentBlockhash {
+    hash: Hash,
+    slot: u64,
+}
+```
+
+Validators and archivers will submit this to indicate that a new storage epoch has passed and that the storage proofs which are current proofs should now be for the previous epoch. Other transactions should check to see that the epoch that they are referencing is accurate according to current chain state.

docs/src/proposals/program-keys-and-signatures.md

@@ -1,169 +0,0 @@
-# Program Keys and Signatures
-
-## Problem
-
-Programs cannot generate their own signatures in `process_instruction`
-as defined in the [Cross-Program Invocations](cross-program-invocation.md)
-design.
-
-Lack of programmatic signature generation limits the kinds of programs
-that can be implemented in Solana.  For example, a program cannot take
-ownership of a TokenAccount and later in a different transaction transfer
-the ownership based on the state of another program.  If two users want
-to make a wager in tokens on the outcome of a game in Solana, they must
-transfer tokens to some intermediary that will honor their agreement.
-Currently there is no way to implement this intermediary as a program
-in Solana.
-
-This capability is necessary for many DeFi applications, since they
-require assets to be transferred to an escrow agent until some event
-occurs that determines the new owner.
-
-* Decentralized Exchanges that transfer assets between matching bid and
-ask orders.
-
-* Auctions that transfer assets to the winner.
-
-* Games or prediction markets that collect and redistribute prizes to
-the winners.
-
-## Proposed Solution
-
-The key to the design is two fold:
-
-1. Allow programs to control specific addresses, called Program
-Addresses, in such a way that it is impossible for any external
-user to generate valid transactions with signatures for those
-addresses.
-
-2. To allow programs to programatically control
-`KeyedAccount::is_signer` value for Program Addresses that are
-present in instructions that is invoked via `process_instruction()`.
-
-Given the two conditions, users can securely transfer or assign
-ownershp of on chain assets to Program Addresses.  Once assigned,
-the program and only the program can execute instructions that
-refences a Program Address with `KeyedAccount::is_signer` set to
-true.
-
-### Private keys for Program Addresses
-
-This address has no private key associated with it, and generating
-a signature for it is impossible.  While it has no private key of
-its own, the program can issue an instruction to set the
-`KeyedAccount::is_signer` flag for this address.
-
-### Hash based generated Program Addresses
-
-All 256 bit values are valid ed25519 curve points, and valid ed25519 public
-keys.  All are equally secure and equally as hard to break.
-Based on this assumption, Program Addresses can be deterministically
-derived from a base seed using a 256 bit preimage resistant hash function.
-
-Deterministic Program Addresses for programs follow a similar derivation
-path as Accounts created with `SystemInstruction::CreateAccountWithSeed`
-which is implemented with `system_instruction::create_address_with_seed`.
-
-For reference the implementation is as follows:
-
-```rust,ignore
-pub fn create_address_with_seed(
-    base: &Pubkey,
-    seed: &str,
-    program_id: &Pubkey,
-) -> Result<Pubkey, SystemError> {
-    if seed.len() > MAX_ADDRESS_SEED_LEN {
-        return Err(SystemError::MaxSeedLengthExceeded);
-    }
-
-    Ok(Pubkey::new(
-        hashv(&[base.as_ref(), seed.as_ref(), program_id.as_ref()]).as_ref(),
-    ))
-}
-```
-
-Programs can deterministically derive any number of addresses by
-using a keyword.  The keyword can symbolically identify how this
-address is used.
-
-```rust,ignore
-//! Generate a derived program address
-//!     * program_id, the program's id
-//!     * key_base, can be any public key chosen by the program
-//!     * keyword, symbolic keyword to identify the key
-//!
-//! The tuple (`key_base`, `keyword`) is used by programs to create user specific
-//! symbolic keys.  For example for the staking contact, the program may need:
-//!     * <user account>/<"withdrawer">
-//!     * <user account>/<"staker">
-//!     * <user account>/<"custodian">
-//! As generated keys to control a single stake account for each user.
-pub fn derive_program_address(
-    program_id: &Pubkey,
-    key_base, &Pubkey,
-    keyword, &str,
-) -> Result<Pubkey, SystemError> {
-
-    // Generate a deterministic base for all program addresses that
-    // are owned by `program_id`.
-    // Hashing twice is recommended to prevent lenght extension attacks.
-    Ok(Pubkey::new(
-        hashv(&[hashv(&[program_id.as_ref(), key_base.as_ref(), keyword.as_ref(),
-            &"ProgramAddress11111111111111111111111111111"]).as_ref()])
-    ))
-}
-```
-
-### Using Program Addresses
-
-Clients can use the `derive_program_address` function to generate
-a destination address.
-
-```rust,ignore
-//deterministically derive the escrow key
-let escrow_pubkey = derive_program_address(&escrow_program_id, &alice_pubkey, &"escrow");
-let message = Message::new(vec![
-    token_instruction::transfer(&alice_pubkey, &escrow_pubkey, 1),
-]);
-//transfer 1 token to escrow
-client.send_message(&[&alice_keypair], &message);
-```
-
-Programs can use the same function to generate the same address.
-Below the program issue a `token_instruction::transfer` from its
-own address as if it had a private key to sign the transaction.
-
-```rust,ignore
-fn transfer_one_token_from_escrow(
-    program_id: &Pubkey,
-    keyed_accounts: &[KeyedAccount]
-) -> Result<()> {
-
-
-    //user supplies the destination
-    let alice_pubkey = keyed_accounts[1].key;
-
-    // Deterministically derive the escrow pubkey.
-    let escrow_pubkey = derive_program_address(program_id, &alice_pubkey, &"escrow");
-
-    //create the transfer instruction
-    let instruction = token_instruction::transfer(&escrow_pubkey, &alice_pubkey, 1);
-
-    // The runtime deterministically derives the key from the current
-    // program id and the supplied keywords.
-    // If the derived key matches a key in the instruction
-    // the `is_signed` flag is set.
-    process_signed_instruction(&instruction,  &[(&alice_pubkey, &"escrow")])?
-}
-```
-
-### Setting `KeyedAccount::is_signer`
-
-The addresses generated with `derive_program_address` are blinded
-and are indistinguishable from any other pubkey.  The only way for
-the runtime to verify that the address belongs to a program is for
-the program to supply the keyword used to generate the address.
-
-The runtime will internally run  `derive_program_address(program_id,
-&alice_pubkey, &"escrow")`, and compare the result against the addresses
-supplied in the instruction.

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,14 +13,14 @@ clap = "2.33.1"
 log = "0.4.8"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ console = "0.10.1"
 indicatif = "0.14.0"
 log = "0.4.8"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
 tar = "0.4.28"
 
 [lib]

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,11 +16,11 @@ clap = "2.33"
 log = "0.4.8"
 serde = "1.0.110"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

genesis-programs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-programs"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana genesis programs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = { version = "0.4.8" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.0" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-vest-program = { path = "../programs/vest", version = "1.2.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.1" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-vest-program = { path = "../programs/vest", version = "1.2.1" }
 
 [lib]
 crate-type = ["lib"]

genesis-programs/src/lib.rs

@@ -16,26 +16,28 @@ use log::*;
 use solana_runtime::bank::{Bank, EnteredEpochCallback};
 
 pub fn get_inflation(operating_mode: OperatingMode, epoch: Epoch) -> Option<Inflation> {
+    let past_epoch_inflation = get_inflation_for_epoch(operating_mode, epoch.saturating_sub(1));
+    let epoch_inflation = get_inflation_for_epoch(operating_mode, epoch);
+
+    if epoch_inflation != past_epoch_inflation || epoch == 0 {
+        Some(epoch_inflation)
+    } else {
+        None
+    }
+}
+
+pub fn get_inflation_for_epoch(operating_mode: OperatingMode, epoch: Epoch) -> Inflation {
     match operating_mode {
-        OperatingMode::Development => {
-            if epoch == 0 {
-                Some(Inflation::default())
-            } else {
-                None
-            }
-        }
+        OperatingMode::Development => Inflation::default(),
         OperatingMode::Stable | OperatingMode::Preview => {
-            if epoch == 0 {
-                // No inflation at epoch 0
-                Some(Inflation::new_disabled())
-            } else if epoch == std::u64::MAX {
+            if epoch == std::u64::MAX {
                 // Inflation starts
-                //
-                // The epoch of std::u64::MAX - 1 is a placeholder and is expected to be reduced in
+                // The epoch of std::u64::MAX is a placeholder and is expected to be reduced in
                 // a future hard fork.
-                Some(Inflation::default())
+                Inflation::default()
             } else {
-                None
+                // No inflation from epoch 0
+                Inflation::new_disabled()
             }
         }
     }

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,14 +15,14 @@ chrono = "0.4"
 serde = "1.0.110"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 tempfile = "3.1.0"
 
 [[bin]]

genesis/src/main.rs

@@ -7,8 +7,8 @@ use solana_clap_utils::{
 };
 use solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account};
 use solana_ledger::{
-    blockstore::create_new_ledger, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
-    poh::compute_hashes_per_tick,
+    blockstore::create_new_ledger, blockstore_db::AccessType,
+    hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, poh::compute_hashes_per_tick,
 };
 use solana_sdk::{
     account::Account,
@@ -540,6 +540,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         &ledger_path,
         &genesis_config,
         max_genesis_archive_unpacked_size,
+        AccessType::PrimaryOnly,
     )?;
 
     println!("{}", genesis_config);

gossip/Cargo.toml

@@ -3,20 +3,20 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-net-utils = { path = "../net-utils", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-net-utils = { path = "../net-utils", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,12 +24,12 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-config-program = { path = "../programs/config", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-config-program = { path = "../programs/config", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempdir = "0.3.7"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs = "2.0.2"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-cli-config = { path = "../cli-config", version = "1.2.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-cli-config = { path = "../cli-config", version = "1.2.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,18 +13,19 @@ bs58 = "0.3.1"
 bytecount = "0.6.0"
 clap = "2.33.1"
 histogram = "*"
+log = { version = "0.4.8" }
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-cli = { path = "../cli", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-cli = { path = "../cli", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
 tempfile = "3.1.0"
 
 [dev-dependencies]

ledger-tool/src/main.rs

@@ -9,12 +9,13 @@ use solana_ledger::{
     bank_forks::{BankForks, SnapshotConfig},
     bank_forks_utils,
     blockstore::Blockstore,
-    blockstore_db::{self, Column, Database},
+    blockstore_db::{self, AccessType, Column, Database},
     blockstore_processor::ProcessOptions,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     rooted_slot_iterator::RootedSlotIterator,
     snapshot_utils,
 };
+use solana_runtime::bank::Bank;
 use solana_sdk::{
     clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
     shred_version::compute_shred_version,
@@ -29,8 +30,11 @@ use std::{
     path::{Path, PathBuf},
     process::{exit, Command, Stdio},
     str::FromStr,
+    sync::Arc,
 };
 
+use log::*;
+
 #[derive(PartialEq)]
 enum LedgerOutputMethod {
     Print,
@@ -519,8 +523,8 @@ fn analyze_storage(database: &Database) -> Result<(), String> {
     Ok(())
 }
 
-fn open_blockstore(ledger_path: &Path) -> Blockstore {
-    match Blockstore::open(ledger_path) {
+fn open_blockstore(ledger_path: &Path, access_type: AccessType) -> Blockstore {
+    match Blockstore::open_with_access_type(ledger_path, access_type) {
         Ok(blockstore) => blockstore,
         Err(err) => {
             eprintln!("Failed to open ledger at {:?}: {:?}", ledger_path, err);
@@ -529,8 +533,8 @@ fn open_blockstore(ledger_path: &Path) -> Blockstore {
     }
 }
 
-fn open_database(ledger_path: &Path) -> Database {
-    match Database::open(&ledger_path.join("rocksdb")) {
+fn open_database(ledger_path: &Path, access_type: AccessType) -> Database {
+    match Database::open(&ledger_path.join("rocksdb"), access_type) {
         Ok(database) => database,
         Err(err) => {
             eprintln!("Unable to read the Ledger rocksdb: {:?}", err);
@@ -553,26 +557,46 @@ fn load_bank_forks(
     ledger_path: &PathBuf,
     genesis_config: &GenesisConfig,
     process_options: ProcessOptions,
+    access_type: AccessType,
 ) -> bank_forks_utils::LoadResult {
+    let blockstore = open_blockstore(&ledger_path, access_type);
+    let snapshot_path = ledger_path.clone().join(if blockstore.is_primary_access() {
+        "snapshot"
+    } else {
+        "snapshot.ledger-tool"
+    });
     let snapshot_config = if arg_matches.is_present("no_snapshot") {
         None
     } else {
         Some(SnapshotConfig {
             snapshot_interval_slots: 0, // Value doesn't matter
             snapshot_package_output_path: ledger_path.clone(),
-            snapshot_path: ledger_path.clone().join("snapshot"),
+            snapshot_path,
             compression: CompressionType::Bzip2,
         })
     };
     let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
+        if !blockstore.is_primary_access() {
+            // Be defenstive, when default account dir is explicitly specified, it's still possible
+            // to wipe the dir possibly shared by the running validator!
+            eprintln!("Error: custom accounts path is not supported under secondary access");
+            exit(1);
+        }
         account_paths.split(',').map(PathBuf::from).collect()
-    } else {
+    } else if blockstore.is_primary_access() {
         vec![ledger_path.join("accounts")]
+    } else {
+        let non_primary_accounts_path = ledger_path.join("accounts.ledger-tool");
+        warn!(
+            "Default accounts path is switched aligning with Blockstore's secondary access: {:?}",
+            non_primary_accounts_path
+        );
+        vec![non_primary_accounts_path]
     };
 
     bank_forks_utils::load(
         &genesis_config,
-        &open_blockstore(&ledger_path),
+        &blockstore,
         account_paths,
         snapshot_config.as_ref(),
         process_options,
@@ -765,6 +789,17 @@ fn main() {
                     .takes_value(true)
                     .help("Output directory for the snapshot"),
             )
+            .arg(
+                Arg::with_name("warp_slot")
+                    .required(false)
+                    .long("warp-slot")
+                    .takes_value(true)
+                    .value_name("WARP_SLOT")
+                    .validator(is_slot)
+                    .help("After loading the snapshot slot warp the ledger to WARP_SLOT, \
+                           which could be a slot in a galaxy far far away"),
+            )
+
         ).subcommand(
             SubCommand::with_name("accounts")
             .about("Print account contents after processing in the ledger")
@@ -862,7 +897,7 @@ fn main() {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
             output_ledger(
-                open_blockstore(&ledger_path),
+                open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
                 starting_slot,
                 allow_dead_slots,
                 LedgerOutputMethod::Print,
@@ -885,7 +920,13 @@ fn main() {
                 ..ProcessOptions::default()
             };
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
-            match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
+            match load_bank_forks(
+                arg_matches,
+                &ledger_path,
+                &genesis_config,
+                process_options,
+                AccessType::TryPrimaryThenSecondary,
+            ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     println!(
                         "{}",
@@ -904,7 +945,7 @@ fn main() {
         ("slot", Some(arg_matches)) => {
             let slots = values_t_or_exit!(arg_matches, "slots", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
-            let blockstore = open_blockstore(&ledger_path);
+            let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
             for slot in slots {
                 println!("Slot {}", slot);
                 if let Err(err) = output_slot(
@@ -921,7 +962,7 @@ fn main() {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
             output_ledger(
-                open_blockstore(&ledger_path),
+                open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
                 starting_slot,
                 allow_dead_slots,
                 LedgerOutputMethod::Json,
@@ -929,7 +970,7 @@ fn main() {
         }
         ("set-dead-slot", Some(arg_matches)) => {
             let slots = values_t_or_exit!(arg_matches, "slots", Slot);
-            let blockstore = open_blockstore(&ledger_path);
+            let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
             for slot in slots {
                 match blockstore.set_dead_slot(slot) {
                     Ok(_) => println!("Slot {} dead", slot),
@@ -954,6 +995,7 @@ fn main() {
                 &ledger_path,
                 &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
+                AccessType::TryPrimaryThenSecondary,
             )
             .unwrap_or_else(|err| {
                 eprintln!("Ledger verification failed: {:?}", err);
@@ -976,6 +1018,7 @@ fn main() {
                 &ledger_path,
                 &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
+                AccessType::TryPrimaryThenSecondary,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let dot = graph_forks(&bank_forks, arg_matches.is_present("include_all_votes"));
@@ -1004,6 +1047,7 @@ fn main() {
         ("create-snapshot", Some(arg_matches)) => {
             let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot);
             let output_directory = value_t_or_exit!(arg_matches, "output_directory", String);
+            let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
 
             let process_options = ProcessOptions {
                 dev_halt_at_slot: Some(snapshot_slot),
@@ -1012,14 +1056,34 @@ fn main() {
                 ..ProcessOptions::default()
             };
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
-            match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
+            match load_bank_forks(
+                arg_matches,
+                &ledger_path,
+                &genesis_config,
+                process_options,
+                AccessType::TryPrimaryThenSecondary,
+            ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
-                    let bank = bank_forks.get(snapshot_slot).unwrap_or_else(|| {
-                        eprintln!("Error: Slot {} is not available", snapshot_slot);
-                        exit(1);
-                    });
+                    let bank = bank_forks
+                        .get(snapshot_slot)
+                        .unwrap_or_else(|| {
+                            eprintln!("Error: Slot {} is not available", snapshot_slot);
+                            exit(1);
+                        })
+                        .clone();
+
+                    let bank = if let Some(warp_slot) = warp_slot {
+                        Arc::new(Bank::warp_from_parent(
+                            &bank,
+                            bank.collector_id(),
+                            warp_slot,
+                        ))
+                    } else {
+                        bank
+                    };
 
                     println!("Creating a snapshot of slot {}", bank.slot());
+                    assert!(bank.is_complete());
                     bank.squash();
                     bank.clean_accounts();
                     bank.update_accounts_hash();
@@ -1046,7 +1110,8 @@ fn main() {
                             snapshot_utils::archive_snapshot_package(&package).map(|ok| {
                                 println!(
                                     "Successfully created snapshot for slot {}: {:?}",
-                                    snapshot_slot, package.tar_output_file
+                                    bank.slot(),
+                                    package.tar_output_file
                                 );
                                 println!(
                                     "Shred version: {}",
@@ -1079,7 +1144,13 @@ fn main() {
             };
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             let include_sysvars = arg_matches.is_present("include_sysvars");
-            match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
+            match load_bank_forks(
+                arg_matches,
+                &ledger_path,
+                &genesis_config,
+                process_options,
+                AccessType::TryPrimaryThenSecondary,
+            ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let slot = bank_forks.working_bank().slot();
                     let bank = bank_forks.get(slot).unwrap_or_else(|| {
@@ -1121,7 +1192,13 @@ fn main() {
                 ..ProcessOptions::default()
             };
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
-            match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
+            match load_bank_forks(
+                arg_matches,
+                &ledger_path,
+                &genesis_config,
+                process_options,
+                AccessType::TryPrimaryThenSecondary,
+            ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let slot = bank_forks.working_bank().slot();
                     let bank = bank_forks.get(slot).unwrap_or_else(|| {
@@ -1185,11 +1262,12 @@ fn main() {
         ("purge", Some(arg_matches)) => {
             let start_slot = value_t_or_exit!(arg_matches, "start_slot", Slot);
             let end_slot = value_t_or_exit!(arg_matches, "end_slot", Slot);
-            let blockstore = open_blockstore(&ledger_path);
+            let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
             blockstore.purge_slots(start_slot, end_slot);
+            blockstore.purge_from_next_slots(start_slot, end_slot);
         }
         ("list-roots", Some(arg_matches)) => {
-            let blockstore = open_blockstore(&ledger_path);
+            let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
             let max_height = if let Some(height) = arg_matches.value_of("max_height") {
                 usize::from_str(height).expect("Maximum height must be a number")
             } else {
@@ -1242,7 +1320,9 @@ fn main() {
                 });
         }
         ("bounds", Some(arg_matches)) => {
-            match open_blockstore(&ledger_path).slot_meta_iterator(0) {
+            match open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary)
+                .slot_meta_iterator(0)
+            {
                 Ok(metas) => {
                     let all = arg_matches.is_present("all");
 
@@ -1268,15 +1348,20 @@ fn main() {
                 }
             }
         }
-        ("analyze-storage", _) => match analyze_storage(&open_database(&ledger_path)) {
-            Ok(()) => {
-                println!("Ok.");
+        ("analyze-storage", _) => {
+            match analyze_storage(&open_database(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+            )) {
+                Ok(()) => {
+                    println!("Ok.");
+                }
+                Err(err) => {
+                    eprintln!("Unable to read the Ledger: {:?}", err);
+                    exit(1);
+                }
             }
-            Err(err) => {
-                eprintln!("Unable to read the Ledger: {:?}", err);
-                exit(1);
-            }
-        },
+        }
         ("", _) => {
             eprintln!("{}", matches.usage());
             exit(1);

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -30,19 +30,19 @@ reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 regex = "1.3.7"
 serde = "1.0.110"
 serde_bytes = "0.11.4"
-solana-transaction-status = { path = "../transaction-status", version = "1.2.0" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-measure = { path = "../measure", version = "1.2.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
-solana-perf = { path = "../perf", version = "1.2.0" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.1" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-measure = { path = "../measure", version = "1.2.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
+solana-perf = { path = "../perf", version = "1.2.1" }
 ed25519-dalek = "1.0.0-pre.3"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
 symlink = "0.1.0"
 tar = "0.4.28"
 thiserror = "1.0"
@@ -59,7 +59,7 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.2.0" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.1" }
 
 [lib]
 crate-type = ["lib"]

ledger/src/blockstore.rs

@@ -4,8 +4,8 @@
 pub use crate::{blockstore_db::BlockstoreError, blockstore_meta::SlotMeta};
 use crate::{
     blockstore_db::{
-        columns as cf, Column, Database, IteratorDirection, IteratorMode, LedgerColumn, Result,
-        WriteBatch,
+        columns as cf, AccessType, Column, Database, IteratorDirection, IteratorMode, LedgerColumn,
+        Result, WriteBatch,
     },
     blockstore_meta::*,
     entry::{create_ticks, Entry},
@@ -56,6 +56,8 @@ use std::{
     time::Duration,
 };
 
+pub mod blockstore_purge;
+
 pub const BLOCKSTORE_DIRECTORY: &str = "rocksdb";
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
@@ -64,6 +66,12 @@ thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::
                     .build()
                     .unwrap()));
 
+thread_local!(static PAR_THREAD_POOL_ALL_CPUS: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
+                    .num_threads(num_cpus::get())
+                    .thread_name(|ix| format!("blockstore_{}", ix))
+                    .build()
+                    .unwrap()));
+
 pub const MAX_COMPLETED_SLOTS_IN_CHANNEL: usize = 100_000;
 pub const MAX_TURBINE_PROPAGATION_IN_MS: u64 = 100;
 pub const MAX_TURBINE_DELAY_IN_TICKS: u64 = MAX_TURBINE_PROPAGATION_IN_MS / MS_PER_TICK;
@@ -75,6 +83,13 @@ const TIMESTAMP_SLOT_RANGE: usize = 16;
 pub const MAX_DATA_SHREDS_PER_SLOT: usize = 32_768;
 
 pub type CompletedSlotsReceiver = Receiver<Vec<u64>>;
+type CompletedRanges = Vec<(u32, u32)>;
+
+#[derive(Clone, Copy)]
+pub enum PurgeType {
+    Exact,
+    PrimaryIndex,
+}
 
 // ledger window
 pub struct Blockstore {
@@ -177,6 +192,17 @@ impl Blockstore {
 
     /// Opens a Ledger in directory, provides "infinite" window of shreds
     pub fn open(ledger_path: &Path) -> Result<Blockstore> {
+        Self::do_open(ledger_path, AccessType::PrimaryOnly)
+    }
+
+    pub fn open_with_access_type(
+        ledger_path: &Path,
+        access_type: AccessType,
+    ) -> Result<Blockstore> {
+        Self::do_open(ledger_path, access_type)
+    }
+
+    fn do_open(ledger_path: &Path, access_type: AccessType) -> Result<Blockstore> {
         fs::create_dir_all(&ledger_path)?;
         let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
 
@@ -185,7 +211,7 @@ impl Blockstore {
         // Open the database
         let mut measure = Measure::start("open");
         info!("Opening database at {:?}", blockstore_path);
-        let db = Database::open(&blockstore_path)?;
+        let db = Database::open(&blockstore_path, access_type)?;
 
         // Create the metadata column family
         let meta_cf = db.column();
@@ -267,7 +293,7 @@ impl Blockstore {
     pub fn open_with_signal(
         ledger_path: &Path,
     ) -> Result<(Self, Receiver<bool>, CompletedSlotsReceiver)> {
-        let mut blockstore = Self::open(ledger_path)?;
+        let mut blockstore = Self::open_with_access_type(ledger_path, AccessType::PrimaryOnly)?;
         let (signal_sender, signal_receiver) = sync_channel(1);
         let (completed_slots_sender, completed_slots_receiver) =
             sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
@@ -301,208 +327,6 @@ impl Blockstore {
         false
     }
 
-    /// Silently deletes all blockstore column families in the range [from_slot,to_slot]
-    /// Dangerous; Use with care:
-    /// Does not check for integrity and does not update slot metas that refer to deleted slots
-    /// Modifies multiple column families simultaneously
-    pub fn purge_slots_with_delay(
-        &self,
-        from_slot: Slot,
-        to_slot: Slot,
-        delay_between_purges: Option<Duration>,
-    ) {
-        // if there's no upper bound, split the purge request into batches of 1000 slots
-        const PURGE_BATCH_SIZE: u64 = 1000;
-        let mut batch_start = from_slot;
-        while batch_start < to_slot {
-            let batch_end = (batch_start + PURGE_BATCH_SIZE).min(to_slot);
-            match self.run_purge(batch_start, batch_end) {
-                Ok(_all_columns_purged) => {
-                    batch_start = batch_end;
-
-                    if let Some(ref duration) = delay_between_purges {
-                        // Cooperate with other blockstore users
-                        std::thread::sleep(*duration);
-                    }
-                }
-                Err(e) => {
-                    error!(
-                        "Error: {:?}; Purge failed in range {:?} to {:?}",
-                        e, batch_start, batch_end
-                    );
-                    break;
-                }
-            }
-        }
-
-        if !self.no_compaction {
-            if let Err(e) = self.compact_storage(from_slot, to_slot) {
-                // This error is not fatal and indicates an internal error
-                error!(
-                    "Error: {:?}; Couldn't compact storage from {:?} to {:?}",
-                    e, from_slot, to_slot
-                );
-            }
-        }
-    }
-
-    pub fn purge_slots(&self, from_slot: Slot, to_slot: Slot) {
-        self.purge_slots_with_delay(from_slot, to_slot, None)
-    }
-
-    // Returns whether or not all columns successfully purged the slot range
-    fn run_purge(&self, from_slot: Slot, to_slot: Slot) -> Result<bool> {
-        let mut write_batch = self
-            .db
-            .batch()
-            .expect("Database Error: Failed to get write batch");
-        // delete range cf is not inclusive
-        let to_slot = to_slot.checked_add(1).unwrap_or_else(|| std::u64::MAX);
-
-        let mut delete_range_timer = Measure::start("delete_range");
-        let mut columns_empty = self
-            .db
-            .delete_range_cf::<cf::SlotMeta>(&mut write_batch, from_slot, to_slot)
-            .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::Root>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::ShredData>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::ShredCode>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::DeadSlots>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::DuplicateSlots>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::ErasureMeta>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::Orphans>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::Index>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false)
-            & self
-                .db
-                .delete_range_cf::<cf::Rewards>(&mut write_batch, from_slot, to_slot)
-                .unwrap_or(false);
-        let mut w_active_transaction_status_index =
-            self.active_transaction_status_index.write().unwrap();
-        if let Some(index) = self.toggle_transaction_status_index(
-            &mut write_batch,
-            &mut w_active_transaction_status_index,
-            to_slot,
-        )? {
-            columns_empty &= self
-                .db
-                .delete_range_cf::<cf::TransactionStatus>(&mut write_batch, index, index + 1)
-                .unwrap_or(false)
-                & self
-                    .db
-                    .delete_range_cf::<cf::AddressSignatures>(&mut write_batch, index, index + 1)
-                    .unwrap_or(false);
-        }
-        delete_range_timer.stop();
-        let mut write_timer = Measure::start("write_batch");
-        if let Err(e) = self.db.write(write_batch) {
-            error!(
-                "Error: {:?} while submitting write batch for slot {:?} retrying...",
-                e, from_slot
-            );
-            return Err(e);
-        }
-        write_timer.stop();
-        datapoint_info!(
-            "blockstore-purge",
-            ("from_slot", from_slot as i64, i64),
-            ("to_slot", to_slot as i64, i64),
-            ("delete_range_us", delete_range_timer.as_us() as i64, i64),
-            ("write_batch_us", write_timer.as_us() as i64, i64)
-        );
-        Ok(columns_empty)
-    }
-
-    pub fn compact_storage(&self, from_slot: Slot, to_slot: Slot) -> Result<bool> {
-        info!("compact_storage: from {} to {}", from_slot, to_slot);
-        let mut compact_timer = Measure::start("compact_range");
-        let result = self
-            .meta_cf
-            .compact_range(from_slot, to_slot)
-            .unwrap_or(false)
-            && self
-                .db
-                .column::<cf::Root>()
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .data_shred_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .code_shred_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .dead_slots_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .duplicate_slots_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .erasure_meta_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .orphans_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .index_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false)
-            && self
-                .transaction_status_cf
-                .compact_range(0, 2)
-                .unwrap_or(false)
-            && self
-                .address_signatures_cf
-                .compact_range(0, 2)
-                .unwrap_or(false)
-            && self
-                .transaction_status_index_cf
-                .compact_range(0, 2)
-                .unwrap_or(false)
-            && self
-                .rewards_cf
-                .compact_range(from_slot, to_slot)
-                .unwrap_or(false);
-        compact_timer.stop();
-        if !result {
-            info!("compact_storage incomplete");
-        }
-        datapoint_info!(
-            "blockstore-compact",
-            ("compact_range_us", compact_timer.as_us() as i64, i64),
-        );
-        Ok(result)
-    }
-
     pub fn erasure_meta(&self, slot: Slot, set_index: u64) -> Result<Option<ErasureMeta>> {
         self.erasure_meta_cf.get((slot, set_index))
     }
@@ -830,6 +654,8 @@ impl Blockstore {
                     &mut index_meta_time,
                     is_trusted,
                     handle_duplicate,
+                    leader_schedule,
+                    false,
                 ) {
                     num_inserted += 1;
                 }
@@ -874,6 +700,8 @@ impl Blockstore {
                             &mut index_meta_time,
                             is_trusted,
                             &handle_duplicate,
+                            leader_schedule,
+                            true,
                         );
                     }
                 }
@@ -956,7 +784,7 @@ impl Blockstore {
             .expect("Couldn't fetch from SlotMeta column family")
         {
             // Clear all slot related information
-            self.run_purge(slot, slot)
+            self.run_purge(slot, slot, PurgeType::PrimaryIndex)
                 .expect("Purge database operations failed");
 
             // Reinsert parts of `slot_meta` that are important to retain, like the `next_slots`
@@ -1086,6 +914,8 @@ impl Blockstore {
         index_meta_time: &mut u64,
         is_trusted: bool,
         handle_duplicate: &F,
+        leader_schedule: Option<&Arc<LeaderScheduleCache>>,
+        is_recovered: bool,
     ) -> bool
     where
         F: Fn(Shred) -> (),
@@ -1106,7 +936,13 @@ impl Blockstore {
             if Self::is_data_shred_present(&shred, slot_meta, &index_meta.data()) {
                 handle_duplicate(shred);
                 return false;
-            } else if !Blockstore::should_insert_data_shred(&shred, slot_meta, &self.last_root) {
+            } else if !Blockstore::should_insert_data_shred(
+                &shred,
+                slot_meta,
+                &self.last_root,
+                leader_schedule,
+                is_recovered,
+            ) {
                 return false;
             }
         }
@@ -1184,6 +1020,8 @@ impl Blockstore {
         shred: &Shred,
         slot_meta: &SlotMeta,
         last_root: &RwLock<u64>,
+        leader_schedule: Option<&Arc<LeaderScheduleCache>>,
+        is_recovered: bool,
     ) -> bool {
         let shred_index = u64::from(shred.index());
         let slot = shred.slot();
@@ -1198,13 +1036,16 @@ impl Blockstore {
         // for the slot
         let last_index = slot_meta.last_index;
         if shred_index >= last_index {
+            let leader_pubkey = leader_schedule
+                .map(|leader_schedule| leader_schedule.slot_leader_at(slot, None))
+                .unwrap_or(None);
             datapoint_error!(
                 "blockstore_error",
                 (
                     "error",
                     format!(
-                        "Slot {}: received index {} >= slot.last_index {}",
-                        slot, shred_index, last_index
+                        "Leader {:?}, slot {}: received index {} >= slot.last_index {}, is_recovered: {}",
+                        leader_pubkey, slot, shred_index, last_index, is_recovered
                     ),
                     String
                 )
@@ -1214,13 +1055,16 @@ impl Blockstore {
         // Check that we do not receive a shred with "last_index" true, but shred_index
         // less than our current received
         if last_in_slot && shred_index < slot_meta.received {
+            let leader_pubkey = leader_schedule
+                .map(|leader_schedule| leader_schedule.slot_leader_at(slot, None))
+                .unwrap_or(None);
             datapoint_error!(
                 "blockstore_error",
                 (
                     "error",
                     format!(
-                        "Slot {}: received shred_index {} < slot.received {}",
-                        slot, shred_index, slot_meta.received
+                        "Leader {:?}, slot {}: received shred_index {} < slot.received {}, is_recovered: {}",
+                        leader_pubkey, slot, shred_index, slot_meta.received, is_recovered
                     ),
                     String
                 )
@@ -2036,29 +1880,11 @@ impl Blockstore {
             return Err(BlockstoreError::DeadSlot);
         }
 
-        // lowest_cleanup_slot is the last slot that was not cleaned up by
-        // LedgerCleanupService
-        let lowest_cleanup_slot = self.lowest_cleanup_slot.read().unwrap();
-        if *lowest_cleanup_slot > 0 && *lowest_cleanup_slot >= slot {
-            return Err(BlockstoreError::SlotCleanedUp);
-        }
-
-        let slot_meta_cf = self.db.column::<cf::SlotMeta>();
-        let slot_meta = slot_meta_cf.get(slot)?;
-        if slot_meta.is_none() {
-            return Ok((vec![], 0, false));
-        }
-
-        let slot_meta = slot_meta.unwrap();
-        // Find all the ranges for the completed data blocks
-        let completed_ranges = Self::get_completed_data_ranges(
-            start_index as u32,
-            &slot_meta.completed_data_indexes[..],
-            slot_meta.consumed as u32,
-        );
+        let (completed_ranges, slot_meta) = self.get_completed_ranges(slot, start_index)?;
         if completed_ranges.is_empty() {
             return Ok((vec![], 0, false));
         }
+        let slot_meta = slot_meta.unwrap();
         let num_shreds = completed_ranges
             .last()
             .map(|(_, end_index)| u64::from(*end_index) - start_index + 1)
@@ -2079,12 +1905,41 @@ impl Blockstore {
         Ok((entries, num_shreds, slot_meta.is_full()))
     }
 
+    fn get_completed_ranges(
+        &self,
+        slot: Slot,
+        start_index: u64,
+    ) -> Result<(CompletedRanges, Option<SlotMeta>)> {
+        // lowest_cleanup_slot is the last slot that was not cleaned up by
+        // LedgerCleanupService
+        let lowest_cleanup_slot = self.lowest_cleanup_slot.read().unwrap();
+        if *lowest_cleanup_slot > slot {
+            return Err(BlockstoreError::SlotCleanedUp);
+        }
+
+        let slot_meta_cf = self.db.column::<cf::SlotMeta>();
+        let slot_meta = slot_meta_cf.get(slot)?;
+        if slot_meta.is_none() {
+            return Ok((vec![], slot_meta));
+        }
+
+        let slot_meta = slot_meta.unwrap();
+        // Find all the ranges for the completed data blocks
+        let completed_ranges = Self::get_completed_data_ranges(
+            start_index as u32,
+            &slot_meta.completed_data_indexes[..],
+            slot_meta.consumed as u32,
+        );
+
+        Ok((completed_ranges, Some(slot_meta)))
+    }
+
     // Get the range of indexes [start_index, end_index] of every completed data block
     fn get_completed_data_ranges(
         mut start_index: u32,
         completed_data_end_indexes: &[u32],
         consumed: u32,
-    ) -> Vec<(u32, u32)> {
+    ) -> CompletedRanges {
         let mut completed_data_ranges = vec![];
         let floor = completed_data_end_indexes
             .iter()
@@ -2166,6 +2021,30 @@ impl Blockstore {
         })
     }
 
+    fn get_any_valid_slot_entries(&self, slot: Slot, start_index: u64) -> Vec<Entry> {
+        let (completed_ranges, slot_meta) = self
+            .get_completed_ranges(slot, start_index)
+            .unwrap_or_default();
+        if completed_ranges.is_empty() {
+            return vec![];
+        }
+        let slot_meta = slot_meta.unwrap();
+
+        let entries: Vec<Vec<Entry>> = PAR_THREAD_POOL_ALL_CPUS.with(|thread_pool| {
+            thread_pool.borrow().install(|| {
+                completed_ranges
+                    .par_iter()
+                    .map(|(start_index, end_index)| {
+                        self.get_entries_in_data_block(slot, *start_index, *end_index, &slot_meta)
+                            .unwrap_or_default()
+                    })
+                    .collect()
+            })
+        });
+
+        entries.into_iter().flatten().collect()
+    }
+
     // Returns slots connecting to any element of the list `slots`.
     pub fn get_slots_since(&self, slots: &[u64]) -> Result<HashMap<u64, Vec<u64>>> {
         // Return error if there was a database error during lookup of any of the
@@ -2308,6 +2187,10 @@ impl Blockstore {
     pub fn storage_size(&self) -> Result<u64> {
         self.db.storage_size()
     }
+
+    pub fn is_primary_access(&self) -> bool {
+        self.db.is_primary_access()
+    }
 }
 
 fn update_slot_meta(
@@ -2740,12 +2623,13 @@ pub fn create_new_ledger(
     ledger_path: &Path,
     genesis_config: &GenesisConfig,
     max_genesis_archive_unpacked_size: u64,
+    access_type: AccessType,
 ) -> Result<Hash> {
     Blockstore::destroy(ledger_path)?;
     genesis_config.write(&ledger_path)?;
 
     // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
-    let blockstore = Blockstore::open(ledger_path)?;
+    let blockstore = Blockstore::open_with_access_type(ledger_path, access_type)?;
     let ticks_per_slot = genesis_config.ticks_per_slot;
     let hashes_per_tick = genesis_config.poh_config.hashes_per_tick.unwrap_or(0);
     let entries = create_ticks(ticks_per_slot, hashes_per_tick, genesis_config.hash());
@@ -2876,7 +2760,11 @@ pub fn get_ledger_path_from_name(name: &str) -> PathBuf {
 #[macro_export]
 macro_rules! create_new_tmp_ledger {
     ($genesis_config:expr) => {
-        $crate::blockstore::create_new_ledger_from_name($crate::tmp_ledger_name!(), $genesis_config)
+        $crate::blockstore::create_new_ledger_from_name(
+            $crate::tmp_ledger_name!(),
+            $genesis_config,
+            $crate::blockstore_db::AccessType::PrimaryOnly,
+        )
     };
 }
 
@@ -2902,12 +2790,17 @@ pub fn verify_shred_slots(slot: Slot, parent_slot: Slot, last_root: Slot) -> boo
 //
 // Note: like `create_new_ledger` the returned ledger will have slot 0 full of ticks (and only
 // ticks)
-pub fn create_new_ledger_from_name(name: &str, genesis_config: &GenesisConfig) -> (PathBuf, Hash) {
+pub fn create_new_ledger_from_name(
+    name: &str,
+    genesis_config: &GenesisConfig,
+    access_type: AccessType,
+) -> (PathBuf, Hash) {
     let ledger_path = get_ledger_path_from_name(name);
     let blockhash = create_new_ledger(
         &ledger_path,
         genesis_config,
         MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+        access_type,
     )
     .unwrap();
     (ledger_path, blockhash)
@@ -3050,7 +2943,7 @@ pub mod tests {
     use std::{iter::FromIterator, time::Duration};
 
     // used for tests only
-    fn make_slot_entries_with_transactions(num_entries: u64) -> Vec<Entry> {
+    pub(crate) fn make_slot_entries_with_transactions(num_entries: u64) -> Vec<Entry> {
         let mut entries: Vec<Entry> = Vec::new();
         for x in 0..num_entries {
             let transaction = Transaction::new_with_compiled_instructions(
@@ -3067,99 +2960,6 @@ pub mod tests {
         entries
     }
 
-    // check that all columns are either empty or start at `min_slot`
-    fn test_all_empty_or_min(blockstore: &Blockstore, min_slot: Slot) {
-        let condition_met = blockstore
-            .db
-            .iter::<cf::SlotMeta>(IteratorMode::Start)
-            .unwrap()
-            .next()
-            .map(|(slot, _)| slot >= min_slot)
-            .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::Root>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::ShredData>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|((slot, _), _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::ShredCode>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|((slot, _), _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::DeadSlots>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::DuplicateSlots>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::ErasureMeta>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|((slot, _), _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::Orphans>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::Index>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::TransactionStatus>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|((primary_index, _, slot), _)| {
-                    slot >= min_slot || (primary_index == 2 && slot == 0)
-                })
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::AddressSignatures>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|((primary_index, _, slot, _), _)| {
-                    slot >= min_slot || (primary_index == 2 && slot == 0)
-                })
-                .unwrap_or(true)
-            & blockstore
-                .db
-                .iter::<cf::Rewards>(IteratorMode::Start)
-                .unwrap()
-                .next()
-                .map(|(slot, _)| slot >= min_slot)
-                .unwrap_or(true);
-        assert!(condition_met);
-    }
-
     #[test]
     fn test_create_new_ledger() {
         let mint_total = 1_000_000_000_000;
@@ -3430,7 +3230,9 @@ pub mod tests {
         assert!(ledger.get_data_shreds(slot, 0, 1, &mut buf).is_ok());
 
         let max_purge_slot = 1;
-        ledger.run_purge(0, max_purge_slot).unwrap();
+        ledger
+            .run_purge(0, max_purge_slot, PurgeType::PrimaryIndex)
+            .unwrap();
         *ledger.lowest_cleanup_slot.write().unwrap() = max_purge_slot;
 
         let mut buf = [0; 4096];
@@ -4693,7 +4495,7 @@ pub mod tests {
                 }
             };
             assert_eq!(
-                Blockstore::should_insert_data_shred(&shred7, &slot_meta, &last_root),
+                Blockstore::should_insert_data_shred(&shred7, &slot_meta, &last_root, None, false),
                 false
             );
 
@@ -4709,7 +4511,7 @@ pub mod tests {
                 panic!("Shred in unexpected format")
             }
             assert_eq!(
-                Blockstore::should_insert_data_shred(&shred7, &slot_meta, &last_root),
+                Blockstore::should_insert_data_shred(&shred7, &slot_meta, &last_root, None, false),
                 false
             );
         }
@@ -4986,49 +4788,6 @@ pub mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
-    #[test]
-    fn test_purge_slots() {
-        let blockstore_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&blockstore_path).unwrap();
-        let (shreds, _) = make_many_slot_entries(0, 50, 5);
-        blockstore.insert_shreds(shreds, None, false).unwrap();
-
-        blockstore.purge_slots(0, 5);
-
-        test_all_empty_or_min(&blockstore, 6);
-
-        blockstore.purge_slots(0, 50);
-
-        // min slot shouldn't matter, blockstore should be empty
-        test_all_empty_or_min(&blockstore, 100);
-        test_all_empty_or_min(&blockstore, 0);
-
-        blockstore
-            .slot_meta_iterator(0)
-            .unwrap()
-            .for_each(|(_, _)| {
-                panic!();
-            });
-
-        drop(blockstore);
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
-    }
-
-    #[test]
-    fn test_purge_huge() {
-        let blockstore_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&blockstore_path).unwrap();
-        let (shreds, _) = make_many_slot_entries(0, 5000, 10);
-        blockstore.insert_shreds(shreds, None, false).unwrap();
-
-        blockstore.purge_slots(0, 4999);
-
-        test_all_empty_or_min(&blockstore, 5000);
-
-        drop(blockstore);
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
-    }
-
     #[test]
     fn test_iter_bounds() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -5744,7 +5503,7 @@ pub mod tests {
             assert_eq!(first_address_entry.0, 0);
             assert_eq!(first_address_entry.2, slot0);
 
-            blockstore.run_purge(0, 8).unwrap();
+            blockstore.run_purge(0, 8, PurgeType::PrimaryIndex).unwrap();
             // First successful prune freezes index 0
             assert_eq!(
                 transaction_status_index_cf.get(0).unwrap().unwrap(),
@@ -5839,7 +5598,9 @@ pub mod tests {
             assert_eq!(index1_first_address_entry.0, 1);
             assert_eq!(index1_first_address_entry.2, slot1);
 
-            blockstore.run_purge(0, 18).unwrap();
+            blockstore
+                .run_purge(0, 18, PurgeType::PrimaryIndex)
+                .unwrap();
             // Successful prune toggles TransactionStatusIndex
             assert_eq!(
                 transaction_status_index_cf.get(0).unwrap().unwrap(),
@@ -5885,169 +5646,6 @@ pub mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
-    #[test]
-    #[allow(clippy::cognitive_complexity)]
-    fn test_purge_transaction_status() {
-        let blockstore_path = get_tmp_ledger_path!();
-        {
-            let blockstore = Blockstore::open(&blockstore_path).unwrap();
-            let transaction_status_index_cf = blockstore.db.column::<cf::TransactionStatusIndex>();
-            let slot = 10;
-            for _ in 0..5 {
-                let random_bytes: Vec<u8> = (0..64).map(|_| rand::random::<u8>()).collect();
-                blockstore
-                    .write_transaction_status(
-                        slot,
-                        Signature::new(&random_bytes),
-                        vec![&Pubkey::new(&random_bytes[0..32])],
-                        vec![&Pubkey::new(&random_bytes[32..])],
-                        &TransactionStatusMeta::default(),
-                    )
-                    .unwrap();
-            }
-            // Purge to freeze index 0
-            blockstore.run_purge(0, 1).unwrap();
-            let mut status_entry_iterator = blockstore
-                .db
-                .iter::<cf::TransactionStatus>(IteratorMode::From(
-                    cf::TransactionStatus::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..5 {
-                let entry = status_entry_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            let mut address_transactions_iterator = blockstore
-                .db
-                .iter::<cf::AddressSignatures>(IteratorMode::From(
-                    (0, Pubkey::default(), 0, Signature::default()),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..10 {
-                let entry = address_transactions_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            assert_eq!(
-                transaction_status_index_cf.get(0).unwrap().unwrap(),
-                TransactionStatusIndexMeta {
-                    max_slot: 10,
-                    frozen: true,
-                }
-            );
-
-            // Low purge should not affect state
-            blockstore.run_purge(0, 5).unwrap();
-            let mut status_entry_iterator = blockstore
-                .db
-                .iter::<cf::TransactionStatus>(IteratorMode::From(
-                    cf::TransactionStatus::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..5 {
-                let entry = status_entry_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            let mut address_transactions_iterator = blockstore
-                .db
-                .iter::<cf::AddressSignatures>(IteratorMode::From(
-                    cf::AddressSignatures::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..10 {
-                let entry = address_transactions_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            assert_eq!(
-                transaction_status_index_cf.get(0).unwrap().unwrap(),
-                TransactionStatusIndexMeta {
-                    max_slot: 10,
-                    frozen: true,
-                }
-            );
-
-            // Test boundary conditions: < slot should not purge statuses; <= slot should
-            blockstore.run_purge(0, 9).unwrap();
-            let mut status_entry_iterator = blockstore
-                .db
-                .iter::<cf::TransactionStatus>(IteratorMode::From(
-                    cf::TransactionStatus::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..5 {
-                let entry = status_entry_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            let mut address_transactions_iterator = blockstore
-                .db
-                .iter::<cf::AddressSignatures>(IteratorMode::From(
-                    cf::AddressSignatures::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            for _ in 0..10 {
-                let entry = address_transactions_iterator.next().unwrap().0;
-                assert_eq!(entry.0, 0);
-                assert_eq!(entry.2, slot);
-            }
-            assert_eq!(
-                transaction_status_index_cf.get(0).unwrap().unwrap(),
-                TransactionStatusIndexMeta {
-                    max_slot: 10,
-                    frozen: true,
-                }
-            );
-
-            blockstore.run_purge(0, 10).unwrap();
-            let mut status_entry_iterator = blockstore
-                .db
-                .iter::<cf::TransactionStatus>(IteratorMode::From(
-                    cf::TransactionStatus::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            let padding_entry = status_entry_iterator.next().unwrap().0;
-            assert_eq!(padding_entry.0, 2);
-            assert_eq!(padding_entry.2, 0);
-            assert!(status_entry_iterator.next().is_none());
-            let mut address_transactions_iterator = blockstore
-                .db
-                .iter::<cf::AddressSignatures>(IteratorMode::From(
-                    cf::AddressSignatures::as_index(0),
-                    IteratorDirection::Forward,
-                ))
-                .unwrap();
-            let padding_entry = address_transactions_iterator.next().unwrap().0;
-            assert_eq!(padding_entry.0, 2);
-            assert_eq!(padding_entry.2, 0);
-            assert!(address_transactions_iterator.next().is_none());
-            assert_eq!(
-                transaction_status_index_cf.get(0).unwrap().unwrap(),
-                TransactionStatusIndexMeta {
-                    max_slot: 0,
-                    frozen: false,
-                }
-            );
-            assert_eq!(
-                transaction_status_index_cf.get(1).unwrap().unwrap(),
-                TransactionStatusIndexMeta {
-                    max_slot: 0,
-                    frozen: true,
-                }
-            );
-        }
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
-    }
-
     #[test]
     fn test_get_transaction_status() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -6239,7 +5837,7 @@ pub mod tests {
             );
         }
 
-        blockstore.run_purge(0, 2).unwrap();
+        blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap();
         *blockstore.lowest_cleanup_slot.write().unwrap() = slot;
         for (transaction, _) in expected_transactions {
             let signature = transaction.signatures[0];
@@ -6275,7 +5873,7 @@ pub mod tests {
                     .unwrap();
             }
             // Purge to freeze index 0
-            blockstore.run_purge(0, 1).unwrap();
+            blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap();
             let slot1 = 20;
             for x in 5..9 {
                 let signature = Signature::new(&[x; 64]);
@@ -6335,7 +5933,9 @@ pub mod tests {
             }
 
             // Purge index 0
-            blockstore.run_purge(0, 10).unwrap();
+            blockstore
+                .run_purge(0, 10, PurgeType::PrimaryIndex)
+                .unwrap();
             assert_eq!(
                 blockstore
                     .get_confirmed_signatures_for_address(address0, 0, 50)
@@ -6472,7 +6072,7 @@ pub mod tests {
                 blockstore.insert_shreds(shreds, None, false).unwrap();
             }
             assert_eq!(blockstore.lowest_slot(), 1);
-            blockstore.run_purge(0, 5).unwrap();
+            blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap();
             assert_eq!(blockstore.lowest_slot(), 6);
         }
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");

ledger/src/blockstore_db.rs

@@ -126,11 +126,22 @@ pub mod columns {
     pub struct Rewards;
 }
 
+pub enum AccessType {
+    PrimaryOnly,
+    TryPrimaryThenSecondary,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ActualAccessType {
+    Primary,
+    Secondary,
+}
+
 #[derive(Debug)]
-struct Rocks(rocksdb::DB);
+struct Rocks(rocksdb::DB, ActualAccessType);
 
 impl Rocks {
-    fn open(path: &Path) -> Result<Rocks> {
+    fn open(path: &Path, access_type: AccessType) -> Result<Rocks> {
         use columns::{
             AddressSignatures, DeadSlots, DuplicateSlots, ErasureMeta, Index, Orphans, Rewards,
             Root, ShredCode, ShredData, SlotMeta, TransactionStatus, TransactionStatusIndex,
@@ -139,7 +150,7 @@ impl Rocks {
         fs::create_dir_all(&path)?;
 
         // Use default database options
-        let db_options = get_db_options();
+        let mut db_options = get_db_options();
 
         // Column family names
         let meta_cf_descriptor = ColumnFamilyDescriptor::new(SlotMeta::NAME, get_cf_options());
@@ -165,23 +176,53 @@ impl Rocks {
         let rewards_cf_descriptor = ColumnFamilyDescriptor::new(Rewards::NAME, get_cf_options());
 
         let cfs = vec![
-            meta_cf_descriptor,
-            dead_slots_cf_descriptor,
-            duplicate_slots_cf_descriptor,
-            erasure_meta_cf_descriptor,
-            orphans_cf_descriptor,
-            root_cf_descriptor,
-            index_cf_descriptor,
-            shred_data_cf_descriptor,
-            shred_code_cf_descriptor,
-            transaction_status_cf_descriptor,
-            address_signatures_cf_descriptor,
-            transaction_status_index_cf_descriptor,
-            rewards_cf_descriptor,
+            (SlotMeta::NAME, meta_cf_descriptor),
+            (DeadSlots::NAME, dead_slots_cf_descriptor),
+            (DuplicateSlots::NAME, duplicate_slots_cf_descriptor),
+            (ErasureMeta::NAME, erasure_meta_cf_descriptor),
+            (Orphans::NAME, orphans_cf_descriptor),
+            (Root::NAME, root_cf_descriptor),
+            (Index::NAME, index_cf_descriptor),
+            (ShredData::NAME, shred_data_cf_descriptor),
+            (ShredCode::NAME, shred_code_cf_descriptor),
+            (TransactionStatus::NAME, transaction_status_cf_descriptor),
+            (AddressSignatures::NAME, address_signatures_cf_descriptor),
+            (
+                TransactionStatusIndex::NAME,
+                transaction_status_index_cf_descriptor,
+            ),
+            (Rewards::NAME, rewards_cf_descriptor),
         ];
 
         // Open the database
-        let db = Rocks(DB::open_cf_descriptors(&db_options, path, cfs)?);
+        let db = match access_type {
+            AccessType::PrimaryOnly => Rocks(
+                DB::open_cf_descriptors(&db_options, path, cfs.into_iter().map(|c| c.1))?,
+                ActualAccessType::Primary,
+            ),
+            AccessType::TryPrimaryThenSecondary => {
+                let names: Vec<_> = cfs.iter().map(|c| c.0).collect();
+
+                match DB::open_cf_descriptors(&db_options, path, cfs.into_iter().map(|c| c.1)) {
+                    Ok(db) => Rocks(db, ActualAccessType::Primary),
+                    Err(err) => {
+                        let secondary_path = path.join("solana-secondary");
+
+                        warn!("Error when opening as primary: {}", err);
+                        warn!("Trying as secondary at : {:?}", secondary_path);
+                        warn!("This active secondary db use may temporarily cause the performance of another db use (like by validator) to degrade");
+
+                        // This is needed according to https://github.com/facebook/rocksdb/wiki/Secondary-instance
+                        db_options.set_max_open_files(-1);
+
+                        Rocks(
+                            DB::open_cf_as_secondary(&db_options, path, &secondary_path, names)?,
+                            ActualAccessType::Secondary,
+                        )
+                    }
+                }
+            }
+        };
 
         Ok(db)
     }
@@ -266,6 +307,10 @@ impl Rocks {
         self.0.write(batch)?;
         Ok(())
     }
+
+    fn is_primary_access(&self) -> bool {
+        self.1 == ActualAccessType::Primary
+    }
 }
 
 pub trait Column {
@@ -337,10 +382,14 @@ impl Column for columns::TransactionStatus {
     }
 
     fn index(key: &[u8]) -> (u64, Signature, Slot) {
-        let index = BigEndian::read_u64(&key[0..8]);
-        let signature = Signature::new(&key[8..72]);
-        let slot = BigEndian::read_u64(&key[72..80]);
-        (index, signature, slot)
+        if key.len() != 80 {
+            Self::as_index(0)
+        } else {
+            let index = BigEndian::read_u64(&key[0..8]);
+            let signature = Signature::new(&key[8..72]);
+            let slot = BigEndian::read_u64(&key[72..80]);
+            (index, signature, slot)
+        }
     }
 
     fn primary_index(index: Self::Index) -> u64 {
@@ -577,8 +626,8 @@ pub struct WriteBatch<'a> {
 }
 
 impl Database {
-    pub fn open(path: &Path) -> Result<Self> {
-        let backend = Arc::new(Rocks::open(path)?);
+    pub fn open(path: &Path, access_type: AccessType) -> Result<Self> {
+        let backend = Arc::new(Rocks::open(path, access_type)?);
 
         Ok(Database {
             backend,
@@ -660,23 +709,19 @@ impl Database {
         Ok(fs_extra::dir::get_size(&self.path)?)
     }
 
-    // Adds a range to delete to the given write batch and returns whether or not the column has reached
-    // its end
-    pub fn delete_range_cf<C>(&self, batch: &mut WriteBatch, from: Slot, to: Slot) -> Result<bool>
+    // Adds a range to delete to the given write batch
+    pub fn delete_range_cf<C>(&self, batch: &mut WriteBatch, from: Slot, to: Slot) -> Result<()>
     where
         C: Column + ColumnName,
     {
         let cf = self.cf_handle::<C>();
         let from_index = C::as_index(from);
         let to_index = C::as_index(to);
-        let result = batch.delete_range_cf::<C>(cf, from_index, to_index);
-        let max_slot = self
-            .iter::<C>(IteratorMode::End)?
-            .next()
-            .map(|(i, _)| C::primary_index(i))
-            .unwrap_or(0);
-        let end = max_slot <= to;
-        result.map(|_| end)
+        batch.delete_range_cf::<C>(cf, from_index, to_index)
+    }
+
+    pub fn is_primary_access(&self) -> bool {
+        self.backend.is_primary_access()
     }
 }
 

ledger/src/blockstore_processor.rs

@@ -336,9 +336,14 @@ pub fn process_blockstore_from_root(
         }
     }
 
-    blockstore
-        .set_roots(&[start_slot])
-        .expect("Couldn't set root slot on startup");
+    // ensure start_slot is rooted for correct replay
+    if blockstore.is_primary_access() {
+        blockstore
+            .set_roots(&[start_slot])
+            .expect("Couldn't set root slot on startup");
+    } else if !blockstore.is_root(start_slot) {
+        panic!("starting slot isn't root and can't update due to being secondary blockstore access: {}", start_slot);
+    }
 
     if let Ok(metas) = blockstore.slot_meta_iterator(start_slot) {
         if let Some((slot, _meta)) = metas.last() {
@@ -786,10 +791,14 @@ fn process_single_slot(
     // see DuplicateSignature errors later in ReplayStage
     confirm_full_slot(blockstore, bank, opts, recyclers, progress).map_err(|err| {
         let slot = bank.slot();
-        blockstore
-            .set_dead_slot(slot)
-            .expect("Failed to mark slot as dead in blockstore");
         warn!("slot {} failed to verify: {}", slot, err);
+        if blockstore.is_primary_access() {
+            blockstore
+                .set_dead_slot(slot)
+                .expect("Failed to mark slot as dead in blockstore");
+        } else if !blockstore.is_dead(slot) {
+            panic!("Failed slot isn't dead and can't update due to being secondary blockstore access: {}", slot);
+        }
         err
     })?;
 

ledger/src/entry.rs

@@ -226,7 +226,7 @@ impl EntryVerificationState {
 
                 verify_check_time.stop();
                 verification_state.duration_ms += gpu_time_ms + verify_check_time.as_ms();
-                inc_new_counter_warn!(
+                inc_new_counter_info!(
                     "entry_verify-duration",
                     verification_state.duration_ms as usize
                 );

ledger/src/hardened_unpack.rs

@@ -77,6 +77,8 @@ where
     let mut total_size: u64 = 0;
     let mut total_count: u64 = 0;
 
+    let mut total_entries = 0;
+    let mut last_log_update = Instant::now();
     for entry in archive.entries()? {
         let mut entry = entry?;
         let path = entry.path()?;
@@ -110,8 +112,15 @@ where
 
         // unpack_in does its own sanitization
         // ref: https://docs.rs/tar/*/tar/struct.Entry.html#method.unpack_in
-        check_unpack_result(entry.unpack_in(&unpack_dir)?, path_str)?
+        check_unpack_result(entry.unpack_in(&unpack_dir)?, path_str)?;
+        total_entries += 1;
+        let now = Instant::now();
+        if now.duration_since(last_log_update).as_secs() >= 10 {
+            info!("unpacked {} entries so far...", total_entries);
+            last_log_update = now;
+        }
     }
+    info!("unpacked {} entries total", total_entries);
 
     Ok(())
 }

ledger/src/snapshot_utils.rs

@@ -17,6 +17,7 @@ use solana_runtime::{
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash, pubkey::Pubkey};
 use std::{
     cmp::Ordering,
+    fmt,
     fs::{self, File},
     io::{BufReader, BufWriter, Error as IOError, ErrorKind, Read, Seek, SeekFrom, Write},
     path::{Path, PathBuf},
@@ -295,7 +296,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
 
 pub fn get_snapshot_paths<P: AsRef<Path>>(snapshot_path: P) -> Vec<SlotSnapshotPaths>
 where
-    P: std::fmt::Debug,
+    P: fmt::Debug,
 {
     match fs::read_dir(&snapshot_path) {
         Ok(paths) => {

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,22 +12,22 @@ homepage = "https://solana.com/"
 itertools = "0.9.0"
 log = "0.4.8"
 rand = "0.7.0"
-solana-config-program = { path = "../programs/config", version = "1.2.0" }
-solana-core = { path = "../core", version = "1.2.0" }
-solana-client = { path = "../client", version = "1.2.0" }
-solana-download-utils = { path = "../download-utils", version = "1.2.0" }
-solana-faucet = { path = "../faucet", version = "1.2.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.0" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.0" }
-solana-ledger = { path = "../ledger", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-runtime = { path = "../runtime", version = "1.2.0" }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.0" }
-solana-vest-program = { path = "../programs/vest", version = "1.2.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.0" }
+solana-config-program = { path = "../programs/config", version = "1.2.1" }
+solana-core = { path = "../core", version = "1.2.1" }
+solana-client = { path = "../client", version = "1.2.1" }
+solana-download-utils = { path = "../download-utils", version = "1.2.1" }
+solana-faucet = { path = "../faucet", version = "1.2.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.1" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.1" }
+solana-ledger = { path = "../ledger", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-runtime = { path = "../runtime", version = "1.2.1" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.1" }
+solana-vest-program = { path = "../programs/vest", version = "1.2.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.1" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.1" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

local-cluster/src/cluster_tests.rs

@@ -57,7 +57,7 @@ pub fn spend_and_verify_all_nodes<S: ::std::hash::BuildHasher>(
             .poll_get_balance_with_commitment(&funding_keypair.pubkey(), CommitmentConfig::recent())
             .expect("balance in source");
         assert!(bal > 0);
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .unwrap();
         let mut transaction =
@@ -103,7 +103,7 @@ pub fn send_many_transactions(
             .poll_get_balance_with_commitment(&funding_keypair.pubkey(), CommitmentConfig::recent())
             .expect("balance in source");
         assert!(bal > 0);
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .unwrap();
         let transfer_amount = thread_rng().gen_range(1, max_tokens_per_transfer);
@@ -242,7 +242,7 @@ pub fn kill_entry_and_spend_and_verify_rest(
             }
 
             let random_keypair = Keypair::new();
-            let (blockhash, _fee_calculator) = client
+            let (blockhash, _fee_calculator, _last_valid_slot) = client
                 .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                 .unwrap();
             let mut transaction = system_transaction::transfer(
@@ -284,7 +284,7 @@ pub fn kill_entry_and_spend_and_verify_rest(
     }
 }
 
-pub fn check_for_new_roots(num_new_roots: usize, contact_infos: &[ContactInfo]) {
+pub fn check_for_new_roots(num_new_roots: usize, contact_infos: &[ContactInfo], test_name: &str) {
     let mut roots = vec![HashSet::new(); contact_infos.len()];
     let mut done = false;
     let mut last_print = Instant::now();
@@ -295,7 +295,7 @@ pub fn check_for_new_roots(num_new_roots: usize, contact_infos: &[ContactInfo])
             roots[i].insert(slot);
             let min_node = roots.iter().map(|r| r.len()).min().unwrap_or(0);
             if last_print.elapsed().as_secs() > 3 {
-                info!("PARTITION_TEST min observed roots {}/16", min_node);
+                info!("{} min observed roots {}/16", test_name, min_node);
                 last_print = Instant::now();
             }
             done = min_node >= num_new_roots;

local-cluster/src/local_cluster.rs

@@ -1,4 +1,7 @@
-use crate::cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo};
+use crate::{
+    cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo},
+    cluster_tests,
+};
 use itertools::izip;
 use log::*;
 use solana_client::thin_client::{create_client, ThinClient};
@@ -137,7 +140,7 @@ impl LocalCluster {
             OperatingMode::Stable | OperatingMode::Preview => {
                 genesis_config.native_instruction_processors =
                     solana_genesis_programs::get_programs(genesis_config.operating_mode, 0)
-                        .unwrap_or_else(|| vec![])
+                        .unwrap_or_default()
             }
             _ => (),
         }
@@ -336,6 +339,25 @@ impl LocalCluster {
         Self::transfer_with_client(&client, source_keypair, dest_pubkey, lamports)
     }
 
+    pub fn check_for_new_roots(&self, num_new_roots: usize, test_name: &str) {
+        let alive_node_contact_infos: Vec<_> = self
+            .validators
+            .values()
+            .map(|v| v.info.contact_info.clone())
+            .collect();
+        assert!(!alive_node_contact_infos.is_empty());
+        info!("{} discovering nodes", test_name);
+        let cluster_nodes = discover_cluster(
+            &alive_node_contact_infos[0].gossip,
+            alive_node_contact_infos.len(),
+        )
+        .unwrap();
+        info!("{} discovered {} nodes", test_name, cluster_nodes.len());
+        info!("{} looking for new roots on all nodes", test_name);
+        cluster_tests::check_for_new_roots(num_new_roots, &alive_node_contact_infos, test_name);
+        info!("{} done waiting for roots", test_name);
+    }
+
     fn transfer_with_client(
         client: &ThinClient,
         source_keypair: &Keypair,
@@ -343,7 +365,7 @@ impl LocalCluster {
         lamports: u64,
     ) -> u64 {
         trace!("getting leader blockhash");
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .unwrap();
         let mut tx =

local-cluster/tests/local_cluster.rs

@@ -207,16 +207,21 @@ fn test_leader_failure_4() {
 /// * `leader_schedule` - An option that specifies whether the cluster should
 /// run with a fixed, predetermined leader schedule
 #[allow(clippy::cognitive_complexity)]
-fn run_cluster_partition(
-    partitions: &[&[(usize, bool)]],
+fn run_cluster_partition<E, F>(
+    partitions: &[&[usize]],
     leader_schedule: Option<(LeaderSchedule, Vec<Arc<Keypair>>)>,
-) {
+    on_partition_start: E,
+    on_partition_resolved: F,
+) where
+    E: Fn(&mut LocalCluster) -> (),
+    F: Fn(&mut LocalCluster) -> (),
+{
     solana_logger::setup();
     info!("PARTITION_TEST!");
     let num_nodes = partitions.len();
     let node_stakes: Vec<_> = partitions
         .iter()
-        .flat_map(|p| p.iter().map(|(stake_weight, _)| 100 * *stake_weight as u64))
+        .flat_map(|p| p.iter().map(|stake_weight| 100 * *stake_weight as u64))
         .collect();
     assert_eq!(node_stakes.len(), num_nodes);
     let cluster_lamports = node_stakes.iter().sum::<u64>() * 2;
@@ -226,7 +231,7 @@ fn run_cluster_partition(
     validator_config.enable_partition = Some(enable_partition.clone());
 
     // Returns:
-    // 1) The keys for the validiators
+    // 1) The keys for the validators
     // 2) The amount of time it would take to iterate through one full iteration of the given
     // leader schedule
     let (validator_keys, leader_schedule_time): (Vec<_>, u64) = {
@@ -252,7 +257,6 @@ fn run_cluster_partition(
         }
     };
 
-    let validator_pubkeys: Vec<_> = validator_keys.iter().map(|v| v.pubkey()).collect();
     let config = ClusterConfig {
         cluster_lamports,
         node_stakes,
@@ -287,7 +291,8 @@ fn run_cluster_partition(
 
         if reached_epoch {
             info!("PARTITION_TEST start partition");
-            enable_partition.clone().store(false, Ordering::Relaxed);
+            enable_partition.store(false, Ordering::Relaxed);
+            on_partition_start(&mut cluster);
             break;
         } else {
             sleep(Duration::from_millis(100));
@@ -298,56 +303,22 @@ fn run_cluster_partition(
     info!("PARTITION_TEST remove partition");
     enable_partition.store(true, Ordering::Relaxed);
 
-    let mut dead_nodes = HashSet::new();
-    let mut alive_node_contact_infos = vec![];
-    let should_exits: Vec<_> = partitions
-        .iter()
-        .flat_map(|p| p.iter().map(|(_, should_exit)| should_exit))
-        .collect();
-    assert_eq!(should_exits.len(), validator_pubkeys.len());
-    let timeout = 10;
-    if timeout > 0 {
-        // Give partitions time to propagate their blocks from during the partition
-        // after the partition resolves
-        let propagation_time = leader_schedule_time;
-        info!("PARTITION_TEST resolving partition. sleeping {}ms", timeout);
-        sleep(Duration::from_millis(10_000));
-        info!(
-            "PARTITION_TEST waiting for blocks to propagate after partition {}ms",
-            propagation_time
-        );
-        sleep(Duration::from_millis(propagation_time));
-        info!("PARTITION_TEST resuming normal operation");
-        for (pubkey, should_exit) in validator_pubkeys.iter().zip(should_exits) {
-            if *should_exit {
-                info!("Killing validator with id: {}", pubkey);
-                cluster.exit_node(pubkey);
-                dead_nodes.insert(*pubkey);
-            } else {
-                alive_node_contact_infos.push(
-                    cluster
-                        .validators
-                        .get(pubkey)
-                        .unwrap()
-                        .info
-                        .contact_info
-                        .clone(),
-                );
-            }
-        }
-    }
-
-    assert_eq!(alive_node_contact_infos.is_empty(), false);
-    info!("PARTITION_TEST discovering nodes");
-    let cluster_nodes = discover_cluster(
-        &alive_node_contact_infos[0].gossip,
-        alive_node_contact_infos.len(),
-    )
-    .unwrap();
-    info!("PARTITION_TEST discovered {} nodes", cluster_nodes.len());
-    info!("PARTITION_TEST looking for new roots on all nodes");
-    cluster_tests::check_for_new_roots(16, &alive_node_contact_infos);
-    info!("PARTITION_TEST done waiting for roots");
+    // Give partitions time to propagate their blocks from during the partition
+    // after the partition resolves
+    let timeout = 10_000;
+    let propagation_time = leader_schedule_time;
+    info!(
+        "PARTITION_TEST resolving partition. sleeping {} ms",
+        timeout
+    );
+    sleep(Duration::from_millis(timeout));
+    info!(
+        "PARTITION_TEST waiting for blocks to propagate after partition {}ms",
+        propagation_time
+    );
+    sleep(Duration::from_millis(propagation_time));
+    info!("PARTITION_TEST resuming normal operation");
+    on_partition_resolved(&mut cluster);
 }
 
 #[allow(unused_attributes)]
@@ -355,7 +326,11 @@ fn run_cluster_partition(
 #[test]
 #[serial]
 fn test_cluster_partition_1_2() {
-    run_cluster_partition(&[&[(1, false)], &[(1, false), (1, false)]], None)
+    let empty = |_: &mut LocalCluster| {};
+    let on_partition_resolved = |cluster: &mut LocalCluster| {
+        cluster.check_for_new_roots(16, &"PARTITION_TEST");
+    };
+    run_cluster_partition(&[&[1], &[1, 1]], None, empty, on_partition_resolved)
 }
 
 #[allow(unused_attributes)]
@@ -363,13 +338,21 @@ fn test_cluster_partition_1_2() {
 #[test]
 #[serial]
 fn test_cluster_partition_1_1() {
-    run_cluster_partition(&[&[(1, false)], &[(1, false)]], None)
+    let empty = |_: &mut LocalCluster| {};
+    let on_partition_resolved = |cluster: &mut LocalCluster| {
+        cluster.check_for_new_roots(16, &"PARTITION_TEST");
+    };
+    run_cluster_partition(&[&[1], &[1]], None, empty, on_partition_resolved)
 }
 
 #[test]
 #[serial]
 fn test_cluster_partition_1_1_1() {
-    run_cluster_partition(&[&[(1, false)], &[(1, false)], &[(1, false)]], None)
+    let empty = |_: &mut LocalCluster| {};
+    let on_partition_resolved = |cluster: &mut LocalCluster| {
+        cluster.check_for_new_roots(16, &"PARTITION_TEST");
+    };
+    run_cluster_partition(&[&[1], &[1], &[1]], None, empty, on_partition_resolved)
 }
 
 #[test]
@@ -387,7 +370,7 @@ fn test_kill_partition() {
     // 5) Check for recovery
     let mut leader_schedule = vec![];
     let num_slots_per_validator = 8;
-    let partitions: [&[(usize, bool)]; 3] = [&[(9, true)], &[(10, false)], &[(10, false)]];
+    let partitions: [&[usize]; 3] = [&[9], &[10], &[10]];
     let validator_keys: Vec<_> = iter::repeat_with(|| Arc::new(Keypair::new()))
         .take(partitions.len())
         .collect();
@@ -406,12 +389,21 @@ fn test_kill_partition() {
     }
     info!("leader_schedule: {}", leader_schedule.len());
 
+    let empty = |_: &mut LocalCluster| {};
+    let validator_to_kill = validator_keys[0].pubkey();
+    let on_partition_resolved = |cluster: &mut LocalCluster| {
+        info!("Killing validator with id: {}", validator_to_kill);
+        cluster.exit_node(&validator_to_kill);
+        cluster.check_for_new_roots(16, &"PARTITION_TEST");
+    };
     run_cluster_partition(
         &partitions,
         Some((
             LeaderSchedule::new_from_schedule(leader_schedule),
             validator_keys,
         )),
+        empty,
+        on_partition_resolved,
     )
 }
 
@@ -605,7 +597,7 @@ fn generate_frozen_account_panic(mut cluster: LocalCluster, frozen_account: Arc<
     let mut i = 0;
     while !solana_runtime::accounts_db::FROZEN_ACCOUNT_PANIC.load(Ordering::Relaxed) {
         // Transfer from frozen account
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .unwrap();
         client
@@ -1113,12 +1105,7 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) {
     let cluster = LocalCluster::new(&cluster_config);
 
     // Check for new roots
-    let alive_node_contact_infos: Vec<_> = cluster
-        .validators
-        .values()
-        .map(|v| v.info.contact_info.clone())
-        .collect();
-    cluster_tests::check_for_new_roots(16, &alive_node_contact_infos);
+    cluster.check_for_new_roots(16, &"test_faulty_node");
 }
 
 #[test]

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "3.1.1"
 clap = "2.33.1"
 serde = "1.0.110"
 serde_json = "1.0.53"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.0"
+version = "1.2.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.2.0" }
-solana-metrics = { path = "../metrics", version = "1.2.0" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
+solana-metrics = { path = "../metrics", version = "1.2.1" }
 
 [target."cfg(unix)".dependencies]
 jemallocator = "0.3.2"

merkle-tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-merkle-tree"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Merkle Tree"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
 fast-math = "0.1"
 
 [dev-dependencies]

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ gethostname = "0.2.1"
 lazy_static = "1.4.0"
 log = "0.4.8"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.2.0" }
+solana-sdk = { path = "../sdk", version = "1.2.1" }
 
 [dev-dependencies]
 rand = "0.7.0"

multinode-demo/bootstrap-validator.sh

@@ -33,6 +33,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --gossip-port ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 = --dev-halt-at-slot ]]; then
+      args+=("$1" "$2")
+      shift 2
     elif [[ $1 = --dynamic-port-range ]]; then
       args+=("$1" "$2")
       shift 2

net-shaper/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-net-shaper"
 description = "The solana cluster network shaping tool"
-version = "1.2.0"
+version = "1.2.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,8 +13,8 @@ publish = false
 clap = "2.33.1"
 serde = "1.0.110"
 serde_json = "1.0.53"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
 rand = "0.7.0"
 
 [[bin]]

net-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-net-utils"
-version = "1.2.0"
+version = "1.2.1"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,9 +18,9 @@ rand = "0.7.0"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 socket2 = "0.3.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.0" }
-solana-logger = { path = "../logger", version = "1.2.0" }
-solana-version = { path = "../version", version = "1.2.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.1" }
+solana-logger = { path = "../logger", version = "1.2.1" }
+solana-version = { path = "../version", version = "1.2.1" }
 tokio = "0.1"
 tokio-codec = "0.1"