update dependence version for gag to latest support windows (#17801) (#17825)

* update dependence version for gag to leatest support windows

* fix compile on windows

* add Cargo.lock

(cherry picked from commit e0ab5ee4f8)

Co-authored-by: Govlzkoy <gotope@users.noreply.github.com>

Cargo.toml

@@ -39,11 +39,11 @@ members = [
     "metrics",
     "net-shaper",
     "notifier",
+    "poh",
     "poh-bench",
     "program-test",
     "programs/secp256k1",
     "programs/bpf_loader",
-    "programs/budget",
     "programs/config",
     "programs/exchange",
     "programs/failure",
@@ -52,6 +52,7 @@ members = [
     "programs/stake",
     "programs/vote",
     "remote-wallet",
+    "rpc",
     "runtime",
     "runtime/store-tool",
     "sdk",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ lazy_static = "1.4.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-config-program = { path = "../programs/config", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,23 +13,23 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,17 +14,18 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
+solana-poh = { path = "../poh", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-streamer = { path = "../streamer", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -4,11 +4,7 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::{
-    banking_stage::{create_test_recorder, BankingStage},
-    poh_recorder::PohRecorder,
-    poh_recorder::WorkingBankEntry,
-};
+use solana_core::banking_stage::BankingStage;
 use solana_gossip::{cluster_info::ClusterInfo, cluster_info::Node};
 use solana_ledger::{
     blockstore::Blockstore,
@@ -17,6 +13,7 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_perf::packet::to_packets_chunked;
+use solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry};
 use solana_runtime::{
     accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
 };
@@ -77,7 +74,7 @@ fn make_accounts_txs(
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
-            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
             if !same_payer {
                 new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
@@ -188,7 +185,7 @@ fn main() {
             genesis_config.hash(),
         );
         // Ignore any pesky duplicate signature errors in the case we are using single-payer
-        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
         fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
@@ -354,7 +351,7 @@ fn main() {
             if bank.slot() > 0 && bank.slot() % 16 == 0 {
                 for tx in transactions.iter_mut() {
                     tx.message.recent_blockhash = bank.last_blockhash();
-                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
                 verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,16 +15,16 @@ borsh = "0.8.1"
 borsh-derive = "0.8.1"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.0" }
-solana-program = { path = "../sdk/program", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.7.1" }
+solana-program = { path = "../sdk/program", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.0" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-banks-server = { path = "../banks-server", version = "=1.7.1" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 
 [dev-dependencies]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,10 +14,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,22 +18,22 @@ rand = "0.7.0"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-genesis = { path = "../genesis", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-genesis = { path = "../genesis", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-streamer = { path = "../streamer", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.11"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-genesis = { path = "../genesis", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-genesis = { path = "../genesis", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/test-bench.sh

@@ -49,6 +49,10 @@ _ "$cargo" nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
 _ "$cargo" nightly bench --manifest-path gossip/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
+# Run poh benches
+_ "$cargo" nightly bench --manifest-path poh/Cargo.toml ${V:+--verbose} \
+  -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
+
 # Run core benches
 _ "$cargo" nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,8 +12,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.8.0"
 uriparse = "0.6.3"

clap-utils/src/keypair.rs

@@ -24,9 +24,11 @@ use {
         },
     },
     std::{
+        cell::RefCell,
         convert::TryFrom,
         error,
         io::{stdin, stdout, Write},
+        ops::Deref,
         process::exit,
         str::FromStr,
         sync::Arc,
@@ -89,33 +91,49 @@ impl CliSignerInfo {
             .collect()
     }
 }
-#[derive(Debug)]
+
+#[derive(Debug, Default)]
 pub struct DefaultSigner {
     pub arg_name: String,
     pub path: String,
+    is_path_checked: RefCell<bool>,
 }
 
 impl DefaultSigner {
-    pub fn new(path: String) -> Self {
+    pub fn new<AN: AsRef<str>, P: AsRef<str>>(arg_name: AN, path: P) -> Self {
+        let arg_name = arg_name.as_ref().to_string();
+        let path = path.as_ref().to_string();
         Self {
-            arg_name: "keypair".to_string(),
+            arg_name,
             path,
+            ..Self::default()
         }
     }
-    pub fn from_path(path: String) -> Result<Self, Box<dyn error::Error>> {
-        std::fs::metadata(&path)
-            .map_err(|_| {
-                std::io::Error::new(
-                    std::io::ErrorKind::Other,
-                    format!(
-                    "No default signer found, run \"solana-keygen new -o {}\" to create a new one",
-                    path
-                ),
-                )
-                .into()
-            })
-            .map(|_| Self::new(path))
+
+    fn path(&self) -> Result<&str, Box<dyn std::error::Error>> {
+        if !self.is_path_checked.borrow().deref() {
+            parse_signer_source(&self.path)
+                .and_then(|s| {
+                    if let SignerSourceKind::Filepath(path) = &s.kind {
+                        std::fs::metadata(path).map(|_| ()).map_err(|e| e.into())
+                    } else {
+                        Ok(())
+                    }
+                })
+                .map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        format!(
+                        "No default signer found, run \"solana-keygen new -o {}\" to create a new one",
+                        self.path
+                    ),
+                    )
+                })?;
+            *self.is_path_checked.borrow_mut() = true;
+        }
+        Ok(&self.path)
     }
+
     pub fn generate_unique_signers(
         &self,
         bulk_signers: Vec<Option<Box<dyn Signer>>>,
@@ -145,7 +163,7 @@ impl DefaultSigner {
         matches: &ArgMatches,
         wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
-        signer_from_path(matches, &self.path, &self.arg_name, wallet_manager)
+        signer_from_path(matches, self.path()?, &self.arg_name, wallet_manager)
     }
 
     pub fn signer_from_path_with_config(
@@ -154,7 +172,13 @@ impl DefaultSigner {
         wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
         config: &SignerFromPathConfig,
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
-        signer_from_path_with_config(matches, &self.path, &self.arg_name, wallet_manager, config)
+        signer_from_path_with_config(
+            matches,
+            self.path()?,
+            &self.arg_name,
+            wallet_manager,
+            config,
+        )
     }
 }
 
@@ -277,7 +301,9 @@ pub(crate) fn parse_signer_source<S: AsRef<str>>(
                     ASK_KEYWORD => Ok(SignerSource::new_legacy(SignerSourceKind::Prompt)),
                     _ => match Pubkey::from_str(source.as_str()) {
                         Ok(pubkey) => Ok(SignerSource::new(SignerSourceKind::Pubkey(pubkey))),
-                        Err(_) => Ok(SignerSource::new(SignerSourceKind::Filepath(source))),
+                        Err(_) => std::fs::metadata(source.as_str())
+                            .map(|_| SignerSource::new(SignerSourceKind::Filepath(source)))
+                            .map_err(|err| err.into()),
                     },
                 }
             }
@@ -751,6 +777,10 @@ mod tests {
         // Catchall into SignerSource::Filepath fails
         let junk = "sometextthatisnotapubkeyorfile".to_string();
         assert!(Pubkey::from_str(&junk).is_err());
+        assert!(matches!(
+            parse_signer_source(&junk),
+            Err(SignerSourceError::IoError(_))
+        ));
 
         let prompt = "prompt:".to_string();
         assert!(matches!(

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,20 +12,20 @@ documentation = "https://docs.rs/solana-cli-output"
 [dependencies]
 base64 = "0.13.0"
 chrono = { version = "0.4.11", features = ["serde"] }
-console = "0.11.3"
+console = "0.14.1"
 humantime = "2.0.1"
 Inflector = "0.11.4"
 indicatif = "0.15.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,7 +16,7 @@ chrono = { version = "0.4.11", features = ["serde"] }
 clap = "2.33.1"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.5", features = ["termination"] }
-console = "0.11.3"
+console = "0.14.1"
 dirs-next = "2.0.0"
 log = "0.4.11"
 Inflector = "0.11.4"
@@ -28,30 +28,30 @@ reqwest = { version = "0.11.2", default-features = false, features = ["blocking"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.0" }
-solana-cli-output = { path = "../cli-output", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-config-program = { path = "../programs/config", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.7.1" }
+solana-cli-output = { path = "../cli-output", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-config-program = { path = "../programs/config", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
 solana_rbpf = "=0.2.11"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.21"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.7.0" }
+solana-core = { path = "../core", version = "=1.7.1" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -32,7 +32,8 @@ use solana_client::{
         RpcLargestAccountsFilter, RpcSendTransactionConfig, RpcTransactionConfig,
         RpcTransactionLogsFilter,
     },
-    rpc_response::RpcKeyedAccount,
+    rpc_request::{RpcError, RpcResponseErrorData},
+    rpc_response::{RpcKeyedAccount, RpcSimulateTransactionResult},
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
@@ -1956,6 +1957,28 @@ where
 {
     match result {
         Err(err) => {
+            // If transaction simulation returns a known Custom InstructionError, decode it
+            if let ClientErrorKind::RpcError(RpcError::RpcResponseError {
+                data:
+                    RpcResponseErrorData::SendTransactionPreflightFailure(
+                        RpcSimulateTransactionResult {
+                            err:
+                                Some(TransactionError::InstructionError(
+                                    _,
+                                    InstructionError::Custom(code),
+                                )),
+                            ..
+                        },
+                    ),
+                ..
+            }) = err.kind()
+            {
+                if let Some(specific_error) = E::decode_custom_error_to_enum(*code) {
+                    return Err(specific_error.into());
+                }
+            }
+            // If the transaction was instead submitted and returned a known Custom
+            // InstructionError, decode it
             if let ClientErrorKind::TransactionError(TransactionError::InstructionError(
                 _,
                 InstructionError::Custom(code),
@@ -2299,7 +2322,7 @@ mod tests {
         let default_keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &default_keypair_file).unwrap();
 
-        let default_signer = DefaultSigner::new(default_keypair_file);
+        let default_signer = DefaultSigner::new("keypair", &default_keypair_file);
 
         let signer_info = default_signer
             .generate_unique_signers(vec![], &matches, &mut None)
@@ -2377,7 +2400,7 @@ mod tests {
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
         let keypair = read_keypair_file(&keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
         // Test Airdrop Subcommand
         let test_airdrop =
             test_commands
@@ -2905,7 +2928,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let default_keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &default_keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(default_keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         //Test Transfer Subcommand, SOL
         let from_keypair = keypair_from_seed(&[0u8; 32]).unwrap();

cli/src/cluster_query.rs

@@ -2098,7 +2098,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner::new(default_keypair_file);
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         let test_cluster_version = test_commands
             .clone()

cli/src/main.rs

@@ -173,12 +173,13 @@ pub fn parse_args<'a>(
         matches.value_of("json_rpc_url").unwrap_or(""),
         &config.json_rpc_url,
     );
+    let default_signer_arg_name = "keypair".to_string();
     let (_, default_signer_path) = CliConfig::compute_keypair_path_setting(
-        matches.value_of("keypair").unwrap_or(""),
+        matches.value_of(&default_signer_arg_name).unwrap_or(""),
         &config.keypair_path,
     );
 
-    let default_signer = DefaultSigner::from_path(default_signer_path.clone())?;
+    let default_signer = DefaultSigner::new(default_signer_arg_name, &default_signer_path);
 
     let CliCommandInfo {
         command,

cli/src/nonce.rs

@@ -596,7 +596,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner::new(default_keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let nonce_account_keypair = Keypair::new();
         write_keypair(&nonce_account_keypair, tmp_file.as_file_mut()).unwrap();

cli/src/program.rs

@@ -2131,7 +2131,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let test_command = test_commands.clone().get_matches_from(vec![
             "test",
@@ -2339,7 +2339,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let test_command = test_commands.clone().get_matches_from(vec![
@@ -2487,7 +2487,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let program_pubkey = Pubkey::new_unique();
         let new_authority_pubkey = Pubkey::new_unique();
@@ -2595,7 +2595,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         let buffer_pubkey = Pubkey::new_unique();
         let new_authority_pubkey = Pubkey::new_unique();
@@ -2652,7 +2652,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file);
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let buffer_pubkey = Pubkey::new_unique();
@@ -2751,7 +2751,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let keypair_file = make_tmp_path("keypair_file");
         write_keypair_file(&default_keypair, &keypair_file).unwrap();
-        let default_signer = DefaultSigner::new(keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &keypair_file);
 
         // defaults
         let buffer_pubkey = Pubkey::new_unique();

cli/src/stake.rs

@@ -2117,7 +2117,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner::new(default_keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let stake_account_keypair = Keypair::new();
         write_keypair(&stake_account_keypair, tmp_file.as_file_mut()).unwrap();

cli/src/vote.rs

@@ -826,7 +826,7 @@ mod tests {
         let default_keypair = Keypair::new();
         let (default_keypair_file, mut tmp_file) = make_tmp_file();
         write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
-        let default_signer = DefaultSigner::new(default_keypair_file.clone());
+        let default_signer = DefaultSigner::new("", &default_keypair_file);
 
         let test_authorize_voter = test_commands.clone().get_matches_from(vec![
             "test",

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -24,14 +24,14 @@ semver = "0.11.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tungstenite = "0.10.1"
@@ -40,7 +40,7 @@ url = "2.1.1"
 [dev-dependencies]
 assert_matches = "1.3.0"
 jsonrpc-http-server = "17.0.0"
-solana-logger = { path = "../logger", version = "=1.7.0" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -22,17 +22,12 @@ bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.1"
 byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
-core_affinity = "0.5.10"
 crossbeam-channel = "0.4"
 ed25519-dalek = "=1.0.1"
 fs_extra = "1.2.0"
 flate2 = "1.0"
 indexmap = { version = "1.5", features = ["rayon"] }
 itertools = "0.9.0"
-jsonrpc-core = "17.0.0"
-jsonrpc-core-client = { version = "17.0.0", features = ["ipc", "ws"] }
-jsonrpc-derive = "17.0.0"
-jsonrpc-http-server = "17.0.0"
 libc = "0.2.81"
 log = "0.4.11"
 lru = "0.6.1"
@@ -44,54 +39,52 @@ rand_chacha = "0.2.2"
 rand_core = "0.6.2"
 raptorq = "1.4.2"
 rayon = "1.5.0"
-regex = "1.3.9"
 retain_mut = "0.1.2"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
-serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
-solana-program-test = { path = "../program-test", version = "=1.7.0" }
-solana-rpc = { path = "../rpc", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-sys-tuner = { path = "../sys-tuner", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-banks-server = { path = "../banks-server", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
+solana-poh = { path = "../poh", version = "=1.7.1" }
+solana-program-test = { path = "../program-test", version = "=1.7.1" }
+solana-rpc = { path = "../rpc", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-streamer = { path = "../streamer", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-tokio = { version = "1", features = ["full"] }
-tokio_02 = { version = "0.2", package = "tokio", features = ["full"] }
-tokio-util = { version = "0.3", features = ["codec"] } # This crate needs to stay in sync with tokio_02, until that dependency can be removed
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.1" }
 trees = "0.2.1"
 
 [dev-dependencies]
+jsonrpc-core = "17.1.0"
+jsonrpc-core-client = { version = "17.1.0", features = ["ipc", "ws"] }
 matches = "0.1.6"
 num_cpus = "1.13.0"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde_json = "1.0.56"
 serial_test = "0.4.0"
+solana-version = { path = "../version", version = "=1.7.1" }
 symlink = "0.1.0"
 systemstat = "0.1.5"
+tokio_02 = { version = "0.2", package = "tokio", features = ["full"] }
 
 [build-dependencies]
 rustc_version = "0.2"
@@ -111,9 +104,6 @@ name = "gen_keys"
 [[bench]]
 name = "sigverify_stage"
 
-[[bench]]
-name = "poh"
-
 [[bench]]
 name = "retransmit_stage"
 

core/benches/banking_stage.rs

@@ -7,8 +7,7 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::banking_stage::{create_test_recorder, BankingStage, BankingStageStats};
-use solana_core::poh_recorder::WorkingBankEntry;
+use solana_core::banking_stage::{BankingStage, BankingStageStats};
 use solana_gossip::cluster_info::ClusterInfo;
 use solana_gossip::cluster_info::Node;
 use solana_ledger::blockstore_processor::process_entries;
@@ -17,6 +16,7 @@ use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
 use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
+use solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry};
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;

core/src/banking_stage.rs

@@ -1,20 +1,13 @@
 //! The `banking_stage` processes Transaction messages. It is intended to be used
 //! to contruct a software pipeline. The stage uses all available CPU cores and
 //! can do its processing in parallel with signature verification on the GPU.
-use crate::{
-    packet_hasher::PacketHasher,
-    poh_recorder::{PohRecorder, PohRecorderError, TransactionRecorder, WorkingBankEntry},
-    poh_service::{self, PohService},
-};
+use crate::packet_hasher::PacketHasher;
 use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError};
 use itertools::Itertools;
 use lru::LruCache;
 use retain_mut::RetainMut;
 use solana_gossip::cluster_info::ClusterInfo;
-use solana_ledger::{
-    blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
-    entry::hash_transactions, leader_schedule_cache::LeaderScheduleCache,
-};
+use solana_ledger::{blockstore_processor::TransactionStatusSender, entry::hash_transactions};
 use solana_measure::measure::Measure;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
 use solana_perf::{
@@ -22,6 +15,7 @@ use solana_perf::{
     packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
     perf_libs,
 };
+use solana_poh::poh_recorder::{PohRecorder, PohRecorderError, TransactionRecorder};
 use solana_runtime::{
     accounts_db::ErrorCounters,
     bank::{
@@ -39,7 +33,6 @@ use solana_sdk::{
         MAX_TRANSACTION_FORWARDING_DELAY_GPU,
     },
     message::Message,
-    poh_config::PohConfig,
     pubkey::Pubkey,
     short_vec::decode_shortu16_len,
     signature::Signature,
@@ -57,8 +50,7 @@ use std::{
     mem::size_of,
     net::UdpSocket,
     ops::DerefMut,
-    sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
-    sync::mpsc::Receiver,
+    sync::atomic::{AtomicU64, AtomicUsize, Ordering},
     sync::{Arc, Mutex},
     thread::{self, Builder, JoinHandle},
     time::Duration,
@@ -1392,66 +1384,29 @@ fn next_leader_tpu_forwards(
     }
 }
 
-pub fn create_test_recorder(
-    bank: &Arc<Bank>,
-    blockstore: &Arc<Blockstore>,
-    poh_config: Option<PohConfig>,
-) -> (
-    Arc<AtomicBool>,
-    Arc<Mutex<PohRecorder>>,
-    PohService,
-    Receiver<WorkingBankEntry>,
-) {
-    let exit = Arc::new(AtomicBool::new(false));
-    let poh_config = Arc::new(poh_config.unwrap_or_default());
-    let (mut poh_recorder, entry_receiver, record_receiver) = PohRecorder::new(
-        bank.tick_height(),
-        bank.last_blockhash(),
-        bank.slot(),
-        Some((4, 4)),
-        bank.ticks_per_slot(),
-        &Pubkey::default(),
-        blockstore,
-        &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
-        &poh_config,
-        exit.clone(),
-    );
-    poh_recorder.set_bank(&bank);
-
-    let poh_recorder = Arc::new(Mutex::new(poh_recorder));
-    let poh_service = PohService::new(
-        poh_recorder.clone(),
-        &poh_config,
-        &exit,
-        bank.ticks_per_slot(),
-        poh_service::DEFAULT_PINNED_CPU_CORE,
-        poh_service::DEFAULT_HASHES_PER_BATCH,
-        record_receiver,
-    );
-
-    (exit, poh_recorder, poh_service, entry_receiver)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{
-        poh_recorder::Record, poh_recorder::WorkingBank,
-        transaction_status_service::TransactionStatusService,
-    };
     use crossbeam_channel::unbounded;
     use itertools::Itertools;
     use solana_gossip::cluster_info::Node;
     use solana_ledger::{
-        blockstore::entries_to_test_shreds,
+        blockstore::{entries_to_test_shreds, Blockstore},
         entry::{next_entry, Entry, EntrySlice},
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
+        leader_schedule_cache::LeaderScheduleCache,
     };
     use solana_perf::packet::to_packets_chunked;
+    use solana_poh::{
+        poh_recorder::{create_test_recorder, Record, WorkingBank, WorkingBankEntry},
+        poh_service::PohService,
+    };
+    use solana_rpc::transaction_status_service::TransactionStatusService;
     use solana_sdk::{
         hash::Hash,
         instruction::InstructionError,
+        poh_config::PohConfig,
         signature::{Keypair, Signer},
         system_instruction::SystemError,
         system_transaction,
@@ -1461,7 +1416,10 @@ mod tests {
     use std::{
         net::SocketAddr,
         path::Path,
-        sync::atomic::{AtomicBool, Ordering},
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::Receiver,
+        },
         thread::sleep,
     };
 

core/src/broadcast_stage.rs

@@ -5,10 +5,7 @@ use self::{
     fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
     standard_broadcast_run::StandardBroadcastRun,
 };
-use crate::{
-    poh_recorder::WorkingBankEntry,
-    result::{Error, Result},
-};
+use crate::result::{Error, Result};
 use crossbeam_channel::{
     Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
     Sender as CrossbeamSender,
@@ -22,6 +19,7 @@ use solana_gossip::{
 use solana_ledger::{blockstore::Blockstore, shred::Shred};
 use solana_measure::measure::Measure;
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
+use solana_poh::poh_recorder::WorkingBankEntry;
 use solana_runtime::bank::Bank;
 use solana_sdk::timing::timestamp;
 use solana_sdk::{clock::Slot, pubkey::Pubkey};

core/src/broadcast_stage/broadcast_utils.rs

@@ -1,6 +1,6 @@
-use crate::poh_recorder::WorkingBankEntry;
 use crate::result::Result;
 use solana_ledger::{entry::Entry, shred::Shred};
+use solana_poh::poh_recorder::WorkingBankEntry;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use std::{

core/src/cluster_info_vote_listener.rs

@@ -1,6 +1,5 @@
 use crate::{
     optimistic_confirmation_verifier::OptimisticConfirmationVerifier,
-    poh_recorder::PohRecorder,
     replay_stage::DUPLICATE_THRESHOLD,
     result::{Error, Result},
     sigverify,
@@ -20,6 +19,7 @@ use solana_gossip::{
 use solana_ledger::blockstore::Blockstore;
 use solana_metrics::inc_new_counter_debug;
 use solana_perf::packet::{self, Packets};
+use solana_poh::poh_recorder::PohRecorder;
 use solana_rpc::{
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     rpc_subscriptions::RpcSubscriptions,

core/src/cluster_slots_service.rs

@@ -1,29 +1,32 @@
 use crate::cluster_slots::ClusterSlots;
+use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
 use solana_gossip::cluster_info::ClusterInfo;
-use solana_ledger::blockstore::{Blockstore, CompletedSlotsReceiver};
+use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_runtime::bank_forks::BankForks;
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
-        mpsc::RecvTimeoutError,
         {Arc, RwLock},
     },
     thread::{self, Builder, JoinHandle},
     time::{Duration, Instant},
 };
 
+pub type ClusterSlotsUpdateReceiver = Receiver<Vec<Slot>>;
+pub type ClusterSlotsUpdateSender = Sender<Vec<Slot>>;
+
 #[derive(Default, Debug)]
 struct ClusterSlotsServiceTiming {
     pub lowest_slot_elapsed: u64,
-    pub update_completed_slots_elapsed: u64,
+    pub process_cluster_slots_updates_elapsed: u64,
 }
 
 impl ClusterSlotsServiceTiming {
-    fn update(&mut self, lowest_slot_elapsed: u64, update_completed_slots_elapsed: u64) {
+    fn update(&mut self, lowest_slot_elapsed: u64, process_cluster_slots_updates_elapsed: u64) {
         self.lowest_slot_elapsed += lowest_slot_elapsed;
-        self.update_completed_slots_elapsed += update_completed_slots_elapsed;
+        self.process_cluster_slots_updates_elapsed += process_cluster_slots_updates_elapsed;
     }
 }
 
@@ -37,12 +40,12 @@ impl ClusterSlotsService {
         cluster_slots: Arc<ClusterSlots>,
         bank_forks: Arc<RwLock<BankForks>>,
         cluster_info: Arc<ClusterInfo>,
-        completed_slots_receiver: CompletedSlotsReceiver,
+        cluster_slots_update_receiver: ClusterSlotsUpdateReceiver,
         exit: Arc<AtomicBool>,
     ) -> Self {
         let id = cluster_info.id();
         Self::initialize_lowest_slot(id, &blockstore, &cluster_info);
-        Self::initialize_epoch_slots(&blockstore, &cluster_info, &completed_slots_receiver);
+        Self::initialize_epoch_slots(&bank_forks, &cluster_info);
         let t_cluster_slots_service = Builder::new()
             .name("solana-cluster-slots-service".to_string())
             .spawn(move || {
@@ -51,7 +54,7 @@ impl ClusterSlotsService {
                     cluster_slots,
                     bank_forks,
                     cluster_info,
-                    completed_slots_receiver,
+                    cluster_slots_update_receiver,
                     exit,
                 )
             })
@@ -71,7 +74,7 @@ impl ClusterSlotsService {
         cluster_slots: Arc<ClusterSlots>,
         bank_forks: Arc<RwLock<BankForks>>,
         cluster_info: Arc<ClusterInfo>,
-        completed_slots_receiver: CompletedSlotsReceiver,
+        cluster_slots_update_receiver: ClusterSlotsUpdateReceiver,
         exit: Arc<AtomicBool>,
     ) {
         let mut cluster_slots_service_timing = ClusterSlotsServiceTiming::default();
@@ -80,7 +83,8 @@ impl ClusterSlotsService {
             if exit.load(Ordering::Relaxed) {
                 break;
             }
-            let slots = match completed_slots_receiver.recv_timeout(Duration::from_millis(200)) {
+            let slots = match cluster_slots_update_receiver.recv_timeout(Duration::from_millis(200))
+            {
                 Ok(slots) => Some(slots),
                 Err(RecvTimeoutError::Timeout) => None,
                 Err(RecvTimeoutError::Disconnected) => {
@@ -94,17 +98,21 @@ impl ClusterSlotsService {
             let lowest_slot = blockstore.lowest_slot();
             Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
             lowest_slot_elapsed.stop();
-            let mut update_completed_slots_elapsed =
-                Measure::start("update_completed_slots_elapsed");
+            let mut process_cluster_slots_updates_elapsed =
+                Measure::start("process_cluster_slots_updates_elapsed");
             if let Some(slots) = slots {
-                Self::update_completed_slots(slots, &completed_slots_receiver, &cluster_info);
+                Self::process_cluster_slots_updates(
+                    slots,
+                    &cluster_slots_update_receiver,
+                    &cluster_info,
+                );
             }
             cluster_slots.update(new_root, &cluster_info, &bank_forks);
-            update_completed_slots_elapsed.stop();
+            process_cluster_slots_updates_elapsed.stop();
 
             cluster_slots_service_timing.update(
                 lowest_slot_elapsed.as_us(),
-                update_completed_slots_elapsed.as_us(),
+                process_cluster_slots_updates_elapsed.as_us(),
             );
 
             if last_stats.elapsed().as_secs() > 2 {
@@ -116,8 +124,8 @@ impl ClusterSlotsService {
                         i64
                     ),
                     (
-                        "update_completed_slots_elapsed",
-                        cluster_slots_service_timing.update_completed_slots_elapsed,
+                        "process_cluster_slots_updates_elapsed",
+                        cluster_slots_service_timing.process_cluster_slots_updates_elapsed,
                         i64
                     ),
                 );
@@ -127,12 +135,12 @@ impl ClusterSlotsService {
         }
     }
 
-    fn update_completed_slots(
+    fn process_cluster_slots_updates(
         mut slots: Vec<Slot>,
-        completed_slots_receiver: &CompletedSlotsReceiver,
+        cluster_slots_update_receiver: &ClusterSlotsUpdateReceiver,
         cluster_info: &ClusterInfo,
     ) {
-        while let Ok(mut more) = completed_slots_receiver.try_recv() {
+        while let Ok(mut more) = cluster_slots_update_receiver.try_recv() {
             slots.append(&mut more);
         }
         #[allow(clippy::stable_sort_primitive)]
@@ -155,30 +163,16 @@ impl ClusterSlotsService {
         cluster_info.push_lowest_slot(*id, lowest_slot);
     }
 
-    fn initialize_epoch_slots(
-        blockstore: &Blockstore,
-        cluster_info: &ClusterInfo,
-        completed_slots_receiver: &CompletedSlotsReceiver,
-    ) {
-        let root = blockstore.last_root();
-        let mut slots: Vec<_> = blockstore
-            .live_slots_iterator(root)
-            .filter_map(|(slot, slot_meta)| {
-                if slot_meta.is_full() {
-                    Some(slot)
-                } else {
-                    None
-                }
-            })
-            .collect();
+    fn initialize_epoch_slots(bank_forks: &RwLock<BankForks>, cluster_info: &ClusterInfo) {
+        // TODO: Should probably incorporate slots that were replayed on startup,
+        // and maybe some that were frozen < snapshot root in case validators restart
+        // from newer snapshots and lose history.
+        let frozen_banks = bank_forks.read().unwrap().frozen_banks();
+        let mut frozen_bank_slots: Vec<Slot> = frozen_banks.keys().cloned().collect();
+        frozen_bank_slots.sort_unstable();
 
-        while let Ok(mut more) = completed_slots_receiver.try_recv() {
-            slots.append(&mut more);
-        }
-        slots.sort_unstable();
-        slots.dedup();
-        if !slots.is_empty() {
-            cluster_info.push_epoch_slots(&slots);
+        if !frozen_bank_slots.is_empty() {
+            cluster_info.push_epoch_slots(&frozen_bank_slots);
         }
     }
 }

core/src/fetch_stage.rs

@@ -1,11 +1,11 @@
 //! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
 
 use crate::banking_stage::HOLD_TRANSACTIONS_SLOT_OFFSET;
-use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
 use solana_perf::packet::PacketsRecycler;
 use solana_perf::recycler::Recycler;
+use solana_poh::poh_recorder::PohRecorder;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::net::UdpSocket;

core/src/ledger_cleanup_service.rs

@@ -207,11 +207,25 @@ impl LedgerCleanupService {
                     );
 
                     let mut purge_time = Measure::start("purge_slots");
+
                     blockstore.purge_slots(
                         purge_first_slot,
                         lowest_cleanup_slot,
-                        PurgeType::PrimaryIndex,
+                        PurgeType::CompactionFilter,
                     );
+                    // Update only after purge operation.
+                    // Safety: This value can be used by compaction_filters shared via Arc<AtomicU64>.
+                    // Compactions are async and run as a multi-threaded background job. However, this
+                    // shouldn't cause consistency issues for iterators and getters because we have
+                    // already expired all affected keys (older than or equal to lowest_cleanup_slot)
+                    // by the above `purge_slots`. According to the general RocksDB design where SST
+                    // files are immutable, even running iterators aren't affected; the database grabs
+                    // a snapshot of the live set of sst files at iterator's creation.
+                    // Also, we passed the PurgeType::CompactionFilter, meaning no delete_range for
+                    // transaction_status and address_signatures CFs. These are fine because they
+                    // don't require strong consistent view for their operation.
+                    blockstore.set_max_expired_slot(lowest_cleanup_slot);
+
                     purge_time.stop();
                     info!("{}", purge_time);
 

core/src/lib.rs

@@ -9,7 +9,6 @@
 
 pub mod accounts_hash_verifier;
 pub mod banking_stage;
-pub mod bigtable_upload_service;
 pub mod broadcast_stage;
 pub mod cache_block_meta_service;
 pub mod cluster_info_vote_listener;
@@ -28,8 +27,6 @@ pub mod ledger_cleanup_service;
 pub mod optimistic_confirmation_verifier;
 pub mod outstanding_requests;
 pub mod packet_hasher;
-pub mod poh_recorder;
-pub mod poh_service;
 pub mod progress_map;
 pub mod repair_response;
 pub mod repair_service;
@@ -40,11 +37,7 @@ pub mod request_response;
 mod result;
 pub mod retransmit_stage;
 pub mod rewards_recorder_service;
-pub mod rpc;
-pub mod rpc_health;
-pub mod rpc_service;
 pub mod sample_performance_service;
-pub mod send_transaction_service;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod shred_fetch_stage;
@@ -54,7 +47,6 @@ pub mod sigverify_stage;
 pub mod snapshot_packager_service;
 pub mod test_validator;
 pub mod tpu;
-pub mod transaction_status_service;
 pub mod tree_diff;
 pub mod tvu;
 pub mod unfrozen_gossip_verified_vote_hashes;
@@ -69,10 +61,6 @@ extern crate log;
 #[macro_use]
 extern crate serde_derive;
 
-#[cfg(test)]
-#[macro_use]
-extern crate serde_json;
-
 #[macro_use]
 extern crate solana_metrics;
 

core/src/repair_service.rs

@@ -5,6 +5,7 @@ use crate::{
     cluster_slots::ClusterSlots,
     outstanding_requests::OutstandingRequests,
     repair_weight::RepairWeight,
+    replay_stage::DUPLICATE_THRESHOLD,
     result::Result,
     serve_repair::{RepairType, ServeRepair},
 };
@@ -15,9 +16,7 @@ use solana_ledger::{
     shred::Nonce,
 };
 use solana_measure::measure::Measure;
-use solana_runtime::{
-    bank::Bank, bank_forks::BankForks, commitment::VOTE_THRESHOLD_SIZE, contains::Contains,
-};
+use solana_runtime::{bank::Bank, bank_forks::BankForks, contains::Contains};
 use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey, timing::timestamp};
 use std::{
     collections::{HashMap, HashSet},
@@ -33,6 +32,8 @@ use std::{
 
 pub type DuplicateSlotsResetSender = CrossbeamSender<Slot>;
 pub type DuplicateSlotsResetReceiver = CrossbeamReceiver<Slot>;
+pub type ConfirmedSlotsSender = CrossbeamSender<Vec<Slot>>;
+pub type ConfirmedSlotsReceiver = CrossbeamReceiver<Vec<Slot>>;
 
 pub type OutstandingRepairs = OutstandingRequests<RepairType>;
 
@@ -569,14 +570,14 @@ impl RepairService {
     ) {
         for slot in new_duplicate_slots {
             warn!(
-                "Cluster completed slot: {}, dumping our current version and repairing",
+                "Cluster confirmed slot: {}, dumping our current version and repairing",
                 slot
             );
             // Clear the slot signatures from status cache for this slot
             root_bank.clear_slot_signatures(*slot);
 
             // Clear the accounts for this slot
-            root_bank.remove_unrooted_slot(*slot);
+            root_bank.remove_unrooted_slots(&[*slot]);
 
             // Clear the slot-related data in blockstore. This will:
             // 1) Clear old shreds allowing new ones to be inserted
@@ -641,7 +642,7 @@ impl RepairService {
                                 })
                                 .sum();
                             if total_completed_slot_stake as f64 / total_stake as f64
-                                > VOTE_THRESHOLD_SIZE
+                                > DUPLICATE_THRESHOLD
                             {
                                 Some(dead_slot)
                             } else {
@@ -1059,7 +1060,7 @@ mod test {
         let serve_repair = ServeRepair::new(cluster_info.clone());
         let valid_repair_peer = Node::new_localhost().info;
 
-        // Signal that this peer has completed the dead slot, and is thus
+        // Signal that this peer has confirmed the dead slot, and is thus
         // a valid target for repair
         let dead_slot = 9;
         let cluster_slots = ClusterSlots::default();

core/src/replay_stage.rs

@@ -8,6 +8,7 @@ use crate::{
     },
     cluster_slot_state_verifier::*,
     cluster_slots::ClusterSlots,
+    cluster_slots_service::ClusterSlotsUpdateSender,
     commitment_service::{AggregateCommitmentService, CommitmentAggregationData},
     consensus::{
         ComputedBankState, Stake, SwitchForkDecision, Tower, VotedStakes, SWITCH_FORK_THRESHOLD,
@@ -15,7 +16,6 @@ use crate::{
     fork_choice::{ForkChoice, SelectVoteAndResetForkResult},
     heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
     latest_validator_votes_for_frozen_banks::LatestValidatorVotesForFrozenBanks,
-    poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
     progress_map::{DuplicateStats, ForkProgress, ProgressMap, PropagatedStats},
     repair_service::DuplicateSlotsResetReceiver,
     result::Result,
@@ -34,6 +34,7 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_info;
+use solana_poh::poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS};
 use solana_rpc::{
     optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSender},
     rpc_subscriptions::RpcSubscriptions,
@@ -292,6 +293,7 @@ impl ReplayStage {
         replay_vote_sender: ReplayVoteSender,
         gossip_duplicate_confirmed_slots_receiver: GossipDuplicateConfirmedSlotsReceiver,
         gossip_verified_vote_hash_receiver: GossipVerifiedVoteHashReceiver,
+        cluster_slots_update_sender: ClusterSlotsUpdateSender,
     ) -> Self {
         let ReplayStageConfig {
             my_pubkey,
@@ -387,6 +389,7 @@ impl ReplayStage {
                         &descendants,
                         &mut unfrozen_gossip_verified_vote_hashes,
                         &mut latest_validator_votes_for_frozen_banks,
+                        &cluster_slots_update_sender,
                     );
                     replay_active_banks_time.stop();
 
@@ -1156,9 +1159,8 @@ impl ReplayStage {
                 // Signal retransmit
                 if Self::should_retransmit(poh_slot, &mut skipped_slots_info.last_retransmit_slot) {
                     datapoint_info!("replay_stage-retransmit", ("slot", bank.slot(), i64),);
-                    retransmit_slots_sender
-                        .send(vec![(bank.slot(), bank.clone())].into_iter().collect())
-                        .unwrap();
+                    let _ = retransmit_slots_sender
+                        .send(vec![(bank.slot(), bank.clone())].into_iter().collect());
                 }
                 return;
             }
@@ -1638,6 +1640,7 @@ impl ReplayStage {
         descendants: &HashMap<Slot, HashSet<Slot>>,
         unfrozen_gossip_verified_vote_hashes: &mut UnfrozenGossipVerifiedVoteHashes,
         latest_validator_votes_for_frozen_banks: &mut LatestValidatorVotesForFrozenBanks,
+        cluster_slots_update_sender: &ClusterSlotsUpdateSender,
     ) -> bool {
         let mut did_complete_bank = false;
         let mut tx_count = 0;
@@ -1725,6 +1728,7 @@ impl ReplayStage {
                 );
                 did_complete_bank = true;
                 info!("bank frozen: {}", bank.slot());
+                let _ = cluster_slots_update_sender.send(vec![*bank_slot]);
                 if let Some(transaction_status_sender) = transaction_status_sender {
                     transaction_status_sender.send_transaction_status_freeze_message(&bank);
                 }
@@ -2470,22 +2474,21 @@ impl ReplayStage {
 }
 
 #[cfg(test)]
-pub(crate) mod tests {
+mod tests {
     use super::*;
     use crate::{
         consensus::test::{initialize_state, VoteSimulator},
         consensus::Tower,
         progress_map::ValidatorStakeInfo,
         replay_stage::ReplayStage,
-        transaction_status_service::TransactionStatusService,
     };
     use crossbeam_channel::unbounded;
     use solana_gossip::{cluster_info::Node, crds::Cursor};
     use solana_ledger::{
         blockstore::make_slot_entries,
         blockstore::{entries_to_test_shreds, BlockstoreError},
-        blockstore_processor, create_new_tmp_ledger,
-        entry::{self, next_entry, Entry},
+        create_new_tmp_ledger,
+        entry::{self, Entry},
         genesis_utils::{create_genesis_config, create_genesis_config_with_leader},
         get_tmp_ledger_path,
         shred::{
@@ -2493,7 +2496,10 @@ pub(crate) mod tests {
             SIZE_OF_COMMON_SHRED_HEADER, SIZE_OF_DATA_SHRED_HEADER, SIZE_OF_DATA_SHRED_PAYLOAD,
         },
     };
-    use solana_rpc::optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank;
+    use solana_rpc::{
+        optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
+        rpc::create_test_transactions_and_populate_blockstore,
+    };
     use solana_runtime::{
         accounts_background_service::AbsRequestSender,
         commitment::BlockCommitment,
@@ -2506,7 +2512,7 @@ pub(crate) mod tests {
         instruction::InstructionError,
         packet::PACKET_DATA_SIZE,
         poh_config::PohConfig,
-        signature::{Keypair, Signature, Signer},
+        signature::{Keypair, Signer},
         system_transaction,
         transaction::TransactionError,
     };
@@ -3297,68 +3303,6 @@ pub(crate) mod tests {
         );
     }
 
-    pub fn create_test_transactions_and_populate_blockstore(
-        keypairs: Vec<&Keypair>,
-        previous_slot: Slot,
-        bank: Arc<Bank>,
-        blockstore: Arc<Blockstore>,
-        max_complete_transaction_status_slot: Arc<AtomicU64>,
-    ) -> Vec<Signature> {
-        let mint_keypair = keypairs[0];
-        let keypair1 = keypairs[1];
-        let keypair2 = keypairs[2];
-        let keypair3 = keypairs[3];
-        let slot = bank.slot();
-        let blockhash = bank.confirmed_last_blockhash().0;
-
-        // Generate transactions for processing
-        // Successful transaction
-        let success_tx =
-            system_transaction::transfer(&mint_keypair, &keypair1.pubkey(), 2, blockhash);
-        let success_signature = success_tx.signatures[0];
-        let entry_1 = next_entry(&blockhash, 1, vec![success_tx]);
-        // Failed transaction, InstructionError
-        let ix_error_tx =
-            system_transaction::transfer(&keypair2, &keypair3.pubkey(), 10, blockhash);
-        let ix_error_signature = ix_error_tx.signatures[0];
-        let entry_2 = next_entry(&entry_1.hash, 1, vec![ix_error_tx]);
-        // Failed transaction
-        let fail_tx =
-            system_transaction::transfer(&mint_keypair, &keypair2.pubkey(), 2, Hash::default());
-        let entry_3 = next_entry(&entry_2.hash, 1, vec![fail_tx]);
-        let mut entries = vec![entry_1, entry_2, entry_3];
-
-        let shreds = entries_to_test_shreds(entries.clone(), slot, previous_slot, true, 0);
-        blockstore.insert_shreds(shreds, None, false).unwrap();
-        blockstore.set_roots(&[slot]).unwrap();
-
-        let (transaction_status_sender, transaction_status_receiver) = unbounded();
-        let (replay_vote_sender, _replay_vote_receiver) = unbounded();
-        let transaction_status_service = TransactionStatusService::new(
-            transaction_status_receiver,
-            max_complete_transaction_status_slot,
-            blockstore,
-            &Arc::new(AtomicBool::new(false)),
-        );
-
-        // Check that process_entries successfully writes can_commit transactions statuses, and
-        // that they are matched properly by get_rooted_block
-        let _result = blockstore_processor::process_entries(
-            &bank,
-            &mut entries,
-            true,
-            Some(&TransactionStatusSender {
-                sender: transaction_status_sender,
-                enable_cpi_and_log_storage: false,
-            }),
-            Some(&replay_vote_sender),
-        );
-
-        transaction_status_service.join().unwrap();
-
-        vec![success_signature, ix_error_signature]
-    }
-
     #[test]
     fn test_write_persist_transaction_status() {
         let GenesisConfigInfo {

core/src/retransmit_stage.rs

@@ -4,10 +4,9 @@
 use crate::{
     cluster_info_vote_listener::VerifiedVoteReceiver,
     cluster_slots::ClusterSlots,
-    cluster_slots_service::ClusterSlotsService,
+    cluster_slots_service::{ClusterSlotsService, ClusterSlotsUpdateReceiver},
     completed_data_sets_service::CompletedDataSetsSender,
-    repair_service::DuplicateSlotsResetSender,
-    repair_service::RepairInfo,
+    repair_service::{DuplicateSlotsResetSender, RepairInfo},
     result::{Error, Result},
     window_service::{should_retransmit_and_persist, WindowService},
 };
@@ -466,9 +465,9 @@ fn retransmit(
 
         let mut retransmit_time = Measure::start("retransmit_to");
         if !packet.meta.forward {
-            ClusterInfo::retransmit_to(&neighbors, packet, sock, true)?;
+            ClusterInfo::retransmit_to(&neighbors, packet, sock, true);
         }
-        ClusterInfo::retransmit_to(&children, packet, sock, packet.meta.forward)?;
+        ClusterInfo::retransmit_to(&children, packet, sock, packet.meta.forward);
         retransmit_time.stop();
         retransmit_total += retransmit_time.as_us();
     }
@@ -592,7 +591,8 @@ impl RetransmitStage {
         repair_socket: Arc<UdpSocket>,
         verified_receiver: Receiver<Vec<Packets>>,
         exit: &Arc<AtomicBool>,
-        completed_slots_receivers: [CompletedSlotsReceiver; 2],
+        rpc_completed_slots_receiver: CompletedSlotsReceiver,
+        cluster_slots_update_receiver: ClusterSlotsUpdateReceiver,
         epoch_schedule: EpochSchedule,
         cfg: Option<Arc<AtomicBool>>,
         shred_version: u16,
@@ -618,8 +618,6 @@ impl RetransmitStage {
             rpc_subscriptions.clone(),
         );
 
-        let [rpc_completed_slots_receiver, cluster_completed_slots_receiver] =
-            completed_slots_receivers;
         let rpc_completed_slots_hdl =
             RpcCompletedSlotsService::spawn(rpc_completed_slots_receiver, rpc_subscriptions);
         let cluster_slots_service = ClusterSlotsService::new(
@@ -627,7 +625,7 @@ impl RetransmitStage {
             cluster_slots.clone(),
             bank_forks.clone(),
             cluster_info.clone(),
-            cluster_completed_slots_receiver,
+            cluster_slots_update_receiver,
             exit.clone(),
         );
 

core/src/test_validator.rs

@@ -1,12 +1,10 @@
 use {
-    crate::{
-        rpc::JsonRpcConfig,
-        validator::{Validator, ValidatorConfig, ValidatorExit, ValidatorStartProgress},
-    },
+    crate::validator::{Validator, ValidatorConfig, ValidatorStartProgress},
     solana_client::rpc_client::RpcClient,
     solana_gossip::{cluster_info::Node, gossip_service::discover_cluster, socketaddr},
     solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger},
     solana_net_utils::PortRange,
+    solana_rpc::rpc::JsonRpcConfig,
     solana_runtime::{
         bank_forks::{ArchiveFormat, SnapshotConfig, SnapshotVersion},
         genesis_utils::create_genesis_config_with_leader_ex,
@@ -18,6 +16,7 @@ use {
         clock::{Slot, DEFAULT_MS_PER_SLOT},
         commitment_config::CommitmentConfig,
         epoch_schedule::EpochSchedule,
+        exit::Exit,
         fee_calculator::{FeeCalculator, FeeRateGovernor},
         hash::Hash,
         native_token::sol_to_lamports,
@@ -79,7 +78,7 @@ pub struct TestValidatorGenesis {
     programs: Vec<ProgramInfo>,
     epoch_schedule: Option<EpochSchedule>,
     node_config: TestValidatorNodeConfig,
-    pub validator_exit: Arc<RwLock<ValidatorExit>>,
+    pub validator_exit: Arc<RwLock<Exit>>,
     pub start_progress: Arc<RwLock<ValidatorStartProgress>>,
     pub authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
     pub max_ledger_shreds: Option<u64>,

core/src/tpu.rs

@@ -9,13 +9,13 @@ use crate::{
         VerifiedVoteSender, VoteTracker,
     },
     fetch_stage::FetchStage,
-    poh_recorder::{PohRecorder, WorkingBankEntry},
     sigverify::TransactionSigVerifier,
     sigverify_stage::SigVerifyStage,
 };
 use crossbeam_channel::unbounded;
 use solana_gossip::cluster_info::ClusterInfo;
 use solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusSender};
+use solana_poh::poh_recorder::{PohRecorder, WorkingBankEntry};
 use solana_rpc::{
     optimistically_confirmed_bank_tracker::BankNotificationSender,
     rpc_subscriptions::RpcSubscriptions,

core/src/tvu.rs

@@ -13,7 +13,6 @@ use crate::{
     completed_data_sets_service::CompletedDataSetsSender,
     consensus::Tower,
     ledger_cleanup_service::LedgerCleanupService,
-    poh_recorder::PohRecorder,
     replay_stage::{ReplayStage, ReplayStageConfig},
     retransmit_stage::RetransmitStage,
     rewards_recorder_service::RewardsRecorderSender,
@@ -29,6 +28,7 @@ use solana_ledger::{
     blockstore_processor::TransactionStatusSender,
     leader_schedule_cache::LeaderScheduleCache,
 };
+use solana_poh::poh_recorder::PohRecorder;
 use solana_rpc::{
     max_slots::MaxSlots, optimistically_confirmed_bank_tracker::BankNotificationSender,
     rpc_subscriptions::RpcSubscriptions,
@@ -111,7 +111,7 @@ impl Tvu {
         tower: Tower,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         exit: &Arc<AtomicBool>,
-        completed_slots_receivers: [CompletedSlotsReceiver; 2],
+        completed_slots_receiver: CompletedSlotsReceiver,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         cfg: Option<Arc<AtomicBool>>,
         transaction_status_sender: Option<TransactionStatusSender>,
@@ -165,6 +165,7 @@ impl Tvu {
         let compaction_interval = tvu_config.rocksdb_compaction_interval;
         let max_compaction_jitter = tvu_config.rocksdb_max_compaction_jitter;
         let (duplicate_slots_sender, duplicate_slots_receiver) = unbounded();
+        let (cluster_slots_update_sender, cluster_slots_update_receiver) = unbounded();
         let retransmit_stage = RetransmitStage::new(
             bank_forks.clone(),
             leader_schedule_cache,
@@ -174,7 +175,8 @@ impl Tvu {
             repair_socket,
             verified_receiver,
             &exit,
-            completed_slots_receivers,
+            completed_slots_receiver,
+            cluster_slots_update_receiver,
             *bank_forks.read().unwrap().working_bank().epoch_schedule(),
             cfg,
             tvu_config.shred_version,
@@ -288,6 +290,7 @@ impl Tvu {
             replay_vote_sender,
             gossip_confirmed_slots_receiver,
             gossip_verified_vote_hash_receiver,
+            cluster_slots_update_sender,
         );
 
         let ledger_cleanup_service = tvu_config.max_ledger_shreds.map(|max_ledger_shreds| {
@@ -338,7 +341,6 @@ impl Tvu {
 #[cfg(test)]
 pub mod tests {
     use super::*;
-    use crate::banking_stage::create_test_recorder;
     use serial_test::serial;
     use solana_gossip::cluster_info::{ClusterInfo, Node};
     use solana_ledger::{
@@ -346,6 +348,7 @@ pub mod tests {
         create_new_tmp_ledger,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
     };
+    use solana_poh::poh_recorder::create_test_recorder;
     use solana_rpc::optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank;
     use solana_runtime::bank::Bank;
     use std::sync::atomic::Ordering;
@@ -373,7 +376,7 @@ pub mod tests {
         let BlockstoreSignals {
             blockstore,
             ledger_signal_receiver,
-            completed_slots_receivers,
+            completed_slots_receiver,
             ..
         } = Blockstore::open_with_signal(&blockstore_path, None, true)
             .expect("Expected to successfully open ledger");
@@ -417,7 +420,7 @@ pub mod tests {
             tower,
             &leader_schedule_cache,
             &exit,
-            completed_slots_receivers,
+            completed_slots_receiver,
             block_commitment_cache,
             None,
             None,

core/src/validator.rs

@@ -6,18 +6,13 @@ use crate::{
     cluster_info_vote_listener::VoteTracker,
     completed_data_sets_service::CompletedDataSetsService,
     consensus::{reconcile_blockstore_roots_with_tower, Tower},
-    poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
-    poh_service::{self, PohService},
     rewards_recorder_service::{RewardsRecorderSender, RewardsRecorderService},
-    rpc::JsonRpcConfig,
-    rpc_service::JsonRpcService,
     sample_performance_service::SamplePerformanceService,
     serve_repair::ServeRepair,
     serve_repair_service::ServeRepairService,
     sigverify,
     snapshot_packager_service::{PendingSnapshotPackage, SnapshotPackagerService},
     tpu::{Tpu, DEFAULT_TPU_COALESCE_MS},
-    transaction_status_service::TransactionStatusService,
     tvu::{Sockets, Tvu, TvuConfig},
 };
 use crossbeam_channel::{bounded, unbounded};
@@ -41,13 +36,20 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_metrics::datapoint_info;
+use solana_poh::{
+    poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
+    poh_service::{self, PohService},
+};
 use solana_rpc::{
     max_slots::MaxSlots,
     optimistically_confirmed_bank_tracker::{
         OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker,
     },
+    rpc::JsonRpcConfig,
     rpc_pubsub_service::{PubSubConfig, PubSubService},
+    rpc_service::JsonRpcService,
     rpc_subscriptions::RpcSubscriptions,
+    transaction_status_service::TransactionStatusService,
 };
 use solana_runtime::{
     accounts_index::AccountSecondaryIndexes,
@@ -59,6 +61,7 @@ use solana_runtime::{
 use solana_sdk::{
     clock::Slot,
     epoch_schedule::MAX_LEADER_SCHEDULE_EPOCH_OFFSET,
+    exit::Exit,
     genesis_config::GenesisConfig,
     hash::Hash,
     pubkey::Pubkey,
@@ -67,10 +70,8 @@ use solana_sdk::{
     timing::timestamp,
 };
 use solana_vote_program::vote_state::VoteState;
-use std::time::Instant;
 use std::{
     collections::HashSet,
-    fmt,
     net::SocketAddr,
     ops::Deref,
     path::{Path, PathBuf},
@@ -78,7 +79,7 @@ use std::{
     sync::mpsc::Receiver,
     sync::{Arc, Mutex, RwLock},
     thread::{sleep, Builder},
-    time::Duration,
+    time::{Duration, Instant},
 };
 
 const MAX_COMPLETED_DATA_SETS_IN_CHANNEL: usize = 100_000;
@@ -135,7 +136,7 @@ pub struct ValidatorConfig {
     pub accounts_db_test_hash_calculation: bool,
     pub accounts_db_use_index_hash_calculation: bool,
     pub tpu_coalesce_ms: u64,
-    pub validator_exit: Arc<RwLock<ValidatorExit>>,
+    pub validator_exit: Arc<RwLock<Exit>>,
     pub no_wait_for_vote_to_start_leader: bool,
 }
 
@@ -191,7 +192,7 @@ impl Default for ValidatorConfig {
             accounts_db_test_hash_calculation: false,
             accounts_db_use_index_hash_calculation: true,
             tpu_coalesce_ms: DEFAULT_TPU_COALESCE_MS,
-            validator_exit: Arc::new(RwLock::new(ValidatorExit::default())),
+            validator_exit: Arc::new(RwLock::new(Exit::default())),
             no_wait_for_vote_to_start_leader: true,
         }
     }
@@ -223,35 +224,6 @@ impl Default for ValidatorStartProgress {
     }
 }
 
-#[derive(Default)]
-pub struct ValidatorExit {
-    exited: bool,
-    exits: Vec<Box<dyn FnOnce() + Send + Sync>>,
-}
-
-impl ValidatorExit {
-    pub fn register_exit(&mut self, exit: Box<dyn FnOnce() + Send + Sync>) {
-        if self.exited {
-            exit();
-        } else {
-            self.exits.push(exit);
-        }
-    }
-
-    pub fn exit(&mut self) {
-        self.exited = true;
-        for exit in self.exits.drain(..) {
-            exit();
-        }
-    }
-}
-
-impl fmt::Debug for ValidatorExit {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} exits", self.exits.len())
-    }
-}
-
 #[derive(Default)]
 struct TransactionHistoryServices {
     transaction_status_sender: Option<TransactionStatusSender>,
@@ -264,7 +236,7 @@ struct TransactionHistoryServices {
 }
 
 pub struct Validator {
-    validator_exit: Arc<RwLock<ValidatorExit>>,
+    validator_exit: Arc<RwLock<Exit>>,
     json_rpc_service: Option<JsonRpcService>,
     pubsub_service: Option<PubSubService>,
     optimistically_confirmed_bank_tracker: Option<OptimisticallyConfirmedBankTracker>,
@@ -387,7 +359,7 @@ impl Validator {
             bank_forks,
             blockstore,
             ledger_signal_receiver,
-            completed_slots_receivers,
+            completed_slots_receiver,
             leader_schedule_cache,
             snapshot_hash,
             TransactionHistoryServices {
@@ -719,7 +691,7 @@ impl Validator {
             tower,
             &leader_schedule_cache,
             &exit,
-            completed_slots_receivers,
+            completed_slots_receiver,
             block_commitment_cache,
             config.enable_partition.clone(),
             transaction_status_sender.clone(),
@@ -1042,7 +1014,7 @@ fn new_banks_from_ledger(
     BankForks,
     Arc<Blockstore>,
     Receiver<bool>,
-    [CompletedSlotsReceiver; 2],
+    CompletedSlotsReceiver,
     LeaderScheduleCache,
     Option<(Slot, Hash)>,
     TransactionHistoryServices,
@@ -1073,7 +1045,7 @@ fn new_banks_from_ledger(
     let BlockstoreSignals {
         mut blockstore,
         ledger_signal_receiver,
-        completed_slots_receivers,
+        completed_slots_receiver,
         ..
     } = Blockstore::open_with_signal(
         ledger_path,
@@ -1225,7 +1197,7 @@ fn new_banks_from_ledger(
         bank_forks,
         blockstore,
         ledger_signal_receiver,
-        completed_slots_receivers,
+        completed_slots_receiver,
         leader_schedule_cache,
         snapshot_hash,
         transaction_history_services,
@@ -1636,9 +1608,11 @@ mod tests {
             }
             drop(blockstore);
 
+            // this purges and compacts all slots greater than or equal to 5
             backup_and_clear_blockstore(&blockstore_path, 5, 2);
 
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            // assert that slots less than 5 aren't affected
             assert!(blockstore.meta(4).unwrap().unwrap().next_slots.is_empty());
             for i in 5..10 {
                 assert!(blockstore

core/tests/ledger_cleanup.rs

@@ -39,6 +39,8 @@ mod tests {
         pub cleanup_blockstore: bool,
         pub emit_cpu_info: bool,
         pub assert_compaction: bool,
+        pub compaction_interval: Option<u64>,
+        pub no_compaction: bool,
     }
 
     #[derive(Clone, Copy, Debug)]
@@ -154,6 +156,11 @@ mod tests {
         let emit_cpu_info = read_env("EMIT_CPU_INFO", true);
         // set default to `true` once compaction is merged
         let assert_compaction = read_env("ASSERT_COMPACTION", false);
+        let compaction_interval = match read_env("COMPACTION_INTERVAL", 0) {
+            maybe_zero if maybe_zero == 0 => None,
+            non_zero => Some(non_zero),
+        };
+        let no_compaction = read_env("NO_COMPACTION", false);
 
         BenchmarkConfig {
             benchmark_slots,
@@ -166,6 +173,8 @@ mod tests {
             cleanup_blockstore,
             emit_cpu_info,
             assert_compaction,
+            compaction_interval,
+            no_compaction,
         }
     }
 
@@ -211,8 +220,13 @@ mod tests {
     fn test_ledger_cleanup_compaction() {
         solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
-        let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap());
+        let mut blockstore = Blockstore::open(&blockstore_path).unwrap();
         let config = get_benchmark_config();
+        if config.no_compaction {
+            blockstore.set_no_compaction(true);
+        }
+        let blockstore = Arc::new(blockstore);
+
         eprintln!("BENCHMARK CONFIG: {:?}", config);
         eprintln!("LEDGER_PATH: {:?}", &blockstore_path);
 
@@ -223,6 +237,8 @@ mod tests {
         let stop_size_bytes = config.stop_size_bytes;
         let stop_size_iterations = config.stop_size_iterations;
         let pre_generate_data = config.pre_generate_data;
+        let compaction_interval = config.compaction_interval;
+
         let batches = benchmark_slots / batch_size;
 
         let (sender, receiver) = channel();
@@ -232,7 +248,7 @@ mod tests {
             blockstore.clone(),
             max_ledger_shreds,
             &exit,
-            None,
+            compaction_interval,
             None,
         );
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/src/developing/clients/jsonrpc-api.md

@@ -350,7 +350,7 @@ Result:
 ### getBlock
 
 **NEW: This method is only available in solana-core v1.7 or newer. Please use
-[getBlock](jsonrpc-api.md#getblock) for solana-core v1.6**
+[getConfirmedBlock](jsonrpc-api.md#getconfirmedblock) for solana-core v1.6**
 
 Returns identity and transaction information about a confirmed block in the ledger
 
@@ -3004,7 +3004,7 @@ curl http://localhost:8899 -X POST -H "Content-Type: application/json" -d '
 
 Result:
 ```json
-{"jsonrpc":"2.0","result":{"solana-core": "1.7.0"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.7.1"},"id":1}
 ```
 
 ### getVoteAccounts

docs/src/running-validator/validator-start.md

@@ -60,7 +60,7 @@ the latest recommended settings are applied.
 To run it:
 
 ```bash
-sudo solana-sys-tuner --user $(whoami) > sys-tuner.log 2>&1 &
+sudo $(command -v solana-sys-tuner) --user $(whoami) > sys-tuner.log 2>&1 &
 ```
 
 #### Manual

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,12 +11,12 @@ edition = "2018"
 
 [dependencies]
 bzip2 = "0.3.3"
-console = "0.11.3"
+console = "0.14.1"
 indicatif = "0.15.0"
 log = "0.4.11"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
 tar = "0.4.28"
 
 [lib]

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,12 +16,12 @@ clap = "2.33"
 log = "0.4.11"
 serde = "1.0.122"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }

frozen-abi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Frozen ABI"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,11 +16,11 @@ log = "0.4.11"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 sha2 = "0.9.2"
-solana-frozen-abi-macro = { path = "macro", version = "=1.7.0" }
+solana-frozen-abi-macro = { path = "macro", version = "=1.7.1" }
 thiserror = "1.0"
 
 [target.'cfg(not(target_arch = "bpf"))'.dependencies]
-solana-logger = { path = "../logger", version = "=1.7.0" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
 generic-array = { version = "0.14.3", default-features = false, features = ["serde", "more_lengths"]}
 memmap2 = "0.1.0"
 

frozen-abi/macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi-macro"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Frozen ABI Macro"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

genesis-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-utils"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Genesis Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ documentation = "https://docs.rs/solana-download-utils"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-download-utils = { path = "../download-utils", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
 
 [lib]
 crate-type = ["lib"]

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,17 +16,16 @@ chrono = "0.4"
 serde = "1.0.122"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-budget-program = { path = "../programs/budget", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.7.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 tempfile = "3.1.0"
 
 [[bin]]

genesis/src/main.rs

@@ -1,8 +1,6 @@
 //! A command-line executable for generating the chain's genesis config.
 #![allow(clippy::integer_arithmetic)]
 
-#[macro_use]
-extern crate solana_budget_program;
 #[macro_use]
 extern crate solana_exchange_program;
 
@@ -492,7 +490,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     );
 
     let native_instruction_processors = if cluster_type == ClusterType::Development {
-        vec![solana_budget_program!(), solana_exchange_program!()]
+        vec![solana_exchange_program!()]
     } else {
         vec![]
     };

gossip/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -25,22 +25,22 @@ rayon = "1.5.0"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-streamer = { path = "../streamer", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 thiserror = "1.0"
 
 [dev-dependencies]

gossip/src/cluster_info.rs

@@ -78,6 +78,7 @@ use {
         result::Result,
         sync::{
             atomic::{AtomicBool, Ordering},
+            mpsc::{Receiver, RecvTimeoutError, Sender},
             {Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard},
         },
         thread::{sleep, Builder, JoinHandle},
@@ -235,7 +236,7 @@ impl Default for ClusterInfo {
 }
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
-struct PruneData {
+pub(crate) struct PruneData {
     /// Pubkey of the node that sent this prune data
     pubkey: Pubkey,
     /// Pubkeys of nodes that should be pruned
@@ -329,7 +330,7 @@ pub(crate) type Ping = ping_pong::Ping<[u8; GOSSIP_PING_TOKEN_SIZE]>;
 #[frozen_abi(digest = "GANv3KVkTYF84kmg1bAuWEZd9MaiYzPquuu13hup3379")]
 #[derive(Serialize, Deserialize, Debug, AbiEnumVisitor, AbiExample)]
 #[allow(clippy::large_enum_variant)]
-enum Protocol {
+pub(crate) enum Protocol {
     /// Gossip protocol messages
     PullRequest(CrdsFilter, CrdsValue),
     PullResponse(Pubkey, Vec<CrdsValue>),
@@ -897,6 +898,8 @@ impl ClusterInfo {
         }
     }
 
+    // TODO: If two threads call into this function then epoch_slot_index has a
+    // race condition and the threads will overwrite each other in crds table.
     pub fn push_epoch_slots(&self, mut update: &[Slot]) {
         let current_slots: Vec<_> = {
             let gossip =
@@ -933,26 +936,28 @@ impl ClusterInfo {
             Some((_wallclock, _slot, index)) => *index,
             None => 0,
         };
+        let self_pubkey = self.id();
+        let mut entries = Vec::default();
         while !update.is_empty() {
             let ix = (epoch_slot_index % crds_value::MAX_EPOCH_SLOTS) as u8;
             let now = timestamp();
             let mut slots = if !reset {
                 self.lookup_epoch_slots(ix)
             } else {
-                EpochSlots::new(self.id(), now)
+                EpochSlots::new(self_pubkey, now)
             };
             let n = slots.fill(update, now);
             update = &update[n..];
             if n > 0 {
-                let entry = CrdsValue::new_signed(CrdsData::EpochSlots(ix, slots), &self.keypair);
-                self.local_message_pending_push_queue
-                    .lock()
-                    .unwrap()
-                    .push(entry);
+                let epoch_slots = CrdsData::EpochSlots(ix, slots);
+                let entry = CrdsValue::new_signed(epoch_slots, &self.keypair);
+                entries.push(entry);
             }
             epoch_slot_index += 1;
             reset = true;
         }
+        let mut gossip = self.gossip.write().unwrap();
+        gossip.process_push_message(&self_pubkey, entries, timestamp());
     }
 
     fn time_gossip_read_lock<'a>(
@@ -1382,12 +1387,7 @@ impl ClusterInfo {
     /// retransmit messages to a list of nodes
     /// # Remarks
     /// We need to avoid having obj locked while doing a io, such as the `send_to`
-    pub fn retransmit_to(
-        peers: &[&ContactInfo],
-        packet: &Packet,
-        s: &UdpSocket,
-        forwarded: bool,
-    ) -> Result<(), GossipError> {
+    pub fn retransmit_to(peers: &[&ContactInfo], packet: &Packet, s: &UdpSocket, forwarded: bool) {
         trace!("retransmit orders {}", peers.len());
         let dests: Vec<_> = if forwarded {
             peers
@@ -1398,22 +1398,28 @@ impl ClusterInfo {
         } else {
             peers.iter().map(|peer| &peer.tvu).collect()
         };
-        let mut sent = 0;
-        while sent < dests.len() {
-            match multicast(s, &packet.data[..packet.meta.size], &dests[sent..]) {
-                Ok(n) => sent += n,
-                Err(e) => {
-                    inc_new_counter_error!(
-                        "cluster_info-retransmit-send_to_error",
-                        dests.len() - sent,
-                        1
-                    );
-                    error!("retransmit result {:?}", e);
-                    return Err(GossipError::Io(e));
+        let mut dests = &dests[..];
+        let data = &packet.data[..packet.meta.size];
+        while !dests.is_empty() {
+            match multicast(s, data, dests) {
+                Ok(n) => dests = &dests[n..],
+                Err(err) => {
+                    inc_new_counter_error!("cluster_info-retransmit-send_to_error", dests.len(), 1);
+                    error!("retransmit multicast: {:?}", err);
+                    break;
                 }
             }
         }
-        Ok(())
+        let mut errs = 0;
+        for dest in dests {
+            if let Err(err) = s.send_to(data, dest) {
+                error!("retransmit send: {}, {:?}", dest, err);
+                errs += 1;
+            }
+        }
+        if errs != 0 {
+            inc_new_counter_error!("cluster_info-retransmit-error", errs, 1);
+        }
     }
 
     fn insert_self(&self) {
@@ -2494,7 +2500,7 @@ impl ClusterInfo {
 
     fn process_packets(
         &self,
-        packets: VecDeque<Packet>,
+        packets: VecDeque<(/*from:*/ SocketAddr, Protocol)>,
         thread_pool: &ThreadPool,
         recycler: &PacketsRecycler,
         response_sender: &PacketSender,
@@ -2504,24 +2510,6 @@ impl ClusterInfo {
         should_check_duplicate_instance: bool,
     ) -> Result<(), GossipError> {
         let _st = ScopedTimer::from(&self.stats.process_gossip_packets_time);
-        self.stats
-            .packets_received_count
-            .add_relaxed(packets.len() as u64);
-        let packets: Vec<_> = thread_pool.install(|| {
-            packets
-                .into_par_iter()
-                .filter_map(|packet| {
-                    let protocol: Protocol =
-                        limited_deserialize(&packet.data[..packet.meta.size]).ok()?;
-                    protocol.sanitize().ok()?;
-                    let protocol = protocol.par_verify()?;
-                    Some((packet.meta.addr(), protocol))
-                })
-                .collect()
-        });
-        self.stats
-            .packets_received_verified_count
-            .add_relaxed(packets.len() as u64);
         // Check if there is a duplicate instance of
         // this node with more recent timestamp.
         let check_duplicate_instance = |values: &[CrdsValue]| {
@@ -2606,12 +2594,54 @@ impl ClusterInfo {
         Ok(())
     }
 
+    // Consumes packets received from the socket, deserializing, sanitizing and
+    // verifying them and then sending them down the channel for the actual
+    // handling of requests/messages.
+    fn run_socket_consume(
+        &self,
+        receiver: &PacketReceiver,
+        sender: &Sender<Vec<(/*from:*/ SocketAddr, Protocol)>>,
+        thread_pool: &ThreadPool,
+    ) -> Result<(), GossipError> {
+        const RECV_TIMEOUT: Duration = Duration::from_secs(1);
+        let packets: Vec<_> = receiver.recv_timeout(RECV_TIMEOUT)?.packets.into();
+        let mut packets = VecDeque::from(packets);
+        for payload in receiver.try_iter() {
+            packets.extend(payload.packets.iter().cloned());
+            let excess_count = packets.len().saturating_sub(MAX_GOSSIP_TRAFFIC);
+            if excess_count > 0 {
+                packets.drain(0..excess_count);
+                self.stats
+                    .gossip_packets_dropped_count
+                    .add_relaxed(excess_count as u64);
+            }
+        }
+        self.stats
+            .packets_received_count
+            .add_relaxed(packets.len() as u64);
+        let verify_packet = |packet: Packet| {
+            let data = &packet.data[..packet.meta.size];
+            let protocol: Protocol = limited_deserialize(data).ok()?;
+            protocol.sanitize().ok()?;
+            let protocol = protocol.par_verify()?;
+            Some((packet.meta.addr(), protocol))
+        };
+        let packets: Vec<_> = {
+            let _st = ScopedTimer::from(&self.stats.verify_gossip_packets_time);
+            thread_pool.install(|| packets.into_par_iter().filter_map(verify_packet).collect())
+        };
+        self.stats
+            .packets_received_verified_count
+            .add_relaxed(packets.len() as u64);
+        Ok(sender.send(packets)?)
+    }
+
     /// Process messages from the network
     fn run_listen(
         &self,
         recycler: &PacketsRecycler,
         bank_forks: Option<&RwLock<BankForks>>,
-        requests_receiver: &PacketReceiver,
+        receiver: &Receiver<Vec<(/*from:*/ SocketAddr, Protocol)>>,
         response_sender: &PacketSender,
         thread_pool: &ThreadPool,
         last_print: &mut Instant,
@@ -2619,10 +2649,9 @@ impl ClusterInfo {
     ) -> Result<(), GossipError> {
         const RECV_TIMEOUT: Duration = Duration::from_secs(1);
         const SUBMIT_GOSSIP_STATS_INTERVAL: Duration = Duration::from_secs(2);
-        let packets: Vec<_> = requests_receiver.recv_timeout(RECV_TIMEOUT)?.packets.into();
-        let mut packets = VecDeque::from(packets);
-        while let Ok(packet) = requests_receiver.try_recv() {
-            packets.extend(packet.packets.iter().cloned());
+        let mut packets = VecDeque::from(receiver.recv_timeout(RECV_TIMEOUT)?);
+        for payload in receiver.try_iter() {
+            packets.extend(payload);
             let excess_count = packets.len().saturating_sub(MAX_GOSSIP_TRAFFIC);
             if excess_count > 0 {
                 packets.drain(0..excess_count);
@@ -2659,10 +2688,35 @@ impl ClusterInfo {
         Ok(())
     }
 
-    pub fn listen(
+    pub(crate) fn start_socket_consume_thread(
+        self: Arc<Self>,
+        receiver: PacketReceiver,
+        sender: Sender<Vec<(/*from:*/ SocketAddr, Protocol)>>,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        let thread_pool = ThreadPoolBuilder::new()
+            .num_threads(get_thread_count().min(8))
+            .thread_name(|i| format!("gossip-consume-{}", i))
+            .build()
+            .unwrap();
+        let run_consume = move || {
+            while !exit.load(Ordering::Relaxed) {
+                match self.run_socket_consume(&receiver, &sender, &thread_pool) {
+                    Err(GossipError::RecvTimeoutError(RecvTimeoutError::Disconnected)) => break,
+                    Err(GossipError::RecvTimeoutError(RecvTimeoutError::Timeout)) => (),
+                    Err(err) => error!("gossip consume: {}", err),
+                    Ok(()) => (),
+                }
+            }
+        };
+        let thread_name = String::from("gossip-consume");
+        Builder::new().name(thread_name).spawn(run_consume).unwrap()
+    }
+
+    pub(crate) fn listen(
         self: Arc<Self>,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
-        requests_receiver: PacketReceiver,
+        requests_receiver: Receiver<Vec<(/*from:*/ SocketAddr, Protocol)>>,
         response_sender: PacketSender,
         should_check_duplicate_instance: bool,
         exit: &Arc<AtomicBool>,
@@ -2689,7 +2743,8 @@ impl ClusterInfo {
                         should_check_duplicate_instance,
                     ) {
                         match err {
-                            GossipError::RecvTimeoutError(_) => {
+                            GossipError::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                            GossipError::RecvTimeoutError(RecvTimeoutError::Timeout) => {
                                 let table_size = self.gossip.read().unwrap().crds.len();
                                 debug!(
                                     "{}: run_listen timeout, table size: {}",
@@ -2703,7 +2758,7 @@ impl ClusterInfo {
                                     self.id()
                                 );
                                 exit.store(true, Ordering::Relaxed);
-                                // TODO: Pass through ValidatorExit here so
+                                // TODO: Pass through Exit here so
                                 // that this will exit cleanly.
                                 std::process::exit(1);
                             }
@@ -3777,7 +3832,6 @@ mod tests {
         let slots = cluster_info.get_epoch_slots(&mut Cursor::default());
         assert!(slots.is_empty());
         cluster_info.push_epoch_slots(&[0]);
-        cluster_info.flush_push_queue();
 
         let mut cursor = Cursor::default();
         let slots = cluster_info.get_epoch_slots(&mut cursor);
@@ -4134,9 +4188,7 @@ mod tests {
             range.push(last + rand::thread_rng().gen_range(1, 32));
         }
         cluster_info.push_epoch_slots(&range[..16000]);
-        cluster_info.flush_push_queue();
         cluster_info.push_epoch_slots(&range[16000..]);
-        cluster_info.flush_push_queue();
         let slots = cluster_info.get_epoch_slots(&mut Cursor::default());
         let slots: Vec<_> = slots.iter().flat_map(|x| x.to_slots(0)).collect();
         assert_eq!(slots, range);

gossip/src/cluster_info_metrics.rs

@@ -118,6 +118,7 @@ pub(crate) struct GossipStats {
     pub(crate) trim_crds_table_failed: Counter,
     pub(crate) trim_crds_table_purged_values_count: Counter,
     pub(crate) tvu_peers: Counter,
+    pub(crate) verify_gossip_packets_time: Counter,
 }
 
 pub(crate) fn submit_gossip_stats(
@@ -171,6 +172,11 @@ pub(crate) fn submit_gossip_stats(
             stats.process_gossip_packets_time.clear(),
             i64
         ),
+        (
+            "verify_gossip_packets_time",
+            stats.verify_gossip_packets_time.clear(),
+            i64
+        ),
         (
             "handle_batch_ping_messages_time",
             stats.handle_batch_ping_messages_time.clear(),

gossip/src/crds_gossip_pull.rs

@@ -13,7 +13,7 @@ use {
     crate::{
         cluster_info::{Ping, CRDS_UNIQUE_PUBKEY_CAPACITY},
         contact_info::ContactInfo,
-        crds::Crds,
+        crds::{Crds, VersionedCrdsValue},
         crds_gossip::{get_stake, get_weight},
         crds_gossip_error::CrdsGossipError,
         crds_value::CrdsValue,
@@ -499,21 +499,24 @@ impl CrdsGossipPull {
                     dropped_requests += 1;
                     return Some(vec![]);
                 }
+                let caller_pubkey = caller.pubkey();
                 let caller_wallclock = caller_wallclock.checked_add(jitter).unwrap_or(0);
+                let pred = |entry: &&VersionedCrdsValue| {
+                    debug_assert!(filter.test_mask(&entry.value_hash));
+                    // Skip values that are too new.
+                    if entry.value.wallclock() > caller_wallclock {
+                        total_skipped += 1;
+                        false
+                    } else {
+                        !filter.filter_contains(&entry.value_hash)
+                            && (entry.value.pubkey() != caller_pubkey
+                                || entry.value.should_force_push(&caller_pubkey))
+                    }
+                };
                 let out: Vec<_> = crds
                     .filter_bitmask(filter.mask, filter.mask_bits)
-                    .filter_map(|item| {
-                        debug_assert!(filter.test_mask(&item.value_hash));
-                        //skip values that are too new
-                        if item.value.wallclock() > caller_wallclock {
-                            total_skipped += 1;
-                            None
-                        } else if filter.filter_contains(&item.value_hash) {
-                            None
-                        } else {
-                            Some(item.value.clone())
-                        }
-                    })
+                    .filter(pred)
+                    .map(|entry| entry.value.clone())
                     .take(output_size_limit)
                     .collect();
                 output_size_limit -= out.len();

gossip/src/gossip_service.rs

@@ -56,11 +56,16 @@ impl GossipService {
             1,
         );
         let (response_sender, response_receiver) = channel();
-        let t_responder = streamer::responder("gossip", gossip_socket, response_receiver);
+        let (consume_sender, listen_receiver) = channel();
+        let t_socket_consume = cluster_info.clone().start_socket_consume_thread(
+            request_receiver,
+            consume_sender,
+            exit.clone(),
+        );
         let t_listen = ClusterInfo::listen(
             cluster_info.clone(),
             bank_forks.clone(),
-            request_receiver,
+            listen_receiver,
             response_sender.clone(),
             should_check_duplicate_instance,
             exit,
@@ -72,7 +77,18 @@ impl GossipService {
             gossip_validators,
             exit,
         );
-        let thread_hdls = vec![t_receiver, t_responder, t_listen, t_gossip];
+        // To work around:
+        // https://github.com/rust-lang/rust/issues/54267
+        // responder thread should start after response_sender.clone(). see:
+        // https://github.com/rust-lang/rust/issues/39364#issuecomment-381446873
+        let t_responder = streamer::responder("gossip", gossip_socket, response_receiver);
+        let thread_hdls = vec![
+            t_receiver,
+            t_responder,
+            t_socket_consume,
+            t_listen,
+            t_gossip,
+        ];
         Self { thread_hdls }
     }
 

gossip/tests/gossip.rs

@@ -209,7 +209,7 @@ pub fn cluster_info_retransmit() {
     p.meta.size = 10;
     let peers = c1.tvu_peers();
     let retransmit_peers: Vec<_> = peers.iter().collect();
-    ClusterInfo::retransmit_to(&retransmit_peers, &p, &tn1, false).unwrap();
+    ClusterInfo::retransmit_to(&retransmit_peers, &p, &tn1, false);
     let res: Vec<_> = [tn1, tn2, tn3]
         .into_par_iter()
         .map(|s| {

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,7 +15,7 @@ bincode = "1.3.1"
 bzip2 = "0.3.3"
 chrono = { version = "0.4.11", features = ["serde"] }
 clap = { version = "2.33.1" }
-console = "0.11.3"
+console = "0.14.1"
 ctrlc = { version = "3.1.5", features = ["termination"] }
 dirs-next = "2.0.0"
 indicatif = "0.15.0"
@@ -25,12 +25,12 @@ reqwest = { version = "0.11.2", default-features = false, features = ["blocking"
 serde = { version = "1.0.122", features = ["derive"] }
 serde_json = "1.0.62"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-config-program = { path = "../programs/config", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-config-program = { path = "../programs/config", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempfile = "3.1.0"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,11 +14,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs-next = "2.0.0"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.7.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -23,18 +23,18 @@ regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-cli-output = { path = "../cli-output", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-cli-output = { path = "../cli-output", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 tempfile = "3.1.0"
 tokio = { version = "1", features = ["full"] }
 

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ edition = "2018"
 bincode = "1.3.1"
 byteorder = "1.3.4"
 chrono = { version = "0.4.11", features = ["serde"] }
-chrono-humanize = "0.1.1"
+chrono-humanize = "0.2.1"
 crossbeam-channel = "0.4"
 dlopen_derive = "0.1.4"
 dlopen = "0.1.8"
@@ -32,24 +32,24 @@ rand_chacha = "0.2.2"
 rayon = "1.5.0"
 reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 serde = "1.0.122"
-serde_bytes = "0.11.4"
+serde_bytes = "0.11.5"
 sha2 = "0.9.2"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.0" }
-solana-storage-proto = { path = "../storage-proto", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.1" }
+solana-storage-proto = { path = "../storage-proto", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
@@ -59,15 +59,15 @@ trees = "0.2.1"
 [dependencies.rocksdb]
 # Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts
 # when also using the bzip2 crate
-version = "0.15.0"
+version = "0.16.0"
 default-features = false
 features = ["lz4"]
 
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-budget-program = { path = "../programs/budget", version = "=1.7.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
 
 [build-dependencies]
 rustc_version = "0.2"

ledger/src/bigtable_upload_service.rs

@@ -1,11 +1,13 @@
-use solana_ledger::blockstore::Blockstore;
-use solana_runtime::commitment::BlockCommitmentCache;
-use std::{
-    sync::atomic::{AtomicBool, Ordering},
-    sync::{Arc, RwLock},
-    thread::{self, Builder, JoinHandle},
+use {
+    crate::{bigtable_upload, blockstore::Blockstore},
+    solana_runtime::commitment::BlockCommitmentCache,
+    std::{
+        sync::atomic::{AtomicBool, Ordering},
+        sync::{Arc, RwLock},
+        thread::{self, Builder, JoinHandle},
+    },
+    tokio::runtime::Runtime,
 };
-use tokio::runtime::Runtime;
 
 // Delay uploading the largest confirmed root for this many slots.  This is done in an attempt to
 // ensure that the `CacheBlockMetaService` has had enough time to add the block time for the root
@@ -68,7 +70,7 @@ impl BigTableUploadService {
                 continue;
             }
 
-            let result = runtime.block_on(solana_ledger::bigtable_upload::upload_confirmed_blocks(
+            let result = runtime.block_on(bigtable_upload::upload_confirmed_blocks(
                 blockstore.clone(),
                 bigtable_ledger_storage.clone(),
                 start_slot,

ledger/src/blockstore.rs

@@ -54,7 +54,7 @@ use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
         mpsc::{sync_channel, Receiver, SyncSender, TrySendError},
-        Arc, Mutex, RwLock,
+        Arc, Mutex, RwLock, RwLockWriteGuard,
     },
 };
 use thiserror::Error;
@@ -85,13 +85,15 @@ pub const MAX_TURBINE_DELAY_IN_TICKS: u64 = MAX_TURBINE_PROPAGATION_IN_MS / MS_P
 // (32K shreds per slot * 4 TX per shred * 2.5 slots per sec)
 pub const MAX_DATA_SHREDS_PER_SLOT: usize = 32_768;
 
-pub type CompletedSlotsReceiver = Receiver<Vec<u64>>;
+pub type CompletedSlotsSender = SyncSender<Vec<Slot>>;
+pub type CompletedSlotsReceiver = Receiver<Vec<Slot>>;
 type CompletedRanges = Vec<(u32, u32)>;
 
 #[derive(Clone, Copy)]
 pub enum PurgeType {
     Exact,
     PrimaryIndex,
+    CompactionFilter,
 }
 
 #[derive(Error, Debug)]
@@ -117,7 +119,7 @@ pub struct CompletedDataSetInfo {
 pub struct BlockstoreSignals {
     pub blockstore: Blockstore,
     pub ledger_signal_receiver: Receiver<bool>,
-    pub completed_slots_receivers: [CompletedSlotsReceiver; 2],
+    pub completed_slots_receiver: CompletedSlotsReceiver,
 }
 
 // ledger window
@@ -143,8 +145,8 @@ pub struct Blockstore {
     last_root: Arc<RwLock<Slot>>,
     insert_shreds_lock: Arc<Mutex<()>>,
     pub new_shreds_signals: Vec<SyncSender<bool>>,
-    pub completed_slots_senders: Vec<SyncSender<Vec<Slot>>>,
-    pub lowest_cleanup_slot: Arc<RwLock<u64>>,
+    pub completed_slots_senders: Vec<CompletedSlotsSender>,
+    pub lowest_cleanup_slot: Arc<RwLock<Slot>>,
     no_compaction: bool,
 }
 
@@ -384,18 +386,16 @@ impl Blockstore {
             enforce_ulimit_nofile,
         )?;
         let (ledger_signal_sender, ledger_signal_receiver) = sync_channel(1);
-        let (completed_slots_sender1, completed_slots_receiver1) =
-            sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
-        let (completed_slots_sender2, completed_slots_receiver2) =
+        let (completed_slots_sender, completed_slots_receiver) =
             sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
 
         blockstore.new_shreds_signals = vec![ledger_signal_sender];
-        blockstore.completed_slots_senders = vec![completed_slots_sender1, completed_slots_sender2];
+        blockstore.completed_slots_senders = vec![completed_slots_sender];
 
         Ok(BlockstoreSignals {
             blockstore,
             ledger_signal_receiver,
-            completed_slots_receivers: [completed_slots_receiver1, completed_slots_receiver2],
+            completed_slots_receiver,
         })
     }
 
@@ -1956,18 +1956,24 @@ impl Blockstore {
             batch.put::<cf::TransactionStatusIndex>(0, &index0)?;
             Ok(None)
         } else {
-            let result = if index0.frozen && to_slot > index0.max_slot {
-                debug!("Pruning transaction index 0 at slot {}", index0.max_slot);
+            let purge_target_primary_index = if index0.frozen && to_slot > index0.max_slot {
+                info!(
+                    "Pruning expired primary index 0 up to slot {} (max requested: {})",
+                    index0.max_slot, to_slot
+                );
                 Some(0)
             } else if index1.frozen && to_slot > index1.max_slot {
-                debug!("Pruning transaction index 1 at slot {}", index1.max_slot);
+                info!(
+                    "Pruning expired primary index 1 up to slot {} (max requested: {})",
+                    index1.max_slot, to_slot
+                );
                 Some(1)
             } else {
                 None
             };
 
-            if result.is_some() {
-                *w_active_transaction_status_index = if index0.frozen { 0 } else { 1 };
+            if let Some(purge_target_primary_index) = purge_target_primary_index {
+                *w_active_transaction_status_index = purge_target_primary_index;
                 if index0.frozen {
                     index0.max_slot = 0
                 };
@@ -1980,16 +1986,17 @@ impl Blockstore {
                 batch.put::<cf::TransactionStatusIndex>(1, &index1)?;
             }
 
-            Ok(result)
+            Ok(purge_target_primary_index)
         }
     }
 
-    fn get_primary_index(
+    fn get_primary_index_to_write(
         &self,
         slot: Slot,
-        w_active_transaction_status_index: &mut u64,
+        // take WriteGuard to require critical section semantics at call site
+        w_active_transaction_status_index: &RwLockWriteGuard<Slot>,
     ) -> Result<u64> {
-        let i = *w_active_transaction_status_index;
+        let i = **w_active_transaction_status_index;
         let mut index_meta = self.transaction_status_index_cf.get(i)?.unwrap();
         if slot > index_meta.max_slot {
             assert!(!index_meta.frozen);
@@ -2028,9 +2035,10 @@ impl Blockstore {
         let status = status.into();
         // This write lock prevents interleaving issues with the transaction_status_index_cf by gating
         // writes to that column
-        let mut w_active_transaction_status_index =
+        let w_active_transaction_status_index =
             self.active_transaction_status_index.write().unwrap();
-        let primary_index = self.get_primary_index(slot, &mut w_active_transaction_status_index)?;
+        let primary_index =
+            self.get_primary_index_to_write(slot, &w_active_transaction_status_index)?;
         self.transaction_status_cf
             .put_protobuf((primary_index, signature, slot), &status)?;
         for address in writable_keys {
@@ -2048,6 +2056,21 @@ impl Blockstore {
         Ok(())
     }
 
+    fn ensure_lowest_cleanup_slot(&self) -> (std::sync::RwLockReadGuard<Slot>, Slot) {
+        // Ensures consistent result by using lowest_cleanup_slot as the lower bound
+        // for reading columns that do not employ strong read consistency with slot-based
+        // delete_range
+        let lowest_cleanup_slot = self.lowest_cleanup_slot.read().unwrap();
+        let lowest_available_slot = (*lowest_cleanup_slot)
+            .checked_add(1)
+            .expect("overflow from trusted value");
+
+        // Make caller hold this lock properly; otherwise LedgerCleanupService can purge/compact
+        // needed slots here at any given moment.
+        // Blockstore callers, like rpc, can process concurrent read queries
+        (lowest_cleanup_slot, lowest_available_slot)
+    }
+
     // Returns a transaction status, as well as a loop counter for unit testing
     fn get_transaction_status_with_counter(
         &self,
@@ -2055,9 +2078,15 @@ impl Blockstore {
         confirmed_unrooted_slots: &[Slot],
     ) -> Result<(Option<(Slot, TransactionStatusMeta)>, u64)> {
         let mut counter = 0;
+        let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot();
+
         for transaction_status_cf_primary_index in 0..=1 {
             let index_iterator = self.transaction_status_cf.iter(IteratorMode::From(
-                (transaction_status_cf_primary_index, signature, 0),
+                (
+                    transaction_status_cf_primary_index,
+                    signature,
+                    lowest_available_slot,
+                ),
                 IteratorDirection::Forward,
             ))?;
             for ((i, sig, slot), _data) in index_iterator {
@@ -2076,6 +2105,8 @@ impl Blockstore {
                 return Ok((status, counter));
             }
         }
+        drop(lock);
+
         Ok((None, counter))
     }
 
@@ -2199,13 +2230,15 @@ impl Blockstore {
         start_slot: Slot,
         end_slot: Slot,
     ) -> Result<Vec<(Slot, Signature)>> {
+        let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot();
+
         let mut signatures: Vec<(Slot, Signature)> = vec![];
         for transaction_status_cf_primary_index in 0..=1 {
             let index_iterator = self.address_signatures_cf.iter(IteratorMode::From(
                 (
                     transaction_status_cf_primary_index,
                     pubkey,
-                    start_slot,
+                    start_slot.max(lowest_available_slot),
                     Signature::default(),
                 ),
                 IteratorDirection::Forward,
@@ -2220,6 +2253,7 @@ impl Blockstore {
                 }
             }
         }
+        drop(lock);
         signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1)));
         Ok(signatures)
     }
@@ -2232,13 +2266,14 @@ impl Blockstore {
         pubkey: Pubkey,
         slot: Slot,
     ) -> Result<Vec<(Slot, Signature)>> {
+        let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot();
         let mut signatures: Vec<(Slot, Signature)> = vec![];
         for transaction_status_cf_primary_index in 0..=1 {
             let index_iterator = self.address_signatures_cf.iter(IteratorMode::From(
                 (
                     transaction_status_cf_primary_index,
                     pubkey,
-                    slot,
+                    slot.max(lowest_available_slot),
                     Signature::default(),
                 ),
                 IteratorDirection::Forward,
@@ -2253,6 +2288,7 @@ impl Blockstore {
                 signatures.push((slot, signature));
             }
         }
+        drop(lock);
         signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1)));
         Ok(signatures)
     }
@@ -4531,7 +4567,7 @@ pub mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let BlockstoreSignals {
             blockstore: ledger,
-            completed_slots_receivers: [recvr, _],
+            completed_slots_receiver: recvr,
             ..
         } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
@@ -4557,7 +4593,7 @@ pub mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let BlockstoreSignals {
             blockstore: ledger,
-            completed_slots_receivers: [recvr, _],
+            completed_slots_receiver: recvr,
             ..
         } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
@@ -4601,7 +4637,7 @@ pub mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let BlockstoreSignals {
             blockstore: ledger,
-            completed_slots_receivers: [recvr, _],
+            completed_slots_receiver: recvr,
             ..
         } = Blockstore::open_with_signal(&ledger_path, None, true).unwrap();
         let ledger = Arc::new(ledger);
@@ -6676,6 +6712,176 @@ pub mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    fn do_test_lowest_cleanup_slot_and_special_cfs(
+        simulate_compaction: bool,
+        simulate_ledger_cleanup_service: bool,
+    ) {
+        solana_logger::setup();
+
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            // TransactionStatus column opens initialized with one entry at index 2
+            let transaction_status_cf = blockstore.db.column::<cf::TransactionStatus>();
+
+            let pre_balances_vec = vec![1, 2, 3];
+            let post_balances_vec = vec![3, 2, 1];
+            let status = TransactionStatusMeta {
+                status: solana_sdk::transaction::Result::<()>::Ok(()),
+                fee: 42u64,
+                pre_balances: pre_balances_vec,
+                post_balances: post_balances_vec,
+                inner_instructions: Some(vec![]),
+                log_messages: Some(vec![]),
+                pre_token_balances: Some(vec![]),
+                post_token_balances: Some(vec![]),
+                rewards: Some(vec![]),
+            }
+            .into();
+
+            let signature1 = Signature::new(&[2u8; 64]);
+            let signature2 = Signature::new(&[3u8; 64]);
+
+            // Insert rooted slots 0..=3 with no fork
+            let meta0 = SlotMeta::new(0, 0);
+            blockstore.meta_cf.put(0, &meta0).unwrap();
+            let meta1 = SlotMeta::new(1, 0);
+            blockstore.meta_cf.put(1, &meta1).unwrap();
+            let meta2 = SlotMeta::new(2, 1);
+            blockstore.meta_cf.put(2, &meta2).unwrap();
+            let meta3 = SlotMeta::new(3, 2);
+            blockstore.meta_cf.put(3, &meta3).unwrap();
+
+            blockstore.set_roots(&[0, 1, 2, 3]).unwrap();
+
+            let lowest_cleanup_slot = 1;
+            let lowest_available_slot = lowest_cleanup_slot + 1;
+
+            transaction_status_cf
+                .put_protobuf((0, signature1, lowest_cleanup_slot), &status)
+                .unwrap();
+
+            transaction_status_cf
+                .put_protobuf((0, signature2, lowest_available_slot), &status)
+                .unwrap();
+
+            let address0 = solana_sdk::pubkey::new_rand();
+            let address1 = solana_sdk::pubkey::new_rand();
+            blockstore
+                .write_transaction_status(
+                    lowest_cleanup_slot,
+                    signature1,
+                    vec![&address0],
+                    vec![],
+                    TransactionStatusMeta::default(),
+                )
+                .unwrap();
+            blockstore
+                .write_transaction_status(
+                    lowest_available_slot,
+                    signature2,
+                    vec![&address1],
+                    vec![],
+                    TransactionStatusMeta::default(),
+                )
+                .unwrap();
+
+            let check_for_missing = || {
+                (
+                    blockstore
+                        .get_transaction_status_with_counter(signature1, &[])
+                        .unwrap()
+                        .0
+                        .is_none(),
+                    blockstore
+                        .find_address_signatures_for_slot(address0, lowest_cleanup_slot)
+                        .unwrap()
+                        .is_empty(),
+                    blockstore
+                        .find_address_signatures(address0, lowest_cleanup_slot, lowest_cleanup_slot)
+                        .unwrap()
+                        .is_empty(),
+                )
+            };
+
+            let assert_existing_always = || {
+                let are_existing_always = (
+                    blockstore
+                        .get_transaction_status_with_counter(signature2, &[])
+                        .unwrap()
+                        .0
+                        .is_some(),
+                    !blockstore
+                        .find_address_signatures_for_slot(address1, lowest_available_slot)
+                        .unwrap()
+                        .is_empty(),
+                    !blockstore
+                        .find_address_signatures(
+                            address1,
+                            lowest_available_slot,
+                            lowest_available_slot,
+                        )
+                        .unwrap()
+                        .is_empty(),
+                );
+                assert_eq!(are_existing_always, (true, true, true));
+            };
+
+            let are_missing = check_for_missing();
+            // should never be missing before the conditional compaction & simulation...
+            assert_eq!(are_missing, (false, false, false));
+            assert_existing_always();
+
+            if simulate_compaction {
+                blockstore.set_max_expired_slot(lowest_cleanup_slot);
+                // force compaction filters to run across whole key range.
+                blockstore
+                    .compact_storage(Slot::min_value(), Slot::max_value())
+                    .unwrap();
+            }
+
+            if simulate_ledger_cleanup_service {
+                *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_cleanup_slot;
+            }
+
+            let are_missing = check_for_missing();
+            if simulate_compaction || simulate_ledger_cleanup_service {
+                // ... when either simulation (or both) is effective, we should observe to be missing
+                // consistently
+                assert_eq!(are_missing, (true, true, true));
+            } else {
+                // ... otherwise, we should observe to be existing...
+                assert_eq!(are_missing, (false, false, false));
+            }
+            assert_existing_always();
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    fn test_lowest_cleanup_slot_and_special_cfs_with_compact_with_ledger_cleanup_service_simulation(
+    ) {
+        do_test_lowest_cleanup_slot_and_special_cfs(true, true);
+    }
+
+    #[test]
+    fn test_lowest_cleanup_slot_and_special_cfs_with_compact_without_ledger_cleanup_service_simulation(
+    ) {
+        do_test_lowest_cleanup_slot_and_special_cfs(true, false);
+    }
+
+    #[test]
+    fn test_lowest_cleanup_slot_and_special_cfs_without_compact_with_ledger_cleanup_service_simulation(
+    ) {
+        do_test_lowest_cleanup_slot_and_special_cfs(false, true);
+    }
+
+    #[test]
+    fn test_lowest_cleanup_slot_and_special_cfs_without_compact_without_ledger_cleanup_service_simulation(
+    ) {
+        do_test_lowest_cleanup_slot_and_special_cfs(false, false);
+    }
+
     #[test]
     fn test_get_rooted_transaction() {
         let slot = 2;

ledger/src/blockstore/blockstore_purge.rs

@@ -32,6 +32,19 @@ impl Blockstore {
         }
     }
 
+    /// Usually this is paired with .purge_slots() but we can't internally call this in
+    /// that function unconditionally. That's because set_max_expired_slot()
+    /// expects to purge older slots by the successive chronological order, while .purge_slots()
+    /// can also be used to purge *future* slots for --hard-fork thing, preserving older
+    /// slots. It'd be quite dangerous to purge older slots in that case.
+    /// So, current legal user of this function is LedgerCleanupService.
+    pub fn set_max_expired_slot(&self, to_slot: Slot) {
+        // convert here from inclusive purged range end to inclusive alive range start to align
+        // with Slot::default() for initial compaction filter behavior consistency
+        let to_slot = to_slot.checked_add(1).unwrap();
+        self.db.set_oldest_slot(to_slot);
+    }
+
     pub fn purge_and_compact_slots(&self, from_slot: Slot, to_slot: Slot) {
         self.purge_slots(from_slot, to_slot, PurgeType::Exact);
         if let Err(e) = self.compact_storage(from_slot, to_slot) {
@@ -165,6 +178,10 @@ impl Blockstore {
             & self
                 .db
                 .delete_range_cf::<cf::PerfSamples>(&mut write_batch, from_slot, to_slot)
+                .is_ok()
+            & self
+                .db
+                .delete_range_cf::<cf::BlockHeight>(&mut write_batch, from_slot, to_slot)
                 .is_ok();
         let mut w_active_transaction_status_index =
             self.active_transaction_status_index.write().unwrap();
@@ -180,6 +197,13 @@ impl Blockstore {
                     to_slot,
                 )?;
             }
+            PurgeType::CompactionFilter => {
+                // No explicit action is required here because this purge type completely and
+                // indefinitely relies on the proper working of compaction filter for those
+                // special column families, never toggling the primary index from the current
+                // one. Overall, this enables well uniformly distributed writes, resulting
+                // in no spiky periodic huge delete_range for them.
+            }
         }
         delete_range_timer.stop();
         let mut write_timer = Measure::start("write_batch");
@@ -193,6 +217,10 @@ impl Blockstore {
         write_timer.stop();
         purge_stats.delete_range += delete_range_timer.as_us();
         purge_stats.write_batch += write_timer.as_us();
+        // only drop w_active_transaction_status_index after we do db.write(write_batch);
+        // otherwise, readers might be confused with inconsistent state between
+        // self.active_transaction_status_index and RockDb's TransactionStatusIndex contents
+        drop(w_active_transaction_status_index);
         Ok(columns_purged)
     }
 
@@ -263,6 +291,10 @@ impl Blockstore {
             && self
                 .perf_samples_cf
                 .compact_range(from_slot, to_slot)
+                .unwrap_or(false)
+            && self
+                .block_height_cf
+                .compact_range(from_slot, to_slot)
                 .unwrap_or(false);
         compact_timer.stop();
         if !result {
@@ -323,18 +355,26 @@ impl Blockstore {
         w_active_transaction_status_index: &mut u64,
         to_slot: Slot,
     ) -> Result<()> {
-        if let Some(index) = self.toggle_transaction_status_index(
+        if let Some(purged_index) = self.toggle_transaction_status_index(
             write_batch,
             w_active_transaction_status_index,
             to_slot,
         )? {
             *columns_purged &= self
                 .db
-                .delete_range_cf::<cf::TransactionStatus>(write_batch, index, index + 1)
+                .delete_range_cf::<cf::TransactionStatus>(
+                    write_batch,
+                    purged_index,
+                    purged_index + 1,
+                )
                 .is_ok()
                 & self
                     .db
-                    .delete_range_cf::<cf::AddressSignatures>(write_batch, index, index + 1)
+                    .delete_range_cf::<cf::AddressSignatures>(
+                        write_batch,
+                        purged_index,
+                        purged_index + 1,
+                    )
                     .is_ok();
         }
         Ok(())

ledger/src/blockstore_db.rs

@@ -5,9 +5,13 @@ use log::*;
 use prost::Message;
 pub use rocksdb::Direction as IteratorDirection;
 use rocksdb::{
-    self, ColumnFamily, ColumnFamilyDescriptor, DBIterator, DBRawIterator, DBRecoveryMode,
-    IteratorMode as RocksIteratorMode, Options, WriteBatch as RWriteBatch, DB,
+    self,
+    compaction_filter::CompactionFilter,
+    compaction_filter_factory::{CompactionFilterContext, CompactionFilterFactory},
+    ColumnFamily, ColumnFamilyDescriptor, CompactionDecision, DBIterator, DBRawIterator,
+    DBRecoveryMode, IteratorMode as RocksIteratorMode, Options, WriteBatch as RWriteBatch, DB,
 };
+
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use solana_runtime::hardened_unpack::UnpackError;
@@ -17,7 +21,17 @@ use solana_sdk::{
     signature::Signature,
 };
 use solana_storage_proto::convert::generated;
-use std::{collections::HashMap, fs, marker::PhantomData, path::Path, sync::Arc};
+use std::{
+    collections::HashMap,
+    ffi::{CStr, CString},
+    fs,
+    marker::PhantomData,
+    path::Path,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
+};
 use thiserror::Error;
 
 const MAX_WRITE_BUFFER_SIZE: u64 = 256 * 1024 * 1024; // 256MB
@@ -58,6 +72,9 @@ const PERF_SAMPLES_CF: &str = "perf_samples";
 /// Column family for BlockHeight
 const BLOCK_HEIGHT_CF: &str = "block_height";
 
+// 1 day is chosen for the same reasoning of DEFAULT_COMPACTION_SLOT_INTERVAL
+const PERIODIC_COMPACTION_SECONDS: u64 = 60 * 60 * 24;
+
 #[derive(Error, Debug)]
 pub enum BlockstoreError {
     ShredForIndexExists,
@@ -208,8 +225,30 @@ impl From<BlockstoreRecoveryMode> for DBRecoveryMode {
     }
 }
 
+#[derive(Default, Clone, Debug)]
+struct OldestSlot(Arc<AtomicU64>);
+
+impl OldestSlot {
+    pub fn set(&self, oldest_slot: Slot) {
+        // this is independently used for compaction_filter without any data dependency.
+        // also, compaction_filters are created via its factories, creating short-lived copies of
+        // this atomic value for the single job of compaction. So, Relaxed store can be justified
+        // in total
+        self.0.store(oldest_slot, Ordering::Relaxed);
+    }
+
+    pub fn get(&self) -> Slot {
+        // copy from the AtomicU64 as a general precaution so that the oldest_slot can not mutate
+        // across single run of compaction for simpler reasoning although this isn't strict
+        // requirement at the moment
+        // also eventual propagation (very Relaxed) load is Ok, because compaction by nature doesn't
+        // require strictly synchronized semantics in this regard
+        self.0.load(Ordering::Relaxed)
+    }
+}
+
 #[derive(Debug)]
-struct Rocks(rocksdb::DB, ActualAccessType);
+struct Rocks(rocksdb::DB, ActualAccessType, OldestSlot);
 
 impl Rocks {
     fn open(
@@ -234,39 +273,75 @@ impl Rocks {
             db_options.set_wal_recovery_mode(recovery_mode.into());
         }
 
+        let oldest_slot = OldestSlot::default();
+
         // Column family names
-        let meta_cf_descriptor =
-            ColumnFamilyDescriptor::new(SlotMeta::NAME, get_cf_options(&access_type));
-        let dead_slots_cf_descriptor =
-            ColumnFamilyDescriptor::new(DeadSlots::NAME, get_cf_options(&access_type));
-        let duplicate_slots_cf_descriptor =
-            ColumnFamilyDescriptor::new(DuplicateSlots::NAME, get_cf_options(&access_type));
-        let erasure_meta_cf_descriptor =
-            ColumnFamilyDescriptor::new(ErasureMeta::NAME, get_cf_options(&access_type));
-        let orphans_cf_descriptor =
-            ColumnFamilyDescriptor::new(Orphans::NAME, get_cf_options(&access_type));
-        let root_cf_descriptor =
-            ColumnFamilyDescriptor::new(Root::NAME, get_cf_options(&access_type));
-        let index_cf_descriptor =
-            ColumnFamilyDescriptor::new(Index::NAME, get_cf_options(&access_type));
-        let shred_data_cf_descriptor =
-            ColumnFamilyDescriptor::new(ShredData::NAME, get_cf_options(&access_type));
-        let shred_code_cf_descriptor =
-            ColumnFamilyDescriptor::new(ShredCode::NAME, get_cf_options(&access_type));
-        let transaction_status_cf_descriptor =
-            ColumnFamilyDescriptor::new(TransactionStatus::NAME, get_cf_options(&access_type));
-        let address_signatures_cf_descriptor =
-            ColumnFamilyDescriptor::new(AddressSignatures::NAME, get_cf_options(&access_type));
-        let transaction_status_index_cf_descriptor =
-            ColumnFamilyDescriptor::new(TransactionStatusIndex::NAME, get_cf_options(&access_type));
-        let rewards_cf_descriptor =
-            ColumnFamilyDescriptor::new(Rewards::NAME, get_cf_options(&access_type));
-        let blocktime_cf_descriptor =
-            ColumnFamilyDescriptor::new(Blocktime::NAME, get_cf_options(&access_type));
-        let perf_samples_cf_descriptor =
-            ColumnFamilyDescriptor::new(PerfSamples::NAME, get_cf_options(&access_type));
-        let block_height_cf_descriptor =
-            ColumnFamilyDescriptor::new(BlockHeight::NAME, get_cf_options(&access_type));
+        let meta_cf_descriptor = ColumnFamilyDescriptor::new(
+            SlotMeta::NAME,
+            get_cf_options::<SlotMeta>(&access_type, &oldest_slot),
+        );
+        let dead_slots_cf_descriptor = ColumnFamilyDescriptor::new(
+            DeadSlots::NAME,
+            get_cf_options::<DeadSlots>(&access_type, &oldest_slot),
+        );
+        let duplicate_slots_cf_descriptor = ColumnFamilyDescriptor::new(
+            DuplicateSlots::NAME,
+            get_cf_options::<DuplicateSlots>(&access_type, &oldest_slot),
+        );
+        let erasure_meta_cf_descriptor = ColumnFamilyDescriptor::new(
+            ErasureMeta::NAME,
+            get_cf_options::<ErasureMeta>(&access_type, &oldest_slot),
+        );
+        let orphans_cf_descriptor = ColumnFamilyDescriptor::new(
+            Orphans::NAME,
+            get_cf_options::<Orphans>(&access_type, &oldest_slot),
+        );
+        let root_cf_descriptor = ColumnFamilyDescriptor::new(
+            Root::NAME,
+            get_cf_options::<Root>(&access_type, &oldest_slot),
+        );
+        let index_cf_descriptor = ColumnFamilyDescriptor::new(
+            Index::NAME,
+            get_cf_options::<Index>(&access_type, &oldest_slot),
+        );
+        let shred_data_cf_descriptor = ColumnFamilyDescriptor::new(
+            ShredData::NAME,
+            get_cf_options::<ShredData>(&access_type, &oldest_slot),
+        );
+        let shred_code_cf_descriptor = ColumnFamilyDescriptor::new(
+            ShredCode::NAME,
+            get_cf_options::<ShredCode>(&access_type, &oldest_slot),
+        );
+        let transaction_status_cf_descriptor = ColumnFamilyDescriptor::new(
+            TransactionStatus::NAME,
+            get_cf_options::<TransactionStatus>(&access_type, &oldest_slot),
+        );
+        let address_signatures_cf_descriptor = ColumnFamilyDescriptor::new(
+            AddressSignatures::NAME,
+            get_cf_options::<AddressSignatures>(&access_type, &oldest_slot),
+        );
+        let transaction_status_index_cf_descriptor = ColumnFamilyDescriptor::new(
+            TransactionStatusIndex::NAME,
+            get_cf_options::<TransactionStatusIndex>(&access_type, &oldest_slot),
+        );
+        let rewards_cf_descriptor = ColumnFamilyDescriptor::new(
+            Rewards::NAME,
+            get_cf_options::<Rewards>(&access_type, &oldest_slot),
+        );
+        let blocktime_cf_descriptor = ColumnFamilyDescriptor::new(
+            Blocktime::NAME,
+            get_cf_options::<Blocktime>(&access_type, &oldest_slot),
+        );
+        let perf_samples_cf_descriptor = ColumnFamilyDescriptor::new(
+            PerfSamples::NAME,
+            get_cf_options::<PerfSamples>(&access_type, &oldest_slot),
+        );
+        let block_height_cf_descriptor = ColumnFamilyDescriptor::new(
+            BlockHeight::NAME,
+            get_cf_options::<BlockHeight>(&access_type, &oldest_slot),
+        );
+        // Don't forget to add to both run_purge_with_stats() and
+        // compact_storage() in ledger/src/blockstore/blockstore_purge.rs!!
 
         let cfs = vec![
             (SlotMeta::NAME, meta_cf_descriptor),
@@ -289,18 +364,18 @@ impl Rocks {
             (PerfSamples::NAME, perf_samples_cf_descriptor),
             (BlockHeight::NAME, block_height_cf_descriptor),
         ];
+        let cf_names: Vec<_> = cfs.iter().map(|c| c.0).collect();
 
         // Open the database
         let db = match access_type {
             AccessType::PrimaryOnly | AccessType::PrimaryOnlyForMaintenance => Rocks(
                 DB::open_cf_descriptors(&db_options, path, cfs.into_iter().map(|c| c.1))?,
                 ActualAccessType::Primary,
+                oldest_slot,
             ),
             AccessType::TryPrimaryThenSecondary => {
-                let names: Vec<_> = cfs.iter().map(|c| c.0).collect();
-
                 match DB::open_cf_descriptors(&db_options, path, cfs.into_iter().map(|c| c.1)) {
-                    Ok(db) => Rocks(db, ActualAccessType::Primary),
+                    Ok(db) => Rocks(db, ActualAccessType::Primary, oldest_slot),
                     Err(err) => {
                         let secondary_path = path.join("solana-secondary");
 
@@ -312,13 +387,75 @@ impl Rocks {
                         db_options.set_max_open_files(-1);
 
                         Rocks(
-                            DB::open_cf_as_secondary(&db_options, path, &secondary_path, names)?,
+                            DB::open_cf_as_secondary(
+                                &db_options,
+                                path,
+                                &secondary_path,
+                                cf_names.clone(),
+                            )?,
                             ActualAccessType::Secondary,
+                            oldest_slot,
                         )
                     }
                 }
             }
         };
+        // this is only needed for LedgerCleanupService. so guard with PrimaryOnly (i.e. running solana-validator)
+        if matches!(access_type, AccessType::PrimaryOnly) {
+            for cf_name in cf_names {
+                // this special column family must be excluded from LedgerCleanupService's rocksdb
+                // compactions
+                if cf_name == TransactionStatusIndex::NAME {
+                    continue;
+                }
+
+                // This is the crux of our write-stall-free storage cleaning strategy with consistent
+                // state view for higher-layers
+                //
+                // For the consistent view, we commit delete_range on pruned slot range by LedgerCleanupService.
+                // simple story here.
+                //
+                // For actual storage cleaning, we employ RocksDB compaction. But default RocksDB compaction
+                // settings don't work well for us. That's because we're using it rather like a really big
+                // (100 GBs) ring-buffer. RocksDB is basically assuming uniform data write over the key space for
+                // efficient compaction, which isn't true for our use as a ring buffer.
+                //
+                // So, we customize the compaction strategy with 2 combined tweaks:
+                // (1) compaction_filter and (2) shortening its periodic cycles.
+                //
+                // Via the compaction_filter, we finally reclaim previously delete_range()-ed storage occupied
+                // by pruned slots. When compaction_filter is set, each SST files are re-compacted periodically
+                // to hunt for keys newly expired by the compaction_filter re-evaluation. But RocksDb's default
+                // `periodic_compaction_seconds` is 30 days, which is too long for our case. So, we
+                // shorten it to a day (24 hours).
+                //
+                // As we write newer SST files over time at rather consistent rate of speed, this
+                // effectively makes each newly-created ssts be re-compacted for the filter at
+                // well-dispersed different timings.
+                // As a whole, we rewrite the whole dataset at every PERIODIC_COMPACTION_SECONDS,
+                // slowly over the duration of PERIODIC_COMPACTION_SECONDS. So, this results in
+                // amortization.
+                // So, there is a bit inefficiency here because we'll rewrite not-so-old SST files
+                // too. But longer period would introduce higher variance of ledger storage sizes over
+                // the long period. And it's much better than the daily IO spike caused by compact_range() by
+                // previous implementation.
+                //
+                // `ttl` and `compact_range`(`ManualCompaction`), doesn't work nicely. That's
+                // because its original intention is delete_range()s to reclaim disk space. So it tries to merge
+                // them with N+1 SST files all way down to the bottommost SSTs, often leading to vastly large amount
+                // (= all) of invalidated SST files, when combined with newer writes happening at the opposite
+                // edge of the key space. This causes a long and heavy disk IOs and possible write
+                // stall and ultimately, the deadly Replay/Banking stage stall at higher layers.
+                db.0.set_options_cf(
+                    db.cf_handle(cf_name),
+                    &[(
+                        "periodic_compaction_seconds",
+                        &format!("{}", PERIODIC_COMPACTION_SECONDS),
+                    )],
+                )
+                .unwrap();
+            }
+        }
 
         Ok(db)
     }
@@ -363,7 +500,7 @@ impl Rocks {
     }
 
     fn get_cf(&self, cf: &ColumnFamily, key: &[u8]) -> Result<Option<Vec<u8>>> {
-        let opt = self.0.get_cf(cf, key)?.map(|db_vec| db_vec.to_vec());
+        let opt = self.0.get_cf(cf, key)?;
         Ok(opt)
     }
 
@@ -415,9 +552,13 @@ pub trait Column {
 
     fn key(index: Self::Index) -> Vec<u8>;
     fn index(key: &[u8]) -> Self::Index;
-    fn primary_index(index: Self::Index) -> Slot;
+    // this return Slot or some u64
+    fn primary_index(index: Self::Index) -> u64;
     #[allow(clippy::wrong_self_convention)]
     fn as_index(slot: Slot) -> Self::Index;
+    fn slot(index: Self::Index) -> Slot {
+        Self::primary_index(index)
+    }
 }
 
 pub trait ColumnName {
@@ -491,6 +632,10 @@ impl Column for columns::TransactionStatus {
         index.0
     }
 
+    fn slot(index: Self::Index) -> Slot {
+        index.2
+    }
+
     #[allow(clippy::wrong_self_convention)]
     fn as_index(index: u64) -> Self::Index {
         (index, Signature::default(), 0)
@@ -528,6 +673,10 @@ impl Column for columns::AddressSignatures {
         index.0
     }
 
+    fn slot(index: Self::Index) -> Slot {
+        index.2
+    }
+
     #[allow(clippy::wrong_self_convention)]
     fn as_index(index: u64) -> Self::Index {
         (index, Pubkey::default(), 0, Signature::default())
@@ -555,6 +704,10 @@ impl Column for columns::TransactionStatusIndex {
         index
     }
 
+    fn slot(_index: Self::Index) -> Slot {
+        unimplemented!()
+    }
+
     #[allow(clippy::wrong_self_convention)]
     fn as_index(slot: u64) -> u64 {
         slot
@@ -855,6 +1008,10 @@ impl Database {
     pub fn is_primary_access(&self) -> bool {
         self.backend.is_primary_access()
     }
+
+    pub fn set_oldest_slot(&self, oldest_slot: Slot) {
+        self.backend.2.set(oldest_slot);
+    }
 }
 
 impl<C> LedgerColumn<C>
@@ -1032,7 +1189,63 @@ impl<'a> WriteBatch<'a> {
     }
 }
 
-fn get_cf_options(access_type: &AccessType) -> Options {
+struct PurgedSlotFilter<C: Column + ColumnName> {
+    oldest_slot: Slot,
+    name: CString,
+    _phantom: PhantomData<C>,
+}
+
+impl<C: Column + ColumnName> CompactionFilter for PurgedSlotFilter<C> {
+    fn filter(&mut self, _level: u32, key: &[u8], _value: &[u8]) -> CompactionDecision {
+        use rocksdb::CompactionDecision::*;
+
+        let slot_in_key = C::slot(C::index(key));
+        // Refer to a comment about periodic_compaction_seconds, especially regarding implicit
+        // periodic execution of compaction_filters
+        if slot_in_key >= self.oldest_slot {
+            Keep
+        } else {
+            Remove
+        }
+    }
+
+    fn name(&self) -> &CStr {
+        &self.name
+    }
+}
+
+struct PurgedSlotFilterFactory<C: Column + ColumnName> {
+    oldest_slot: OldestSlot,
+    name: CString,
+    _phantom: PhantomData<C>,
+}
+
+impl<C: Column + ColumnName> CompactionFilterFactory for PurgedSlotFilterFactory<C> {
+    type Filter = PurgedSlotFilter<C>;
+
+    fn create(&mut self, _context: CompactionFilterContext) -> Self::Filter {
+        let copied_oldest_slot = self.oldest_slot.get();
+        PurgedSlotFilter::<C> {
+            oldest_slot: copied_oldest_slot,
+            name: CString::new(format!(
+                "purged_slot_filter({}, {:?})",
+                C::NAME,
+                copied_oldest_slot
+            ))
+            .unwrap(),
+            _phantom: PhantomData::default(),
+        }
+    }
+
+    fn name(&self) -> &CStr {
+        &self.name
+    }
+}
+
+fn get_cf_options<C: 'static + Column + ColumnName>(
+    access_type: &AccessType,
+    oldest_slot: &OldestSlot,
+) -> Options {
     let mut options = Options::default();
     // 256 * 8 = 2GB. 6 of these columns should take at most 12GB of RAM
     options.set_max_write_buffer_number(8);
@@ -1046,6 +1259,19 @@ fn get_cf_options(access_type: &AccessType) -> Options {
     options.set_level_zero_file_num_compaction_trigger(file_num_compaction_trigger as i32);
     options.set_max_bytes_for_level_base(total_size_base);
     options.set_target_file_size_base(file_size_base);
+
+    // TransactionStatusIndex must be excluded from LedgerCleanupService's rocksdb
+    // compactions....
+    if matches!(access_type, AccessType::PrimaryOnly)
+        && C::NAME != columns::TransactionStatusIndex::NAME
+    {
+        options.set_compaction_filter_factory(PurgedSlotFilterFactory::<C> {
+            oldest_slot: oldest_slot.clone(),
+            name: CString::new(format!("purged_slot_filter_factory({})", C::NAME)).unwrap(),
+            _phantom: PhantomData::default(),
+        });
+    }
+
     if matches!(access_type, AccessType::PrimaryOnlyForMaintenance) {
         options.set_disable_auto_compactions(true);
     }
@@ -1077,3 +1303,57 @@ fn get_db_options(access_type: &AccessType) -> Options {
 
     options
 }
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use crate::blockstore_db::columns::ShredData;
+
+    #[test]
+    fn test_compaction_filter() {
+        // this doesn't implement Clone...
+        let dummy_compaction_filter_context = || CompactionFilterContext {
+            is_full_compaction: true,
+            is_manual_compaction: true,
+        };
+        let oldest_slot = OldestSlot::default();
+
+        let mut factory = PurgedSlotFilterFactory::<ShredData> {
+            oldest_slot: oldest_slot.clone(),
+            name: CString::new("test compaction filter").unwrap(),
+            _phantom: PhantomData::default(),
+        };
+        let mut compaction_filter = factory.create(dummy_compaction_filter_context());
+
+        let dummy_level = 0;
+        let key = ShredData::key(ShredData::as_index(0));
+        let dummy_value = vec![];
+
+        // we can't use assert_matches! because CompactionDecision doesn't implement Debug
+        assert!(matches!(
+            compaction_filter.filter(dummy_level, &key, &dummy_value),
+            CompactionDecision::Keep
+        ));
+
+        // mutating oledst_slot doen't affect existing compaction filters...
+        oldest_slot.set(1);
+        assert!(matches!(
+            compaction_filter.filter(dummy_level, &key, &dummy_value),
+            CompactionDecision::Keep
+        ));
+
+        // recreating compaction filter starts to expire the key
+        let mut compaction_filter = factory.create(dummy_compaction_filter_context());
+        assert!(matches!(
+            compaction_filter.filter(dummy_level, &key, &dummy_value),
+            CompactionDecision::Remove
+        ));
+
+        // newer key shouldn't be removed
+        let key = ShredData::key(ShredData::as_index(1));
+        matches!(
+            compaction_filter.filter(dummy_level, &key, &dummy_value),
+            CompactionDecision::Keep
+        );
+    }
+}

ledger/src/entry.rs

@@ -724,40 +724,16 @@ pub fn next_entry(prev_hash: &Hash, num_hashes: u64, transactions: Vec<Transacti
 mod tests {
     use super::*;
     use crate::entry::Entry;
-    use chrono::prelude::Utc;
-    use solana_budget_program::budget_instruction;
     use solana_sdk::{
         hash::{hash, new_rand as hash_new_rand, Hash},
         message::Message,
         packet::PACKET_DATA_SIZE,
+        pubkey::Pubkey,
         signature::{Keypair, Signer},
-        system_transaction,
+        system_instruction, system_transaction,
         transaction::Transaction,
     };
 
-    fn create_sample_payment(keypair: &Keypair, hash: Hash) -> Transaction {
-        let pubkey = keypair.pubkey();
-        let budget_contract = Keypair::new();
-        let budget_pubkey = budget_contract.pubkey();
-        let ixs = budget_instruction::payment(&pubkey, &pubkey, &budget_pubkey, 1);
-        let message = Message::new(&ixs, Some(&pubkey));
-        Transaction::new(&[keypair, &budget_contract], message, hash)
-    }
-
-    fn create_sample_timestamp(keypair: &Keypair, hash: Hash) -> Transaction {
-        let pubkey = keypair.pubkey();
-        let ix = budget_instruction::apply_timestamp(&pubkey, &pubkey, &pubkey, Utc::now());
-        let message = Message::new(&[ix], Some(&pubkey));
-        Transaction::new(&[keypair], message, hash)
-    }
-
-    fn create_sample_apply_signature(keypair: &Keypair, hash: Hash) -> Transaction {
-        let pubkey = keypair.pubkey();
-        let ix = budget_instruction::apply_signature(&pubkey, &pubkey, &pubkey);
-        let message = Message::new(&[ix], Some(&pubkey));
-        Transaction::new(&[keypair], message, hash)
-    }
-
     #[test]
     fn test_entry_verify() {
         let zero = Hash::default();
@@ -819,23 +795,6 @@ mod tests {
         assert!(e0.verify(&zero));
     }
 
-    #[test]
-    fn test_witness_reorder_attack() {
-        let zero = Hash::default();
-
-        // First, verify entries
-        let keypair = Keypair::new();
-        let tx0 = create_sample_timestamp(&keypair, zero);
-        let tx1 = create_sample_apply_signature(&keypair, zero);
-        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
-        assert!(e0.verify(&zero));
-
-        // Next, swap two witness transactions and ensure verification fails.
-        e0.transactions[0] = tx1; // <-- attack
-        e0.transactions[1] = tx0;
-        assert!(!e0.verify(&zero));
-    }
-
     #[test]
     fn test_next_entry() {
         let zero = Hash::default();
@@ -848,7 +807,7 @@ mod tests {
         assert_eq!(tick.hash, zero);
 
         let keypair = Keypair::new();
-        let tx0 = create_sample_timestamp(&keypair, zero);
+        let tx0 = system_transaction::transfer(&keypair, &Pubkey::new_unique(), 42, zero);
         let entry0 = next_entry(&zero, 1, vec![tx0.clone()]);
         assert_eq!(entry0.num_hashes, 1);
         assert_eq!(entry0.hash, next_hash(&zero, 1, &[tx0]));
@@ -904,9 +863,10 @@ mod tests {
         let zero = Hash::default();
         let one = hash(&zero.as_ref());
         let two = hash(&one.as_ref());
-        let alice_pubkey = Keypair::new();
-        let tx0 = create_sample_payment(&alice_pubkey, one);
-        let tx1 = create_sample_timestamp(&alice_pubkey, one);
+        let alice_keypair = Keypair::new();
+        let bob_keypair = Keypair::new();
+        let tx0 = system_transaction::transfer(&alice_keypair, &bob_keypair.pubkey(), 1, one);
+        let tx1 = system_transaction::transfer(&bob_keypair, &alice_keypair.pubkey(), 1, one);
         assert!(vec![][..].verify(&one)); // base case
         assert!(vec![next_entry(&one, 1, vec![tx0.clone()])][..].verify(&one)); // singleton case 1
         assert!(!vec![next_entry(&one, 1, vec![tx0.clone()])][..].verify(&two)); // singleton case 2, bad
@@ -931,17 +891,14 @@ mod tests {
         let recent_blockhash = hash_new_rand(&mut rng);
         let keypair = Keypair::new();
         let pubkey = keypair.pubkey();
-        let budget_contract = Keypair::new();
-        let budget_pubkey = budget_contract.pubkey();
         let make_transaction = |size| {
             let ixs: Vec<_> = std::iter::repeat_with(|| {
-                budget_instruction::payment(&pubkey, &pubkey, &budget_pubkey, 1)
+                system_instruction::transfer(&pubkey, &Pubkey::new_unique(), 1)
             })
             .take(size)
-            .flat_map(|x| x.into_iter())
             .collect();
             let message = Message::new(&ixs[..], Some(&pubkey));
-            Transaction::new(&[&keypair, &budget_contract], message, recent_blockhash)
+            Transaction::new(&[&keypair], message, recent_blockhash)
         };
         // Small transaction.
         {
@@ -954,7 +911,7 @@ mod tests {
         }
         // Big transaction.
         {
-            let tx = make_transaction(15);
+            let tx = make_transaction(25);
             let entries = vec![next_entry(&recent_blockhash, 1, vec![tx.clone()])];
             assert!(bincode::serialized_size(&tx).unwrap() > PACKET_DATA_SIZE as u64);
             assert!(entries[..]
@@ -963,7 +920,7 @@ mod tests {
         }
         // Assert that verify fails as soon as serialized
         // size exceeds packet data size.
-        for size in 1..20 {
+        for size in 1..30 {
             let tx = make_transaction(size);
             let entries = vec![next_entry(&recent_blockhash, 1, vec![tx.clone()])];
             assert_eq!(

ledger/src/lib.rs

@@ -5,6 +5,7 @@ extern crate solana_bpf_loader_program;
 
 pub mod bank_forks_utils;
 pub mod bigtable_upload;
+pub mod bigtable_upload_service;
 pub mod block_error;
 #[macro_use]
 pub mod blockstore;

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,26 +12,27 @@ documentation = "https://docs.rs/solana-local-cluster"
 [dependencies]
 crossbeam-channel = "0.4"
 itertools = "0.9.0"
-gag = "0.1.10"
+gag = "1.0.0"
 fs_extra = "1.2.0"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-config-program = { path = "../programs/config", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-download-utils = { path = "../download-utils", version = "=1.7.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-config-program = { path = "../programs/config", version = "=1.7.1" }
+solana-core = { path = "../core", version = "=1.7.1" }
+solana-client = { path = "../client", version = "=1.7.1" }
+solana-download-utils = { path = "../download-utils", version = "=1.7.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.7.1" }
+solana-faucet = { path = "../faucet", version = "=1.7.1" }
+solana-gossip = { path = "../gossip", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.1" }
+solana-rpc = { path = "../rpc", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.0" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

local-cluster/src/cluster_tests.rs

@@ -1,13 +1,12 @@
-use log::*;
 /// Cluster independent integration tests
 ///
 /// All tests must start from an entry point and a funding keypair and
 /// discover the rest of the network.
+use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
 use solana_client::thin_client::create_client;
 use solana_core::consensus::VOTE_THRESHOLD_DEPTH;
-use solana_core::validator::ValidatorExit;
 use solana_gossip::{
     cluster_info::VALIDATOR_PORT_RANGE, contact_info::ContactInfo, gossip_service::discover_cluster,
 };
@@ -20,6 +19,7 @@ use solana_sdk::{
     clock::{self, Slot, NUM_CONSECUTIVE_LEADER_SLOTS},
     commitment_config::CommitmentConfig,
     epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
+    exit::Exit,
     hash::Hash,
     poh_config::PohConfig,
     pubkey::Pubkey,
@@ -178,7 +178,7 @@ pub fn sleep_n_epochs(
 
 pub fn kill_entry_and_spend_and_verify_rest(
     entry_point_info: &ContactInfo,
-    entry_point_validator_exit: &Arc<RwLock<ValidatorExit>>,
+    entry_point_validator_exit: &Arc<RwLock<Exit>>,
     funding_keypair: &Keypair,
     nodes: usize,
     slot_millis: u64,

local-cluster/src/validator_configs.rs

@@ -1,4 +1,5 @@
-use solana_core::validator::{ValidatorConfig, ValidatorExit};
+use solana_core::validator::ValidatorConfig;
+use solana_sdk::exit::Exit;
 use std::sync::{Arc, RwLock};
 
 pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
@@ -52,7 +53,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation,
         accounts_db_use_index_hash_calculation: config.accounts_db_use_index_hash_calculation,
         tpu_coalesce_ms: config.tpu_coalesce_ms,
-        validator_exit: Arc::new(RwLock::new(ValidatorExit::default())),
+        validator_exit: Arc::new(RwLock::new(Exit::default())),
         poh_hashes_per_batch: config.poh_hashes_per_batch,
         no_wait_for_vote_to_start_leader: config.no_wait_for_vote_to_start_leader,
     }

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "4.0.9"
 clap = "2.33.1"
 serde = "1.0.122"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-measure"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.11"
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

merkle-root-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-merkle-root-bench"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 log = "0.4.11"
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 clap = "2.33.1"
 
 [package.metadata.docs.rs]

merkle-tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-merkle-tree"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Merkle Tree"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,7 +10,7 @@ documentation = "https://docs.rs/solana-merkle-tree"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../sdk/program", version = "=1.7.0" }
+solana-program = { path = "../sdk/program", version = "=1.7.1" }
 fast-math = "0.1"
 
 # This can go once the BPF toolchain target Rust 1.42.0+

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,7 +15,7 @@ gethostname = "0.2.1"
 lazy_static = "1.4.0"
 log = "0.4.11"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
 
 [dev-dependencies]
 rand = "0.7.0"

net-shaper/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-net-shaper"
 description = "The solana cluster network shaping tool"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,8 +13,8 @@ publish = false
 clap = "2.33.1"
 serde = "1.0.122"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
 rand = "0.7.0"
 
 [[bin]]

net-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-net-utils"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,9 +18,9 @@ rand = "0.7.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 socket2 = "0.3.17"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
 tokio = { version = "1", features = ["full"] }
 url = "2.1.1"
 

net/net.sh

@@ -102,7 +102,10 @@ Operate a configured testnet
    --cluster-type development|devnet|testnet|mainnet-beta
                                       - Specify whether or not to launch the cluster in "development" mode with all features enabled at epoch 0,
                                         or various other live clusters' feature set (default: development)
-   --warp-slot WARP_SLOT              - Boot from a snapshot that has warped ahead to WARP_SLOT rather than a slot 0 genesis.
+   --slots-per-epoch SLOTS
+                                      - Override the number of slots in an epoch
+   --warp-slot WARP_SLOT
+                                      - Boot from a snapshot that has warped ahead to WARP_SLOT rather than a slot 0 genesis.
  sanity/start-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
    -o noInstallCheck    - Skip solana-install sanity
@@ -822,6 +825,9 @@ while [[ -n $1 ]]; do
       esac
       genesisOptions="$genesisOptions $1 $2"
       shift 2
+    elif [[ $1 = --slots-per-epoch ]]; then
+      genesisOptions="$genesisOptions $1 $2"
+      shift 2
     elif [[ $1 = --no-snapshot-fetch ]]; then
       maybeNoSnapshot="$1"
       shift 1

notifier/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-notifier"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Notifier"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

perf/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-perf"
-version = "1.7.0"
+version = "1.7.1"
 description = "Solana Performance APIs"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,20 +10,20 @@ documentation = "https://docs.rs/solana-perf"
 edition = "2018"
 
 [dependencies]
-rand = "0.7.0"
-dlopen = "0.1.8"
 bincode = "1.3.1"
-rayon = "1.5.0"
-serde = "1.0.122"
+curve25519-dalek = { version = "2" }
+dlopen = "0.1.8"
 dlopen_derive = "0.1.4"
 lazy_static = "1.4.0"
 log = "0.4.11"
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.0" }
-solana-budget-program = { path = "../programs/budget", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-curve25519-dalek = { version = "2" }
+rand = "0.7.0"
+rayon = "1.5.0"
+serde = "1.0.126"
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }
 
 [lib]
 name = "solana_perf"

perf/src/test_tx.rs

@@ -22,7 +22,7 @@ pub fn test_multisig_tx() -> Transaction {
 
     let transfer_instruction = SystemInstruction::Transfer { lamports };
 
-    let program_ids = vec![system_program::id(), solana_budget_program::id()];
+    let program_ids = vec![system_program::id(), solana_stake_program::id()];
 
     let instructions = vec![CompiledInstruction::new(
         0,

poh-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-poh-bench"
-version = "1.7.0"
+version = "1.7.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,13 +13,13 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-version = { path = "../version", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

poh/.gitignore

@@ -0,0 +1,2 @@
+/target/
+/farf/

poh/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "solana-poh"
+version = "1.7.1"
+description = "Solana PoH"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-poh"
+edition = "2018"
+
+[dependencies]
+core_affinity = "0.5.10"
+crossbeam-channel = "0.4"
+log = "0.4.11"
+solana-ledger = { path = "../ledger", version = "=1.7.1" }
+solana-measure = { path = "../measure", version = "=1.7.1" }
+solana-metrics = { path = "../metrics", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-sys-tuner = { path = "../sys-tuner", version = "=1.7.1" }
+thiserror = "1.0"
+
+[dev-dependencies]
+bincode = "1.3.1"
+matches = "0.1.6"
+rand = "0.7.0"
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-perf = { path = "../perf", version = "=1.7.1" }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_poh"
+
+[[bench]]
+name = "poh"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

poh/benches/poh.rs

@@ -3,12 +3,16 @@
 #![feature(test)]
 extern crate test;
 
-use solana_core::poh_service::DEFAULT_HASHES_PER_BATCH;
-use solana_ledger::poh::Poh;
-use solana_sdk::hash::Hash;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex};
-use test::Bencher;
+use {
+    solana_ledger::poh::Poh,
+    solana_poh::poh_service::DEFAULT_HASHES_PER_BATCH,
+    solana_sdk::hash::Hash,
+    std::sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, Mutex,
+    },
+    test::Bencher,
+};
 
 const NUM_HASHES: u64 = 30_000; // Should require ~10ms on a 2017 MacBook Pro
 

poh/benches/poh_verify.rs

@@ -1,11 +1,15 @@
 #![feature(test)]
 extern crate test;
 
-use solana_ledger::entry::{next_entry_mut, Entry, EntrySlice};
-use solana_sdk::hash::{hash, Hash};
-use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::system_transaction;
-use test::Bencher;
+use {
+    solana_ledger::entry::{next_entry_mut, Entry, EntrySlice},
+    solana_sdk::{
+        hash::{hash, Hash},
+        signature::{Keypair, Signer},
+        system_transaction,
+    },
+    test::Bencher,
+};
 
 const NUM_HASHES: u64 = 400;
 const NUM_ENTRIES: usize = 800;

poh/src/lib.rs

@@ -0,0 +1,10 @@
+#![allow(clippy::integer_arithmetic)]
+pub mod poh_recorder;
+pub mod poh_service;
+
+#[macro_use]
+extern crate solana_metrics;
+
+#[cfg(test)]
+#[macro_use]
+extern crate matches;

poh/src/poh_recorder.rs

@@ -10,30 +10,32 @@
 //! For Entries:
 //! * recorded entry must be >= WorkingBank::min_tick_height && entry must be < WorkingBank::max_tick_height
 //!
-use crate::poh_service::PohService;
-use crossbeam_channel::{
-    unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
-};
-use solana_ledger::blockstore::Blockstore;
-use solana_ledger::entry::Entry;
-use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
-use solana_ledger::poh::Poh;
-use solana_runtime::bank::Bank;
 pub use solana_sdk::clock::Slot;
-use solana_sdk::clock::NUM_CONSECUTIVE_LEADER_SLOTS;
-use solana_sdk::hash::Hash;
-use solana_sdk::poh_config::PohConfig;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::timing;
-use solana_sdk::transaction::Transaction;
-use std::cmp;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    mpsc::{channel, Receiver, SendError, Sender, SyncSender},
-    {Arc, Mutex},
+use {
+    crate::poh_service::PohService,
+    crossbeam_channel::{
+        unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
+    },
+    log::*,
+    solana_ledger::{
+        blockstore::Blockstore, entry::Entry, leader_schedule_cache::LeaderScheduleCache, poh::Poh,
+    },
+    solana_runtime::bank::Bank,
+    solana_sdk::{
+        clock::NUM_CONSECUTIVE_LEADER_SLOTS, hash::Hash, poh_config::PohConfig, pubkey::Pubkey,
+        timing, transaction::Transaction,
+    },
+    std::{
+        cmp,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::{channel, Receiver, SendError, Sender, SyncSender},
+            {Arc, Mutex},
+        },
+        time::{Duration, Instant},
+    },
+    thiserror::Error,
 };
-use std::time::{Duration, Instant};
-use thiserror::Error;
 
 pub const GRACE_TICKS_FACTOR: u64 = 2;
 pub const MAX_GRACE_SLOTS: u64 = 2;
@@ -726,22 +728,67 @@ impl PohRecorder {
         })
     }
 
-    #[cfg(test)]
+    // Used in tests
     pub fn schedule_dummy_max_height_reached_failure(&mut self) {
         self.reset(Hash::default(), 1, None);
     }
 }
 
+pub fn create_test_recorder(
+    bank: &Arc<Bank>,
+    blockstore: &Arc<Blockstore>,
+    poh_config: Option<PohConfig>,
+) -> (
+    Arc<AtomicBool>,
+    Arc<Mutex<PohRecorder>>,
+    PohService,
+    Receiver<WorkingBankEntry>,
+) {
+    let exit = Arc::new(AtomicBool::new(false));
+    let poh_config = Arc::new(poh_config.unwrap_or_default());
+    let (mut poh_recorder, entry_receiver, record_receiver) = PohRecorder::new(
+        bank.tick_height(),
+        bank.last_blockhash(),
+        bank.slot(),
+        Some((4, 4)),
+        bank.ticks_per_slot(),
+        &Pubkey::default(),
+        blockstore,
+        &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
+        &poh_config,
+        exit.clone(),
+    );
+    poh_recorder.set_bank(&bank);
+
+    let poh_recorder = Arc::new(Mutex::new(poh_recorder));
+    let poh_service = PohService::new(
+        poh_recorder.clone(),
+        &poh_config,
+        &exit,
+        bank.ticks_per_slot(),
+        crate::poh_service::DEFAULT_PINNED_CPU_CORE,
+        crate::poh_service::DEFAULT_HASHES_PER_BATCH,
+        record_receiver,
+    );
+
+    (exit, poh_recorder, poh_service, entry_receiver)
+}
+
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use bincode::serialize;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta, get_tmp_ledger_path};
-    use solana_perf::test_tx::test_tx;
-    use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
-    use solana_sdk::hash::hash;
-    use std::sync::mpsc::sync_channel;
+    use {
+        super::*,
+        bincode::serialize,
+        solana_ledger::{
+            blockstore::Blockstore,
+            blockstore_meta::SlotMeta,
+            genesis_utils::{create_genesis_config, GenesisConfigInfo},
+            get_tmp_ledger_path,
+        },
+        solana_perf::test_tx::test_tx,
+        solana_sdk::{clock::DEFAULT_TICKS_PER_SLOT, hash::hash},
+        std::sync::mpsc::sync_channel,
+    };
 
     #[test]
     fn test_poh_recorder_no_zero_tick() {

poh/src/poh_service.rs

@@ -1,14 +1,21 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
-use crate::poh_recorder::{PohRecorder, Record};
-use crossbeam_channel::Receiver;
-use solana_ledger::poh::Poh;
-use solana_measure::measure::Measure;
-use solana_sdk::poh_config::PohConfig;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex};
-use std::thread::{self, sleep, Builder, JoinHandle};
-use std::time::{Duration, Instant};
+use {
+    crate::poh_recorder::{PohRecorder, Record},
+    crossbeam_channel::Receiver,
+    log::*,
+    solana_ledger::poh::Poh,
+    solana_measure::measure::Measure,
+    solana_sdk::poh_config::PohConfig,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, Mutex,
+        },
+        thread::{self, sleep, Builder, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
 
 pub struct PohService {
     tick_producer: JoinHandle<()>,
@@ -348,20 +355,22 @@ impl PohService {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::poh_recorder::WorkingBank;
-    use rand::{thread_rng, Rng};
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
-    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
-    use solana_measure::measure::Measure;
-    use solana_perf::test_tx::test_tx;
-    use solana_runtime::bank::Bank;
-    use solana_sdk::clock;
-    use solana_sdk::hash::hash;
-    use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::timing;
-    use std::time::Duration;
+    use {
+        super::*,
+        crate::poh_recorder::WorkingBank,
+        rand::{thread_rng, Rng},
+        solana_ledger::{
+            blockstore::Blockstore,
+            genesis_utils::{create_genesis_config, GenesisConfigInfo},
+            get_tmp_ledger_path,
+            leader_schedule_cache::LeaderScheduleCache,
+        },
+        solana_measure::measure::Measure,
+        solana_perf::test_tx::test_tx,
+        solana_runtime::bank::Bank,
+        solana_sdk::{clock, hash::hash, pubkey::Pubkey, timing},
+        std::time::Duration,
+    };
 
     #[test]
     #[ignore]

program-test/Cargo.toml

@@ -5,28 +5,28 @@ edition = "2018"
 license = "Apache-2.0"
 name = "solana-program-test"
 repository = "https://github.com/solana-labs/solana"
-version = "1.7.0"
+version = "1.7.1"
 
 [dependencies]
 async-trait = "0.1.42"
 base64 = "0.12.3"
 bincode = "1.3.1"
 chrono = "0.4.19"
-chrono-humanize = "0.1.1"
+chrono-humanize = "0.2.1"
 log = "0.4.11"
 mio = "0.7.6"
 serde = "1.0.112"
 serde_derive = "1.0.103"
-solana-banks-client = { path = "../banks-client", version = "=1.7.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
+solana-banks-client = { path = "../banks-client", version = "=1.7.1" }
+solana-banks-server = { path = "../banks-server", version = "=1.7.1" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.1" }
+solana-logger = { path = "../logger", version = "=1.7.1" }
+solana-runtime = { path = "../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../sdk", version = "=1.7.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.7.1" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 
 [dev-dependencies]
 assert_matches = "1.3.0"
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
+solana-stake-program = { path = "../programs/stake", version = "=1.7.1" }

program-test/src/lib.rs

@@ -393,6 +393,16 @@ impl solana_sdk::program_stubs::SyscallStubs for SyscallStubs {
 }
 
 pub fn find_file(filename: &str) -> Option<PathBuf> {
+    for dir in default_shared_object_dirs() {
+        let candidate = dir.join(&filename);
+        if candidate.exists() {
+            return Some(candidate);
+        }
+    }
+    None
+}
+
+fn default_shared_object_dirs() -> Vec<PathBuf> {
     let mut search_path = vec![];
     if let Ok(bpf_out_dir) = std::env::var("BPF_OUT_DIR") {
         search_path.push(PathBuf::from(bpf_out_dir));
@@ -401,15 +411,8 @@ pub fn find_file(filename: &str) -> Option<PathBuf> {
     if let Ok(dir) = std::env::current_dir() {
         search_path.push(dir);
     }
-    trace!("search path: {:?}", search_path);
-
-    for path in search_path {
-        let candidate = path.join(&filename);
-        if candidate.exists() {
-            return Some(candidate);
-        }
-    }
-    None
+    trace!("BPF .so search path: {:?}", search_path);
+    search_path
 }
 
 pub fn read_file<P: AsRef<Path>>(path: P) -> Vec<u8> {
@@ -502,6 +505,13 @@ impl Default for ProgramTest {
 }
 
 impl ProgramTest {
+    /// Create a `ProgramTest`.
+    ///
+    /// This is a wrapper around [`default`] and [`add_program`]. See their documentation for more
+    /// details.
+    ///
+    /// [`default`]: #method.default
+    /// [`add_program`]: #method.add_program
     pub fn new(
         program_name: &str,
         program_id: Pubkey,
@@ -579,7 +589,7 @@ impl ProgramTest {
 
     /// Add a BPF program to the test environment.
     ///
-    /// `program_name` will also used to locate the BPF shared object in the current or fixtures
+    /// `program_name` will also be used to locate the BPF shared object in the current or fixtures
     /// directory.
     ///
     /// If `process_instruction` is provided, the natively built-program may be used instead of the
@@ -590,20 +600,7 @@ impl ProgramTest {
         program_id: Pubkey,
         process_instruction: Option<ProcessInstructionWithContext>,
     ) {
-        let loader = solana_sdk::bpf_loader::id();
-        let program_file = find_file(&format!("{}.so", program_name));
-
-        if process_instruction.is_none() && program_file.is_none() {
-            panic!("Unable to add program {} ({})", program_name, program_id);
-        }
-
-        if (program_file.is_some() && self.prefer_bpf) || process_instruction.is_none() {
-            let program_file = program_file.unwrap_or_else(|| {
-                panic!(
-                    "Program file data not available for {} ({})",
-                    program_name, program_id
-                );
-            });
+        let add_bpf = |this: &mut ProgramTest, program_file: PathBuf| {
             let data = read_file(&program_file);
             info!(
                 "\"{}\" BPF program from {}{}",
@@ -627,28 +624,87 @@ impl ProgramTest {
                     .unwrap_or_else(|| "".to_string())
             );
 
-            self.add_account(
+            this.add_account(
                 program_id,
                 Account {
                     lamports: Rent::default().minimum_balance(data.len()).min(1),
                     data,
-                    owner: loader,
+                    owner: solana_sdk::bpf_loader::id(),
                     executable: true,
                     rent_epoch: 0,
                 },
             );
-        } else {
+        };
+
+        let add_native = |this: &mut ProgramTest, process_fn: ProcessInstructionWithContext| {
             info!("\"{}\" program loaded as native code", program_name);
-            self.builtins.push(Builtin::new(
+            this.builtins
+                .push(Builtin::new(program_name, program_id, process_fn));
+        };
+
+        let warn_invalid_program_name = || {
+            let valid_program_names = default_shared_object_dirs()
+                .iter()
+                .filter_map(|dir| dir.read_dir().ok())
+                .flat_map(|read_dir| {
+                    read_dir.filter_map(|entry| {
+                        let path = entry.ok()?.path();
+                        if !path.is_file() {
+                            return None;
+                        }
+                        match path.extension()?.to_str()? {
+                            "so" => Some(path.file_stem()?.to_os_string()),
+                            _ => None,
+                        }
+                    })
+                })
+                .collect::<Vec<_>>();
+
+            if valid_program_names.is_empty() {
+                // This should be unreachable as `test-bpf` should guarantee at least one shared
+                // object exists somewhere.
+                warn!("No BPF shared objects found.");
+                return;
+            }
+
+            warn!(
+                "Possible bogus program name. Ensure the program name ({}) \
+                matches one of the following recognizable program names:",
                 program_name,
-                program_id,
-                process_instruction.unwrap_or_else(|| {
-                    panic!(
-                        "Program processor not available for {} ({})",
-                        program_name, program_id
-                    );
-                }),
-            ));
+            );
+            for name in valid_program_names {
+                warn!(" - {}", name.to_str().unwrap());
+            }
+        };
+
+        let program_file = find_file(&format!("{}.so", program_name));
+        match (self.prefer_bpf, program_file, process_instruction) {
+            // If BPF is preferred (i.e., `test-bpf` is invoked) and a BPF shared object exists,
+            // use that as the program data.
+            (true, Some(file), _) => add_bpf(self, file),
+
+            // If BPF is not required (i.e., we were invoked with `test`), use the provided
+            // processor function as is.
+            //
+            // TODO: figure out why tests hang if a processor panics when running native code.
+            (false, _, Some(process)) => add_native(self, process),
+
+            // Invalid: `test-bpf` invocation with no matching BPF shared object.
+            (true, None, _) => {
+                warn_invalid_program_name();
+                panic!(
+                    "Program file data not available for {} ({})",
+                    program_name, program_id
+                );
+            }
+
+            // Invalid: regular `test` invocation without a processor.
+            (false, _, None) => {
+                panic!(
+                    "Program processor not available for {} ({})",
+                    program_name, program_id
+                );
+            }
         }
     }
 

programs/bpf/Cargo.lock

@@ -381,9 +381,9 @@ dependencies = [
 
 [[package]]
 name = "chrono-humanize"
-version = "0.1.2"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8164ae3089baf04ff71f32aeb70213283dcd236dce8bc976d00b17a458f5f71c"
+checksum = "2eddc119501d583fd930cb92144e605f44e0252c38dd89d9247fffa1993375cb"
 dependencies = [
  "chrono",
 ]
@@ -458,6 +458,21 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "console"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "regex",
+ "terminal_size",
+ "unicode-width",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "const_fn"
 version = "0.4.5"
@@ -2282,21 +2297,20 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.3.9"
+version = "1.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
+checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
 dependencies = [
  "aho-corasick",
  "memchr",
  "regex-syntax",
- "thread_local",
 ]
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.18"
+version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
+checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
 [[package]]
 name = "remove_dir_all"
@@ -2536,9 +2550,9 @@ dependencies = [
 
 [[package]]
 name = "serde_bytes"
-version = "0.11.4"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bf487fbf5c6239d7ea2ff8b10cb6b811cd4b5080d1c2aeed1dec18753c06e10"
+checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9"
 dependencies = [
  "serde",
 ]
@@ -2688,7 +2702,7 @@ dependencies = [
 
 [[package]]
 name = "solana-account-decoder"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "Inflector",
  "base64 0.12.3",
@@ -2710,7 +2724,7 @@ dependencies = [
 
 [[package]]
 name = "solana-banks-client"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "borsh",
@@ -2718,7 +2732,7 @@ dependencies = [
  "futures 0.3.12",
  "mio 0.7.7",
  "solana-banks-interface",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
  "solana-sdk",
  "tarpc",
  "tokio 1.4.0",
@@ -2727,7 +2741,7 @@ dependencies = [
 
 [[package]]
 name = "solana-banks-interface"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "mio 0.7.7",
  "serde",
@@ -2737,7 +2751,7 @@ dependencies = [
 
 [[package]]
 name = "solana-banks-server"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "futures 0.3.12",
@@ -2755,7 +2769,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-loader-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
@@ -2773,7 +2787,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-programs"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
@@ -2785,7 +2799,7 @@ dependencies = [
  "solana-account-decoder",
  "solana-bpf-loader-program",
  "solana-cli-output",
- "solana-logger 1.7.0",
+ "solana-logger 1.7.1",
  "solana-measure",
  "solana-runtime",
  "solana-sdk",
@@ -2796,264 +2810,281 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-128bit"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "solana-bpf-rust-128bit-dep",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-128bit-dep"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-alloc"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-call-depth"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-caller-access"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-custom-heap"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-dep-crate"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "byteorder 1.3.4",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-deprecated-loader"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-dup-accounts"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-error-handling"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "num-derive 0.2.5",
  "num-traits",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
  "thiserror",
 ]
 
 [[package]]
 name = "solana-bpf-rust-external-spend"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-finalize"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-instruction-introspection"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "solana-bpf-rust-invoked",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke-and-error"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke-and-ok"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke-and-return"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoked"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-iter"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-many-args"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "solana-bpf-rust-many-args-dep",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-many-args-dep"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-mem"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
+ "solana-program-test",
+ "solana-sdk",
+]
+
+[[package]]
+name = "solana-bpf-rust-membuiltins"
+version = "1.7.1"
+dependencies = [
+ "solana-bpf-rust-mem",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-noop"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-panic"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-param-passing"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "solana-bpf-rust-param-passing-dep",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-param-passing-dep"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-rand"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "getrandom 0.1.14",
  "rand 0.7.3",
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
+]
+
+[[package]]
+name = "solana-bpf-rust-ro-account_modify"
+version = "1.7.1"
+dependencies = [
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-ro-modify"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sanity"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sha"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-spoof1"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-spoof1-system"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sysvar"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
  "solana-program-test",
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-upgradeable"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-bpf-rust-upgraded"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
- "solana-program 1.7.0",
+ "solana-program 1.7.1",
 ]
 
 [[package]]
 name = "solana-clap-utils"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "chrono",
  "clap",
@@ -3068,7 +3099,7 @@ dependencies = [
 
 [[package]]
 name = "solana-cli-config"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "dirs-next",
  "lazy_static",
@@ -3080,12 +3111,12 @@ dependencies = [
 
 [[package]]
 name = "solana-cli-output"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "Inflector",
  "base64 0.13.0",
  "chrono",
- "console 0.11.3",
+ "console 0.14.1",
  "humantime",
  "indicatif",
  "serde",
@@ -3103,7 +3134,7 @@ dependencies = [
 
 [[package]]
 name = "solana-client"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "base64 0.13.0",
  "bincode",
@@ -3135,7 +3166,7 @@ dependencies = [
 
 [[package]]
 name = "solana-config-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "chrono",
@@ -3148,7 +3179,7 @@ dependencies = [
 
 [[package]]
 name = "solana-crate-features"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "backtrace",
  "bytes 0.4.12",
@@ -3171,7 +3202,7 @@ dependencies = [
 
 [[package]]
 name = "solana-faucet"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
@@ -3181,7 +3212,7 @@ dependencies = [
  "serde_derive",
  "solana-clap-utils",
  "solana-cli-config",
- "solana-logger 1.7.0",
+ "solana-logger 1.7.1",
  "solana-metrics",
  "solana-sdk",
  "solana-version",
@@ -3212,7 +3243,7 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bs58",
  "bv",
@@ -3223,8 +3254,8 @@ dependencies = [
  "serde",
  "serde_derive",
  "sha2 0.9.2",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
  "thiserror",
 ]
 
@@ -3243,7 +3274,7 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi-macro"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "proc-macro2 1.0.24",
  "quote 1.0.6",
@@ -3264,7 +3295,7 @@ dependencies = [
 
 [[package]]
 name = "solana-logger"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "env_logger",
  "lazy_static",
@@ -3273,7 +3304,7 @@ dependencies = [
 
 [[package]]
 name = "solana-measure"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "log",
  "solana-metrics",
@@ -3282,7 +3313,7 @@ dependencies = [
 
 [[package]]
 name = "solana-metrics"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "env_logger",
  "gethostname",
@@ -3294,7 +3325,7 @@ dependencies = [
 
 [[package]]
 name = "solana-net-utils"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "clap",
@@ -3305,7 +3336,7 @@ dependencies = [
  "serde_derive",
  "socket2",
  "solana-clap-utils",
- "solana-logger 1.7.0",
+ "solana-logger 1.7.1",
  "solana-version",
  "tokio 1.4.0",
  "url",
@@ -3345,7 +3376,7 @@ dependencies = [
 
 [[package]]
 name = "solana-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "blake3",
@@ -3368,16 +3399,16 @@ dependencies = [
  "serde_derive",
  "sha2 0.9.2",
  "sha3",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
- "solana-sdk-macro 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
+ "solana-sdk-macro 1.7.1",
  "thiserror",
 ]
 
 [[package]]
 name = "solana-program-test"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "async-trait",
  "base64 0.12.3",
@@ -3391,7 +3422,7 @@ dependencies = [
  "solana-banks-client",
  "solana-banks-server",
  "solana-bpf-loader-program",
- "solana-logger 1.7.0",
+ "solana-logger 1.7.1",
  "solana-runtime",
  "solana-sdk",
  "solana-vote-program",
@@ -3401,7 +3432,7 @@ dependencies = [
 
 [[package]]
 name = "solana-rayon-threadlimit"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "lazy_static",
  "num_cpus",
@@ -3409,10 +3440,10 @@ dependencies = [
 
 [[package]]
 name = "solana-remote-wallet"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "base32",
- "console 0.11.3",
+ "console 0.14.1",
  "dialoguer",
  "hidapi",
  "log",
@@ -3428,7 +3459,7 @@ dependencies = [
 
 [[package]]
 name = "solana-runtime"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "arrayref",
  "bincode",
@@ -3458,9 +3489,9 @@ dependencies = [
  "serde",
  "serde_derive",
  "solana-config-program",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
  "solana-measure",
  "solana-metrics",
  "solana-rayon-threadlimit",
@@ -3477,7 +3508,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "assert_matches",
  "bincode",
@@ -3513,11 +3544,11 @@ dependencies = [
  "sha2 0.9.2",
  "sha3",
  "solana-crate-features",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
- "solana-program 1.7.0",
- "solana-sdk-macro 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
+ "solana-program 1.7.1",
+ "solana-sdk-macro 1.7.1",
  "thiserror",
  "uriparse",
 ]
@@ -3537,7 +3568,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-macro"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bs58",
  "proc-macro2 1.0.24",
@@ -3548,20 +3579,20 @@ dependencies = [
 
 [[package]]
 name = "solana-secp256k1-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "digest 0.9.0",
  "libsecp256k1",
  "rand 0.7.3",
  "sha3",
- "solana-logger 1.7.0",
+ "solana-logger 1.7.1",
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-stake-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "log",
@@ -3571,8 +3602,8 @@ dependencies = [
  "serde",
  "serde_derive",
  "solana-config-program",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
  "solana-metrics",
  "solana-sdk",
  "solana-vote-program",
@@ -3581,7 +3612,7 @@ dependencies = [
 
 [[package]]
 name = "solana-transaction-status"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "Inflector",
  "base64 0.12.3",
@@ -3604,21 +3635,21 @@ dependencies = [
 
 [[package]]
 name = "solana-version"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "log",
  "rustc_version",
  "serde",
  "serde_derive",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-vote-program"
-version = "1.7.0"
+version = "1.7.1"
 dependencies = [
  "bincode",
  "log",
@@ -3627,9 +3658,9 @@ dependencies = [
  "rustc_version",
  "serde",
  "serde_derive",
- "solana-frozen-abi 1.7.0",
- "solana-frozen-abi-macro 1.7.0",
- "solana-logger 1.7.0",
+ "solana-frozen-abi 1.7.1",
+ "solana-frozen-abi-macro 1.7.1",
+ "solana-logger 1.7.1",
  "solana-metrics",
  "solana-sdk",
  "thiserror",
@@ -3878,15 +3909,6 @@ dependencies = [
  "syn 1.0.67",
 ]
 
-[[package]]
-name = "thread_local"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
-dependencies = [
- "lazy_static",
-]
-
 [[package]]
 name = "time"
 version = "0.1.43"

programs/bpf/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-bpf-programs"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.0"
+version = "1.7.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -26,15 +26,15 @@ itertools = "0.10.0"
 log = "0.4.11"
 miow = "0.2.2"
 net2 = "0.2.37"
-solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.7.0" }
-solana-cli-output = { path = "../../cli-output", version = "=1.7.0" }
-solana-logger = { path = "../../logger", version = "=1.7.0" }
-solana-measure = { path = "../../measure", version = "=1.7.0" }
+solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.7.1" }
+solana-cli-output = { path = "../../cli-output", version = "=1.7.1" }
+solana-logger = { path = "../../logger", version = "=1.7.1" }
+solana-measure = { path = "../../measure", version = "=1.7.1" }
 solana_rbpf = "=0.2.11"
-solana-runtime = { path = "../../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../../sdk", version = "=1.7.0" }
-solana-transaction-status = { path = "../../transaction-status", version = "=1.7.0" }
-solana-account-decoder = { path = "../../account-decoder", version = "=1.7.0" }
+solana-runtime = { path = "../../runtime", version = "=1.7.1" }
+solana-sdk = { path = "../../sdk", version = "=1.7.1" }
+solana-transaction-status = { path = "../../transaction-status", version = "=1.7.1" }
+solana-account-decoder = { path = "../../account-decoder", version = "=1.7.1" }
 
 
 [[bench]]
@@ -64,12 +64,14 @@ members = [
     "rust/many_args",
     "rust/many_args_dep",
     "rust/mem",
+    "rust/membuiltins",
     "rust/noop",
     "rust/panic",
     "rust/param_passing",
     "rust/param_passing_dep",
     "rust/rand",
     "rust/ro_modify",
+    "rust/ro_account_modify",
     "rust/sanity",
     "rust/sha",
     "rust/spoof1",

programs/bpf/build.rs

@@ -78,11 +78,13 @@ fn main() {
             "iter",
             "many_args",
             "mem",
+            "membuiltins",
             "noop",
             "panic",
             "param_passing",
             "rand",
             "ro_modify",
+            "ro_account_modify",
             "sanity",
             "sha",
             "spoof1",